promptfoo 0.100.4 → 0.100.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +13 -14
- package/dist/src/app/assets/{index-BQ5_IXki.js → index-DilfRyVV.js} +250 -250
- package/dist/src/app/assets/{index.es-D5Pwj_C4.js → index.es-BlWn08uT.js} +1 -1
- package/dist/src/app/assets/{sync-OQfXnPYr.js → sync-Bi0uUoUZ.js} +1 -1
- package/dist/src/app/index.html +1 -1
- package/dist/src/assertions/answerRelevance.d.ts.map +1 -1
- package/dist/src/assertions/answerRelevance.js +3 -3
- package/dist/src/assertions/answerRelevance.js.map +1 -1
- package/dist/src/assertions/bleu.d.ts +10 -0
- package/dist/src/assertions/bleu.d.ts.map +1 -1
- package/dist/src/assertions/bleu.js +2 -12
- package/dist/src/assertions/bleu.js.map +1 -1
- package/dist/src/assertions/classifier.d.ts.map +1 -1
- package/dist/src/assertions/classifier.js +2 -2
- package/dist/src/assertions/classifier.js.map +1 -1
- package/dist/src/assertions/contains.d.ts.map +1 -1
- package/dist/src/assertions/contains.js +13 -13
- package/dist/src/assertions/contains.js.map +1 -1
- package/dist/src/assertions/contextFaithfulness.d.ts.map +1 -1
- package/dist/src/assertions/contextFaithfulness.js +5 -5
- package/dist/src/assertions/contextFaithfulness.js.map +1 -1
- package/dist/src/assertions/contextRecall.d.ts.map +1 -1
- package/dist/src/assertions/contextRecall.js +3 -3
- package/dist/src/assertions/contextRecall.js.map +1 -1
- package/dist/src/assertions/contextRelevance.d.ts.map +1 -1
- package/dist/src/assertions/contextRelevance.js +4 -4
- package/dist/src/assertions/contextRelevance.js.map +1 -1
- package/dist/src/assertions/factuality.d.ts.map +1 -1
- package/dist/src/assertions/factuality.js +4 -4
- package/dist/src/assertions/factuality.js.map +1 -1
- package/dist/src/assertions/index.d.ts.map +1 -1
- package/dist/src/assertions/index.js +3 -3
- package/dist/src/assertions/index.js.map +1 -1
- package/dist/src/assertions/javascript.d.ts.map +1 -1
- package/dist/src/assertions/javascript.js +3 -3
- package/dist/src/assertions/javascript.js.map +1 -1
- package/dist/src/assertions/json.d.ts.map +1 -1
- package/dist/src/assertions/json.js +3 -3
- package/dist/src/assertions/json.js.map +1 -1
- package/dist/src/assertions/levenshtein.d.ts.map +1 -1
- package/dist/src/assertions/levenshtein.js +2 -2
- package/dist/src/assertions/levenshtein.js.map +1 -1
- package/dist/src/assertions/llmRubric.d.ts.map +1 -1
- package/dist/src/assertions/llmRubric.js +2 -2
- package/dist/src/assertions/llmRubric.js.map +1 -1
- package/dist/src/assertions/modelGradedClosedQa.d.ts.map +1 -1
- package/dist/src/assertions/modelGradedClosedQa.js +4 -4
- package/dist/src/assertions/modelGradedClosedQa.js.map +1 -1
- package/dist/src/assertions/moderation.d.ts.map +1 -1
- package/dist/src/assertions/moderation.js +3 -3
- package/dist/src/assertions/moderation.js.map +1 -1
- package/dist/src/assertions/python.d.ts.map +1 -1
- package/dist/src/assertions/python.js +2 -2
- package/dist/src/assertions/python.js.map +1 -1
- package/dist/src/assertions/redteam.d.ts.map +1 -1
- package/dist/src/assertions/redteam.js +3 -3
- package/dist/src/assertions/redteam.js.map +1 -1
- package/dist/src/assertions/regex.d.ts.map +1 -1
- package/dist/src/assertions/regex.js +3 -3
- package/dist/src/assertions/regex.js.map +1 -1
- package/dist/src/assertions/rouge.d.ts.map +1 -1
- package/dist/src/assertions/rouge.js +2 -2
- package/dist/src/assertions/rouge.js.map +1 -1
- package/dist/src/assertions/similar.d.ts.map +1 -1
- package/dist/src/assertions/similar.js +2 -2
- package/dist/src/assertions/similar.js.map +1 -1
- package/dist/src/assertions/startsWith.d.ts.map +1 -1
- package/dist/src/assertions/startsWith.js +3 -3
- package/dist/src/assertions/startsWith.js.map +1 -1
- package/dist/src/assertions/webhook.d.ts.map +1 -1
- package/dist/src/assertions/webhook.js +3 -3
- package/dist/src/assertions/webhook.js.map +1 -1
- package/dist/src/commands/eval.d.ts.map +1 -1
- package/dist/src/commands/eval.js +2 -2
- package/dist/src/commands/eval.js.map +1 -1
- package/dist/src/commands/share.d.ts.map +1 -1
- package/dist/src/commands/share.js +2 -2
- package/dist/src/commands/share.js.map +1 -1
- package/dist/src/commands/show.js +2 -2
- package/dist/src/commands/show.js.map +1 -1
- package/dist/src/evaluator.d.ts.map +1 -1
- package/dist/src/evaluator.js +12 -16
- package/dist/src/evaluator.js.map +1 -1
- package/dist/src/evaluatorHelpers.d.ts.map +1 -1
- package/dist/src/evaluatorHelpers.js +3 -3
- package/dist/src/evaluatorHelpers.js.map +1 -1
- package/dist/src/fetch.js +3 -3
- package/dist/src/fetch.js.map +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -2
- package/dist/src/index.js.map +1 -1
- package/dist/src/integrations/portkey.js +2 -2
- package/dist/src/integrations/portkey.js.map +1 -1
- package/dist/src/matchers.d.ts.map +1 -1
- package/dist/src/matchers.js +35 -25
- package/dist/src/matchers.js.map +1 -1
- package/dist/src/models/eval.d.ts.map +1 -1
- package/dist/src/models/eval.js +8 -8
- package/dist/src/models/eval.js.map +1 -1
- package/dist/src/prompts/index.d.ts.map +1 -1
- package/dist/src/prompts/index.js +3 -3
- package/dist/src/prompts/index.js.map +1 -1
- package/dist/src/prompts/processors/javascript.d.ts.map +1 -1
- package/dist/src/prompts/processors/javascript.js +2 -2
- package/dist/src/prompts/processors/javascript.js.map +1 -1
- package/dist/src/prompts/processors/python.d.ts.map +1 -1
- package/dist/src/prompts/processors/python.js +3 -3
- package/dist/src/prompts/processors/python.js.map +1 -1
- package/dist/src/prompts/processors/string.d.ts.map +1 -1
- package/dist/src/prompts/processors/string.js +2 -2
- package/dist/src/prompts/processors/string.js.map +1 -1
- package/dist/src/prompts/processors/yaml.d.ts.map +1 -1
- package/dist/src/prompts/processors/yaml.js +0 -1
- package/dist/src/prompts/processors/yaml.js.map +1 -1
- package/dist/src/providers/azure.d.ts.map +1 -1
- package/dist/src/providers/azure.js +6 -6
- package/dist/src/providers/azure.js.map +1 -1
- package/dist/src/providers/bedrock.d.ts.map +1 -1
- package/dist/src/providers/bedrock.js +20 -9
- package/dist/src/providers/bedrock.js.map +1 -1
- package/dist/src/providers/browser.d.ts.map +1 -1
- package/dist/src/providers/browser.js +9 -9
- package/dist/src/providers/browser.js.map +1 -1
- package/dist/src/providers/cloudflare-ai.d.ts.map +1 -1
- package/dist/src/providers/cloudflare-ai.js +4 -4
- package/dist/src/providers/cloudflare-ai.js.map +1 -1
- package/dist/src/providers/defaults.d.ts +5 -0
- package/dist/src/providers/defaults.d.ts.map +1 -1
- package/dist/src/providers/defaults.js +44 -16
- package/dist/src/providers/defaults.js.map +1 -1
- package/dist/src/providers/http.d.ts +36 -13
- package/dist/src/providers/http.d.ts.map +1 -1
- package/dist/src/providers/http.js +25 -14
- package/dist/src/providers/http.js.map +1 -1
- package/dist/src/providers/scriptCompletion.d.ts.map +1 -1
- package/dist/src/providers/scriptCompletion.js +2 -2
- package/dist/src/providers/scriptCompletion.js.map +1 -1
- package/dist/src/providers/sequence.d.ts.map +1 -1
- package/dist/src/providers/sequence.js +3 -3
- package/dist/src/providers/sequence.js.map +1 -1
- package/dist/src/providers/simulatedUser.d.ts.map +1 -1
- package/dist/src/providers/simulatedUser.js +2 -2
- package/dist/src/providers/simulatedUser.js.map +1 -1
- package/dist/src/providers/watsonx.d.ts.map +1 -1
- package/dist/src/providers/watsonx.js +3 -3
- package/dist/src/providers/watsonx.js.map +1 -1
- package/dist/src/providers/websocket.d.ts.map +1 -1
- package/dist/src/providers/websocket.js +2 -2
- package/dist/src/providers/websocket.js.map +1 -1
- package/dist/src/providers/xai.d.ts.map +1 -1
- package/dist/src/providers/xai.js +2 -2
- package/dist/src/providers/xai.js.map +1 -1
- package/dist/src/providers.d.ts.map +1 -1
- package/dist/src/providers.js +3 -3
- package/dist/src/providers.js.map +1 -1
- package/dist/src/redteam/commands/generate.d.ts.map +1 -1
- package/dist/src/redteam/commands/generate.js +2 -2
- package/dist/src/redteam/commands/generate.js.map +1 -1
- package/dist/src/redteam/constants.d.ts +1 -0
- package/dist/src/redteam/constants.d.ts.map +1 -1
- package/dist/src/redteam/constants.js +9 -1
- package/dist/src/redteam/constants.js.map +1 -1
- package/dist/src/redteam/extraction/purpose.d.ts.map +1 -1
- package/dist/src/redteam/extraction/purpose.js +11 -5
- package/dist/src/redteam/extraction/purpose.js.map +1 -1
- package/dist/src/redteam/extraction/util.d.ts.map +1 -1
- package/dist/src/redteam/extraction/util.js +2 -2
- package/dist/src/redteam/extraction/util.js.map +1 -1
- package/dist/src/redteam/index.d.ts.map +1 -1
- package/dist/src/redteam/index.js +3 -3
- package/dist/src/redteam/index.js.map +1 -1
- package/dist/src/redteam/plugins/base.d.ts.map +1 -1
- package/dist/src/redteam/plugins/base.js +5 -5
- package/dist/src/redteam/plugins/base.js.map +1 -1
- package/dist/src/redteam/plugins/crossSessionLeak.js +1 -1
- package/dist/src/redteam/plugins/crossSessionLeak.js.map +1 -1
- package/dist/src/redteam/plugins/harmful/aligned.d.ts.map +1 -1
- package/dist/src/redteam/plugins/harmful/aligned.js +2 -2
- package/dist/src/redteam/plugins/harmful/aligned.js.map +1 -1
- package/dist/src/redteam/plugins/index.d.ts.map +1 -1
- package/dist/src/redteam/plugins/index.js +6 -6
- package/dist/src/redteam/plugins/index.js.map +1 -1
- package/dist/src/redteam/plugins/indirectPromptInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/indirectPromptInjection.js +2 -2
- package/dist/src/redteam/plugins/indirectPromptInjection.js.map +1 -1
- package/dist/src/redteam/plugins/intent.d.ts.map +1 -1
- package/dist/src/redteam/plugins/intent.js +3 -3
- package/dist/src/redteam/plugins/intent.js.map +1 -1
- package/dist/src/redteam/plugins/policy.d.ts.map +1 -1
- package/dist/src/redteam/plugins/policy.js +2 -2
- package/dist/src/redteam/plugins/policy.js.map +1 -1
- package/dist/src/redteam/plugins/promptExtraction.d.ts.map +1 -1
- package/dist/src/redteam/plugins/promptExtraction.js +2 -2
- package/dist/src/redteam/plugins/promptExtraction.js.map +1 -1
- package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
- package/dist/src/redteam/providers/crescendo/index.js +31 -16
- package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
- package/dist/src/redteam/providers/goat.d.ts.map +1 -1
- package/dist/src/redteam/providers/goat.js +18 -7
- package/dist/src/redteam/providers/goat.js.map +1 -1
- package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
- package/dist/src/redteam/providers/iterative.js +18 -9
- package/dist/src/redteam/providers/iterative.js.map +1 -1
- package/dist/src/redteam/providers/iterativeImage.d.ts.map +1 -1
- package/dist/src/redteam/providers/iterativeImage.js +8 -8
- package/dist/src/redteam/providers/iterativeImage.js.map +1 -1
- package/dist/src/redteam/providers/iterativeTree.d.ts.map +1 -1
- package/dist/src/redteam/providers/iterativeTree.js +19 -10
- package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
- package/dist/src/redteam/providers/shared.d.ts +13 -5
- package/dist/src/redteam/providers/shared.d.ts.map +1 -1
- package/dist/src/redteam/providers/shared.js +31 -3
- package/dist/src/redteam/providers/shared.js.map +1 -1
- package/dist/src/redteam/strategies/citation.d.ts.map +1 -1
- package/dist/src/redteam/strategies/citation.js +2 -2
- package/dist/src/redteam/strategies/citation.js.map +1 -1
- package/dist/src/redteam/strategies/mathPrompt.d.ts.map +1 -1
- package/dist/src/redteam/strategies/mathPrompt.js +6 -3
- package/dist/src/redteam/strategies/mathPrompt.js.map +1 -1
- package/dist/src/redteam/strategies/multilingual.d.ts.map +1 -1
- package/dist/src/redteam/strategies/multilingual.js +16 -6
- package/dist/src/redteam/strategies/multilingual.js.map +1 -1
- package/dist/src/redteam/strategies/singleTurnComposite.d.ts.map +1 -1
- package/dist/src/redteam/strategies/singleTurnComposite.js +8 -2
- package/dist/src/redteam/strategies/singleTurnComposite.js.map +1 -1
- package/dist/src/server/routes/configs.js +8 -2
- package/dist/src/server/routes/configs.js.map +1 -1
- package/dist/src/server/routes/eval.js +9 -6
- package/dist/src/server/routes/eval.js.map +1 -1
- package/dist/src/server/routes/providers.d.ts.map +1 -1
- package/dist/src/server/routes/providers.js +16 -13
- package/dist/src/server/routes/providers.js.map +1 -1
- package/dist/src/server/server.js +2 -2
- package/dist/src/server/server.js.map +1 -1
- package/dist/src/share.d.ts.map +1 -1
- package/dist/src/share.js +2 -2
- package/dist/src/share.js.map +1 -1
- package/dist/src/testCases.d.ts.map +1 -1
- package/dist/src/testCases.js +9 -5
- package/dist/src/testCases.js.map +1 -1
- package/dist/src/util/config/load.d.ts.map +1 -1
- package/dist/src/util/config/load.js +9 -9
- package/dist/src/util/config/load.js.map +1 -1
- package/dist/src/util/convertEvalResultsToTable.d.ts.map +1 -1
- package/dist/src/util/convertEvalResultsToTable.js +4 -4
- package/dist/src/util/convertEvalResultsToTable.js.map +1 -1
- package/dist/src/util/index.d.ts.map +1 -1
- package/dist/src/util/index.js +7 -7
- package/dist/src/util/index.js.map +1 -1
- package/dist/src/util/invariant.d.ts +15 -0
- package/dist/src/util/invariant.d.ts.map +1 -0
- package/dist/src/util/invariant.js +26 -0
- package/dist/src/util/invariant.js.map +1 -0
- package/dist/src/util/json.js +2 -2
- package/dist/src/util/json.js.map +1 -1
- package/dist/test/evaluator.test.js +103 -0
- package/dist/test/evaluator.test.js.map +1 -1
- package/dist/test/matchers.test.js +52 -0
- package/dist/test/matchers.test.js.map +1 -1
- package/dist/test/prompts/processors/javascript.test.js +2 -2
- package/dist/test/prompts/processors/javascript.test.js.map +1 -1
- package/dist/test/providers/http.test.js +241 -9
- package/dist/test/providers/http.test.js.map +1 -1
- package/dist/test/redteam/extraction/entities.test.js +6 -0
- package/dist/test/redteam/extraction/entities.test.js.map +1 -1
- package/dist/test/redteam/extraction/purpose.test.js +6 -0
- package/dist/test/redteam/extraction/purpose.test.js.map +1 -1
- package/dist/test/redteam/extraction/util.test.js +21 -0
- package/dist/test/redteam/extraction/util.test.js.map +1 -1
- package/dist/test/redteam/providers/shared.test.js +243 -72
- package/dist/test/redteam/providers/shared.test.js.map +1 -1
- package/dist/test/redteam/strategies/multilingual.test.js +11 -7
- package/dist/test/redteam/strategies/multilingual.test.js.map +1 -1
- package/dist/test/server/eval.test.js +9 -9
- package/dist/test/server/eval.test.js.map +1 -1
- package/dist/test/server/providers.test.js +55 -26
- package/dist/test/server/providers.test.js.map +1 -1
- package/dist/test/testCases.test.js +32 -0
- package/dist/test/testCases.test.js.map +1 -1
- package/dist/test/util/invariant.test.d.ts +2 -0
- package/dist/test/util/invariant.test.d.ts.map +1 -0
- package/dist/test/util/invariant.test.js +36 -0
- package/dist/test/util/invariant.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +13 -14
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contextFaithfulness.js","sourceRoot":"","sources":["../../../src/assertions/contextFaithfulness.ts"],"names":[],"mappings":";;;;;AAIA,8DA6BC;AAjCD,
|
|
1
|
+
{"version":3,"file":"contextFaithfulness.js","sourceRoot":"","sources":["../../../src/assertions/contextFaithfulness.ts"],"names":[],"mappings":";;;;;AAIA,8DA6BC;AAjCD,0CAAyD;AAEzD,kEAA0C;AAEnC,KAAK,UAAU,yBAAyB,CAAC,EAC9C,SAAS,EACT,IAAI,EACJ,MAAM,GACU;IAChB,IAAA,mBAAS,EAAC,IAAI,CAAC,IAAI,EAAE,6DAA6D,CAAC,CAAC;IACpF,IAAA,mBAAS,EACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,QAAQ,EACnC,2DAA2D,CAC5D,CAAC;IACF,IAAA,mBAAS,EACP,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,EACrC,6DAA6D,CAC9D,CAAC;IACF,IAAA,mBAAS,EACP,OAAO,MAAM,KAAK,QAAQ,EAC1B,+DAA+D,CAChE,CAAC;IAEF,OAAO;QACL,SAAS;QACT,GAAG,CAAC,MAAM,IAAA,qCAA0B,EAClC,IAAI,CAAC,IAAI,CAAC,KAAK,EACf,MAAM,EACN,IAAI,CAAC,IAAI,CAAC,OAAO,EACjB,SAAS,CAAC,SAAS,IAAI,CAAC,EACxB,IAAI,CAAC,OAAO,CACb,CAAC;KACH,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contextRecall.d.ts","sourceRoot":"","sources":["../../../src/assertions/contextRecall.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"contextRecall.d.ts","sourceRoot":"","sources":["../../../src/assertions/contextRecall.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,eAAO,MAAM,mBAAmB,gDAK7B,eAAe,KAAG,OAAO,CAAC,aAAa,CAgBzC,CAAC"}
|
|
@@ -4,11 +4,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handleContextRecall = void 0;
|
|
7
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
7
|
const matchers_1 = require("../matchers");
|
|
8
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
9
9
|
const handleContextRecall = async ({ assertion, renderedValue, prompt, test, }) => {
|
|
10
|
-
(0,
|
|
11
|
-
(0,
|
|
10
|
+
(0, invariant_1.default)(typeof renderedValue === 'string', 'context-recall assertion type must have a string value');
|
|
11
|
+
(0, invariant_1.default)(prompt, 'context-recall assertion type must have a prompt');
|
|
12
12
|
return {
|
|
13
13
|
assertion,
|
|
14
14
|
...(await (0, matchers_1.matchesContextRecall)(typeof test.vars?.context === 'string' ? test.vars.context : prompt, renderedValue, assertion.threshold || 0, test.options, test.vars)),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contextRecall.js","sourceRoot":"","sources":["../../../src/assertions/contextRecall.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"contextRecall.js","sourceRoot":"","sources":["../../../src/assertions/contextRecall.ts"],"names":[],"mappings":";;;;;;AAAA,0CAAmD;AAEnD,kEAA0C;AAEnC,MAAM,mBAAmB,GAAG,KAAK,EAAE,EACxC,SAAS,EACT,aAAa,EACb,MAAM,EACN,IAAI,GACY,EAA0B,EAAE;IAC5C,IAAA,mBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,EACjC,wDAAwD,CACzD,CAAC;IACF,IAAA,mBAAS,EAAC,MAAM,EAAE,kDAAkD,CAAC,CAAC;IACtE,OAAO;QACL,SAAS;QACT,GAAG,CAAC,MAAM,IAAA,+BAAoB,EAC5B,OAAO,IAAI,CAAC,IAAI,EAAE,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EACnE,aAAa,EACb,SAAS,CAAC,SAAS,IAAI,CAAC,EACxB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,IAAI,CACV,CAAC;KACH,CAAC;AACJ,CAAC,CAAC;AArBW,QAAA,mBAAmB,uBAqB9B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contextRelevance.d.ts","sourceRoot":"","sources":["../../../src/assertions/contextRelevance.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"contextRelevance.d.ts","sourceRoot":"","sources":["../../../src/assertions/contextRelevance.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,eAAO,MAAM,sBAAsB,yBAGhC,eAAe,KAAG,OAAO,CAAC,aAAa,CAoBzC,CAAC"}
|
|
@@ -4,12 +4,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handleContextRelevance = void 0;
|
|
7
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
7
|
const matchers_1 = require("../matchers");
|
|
8
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
9
9
|
const handleContextRelevance = async ({ assertion, test, }) => {
|
|
10
|
-
(0,
|
|
11
|
-
(0,
|
|
12
|
-
(0,
|
|
10
|
+
(0, invariant_1.default)(test.vars, 'context-relevance assertion type must have a vars object');
|
|
11
|
+
(0, invariant_1.default)(typeof test.vars.query === 'string', 'context-relevance assertion type must have a query var');
|
|
12
|
+
(0, invariant_1.default)(typeof test.vars.context === 'string', 'context-relevance assertion type must have a context var');
|
|
13
13
|
return {
|
|
14
14
|
assertion,
|
|
15
15
|
...(await (0, matchers_1.matchesContextRelevance)(test.vars.query, test.vars.context, assertion.threshold || 0, test.options)),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contextRelevance.js","sourceRoot":"","sources":["../../../src/assertions/contextRelevance.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"contextRelevance.js","sourceRoot":"","sources":["../../../src/assertions/contextRelevance.ts"],"names":[],"mappings":";;;;;;AAAA,0CAAsD;AAEtD,kEAA0C;AAEnC,MAAM,sBAAsB,GAAG,KAAK,EAAE,EAC3C,SAAS,EACT,IAAI,GACY,EAA0B,EAAE;IAC5C,IAAA,mBAAS,EAAC,IAAI,CAAC,IAAI,EAAE,0DAA0D,CAAC,CAAC;IACjF,IAAA,mBAAS,EACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,QAAQ,EACnC,wDAAwD,CACzD,CAAC;IACF,IAAA,mBAAS,EACP,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,EACrC,0DAA0D,CAC3D,CAAC;IAEF,OAAO;QACL,SAAS;QACT,GAAG,CAAC,MAAM,IAAA,kCAAuB,EAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,EACf,IAAI,CAAC,IAAI,CAAC,OAAO,EACjB,SAAS,CAAC,SAAS,IAAI,CAAC,EACxB,IAAI,CAAC,OAAO,CACb,CAAC;KACH,CAAC;AACJ,CAAC,CAAC;AAvBW,QAAA,sBAAsB,0BAuBjC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"factuality.d.ts","sourceRoot":"","sources":["../../../src/assertions/factuality.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"factuality.d.ts","sourceRoot":"","sources":["../../../src/assertions/factuality.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAI/D,eAAO,MAAM,gBAAgB,8DAM1B,eAAe,KAAG,OAAO,CAAC,aAAa,CAiBzC,CAAC"}
|
|
@@ -4,15 +4,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handleFactuality = void 0;
|
|
7
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
7
|
const matchers_1 = require("../matchers");
|
|
8
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
9
9
|
const templates_1 = require("../util/templates");
|
|
10
10
|
const handleFactuality = async ({ assertion, renderedValue, outputString, test, prompt, }) => {
|
|
11
|
-
(0,
|
|
12
|
-
(0,
|
|
11
|
+
(0, invariant_1.default)(typeof renderedValue === 'string', 'factuality assertion type must have a string value');
|
|
12
|
+
(0, invariant_1.default)(prompt, 'factuality assertion type must have a prompt');
|
|
13
13
|
if (test.options?.rubricPrompt) {
|
|
14
14
|
// Substitute vars in prompt
|
|
15
|
-
(0,
|
|
15
|
+
(0, invariant_1.default)(typeof test.options.rubricPrompt === 'string', 'rubricPrompt must be a string');
|
|
16
16
|
const nunjucks = (0, templates_1.getNunjucksEngine)();
|
|
17
17
|
test.options.rubricPrompt = nunjucks.renderString(test.options.rubricPrompt, test.vars || {});
|
|
18
18
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"factuality.js","sourceRoot":"","sources":["../../../src/assertions/factuality.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"factuality.js","sourceRoot":"","sources":["../../../src/assertions/factuality.ts"],"names":[],"mappings":";;;;;;AAAA,0CAAgD;AAEhD,kEAA0C;AAC1C,iDAAsD;AAE/C,MAAM,gBAAgB,GAAG,KAAK,EAAE,EACrC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,IAAI,EACJ,MAAM,GACU,EAA0B,EAAE;IAC5C,IAAA,mBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,EACjC,oDAAoD,CACrD,CAAC;IACF,IAAA,mBAAS,EAAC,MAAM,EAAE,8CAA8C,CAAC,CAAC;IAClE,IAAI,IAAI,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC;QAC/B,4BAA4B;QAC5B,IAAA,mBAAS,EAAC,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,KAAK,QAAQ,EAAE,+BAA+B,CAAC,CAAC;QAC1F,MAAM,QAAQ,GAAG,IAAA,6BAAiB,GAAE,CAAC;QACrC,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,OAAO;QACL,SAAS;QACT,GAAG,CAAC,MAAM,IAAA,4BAAiB,EAAC,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;KAC3F,CAAC;AACJ,CAAC,CAAC;AAvBW,QAAA,gBAAgB,oBAuB3B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/assertions/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/assertions/index.ts"],"names":[],"mappings":"AAQA,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,qBAAqB,EACrB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,aAAa,CAAC;AAIrB,OAAO,KAAK,EAIV,gBAAgB,EACjB,MAAM,UAAU,CAAC;AAClB,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,SAAS,EAEd,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,UAAU,CAAC;AA6ClB,eAAO,MAAM,4BAA4B,k8CASvC,CAAC;AAIH,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,QAAQ,EACR,SAAS,EACT,IAAI,EACJ,SAAS,EACT,gBAAgB,GACjB,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,SAAS,EAAE,SAAS,CAAC;IACrB,IAAI,EAAE,cAAc,CAAC;IACrB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,aAAa,CAAC,CAgLzB;AAED,wBAAsB,aAAa,CAAC,EAClC,MAAM,EACN,QAAQ,EACR,gBAAgB,EAChB,IAAI,EACJ,SAAS,GACV,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,IAAI,EAAE,cAAc,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,aAAa,CAAC,CAkFzB;AAED,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,cAAc,EACpB,SAAS,EAAE,SAAS,EACpB,OAAO,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,aAAa,EAAE,CAAC,CAa1B;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAU3E;;;;;;;;;;;;;;;;AAGD,wBAcE"}
|
|
@@ -12,7 +12,6 @@ const async_1 = __importDefault(require("async"));
|
|
|
12
12
|
const fs_1 = __importDefault(require("fs"));
|
|
13
13
|
const js_yaml_1 = __importDefault(require("js-yaml"));
|
|
14
14
|
const path_1 = __importDefault(require("path"));
|
|
15
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
16
15
|
const cliState_1 = __importDefault(require("../cliState"));
|
|
17
16
|
const envars_1 = require("../envars");
|
|
18
17
|
const esm_1 = require("../esm");
|
|
@@ -22,6 +21,7 @@ const packageParser_1 = require("../providers/packageParser");
|
|
|
22
21
|
const pythonUtils_1 = require("../python/pythonUtils");
|
|
23
22
|
const telemetry_1 = __importDefault(require("../telemetry"));
|
|
24
23
|
const file_1 = require("../util/file");
|
|
24
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
25
25
|
const templates_1 = require("../util/templates");
|
|
26
26
|
const transform_1 = require("../util/transform");
|
|
27
27
|
const AssertionsResult_1 = require("./AssertionsResult");
|
|
@@ -69,7 +69,7 @@ const nunjucks = (0, templates_1.getNunjucksEngine)();
|
|
|
69
69
|
async function runAssertion({ prompt, provider, assertion, test, latencyMs, providerResponse, }) {
|
|
70
70
|
const { cost, logProbs, output: originalOutput } = providerResponse;
|
|
71
71
|
let output = originalOutput;
|
|
72
|
-
(0,
|
|
72
|
+
(0, invariant_1.default)(assertion.type, `Assertion must have a type: ${JSON.stringify(assertion)}`);
|
|
73
73
|
const inverse = assertion.type.startsWith('not-');
|
|
74
74
|
const baseType = inverse
|
|
75
75
|
? assertion.type.slice(4)
|
|
@@ -293,7 +293,7 @@ async function runAssertions({ prompt, provider, providerResponse, test, latency
|
|
|
293
293
|
return mainAssertResult.testResult();
|
|
294
294
|
}
|
|
295
295
|
async function runCompareAssertion(test, assertion, outputs) {
|
|
296
|
-
(0,
|
|
296
|
+
(0, invariant_1.default)(typeof assertion.value === 'string', 'select-best must have a string value');
|
|
297
297
|
test = (0, utils_1.getFinalTest)(test, assertion);
|
|
298
298
|
const comparisonResults = await (0, matchers_1.matchesSelectBest)(assertion.value, outputs, test.options, test.vars);
|
|
299
299
|
return comparisonResults.map((result) => ({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/assertions/index.ts"],"names":[],"mappings":";;;;;;AA8FA,oCA8LC;AAED,sCA8FC;AAED,kDAiBC;AAED,wCAUC;AA3ZD,kDAA0B;AAC1B,4CAAoB;AACpB,sDAA2B;AAC3B,gDAAwB;AACxB,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/assertions/index.ts"],"names":[],"mappings":";;;;;;AA8FA,oCA8LC;AAED,sCA8FC;AAED,kDAiBC;AAED,wCAUC;AA3ZD,kDAA0B;AAC1B,4CAAoB;AACpB,sDAA2B;AAC3B,gDAAwB;AACxB,2DAAmC;AACnC,sCAAsC;AACtC,gCAAsC;AACtC,uDAA+B;AAC/B,0CAYqB;AACrB,8DAA4E;AAC5E,uDAAkD;AAClD,6DAAqC;AAcrC,uCAAgD;AAChD,kEAA0C;AAC1C,iDAAsD;AACtD,iDAA8C;AAC9C,yDAAsD;AACtD,uDAA0D;AAC1D,iCAAyC;AACzC,6CAAgD;AAChD,yCAOoB;AACpB,+DAAkE;AAClE,mDAAsD;AACtD,yDAA4D;AAC5D,iCAAoC;AACpC,qCAAwC;AACxC,6CAAgD;AAChD,6CAAgD;AAChD,iCAA0D;AAC1D,uCAA0C;AAC1C,+CAAkD;AAClD,2CAA8C;AAC9C,+DAAkE;AAClE,6CAAgD;AAChD,qCAAyF;AACzF,6CAAuE;AACvE,qCAAwC;AACxC,uCAA0C;AAC1C,mCAAsC;AACtC,mCAA2C;AAC3C,uCAA0C;AAC1C,+BAAuD;AACvD,6CAAgD;AAChD,mCAA2E;AAC3E,uCAA0C;AAC1C,+BAAoC;AAEpC,MAAM,0BAA0B,GAAG,IAAA,kBAAS,EAAC,sCAAsC,EAAE,CAAC,CAAC,CAAC;AAE3E,QAAA,4BAA4B,GAAG,IAAI,GAAG,CAAgB;IACjE,kBAAkB;IAClB,sBAAsB;IACtB,gBAAgB;IAChB,mBAAmB;IACnB,YAAY;IACZ,YAAY;IACZ,uBAAuB;IACvB,yBAAyB;CAC1B,CAAC,CAAC;AAEH,MAAM,QAAQ,GAAG,IAAA,6BAAiB,GAAE,CAAC;AAE9B,KAAK,UAAU,YAAY,CAAC,EACjC,MAAM,EACN,QAAQ,EACR,SAAS,EACT,IAAI,EACJ,SAAS,EACT,gBAAgB,GAQjB;IACC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,gBAAgB,CAAC;IACpE,IAAI,MAAM,GAAG,cAAc,CAAC;IAE5B,IAAA,mBAAS,EAAC,SAAS,CAAC,IAAI,EAAE,+BAA+B,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAEtF,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,OAAO;QACtB,CAAC,CAAE,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAmB;QAC5C,CAAC,CAAE,SAAS,CAAC,IAAsB,CAAC;IAEtC,mBAAS,CAAC,MAAM,CAAC,gBAAgB,EAAE;QACjC,IAAI,EAAE,QAAQ;KACf,CAAC,CAAC;IAEH,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;QACxB,MAAM,GAAG,MAAM,IAAA,qBAAS,EAAC,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE;YACpD,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAkC;QAC7C,MAAM;QACN,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;QACrB,IAAI;QACJ,QAAQ;QACR,QAAQ;QACR,gBAAgB;QAChB,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC;IAEF,0BAA0B;IAC1B,IAAI,aAAa,GAAG,SAAS,CAAC,KAAK,CAAC;IACpC,IAAI,eAA+E,CAAC;IACpF,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,IAAI,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,kBAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;YACzC,MAAM,QAAQ,GAAG,cAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YAE/E,IAAI,IAAA,uBAAgB,EAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/B,MAAM,cAAc,GAAG,MAAM,IAAA,kBAAY,EAAC,QAAQ,CAAC,CAAC;gBACpD,IAAI,OAAO,cAAc,KAAK,UAAU,EAAE,CAAC;oBACzC,eAAe,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC3E,CAAC;qBAAM,IAAI,cAAc,CAAC,OAAO,IAAI,OAAO,cAAc,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;oBAClF,eAAe,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;gBACnF,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CACb,wBAAwB,QAAQ,gEAAgE,CACjG,CAAC;gBACJ,CAAC;gBACD,gBAAM,CAAC,KAAK,CAAC,qBAAqB,QAAQ,YAAY,eAAe,EAAE,CAAC,CAAC;YAC3E,CAAC;iBAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpC,IAAI,CAAC;oBACH,MAAM,kBAAkB,GAAG,MAAM,IAAA,uBAAS,EAAC,QAAQ,EAAE,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;oBACtF,eAAe,GAAG,kBAAkB,CAAC;oBACrC,gBAAM,CAAC,KAAK,CAAC,iBAAiB,QAAQ,YAAY,eAAe,EAAE,CAAC,CAAC;gBACvE,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,OAAO;wBACL,IAAI,EAAE,KAAK;wBACX,KAAK,EAAE,CAAC;wBACR,MAAM,EAAG,KAAe,CAAC,OAAO;wBAChC,SAAS;qBACV,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,aAAa,GAAG,IAAA,4BAAoB,EAAC,aAAa,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;aAAM,IAAI,IAAA,6BAAa,EAAC,aAAa,CAAC,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,kBAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;YACzC,MAAM,cAAc,GAAG,MAAM,IAAA,+BAAe,EAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;YACtE,IAAI,OAAO,cAAc,KAAK,UAAU,EAAE,CAAC;gBACzC,MAAM,IAAI,KAAK,CACb,wBAAwB,aAAa,kCAAkC,OAAO,cAAc,EAAE,CAC/F,CAAC;YACJ,CAAC;YAED,eAAe,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QAC3E,CAAC;aAAM,CAAC;YACN,6BAA6B;YAC7B,aAAa,GAAG,QAAQ,CAAC,YAAY,CAAC,aAAa,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;SAAM,IAAI,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QACzD,oCAAoC;QACpC,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACtC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAC1B,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC5B,OAAO,IAAA,4BAAoB,EAAC,CAAC,CAAC,CAAC;gBACjC,CAAC;gBACD,OAAO,QAAQ,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YACnD,CAAC;YACD,OAAO,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,eAAe,GAAoB;QACvC,SAAS;QACT,QAAQ;QACR,OAAO;QACP,IAAI;QACJ,OAAO;QACP,SAAS;QACT,QAAQ;QACR,MAAM;QACN,YAAY,EAAE,IAAA,oBAAY,EAAC,MAAM,CAAC;QAClC,MAAM;QACN,QAAQ;QACR,gBAAgB;QAChB,aAAa;QACb,IAAI,EAAE,IAAA,oBAAY,EAAC,IAAI,EAAE,SAAS,CAAC;QACnC,eAAe;KAChB,CAAC;IAEF,MAAM,iBAAiB,GAGnB;QACF,kBAAkB,EAAE,uCAAqB;QACzC,IAAI,EAAE,sBAAe;QACrB,UAAU,EAAE,6BAAgB;QAC5B,QAAQ,EAAE,yBAAc;QACxB,cAAc,EAAE,4BAAiB;QACjC,cAAc,EAAE,4BAAiB;QACjC,eAAe,EAAE,yBAAkB;QACnC,cAAc,EAAE,uBAAiB;QACjC,cAAc,EAAE,iBAAW;QAC3B,sBAAsB,EAAE,+CAAyB;QACjD,gBAAgB,EAAE,mCAAmB;QACrC,mBAAmB,EAAE,yCAAsB;QAC3C,IAAI,EAAE,iBAAU;QAChB,MAAM,EAAE,qBAAY;QACpB,UAAU,EAAE,6BAAgB;QAC5B,SAAS,EAAE,0BAAe;QAC1B,eAAe,EAAE,6BAAkB;QACnC,eAAe,EAAE,6BAAkB;QACnC,SAAS,EAAE,mBAAY;QACvB,QAAQ,EAAE,iBAAW;QACrB,+BAA+B,EAAE,wCAA+B;QAChE,4BAA4B,EAAE,qCAA4B;QAC1D,QAAQ,EAAE,iBAAW;QACrB,UAAU,EAAE,6BAAgB;QAC5B,OAAO,EAAE,uBAAa;QACtB,WAAW,EAAE,+BAAiB;QAC9B,YAAY,EAAE,2BAAe;QAC7B,uBAAuB,EAAE,+CAAyB;QAClD,yBAAyB,EAAE,6BAAgB;QAC3C,UAAU,EAAE,6BAAgB;QAC5B,UAAU,EAAE,6BAAgB;QAC5B,kBAAkB,EAAE,kCAAqB;QACzC,MAAM,EAAE,qBAAY;QACpB,KAAK,EAAE,mBAAW;QAClB,SAAS,EAAE,wBAAgB;QAC3B,OAAO,EAAE,uBAAa;QACtB,aAAa,EAAE,6BAAgB;QAC/B,OAAO,EAAE,uBAAa;KACvB,CAAC;IAEF,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAA0C,CAAC,CAAC;IAC9E,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,CAAC;QAE9C,wEAAwE;QACxE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,GAAG,MAAM;gBACT,IAAI,EAAE,IAAI,EAAE,qCAAqC;aAClD,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC,EAAE,CAAC;QAC9C,OAAO,IAAA,uBAAa,EAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;AAC/D,CAAC;AAEM,KAAK,UAAU,aAAa,CAAC,EAClC,MAAM,EACN,QAAQ,EACR,gBAAgB,EAChB,IAAI,EACJ,SAAS,GAOV;IACC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3C,OAAO,mCAAgB,CAAC,eAAe,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,mCAAgB,CAAC;QAC5C,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC;IACH,MAAM,gBAAgB,GAAuB,EAAE,CAAC;IAChD,MAAM,OAAO,GAIP,IAAI,CAAC,MAAM;SACd,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE;QACpB,IAAI,SAAS,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACpC,MAAM,eAAe,GAAG,IAAI,mCAAgB,CAAC;gBAC3C,SAAS,EAAE,SAAS,CAAC,SAAS;gBAC9B,kBAAkB,EAAE;oBAClB,YAAY,EAAE,SAAS;oBACvB,KAAK,EAAE,CAAC;iBACT;aACF,CAAC,CAAC;YAEH,gBAAgB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAEvC,OAAO,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE;gBAC3C,OAAO;oBACL,SAAS,EAAE,SAAS;oBACpB,YAAY,EAAE,eAAe;oBAC7B,KAAK,EAAE,CAAC;iBACT,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IACjE,CAAC,CAAC;SACD,IAAI,EAAE,CAAC;IAEV,MAAM,eAAK,CAAC,cAAc,CACxB,OAAO,EACP,0BAA0B,EAC1B,KAAK,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,KAAK,EAAE,EAAE,EAAE;QAC3C,IAAI,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACzC,yFAAyF;YACzF,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,MAAM;YACN,QAAQ;YACR,gBAAgB;YAChB,SAAS;YACT,IAAI;YACJ,SAAS;SACV,CAAC,CAAC;QAEH,YAAY,CAAC,SAAS,CAAC;YACrB,KAAK;YACL,MAAM;YACN,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,MAAM,EAAE,SAAS,CAAC,MAAM;SACzB,CAAC,CAAC;IACL,CAAC,CACF,CAAC;IAEF,gBAAgB,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,EAAE;QAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,CAAC;QAC5C,MAAM,EACJ,KAAK,EACL,YAAY,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GACjC,GAAG,eAAe,CAAC,kBAAmB,CAAC;QAExC,gBAAgB,CAAC,SAAS,CAAC;YACzB,KAAK;YACL,MAAM;YACN,MAAM;YACN,MAAM;SACP,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,gBAAgB,CAAC,UAAU,EAAE,CAAC;AACvC,CAAC;AAEM,KAAK,UAAU,mBAAmB,CACvC,IAAoB,EACpB,SAAoB,EACpB,OAAiB;IAEjB,IAAA,mBAAS,EAAC,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,EAAE,sCAAsC,CAAC,CAAC;IACvF,IAAI,GAAG,IAAA,oBAAY,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACrC,MAAM,iBAAiB,GAAG,MAAM,IAAA,4BAAiB,EAC/C,SAAS,CAAC,KAAK,EACf,OAAO,EACP,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,IAAI,CACV,CAAC;IACF,OAAO,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACxC,GAAG,MAAM;QACT,SAAS;KACV,CAAC,CAAC,CAAC;AACN,CAAC;AAEM,KAAK,UAAU,cAAc,CAAC,QAAgB;IACnD,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,iBAAI,CAAC,IAAI,CAAC,YAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAgB,CAAC;QAChF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,kCAAkC,QAAQ,MAAM,GAAG,EAAE,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,2DAA2D;AAC3D,kBAAe;IACb,YAAY;IACZ,aAAa;IACb,iBAAiB,EAAjB,4BAAiB;IACjB,qBAAqB,EAArB,gCAAqB;IACrB,gBAAgB,EAAhB,2BAAgB;IAChB,iBAAiB,EAAjB,4BAAiB;IACjB,eAAe,EAAf,0BAAe;IACf,sBAAsB,EAAtB,iCAAsB;IACtB,oBAAoB,EAApB,+BAAoB;IACpB,uBAAuB,EAAvB,kCAAuB;IACvB,0BAA0B,EAA1B,qCAA0B;IAC1B,wBAAwB,EAAE,4BAAiB;IAC3C,iBAAiB,EAAjB,4BAAiB;CAClB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"javascript.d.ts","sourceRoot":"","sources":["../../../src/assertions/javascript.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"javascript.d.ts","sourceRoot":"","sources":["../../../src/assertions/javascript.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,KAAK,aAAa,EAAmB,MAAM,UAAU,CAAC;AAgB/D,eAAO,MAAM,gBAAgB,2FAQ1B,eAAe,KAAG,OAAO,CAAC,aAAa,CA2EzC,CAAC"}
|
|
@@ -4,8 +4,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handleJavascript = void 0;
|
|
7
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
7
|
const types_1 = require("../types");
|
|
8
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
9
9
|
const validateResult = async (result) => {
|
|
10
10
|
result = await Promise.resolve(result);
|
|
11
11
|
if (typeof result === 'boolean' || typeof result === 'number' || (0, types_1.isGradingResult)(result)) {
|
|
@@ -32,7 +32,7 @@ const handleJavascript = async ({ assertion, renderedValue, valueFromScript, con
|
|
|
32
32
|
}
|
|
33
33
|
return ret;
|
|
34
34
|
}
|
|
35
|
-
(0,
|
|
35
|
+
(0, invariant_1.default)(typeof renderedValue === 'string', 'javascript assertion must have a string value');
|
|
36
36
|
/**
|
|
37
37
|
* Removes trailing newline from the rendered value.
|
|
38
38
|
* This is necessary for handling multi-line string literals in YAML
|
|
@@ -50,7 +50,7 @@ const handleJavascript = async ({ assertion, renderedValue, valueFromScript, con
|
|
|
50
50
|
result = await validateResult(customFunction(output, context));
|
|
51
51
|
}
|
|
52
52
|
else {
|
|
53
|
-
(0,
|
|
53
|
+
(0, invariant_1.default)(typeof valueFromScript === 'boolean' ||
|
|
54
54
|
typeof valueFromScript === 'number' ||
|
|
55
55
|
typeof valueFromScript === 'object', `Javascript assertion script must return a boolean, number, or object (${assertion.value})`);
|
|
56
56
|
result = await validateResult(valueFromScript);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"javascript.js","sourceRoot":"","sources":["../../../src/assertions/javascript.ts"],"names":[],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"javascript.js","sourceRoot":"","sources":["../../../src/assertions/javascript.ts"],"names":[],"mappings":";;;;;;AACA,oCAA+D;AAC/D,kEAA0C;AAE1C,MAAM,cAAc,GAAG,KAAK,EAAE,MAAW,EAA6C,EAAE;IACtF,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,OAAO,MAAM,KAAK,SAAS,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,IAAA,uBAAe,EAAC,MAAM,CAAC,EAAE,CAAC;QACzF,OAAO,MAAM,CAAC;IAChB,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CACb,oFAAoF,OAAO,MAAM,KAAK,IAAI,CAAC,SAAS,CAClH,MAAM,CACP,EAAE,CACJ,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AAEK,MAAM,gBAAgB,GAAG,KAAK,EAAE,EACrC,SAAS,EACT,aAAa,EACb,eAAe,EACf,OAAO,EACP,YAAY,EACZ,MAAM,EACN,OAAO,GACS,EAA0B,EAAE;IAC5C,IAAI,IAAI,CAAC;IACT,IAAI,KAAK,CAAC;IACV,IAAI,CAAC;QACH,IAAI,OAAO,SAAS,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YAC1C,IAAI,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACjD,GAAG,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;gBACnB,yEAAyE;gBACzE,MAAM,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAClD,GAAG,CAAC,SAAS,GAAG;oBACd,IAAI,EAAE,YAAY;oBAClB,KAAK,EAAE,cAAc,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,cAAc;iBACzF,CAAC;YACJ,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QACD,IAAA,mBAAS,EAAC,OAAO,aAAa,KAAK,QAAQ,EAAE,+CAA+C,CAAC,CAAC;QAE9F;;;;;;;;WAQG;QACH,aAAa,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC;QAExC,IAAI,MAAwC,CAAC;QAC7C,IAAI,OAAO,eAAe,KAAK,WAAW,EAAE,CAAC;YAC3C,MAAM,YAAY,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,aAAa,EAAE,CAAC;YAC9F,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC,QAAQ,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;YACvE,MAAM,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,IAAA,mBAAS,EACP,OAAO,eAAe,KAAK,SAAS;gBAClC,OAAO,eAAe,KAAK,QAAQ;gBACnC,OAAO,eAAe,KAAK,QAAQ,EACrC,yEAAyE,SAAS,CAAC,KAAK,GAAG,CAC5F,CAAC;YACF,MAAM,GAAG,MAAM,cAAc,CAAC,eAAe,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,GAAG,MAAM,KAAK,OAAO,CAAC;YAC1B,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;aAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtC,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACxE,KAAK,GAAG,MAAM,CAAC;QACjB,CAAC;aAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtC,OAAO,MAAM,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,gCAAiC,GAAa,CAAC,OAAO;eACpD,GAAa,CAAC,KAAK;EACjC,aAAa,EAAE;YACX,SAAS;SACV,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI;QACJ,KAAK;QACL,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,4BAA4B,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;EAC5D,aAAa,EAAE;QACb,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AAnFW,QAAA,gBAAgB,oBAmF3B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/assertions/json.ts"],"names":[],"mappings":"AACA,OAAO,GAAG,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/assertions/json.ts"],"names":[],"mappings":"AACA,OAAO,GAAG,MAAM,KAAK,CAAC;AAItB,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAM/D,wBAAgB,QAAQ,IAAI,IAAI,CAK/B;AAED,wBAAgB,MAAM,IAAI,GAAG,CAS5B;AAED,wBAAgB,YAAY,CAAC,EAC3B,YAAY,EACZ,aAAa,EACb,OAAO,EACP,eAAe,EACf,SAAS,GACV,EAAE,eAAe,GAAG,aAAa,CA8CjC;AAED,wBAAgB,kBAAkB,CAAC,EACjC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,EACP,eAAe,GAChB,EAAE,eAAe,GAAG,aAAa,CAsCjC"}
|
|
@@ -10,8 +10,8 @@ exports.handleContainsJson = handleContainsJson;
|
|
|
10
10
|
const ajv_1 = __importDefault(require("ajv"));
|
|
11
11
|
const ajv_formats_1 = __importDefault(require("ajv-formats"));
|
|
12
12
|
const js_yaml_1 = __importDefault(require("js-yaml"));
|
|
13
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
14
13
|
const envars_1 = require("../envars");
|
|
14
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
15
15
|
const json_1 = require("../util/json");
|
|
16
16
|
let ajvInstance = null;
|
|
17
17
|
function resetAjv() {
|
|
@@ -46,7 +46,7 @@ function handleIsJson({ outputString, renderedValue, inverse, valueFromScript, a
|
|
|
46
46
|
if (renderedValue.startsWith('file://')) {
|
|
47
47
|
// Reference the JSON schema from external file
|
|
48
48
|
const schema = valueFromScript;
|
|
49
|
-
(0,
|
|
49
|
+
(0, invariant_1.default)(schema, 'is-json references a file that does not export a JSON schema');
|
|
50
50
|
validate = getAjv().compile(schema);
|
|
51
51
|
}
|
|
52
52
|
else {
|
|
@@ -88,7 +88,7 @@ function handleContainsJson({ assertion, renderedValue, outputString, inverse, v
|
|
|
88
88
|
if (renderedValue.startsWith('file://')) {
|
|
89
89
|
// Reference the JSON schema from external file
|
|
90
90
|
const schema = valueFromScript;
|
|
91
|
-
(0,
|
|
91
|
+
(0, invariant_1.default)(schema, 'contains-json references a file that does not export a JSON schema');
|
|
92
92
|
validate = getAjv().compile(schema);
|
|
93
93
|
}
|
|
94
94
|
else {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../../src/assertions/json.ts"],"names":[],"mappings":";;;;;AAWA,4BAKC;AAED,wBASC;AAED,oCAoDC;AAED,gDA4CC;AA9HD,8CAAsB;AACtB,8DAAqC;AACrC,sDAA2B;AAC3B,
|
|
1
|
+
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../../src/assertions/json.ts"],"names":[],"mappings":";;;;;AAWA,4BAKC;AAED,wBASC;AAED,oCAoDC;AAED,gDA4CC;AA9HD,8CAAsB;AACtB,8DAAqC;AACrC,sDAA2B;AAC3B,sCAAuC;AAEvC,kEAA0C;AAC1C,uCAAkD;AAElD,IAAI,WAAW,GAAe,IAAI,CAAC;AAEnC,SAAgB,QAAQ;IACtB,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,WAAW,GAAG,IAAI,CAAC;AACrB,CAAC;AAED,SAAgB,MAAM;IACpB,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,UAAU,GAAyC;YACvD,YAAY,EAAE,CAAC,IAAA,mBAAU,EAAC,mCAAmC,CAAC;SAC/D,CAAC;QACF,WAAW,GAAG,IAAI,aAAG,CAAC,UAAU,CAAC,CAAC;QAClC,IAAA,qBAAU,EAAC,WAAW,CAAC,CAAC;IAC1B,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAgB,YAAY,CAAC,EAC3B,YAAY,EACZ,aAAa,EACb,OAAO,EACP,eAAe,EACf,SAAS,GACO;IAChB,IAAI,UAAU,CAAC;IACf,IAAI,IAAI,CAAC;IACT,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACtC,IAAI,GAAG,CAAC,OAAO,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,GAAG,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC;QAC1B,IAAI,QAA0B,CAAC;QAC/B,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;YACtC,IAAI,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBACxC,+CAA+C;gBAC/C,MAAM,MAAM,GAAG,eAAe,CAAC;gBAC/B,IAAA,mBAAS,EAAC,MAAM,EAAE,8DAA8D,CAAC,CAAC;gBAClF,QAAQ,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,MAAgB,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,iBAAI,CAAC,IAAI,CAAC,aAAa,CAAW,CAAC;gBAClD,QAAQ,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;YAC7C,QAAQ,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QAC7C,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC1E,CAAC;QACD,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC5B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,yDAAyD,MAAM,EAAE,CAAC,UAAU,CAClF,QAAQ,CAAC,MAAM,CAChB,EAAE;gBACH,SAAS;aACV,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,kCAAkC;QACtE,SAAS;KACV,CAAC;AACJ,CAAC;AAED,SAAgB,kBAAkB,CAAC,EACjC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,EACP,eAAe,GACC;IAChB,IAAI,YAAY,GAAG,uCAAuC,CAAC;IAC3D,MAAM,WAAW,GAAG,IAAA,yBAAkB,EAAC,YAAY,CAAC,CAAC;IACrD,IAAI,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;IACvE,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,QAA0B,CAAC;YAC/B,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;gBACtC,IAAI,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;oBACxC,+CAA+C;oBAC/C,MAAM,MAAM,GAAG,eAAe,CAAC;oBAC/B,IAAA,mBAAS,EAAC,MAAM,EAAE,oEAAoE,CAAC,CAAC;oBACxF,QAAQ,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,MAAgB,CAAC,CAAC;gBAChD,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,GAAG,iBAAI,CAAC,IAAI,CAAC,aAAa,CAAW,CAAC;oBAClD,QAAQ,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;iBAAM,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;gBAC7C,QAAQ,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;YAChF,CAAC;YACD,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;YAC5B,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM;YACR,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,yDAAyD,MAAM,EAAE,CAAC,UAAU,CACzF,QAAQ,CAAC,MAAM,CAChB,EAAE,CAAC;YACN,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,YAAY;QAChD,SAAS;KACV,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"levenshtein.d.ts","sourceRoot":"","sources":["../../../src/assertions/levenshtein.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"levenshtein.d.ts","sourceRoot":"","sources":["../../../src/assertions/levenshtein.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,wBAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,aAAa,EACb,YAAY,GACb,EAAE,eAAe,GAAG,aAAa,CAejC"}
|
|
@@ -5,9 +5,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handleLevenshtein = handleLevenshtein;
|
|
7
7
|
const fastest_levenshtein_1 = require("fastest-levenshtein");
|
|
8
|
-
const
|
|
8
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
9
9
|
function handleLevenshtein({ assertion, renderedValue, outputString, }) {
|
|
10
|
-
(0,
|
|
10
|
+
(0, invariant_1.default)(typeof renderedValue === 'string', '"levenshtein" assertion type must have a string value');
|
|
11
11
|
const levDistance = (0, fastest_levenshtein_1.distance)(outputString, renderedValue);
|
|
12
12
|
const pass = levDistance <= (assertion.threshold || 5);
|
|
13
13
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"levenshtein.js","sourceRoot":"","sources":["../../../src/assertions/levenshtein.ts"],"names":[],"mappings":";;;;;AAIA,8CAmBC;AAvBD,6DAA+C;
|
|
1
|
+
{"version":3,"file":"levenshtein.js","sourceRoot":"","sources":["../../../src/assertions/levenshtein.ts"],"names":[],"mappings":";;;;;AAIA,8CAmBC;AAvBD,6DAA+C;AAE/C,kEAA0C;AAE1C,SAAgB,iBAAiB,CAAC,EAChC,SAAS,EACT,aAAa,EACb,YAAY,GACI;IAChB,IAAA,mBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,EACjC,uDAAuD,CACxD,CAAC;IACF,MAAM,WAAW,GAAG,IAAA,8BAAQ,EAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG,WAAW,IAAI,CAAC,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC;IACvD,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,wBAAwB,WAAW,8BAA8B,SAAS,CAAC,SAAS,IAAI,CAAC,EAAE;QAC/F,SAAS;KACV,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llmRubric.d.ts","sourceRoot":"","sources":["../../../src/assertions/llmRubric.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"llmRubric.d.ts","sourceRoot":"","sources":["../../../src/assertions/llmRubric.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,eAAO,MAAM,eAAe,sDAKzB,eAAe,KAAG,OAAO,CAAC,aAAa,CAezC,CAAC"}
|
|
@@ -4,10 +4,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handleLlmRubric = void 0;
|
|
7
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
7
|
const matchers_1 = require("../matchers");
|
|
8
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
9
9
|
const handleLlmRubric = async ({ assertion, renderedValue, outputString, test, }) => {
|
|
10
|
-
(0,
|
|
10
|
+
(0, invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'undefined', '"llm-rubric" assertion type must have a string value');
|
|
11
11
|
if (test.options?.rubricPrompt && typeof test.options.rubricPrompt === 'object') {
|
|
12
12
|
test.options.rubricPrompt = JSON.stringify(test.options.rubricPrompt);
|
|
13
13
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llmRubric.js","sourceRoot":"","sources":["../../../src/assertions/llmRubric.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"llmRubric.js","sourceRoot":"","sources":["../../../src/assertions/llmRubric.ts"],"names":[],"mappings":";;;;;;AAAA,0CAA+C;AAE/C,kEAA0C;AAEnC,MAAM,eAAe,GAAG,KAAK,EAAE,EACpC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,IAAI,GACY,EAA0B,EAAE;IAC5C,IAAA,mBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,IAAI,OAAO,aAAa,KAAK,WAAW,EACzE,sDAAsD,CACvD,CAAC;IACF,IAAI,IAAI,CAAC,OAAO,EAAE,YAAY,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;QAChF,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACxE,CAAC;IAED,8EAA8E;IAC9E,SAAS,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC;IAChE,OAAO;QACL,SAAS;QACT,GAAG,CAAC,MAAM,IAAA,2BAAgB,EAAC,aAAa,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;KACxF,CAAC;AACJ,CAAC,CAAC;AApBW,QAAA,eAAe,mBAoB1B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"modelGradedClosedQa.d.ts","sourceRoot":"","sources":["../../../src/assertions/modelGradedClosedQa.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"modelGradedClosedQa.d.ts","sourceRoot":"","sources":["../../../src/assertions/modelGradedClosedQa.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAI/D,eAAO,MAAM,yBAAyB,8DAMnC,eAAe,KAAG,OAAO,CAAC,aAAa,CAiBzC,CAAC"}
|
|
@@ -4,15 +4,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handleModelGradedClosedQa = void 0;
|
|
7
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
7
|
const matchers_1 = require("../matchers");
|
|
8
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
9
9
|
const templates_1 = require("../util/templates");
|
|
10
10
|
const handleModelGradedClosedQa = async ({ assertion, renderedValue, outputString, test, prompt, }) => {
|
|
11
|
-
(0,
|
|
12
|
-
(0,
|
|
11
|
+
(0, invariant_1.default)(typeof renderedValue === 'string', 'model-graded-closedqa assertion type must have a string value');
|
|
12
|
+
(0, invariant_1.default)(prompt, 'model-graded-closedqa assertion type must have a prompt');
|
|
13
13
|
if (test.options?.rubricPrompt) {
|
|
14
14
|
// Substitute vars in prompt
|
|
15
|
-
(0,
|
|
15
|
+
(0, invariant_1.default)(typeof test.options.rubricPrompt === 'string', 'rubricPrompt must be a string');
|
|
16
16
|
const nunjucks = (0, templates_1.getNunjucksEngine)();
|
|
17
17
|
test.options.rubricPrompt = nunjucks.renderString(test.options.rubricPrompt, test.vars || {});
|
|
18
18
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"modelGradedClosedQa.js","sourceRoot":"","sources":["../../../src/assertions/modelGradedClosedQa.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"modelGradedClosedQa.js","sourceRoot":"","sources":["../../../src/assertions/modelGradedClosedQa.ts"],"names":[],"mappings":";;;;;;AAAA,0CAA8C;AAE9C,kEAA0C;AAC1C,iDAAsD;AAE/C,MAAM,yBAAyB,GAAG,KAAK,EAAE,EAC9C,SAAS,EACT,aAAa,EACb,YAAY,EACZ,IAAI,EACJ,MAAM,GACU,EAA0B,EAAE;IAC5C,IAAA,mBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,EACjC,+DAA+D,CAChE,CAAC;IACF,IAAA,mBAAS,EAAC,MAAM,EAAE,yDAAyD,CAAC,CAAC;IAC7E,IAAI,IAAI,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC;QAC/B,4BAA4B;QAC5B,IAAA,mBAAS,EAAC,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,KAAK,QAAQ,EAAE,+BAA+B,CAAC,CAAC;QAC1F,MAAM,QAAQ,GAAG,IAAA,6BAAiB,GAAE,CAAC;QACrC,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,OAAO;QACL,SAAS;QACT,GAAG,CAAC,MAAM,IAAA,0BAAe,EAAC,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;KACzF,CAAC;AACJ,CAAC,CAAC;AAvBW,QAAA,yBAAyB,6BAuBpC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"moderation.d.ts","sourceRoot":"","sources":["../../../src/assertions/moderation.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"moderation.d.ts","sourceRoot":"","sources":["../../../src/assertions/moderation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,eAAO,MAAM,gBAAgB,iEAM1B,eAAe,KAAG,OAAO,CAAC,aAAa,CAuCzC,CAAC"}
|
|
@@ -4,14 +4,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handleModeration = void 0;
|
|
7
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
7
|
const matchers_1 = require("../matchers");
|
|
9
8
|
const shared_1 = require("../providers/shared");
|
|
9
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
10
10
|
const handleModeration = async ({ assertion, test, outputString, providerResponse, prompt, }) => {
|
|
11
11
|
// Some redteam techniques override the actual prompt that is used, so we need to assess that prompt for moderation.
|
|
12
12
|
const promptToModerate = providerResponse.metadata?.redteamFinalPrompt || prompt;
|
|
13
|
-
(0,
|
|
14
|
-
(0,
|
|
13
|
+
(0, invariant_1.default)(promptToModerate, 'moderation assertion type must have a prompt');
|
|
14
|
+
(0, invariant_1.default)(!assertion.value || (Array.isArray(assertion.value) && typeof assertion.value[0] === 'string'), 'moderation assertion value must be a string array if set');
|
|
15
15
|
if (promptToModerate[0] === '[' || promptToModerate[0] === '{') {
|
|
16
16
|
// Try to extract the last user message from OpenAI-style prompts.
|
|
17
17
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"moderation.js","sourceRoot":"","sources":["../../../src/assertions/moderation.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"moderation.js","sourceRoot":"","sources":["../../../src/assertions/moderation.ts"],"names":[],"mappings":";;;;;;AAAA,0CAAgD;AAChD,gDAAsD;AAEtD,kEAA0C;AAEnC,MAAM,gBAAgB,GAAG,KAAK,EAAE,EACrC,SAAS,EACT,IAAI,EACJ,YAAY,EACZ,gBAAgB,EAChB,MAAM,GACU,EAA0B,EAAE;IAC5C,oHAAoH;IACpH,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,kBAAkB,IAAI,MAAM,CAAC;IACjF,IAAA,mBAAS,EAAC,gBAAgB,EAAE,8CAA8C,CAAC,CAAC;IAC5E,IAAA,mBAAS,EACP,CAAC,SAAS,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,EAC9F,0DAA0D,CAC3D,CAAC;IACF,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;QAC/D,kEAAkE;QAClE,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,IAAA,wBAAe,EAClC,gBAAgB,EAChB,IAAI,CACL,CAAC;YACF,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,MAAM,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;YACzD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IAED,MAAM,gBAAgB,GAAG,MAAM,IAAA,4BAAiB,EAC9C;QACE,UAAU,EAAE,gBAAgB;QAC5B,iBAAiB,EAAE,YAAY;QAC/B,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;KAClE,EACD,IAAI,CAAC,OAAO,CACb,CAAC;IAEF,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAC;IACnC,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,gBAAgB,CAAC,KAAK;QAC7B,MAAM,EAAE,gBAAgB,CAAC,MAAM;QAC/B,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AA7CW,QAAA,gBAAgB,oBA6C3B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"python.d.ts","sourceRoot":"","sources":["../../../src/assertions/python.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"python.d.ts","sourceRoot":"","sources":["../../../src/assertions/python.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,KAAK,aAAa,EAAmB,MAAM,UAAU,CAAC;AAG/D,eAAO,MAAM,YAAY,oEAMtB,eAAe,KAAG,OAAO,CAAC,aAAa,CA0GzC,CAAC"}
|
|
@@ -4,11 +4,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handlePython = void 0;
|
|
7
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
7
|
const wrapper_1 = require("../python/wrapper");
|
|
9
8
|
const types_1 = require("../types");
|
|
9
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
10
10
|
const handlePython = async ({ assertion, renderedValue, valueFromScript, context, output, }) => {
|
|
11
|
-
(0,
|
|
11
|
+
(0, invariant_1.default)(typeof renderedValue === 'string', 'python assertion must have a string value');
|
|
12
12
|
let pass;
|
|
13
13
|
let score;
|
|
14
14
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"python.js","sourceRoot":"","sources":["../../../src/assertions/python.ts"],"names":[],"mappings":";;;;;;AAAA
|
|
1
|
+
{"version":3,"file":"python.js","sourceRoot":"","sources":["../../../src/assertions/python.ts"],"names":[],"mappings":";;;;;;AAAA,+CAAkD;AAElD,oCAA+D;AAC/D,kEAA0C;AAEnC,MAAM,YAAY,GAAG,KAAK,EAAE,EACjC,SAAS,EACT,aAAa,EACb,eAAe,EACf,OAAO,EACP,MAAM,GACU,EAA0B,EAAE;IAC5C,IAAA,mBAAS,EAAC,OAAO,aAAa,KAAK,QAAQ,EAAE,2CAA2C,CAAC,CAAC;IAC1F,IAAI,IAAI,CAAC;IACT,IAAI,KAAK,CAAC;IACV,IAAI,CAAC;QACH,IAAI,MAAsE,CAAC;QAC3E,IAAI,OAAO,eAAe,KAAK,WAAW,EAAE,CAAC;YAC3C,MAAM,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACjD,IAAI,WAAW,GAAG,MAAM,CAAC;YACzB,IAAI,WAAW,EAAE,CAAC;gBAChB,0DAA0D;gBAC1D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBACnD,IAAI,KAAK,EAAE,CAAC;oBACV,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;YAED,MAAM,YAAY,GAAG;;;EAIzB,WAAW;gBACT,CAAC,CAAC,aAAa;qBACV,KAAK,CAAC,IAAI,CAAC;qBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,WAAW,GAAG,IAAI,EAAE,CAAC;qBACtC,IAAI,CAAC,IAAI,CAAC;gBACf,CAAC,CAAC,cAAc,aAAa,EACjC;CACC,CAAC;YACI,MAAM,GAAG,MAAM,IAAA,uBAAa,EAAC,YAAY,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QACxE,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,eAAe,CAAC;QAC3B,CAAC;QAED,IACE,CAAC,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC;YACvC,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,EAC/D,CAAC;YACD,IAAI,GAAG,IAAI,CAAC;YACZ,KAAK,GAAG,GAAG,CAAC;QACd,CAAC;aAAM,IACL,CAAC,OAAO,MAAM,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC;YACxC,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,EAChE,CAAC;YACD,IAAI,GAAG,KAAK,CAAC;YACb,KAAK,GAAG,GAAG,CAAC;QACd,CAAC;aAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAChE,IAAI,MAAM,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,yBAAyB,MAAM,EAAE,CAAC,CAAC;YACzE,CAAC;YACD,IAAI,CAAC,IAAA,uBAAe,EAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,iGAAiG,MAAM,EAAE,CAC1G,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;aAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtC,IAAI,CAAC,IAAA,uBAAe,EAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,kGAAkG,IAAI,CAAC,SAAS,CAC9G,MAAM,EACN,IAAI,EACJ,CAAC,CACF,EAAE,CACJ,CAAC;YACJ,CAAC;YACD,MAAM,mBAAmB,GAAG,MAA0C,CAAC;YACvE,IAAI,SAAS,CAAC,SAAS,IAAI,mBAAmB,CAAC,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC3E,mBAAmB,CAAC,IAAI,GAAG,KAAK,CAAC;gBACjC,mBAAmB,CAAC,MAAM,GAAG,gBAAgB,mBAAmB,CAAC,KAAK,2BAA2B,SAAS,CAAC,SAAS,EAAE,CAAC;YACzH,CAAC;YACD,OAAO;gBACL,GAAG,mBAAmB;gBACtB,SAAS;aACV,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YAC1C,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;YACtE,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CACb,kGAAkG,MAAM,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,SAAS,CAAC,SAAS,KAAK,WAAW,IAAI,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC9E,IAAI,GAAG,KAAK,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,iCAAkC,GAAa,CAAC,OAAO,EAAE;YACjE,SAAS;SACV,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI;QACJ,KAAK;QACL,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,wBAAwB,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,KAAK,EAAE;QACzE,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AAhHW,QAAA,YAAY,gBAgHvB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/assertions/redteam.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/assertions/redteam.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,eAAO,MAAM,aAAa,kFAQvB,eAAe,KAAG,OAAO,CAAC,aAAa,CAwBzC,CAAC"}
|
|
@@ -4,12 +4,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handleRedteam = void 0;
|
|
7
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
7
|
const graders_1 = require("../redteam/graders");
|
|
8
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
9
9
|
const handleRedteam = async ({ assertion, baseType, test, prompt, outputString, provider, renderedValue, }) => {
|
|
10
10
|
const grader = (0, graders_1.getGraderById)(assertion.type);
|
|
11
|
-
(0,
|
|
12
|
-
(0,
|
|
11
|
+
(0, invariant_1.default)(grader, `Unknown grader: ${baseType}`);
|
|
12
|
+
(0, invariant_1.default)(prompt, `Grader ${baseType} must have a prompt`);
|
|
13
13
|
const { grade, rubric, suggestions } = await grader.getResult(prompt, outputString, test, provider, renderedValue);
|
|
14
14
|
return {
|
|
15
15
|
assertion: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/assertions/redteam.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/assertions/redteam.ts"],"names":[],"mappings":";;;;;;AAAA,gDAAmD;AAEnD,kEAA0C;AAEnC,MAAM,aAAa,GAAG,KAAK,EAAE,EAClC,SAAS,EACT,QAAQ,EACR,IAAI,EACJ,MAAM,EACN,YAAY,EACZ,QAAQ,EACR,aAAa,GACG,EAA0B,EAAE;IAC5C,MAAM,MAAM,GAAG,IAAA,uBAAa,EAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAA,mBAAS,EAAC,MAAM,EAAE,mBAAmB,QAAQ,EAAE,CAAC,CAAC;IACjD,IAAA,mBAAS,EAAC,MAAM,EAAE,UAAU,QAAQ,qBAAqB,CAAC,CAAC;IAC3D,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAC3D,MAAM,EACN,YAAY,EACZ,IAAI,EACJ,QAAQ,EACR,aAAa,CACd,CAAC;IACF,OAAO;QACL,SAAS,EAAE;YACT,GAAG,SAAS;YACZ,KAAK,EAAE,MAAM;SACd;QACD,GAAG,KAAK;QACR,WAAW;QACX,QAAQ,EAAE;YACR,6CAA6C;YAC7C,GAAG,IAAI,CAAC,QAAQ;YAChB,GAAG,KAAK,CAAC,QAAQ;SAClB;KACF,CAAC;AACJ,CAAC,CAAC;AAhCW,QAAA,aAAa,iBAgCxB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"regex.d.ts","sourceRoot":"","sources":["../../../src/assertions/regex.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"regex.d.ts","sourceRoot":"","sources":["../../../src/assertions/regex.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,eAAO,MAAM,WAAW,yDAKrB,eAAe,KAAG,aAuBpB,CAAC"}
|
|
@@ -4,10 +4,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handleRegex = void 0;
|
|
7
|
-
const
|
|
7
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
8
8
|
const handleRegex = ({ assertion, renderedValue, outputString, inverse, }) => {
|
|
9
|
-
(0,
|
|
10
|
-
(0,
|
|
9
|
+
(0, invariant_1.default)(renderedValue, '"regex" assertion type must have a string value');
|
|
10
|
+
(0, invariant_1.default)(typeof renderedValue === 'string', '"regex" assertion type must have a string value');
|
|
11
11
|
let regex;
|
|
12
12
|
try {
|
|
13
13
|
regex = new RegExp(renderedValue);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"regex.js","sourceRoot":"","sources":["../../../src/assertions/regex.ts"],"names":[],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"regex.js","sourceRoot":"","sources":["../../../src/assertions/regex.ts"],"names":[],"mappings":";;;;;;AACA,kEAA0C;AAEnC,MAAM,WAAW,GAAG,CAAC,EAC1B,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAAiB,EAAE;IACnC,IAAA,mBAAS,EAAC,aAAa,EAAE,iDAAiD,CAAC,CAAC;IAC5E,IAAA,mBAAS,EAAC,OAAO,aAAa,KAAK,QAAQ,EAAE,iDAAiD,CAAC,CAAC;IAChG,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE;YAC5F,SAAS;SACV,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,OAAO,CAAC;IAClD,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,gBAAgB,aAAa,GAAG;QAC/E,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AA5BW,QAAA,WAAW,eA4BtB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rouge.d.ts","sourceRoot":"","sources":["../../../src/assertions/rouge.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"rouge.d.ts","sourceRoot":"","sources":["../../../src/assertions/rouge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,wBAAgB,gBAAgB,CAAC,EAC/B,QAAQ,EACR,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACR,EAAE,eAAe,GAAG,aAAa,CAkBjC"}
|
|
@@ -38,9 +38,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
39
|
exports.handleRougeScore = handleRougeScore;
|
|
40
40
|
const rouge = __importStar(require("js-rouge"));
|
|
41
|
-
const
|
|
41
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
42
42
|
function handleRougeScore({ baseType, assertion, renderedValue, outputString, inverse, }) {
|
|
43
|
-
(0,
|
|
43
|
+
(0, invariant_1.default)(typeof renderedValue === 'string', '"rouge" assertion type must be a string value');
|
|
44
44
|
const fnName = baseType[baseType.length - 1];
|
|
45
45
|
const rougeMethod = rouge[fnName];
|
|
46
46
|
const score = rougeMethod(outputString, renderedValue, {});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rouge.js","sourceRoot":"","sources":["../../../src/assertions/rouge.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,4CAwBC;AA5BD,gDAAkC;
|
|
1
|
+
{"version":3,"file":"rouge.js","sourceRoot":"","sources":["../../../src/assertions/rouge.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,4CAwBC;AA5BD,gDAAkC;AAElC,kEAA0C;AAE1C,SAAgB,gBAAgB,CAAC,EAC/B,QAAQ,EACR,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS;IAChB,IAAA,mBAAS,EAAC,OAAO,aAAa,KAAK,QAAQ,EAAE,+CAA+C,CAAC,CAAC;IAC9F,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAoB,CAAC;IAChE,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,EAAE,aAAa,EAAE,EAAE,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;IAC/D,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK;QAClC,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,UAAU,KAAK,CAAC,OAAO,CAC9C,CAAC,CACF,0CAA0C,SAAS,CAAC,SAAS,IAAI,IAAI,EAAE;YAC1E,CAAC,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,UAAU,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BACjD,SAAS,CAAC,SAAS,IAAI,IACzB,EAAE;QACN,SAAS;KACV,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"similar.d.ts","sourceRoot":"","sources":["../../../src/assertions/similar.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"similar.d.ts","sourceRoot":"","sources":["../../../src/assertions/similar.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,eAAO,MAAM,aAAa,+DAMvB,eAAe,KAAG,OAAO,CAAC,aAAa,CA2CzC,CAAC"}
|
|
@@ -4,10 +4,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.handleSimilar = void 0;
|
|
7
|
-
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
7
|
const matchers_1 = require("../matchers");
|
|
8
|
+
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
9
9
|
const handleSimilar = async ({ assertion, renderedValue, outputString, inverse, test, }) => {
|
|
10
|
-
(0,
|
|
10
|
+
(0, invariant_1.default)(typeof renderedValue === 'string' || Array.isArray(renderedValue), 'Similarity assertion type must have a string or array of strings value');
|
|
11
11
|
if (Array.isArray(renderedValue)) {
|
|
12
12
|
let minScore = Infinity;
|
|
13
13
|
for (const value of renderedValue) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"similar.js","sourceRoot":"","sources":["../../../src/assertions/similar.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"similar.js","sourceRoot":"","sources":["../../../src/assertions/similar.ts"],"names":[],"mappings":";;;;;;AAAA,0CAAgD;AAEhD,kEAA0C;AAEnC,MAAM,aAAa,GAAG,KAAK,EAAE,EAClC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,EACP,IAAI,GACY,EAA0B,EAAE;IAC5C,IAAA,mBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EACjE,wEAAwE,CACzE,CAAC;IACF,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QACjC,IAAI,QAAQ,GAAG,QAAQ,CAAC;QACxB,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,MAAM,IAAA,4BAAiB,EACpC,KAAK,EACL,YAAY,EACZ,SAAS,CAAC,SAAS,IAAI,IAAI,EAC3B,OAAO,EACP,IAAI,CAAC,OAAO,CACb,CAAC;YACF,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,OAAO;oBACL,SAAS;oBACT,GAAG,MAAM;iBACV,CAAC;YACJ,CAAC;YACD,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,EAAE,CAAC;gBAC5B,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;YAC1B,CAAC;QACH,CAAC;QACD,OAAO;YACL,SAAS;YACT,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,QAAQ;YACf,MAAM,EAAE,0DAA0D;SACnE,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,SAAS;YACT,GAAG,CAAC,MAAM,IAAA,4BAAiB,EACzB,aAAa,EACb,YAAY,EACZ,SAAS,CAAC,SAAS,IAAI,IAAI,EAC3B,OAAO,EACP,IAAI,CAAC,OAAO,CACb,CAAC;SACH,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AAjDW,QAAA,aAAa,iBAiDxB"}
|