promptfoo 0.119.0 → 0.119.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +10 -10
- package/dist/src/app/assets/index-DqlKP0sF.js +72 -0
- package/dist/src/app/assets/{index-BmEhbU1h.js.map → index-DqlKP0sF.js.map} +1 -1
- package/dist/src/app/assets/{source-map-support-5rOGWFat.js → source-map-support-eCtSvaxE.js} +1 -1
- package/dist/src/app/assets/{source-map-support-5rOGWFat.js.map → source-map-support-eCtSvaxE.js.map} +1 -1
- package/dist/src/app/index.html +1 -1
- package/dist/src/assertions/answerRelevance.d.ts +1 -1
- package/dist/src/assertions/answerRelevance.d.ts.map +1 -1
- package/dist/src/assertions/answerRelevance.js +2 -2
- package/dist/src/assertions/answerRelevance.js.map +1 -1
- package/dist/src/assertions/contextFaithfulness.d.ts +1 -1
- package/dist/src/assertions/contextFaithfulness.d.ts.map +1 -1
- package/dist/src/assertions/contextFaithfulness.js +2 -2
- package/dist/src/assertions/contextFaithfulness.js.map +1 -1
- package/dist/src/assertions/contextRecall.d.ts +1 -1
- package/dist/src/assertions/contextRecall.d.ts.map +1 -1
- package/dist/src/assertions/contextRecall.js +2 -2
- package/dist/src/assertions/contextRecall.js.map +1 -1
- package/dist/src/assertions/contextRelevance.d.ts +1 -1
- package/dist/src/assertions/contextRelevance.d.ts.map +1 -1
- package/dist/src/assertions/contextRelevance.js +2 -2
- package/dist/src/assertions/contextRelevance.js.map +1 -1
- package/dist/src/assertions/factuality.d.ts +1 -1
- package/dist/src/assertions/factuality.d.ts.map +1 -1
- package/dist/src/assertions/factuality.js +2 -2
- package/dist/src/assertions/factuality.js.map +1 -1
- package/dist/src/assertions/geval.d.ts +1 -1
- package/dist/src/assertions/geval.d.ts.map +1 -1
- package/dist/src/assertions/geval.js +3 -3
- package/dist/src/assertions/geval.js.map +1 -1
- package/dist/src/assertions/index.d.ts +2 -2
- package/dist/src/assertions/index.d.ts.map +1 -1
- package/dist/src/assertions/index.js +12 -3
- package/dist/src/assertions/index.js.map +1 -1
- package/dist/src/assertions/javascript.d.ts +1 -1
- package/dist/src/assertions/javascript.d.ts.map +1 -1
- package/dist/src/assertions/javascript.js +3 -3
- package/dist/src/assertions/javascript.js.map +1 -1
- package/dist/src/assertions/llmRubric.d.ts +1 -1
- package/dist/src/assertions/llmRubric.d.ts.map +1 -1
- package/dist/src/assertions/llmRubric.js +2 -2
- package/dist/src/assertions/llmRubric.js.map +1 -1
- package/dist/src/assertions/modelGradedClosedQa.d.ts +1 -1
- package/dist/src/assertions/modelGradedClosedQa.d.ts.map +1 -1
- package/dist/src/assertions/modelGradedClosedQa.js +2 -2
- package/dist/src/assertions/modelGradedClosedQa.js.map +1 -1
- package/dist/src/assertions/python.d.ts +1 -1
- package/dist/src/assertions/python.d.ts.map +1 -1
- package/dist/src/assertions/python.js +2 -2
- package/dist/src/assertions/python.js.map +1 -1
- package/dist/src/assertions/ruby.d.ts +1 -1
- package/dist/src/assertions/ruby.d.ts.map +1 -1
- package/dist/src/assertions/ruby.js +2 -2
- package/dist/src/assertions/ruby.js.map +1 -1
- package/dist/src/assertions/traceErrorSpans.d.ts +1 -1
- package/dist/src/assertions/traceErrorSpans.d.ts.map +1 -1
- package/dist/src/assertions/traceErrorSpans.js +3 -3
- package/dist/src/assertions/traceErrorSpans.js.map +1 -1
- package/dist/src/assertions/traceSpanCount.d.ts +1 -1
- package/dist/src/assertions/traceSpanCount.d.ts.map +1 -1
- package/dist/src/assertions/traceSpanCount.js +3 -3
- package/dist/src/assertions/traceSpanCount.js.map +1 -1
- package/dist/src/assertions/traceSpanDuration.d.ts +1 -1
- package/dist/src/assertions/traceSpanDuration.d.ts.map +1 -1
- package/dist/src/assertions/traceSpanDuration.js +3 -3
- package/dist/src/assertions/traceSpanDuration.js.map +1 -1
- package/dist/src/commands/init.d.ts +1 -0
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/init.js +29 -0
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/commands/share.d.ts.map +1 -1
- package/dist/src/commands/share.js +23 -5
- package/dist/src/commands/share.js.map +1 -1
- package/dist/src/database/tables.d.ts +136 -48
- package/dist/src/database/tables.d.ts.map +1 -1
- package/dist/src/envars.d.ts +4 -0
- package/dist/src/envars.d.ts.map +1 -1
- package/dist/src/envars.js.map +1 -1
- package/dist/src/evaluator.d.ts +1 -1
- package/dist/src/evaluator.d.ts.map +1 -1
- package/dist/src/evaluator.js +20 -3
- package/dist/src/evaluator.js.map +1 -1
- package/dist/src/generated/constants.js +1 -1
- package/dist/src/matchers.d.ts +10 -10
- package/dist/src/matchers.d.ts.map +1 -1
- package/dist/src/matchers.js +81 -30
- package/dist/src/matchers.js.map +1 -1
- package/dist/src/microsoftSharepoint.d.ts +12 -0
- package/dist/src/microsoftSharepoint.d.ts.map +1 -0
- package/dist/src/microsoftSharepoint.js +154 -0
- package/dist/src/microsoftSharepoint.js.map +1 -0
- package/dist/src/models/eval.d.ts.map +1 -1
- package/dist/src/models/eval.js +26 -26
- package/dist/src/models/eval.js.map +1 -1
- package/dist/src/providers/google/live.d.ts +8 -0
- package/dist/src/providers/google/live.d.ts.map +1 -1
- package/dist/src/providers/google/live.js +44 -16
- package/dist/src/providers/google/live.js.map +1 -1
- package/dist/src/providers/http.d.ts +2 -2
- package/dist/src/providers/http.d.ts.map +1 -1
- package/dist/src/providers/http.js +16 -0
- package/dist/src/providers/http.js.map +1 -1
- package/dist/src/providers/index.d.ts +1 -1
- package/dist/src/providers/index.d.ts.map +1 -1
- package/dist/src/providers/index.js +42 -6
- package/dist/src/providers/index.js.map +1 -1
- package/dist/src/providers/simulatedUser.d.ts +20 -0
- package/dist/src/providers/simulatedUser.d.ts.map +1 -1
- package/dist/src/providers/simulatedUser.js +92 -1
- package/dist/src/providers/simulatedUser.js.map +1 -1
- package/dist/src/python/persistent_wrapper.py +95 -11
- package/dist/src/python/worker.d.ts +1 -2
- package/dist/src/python/worker.d.ts.map +1 -1
- package/dist/src/python/worker.js +2 -2
- package/dist/src/python/worker.js.map +1 -1
- package/dist/src/redteam/commands/generate.d.ts.map +1 -1
- package/dist/src/redteam/commands/generate.js +0 -1
- package/dist/src/redteam/commands/generate.js.map +1 -1
- package/dist/src/redteam/constants/frameworks.d.ts +1 -1
- package/dist/src/redteam/constants/frameworks.d.ts.map +1 -1
- package/dist/src/redteam/constants/frameworks.js +49 -12
- package/dist/src/redteam/constants/frameworks.js.map +1 -1
- package/dist/src/redteam/constants/metadata.d.ts.map +1 -1
- package/dist/src/redteam/constants/metadata.js +61 -4
- package/dist/src/redteam/constants/metadata.js.map +1 -1
- package/dist/src/redteam/constants/plugins.d.ts +10 -4
- package/dist/src/redteam/constants/plugins.d.ts.map +1 -1
- package/dist/src/redteam/constants/plugins.js +26 -1
- package/dist/src/redteam/constants/plugins.js.map +1 -1
- package/dist/src/redteam/constants/strategies.d.ts +11 -3
- package/dist/src/redteam/constants/strategies.d.ts.map +1 -1
- package/dist/src/redteam/constants/strategies.js +21 -3
- package/dist/src/redteam/constants/strategies.js.map +1 -1
- package/dist/src/redteam/graders.d.ts.map +1 -1
- package/dist/src/redteam/graders.js +14 -0
- package/dist/src/redteam/graders.js.map +1 -1
- package/dist/src/redteam/index.d.ts +1 -22
- package/dist/src/redteam/index.d.ts.map +1 -1
- package/dist/src/redteam/index.js +200 -138
- package/dist/src/redteam/index.js.map +1 -1
- package/dist/src/redteam/plugins/aegis.d.ts +3 -3
- package/dist/src/redteam/plugins/aegis.d.ts.map +1 -1
- package/dist/src/redteam/plugins/aegis.js +2 -2
- package/dist/src/redteam/plugins/aegis.js.map +1 -1
- package/dist/src/redteam/plugins/asciiSmuggling.d.ts +3 -3
- package/dist/src/redteam/plugins/asciiSmuggling.d.ts.map +1 -1
- package/dist/src/redteam/plugins/asciiSmuggling.js +1 -1
- package/dist/src/redteam/plugins/asciiSmuggling.js.map +1 -1
- package/dist/src/redteam/plugins/base.d.ts +6 -1
- package/dist/src/redteam/plugins/base.d.ts.map +1 -1
- package/dist/src/redteam/plugins/base.js +18 -15
- package/dist/src/redteam/plugins/base.js.map +1 -1
- package/dist/src/redteam/plugins/beavertails.d.ts +3 -3
- package/dist/src/redteam/plugins/beavertails.d.ts.map +1 -1
- package/dist/src/redteam/plugins/beavertails.js +7 -2
- package/dist/src/redteam/plugins/beavertails.js.map +1 -1
- package/dist/src/redteam/plugins/crossSessionLeak.d.ts +3 -3
- package/dist/src/redteam/plugins/crossSessionLeak.d.ts.map +1 -1
- package/dist/src/redteam/plugins/crossSessionLeak.js +1 -1
- package/dist/src/redteam/plugins/crossSessionLeak.js.map +1 -1
- package/dist/src/redteam/plugins/goalMisalignment.d.ts +18 -0
- package/dist/src/redteam/plugins/goalMisalignment.d.ts.map +1 -0
- package/dist/src/redteam/plugins/goalMisalignment.js +548 -0
- package/dist/src/redteam/plugins/goalMisalignment.js.map +1 -0
- package/dist/src/redteam/plugins/harmful/graders.d.ts +3 -3
- package/dist/src/redteam/plugins/harmful/graders.d.ts.map +1 -1
- package/dist/src/redteam/plugins/harmful/graders.js +2 -2
- package/dist/src/redteam/plugins/harmful/graders.js.map +1 -1
- package/dist/src/redteam/plugins/insurance/coverageDiscrimination.d.ts +11 -0
- package/dist/src/redteam/plugins/insurance/coverageDiscrimination.d.ts.map +1 -0
- package/dist/src/redteam/plugins/insurance/coverageDiscrimination.js +81 -0
- package/dist/src/redteam/plugins/insurance/coverageDiscrimination.js.map +1 -0
- package/dist/src/redteam/plugins/insurance/networkMisinformation.d.ts +11 -0
- package/dist/src/redteam/plugins/insurance/networkMisinformation.d.ts.map +1 -0
- package/dist/src/redteam/plugins/insurance/networkMisinformation.js +81 -0
- package/dist/src/redteam/plugins/insurance/networkMisinformation.js.map +1 -0
- package/dist/src/redteam/plugins/insurance/phiDisclosure.d.ts +11 -0
- package/dist/src/redteam/plugins/insurance/phiDisclosure.d.ts.map +1 -0
- package/dist/src/redteam/plugins/insurance/phiDisclosure.js +78 -0
- package/dist/src/redteam/plugins/insurance/phiDisclosure.js.map +1 -0
- package/dist/src/redteam/plugins/intent.d.ts +3 -3
- package/dist/src/redteam/plugins/intent.d.ts.map +1 -1
- package/dist/src/redteam/plugins/intent.js +2 -2
- package/dist/src/redteam/plugins/intent.js.map +1 -1
- package/dist/src/redteam/plugins/pharmacy/controlledSubstanceCompliance.d.ts +11 -0
- package/dist/src/redteam/plugins/pharmacy/controlledSubstanceCompliance.d.ts.map +1 -0
- package/dist/src/redteam/plugins/pharmacy/controlledSubstanceCompliance.js +76 -0
- package/dist/src/redteam/plugins/pharmacy/controlledSubstanceCompliance.js.map +1 -0
- package/dist/src/redteam/plugins/pharmacy/dosageCalculation.d.ts +11 -0
- package/dist/src/redteam/plugins/pharmacy/dosageCalculation.d.ts.map +1 -0
- package/dist/src/redteam/plugins/pharmacy/dosageCalculation.js +75 -0
- package/dist/src/redteam/plugins/pharmacy/dosageCalculation.js.map +1 -0
- package/dist/src/redteam/plugins/pharmacy/drugInteraction.d.ts +11 -0
- package/dist/src/redteam/plugins/pharmacy/drugInteraction.d.ts.map +1 -0
- package/dist/src/redteam/plugins/pharmacy/drugInteraction.js +69 -0
- package/dist/src/redteam/plugins/pharmacy/drugInteraction.js.map +1 -0
- package/dist/src/redteam/plugins/policy/index.d.ts +3 -3
- package/dist/src/redteam/plugins/policy/index.d.ts.map +1 -1
- package/dist/src/redteam/plugins/policy/index.js +2 -2
- package/dist/src/redteam/plugins/policy/index.js.map +1 -1
- package/dist/src/redteam/plugins/policy/utils.d.ts +2 -1
- package/dist/src/redteam/plugins/policy/utils.d.ts.map +1 -1
- package/dist/src/redteam/plugins/policy/utils.js +21 -3
- package/dist/src/redteam/plugins/policy/utils.js.map +1 -1
- package/dist/src/redteam/plugins/policy/utils.test.js +33 -2
- package/dist/src/redteam/plugins/policy/utils.test.js.map +1 -1
- package/dist/src/redteam/providers/crescendo/index.d.ts +6 -1
- package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
- package/dist/src/redteam/providers/crescendo/index.js +63 -21
- package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
- package/dist/src/redteam/providers/custom/index.d.ts +2 -1
- package/dist/src/redteam/providers/custom/index.d.ts.map +1 -1
- package/dist/src/redteam/providers/custom/index.js +1 -14
- package/dist/src/redteam/providers/custom/index.js.map +1 -1
- package/dist/src/redteam/providers/goat.d.ts +6 -0
- package/dist/src/redteam/providers/goat.d.ts.map +1 -1
- package/dist/src/redteam/providers/goat.js +59 -2
- package/dist/src/redteam/providers/goat.js.map +1 -1
- package/dist/src/redteam/providers/iterative.d.ts +5 -1
- package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
- package/dist/src/redteam/providers/iterative.js +79 -34
- package/dist/src/redteam/providers/iterative.js.map +1 -1
- package/dist/src/redteam/providers/iterativeImage.d.ts.map +1 -1
- package/dist/src/redteam/providers/iterativeImage.js +0 -3
- package/dist/src/redteam/providers/iterativeImage.js.map +1 -1
- package/dist/src/redteam/providers/iterativeTree.d.ts +2 -1
- package/dist/src/redteam/providers/iterativeTree.d.ts.map +1 -1
- package/dist/src/redteam/providers/iterativeTree.js +3 -20
- package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
- package/dist/src/redteam/providers/shared.d.ts +3 -0
- package/dist/src/redteam/providers/shared.d.ts.map +1 -1
- package/dist/src/redteam/providers/shared.js.map +1 -1
- package/dist/src/redteam/providers/traceFormatting.d.ts +6 -0
- package/dist/src/redteam/providers/traceFormatting.d.ts.map +1 -0
- package/dist/src/redteam/providers/traceFormatting.js +55 -0
- package/dist/src/redteam/providers/traceFormatting.js.map +1 -0
- package/dist/src/redteam/providers/tracingOptions.d.ts +22 -0
- package/dist/src/redteam/providers/tracingOptions.d.ts.map +1 -0
- package/dist/src/redteam/providers/tracingOptions.js +54 -0
- package/dist/src/redteam/providers/tracingOptions.js.map +1 -0
- package/dist/src/redteam/shared.d.ts.map +1 -1
- package/dist/src/redteam/shared.js +3 -1
- package/dist/src/redteam/shared.js.map +1 -1
- package/dist/src/redteam/sharedFrontend.d.ts.map +1 -1
- package/dist/src/redteam/sharedFrontend.js +1 -0
- package/dist/src/redteam/sharedFrontend.js.map +1 -1
- package/dist/src/redteam/strategies/index.d.ts.map +1 -1
- package/dist/src/redteam/strategies/index.js +4 -11
- package/dist/src/redteam/strategies/index.js.map +1 -1
- package/dist/src/redteam/strategies/layer.js +1 -1
- package/dist/src/redteam/strategies/layer.js.map +1 -1
- package/dist/src/redteam/strategies/multilingual.d.ts +19 -0
- package/dist/src/redteam/strategies/multilingual.d.ts.map +1 -1
- package/dist/src/redteam/strategies/multilingual.js +32 -0
- package/dist/src/redteam/strategies/multilingual.js.map +1 -1
- package/dist/src/redteam/strategies/simpleImage.d.ts.map +1 -1
- package/dist/src/redteam/strategies/simpleImage.js +75 -2
- package/dist/src/redteam/strategies/simpleImage.js.map +1 -1
- package/dist/src/redteam/types.d.ts +19 -3
- package/dist/src/redteam/types.d.ts.map +1 -1
- package/dist/src/server/utils/evalTableUtils.d.ts.map +1 -1
- package/dist/src/server/utils/evalTableUtils.js +24 -16
- package/dist/src/server/utils/evalTableUtils.js.map +1 -1
- package/dist/src/share.d.ts.map +1 -1
- package/dist/src/share.js +6 -0
- package/dist/src/share.js.map +1 -1
- package/dist/src/tracing/evaluatorTracing.d.ts.map +1 -1
- package/dist/src/tracing/evaluatorTracing.js +3 -1
- package/dist/src/tracing/evaluatorTracing.js.map +1 -1
- package/dist/src/tracing/otlpReceiver.d.ts.map +1 -1
- package/dist/src/tracing/otlpReceiver.js +11 -0
- package/dist/src/tracing/otlpReceiver.js.map +1 -1
- package/dist/src/tracing/store.d.ts +10 -0
- package/dist/src/tracing/store.d.ts.map +1 -1
- package/dist/src/tracing/store.js +174 -3
- package/dist/src/tracing/store.js.map +1 -1
- package/dist/src/tracing/traceContext.d.ts +37 -0
- package/dist/src/tracing/traceContext.d.ts.map +1 -0
- package/dist/src/tracing/traceContext.js +159 -0
- package/dist/src/tracing/traceContext.js.map +1 -0
- package/dist/src/types/env.d.ts +12 -0
- package/dist/src/types/env.d.ts.map +1 -1
- package/dist/src/types/env.js +4 -0
- package/dist/src/types/env.js.map +1 -1
- package/dist/src/types/index.d.ts +1425 -286
- package/dist/src/types/index.d.ts.map +1 -1
- package/dist/src/types/index.js.map +1 -1
- package/dist/src/types/providers.d.ts +1 -0
- package/dist/src/types/providers.d.ts.map +1 -1
- package/dist/src/types/providers.js.map +1 -1
- package/dist/src/util/cloud.d.ts +21 -0
- package/dist/src/util/cloud.d.ts.map +1 -1
- package/dist/src/util/cloud.js +125 -0
- package/dist/src/util/cloud.js.map +1 -1
- package/dist/src/util/database.d.ts +8 -0
- package/dist/src/util/database.d.ts.map +1 -1
- package/dist/src/util/index.d.ts +22 -0
- package/dist/src/util/index.d.ts.map +1 -1
- package/dist/src/util/index.js +78 -0
- package/dist/src/util/index.js.map +1 -1
- package/dist/src/util/sanitizer.d.ts.map +1 -1
- package/dist/src/util/sanitizer.js +16 -0
- package/dist/src/util/sanitizer.js.map +1 -1
- package/dist/src/util/testCaseReader.d.ts.map +1 -1
- package/dist/src/util/testCaseReader.js +7 -0
- package/dist/src/util/testCaseReader.js.map +1 -1
- package/dist/src/validators/providers.d.ts +80 -0
- package/dist/src/validators/providers.d.ts.map +1 -1
- package/dist/src/validators/redteam.d.ts +59 -23
- package/dist/src/validators/redteam.d.ts.map +1 -1
- package/dist/src/validators/redteam.js +67 -2
- package/dist/src/validators/redteam.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +10 -10
- package/dist/src/app/assets/index-BmEhbU1h.js +0 -72
|
@@ -16,28 +16,64 @@ const cliState_1 = __importDefault(require("../cliState"));
|
|
|
16
16
|
const logger_1 = __importDefault(require("../logger"));
|
|
17
17
|
const cloud_1 = require("../util/cloud");
|
|
18
18
|
const file_1 = require("../util/file");
|
|
19
|
+
const index_1 = require("../util/index");
|
|
19
20
|
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
20
21
|
const templates_1 = require("../util/templates");
|
|
21
22
|
const registry_1 = require("./registry");
|
|
22
23
|
// FIXME(ian): Make loadApiProvider handle all the different provider types (string, ProviderOptions, ApiProvider, etc), rather than the callers.
|
|
23
24
|
async function loadApiProvider(providerPath, context = {}) {
|
|
24
25
|
const { options = {}, basePath, env } = context;
|
|
26
|
+
// Render ONLY environment variable templates at load time (e.g., {{ env.AZURE_ENDPOINT }})
|
|
27
|
+
// This allows constructors to access real env values while preserving runtime templates
|
|
28
|
+
// like {{ vars.* }} for per-test customization at callApi() time
|
|
29
|
+
const renderedConfig = options.config ? (0, index_1.renderEnvOnlyInObject)(options.config) : undefined;
|
|
30
|
+
const renderedId = options.id ? (0, index_1.renderEnvOnlyInObject)(options.id) : undefined;
|
|
25
31
|
const providerOptions = {
|
|
26
|
-
id:
|
|
32
|
+
id: renderedId,
|
|
27
33
|
config: {
|
|
28
|
-
...
|
|
34
|
+
...renderedConfig,
|
|
29
35
|
basePath,
|
|
30
36
|
},
|
|
31
37
|
env,
|
|
32
38
|
};
|
|
39
|
+
// Validate linkedTargetId if present (Promptfoo Cloud feature)
|
|
40
|
+
if (providerOptions.config?.linkedTargetId) {
|
|
41
|
+
await (0, cloud_1.validateLinkedTargetId)(providerOptions.config.linkedTargetId);
|
|
42
|
+
}
|
|
33
43
|
const renderedProviderPath = (0, templates_1.getNunjucksEngine)().renderString(providerPath, {});
|
|
34
44
|
if ((0, cloud_1.isCloudProvider)(renderedProviderPath)) {
|
|
35
45
|
const cloudDatabaseId = (0, cloud_1.getCloudDatabaseId)(renderedProviderPath);
|
|
36
|
-
const
|
|
37
|
-
if ((0, cloud_1.isCloudProvider)(
|
|
38
|
-
throw new Error(`This cloud provider ${cloudDatabaseId} points to another cloud provider: ${
|
|
46
|
+
const cloudProvider = await (0, cloud_1.getProviderFromCloud)(cloudDatabaseId);
|
|
47
|
+
if ((0, cloud_1.isCloudProvider)(cloudProvider.id)) {
|
|
48
|
+
throw new Error(`This cloud provider ${cloudDatabaseId} points to another cloud provider: ${cloudProvider.id}. This is not allowed. A cloud provider should point to a specific provider, not another cloud provider.`);
|
|
39
49
|
}
|
|
40
|
-
|
|
50
|
+
// Merge local config overrides with cloud provider config
|
|
51
|
+
// Local config takes precedence to allow per-eval customization
|
|
52
|
+
const mergedOptions = {
|
|
53
|
+
...cloudProvider,
|
|
54
|
+
config: {
|
|
55
|
+
...cloudProvider.config,
|
|
56
|
+
...options.config,
|
|
57
|
+
},
|
|
58
|
+
// Allow local overrides for these fields
|
|
59
|
+
label: options.label ?? cloudProvider.label,
|
|
60
|
+
transform: options.transform ?? cloudProvider.transform,
|
|
61
|
+
delay: options.delay ?? cloudProvider.delay,
|
|
62
|
+
prompts: options.prompts ?? cloudProvider.prompts,
|
|
63
|
+
// Merge all three env sources: context (base) -> cloud -> local (highest priority)
|
|
64
|
+
env: {
|
|
65
|
+
...env, // Context env (from testSuite.env - proxies, tracing IDs, etc.)
|
|
66
|
+
...cloudProvider.env, // Cloud provider env overrides context
|
|
67
|
+
...options.env, // Local env overrides everything
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
logger_1.default.debug(`[Cloud Provider] Loaded ${cloudDatabaseId}, resolved to ${cloudProvider.id}${options.config ? ' with local config overrides' : ''}`);
|
|
71
|
+
const mergedContext = {
|
|
72
|
+
...context,
|
|
73
|
+
options: mergedOptions,
|
|
74
|
+
env: mergedOptions.env,
|
|
75
|
+
};
|
|
76
|
+
return loadApiProvider(cloudProvider.id, mergedContext);
|
|
41
77
|
}
|
|
42
78
|
if (renderedProviderPath.startsWith('file://') &&
|
|
43
79
|
(renderedProviderPath.endsWith('.yaml') ||
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/providers/index.ts"],"names":[],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/providers/index.ts"],"names":[],"mappings":";;;;;AAyBA,0CAuHC;AAcD,0CAiCC;AAkCD,4CAwEC;AAUD,wCAsBC;AAzUD,4CAAoB;AACpB,gDAAwB;AAExB,kDAA0B;AAC1B,oDAA4B;AAC5B,sDAA2B;AAC3B,2DAAmC;AACnC,uDAA+B;AAC/B,yCAKuB;AACvB,uCAA+D;AAC/D,yCAAsD;AACtD,kEAA0C;AAC1C,iDAAsD;AACtD,yCAAyC;AAMzC,iJAAiJ;AAC1I,KAAK,UAAU,eAAe,CACnC,YAAoB,EACpB,UAAkC,EAAE;IAEpC,MAAM,EAAE,OAAO,GAAG,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAEhD,2FAA2F;IAC3F,wFAAwF;IACxF,iEAAiE;IACjE,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAA,6BAAqB,EAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1F,MAAM,UAAU,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAA,6BAAqB,EAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE9E,MAAM,eAAe,GAAoB;QACvC,EAAE,EAAE,UAAU;QACd,MAAM,EAAE;YACN,GAAG,cAAc;YACjB,QAAQ;SACT;QACD,GAAG;KACJ,CAAC;IAEF,+DAA+D;IAC/D,IAAI,eAAe,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QAC3C,MAAM,IAAA,8BAAsB,EAAC,eAAe,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,oBAAoB,GAAG,IAAA,6BAAiB,GAAE,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAEhF,IAAI,IAAA,uBAAe,EAAC,oBAAoB,CAAC,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,IAAA,0BAAkB,EAAC,oBAAoB,CAAC,CAAC;QAEjE,MAAM,aAAa,GAAG,MAAM,IAAA,4BAAoB,EAAC,eAAe,CAAC,CAAC;QAClE,IAAI,IAAA,uBAAe,EAAC,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,uBAAuB,eAAe,sCAAsC,aAAa,CAAC,EAAE,0GAA0G,CACvM,CAAC;QACJ,CAAC;QAED,0DAA0D;QAC1D,gEAAgE;QAChE,MAAM,aAAa,GAAoB;YACrC,GAAG,aAAa;YAChB,MAAM,EAAE;gBACN,GAAG,aAAa,CAAC,MAAM;gBACvB,GAAG,OAAO,CAAC,MAAM;aAClB;YACD,yCAAyC;YACzC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,aAAa,CAAC,KAAK;YAC3C,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,aAAa,CAAC,SAAS;YACvD,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,aAAa,CAAC,KAAK;YAC3C,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,aAAa,CAAC,OAAO;YACjD,mFAAmF;YACnF,GAAG,EAAE;gBACH,GAAG,GAAG,EAAE,gEAAgE;gBACxE,GAAG,aAAa,CAAC,GAAG,EAAE,uCAAuC;gBAC7D,GAAG,OAAO,CAAC,GAAG,EAAE,iCAAiC;aAClD;SACF,CAAC;QAEF,gBAAM,CAAC,KAAK,CACV,2BAA2B,eAAe,iBAAiB,aAAa,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,EAAE,EAAE,CACrI,CAAC;QAEF,MAAM,aAAa,GAAG;YACpB,GAAG,OAAO;YACV,OAAO,EAAE,aAAa;YACtB,GAAG,EAAE,aAAa,CAAC,GAAG;SACvB,CAAC;QAEF,OAAO,eAAe,CAAC,aAAa,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC;IAC1D,CAAC;IAED,IACE,oBAAoB,CAAC,UAAU,CAAC,SAAS,CAAC;QAC1C,CAAC,oBAAoB,CAAC,QAAQ,CAAC,OAAO,CAAC;YACrC,oBAAoB,CAAC,QAAQ,CAAC,MAAM,CAAC;YACrC,oBAAoB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EACzC,CAAC;QACD,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC9D,MAAM,UAAU,GAAG,cAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAC1C,CAAC,CAAC,QAAQ;YACV,CAAC,CAAC,cAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAC;QACnD,MAAM,UAAU,GAAG,iBAAI,CAAC,IAAI,CAAC,YAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;QAClE,MAAM,WAAW,GAAG,IAAA,sCAA+B,EAAC,UAAU,CAAoB,CAAC;QACnF,IAAA,mBAAS,EAAC,WAAW,EAAE,mBAAmB,QAAQ,eAAe,CAAC,CAAC;QAEnE,6DAA6D;QAC7D,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/B,oEAAoE;YACpE,MAAM,IAAI,KAAK,CACb,+BAA+B,QAAQ,oDAAoD,CAC5F,CAAC;QACJ,CAAC;QAED,IAAA,mBAAS,EAAC,WAAW,CAAC,EAAE,EAAE,mBAAmB,QAAQ,kBAAkB,CAAC,CAAC;QACzE,gBAAM,CAAC,IAAI,CAAC,mBAAmB,WAAW,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC,CAAC;QAClE,OAAO,eAAe,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,sBAAW,EAAE,CAAC;QAClC,IAAI,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,oBAAoB,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YACjF,GAAG,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;YAClC,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,GAAG,CAAC,KAAK,KAAT,GAAG,CAAC,KAAK,GAAK,IAAA,6BAAiB,GAAE,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,EAAC;YAChF,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,IAAA,gBAAM,EAAA;mCACM,eAAK,CAAC,IAAI,CAAC,YAAY,CAAC;;MAErD,eAAK,CAAC,KAAK,CAAC,IAAA,gBAAM,EAAA;;;2DAGmC,CAAC,IAAI,eAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC;GAChH,CAAC;IACF,gBAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3B,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;AAChC,CAAC;AAUD;;;GAGG;AACI,KAAK,UAAU,eAAe,CACnC,QAAa,EACb,WAAwC,EACxC,UAAyB,EAAE;IAE3B,qDAAqD;IACrD,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,2FAA2F;QAC3F,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,OAAO,CAAC,GAAG;YAChB,CAAC,CAAC,MAAM,eAAe,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;YACvD,CAAC,CAAC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;SAAM,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,QAA2B,CAAC;QAC3C,IAAA,mBAAS,EAAC,MAAM,CAAC,EAAE,EAAE,iCAAiC,CAAC,CAAC;QACxD,MAAM,WAAW,GAA2B,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAChE,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,WAAW,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;QAChC,CAAC;QACD,OAAO,MAAM,eAAe,CAAC,MAAM,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;IACvD,CAAC;SAAM,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC1C,OAAO,OAAO,CAAC,GAAG;YAChB,CAAC,CAAC,MAAM,eAAe,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;YACvD,CAAC,CAAC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,qBAAqB,CAClC,QAAgB,EAChB,UAGI,EAAE;IAEN,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,cAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAC9C,CAAC,CAAC,YAAY;QACd,CAAC,CAAC,cAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,CAAC;IAEvD,MAAM,UAAU,GAAG,iBAAI,CAAC,IAAI,CAAC,YAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;IAClE,MAAM,WAAW,GAAG,IAAA,sCAA+B,EAAC,UAAU,CAEzC,CAAC;IACtB,IAAA,mBAAS,EAAC,WAAW,EAAE,mBAAmB,YAAY,eAAe,CAAC,CAAC;IAEvE,MAAM,OAAO,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,EAAuB,CAAC;IAC1D,OAAO,OAAO,CAAC,GAAG,CAChB,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACrB,IAAA,mBAAS,EAAC,MAAM,CAAC,EAAE,EAAE,sBAAsB,YAAY,kBAAkB,CAAC,CAAC;QAC3E,OAAO,eAAe,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;IACxE,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,gBAAgB,CACpC,aAA2C,EAC3C,UAGI,EAAE;IAEN,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAE7B,MAAM,GAAG,GAAG;QACV,GAAG,kBAAQ,CAAC,MAAM,EAAE,GAAG;QACvB,GAAG,OAAO,CAAC,GAAG;KACf,CAAC;IAEF,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,4CAA4C;QAC5C,IACE,aAAa,CAAC,UAAU,CAAC,SAAS,CAAC;YACnC,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAC9B,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC9B,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAClC,CAAC;YACD,OAAO,qBAAqB,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,CAAC,MAAM,eAAe,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IACnE,CAAC;SAAM,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE,CAAC;QAC/C,OAAO;YACL;gBACE,EAAE,EAAE,GAAG,EAAE,CAAC,iBAAiB;gBAC3B,OAAO,EAAE,aAAa;aACvB;SACF,CAAC;IACJ,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QACxC,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CACvC,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE;YACxC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBACjC,IACE,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC9B,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EACvF,CAAC;oBACD,OAAO,qBAAqB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC5D,CAAC;gBACD,OAAO,CAAC,MAAM,eAAe,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;gBACnC,OAAO;oBACL;wBACE,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,mBAAmB,GAAG,EAAE;wBAC3E,OAAO,EAAE,QAAQ;qBAClB;iBACF,CAAC;YACJ,CAAC;YACD,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAChB,iCAAiC;gBACjC,OAAO;oBACL,MAAM,eAAe,CAAE,QAA4B,CAAC,EAAG,EAAE;wBACvD,OAAO,EAAE,QAAQ;wBACjB,QAAQ;wBACR,GAAG;qBACJ,CAAC;iBACH,CAAC;YACJ,CAAC;YACD,yDAAyD;YACzD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,cAAc,GAAI,QAA+B,CAAC,EAAE,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,EAAE,GAAG,cAAc,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;YACnE,OAAO,CAAC,MAAM,eAAe,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,CACH,CAAC;QACF,OAAO,eAAe,CAAC,IAAI,EAAE,CAAC;IAChC,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,cAAc,CAAC,aAA2C;IACxE,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,CAAC,aAAa,CAAC,CAAC;IACzB,CAAC;SAAM,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE,CAAC;QAC/C,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC7B,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QACxC,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;YACzC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBACjC,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;gBACnC,OAAO,QAAQ,CAAC,KAAK,IAAI,mBAAmB,GAAG,EAAE,CAAC;YACpD,CAAC;YACD,IAAK,QAA4B,CAAC,EAAE,EAAE,CAAC;gBACrC,OAAQ,QAA4B,CAAC,EAAG,CAAC;YAC3C,CAAC;YACD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,cAAc,GAAI,QAA+B,CAAC,EAAE,CAAC,CAAC;YAC5D,OAAO,cAAc,CAAC,EAAE,IAAI,EAAE,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC"}
|
|
@@ -9,6 +9,12 @@ type AgentProviderOptions = ProviderOptions & {
|
|
|
9
9
|
instructions?: string;
|
|
10
10
|
maxTurns?: number;
|
|
11
11
|
stateful?: boolean;
|
|
12
|
+
/**
|
|
13
|
+
* Pre-defined conversation history to start from.
|
|
14
|
+
* Can be an array of Message objects or a file:// path to JSON/YAML.
|
|
15
|
+
* Useful for testing specific conversation states or reproducing bugs.
|
|
16
|
+
*/
|
|
17
|
+
initialMessages?: Message[] | string;
|
|
12
18
|
};
|
|
13
19
|
};
|
|
14
20
|
/**
|
|
@@ -20,6 +26,7 @@ export declare class SimulatedUser implements ApiProvider {
|
|
|
20
26
|
private readonly maxTurns;
|
|
21
27
|
private readonly rawInstructions;
|
|
22
28
|
private readonly stateful;
|
|
29
|
+
private readonly configInitialMessages?;
|
|
23
30
|
/**
|
|
24
31
|
* Because the SimulatedUser is inherited by the RedteamMischievousUserProvider, and different
|
|
25
32
|
* Cloud tasks are used for each, the taskId needs to be explicitly defined/scoped.
|
|
@@ -27,6 +34,19 @@ export declare class SimulatedUser implements ApiProvider {
|
|
|
27
34
|
readonly taskId: string;
|
|
28
35
|
constructor({ id, label, config }: AgentProviderOptions);
|
|
29
36
|
id(): string;
|
|
37
|
+
/**
|
|
38
|
+
* Validates that a message has the required structure.
|
|
39
|
+
*/
|
|
40
|
+
private isValidMessage;
|
|
41
|
+
/**
|
|
42
|
+
* Validates and filters an array of messages, logging warnings for invalid entries.
|
|
43
|
+
*/
|
|
44
|
+
private validateMessages;
|
|
45
|
+
/**
|
|
46
|
+
* Resolves initial messages from either an array or a file:// path.
|
|
47
|
+
* Supports loading messages from JSON and YAML files.
|
|
48
|
+
*/
|
|
49
|
+
private resolveInitialMessages;
|
|
30
50
|
private sendMessageToUser;
|
|
31
51
|
private sendMessageToAgent;
|
|
32
52
|
callApi(_prompt: string, context?: CallApiContextParams, _callApiOptions?: CallApiOptionsParams): Promise<ProviderResponse>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"simulatedUser.d.ts","sourceRoot":"","sources":["../../../src/providers/simulatedUser.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"simulatedUser.d.ts","sourceRoot":"","sources":["../../../src/providers/simulatedUser.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EACV,WAAW,EACX,oBAAoB,EACpB,oBAAoB,EACpB,eAAe,EACf,gBAAgB,EAChB,UAAU,EACX,MAAM,gBAAgB,CAAC;AAExB,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,KAAK,oBAAoB,GAAG,eAAe,GAAG;IAC5C,MAAM,CAAC,EAAE;QACP,YAAY,CAAC,EAAE,eAAe,CAAC;QAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB;;;;WAIG;QACH,eAAe,CAAC,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;KACtC,CAAC;CACH,CAAC;AAEF;;;GAGG;AACH,qBAAa,aAAc,YAAW,WAAW;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAqB;IAE5D;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAS;gBAEpB,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,oBAAoB;IAQvD,EAAE;IAIF;;OAEG;IACH,OAAO,CAAC,cAAc;IAStB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAcxB;;;OAGG;IACH,OAAO,CAAC,sBAAsB;YA4DhB,iBAAiB;YA8BjB,kBAAkB;IA2C1B,OAAO,CAGX,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,oBAAoB,EAC9B,eAAe,CAAC,EAAE,oBAAoB,GACrC,OAAO,CAAC,gBAAgB,CAAC;IAsF5B,QAAQ;IAIR,eAAe,CACb,QAAQ,EAAE,OAAO,EAAE,EACnB,UAAU,EAAE,UAAU,EACtB,mBAAmB,EAAE,gBAAgB,EACrC,SAAS,CAAC,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgBrB"}
|
|
@@ -7,6 +7,7 @@ exports.SimulatedUser = void 0;
|
|
|
7
7
|
const logger_1 = __importDefault(require("../logger"));
|
|
8
8
|
const util_1 = require("../redteam/util");
|
|
9
9
|
const invariant_1 = __importDefault(require("../util/invariant"));
|
|
10
|
+
const file_1 = require("../util/file");
|
|
10
11
|
const templates_1 = require("../util/templates");
|
|
11
12
|
const time_1 = require("../util/time");
|
|
12
13
|
const tokenUsageUtils_1 = require("../util/tokenUsageUtils");
|
|
@@ -26,10 +27,84 @@ class SimulatedUser {
|
|
|
26
27
|
this.maxTurns = config.maxTurns ?? 10;
|
|
27
28
|
this.rawInstructions = config.instructions || '{{instructions}}';
|
|
28
29
|
this.stateful = config.stateful ?? false;
|
|
30
|
+
this.configInitialMessages = config.initialMessages;
|
|
29
31
|
}
|
|
30
32
|
id() {
|
|
31
33
|
return this.identifier;
|
|
32
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* Validates that a message has the required structure.
|
|
37
|
+
*/
|
|
38
|
+
isValidMessage(msg) {
|
|
39
|
+
return (msg &&
|
|
40
|
+
typeof msg === 'object' &&
|
|
41
|
+
typeof msg.content === 'string' &&
|
|
42
|
+
(msg.role === 'user' || msg.role === 'assistant' || msg.role === 'system'));
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Validates and filters an array of messages, logging warnings for invalid entries.
|
|
46
|
+
*/
|
|
47
|
+
validateMessages(messages) {
|
|
48
|
+
const validMessages = [];
|
|
49
|
+
for (let i = 0; i < messages.length; i++) {
|
|
50
|
+
if (this.isValidMessage(messages[i])) {
|
|
51
|
+
validMessages.push(messages[i]);
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
logger_1.default.warn(`[SimulatedUser] Invalid message at index ${i}, skipping. Expected {role: 'user'|'assistant'|'system', content: string}, got: ${JSON.stringify(messages[i]).substring(0, 100)}`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return validMessages;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Resolves initial messages from either an array or a file:// path.
|
|
61
|
+
* Supports loading messages from JSON and YAML files.
|
|
62
|
+
*/
|
|
63
|
+
resolveInitialMessages(initialMessages) {
|
|
64
|
+
if (!initialMessages) {
|
|
65
|
+
return [];
|
|
66
|
+
}
|
|
67
|
+
// If it's already an array, validate and return it
|
|
68
|
+
if (Array.isArray(initialMessages)) {
|
|
69
|
+
return this.validateMessages(initialMessages);
|
|
70
|
+
}
|
|
71
|
+
// If it's a string, handle different cases
|
|
72
|
+
if (typeof initialMessages === 'string') {
|
|
73
|
+
if (initialMessages.trim() === '') {
|
|
74
|
+
return [];
|
|
75
|
+
}
|
|
76
|
+
// Case 1: file:// reference (JSON/YAML)
|
|
77
|
+
if (initialMessages.startsWith('file://')) {
|
|
78
|
+
try {
|
|
79
|
+
const resolved = (0, file_1.maybeLoadConfigFromExternalFile)(initialMessages);
|
|
80
|
+
if (Array.isArray(resolved)) {
|
|
81
|
+
return this.validateMessages(resolved);
|
|
82
|
+
}
|
|
83
|
+
logger_1.default.warn(`[SimulatedUser] Expected array of messages from file, got: ${typeof resolved}. Value: ${JSON.stringify(resolved).substring(0, 200)}`);
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
logger_1.default.warn(`[SimulatedUser] Failed to load initialMessages from file: ${error instanceof Error ? error.message : error}`);
|
|
87
|
+
}
|
|
88
|
+
return [];
|
|
89
|
+
}
|
|
90
|
+
// Case 2: Stringified JSON array (happens when file was already loaded but then stringified for storage/transport)
|
|
91
|
+
if (initialMessages.trim().startsWith('[')) {
|
|
92
|
+
try {
|
|
93
|
+
const parsed = JSON.parse(initialMessages);
|
|
94
|
+
if (Array.isArray(parsed)) {
|
|
95
|
+
return this.validateMessages(parsed);
|
|
96
|
+
}
|
|
97
|
+
logger_1.default.warn(`[SimulatedUser] Parsed JSON but got ${typeof parsed} instead of array. Value: ${initialMessages.substring(0, 200)}`);
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
logger_1.default.warn(`[SimulatedUser] Failed to parse initialMessages as JSON: ${error}. Value: ${initialMessages.substring(0, 200)}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
logger_1.default.warn(`[SimulatedUser] initialMessages is a string but could not be resolved: ${initialMessages.substring(0, 200)}`);
|
|
104
|
+
return [];
|
|
105
|
+
}
|
|
106
|
+
return [];
|
|
107
|
+
}
|
|
33
108
|
async sendMessageToUser(messages, userProvider) {
|
|
34
109
|
logger_1.default.debug('[SimulatedUser] Sending message to simulated user provider');
|
|
35
110
|
const flippedMessages = messages.map((message) => {
|
|
@@ -91,10 +166,26 @@ class SimulatedUser {
|
|
|
91
166
|
const instructions = (0, templates_1.getNunjucksEngine)().renderString(this.rawInstructions, context?.vars);
|
|
92
167
|
const userProvider = new promptfoo_1.PromptfooSimulatedUserProvider({ instructions }, this.taskId);
|
|
93
168
|
logger_1.default.debug(`[SimulatedUser] Formatted user instructions: ${instructions}`);
|
|
94
|
-
|
|
169
|
+
// Support initial messages from either vars.initialMessages (per-test) or config.initialMessages (provider-level)
|
|
170
|
+
// vars.initialMessages takes precedence over config.initialMessages
|
|
171
|
+
// Both can be arrays or file:// paths
|
|
172
|
+
const varsInitialMessages = context?.vars?.initialMessages;
|
|
173
|
+
const initialMessages = this.resolveInitialMessages(varsInitialMessages || this.configInitialMessages);
|
|
174
|
+
const messages = [...initialMessages];
|
|
175
|
+
if (initialMessages.length > 0) {
|
|
176
|
+
logger_1.default.debug(`[SimulatedUser] Starting with ${initialMessages.length} initial messages`);
|
|
177
|
+
}
|
|
95
178
|
const maxTurns = this.maxTurns;
|
|
96
179
|
const tokenUsage = (0, tokenUsageUtils_1.createEmptyTokenUsage)();
|
|
97
180
|
let agentResponse;
|
|
181
|
+
// If initial messages end with user message, agent needs to respond first to avoid consecutive user messages
|
|
182
|
+
const lastInitialRole = messages.length > 0 ? messages[messages.length - 1].role : null;
|
|
183
|
+
if (lastInitialRole === 'user') {
|
|
184
|
+
logger_1.default.debug('[SimulatedUser] Initial messages end with user message, getting agent response first');
|
|
185
|
+
agentResponse = await this.sendMessageToAgent(messages, context.originalProvider, context);
|
|
186
|
+
messages.push({ role: 'assistant', content: String(agentResponse.output ?? '') });
|
|
187
|
+
(0, tokenUsageUtils_1.accumulateResponseTokenUsage)(tokenUsage, agentResponse);
|
|
188
|
+
}
|
|
98
189
|
for (let i = 0; i < maxTurns; i++) {
|
|
99
190
|
logger_1.default.debug(`[SimulatedUser] Turn ${i + 1} of ${maxTurns}`);
|
|
100
191
|
// NOTE: Simulated-user provider acts as a judge to determine whether the instruction goal is satisfied.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"simulatedUser.js","sourceRoot":"","sources":["../../../src/providers/simulatedUser.ts"],"names":[],"mappings":";;;;;;AAAA,uDAA+B;AAC/B,0CAA+C;AAC/C,kEAA0C;AAC1C,iDAAsD;AACtD,uCAAqC;AACrC,6DAA8F;AAC9F,2CAA6D;
|
|
1
|
+
{"version":3,"file":"simulatedUser.js","sourceRoot":"","sources":["../../../src/providers/simulatedUser.ts"],"names":[],"mappings":";;;;;;AAAA,uDAA+B;AAC/B,0CAA+C;AAC/C,kEAA0C;AAC1C,uCAA+D;AAC/D,iDAAsD;AACtD,uCAAqC;AACrC,6DAA8F;AAC9F,2CAA6D;AA+B7D;;;GAGG;AACH,MAAa,aAAa;IAaxB,YAAY,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAwB;QANvD;;;WAGG;QACM,WAAM,GAAW,KAAK,CAAC;QAG9B,IAAI,CAAC,UAAU,GAAG,EAAE,IAAI,KAAK,IAAI,gBAAgB,CAAC;QAClD,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,YAAY,IAAI,kBAAkB,CAAC;QACjE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,KAAK,CAAC;QACzC,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,eAAe,CAAC;IACtD,CAAC;IAED,EAAE;QACA,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,GAAQ;QAC7B,OAAO,CACL,GAAG;YACH,OAAO,GAAG,KAAK,QAAQ;YACvB,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;YAC/B,CAAC,GAAG,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAC3E,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,QAAe;QACtC,MAAM,aAAa,GAAc,EAAE,CAAC;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAM,CAAC,IAAI,CACT,4CAA4C,CAAC,mFAAmF,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAChL,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAED;;;OAGG;IACK,sBAAsB,CAAC,eAA+C;QAC5E,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,mDAAmD;QACnD,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;QAChD,CAAC;QAED,2CAA2C;QAC3C,IAAI,OAAO,eAAe,KAAK,QAAQ,EAAE,CAAC;YACxC,IAAI,eAAe,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBAClC,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,wCAAwC;YACxC,IAAI,eAAe,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC1C,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,IAAA,sCAA+B,EAAC,eAAe,CAAC,CAAC;oBAClE,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5B,OAAO,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;oBACzC,CAAC;oBACD,gBAAM,CAAC,IAAI,CACT,8DAA8D,OAAO,QAAQ,YAAY,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACtI,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,gBAAM,CAAC,IAAI,CACT,6DAA6D,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,CAC9G,CAAC;gBACJ,CAAC;gBACD,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,mHAAmH;YACnH,IAAI,eAAe,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3C,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;oBAC3C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC1B,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;oBACvC,CAAC;oBACD,gBAAM,CAAC,IAAI,CACT,uCAAuC,OAAO,MAAM,6BAA6B,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACrH,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,gBAAM,CAAC,IAAI,CACT,4DAA4D,KAAK,YAAY,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACjH,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,gBAAM,CAAC,IAAI,CACT,0EAA0E,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC9G,CAAC;YACF,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,QAAmB,EACnB,YAA4C;QAE5C,gBAAM,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAE3E,MAAM,eAAe,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC/C,OAAO;gBACL,IAAI,EAAE,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM;gBACpD,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC;QAE7E,uDAAuD;QACvD,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO;gBACL,QAAQ;gBACR,KAAK,EAAE,QAAQ,CAAC,KAAK;aACtB,CAAC;QACJ,CAAC;QAED,gBAAM,CAAC,KAAK,CAAC,SAAS,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACzC,OAAO;YACL,QAAQ,EAAE,CAAC,GAAG,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;YACjF,UAAU,EAAE,QAAQ,CAAC,UAAU;SAChC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAmB,EACnB,cAA2B,EAC3B,OAA6B;QAE7B,IAAA,mBAAS,EAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,uCAAuC,CAAC,CAAC;QAEzE,kEAAkE;QAClE,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;QACvC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;QAC/B,MAAM,cAAc,GAAG,IAAA,6BAAiB,GAAE,CAAC,YAAY,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAEhF,0BAA0B;QAC1B,wFAAwF;QACxF,6FAA6F;QAC7F,4EAA4E;QAC5E,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ;YAChC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS;gBACvB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBACpF,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;oBACb,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE;oBAC3C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE;iBACjE,CAAC;YACN,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC;QAE/E,gBAAM,CAAC,KAAK,CAAC,uDAAuD,YAAY,EAAE,CAAC,CAAC;QAEpF,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAErE,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YACvB,OAAO,GAAG,OAAO,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC;YACxE,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;QAC9C,CAAC;QAED,IAAI,cAAc,CAAC,KAAK,EAAE,CAAC;YACzB,gBAAM,CAAC,KAAK,CAAC,gCAAgC,cAAc,CAAC,KAAK,IAAI,CAAC,CAAC;YACvE,MAAM,IAAA,YAAK,EAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;QAED,gBAAM,CAAC,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1D,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,OAAO;IACX,0FAA0F;IAC1F,oGAAoG;IACpG,OAAe,EACf,OAA8B,EAC9B,eAAsC;QAEtC,IAAA,mBAAS,EAAC,OAAO,EAAE,gBAAgB,EAAE,qCAAqC,CAAC,CAAC;QAE5E,MAAM,YAAY,GAAG,IAAA,6BAAiB,GAAE,CAAC,YAAY,CAAC,IAAI,CAAC,eAAe,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QAE3F,MAAM,YAAY,GAAG,IAAI,0CAA8B,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAEvF,gBAAM,CAAC,KAAK,CAAC,gDAAgD,YAAY,EAAE,CAAC,CAAC;QAE7E,kHAAkH;QAClH,oEAAoE;QACpE,sCAAsC;QACtC,MAAM,mBAAmB,GAAG,OAAO,EAAE,IAAI,EAAE,eAAiD,CAAC;QAC7F,MAAM,eAAe,GAAG,IAAI,CAAC,sBAAsB,CACjD,mBAAmB,IAAI,IAAI,CAAC,qBAAqB,CAClD,CAAC;QACF,MAAM,QAAQ,GAAc,CAAC,GAAG,eAAe,CAAC,CAAC;QAEjD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,gBAAM,CAAC,KAAK,CAAC,iCAAiC,eAAe,CAAC,MAAM,mBAAmB,CAAC,CAAC;QAC3F,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE/B,MAAM,UAAU,GAAG,IAAA,uCAAqB,GAAE,CAAC;QAE3C,IAAI,aAA2C,CAAC;QAEhD,6GAA6G;QAC7G,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QACxF,IAAI,eAAe,KAAK,MAAM,EAAE,CAAC;YAC/B,gBAAM,CAAC,KAAK,CACV,sFAAsF,CACvF,CAAC;YACF,aAAa,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;YAC3F,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;YAClF,IAAA,8CAA4B,EAAC,UAAU,EAAE,aAAa,CAAC,CAAC;QAC1D,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,gBAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,GAAG,CAAC,OAAO,QAAQ,EAAE,CAAC,CAAC;YAE7D,wGAAwG;YACxG,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YAExE,kDAAkD;YAClD,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrB,OAAO;oBACL,KAAK,EAAE,UAAU,CAAC,KAAK;oBACvB,UAAU;iBACX,CAAC;YACJ,CAAC;YAED,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,UAAU,CAAC;YAChD,MAAM,WAAW,GAAG,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAE9D,iFAAiF;YACjF,IACE,WAAW,CAAC,OAAO;gBACnB,OAAO,WAAW,CAAC,OAAO,KAAK,QAAQ;gBACvC,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAC1C,CAAC;gBACD,MAAM;YACR,CAAC;YAED,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAE3B,aAAa,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAC3C,cAAc,EACd,OAAO,CAAC,gBAAgB,EACxB,OAAO,CACR,CAAC;YAEF,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;YAElF,IAAA,8CAA4B,EAAC,UAAU,EAAE,aAAa,CAAC,CAAC;QAC1D,CAAC;QAED,OAAO,IAAI,CAAC,eAAe,CACzB,QAAQ,EACR,UAAU,EACV,aAAiC,EACjC,IAAA,mBAAY,EAAC,aAAa,EAAE,OAAO,CAAC,CACrC,CAAC;IACJ,CAAC;IAED,QAAQ;QACN,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,eAAe,CACb,QAAmB,EACnB,UAAsB,EACtB,mBAAqC,EACrC,SAAkB;QAElB,OAAO;YACL,MAAM,EAAE,QAAQ;iBACb,GAAG,CACF,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,OAAO,EAAE,CAC1F;iBACA,IAAI,CAAC,SAAS,CAAC;YAClB,UAAU;YACV,QAAQ,EAAE;gBACR,QAAQ;gBACR,SAAS;aACV;YACD,UAAU,EAAE,mBAAmB,CAAC,UAAU;SAC3C,CAAC;IACJ,CAAC;CACF;AAnTD,sCAmTC"}
|
|
@@ -42,12 +42,64 @@ def load_user_module(script_path):
|
|
|
42
42
|
|
|
43
43
|
def get_callable(module, method_name):
|
|
44
44
|
"""Get the callable method from module, supporting 'Class.method' syntax."""
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
45
|
+
try:
|
|
46
|
+
if "." in method_name:
|
|
47
|
+
class_name, classmethod_name = method_name.split(".", 1)
|
|
48
|
+
cls = getattr(module, class_name)
|
|
49
|
+
return getattr(cls, classmethod_name)
|
|
50
|
+
else:
|
|
51
|
+
return getattr(module, method_name)
|
|
52
|
+
except AttributeError as e:
|
|
53
|
+
# Provide helpful error message when function not found
|
|
54
|
+
available_funcs = [
|
|
55
|
+
name
|
|
56
|
+
for name in dir(module)
|
|
57
|
+
if callable(getattr(module, name, None)) and not name.startswith("_")
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
error_lines = [
|
|
61
|
+
f"Function '{method_name}' not found in module '{module.__name__}'",
|
|
62
|
+
"",
|
|
63
|
+
f"Available functions in your module: {', '.join(available_funcs) if available_funcs else '(none)'}",
|
|
64
|
+
"",
|
|
65
|
+
"Expected function names for promptfoo:",
|
|
66
|
+
" • call_api(prompt, options, context) - for chat/completions",
|
|
67
|
+
" • call_embedding_api(prompt, options) - for embeddings",
|
|
68
|
+
" • call_classification_api(prompt, options) - for classification",
|
|
69
|
+
"",
|
|
70
|
+
]
|
|
71
|
+
|
|
72
|
+
# Fuzzy match suggestion
|
|
73
|
+
if available_funcs:
|
|
74
|
+
# Check for common mistakes
|
|
75
|
+
method_lower = method_name.lower()
|
|
76
|
+
for func in available_funcs:
|
|
77
|
+
func_lower = func.lower()
|
|
78
|
+
# Check if user used 'get_' instead of 'call_'
|
|
79
|
+
if method_lower.replace("call_", "") == func_lower.replace("get_", ""):
|
|
80
|
+
error_lines.append(
|
|
81
|
+
f"💡 Did you mean to rename '{func}' to '{method_name}'?"
|
|
82
|
+
)
|
|
83
|
+
break
|
|
84
|
+
# Check if function name is similar (missing 'call_' prefix)
|
|
85
|
+
elif method_lower.replace("call_", "") == func_lower:
|
|
86
|
+
error_lines.append(
|
|
87
|
+
f"💡 Did you mean to rename '{func}' to '{method_name}'?"
|
|
88
|
+
)
|
|
89
|
+
break
|
|
90
|
+
# Check if it's just a typo (Levenshtein-like)
|
|
91
|
+
elif (
|
|
92
|
+
len(set(method_lower) & set(func_lower)) > len(method_lower) * 0.6
|
|
93
|
+
and abs(len(method_lower) - len(func_lower)) <= 3
|
|
94
|
+
):
|
|
95
|
+
error_lines.append(f"💡 Did you mean '{func}'?")
|
|
96
|
+
break
|
|
97
|
+
|
|
98
|
+
error_lines.append(
|
|
99
|
+
"\nSee https://www.promptfoo.dev/docs/providers/python/ for details."
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
raise AttributeError("\n".join(error_lines)) from e
|
|
51
103
|
|
|
52
104
|
|
|
53
105
|
def call_method(method_callable, args):
|
|
@@ -72,8 +124,11 @@ def main():
|
|
|
72
124
|
# Load user module once
|
|
73
125
|
try:
|
|
74
126
|
user_module = load_user_module(script_path)
|
|
75
|
-
#
|
|
76
|
-
|
|
127
|
+
# Note: We don't validate the default function exists at initialization.
|
|
128
|
+
# With the persistent worker protocol supporting dynamic function calls per request,
|
|
129
|
+
# users may only define specific functions (e.g., call_embedding_api for embeddings-only).
|
|
130
|
+
# Functions are validated when actually called in handle_call(), providing clear
|
|
131
|
+
# error messages with available functions and suggestions.
|
|
77
132
|
except Exception as e:
|
|
78
133
|
print(f"ERROR: Failed to load module: {e}", file=sys.stderr, flush=True)
|
|
79
134
|
print(traceback.format_exc(), file=sys.stderr, flush=True)
|
|
@@ -108,17 +163,23 @@ def main():
|
|
|
108
163
|
|
|
109
164
|
def handle_call(command_line, user_module, default_function_name):
|
|
110
165
|
"""Handle a CALL command."""
|
|
166
|
+
response_file = None
|
|
111
167
|
try:
|
|
112
168
|
# Parse command: "CALL:<function_name>:<request_file>:<response_file>"
|
|
113
169
|
# or legacy: "CALL:<request_file>:<response_file>"
|
|
114
170
|
parts = command_line.split(":", 3)
|
|
115
171
|
|
|
172
|
+
# Extract response_file first (always the last part) so it's available even if validation fails
|
|
173
|
+
# This ensures we can write an error response file if command format is invalid
|
|
174
|
+
if len(parts) >= 3:
|
|
175
|
+
response_file = parts[-1]
|
|
176
|
+
|
|
116
177
|
if len(parts) == 4:
|
|
117
178
|
# New format: CALL:<function_name>:<request_file>:<response_file>
|
|
118
|
-
_, function_name, request_file,
|
|
179
|
+
_, function_name, request_file, _ = parts
|
|
119
180
|
elif len(parts) == 3:
|
|
120
181
|
# Legacy format: CALL:<request_file>:<response_file>
|
|
121
|
-
_, request_file,
|
|
182
|
+
_, request_file, _ = parts
|
|
122
183
|
function_name = default_function_name
|
|
123
184
|
else:
|
|
124
185
|
raise ValueError(f"Invalid CALL command format: {command_line}")
|
|
@@ -174,7 +235,30 @@ def handle_call(command_line, user_module, default_function_name):
|
|
|
174
235
|
except Exception as e:
|
|
175
236
|
print(f"ERROR handling call: {e}", file=sys.stderr, flush=True)
|
|
176
237
|
print(traceback.format_exc(), file=sys.stderr, flush=True)
|
|
177
|
-
|
|
238
|
+
|
|
239
|
+
# Write error response if we have response_file
|
|
240
|
+
# This ensures Node.js always has a file to read, preventing ENOENT errors
|
|
241
|
+
# when errors occur before the normal response file write (e.g., invalid command,
|
|
242
|
+
# non-existent function, file I/O errors)
|
|
243
|
+
if response_file:
|
|
244
|
+
try:
|
|
245
|
+
error_response = {
|
|
246
|
+
"type": "error",
|
|
247
|
+
"error": str(e),
|
|
248
|
+
"traceback": traceback.format_exc(),
|
|
249
|
+
}
|
|
250
|
+
with open(response_file, "w", encoding="utf-8") as f:
|
|
251
|
+
json.dump(error_response, f, ensure_ascii=False)
|
|
252
|
+
f.flush()
|
|
253
|
+
os.fsync(f.fileno())
|
|
254
|
+
except Exception as write_error:
|
|
255
|
+
print(
|
|
256
|
+
f"ERROR: Failed to write error response: {write_error}",
|
|
257
|
+
file=sys.stderr,
|
|
258
|
+
flush=True,
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
# Signal done after attempting to write error
|
|
178
262
|
print("DONE", flush=True)
|
|
179
263
|
|
|
180
264
|
|
|
@@ -12,8 +12,7 @@ export declare class PythonWorker {
|
|
|
12
12
|
private readonly maxCrashes;
|
|
13
13
|
private pendingRequest;
|
|
14
14
|
private requestTimeout;
|
|
15
|
-
constructor(scriptPath: string, functionName: string, pythonPath?: string | undefined, timeout?: number,
|
|
16
|
-
onReady?: (() => void) | undefined);
|
|
15
|
+
constructor(scriptPath: string, functionName: string, pythonPath?: string | undefined, timeout?: number, onReady?: (() => void) | undefined);
|
|
17
16
|
initialize(): Promise<void>;
|
|
18
17
|
private startWorker;
|
|
19
18
|
call(functionName: string, args: any[]): Promise<any>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../../src/python/worker.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../../src/python/worker.ts"],"names":[],"mappings":"AASA,qBAAa,YAAY;IAcrB,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,OAAO,CAAC;IAjBlB,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,IAAI,CAAkB;IAC9B,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,cAAc,CAGN;IAChB,OAAO,CAAC,cAAc,CAA+B;gBAG3C,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,EACpB,UAAU,CAAC,EAAE,MAAM,YAAA,EACnB,OAAO,GAAE,MAA2B,EACpC,OAAO,CAAC,GAAE,MAAM,IAAI,aAAA;IAGxB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAInB,WAAW;IAsDnB,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC;YAsB7C,WAAW;IAqFzB,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,UAAU;IAOlB,OAAO,CAAC,WAAW;IAmBnB,OAAO,IAAI,OAAO;IAIlB,MAAM,IAAI,OAAO;IAIX,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAoChC"}
|
|
@@ -9,11 +9,11 @@ const fs_1 = __importDefault(require("fs"));
|
|
|
9
9
|
const path_1 = __importDefault(require("path"));
|
|
10
10
|
const os_1 = __importDefault(require("os"));
|
|
11
11
|
const logger_1 = __importDefault(require("../logger"));
|
|
12
|
+
const shared_1 = require("../providers/shared");
|
|
12
13
|
const json_1 = require("../util/json");
|
|
13
14
|
const pythonUtils_1 = require("./pythonUtils");
|
|
14
15
|
class PythonWorker {
|
|
15
|
-
constructor(scriptPath, functionName, pythonPath, timeout =
|
|
16
|
-
onReady) {
|
|
16
|
+
constructor(scriptPath, functionName, pythonPath, timeout = shared_1.REQUEST_TIMEOUT_MS, onReady) {
|
|
17
17
|
this.scriptPath = scriptPath;
|
|
18
18
|
this.functionName = functionName;
|
|
19
19
|
this.pythonPath = pythonPath;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/python/worker.ts"],"names":[],"mappings":";;;;;;AAAA,+CAA2C;AAC3C,4CAAoB;AACpB,gDAAwB;AACxB,4CAAoB;AACpB,uDAA+B;AAC/B,uCAAiD;AACjD,+CAAmD;AAEnD,MAAa,YAAY;IAavB,YACU,UAAkB,EAClB,YAAoB,EACpB,UAAmB,EACnB,UAAkB,
|
|
1
|
+
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../src/python/worker.ts"],"names":[],"mappings":";;;;;;AAAA,+CAA2C;AAC3C,4CAAoB;AACpB,gDAAwB;AACxB,4CAAoB;AACpB,uDAA+B;AAC/B,gDAAyD;AACzD,uCAAiD;AACjD,+CAAmD;AAEnD,MAAa,YAAY;IAavB,YACU,UAAkB,EAClB,YAAoB,EACpB,UAAmB,EACnB,UAAkB,2BAAkB,EACpC,OAAoB;QAJpB,eAAU,GAAV,UAAU,CAAQ;QAClB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,eAAU,GAAV,UAAU,CAAS;QACnB,YAAO,GAAP,OAAO,CAA6B;QACpC,YAAO,GAAP,OAAO,CAAa;QAjBtB,YAAO,GAAuB,IAAI,CAAC;QACnC,UAAK,GAAY,KAAK,CAAC;QACvB,SAAI,GAAY,KAAK,CAAC;QACtB,iBAAY,GAAY,KAAK,CAAC;QAC9B,eAAU,GAAW,CAAC,CAAC;QACd,eAAU,GAAW,CAAC,CAAC;QAChC,mBAAc,GAGX,IAAI,CAAC;QACR,mBAAc,GAA0B,IAAI,CAAC;IAQlD,CAAC;IAEJ,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,MAAM,WAAW,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAElE,sFAAsF;QACtF,MAAM,kBAAkB,GAAG,MAAM,IAAA,gCAAkB,EACjD,IAAI,CAAC,UAAU,IAAI,QAAQ,EAC3B,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,CACpC,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,0BAAW,CAAC,WAAW,EAAE;YAC1C,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,kBAAkB;YAC9B,IAAI,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;YAC1C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;QAEH,0BAA0B;QAC1B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;gBACnC,MAAM,CAAC,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC,CAAC;YACpE,CAAC,EAAE,KAAK,CAAC,CAAC;YAEV,IAAI,CAAC,OAAQ,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAe,EAAE,EAAE;gBAC9C,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,OAAO,EAAE,CAAC;oBAC/B,YAAY,CAAC,YAAY,CAAC,CAAC;oBAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;oBAClB,gBAAM,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;oBAC3D,+DAA+D;oBAC/D,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;wBACjB,IAAI,CAAC,OAAO,EAAE,CAAC;oBACjB,CAAC;oBACD,OAAO,EAAE,CAAC;gBACZ,CAAC;qBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBACtC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAChC,YAAY,CAAC,YAAY,CAAC,CAAC;gBAC3B,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBAC7B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;oBACvB,IAAI,CAAC,WAAW,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxC,gBAAM,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC3D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,YAAoB,EAAE,IAAW;QAC1C,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACpC,CAAC;QAED,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;QAC1F,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;YAClB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACxB,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;YAC7B,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,YAAoB,EAAE,IAAW;QACzD,MAAM,WAAW,GAAG,cAAI,CAAC,IAAI,CAC3B,YAAE,CAAC,MAAM,EAAE,EACX,wBAAwB,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CACjF,CAAC;QACF,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAC5B,YAAE,CAAC,MAAM,EAAE,EACX,yBAAyB,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAClF,CAAC;QAEF,IAAI,CAAC;YACH,gBAAgB;YAChB,YAAE,CAAC,aAAa,CAAC,WAAW,EAAE,IAAA,wBAAiB,EAAC,IAAI,CAAW,EAAE,OAAO,CAAC,CAAC;YAE1E,uCAAuC;YACvC,qEAAqE;YACrE,MAAM,OAAO,GAAG,QAAQ,YAAY,IAAI,WAAW,IAAI,YAAY,EAAE,CAAC;YACtE,IAAI,CAAC,OAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAE5B,gBAAgB;YAChB,MAAM,IAAI,OAAO,CAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACzC,IAAI,CAAC,cAAc,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;YAC5C,CAAC,CAAC,CAAC;YAEH,+CAA+C;YAC/C,4FAA4F;YAC5F,IAAI,YAAoB,CAAC;YACzB,IAAI,SAAc,CAAC;YAEnB,6HAA6H;YAC7H,iEAAiE;YACjE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC5F,IAAI,CAAC;oBACH,YAAY,GAAG,YAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;oBACtD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;wBAChB,gBAAM,CAAC,KAAK,CAAC,2CAA2C,OAAO,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC;oBAC1F,CAAC;oBACD,MAAM;gBACR,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,SAAS,GAAG,KAAK,CAAC;oBAClB,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC5B,kEAAkE;wBAClE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;wBAC3D,SAAS;oBACX,CAAC;oBACD,gCAAgC;oBAChC,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;YAED,yDAAyD;YACzD,IAAI,CAAC,YAAa,EAAE,CAAC;gBACnB,MAAM,OAAO,GAAG,cAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;gBAC3C,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,YAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC,CAAC;oBACvF,gBAAM,CAAC,KAAK,CACV,oEAAoE,cAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5I,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,gBAAM,CAAC,KAAK,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAC;gBAChE,CAAC;gBACD,MAAM,SAAS,CAAC;YAClB,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAa,CAAC,CAAC;YAE3C,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC,CAAC;YAClF,CAAC;YAED,OAAO,QAAQ,CAAC,IAAI,CAAC;QACvB,CAAC;gBAAS,CAAC;YACT,qBAAqB;YACrB,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC3C,IAAI,CAAC;oBACH,IAAI,YAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;wBACxB,YAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;oBACtB,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,gBAAM,CAAC,KAAK,CAAC,kBAAkB,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,aAAa;QACnB,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YAC/B,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE;gBACpC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YACvE,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACjB,wDAAwD;YACxD,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC9B,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,UAAU;QAChB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;YACxD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,gBAAM,CAAC,IAAI,CAAC,0BAA0B,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,kBAAkB,CAAC,CAAC;YAC5F,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/B,gBAAM,CAAC,KAAK,CAAC,6BAA6B,GAAG,EAAE,CAAC,CAAC;YACnD,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,UAAU,yBAAyB,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;YAEzB,wCAAwC;YACxC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACxB,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;gBAC9D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;YAC7B,CAAC;YAED,qEAAqE;YACrE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE9B,6BAA6B;YAC7B,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;oBAC5B,IAAI,CAAC,OAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC7C,CAAC,CAAC;gBACF,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;aAClE,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gBAAM,CAAC,KAAK,CAAC,iCAAiC,KAAK,EAAE,CAAC,CAAC;QACzD,CAAC;gBAAS,CAAC;YACT,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,CAAC;YACD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;YAClB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;CACF;AA1QD,oCA0QC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../../src/redteam/commands/generate.ts"],"names":[],"mappings":"AA+CA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,KAAK,EAA0B,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,KAAK,EAEV,yBAAyB,EAK1B,MAAM,UAAU,CAAC;AA+ClB,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,OAAO,CAAC,yBAAyB,CAAC,GAC1C,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../../src/redteam/commands/generate.ts"],"names":[],"mappings":"AA+CA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,KAAK,EAA0B,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,KAAK,EAEV,yBAAyB,EAK1B,MAAM,UAAU,CAAC;AA+ClB,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,OAAO,CAAC,yBAAyB,CAAC,GAC1C,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAygBxC;AAED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,SAAS,GAAG,UAAU,EAC/B,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EACrC,iBAAiB,EAAE,MAAM,GAAG,SAAS,QA0JtC"}
|
|
@@ -350,7 +350,6 @@ async function doGenerateRedteam(options) {
|
|
|
350
350
|
const { testCases: redteamTests, purpose, entities, injectVar: finalInjectVar, } = await (0, __1.synthesize)({
|
|
351
351
|
...parsedConfig.data,
|
|
352
352
|
purpose: enhancedPurpose,
|
|
353
|
-
language: config.language,
|
|
354
353
|
numTests: config.numTests,
|
|
355
354
|
prompts: testSuite.prompts.map((prompt) => prompt.raw),
|
|
356
355
|
maxConcurrency: config.maxConcurrency,
|