promptfoo 0.73.9 → 0.74.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -1
- package/dist/package.json +1 -1
- package/dist/src/assertions/AssertionsResult.d.ts +1 -1
- package/dist/src/assertions/AssertionsResult.d.ts.map +1 -1
- package/dist/src/assertions/validateAssertions.d.ts +1 -1
- package/dist/src/assertions/validateAssertions.d.ts.map +1 -1
- package/dist/src/assertions.d.ts +2 -1
- package/dist/src/assertions.d.ts.map +1 -1
- package/dist/src/assertions.js.map +1 -1
- package/dist/src/commands/cache.d.ts +1 -1
- package/dist/src/commands/cache.d.ts.map +1 -1
- package/dist/src/commands/config.d.ts +1 -1
- package/dist/src/commands/config.d.ts.map +1 -1
- package/dist/src/commands/delete.d.ts +1 -1
- package/dist/src/commands/delete.d.ts.map +1 -1
- package/dist/src/commands/eval/filterFailingTests.d.ts +1 -1
- package/dist/src/commands/eval/filterFailingTests.d.ts.map +1 -1
- package/dist/src/commands/eval/filterTests.d.ts +1 -1
- package/dist/src/commands/eval/filterTests.d.ts.map +1 -1
- package/dist/src/commands/eval.d.ts +2 -2
- package/dist/src/commands/eval.d.ts.map +1 -1
- package/dist/src/commands/eval.js.map +1 -1
- package/dist/src/commands/export.d.ts +1 -1
- package/dist/src/commands/export.d.ts.map +1 -1
- package/dist/src/commands/feedback.d.ts +1 -1
- package/dist/src/commands/feedback.d.ts.map +1 -1
- package/dist/src/commands/generate/dataset.d.ts +2 -2
- package/dist/src/commands/generate/dataset.d.ts.map +1 -1
- package/dist/src/commands/generate/redteam.d.ts +3 -3
- package/dist/src/commands/generate/redteam.d.ts.map +1 -1
- package/dist/src/commands/generate/redteam.js +13 -6
- package/dist/src/commands/generate/redteam.js.map +1 -1
- package/dist/src/commands/import.d.ts +1 -1
- package/dist/src/commands/import.d.ts.map +1 -1
- package/dist/src/commands/init.d.ts +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/list.d.ts +1 -1
- package/dist/src/commands/list.d.ts.map +1 -1
- package/dist/src/commands/redteam.d.ts +2 -1
- package/dist/src/commands/redteam.d.ts.map +1 -1
- package/dist/src/commands/redteam.js +200 -193
- package/dist/src/commands/redteam.js.map +1 -1
- package/dist/src/commands/share.d.ts +1 -1
- package/dist/src/commands/share.d.ts.map +1 -1
- package/dist/src/commands/show.d.ts +1 -1
- package/dist/src/commands/show.d.ts.map +1 -1
- package/dist/src/commands/version.d.ts +1 -1
- package/dist/src/commands/version.d.ts.map +1 -1
- package/dist/src/commands/view.d.ts +1 -1
- package/dist/src/commands/view.d.ts.map +1 -1
- package/dist/src/config.d.ts +1 -1
- package/dist/src/config.d.ts.map +1 -1
- package/dist/src/database/operations.d.ts +56 -240
- package/dist/src/database/operations.d.ts.map +1 -1
- package/dist/src/database/types.d.ts +1 -1
- package/dist/src/database/types.d.ts.map +1 -1
- package/dist/src/globalConfig.d.ts +1 -1
- package/dist/src/globalConfig.d.ts.map +1 -1
- package/dist/src/onboarding.d.ts.map +1 -1
- package/dist/src/onboarding.js +11 -0
- package/dist/src/onboarding.js.map +1 -1
- package/dist/src/prompts/grading.js +1 -1
- package/dist/src/prompts/processors/javascript.d.ts +1 -1
- package/dist/src/prompts/processors/javascript.d.ts.map +1 -1
- package/dist/src/prompts/processors/json.d.ts +1 -1
- package/dist/src/prompts/processors/json.d.ts.map +1 -1
- package/dist/src/prompts/processors/jsonl.d.ts +1 -1
- package/dist/src/prompts/processors/jsonl.d.ts.map +1 -1
- package/dist/src/prompts/processors/python.d.ts.map +1 -1
- package/dist/src/prompts/processors/python.js.map +1 -1
- package/dist/src/prompts/processors/string.d.ts +1 -1
- package/dist/src/prompts/processors/string.d.ts.map +1 -1
- package/dist/src/prompts/processors/text.d.ts +1 -1
- package/dist/src/prompts/processors/text.d.ts.map +1 -1
- package/dist/src/prompts/processors/yaml.d.ts +1 -1
- package/dist/src/prompts/processors/yaml.d.ts.map +1 -1
- package/dist/src/prompts/utils.d.ts +1 -1
- package/dist/src/prompts/utils.d.ts.map +1 -1
- package/dist/src/providers/anthropic.js +1 -1
- package/dist/src/providers/anthropic.js.map +1 -1
- package/dist/src/providers/azureopenai.d.ts.map +1 -1
- package/dist/src/providers/azureopenai.js +6 -2
- package/dist/src/providers/azureopenai.js.map +1 -1
- package/dist/src/providers/bedrock.d.ts.map +1 -1
- package/dist/src/providers/bedrock.js +2 -2
- package/dist/src/providers/bedrock.js.map +1 -1
- package/dist/src/providers/huggingface.d.ts +1 -1
- package/dist/src/providers/huggingface.d.ts.map +1 -1
- package/dist/src/providers/manualInput.d.ts +1 -1
- package/dist/src/providers/manualInput.d.ts.map +1 -1
- package/dist/src/providers/mistral.d.ts +1 -1
- package/dist/src/providers/mistral.d.ts.map +1 -1
- package/dist/src/providers/openai.d.ts +1 -1
- package/dist/src/providers/openai.d.ts.map +1 -1
- package/dist/src/providers/openai.js +8 -3
- package/dist/src/providers/openai.js.map +1 -1
- package/dist/src/providers/shared.d.ts +1 -0
- package/dist/src/providers/shared.d.ts.map +1 -1
- package/dist/src/providers/shared.js +47 -0
- package/dist/src/providers/shared.js.map +1 -1
- package/dist/src/providers/vertex.d.ts.map +1 -1
- package/dist/src/providers/vertex.js.map +1 -1
- package/dist/src/providers.d.ts +2 -1
- package/dist/src/providers.d.ts.map +1 -1
- package/dist/src/python/pythonUtils.d.ts.map +1 -1
- package/dist/src/python/pythonUtils.js.map +1 -1
- package/dist/src/redteam/constants.d.ts +1 -1
- package/dist/src/redteam/constants.js +3 -3
- package/dist/src/redteam/constants.js.map +1 -1
- package/dist/src/redteam/extraction/entities.d.ts +1 -1
- package/dist/src/redteam/extraction/entities.d.ts.map +1 -1
- package/dist/src/redteam/extraction/purpose.d.ts +1 -1
- package/dist/src/redteam/extraction/purpose.d.ts.map +1 -1
- package/dist/src/redteam/extraction/util.d.ts +1 -1
- package/dist/src/redteam/extraction/util.d.ts.map +1 -1
- package/dist/src/redteam/graders/base.d.ts +1 -1
- package/dist/src/redteam/graders/base.d.ts.map +1 -1
- package/dist/src/redteam/graders/harmful.d.ts +1 -1
- package/dist/src/redteam/graders/harmful.d.ts.map +1 -1
- package/dist/src/redteam/graders/index.d.ts +1 -1
- package/dist/src/redteam/graders/index.d.ts.map +1 -1
- package/dist/src/redteam/index.d.ts +3 -3
- package/dist/src/redteam/index.d.ts.map +1 -1
- package/dist/src/redteam/index.js +20 -8
- package/dist/src/redteam/index.js.map +1 -1
- package/dist/src/redteam/plugins/base.d.ts +4 -4
- package/dist/src/redteam/plugins/base.d.ts.map +1 -1
- package/dist/src/redteam/plugins/base.js +1 -1
- package/dist/src/redteam/plugins/base.js.map +1 -1
- package/dist/src/redteam/plugins/competitors.d.ts +2 -2
- package/dist/src/redteam/plugins/competitors.d.ts.map +1 -1
- package/dist/src/redteam/plugins/competitors.js +2 -3
- package/dist/src/redteam/plugins/competitors.js.map +1 -1
- package/dist/src/redteam/plugins/contracts.d.ts +2 -2
- package/dist/src/redteam/plugins/contracts.d.ts.map +1 -1
- package/dist/src/redteam/plugins/contracts.js +2 -3
- package/dist/src/redteam/plugins/contracts.js.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.d.ts +2 -2
- package/dist/src/redteam/plugins/debugAccess.d.ts.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.js +2 -3
- package/dist/src/redteam/plugins/debugAccess.js.map +1 -1
- package/dist/src/redteam/plugins/excessiveAgency.d.ts +2 -2
- package/dist/src/redteam/plugins/excessiveAgency.d.ts.map +1 -1
- package/dist/src/redteam/plugins/excessiveAgency.js +2 -3
- package/dist/src/redteam/plugins/excessiveAgency.js.map +1 -1
- package/dist/src/redteam/plugins/hallucination.d.ts +2 -2
- package/dist/src/redteam/plugins/hallucination.d.ts.map +1 -1
- package/dist/src/redteam/plugins/hallucination.js +2 -3
- package/dist/src/redteam/plugins/hallucination.js.map +1 -1
- package/dist/src/redteam/plugins/harmful.d.ts.map +1 -1
- package/dist/src/redteam/plugins/harmful.js +3 -1
- package/dist/src/redteam/plugins/harmful.js.map +1 -1
- package/dist/src/redteam/plugins/hijacking.d.ts +2 -2
- package/dist/src/redteam/plugins/hijacking.d.ts.map +1 -1
- package/dist/src/redteam/plugins/hijacking.js +2 -3
- package/dist/src/redteam/plugins/hijacking.js.map +1 -1
- package/dist/src/redteam/plugins/imitation.d.ts +1 -1
- package/dist/src/redteam/plugins/imitation.d.ts.map +1 -1
- package/dist/src/redteam/plugins/imitation.js +2 -3
- package/dist/src/redteam/plugins/imitation.js.map +1 -1
- package/dist/src/redteam/plugins/overreliance.d.ts +2 -2
- package/dist/src/redteam/plugins/overreliance.d.ts.map +1 -1
- package/dist/src/redteam/plugins/overreliance.js +2 -3
- package/dist/src/redteam/plugins/overreliance.js.map +1 -1
- package/dist/src/redteam/plugins/politics.d.ts +2 -2
- package/dist/src/redteam/plugins/politics.d.ts.map +1 -1
- package/dist/src/redteam/plugins/politics.js +2 -3
- package/dist/src/redteam/plugins/politics.js.map +1 -1
- package/dist/src/redteam/plugins/rbac.d.ts +2 -2
- package/dist/src/redteam/plugins/rbac.d.ts.map +1 -1
- package/dist/src/redteam/plugins/rbac.js +2 -3
- package/dist/src/redteam/plugins/rbac.js.map +1 -1
- package/dist/src/redteam/plugins/shellInjection.d.ts +2 -2
- package/dist/src/redteam/plugins/shellInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/shellInjection.js +2 -3
- package/dist/src/redteam/plugins/shellInjection.js.map +1 -1
- package/dist/src/redteam/plugins/sqlInjection.d.ts +2 -2
- package/dist/src/redteam/plugins/sqlInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/sqlInjection.js +2 -3
- package/dist/src/redteam/plugins/sqlInjection.js.map +1 -1
- package/dist/src/redteam/strategies/injections.d.ts +1 -1
- package/dist/src/redteam/strategies/injections.d.ts.map +1 -1
- package/dist/src/redteam/strategies/iterative.d.ts +1 -1
- package/dist/src/redteam/strategies/iterative.d.ts.map +1 -1
- package/dist/src/testCases.d.ts.map +1 -1
- package/dist/src/{types.d.ts → types/index.d.ts} +1496 -6518
- package/dist/src/{types.d.ts.map → types/index.d.ts.map} +1 -1
- package/dist/src/{types.js → types/index.js} +38 -172
- package/dist/src/types/index.js.map +1 -0
- package/dist/src/types/prompts.d.ts +29 -0
- package/dist/src/types/prompts.d.ts.map +1 -0
- package/dist/src/types/prompts.js +3 -0
- package/dist/src/types/prompts.js.map +1 -0
- package/dist/src/types/providers.d.ts +126 -0
- package/dist/src/types/providers.d.ts.map +1 -0
- package/dist/src/types/providers.js +11 -0
- package/dist/src/types/providers.js.map +1 -0
- package/dist/src/types/redteam.d.ts +51 -0
- package/dist/src/types/redteam.d.ts.map +1 -0
- package/dist/src/types/redteam.js +3 -0
- package/dist/src/types/redteam.js.map +1 -0
- package/dist/src/types/shared.d.ts +8 -0
- package/dist/src/types/shared.d.ts.map +1 -0
- package/dist/src/types/shared.js +3 -0
- package/dist/src/types/shared.js.map +1 -0
- package/dist/src/util/index.d.ts +20 -96
- package/dist/src/util/index.d.ts.map +1 -1
- package/dist/src/util/index.js +15 -8
- package/dist/src/util/index.js.map +1 -1
- package/dist/src/util/templates.d.ts +1 -1
- package/dist/src/util/templates.d.ts.map +1 -1
- package/dist/src/util/transform.d.ts +1 -1
- package/dist/src/util/transform.d.ts.map +1 -1
- package/dist/src/validators/prompts.d.ts +60 -0
- package/dist/src/validators/prompts.d.ts.map +1 -0
- package/dist/src/validators/prompts.js +32 -0
- package/dist/src/validators/prompts.js.map +1 -0
- package/dist/src/validators/providers.d.ts +1145 -0
- package/dist/src/validators/providers.d.ts.map +1 -0
- package/dist/src/validators/providers.js +133 -0
- package/dist/src/validators/providers.js.map +1 -0
- package/dist/src/validators/redteam.d.ts +517 -0
- package/dist/src/validators/redteam.d.ts.map +1 -0
- package/dist/src/{redteam/types.js → validators/redteam.js} +31 -26
- package/dist/src/validators/redteam.js.map +1 -0
- package/dist/src/validators/shared.d.ts +19 -0
- package/dist/src/validators/shared.d.ts.map +1 -0
- package/dist/src/validators/shared.js +12 -0
- package/dist/src/validators/shared.js.map +1 -0
- package/dist/src/web/nextui/404/index.html +1 -1
- package/dist/src/web/nextui/404.html +1 -1
- package/dist/src/web/nextui/_next/static/chunks/949-e778f1d346946d00.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/app/eval/[id]/{page-3f5dd9507c5c905d.js → page-9029528e35960a91.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/{layout-881a16d7fe67581c.js → layout-517c4420e852da35.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/page-81e3b8fee37eaf67.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/app/report/{page-95d5e86c47bb3593.js → page-36e98231504ad757.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/main-app-0dc90092cd7acaaf.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/{webpack-b0363d9598537efc.js → webpack-d3dba20c5c1ca02e.js} +1 -1
- package/dist/src/web/nextui/_next/static/css/32384cb640d2265e.css +1 -0
- package/dist/src/web/nextui/auth/login/index.html +1 -1
- package/dist/src/web/nextui/auth/login/index.txt +4 -4
- package/dist/src/web/nextui/auth/signup/index.html +1 -1
- package/dist/src/web/nextui/auth/signup/index.txt +4 -4
- package/dist/src/web/nextui/datasets/index.html +1 -1
- package/dist/src/web/nextui/datasets/index.txt +4 -4
- package/dist/src/web/nextui/eval/index.html +1 -1
- package/dist/src/web/nextui/eval/index.txt +5 -5
- package/dist/src/web/nextui/index.html +1 -1
- package/dist/src/web/nextui/index.txt +4 -4
- package/dist/src/web/nextui/progress/index.html +1 -1
- package/dist/src/web/nextui/progress/index.txt +4 -4
- package/dist/src/web/nextui/prompts/index.html +1 -1
- package/dist/src/web/nextui/prompts/index.txt +4 -4
- package/dist/src/web/nextui/report/index.html +1 -1
- package/dist/src/web/nextui/report/index.txt +7 -7
- package/dist/src/web/nextui/setup/index.html +1 -1
- package/dist/src/web/nextui/setup/index.txt +5 -5
- package/dist/src/web/server.d.ts.map +1 -1
- package/dist/src/web/server.js.map +1 -1
- package/dist/test/cache.test.js.map +1 -1
- package/dist/test/providers.bedrock.test.js.map +1 -1
- package/dist/test/providers.shared.test.d.ts +2 -0
- package/dist/test/providers.shared.test.d.ts.map +1 -0
- package/dist/test/providers.shared.test.js +101 -0
- package/dist/test/providers.shared.test.js.map +1 -0
- package/dist/test/redteam/plugins/base.test.js +2 -3
- package/dist/test/redteam/plugins/base.test.js.map +1 -1
- package/dist/test/redteam/validators.test.d.ts +2 -0
- package/dist/test/redteam/validators.test.d.ts.map +1 -0
- package/dist/test/redteam/{types.test.js → validators.test.js} +84 -29
- package/dist/test/redteam/validators.test.js.map +1 -0
- package/dist/test/utils.d.ts +1 -1
- package/dist/test/utils.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/src/redteam/types.d.ts +0 -215
- package/dist/src/redteam/types.d.ts.map +0 -1
- package/dist/src/redteam/types.js.map +0 -1
- package/dist/src/types.js.map +0 -1
- package/dist/src/web/nextui/_next/static/chunks/347-49b79b8c81884e85.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/app/page-7e8faab5d212f727.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/main-app-4398f7af7ad8387f.js +0 -1
- package/dist/src/web/nextui/_next/static/css/e17fdafd8599f69b.css +0 -1
- package/dist/test/redteam/types.test.d.ts +0 -2
- package/dist/test/redteam/types.test.d.ts.map +0 -1
- package/dist/test/redteam/types.test.js.map +0 -1
- /package/dist/src/web/nextui/_next/static/{rXJSLz3x5ieAU1Syln-RX → 9g6bpuNNR0WtmBBAmnqTc}/_buildManifest.js +0 -0
- /package/dist/src/web/nextui/_next/static/{rXJSLz3x5ieAU1Syln-RX → 9g6bpuNNR0WtmBBAmnqTc}/_ssgManifest.js +0 -0
- /package/dist/src/web/nextui/_next/static/chunks/{258-f5d5d6dbebb8ae22.js → 258-280986e9c8fae219.js} +0 -0
|
@@ -145,34 +145,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
145
145
|
columnType: "SQLiteTextJson";
|
|
146
146
|
data: Partial<{
|
|
147
147
|
prompts: string | Record<string, string> | (string | {
|
|
148
|
-
label: string;
|
|
149
148
|
raw: string;
|
|
149
|
+
label: string;
|
|
150
150
|
function?: ((args_0: {
|
|
151
151
|
vars: Record<string, any>;
|
|
152
|
-
provider?:
|
|
153
|
-
id: (...args: unknown[]) => string;
|
|
154
|
-
callApi: {
|
|
155
|
-
(prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
|
|
156
|
-
label?: string;
|
|
157
|
-
};
|
|
158
|
-
delay?: number | undefined;
|
|
159
|
-
label?: string | undefined;
|
|
160
|
-
transform?: string | undefined;
|
|
161
|
-
callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
|
|
162
|
-
error?: string | undefined;
|
|
163
|
-
embedding?: number[] | undefined;
|
|
164
|
-
tokenUsage?: {
|
|
165
|
-
cached?: number | undefined;
|
|
166
|
-
completion?: number | undefined;
|
|
167
|
-
prompt?: number | undefined;
|
|
168
|
-
total?: number | undefined;
|
|
169
|
-
} | undefined;
|
|
170
|
-
}>) | undefined;
|
|
171
|
-
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
|
|
172
|
-
error?: string | undefined;
|
|
173
|
-
classification?: Record<string, number> | undefined;
|
|
174
|
-
}>) | undefined;
|
|
175
|
-
} | undefined;
|
|
152
|
+
provider?: import("../types").ApiProvider | undefined;
|
|
176
153
|
}, ...args_1: unknown[]) => Promise<any>) | undefined;
|
|
177
154
|
id?: string | undefined;
|
|
178
155
|
display?: string | undefined;
|
|
@@ -180,34 +157,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
180
157
|
providers: string | (((args_0: string, args_1: {
|
|
181
158
|
vars: Record<string, string | {}>;
|
|
182
159
|
prompt: {
|
|
183
|
-
label: string;
|
|
184
160
|
raw: string;
|
|
161
|
+
label: string;
|
|
185
162
|
function?: ((args_0: {
|
|
186
163
|
vars: Record<string, any>;
|
|
187
|
-
provider?:
|
|
188
|
-
id: (...args: unknown[]) => string;
|
|
189
|
-
callApi: {
|
|
190
|
-
(prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
|
|
191
|
-
label?: string;
|
|
192
|
-
};
|
|
193
|
-
delay?: number | undefined;
|
|
194
|
-
label?: string | undefined;
|
|
195
|
-
transform?: string | undefined;
|
|
196
|
-
callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
|
|
197
|
-
error?: string | undefined;
|
|
198
|
-
embedding?: number[] | undefined;
|
|
199
|
-
tokenUsage?: {
|
|
200
|
-
cached?: number | undefined;
|
|
201
|
-
completion?: number | undefined;
|
|
202
|
-
prompt?: number | undefined;
|
|
203
|
-
total?: number | undefined;
|
|
204
|
-
} | undefined;
|
|
205
|
-
}>) | undefined;
|
|
206
|
-
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
|
|
207
|
-
error?: string | undefined;
|
|
208
|
-
classification?: Record<string, number> | undefined;
|
|
209
|
-
}>) | undefined;
|
|
210
|
-
} | undefined;
|
|
164
|
+
provider?: import("../types").ApiProvider | undefined;
|
|
211
165
|
}, ...args_1: unknown[]) => Promise<any>) | undefined;
|
|
212
166
|
id?: string | undefined;
|
|
213
167
|
display?: string | undefined;
|
|
@@ -219,30 +173,15 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
219
173
|
originalProvider?: any;
|
|
220
174
|
} | undefined, args_2: {
|
|
221
175
|
includeLogProbs?: boolean | undefined;
|
|
222
|
-
} | undefined, ...args_3: unknown[]) => Promise<{
|
|
223
|
-
output?: any;
|
|
224
|
-
error?: string | undefined;
|
|
225
|
-
cached?: boolean | undefined;
|
|
226
|
-
tokenUsage?: {
|
|
227
|
-
cached?: number | undefined;
|
|
228
|
-
completion?: number | undefined;
|
|
229
|
-
prompt?: number | undefined;
|
|
230
|
-
total?: number | undefined;
|
|
231
|
-
} | undefined;
|
|
232
|
-
cost?: number | undefined;
|
|
233
|
-
logProbs?: number[] | undefined;
|
|
234
|
-
metadata?: import("zod").objectOutputType<{
|
|
235
|
-
redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
|
|
236
|
-
}, import("zod").ZodAny, "strip"> | undefined;
|
|
237
|
-
}>) & {
|
|
176
|
+
} | undefined, ...args_3: unknown[]) => Promise<import("../types").ProviderResponse>) & {
|
|
238
177
|
label?: string | undefined;
|
|
239
178
|
}) | (string | {
|
|
240
179
|
id?: string | undefined;
|
|
180
|
+
label?: string | undefined;
|
|
241
181
|
config?: any;
|
|
242
182
|
prompts?: string[] | undefined;
|
|
243
|
-
delay?: number | undefined;
|
|
244
|
-
label?: string | undefined;
|
|
245
183
|
transform?: string | undefined;
|
|
184
|
+
delay?: number | undefined;
|
|
246
185
|
env?: {
|
|
247
186
|
ANTHROPIC_API_KEY?: string | undefined;
|
|
248
187
|
BAM_API_KEY?: string | undefined;
|
|
@@ -279,34 +218,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
279
218
|
} | (((args_0: string, args_1: {
|
|
280
219
|
vars: Record<string, string | {}>;
|
|
281
220
|
prompt: {
|
|
282
|
-
label: string;
|
|
283
221
|
raw: string;
|
|
222
|
+
label: string;
|
|
284
223
|
function?: ((args_0: {
|
|
285
224
|
vars: Record<string, any>;
|
|
286
|
-
provider?:
|
|
287
|
-
id: (...args: unknown[]) => string;
|
|
288
|
-
callApi: {
|
|
289
|
-
(prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
|
|
290
|
-
label?: string;
|
|
291
|
-
};
|
|
292
|
-
delay?: number | undefined;
|
|
293
|
-
label?: string | undefined;
|
|
294
|
-
transform?: string | undefined;
|
|
295
|
-
callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
|
|
296
|
-
error?: string | undefined;
|
|
297
|
-
embedding?: number[] | undefined;
|
|
298
|
-
tokenUsage?: {
|
|
299
|
-
cached?: number | undefined;
|
|
300
|
-
completion?: number | undefined;
|
|
301
|
-
prompt?: number | undefined;
|
|
302
|
-
total?: number | undefined;
|
|
303
|
-
} | undefined;
|
|
304
|
-
}>) | undefined;
|
|
305
|
-
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
|
|
306
|
-
error?: string | undefined;
|
|
307
|
-
classification?: Record<string, number> | undefined;
|
|
308
|
-
}>) | undefined;
|
|
309
|
-
} | undefined;
|
|
225
|
+
provider?: import("../types").ApiProvider | undefined;
|
|
310
226
|
}, ...args_1: unknown[]) => Promise<any>) | undefined;
|
|
311
227
|
id?: string | undefined;
|
|
312
228
|
display?: string | undefined;
|
|
@@ -318,30 +234,15 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
318
234
|
originalProvider?: any;
|
|
319
235
|
} | undefined, args_2: {
|
|
320
236
|
includeLogProbs?: boolean | undefined;
|
|
321
|
-
} | undefined, ...args_3: unknown[]) => Promise<{
|
|
322
|
-
output?: any;
|
|
323
|
-
error?: string | undefined;
|
|
324
|
-
cached?: boolean | undefined;
|
|
325
|
-
tokenUsage?: {
|
|
326
|
-
cached?: number | undefined;
|
|
327
|
-
completion?: number | undefined;
|
|
328
|
-
prompt?: number | undefined;
|
|
329
|
-
total?: number | undefined;
|
|
330
|
-
} | undefined;
|
|
331
|
-
cost?: number | undefined;
|
|
332
|
-
logProbs?: number[] | undefined;
|
|
333
|
-
metadata?: import("zod").objectOutputType<{
|
|
334
|
-
redteamFinalPrompt: import("zod").ZodOptional<import("zod").ZodString>;
|
|
335
|
-
}, import("zod").ZodAny, "strip"> | undefined;
|
|
336
|
-
}>) & {
|
|
237
|
+
} | undefined, ...args_3: unknown[]) => Promise<import("../types").ProviderResponse>) & {
|
|
337
238
|
label?: string | undefined;
|
|
338
239
|
}) | Record<string, {
|
|
339
240
|
id?: string | undefined;
|
|
241
|
+
label?: string | undefined;
|
|
340
242
|
config?: any;
|
|
341
243
|
prompts?: string[] | undefined;
|
|
342
|
-
delay?: number | undefined;
|
|
343
|
-
label?: string | undefined;
|
|
344
244
|
transform?: string | undefined;
|
|
245
|
+
delay?: number | undefined;
|
|
345
246
|
env?: {
|
|
346
247
|
ANTHROPIC_API_KEY?: string | undefined;
|
|
347
248
|
BAM_API_KEY?: string | undefined;
|
|
@@ -401,13 +302,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
401
302
|
disableVarExpansion?: boolean | undefined;
|
|
402
303
|
disableConversationVar?: boolean | undefined;
|
|
403
304
|
}) | undefined;
|
|
305
|
+
vars?: Record<string, string | any[] | string[] | {}> | undefined;
|
|
404
306
|
provider?: string | {
|
|
405
307
|
id?: string | undefined;
|
|
308
|
+
label?: string | undefined;
|
|
406
309
|
config?: any;
|
|
407
310
|
prompts?: string[] | undefined;
|
|
408
|
-
delay?: number | undefined;
|
|
409
|
-
label?: string | undefined;
|
|
410
311
|
transform?: string | undefined;
|
|
312
|
+
delay?: number | undefined;
|
|
411
313
|
env?: {
|
|
412
314
|
ANTHROPIC_API_KEY?: string | undefined;
|
|
413
315
|
BAM_API_KEY?: string | undefined;
|
|
@@ -442,30 +344,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
442
344
|
CLOUDFLARE_ACCOUNT_ID?: string | undefined;
|
|
443
345
|
} | undefined;
|
|
444
346
|
} | {
|
|
347
|
+
callApi: import("../types").CallApiFunction;
|
|
445
348
|
id: (...args: unknown[]) => string;
|
|
446
|
-
callApi: {
|
|
447
|
-
(prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
|
|
448
|
-
label?: string;
|
|
449
|
-
};
|
|
450
|
-
delay?: number | undefined;
|
|
451
349
|
label?: string | undefined;
|
|
452
350
|
transform?: string | undefined;
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
tokenUsage?: {
|
|
457
|
-
cached?: number | undefined;
|
|
458
|
-
completion?: number | undefined;
|
|
459
|
-
prompt?: number | undefined;
|
|
460
|
-
total?: number | undefined;
|
|
461
|
-
} | undefined;
|
|
462
|
-
}>) | undefined;
|
|
463
|
-
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
|
|
464
|
-
error?: string | undefined;
|
|
465
|
-
classification?: Record<string, number> | undefined;
|
|
466
|
-
}>) | undefined;
|
|
351
|
+
delay?: number | undefined;
|
|
352
|
+
callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
|
|
353
|
+
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
|
|
467
354
|
} | undefined;
|
|
468
|
-
vars?: Record<string, string | any[] | string[] | {}> | undefined;
|
|
469
355
|
metadata?: Record<string, any> | undefined;
|
|
470
356
|
description?: string | undefined;
|
|
471
357
|
providerOutput?: string | {} | undefined;
|
|
@@ -563,13 +449,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
563
449
|
disableVarExpansion?: boolean | undefined;
|
|
564
450
|
disableConversationVar?: boolean | undefined;
|
|
565
451
|
}) | undefined;
|
|
452
|
+
vars?: Record<string, string | any[] | string[] | {}> | undefined;
|
|
566
453
|
provider?: string | {
|
|
567
454
|
id?: string | undefined;
|
|
455
|
+
label?: string | undefined;
|
|
568
456
|
config?: any;
|
|
569
457
|
prompts?: string[] | undefined;
|
|
570
|
-
delay?: number | undefined;
|
|
571
|
-
label?: string | undefined;
|
|
572
458
|
transform?: string | undefined;
|
|
459
|
+
delay?: number | undefined;
|
|
573
460
|
env?: {
|
|
574
461
|
ANTHROPIC_API_KEY?: string | undefined;
|
|
575
462
|
BAM_API_KEY?: string | undefined;
|
|
@@ -604,30 +491,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
604
491
|
CLOUDFLARE_ACCOUNT_ID?: string | undefined;
|
|
605
492
|
} | undefined;
|
|
606
493
|
} | {
|
|
494
|
+
callApi: import("../types").CallApiFunction;
|
|
607
495
|
id: (...args: unknown[]) => string;
|
|
608
|
-
callApi: {
|
|
609
|
-
(prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
|
|
610
|
-
label?: string;
|
|
611
|
-
};
|
|
612
|
-
delay?: number | undefined;
|
|
613
496
|
label?: string | undefined;
|
|
614
497
|
transform?: string | undefined;
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
tokenUsage?: {
|
|
619
|
-
cached?: number | undefined;
|
|
620
|
-
completion?: number | undefined;
|
|
621
|
-
prompt?: number | undefined;
|
|
622
|
-
total?: number | undefined;
|
|
623
|
-
} | undefined;
|
|
624
|
-
}>) | undefined;
|
|
625
|
-
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
|
|
626
|
-
error?: string | undefined;
|
|
627
|
-
classification?: Record<string, number> | undefined;
|
|
628
|
-
}>) | undefined;
|
|
498
|
+
delay?: number | undefined;
|
|
499
|
+
callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
|
|
500
|
+
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
|
|
629
501
|
} | undefined;
|
|
630
|
-
vars?: Record<string, string | any[] | string[] | {}> | undefined;
|
|
631
502
|
metadata?: Record<string, any> | undefined;
|
|
632
503
|
description?: string | undefined;
|
|
633
504
|
providerOutput?: string | {} | undefined;
|
|
@@ -689,13 +560,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
689
560
|
disableVarExpansion?: boolean | undefined;
|
|
690
561
|
disableConversationVar?: boolean | undefined;
|
|
691
562
|
}) | undefined;
|
|
563
|
+
vars?: Record<string, string | any[] | string[] | {}> | undefined;
|
|
692
564
|
provider?: string | {
|
|
693
565
|
id?: string | undefined;
|
|
566
|
+
label?: string | undefined;
|
|
694
567
|
config?: any;
|
|
695
568
|
prompts?: string[] | undefined;
|
|
696
|
-
delay?: number | undefined;
|
|
697
|
-
label?: string | undefined;
|
|
698
569
|
transform?: string | undefined;
|
|
570
|
+
delay?: number | undefined;
|
|
699
571
|
env?: {
|
|
700
572
|
ANTHROPIC_API_KEY?: string | undefined;
|
|
701
573
|
BAM_API_KEY?: string | undefined;
|
|
@@ -730,30 +602,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
730
602
|
CLOUDFLARE_ACCOUNT_ID?: string | undefined;
|
|
731
603
|
} | undefined;
|
|
732
604
|
} | {
|
|
605
|
+
callApi: import("../types").CallApiFunction;
|
|
733
606
|
id: (...args: unknown[]) => string;
|
|
734
|
-
callApi: {
|
|
735
|
-
(prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
|
|
736
|
-
label?: string;
|
|
737
|
-
};
|
|
738
|
-
delay?: number | undefined;
|
|
739
607
|
label?: string | undefined;
|
|
740
608
|
transform?: string | undefined;
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
tokenUsage?: {
|
|
745
|
-
cached?: number | undefined;
|
|
746
|
-
completion?: number | undefined;
|
|
747
|
-
prompt?: number | undefined;
|
|
748
|
-
total?: number | undefined;
|
|
749
|
-
} | undefined;
|
|
750
|
-
}>) | undefined;
|
|
751
|
-
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
|
|
752
|
-
error?: string | undefined;
|
|
753
|
-
classification?: Record<string, number> | undefined;
|
|
754
|
-
}>) | undefined;
|
|
609
|
+
delay?: number | undefined;
|
|
610
|
+
callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
|
|
611
|
+
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
|
|
755
612
|
} | undefined;
|
|
756
|
-
vars?: Record<string, string | any[] | string[] | {}> | undefined;
|
|
757
613
|
metadata?: Record<string, any> | undefined;
|
|
758
614
|
description?: string | undefined;
|
|
759
615
|
providerOutput?: string | {} | undefined;
|
|
@@ -817,13 +673,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
817
673
|
disableVarExpansion?: boolean | undefined;
|
|
818
674
|
disableConversationVar?: boolean | undefined;
|
|
819
675
|
}) | undefined;
|
|
676
|
+
vars?: Record<string, string | any[] | string[] | {}> | undefined;
|
|
820
677
|
provider?: string | {
|
|
821
678
|
id?: string | undefined;
|
|
679
|
+
label?: string | undefined;
|
|
822
680
|
config?: any;
|
|
823
681
|
prompts?: string[] | undefined;
|
|
824
|
-
delay?: number | undefined;
|
|
825
|
-
label?: string | undefined;
|
|
826
682
|
transform?: string | undefined;
|
|
683
|
+
delay?: number | undefined;
|
|
827
684
|
env?: {
|
|
828
685
|
ANTHROPIC_API_KEY?: string | undefined;
|
|
829
686
|
BAM_API_KEY?: string | undefined;
|
|
@@ -858,30 +715,14 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
858
715
|
CLOUDFLARE_ACCOUNT_ID?: string | undefined;
|
|
859
716
|
} | undefined;
|
|
860
717
|
} | {
|
|
718
|
+
callApi: import("../types").CallApiFunction;
|
|
861
719
|
id: (...args: unknown[]) => string;
|
|
862
|
-
callApi: {
|
|
863
|
-
(prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
|
|
864
|
-
label?: string;
|
|
865
|
-
};
|
|
866
|
-
delay?: number | undefined;
|
|
867
720
|
label?: string | undefined;
|
|
868
721
|
transform?: string | undefined;
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
tokenUsage?: {
|
|
873
|
-
cached?: number | undefined;
|
|
874
|
-
completion?: number | undefined;
|
|
875
|
-
prompt?: number | undefined;
|
|
876
|
-
total?: number | undefined;
|
|
877
|
-
} | undefined;
|
|
878
|
-
}>) | undefined;
|
|
879
|
-
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
|
|
880
|
-
error?: string | undefined;
|
|
881
|
-
classification?: Record<string, number> | undefined;
|
|
882
|
-
}>) | undefined;
|
|
722
|
+
delay?: number | undefined;
|
|
723
|
+
callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
|
|
724
|
+
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
|
|
883
725
|
} | undefined;
|
|
884
|
-
vars?: Record<string, string | any[] | string[] | {}> | undefined;
|
|
885
726
|
metadata?: Record<string, any> | undefined;
|
|
886
727
|
providerOutput?: string | {} | undefined;
|
|
887
728
|
assert?: ({
|
|
@@ -922,17 +763,7 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
922
763
|
value: string | ((args_0: Record<string, number>, args_1: import("../types").RunEvalOptions, ...args_2: unknown[]) => number);
|
|
923
764
|
name: string;
|
|
924
765
|
}[] | undefined;
|
|
925
|
-
redteam?:
|
|
926
|
-
plugins: any[];
|
|
927
|
-
strategies: ({
|
|
928
|
-
id: string;
|
|
929
|
-
} | {
|
|
930
|
-
id: string;
|
|
931
|
-
})[];
|
|
932
|
-
provider?: string | undefined;
|
|
933
|
-
injectVar?: string | undefined;
|
|
934
|
-
purpose?: string | undefined;
|
|
935
|
-
} | undefined;
|
|
766
|
+
redteam?: import("../types").RedteamConfig | undefined;
|
|
936
767
|
outputPath?: string | string[] | undefined;
|
|
937
768
|
sharing?: boolean | {
|
|
938
769
|
apiBaseUrl?: string | undefined;
|
|
@@ -940,9 +771,9 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
940
771
|
} | undefined;
|
|
941
772
|
evaluateOptions?: {
|
|
942
773
|
repeat?: number | undefined;
|
|
943
|
-
cache?: boolean | undefined;
|
|
944
|
-
maxConcurrency?: number | undefined;
|
|
945
774
|
delay?: number | undefined;
|
|
775
|
+
maxConcurrency?: number | undefined;
|
|
776
|
+
cache?: boolean | undefined;
|
|
946
777
|
interactiveProviders?: boolean | undefined;
|
|
947
778
|
generateSuggestions?: boolean | undefined;
|
|
948
779
|
showProgressBar?: boolean | undefined;
|
|
@@ -951,22 +782,21 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
951
782
|
} | undefined;
|
|
952
783
|
commandLineOptions?: {
|
|
953
784
|
repeat?: string | undefined;
|
|
954
|
-
|
|
785
|
+
vars?: string | undefined;
|
|
955
786
|
config?: string[] | undefined;
|
|
956
|
-
envFile?: string | undefined;
|
|
957
|
-
output?: string[] | undefined;
|
|
958
|
-
write?: boolean | undefined;
|
|
959
787
|
prompts?: string[] | undefined;
|
|
788
|
+
delay?: string | undefined;
|
|
789
|
+
output?: string[] | undefined;
|
|
960
790
|
providers?: string[] | undefined;
|
|
961
791
|
maxConcurrency?: string | undefined;
|
|
962
|
-
delay?: string | undefined;
|
|
963
|
-
vars?: string | undefined;
|
|
964
792
|
tests?: string | undefined;
|
|
965
793
|
assertions?: string | undefined;
|
|
966
794
|
modelOutputs?: string | undefined;
|
|
967
795
|
verbose?: boolean | undefined;
|
|
968
796
|
grader?: string | undefined;
|
|
969
797
|
tableCellMaxLength?: string | undefined;
|
|
798
|
+
write?: boolean | undefined;
|
|
799
|
+
cache?: boolean | undefined;
|
|
970
800
|
table?: boolean | undefined;
|
|
971
801
|
share?: boolean | undefined;
|
|
972
802
|
progressBar?: boolean | undefined;
|
|
@@ -980,6 +810,7 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
|
|
|
980
810
|
generateSuggestions?: boolean | undefined;
|
|
981
811
|
promptPrefix?: string | undefined;
|
|
982
812
|
promptSuffix?: string | undefined;
|
|
813
|
+
envFile?: string | undefined;
|
|
983
814
|
} | undefined;
|
|
984
815
|
}>;
|
|
985
816
|
driverParam: string;
|
|
@@ -1087,13 +918,14 @@ export declare const datasets: import("drizzle-orm/sqlite-core").SQLiteTableWith
|
|
|
1087
918
|
disableVarExpansion?: boolean | undefined;
|
|
1088
919
|
disableConversationVar?: boolean | undefined;
|
|
1089
920
|
}) | undefined;
|
|
921
|
+
vars?: Record<string, string | any[] | string[] | {}> | undefined;
|
|
1090
922
|
provider?: string | {
|
|
1091
923
|
id?: string | undefined;
|
|
924
|
+
label?: string | undefined;
|
|
1092
925
|
config?: any;
|
|
1093
926
|
prompts?: string[] | undefined;
|
|
1094
|
-
delay?: number | undefined;
|
|
1095
|
-
label?: string | undefined;
|
|
1096
927
|
transform?: string | undefined;
|
|
928
|
+
delay?: number | undefined;
|
|
1097
929
|
env?: {
|
|
1098
930
|
ANTHROPIC_API_KEY?: string | undefined;
|
|
1099
931
|
BAM_API_KEY?: string | undefined;
|
|
@@ -1128,30 +960,14 @@ export declare const datasets: import("drizzle-orm/sqlite-core").SQLiteTableWith
|
|
|
1128
960
|
CLOUDFLARE_ACCOUNT_ID?: string | undefined;
|
|
1129
961
|
} | undefined;
|
|
1130
962
|
} | {
|
|
963
|
+
callApi: import("../types").CallApiFunction;
|
|
1131
964
|
id: (...args: unknown[]) => string;
|
|
1132
|
-
callApi: {
|
|
1133
|
-
(prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
|
|
1134
|
-
label?: string;
|
|
1135
|
-
};
|
|
1136
|
-
delay?: number | undefined;
|
|
1137
965
|
label?: string | undefined;
|
|
1138
966
|
transform?: string | undefined;
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
tokenUsage?: {
|
|
1143
|
-
cached?: number | undefined;
|
|
1144
|
-
completion?: number | undefined;
|
|
1145
|
-
prompt?: number | undefined;
|
|
1146
|
-
total?: number | undefined;
|
|
1147
|
-
} | undefined;
|
|
1148
|
-
}>) | undefined;
|
|
1149
|
-
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
|
|
1150
|
-
error?: string | undefined;
|
|
1151
|
-
classification?: Record<string, number> | undefined;
|
|
1152
|
-
}>) | undefined;
|
|
967
|
+
delay?: number | undefined;
|
|
968
|
+
callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
|
|
969
|
+
callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
|
|
1153
970
|
} | undefined;
|
|
1154
|
-
vars?: Record<string, string | any[] | string[] | {}> | undefined;
|
|
1155
971
|
metadata?: Record<string, any> | undefined;
|
|
1156
972
|
description?: string | undefined;
|
|
1157
973
|
providerOutput?: string | {} | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../../src/database/operations.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAiB,MAAM,UAAU,CAAC;AAI/D,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMlB,CAAC;AAEH,eAAO,MAAM,KAAK
|
|
1
|
+
{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../../src/database/operations.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAiB,MAAM,UAAU,CAAC;AAI/D,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMlB,CAAC;AAEH,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAShB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe1B,CAAC;AAEF,eAAO,MAAM,gBAAgB;;EAE1B,CAAC;AAIJ,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMnB,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe3B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;EAE3B,CAAC;AAIJ,eAAO,MAAM,cAAc;;;EAGxB,CAAC;AAEJ,eAAO,MAAM,uBAAuB;;;EASjC,CAAC;AAEJ,eAAO,MAAM,wBAAwB;;;EASlC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/database/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/database/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,eAAe,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,OAAO,EAAE,eAAe,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { GlobalConfig } from './configTypes';
|
|
1
|
+
import type { GlobalConfig } from './configTypes';
|
|
2
2
|
export declare function resetGlobalConfig(): void;
|
|
3
3
|
export declare function readGlobalConfig(): GlobalConfig;
|
|
4
4
|
export declare function writeGlobalConfig(config: GlobalConfig): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"globalConfig.d.ts","sourceRoot":"","sources":["../../src/globalConfig.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"globalConfig.d.ts","sourceRoot":"","sources":["../../src/globalConfig.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAKlD,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED,wBAAgB,gBAAgB,IAAI,YAAY,CAiB/C;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAK5D;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,aAAa,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAcnF;AAED,wBAAgB,mBAAmB,IAAI,OAAO,CAa7C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"onboarding.d.ts","sourceRoot":"","sources":["../../src/onboarding.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"onboarding.d.ts","sourceRoot":"","sources":["../../src/onboarding.ts"],"names":[],"mappings":"AASA,eAAO,MAAM,eAAe,6hFAmF3B,CAAC;AAEF,eAAO,MAAM,eAAe,muCA8B3B,CAAC;AAEF,eAAO,MAAM,mBAAmB,0/BAsC/B,CAAC;AAEF,eAAO,MAAM,aAAa,gTAQzB,CAAC;AAEF,eAAO,MAAM,UAAU,upBAgBtB,CAAC;AAEF,eAAO,MAAM,cAAc,moBAkB1B,CAAC;AAEF,eAAO,MAAM,cAAc,0QAU1B,CAAC;AAEF,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,EAAE,WAAW,GAAE,OAAc;;;;;GAiP3F"}
|
package/dist/src/onboarding.js
CHANGED
|
@@ -10,6 +10,7 @@ const rawlist_1 = __importDefault(require("@inquirer/rawlist"));
|
|
|
10
10
|
const chalk_1 = __importDefault(require("chalk"));
|
|
11
11
|
const fs_1 = __importDefault(require("fs"));
|
|
12
12
|
const path_1 = __importDefault(require("path"));
|
|
13
|
+
const redteam_1 = require("./commands/redteam");
|
|
13
14
|
const logger_1 = __importDefault(require("./logger"));
|
|
14
15
|
const templates_1 = require("./util/templates");
|
|
15
16
|
exports.CONFIG_TEMPLATE = `# Learn more about building a configuration: https://promptfoo.dev/docs/configuration/guide
|
|
@@ -251,8 +252,18 @@ async function createDummyFiles(directory, interactive = true) {
|
|
|
251
252
|
{ name: 'Improve prompt and model performance', value: 'compare' },
|
|
252
253
|
{ name: 'Improve RAG performance', value: 'rag' },
|
|
253
254
|
{ name: 'Improve agent/chain of thought performance', value: 'agent' },
|
|
255
|
+
{ name: 'Run a red team evaluation', value: 'redteam' },
|
|
254
256
|
],
|
|
255
257
|
});
|
|
258
|
+
if (action === 'redteam') {
|
|
259
|
+
await (0, redteam_1.redteamInit)(directory || '.');
|
|
260
|
+
return {
|
|
261
|
+
numPrompts: 0,
|
|
262
|
+
providerPrefixes: [],
|
|
263
|
+
action: 'redteam',
|
|
264
|
+
language: 'not_applicable',
|
|
265
|
+
};
|
|
266
|
+
}
|
|
256
267
|
language = 'not_sure';
|
|
257
268
|
if (action === 'rag' || action === 'agent') {
|
|
258
269
|
language = await (0, rawlist_1.default)({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"onboarding.js","sourceRoot":"","sources":["../../src/onboarding.ts"],"names":[],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"onboarding.js","sourceRoot":"","sources":["../../src/onboarding.ts"],"names":[],"mappings":";;;;;;AAkOA,4CAiPC;AAndD,kEAA0C;AAC1C,gEAAwC;AACxC,kDAA0B;AAC1B,4CAAoB;AACpB,gDAAwB;AACxB,gDAAiD;AACjD,sDAA8B;AAC9B,gDAAqD;AAExC,QAAA,eAAe,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmF9B,CAAC;AAEW,QAAA,eAAe,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8B9B,CAAC;AAEW,QAAA,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsClC,CAAC;AAEW,QAAA,aAAa,GAAG;;;;;;;;CAQ5B,CAAC;AAEW,QAAA,UAAU,GAAG;;;;;;;;;;;;;;;;CAgBzB,CAAC;AAEW,QAAA,cAAc,GAAG;;;;;;;;;;;;;;;;;;CAkB7B,CAAC;AAEW,QAAA,cAAc,GAAG;;;;;;;;;;CAU7B,CAAC;AAEK,KAAK,UAAU,gBAAgB,CAAC,SAAwB,EAAE,cAAuB,IAAI;IAC1F,IAAI,SAAS,EAAE,CAAC;QACd,yCAAyC;QACzC,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,YAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,gBAAM,CAAC,IAAI,CAAC,sBAAsB,SAAS,MAAM,CAAC,CAAC;YACnD,YAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;SAAM,CAAC;QACN,SAAS,GAAG,GAAG,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAwB,EAAE,CAAC;IAC1C,IAAI,MAAc,CAAC;IACnB,IAAI,QAAgB,CAAC;IACrB,IAAI,WAAW,EAAE,CAAC;QAChB,kBAAkB;QAClB,MAAM,GAAG,MAAM,IAAA,iBAAO,EAAC;YACrB,OAAO,EAAE,4BAA4B;YACrC,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,SAAS,EAAE;gBAC1C,EAAE,IAAI,EAAE,sCAAsC,EAAE,KAAK,EAAE,SAAS,EAAE;gBAClE,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,KAAK,EAAE;gBACjD,EAAE,IAAI,EAAE,4CAA4C,EAAE,KAAK,EAAE,OAAO,EAAE;gBACtE,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,SAAS,EAAE;aACxD;SACF,CAAC,CAAC;QAEH,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAA,qBAAW,EAAC,SAAS,IAAI,GAAG,CAAC,CAAC;YACpC,OAAO;gBACL,UAAU,EAAE,CAAC;gBACb,gBAAgB,EAAE,EAAE;gBACpB,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,gBAAgB;aAC3B,CAAC;QACJ,CAAC;QAED,QAAQ,GAAG,UAAU,CAAC;QACtB,IAAI,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC3C,QAAQ,GAAG,MAAM,IAAA,iBAAO,EAAC;gBACvB,OAAO,EAAE,0DAA0D;gBACnE,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,UAAU,EAAE;oBAC3C,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;oBACnC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY,EAAE;iBAC5C;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAmD;YAC9D,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,oBAAoB,EAAE,eAAe,CAAC,EAAE;YACxE;gBACE,IAAI,EAAE,4CAA4C;gBAClD,KAAK,EACH,MAAM,KAAK,OAAO;oBAChB,CAAC,CAAC;wBACE;4BACE,EAAE,EAAE,eAAe;4BACnB,MAAM,EAAE;gCACN,KAAK,EAAE;oCACL;wCACE,IAAI,EAAE,UAAU;wCAChB,QAAQ,EAAE;4CACR,IAAI,EAAE,qBAAqB;4CAC3B,WAAW,EAAE,6CAA6C;4CAC1D,UAAU,EAAE;gDACV,IAAI,EAAE,QAAQ;gDACd,UAAU,EAAE;oDACV,QAAQ,EAAE;wDACR,IAAI,EAAE,QAAQ;wDACd,WAAW,EAAE,4CAA4C;qDAC1D;iDACF;gDACD,QAAQ,EAAE,CAAC,UAAU,CAAC;6CACvB;yCACF;qCACF;iCACF;6BACF;yBACF;qBACF;oBACH,CAAC,CAAC,CAAC,oBAAoB,EAAE,eAAe,CAAC;aAC9C;YACD;gBACE,IAAI,EAAE,6CAA6C;gBACnD,KAAK,EAAE;oBACL,+CAA+C;oBAC/C,2CAA2C;iBAC5C;aACF;YACD;gBACE,IAAI,EAAE,sCAAsC;gBAC5C,KAAK,EAAE;oBACL,iEAAiE;oBACjE,8DAA8D;oBAC9D,gDAAgD;iBACjD;aACF;YACD;gBACE,IAAI,EAAE,qBAAqB;gBAC3B,KAAK,EAAE,CAAC,oBAAoB,CAAC;aAC9B;YACD;gBACE,IAAI,EAAE,yBAAyB;gBAC/B,KAAK,EAAE,CAAC,oBAAoB,CAAC;aAC9B;YACD;gBACE,IAAI,EAAE,kBAAkB;gBACxB,KAAK,EAAE,CAAC,kBAAkB,CAAC;aAC5B;YACD;gBACE,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,CAAC,kCAAkC,CAAC;aAC5C;YACD;gBACE,IAAI,EAAE,yCAAyC;gBAC/C,KAAK,EAAE;oBACL,gDAAgD;oBAChD,+CAA+C;iBAChD;aACF;YACD;gBACE,IAAI,EAAE,qCAAqC;gBAC3C,KAAK,EAAE,CAAC,kBAAkB,EAAE,uBAAuB,CAAC;aACrD;YACD,EAAE,IAAI,EAAE,wCAAwC,EAAE,KAAK,EAAE,CAAC,mBAAmB,CAAC,EAAE;YAChF;gBACE,IAAI,EAAE,gCAAgC;gBACtC,KAAK,EAAE,CAAC,oBAAoB,EAAE,2BAA2B,CAAC;aAC3D;SACF,CAAC;QACF,MAAM,eAAe,GAAwB,MAAM,IAAA,kBAAQ,EAAC;YAC1D,OAAO,EAAE,oEAAoE;YAC7E,OAAO;SACR,CAAC,CAAC;QACH,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC;YAC7C,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,SAAS,CAAC,IAAI,CACZ,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CACjF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;YACnC,CAAC;YAED,IACE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAC1F,CAAC;gBACD,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,2BAAmB,CAAC,CAAC;gBAC1F,gBAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACrC,CAAC;YACD,IACE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,EACxF,CAAC;gBACD,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,qBAAa,CAAC,CAAC;gBACpF,gBAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACrC,CAAC;YACD,IACE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAC1F,CAAC;gBACD,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,uBAAe,CAAC,CAAC;gBACtF,gBAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACrC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAC9C,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CACV,6FAA6F,CAC9F,CAAC;QACJ,CAAC;aAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YAC3C,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;gBAC9B,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,sBAAc,CAAC,CAAC;gBACpF,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,kBAAU,CAAC,CAAC;gBAChF,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,SAAS,CAAC;QACnB,QAAQ,GAAG,UAAU,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QAC7D,SAAS,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACrC,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,QAAQ,GAAG,IAAA,6BAAiB,GAAE,CAAC;IACrC,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,uBAAe,EAAE;QACpD,OAAO;QACP,SAAS;QACT,IAAI,EAAE,MAAM;QACZ,QAAQ;KACT,CAAC,CAAC;IACH,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,sBAAsB,CAAC,EAAE,MAAM,CAAC,CAAC;IACtF,YAAE,CAAC,aAAa,CAAC,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,sBAAc,CAAC,CAAC;IAEnF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IACxD,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,gBAAgB,CAAC;IAC1E,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACtB,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,KAAK,CACT,uCAAuC,eAAK,CAAC,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAClF,CACF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CAAC,qCAAqC,SAAS,EAAE,CAAC,CAAC;QAC9D,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,KAAK,CACT,SAAS,eAAK,CAAC,IAAI,CAAC,MAAM,SAAS,EAAE,CAAC,iBAAiB,eAAK,CAAC,IAAI,CAC/D,UAAU,CACX,oBAAoB,CACtB,CACF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,gBAAgB,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC7F,MAAM;QACN,QAAQ;KACT,CAAC;AACJ,CAAC"}
|
|
@@ -35,7 +35,7 @@ exports.DEFAULT_GRADING_PROMPT = JSON.stringify([
|
|
|
35
35
|
|
|
36
36
|
Output: Hello world
|
|
37
37
|
Rubric: Content contains a greeting
|
|
38
|
-
{"reason": "the content contains the word '
|
|
38
|
+
{"reason": "the content contains the word 'Hello'", "pass": true, "score": 1.0}
|
|
39
39
|
|
|
40
40
|
Output: Avast ye swabs, repel the invaders!
|
|
41
41
|
Rubric: Does not speak like a pirate
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ApiProvider, Prompt, PromptFunctionContext } from '../../types';
|
|
1
|
+
import type { ApiProvider, Prompt, PromptFunctionContext } from '../../types';
|
|
2
2
|
export declare const transformContext: (context: {
|
|
3
3
|
vars: Record<string, string | object>;
|
|
4
4
|
provider?: ApiProvider;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"javascript.d.ts","sourceRoot":"","sources":["../../../../src/prompts/processors/javascript.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"javascript.d.ts","sourceRoot":"","sources":["../../../../src/prompts/processors/javascript.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAE9E,eAAO,MAAM,gBAAgB,YAAa;IACxC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,WAAW,CAAC;CACxB,KAAG,qBAMH,CAAC;AAEF;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EACvB,YAAY,EAAE,MAAM,GAAG,SAAS,GAC/B,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB"}
|