promptfoo 0.69.0 → 0.69.1

package/dist/src/types.d.ts

@@ -1,103 +1,490 @@
-import type { fetchWithCache, getCache } from './cache';
-import type logger from './logger';
+import { z } from 'zod';
 export type FilePath = string;
-export interface CommandLineOptions {
-    prompts?: FilePath[];
-    providers: FilePath[];
-    output: FilePath[];
+export declare const CommandLineOptionsSchema: z.ZodObject<{
+    prompts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    providers: z.ZodArray<z.ZodString, "many">;
+    output: z.ZodArray<z.ZodString, "many">;
+    maxConcurrency: z.ZodString;
+    repeat: z.ZodString;
+    delay: z.ZodString;
+    vars: z.ZodOptional<z.ZodString>;
+    tests: z.ZodOptional<z.ZodString>;
+    config: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    assertions: z.ZodOptional<z.ZodString>;
+    modelOutputs: z.ZodOptional<z.ZodString>;
+    verbose: z.ZodOptional<z.ZodBoolean>;
+    grader: z.ZodOptional<z.ZodString>;
+    tableCellMaxLength: z.ZodOptional<z.ZodString>;
+    write: z.ZodOptional<z.ZodBoolean>;
+    cache: z.ZodOptional<z.ZodBoolean>;
+    table: z.ZodOptional<z.ZodBoolean>;
+    share: z.ZodOptional<z.ZodBoolean>;
+    progressBar: z.ZodOptional<z.ZodBoolean>;
+    watch: z.ZodOptional<z.ZodBoolean>;
+    interactiveProviders: z.ZodOptional<z.ZodBoolean>;
+    filterFailing: z.ZodOptional<z.ZodString>;
+    filterFirstN: z.ZodOptional<z.ZodString>;
+    filterPattern: z.ZodOptional<z.ZodString>;
+    filterProviders: z.ZodOptional<z.ZodString>;
+    var: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    generateSuggestions: z.ZodOptional<z.ZodBoolean>;
+    promptPrefix: z.ZodOptional<z.ZodString>;
+    promptSuffix: z.ZodOptional<z.ZodString>;
+    envFile: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    providers: string[];
+    output: string[];
     maxConcurrency: string;
     repeat: string;
     delay: string;
-    vars?: FilePath;
-    tests?: FilePath;
-    config?: FilePath[];
-    assertions?: FilePath;
-    modelOutputs?: FilePath;
-    verbose?: boolean;
-    grader?: string;
-    tableCellMaxLength?: string;
-    write?: boolean;
-    cache?: boolean;
-    table?: boolean;
-    share?: boolean;
-    progressBar?: boolean;
-    watch?: boolean;
-    interactiveProviders?: boolean;
-    filterFailing?: string;
-    filterFirstN?: string;
-    filterPattern?: string;
-    filterProviders?: string;
-    var?: Record<string, string>;
-    generateSuggestions?: boolean;
-    promptPrefix?: string;
-    promptSuffix?: string;
-    envFile?: FilePath;
-}
-export interface EnvOverrides {
-    ANTHROPIC_API_KEY?: string;
-    BAM_API_KEY?: string;
-    BAM_API_HOST?: string;
-    AZURE_OPENAI_API_HOST?: string;
-    AZURE_OPENAI_API_KEY?: string;
-    AZURE_OPENAI_API_BASE_URL?: string;
-    AZURE_OPENAI_BASE_URL?: string;
-    AWS_BEDROCK_REGION?: string;
-    COHERE_API_KEY?: string;
-    OPENAI_API_KEY?: string;
-    OPENAI_API_HOST?: string;
-    OPENAI_API_BASE_URL?: string;
-    OPENAI_BASE_URL?: string;
-    OPENAI_ORGANIZATION?: string;
-    REPLICATE_API_KEY?: string;
-    REPLICATE_API_TOKEN?: string;
-    LOCALAI_BASE_URL?: string;
-    MISTRAL_API_HOST?: string;
-    MISTRAL_API_BASE_URL?: string;
-    PALM_API_KEY?: string;
-    PALM_API_HOST?: string;
-    GOOGLE_API_KEY?: string;
-    GOOGLE_API_HOST?: string;
-    VERTEX_API_KEY?: string;
-    VERTEX_API_HOST?: string;
-    VERTEX_PROJECT_ID?: string;
-    VERTEX_REGION?: string;
-    VERTEX_PUBLISHER?: string;
-    MISTRAL_API_KEY?: string;
-    CLOUDFLARE_API_KEY?: string;
-    CLOUDFLARE_ACCOUNT_ID?: string;
-}
-export interface ProviderOptions {
-    id?: ProviderId;
-    label?: ProviderLabel;
+    prompts?: string[] | undefined;
+    vars?: string | undefined;
+    tests?: string | undefined;
+    config?: string[] | undefined;
+    assertions?: string | undefined;
+    modelOutputs?: string | undefined;
+    verbose?: boolean | undefined;
+    grader?: string | undefined;
+    tableCellMaxLength?: string | undefined;
+    write?: boolean | undefined;
+    cache?: boolean | undefined;
+    table?: boolean | undefined;
+    share?: boolean | undefined;
+    progressBar?: boolean | undefined;
+    watch?: boolean | undefined;
+    interactiveProviders?: boolean | undefined;
+    filterFailing?: string | undefined;
+    filterFirstN?: string | undefined;
+    filterPattern?: string | undefined;
+    filterProviders?: string | undefined;
+    var?: Record<string, string> | undefined;
+    generateSuggestions?: boolean | undefined;
+    promptPrefix?: string | undefined;
+    promptSuffix?: string | undefined;
+    envFile?: string | undefined;
+}, {
+    providers: string[];
+    output: string[];
+    maxConcurrency: string;
+    repeat: string;
+    delay: string;
+    prompts?: string[] | undefined;
+    vars?: string | undefined;
+    tests?: string | undefined;
+    config?: string[] | undefined;
+    assertions?: string | undefined;
+    modelOutputs?: string | undefined;
+    verbose?: boolean | undefined;
+    grader?: string | undefined;
+    tableCellMaxLength?: string | undefined;
+    write?: boolean | undefined;
+    cache?: boolean | undefined;
+    table?: boolean | undefined;
+    share?: boolean | undefined;
+    progressBar?: boolean | undefined;
+    watch?: boolean | undefined;
+    interactiveProviders?: boolean | undefined;
+    filterFailing?: string | undefined;
+    filterFirstN?: string | undefined;
+    filterPattern?: string | undefined;
+    filterProviders?: string | undefined;
+    var?: Record<string, string> | undefined;
+    generateSuggestions?: boolean | undefined;
+    promptPrefix?: string | undefined;
+    promptSuffix?: string | undefined;
+    envFile?: string | undefined;
+}>;
+export type CommandLineOptions = z.infer<typeof CommandLineOptionsSchema>;
+export declare const EnvOverridesSchema: z.ZodObject<{
+    ANTHROPIC_API_KEY: z.ZodOptional<z.ZodString>;
+    BAM_API_KEY: z.ZodOptional<z.ZodString>;
+    BAM_API_HOST: z.ZodOptional<z.ZodString>;
+    AZURE_OPENAI_API_HOST: z.ZodOptional<z.ZodString>;
+    AZURE_OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
+    AZURE_OPENAI_API_BASE_URL: z.ZodOptional<z.ZodString>;
+    AZURE_OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
+    AWS_BEDROCK_REGION: z.ZodOptional<z.ZodString>;
+    COHERE_API_KEY: z.ZodOptional<z.ZodString>;
+    OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
+    OPENAI_API_HOST: z.ZodOptional<z.ZodString>;
+    OPENAI_API_BASE_URL: z.ZodOptional<z.ZodString>;
+    OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
+    OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
+    REPLICATE_API_KEY: z.ZodOptional<z.ZodString>;
+    REPLICATE_API_TOKEN: z.ZodOptional<z.ZodString>;
+    LOCALAI_BASE_URL: z.ZodOptional<z.ZodString>;
+    MISTRAL_API_HOST: z.ZodOptional<z.ZodString>;
+    MISTRAL_API_BASE_URL: z.ZodOptional<z.ZodString>;
+    PALM_API_KEY: z.ZodOptional<z.ZodString>;
+    PALM_API_HOST: z.ZodOptional<z.ZodString>;
+    GOOGLE_API_KEY: z.ZodOptional<z.ZodString>;
+    GOOGLE_API_HOST: z.ZodOptional<z.ZodString>;
+    VERTEX_API_KEY: z.ZodOptional<z.ZodString>;
+    VERTEX_API_HOST: z.ZodOptional<z.ZodString>;
+    VERTEX_PROJECT_ID: z.ZodOptional<z.ZodString>;
+    VERTEX_REGION: z.ZodOptional<z.ZodString>;
+    VERTEX_PUBLISHER: z.ZodOptional<z.ZodString>;
+    MISTRAL_API_KEY: z.ZodOptional<z.ZodString>;
+    CLOUDFLARE_API_KEY: z.ZodOptional<z.ZodString>;
+    CLOUDFLARE_ACCOUNT_ID: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    ANTHROPIC_API_KEY?: string | undefined;
+    BAM_API_KEY?: string | undefined;
+    BAM_API_HOST?: string | undefined;
+    AZURE_OPENAI_API_HOST?: string | undefined;
+    AZURE_OPENAI_API_KEY?: string | undefined;
+    AZURE_OPENAI_API_BASE_URL?: string | undefined;
+    AZURE_OPENAI_BASE_URL?: string | undefined;
+    AWS_BEDROCK_REGION?: string | undefined;
+    COHERE_API_KEY?: string | undefined;
+    OPENAI_API_KEY?: string | undefined;
+    OPENAI_API_HOST?: string | undefined;
+    OPENAI_API_BASE_URL?: string | undefined;
+    OPENAI_BASE_URL?: string | undefined;
+    OPENAI_ORGANIZATION?: string | undefined;
+    REPLICATE_API_KEY?: string | undefined;
+    REPLICATE_API_TOKEN?: string | undefined;
+    LOCALAI_BASE_URL?: string | undefined;
+    MISTRAL_API_HOST?: string | undefined;
+    MISTRAL_API_BASE_URL?: string | undefined;
+    PALM_API_KEY?: string | undefined;
+    PALM_API_HOST?: string | undefined;
+    GOOGLE_API_KEY?: string | undefined;
+    GOOGLE_API_HOST?: string | undefined;
+    VERTEX_API_KEY?: string | undefined;
+    VERTEX_API_HOST?: string | undefined;
+    VERTEX_PROJECT_ID?: string | undefined;
+    VERTEX_REGION?: string | undefined;
+    VERTEX_PUBLISHER?: string | undefined;
+    MISTRAL_API_KEY?: string | undefined;
+    CLOUDFLARE_API_KEY?: string | undefined;
+    CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+}, {
+    ANTHROPIC_API_KEY?: string | undefined;
+    BAM_API_KEY?: string | undefined;
+    BAM_API_HOST?: string | undefined;
+    AZURE_OPENAI_API_HOST?: string | undefined;
+    AZURE_OPENAI_API_KEY?: string | undefined;
+    AZURE_OPENAI_API_BASE_URL?: string | undefined;
+    AZURE_OPENAI_BASE_URL?: string | undefined;
+    AWS_BEDROCK_REGION?: string | undefined;
+    COHERE_API_KEY?: string | undefined;
+    OPENAI_API_KEY?: string | undefined;
+    OPENAI_API_HOST?: string | undefined;
+    OPENAI_API_BASE_URL?: string | undefined;
+    OPENAI_BASE_URL?: string | undefined;
+    OPENAI_ORGANIZATION?: string | undefined;
+    REPLICATE_API_KEY?: string | undefined;
+    REPLICATE_API_TOKEN?: string | undefined;
+    LOCALAI_BASE_URL?: string | undefined;
+    MISTRAL_API_HOST?: string | undefined;
+    MISTRAL_API_BASE_URL?: string | undefined;
+    PALM_API_KEY?: string | undefined;
+    PALM_API_HOST?: string | undefined;
+    GOOGLE_API_KEY?: string | undefined;
+    GOOGLE_API_HOST?: string | undefined;
+    VERTEX_API_KEY?: string | undefined;
+    VERTEX_API_HOST?: string | undefined;
+    VERTEX_PROJECT_ID?: string | undefined;
+    VERTEX_REGION?: string | undefined;
+    VERTEX_PUBLISHER?: string | undefined;
+    MISTRAL_API_KEY?: string | undefined;
+    CLOUDFLARE_API_KEY?: string | undefined;
+    CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+}>;
+export type EnvOverrides = z.infer<typeof EnvOverridesSchema>;
+export declare const ProviderOptionsSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
+    label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
+    config: z.ZodOptional<z.ZodAny>;
+    prompts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    transform: z.ZodOptional<z.ZodString>;
+    delay: z.ZodOptional<z.ZodNumber>;
+    env: z.ZodOptional<z.ZodObject<{
+        ANTHROPIC_API_KEY: z.ZodOptional<z.ZodString>;
+        BAM_API_KEY: z.ZodOptional<z.ZodString>;
+        BAM_API_HOST: z.ZodOptional<z.ZodString>;
+        AZURE_OPENAI_API_HOST: z.ZodOptional<z.ZodString>;
+        AZURE_OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
+        AZURE_OPENAI_API_BASE_URL: z.ZodOptional<z.ZodString>;
+        AZURE_OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
+        AWS_BEDROCK_REGION: z.ZodOptional<z.ZodString>;
+        COHERE_API_KEY: z.ZodOptional<z.ZodString>;
+        OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
+        OPENAI_API_HOST: z.ZodOptional<z.ZodString>;
+        OPENAI_API_BASE_URL: z.ZodOptional<z.ZodString>;
+        OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
+        OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
+        REPLICATE_API_KEY: z.ZodOptional<z.ZodString>;
+        REPLICATE_API_TOKEN: z.ZodOptional<z.ZodString>;
+        LOCALAI_BASE_URL: z.ZodOptional<z.ZodString>;
+        MISTRAL_API_HOST: z.ZodOptional<z.ZodString>;
+        MISTRAL_API_BASE_URL: z.ZodOptional<z.ZodString>;
+        PALM_API_KEY: z.ZodOptional<z.ZodString>;
+        PALM_API_HOST: z.ZodOptional<z.ZodString>;
+        GOOGLE_API_KEY: z.ZodOptional<z.ZodString>;
+        GOOGLE_API_HOST: z.ZodOptional<z.ZodString>;
+        VERTEX_API_KEY: z.ZodOptional<z.ZodString>;
+        VERTEX_API_HOST: z.ZodOptional<z.ZodString>;
+        VERTEX_PROJECT_ID: z.ZodOptional<z.ZodString>;
+        VERTEX_REGION: z.ZodOptional<z.ZodString>;
+        VERTEX_PUBLISHER: z.ZodOptional<z.ZodString>;
+        MISTRAL_API_KEY: z.ZodOptional<z.ZodString>;
+        CLOUDFLARE_API_KEY: z.ZodOptional<z.ZodString>;
+        CLOUDFLARE_ACCOUNT_ID: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    }, {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    }>>;
+}, "strict", z.ZodTypeAny, {
+    prompts?: string[] | undefined;
+    delay?: number | undefined;
     config?: any;
-    prompts?: string[];
-    transform?: string;
-    delay?: number;
-    env?: EnvOverrides;
-}
-export interface CallApiContextParams {
-    vars: Record<string, string | object>;
-    logger?: typeof logger;
-    fetchWithCache?: typeof fetchWithCache;
-    getCache?: typeof getCache;
-}
-export interface CallApiOptionsParams {
-    includeLogProbs?: boolean;
-    originalProvider?: ApiProvider;
-}
+    id?: string | undefined;
+    label?: string | undefined;
+    transform?: string | undefined;
+    env?: {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    } | undefined;
+}, {
+    prompts?: string[] | undefined;
+    delay?: number | undefined;
+    config?: any;
+    id?: string | undefined;
+    label?: string | undefined;
+    transform?: string | undefined;
+    env?: {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    } | undefined;
+}>;
+export type ProviderOptions = z.infer<typeof ProviderOptionsSchema>;
+export declare const CallApiContextParamsSchema: z.ZodObject<{
+    vars: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodObject<{}, "strip", z.ZodTypeAny, {}, {}>]>>;
+    logger: z.ZodOptional<z.ZodAny>;
+    fetchWithCache: z.ZodOptional<z.ZodAny>;
+    getCache: z.ZodOptional<z.ZodAny>;
+}, "strip", z.ZodTypeAny, {
+    vars: Record<string, string | {}>;
+    logger?: any;
+    fetchWithCache?: any;
+    getCache?: any;
+}, {
+    vars: Record<string, string | {}>;
+    logger?: any;
+    fetchWithCache?: any;
+    getCache?: any;
+}>;
+export type CallApiContextParams = z.infer<typeof CallApiContextParamsSchema>;
+export declare const CallApiOptionsParamsSchema: z.ZodObject<{
+    includeLogProbs: z.ZodOptional<z.ZodBoolean>;
+    originalProvider: z.ZodOptional<z.ZodAny>;
+}, "strip", z.ZodTypeAny, {
+    includeLogProbs?: boolean | undefined;
+    originalProvider?: any;
+}, {
+    includeLogProbs?: boolean | undefined;
+    originalProvider?: any;
+}>;
+export type CallApiOptionsParams = z.infer<typeof CallApiOptionsParamsSchema>;
 type CallApiFunction = {
     (prompt: string, context?: CallApiContextParams, options?: CallApiOptionsParams): Promise<ProviderResponse>;
     label?: string;
 };
-export interface ApiProvider {
-    id: () => string;
+export declare const ApiProviderSchema: z.ZodObject<{
+    id: z.ZodFunction<z.ZodTuple<[], z.ZodUnknown>, z.ZodString>;
+    callApi: z.ZodType<CallApiFunction, z.ZodTypeDef, CallApiFunction>;
+    callEmbeddingApi: z.ZodOptional<z.ZodFunction<z.ZodTuple<[z.ZodString], z.ZodUnknown>, z.ZodPromise<z.ZodType<{
+        error?: string | undefined;
+        tokenUsage?: {
+            cached?: number | undefined;
+            completion?: number | undefined;
+            prompt?: number | undefined;
+            total?: number | undefined;
+        } | undefined;
+        embedding?: number[] | undefined;
+    }, z.ZodTypeDef, {
+        error?: string | undefined;
+        tokenUsage?: {
+            cached?: number | undefined;
+            completion?: number | undefined;
+            prompt?: number | undefined;
+            total?: number | undefined;
+        } | undefined;
+        embedding?: number[] | undefined;
+    }>>>>;
+    callClassificationApi: z.ZodOptional<z.ZodFunction<z.ZodTuple<[z.ZodString], z.ZodUnknown>, z.ZodPromise<z.ZodType<{
+        error?: string | undefined;
+        classification?: Record<string, number> | undefined;
+    }, z.ZodTypeDef, {
+        error?: string | undefined;
+        classification?: Record<string, number> | undefined;
+    }>>>>;
+    label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
+    transform: z.ZodOptional<z.ZodString>;
+    delay: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    id: (...args: unknown[]) => string;
     callApi: CallApiFunction;
-    callEmbeddingApi?: (prompt: string) => Promise<ProviderEmbeddingResponse>;
-    callClassificationApi?: (prompt: string) => Promise<ProviderClassificationResponse>;
-    label?: ProviderLabel;
-    transform?: string;
-    delay?: number;
-}
+    delay?: number | undefined;
+    label?: string | undefined;
+    transform?: string | undefined;
+    callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+        error?: string | undefined;
+        tokenUsage?: {
+            cached?: number | undefined;
+            completion?: number | undefined;
+            prompt?: number | undefined;
+            total?: number | undefined;
+        } | undefined;
+        embedding?: number[] | undefined;
+    }>) | undefined;
+    callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+        error?: string | undefined;
+        classification?: Record<string, number> | undefined;
+    }>) | undefined;
+}, {
+    id: (...args: unknown[]) => string;
+    callApi: CallApiFunction;
+    delay?: number | undefined;
+    label?: string | undefined;
+    transform?: string | undefined;
+    callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+        error?: string | undefined;
+        tokenUsage?: {
+            cached?: number | undefined;
+            completion?: number | undefined;
+            prompt?: number | undefined;
+            total?: number | undefined;
+        } | undefined;
+        embedding?: number[] | undefined;
+    }>) | undefined;
+    callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+        error?: string | undefined;
+        classification?: Record<string, number> | undefined;
+    }>) | undefined;
+}>;
+export type ApiProvider = z.infer<typeof ApiProviderSchema>;
 export declare function isApiProvider(provider: any): provider is ApiProvider;
 export declare function isProviderOptions(provider: any): provider is ProviderOptions;
 export interface ApiEmbeddingProvider extends ApiProvider {
@@ -112,35 +499,163 @@ export interface ApiClassificationProvider extends ApiProvider {
 export interface ApiModerationProvider extends ApiProvider {
     callModerationApi: (prompt: string, response: string) => Promise<ProviderModerationResponse>;
 }
-export interface TokenUsage {
-    total: number;
-    prompt: number;
-    completion: number;
-    cached?: number;
-}
-export interface ProviderResponse {
-    error?: string;
-    output?: string | object;
-    tokenUsage?: Partial<TokenUsage>;
-    cost?: number;
-    cached?: boolean;
-    logProbs?: number[];
-    metadata?: Record<string, any>;
-}
-export interface ProviderEmbeddingResponse {
-    error?: string;
-    embedding?: number[];
-    tokenUsage?: Partial<TokenUsage>;
-}
-export interface ProviderSimilarityResponse {
-    error?: string;
-    similarity?: number;
-    tokenUsage?: Partial<TokenUsage>;
-}
-export interface ProviderClassificationResponse {
-    error?: string;
-    classification?: Record<string, number>;
-}
+declare const TokenUsageSchema: z.ZodObject<{
+    cached: z.ZodOptional<z.ZodNumber>;
+    completion: z.ZodOptional<z.ZodNumber>;
+    prompt: z.ZodOptional<z.ZodNumber>;
+    total: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    cached?: number | undefined;
+    completion?: number | undefined;
+    prompt?: number | undefined;
+    total?: number | undefined;
+}, {
+    cached?: number | undefined;
+    completion?: number | undefined;
+    prompt?: number | undefined;
+    total?: number | undefined;
+}>;
+export type TokenUsage = z.infer<typeof TokenUsageSchema>;
+declare const ProviderResponseSchema: z.ZodObject<{
+    cached: z.ZodOptional<z.ZodBoolean>;
+    cost: z.ZodOptional<z.ZodNumber>;
+    error: z.ZodOptional<z.ZodString>;
+    logProbs: z.ZodOptional<z.ZodArray<z.ZodNumber, "many">>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    output: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodAny]>>;
+    tokenUsage: z.ZodOptional<z.ZodObject<{
+        cached: z.ZodOptional<z.ZodNumber>;
+        completion: z.ZodOptional<z.ZodNumber>;
+        prompt: z.ZodOptional<z.ZodNumber>;
+        total: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    }, {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    output?: any;
+    cached?: boolean | undefined;
+    cost?: number | undefined;
+    error?: string | undefined;
+    logProbs?: number[] | undefined;
+    metadata?: Record<string, any> | undefined;
+    tokenUsage?: {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    } | undefined;
+}, {
+    output?: any;
+    cached?: boolean | undefined;
+    cost?: number | undefined;
+    error?: string | undefined;
+    logProbs?: number[] | undefined;
+    metadata?: Record<string, any> | undefined;
+    tokenUsage?: {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    } | undefined;
+}>;
+export type ProviderResponse = z.infer<typeof ProviderResponseSchema>;
+declare const ProviderEmbeddingResponseSchema: z.ZodObject<{
+    error: z.ZodOptional<z.ZodString>;
+    embedding: z.ZodOptional<z.ZodArray<z.ZodNumber, "many">>;
+    tokenUsage: z.ZodOptional<z.ZodObject<{
+        cached: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        completion: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        prompt: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        total: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+    }, "strip", z.ZodTypeAny, {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    }, {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    error?: string | undefined;
+    tokenUsage?: {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    } | undefined;
+    embedding?: number[] | undefined;
+}, {
+    error?: string | undefined;
+    tokenUsage?: {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    } | undefined;
+    embedding?: number[] | undefined;
+}>;
+export type ProviderEmbeddingResponse = z.infer<typeof ProviderEmbeddingResponseSchema>;
+declare const ProviderSimilarityResponseSchema: z.ZodObject<{
+    error: z.ZodOptional<z.ZodString>;
+    similarity: z.ZodOptional<z.ZodNumber>;
+    tokenUsage: z.ZodOptional<z.ZodObject<{
+        cached: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        completion: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        prompt: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+        total: z.ZodOptional<z.ZodOptional<z.ZodNumber>>;
+    }, "strip", z.ZodTypeAny, {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    }, {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    error?: string | undefined;
+    tokenUsage?: {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    } | undefined;
+    similarity?: number | undefined;
+}, {
+    error?: string | undefined;
+    tokenUsage?: {
+        cached?: number | undefined;
+        completion?: number | undefined;
+        prompt?: number | undefined;
+        total?: number | undefined;
+    } | undefined;
+    similarity?: number | undefined;
+}>;
+export type ProviderSimilarityResponse = z.infer<typeof ProviderSimilarityResponseSchema>;
+declare const ProviderClassificationResponseSchema: z.ZodObject<{
+    error: z.ZodOptional<z.ZodString>;
+    classification: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
+}, "strip", z.ZodTypeAny, {
+    error?: string | undefined;
+    classification?: Record<string, number> | undefined;
+}, {
+    error?: string | undefined;
+    classification?: Record<string, number> | undefined;
+}>;
+export type ProviderClassificationResponse = z.infer<typeof ProviderClassificationResponseSchema>;
 export interface ModerationFlag {
     code: string;
     description: string;
@@ -156,17 +671,50 @@ export interface CsvRow {
 export type VarMapping = Record<string, string>;
 export type ProviderType = 'embedding' | 'classification' | 'text' | 'moderation';
 export type ProviderTypeMap = Partial<Record<ProviderType, string | ProviderOptions | ApiProvider>>;
-export interface GradingConfig {
-    rubricPrompt?: string | string[];
-    provider?: string | ProviderOptions | ApiProvider | ProviderTypeMap;
+declare const GradingConfigSchema: z.ZodObject<{
+    rubricPrompt: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
+    provider: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodAny, z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodString, z.ZodAny]>>>]>>;
+    factuality: z.ZodOptional<z.ZodObject<{
+        subset: z.ZodOptional<z.ZodNumber>;
+        superset: z.ZodOptional<z.ZodNumber>;
+        agree: z.ZodOptional<z.ZodNumber>;
+        disagree: z.ZodOptional<z.ZodNumber>;
+        differButFactual: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        subset?: number | undefined;
+        superset?: number | undefined;
+        agree?: number | undefined;
+        disagree?: number | undefined;
+        differButFactual?: number | undefined;
+    }, {
+        subset?: number | undefined;
+        superset?: number | undefined;
+        agree?: number | undefined;
+        disagree?: number | undefined;
+        differButFactual?: number | undefined;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    rubricPrompt?: string | string[] | undefined;
+    provider?: any;
     factuality?: {
-        subset?: number;
-        superset?: number;
-        agree?: number;
-        disagree?: number;
-        differButFactual?: number;
-    };
-}
+        subset?: number | undefined;
+        superset?: number | undefined;
+        agree?: number | undefined;
+        disagree?: number | undefined;
+        differButFactual?: number | undefined;
+    } | undefined;
+}, {
+    rubricPrompt?: string | string[] | undefined;
+    provider?: any;
+    factuality?: {
+        subset?: number | undefined;
+        superset?: number | undefined;
+        agree?: number | undefined;
+        disagree?: number | undefined;
+        differButFactual?: number | undefined;
+    } | undefined;
+}>;
+export type GradingConfig = z.infer<typeof GradingConfigSchema>;
 export interface PromptConfig {
     prefix?: string;
     suffix?: string;
@@ -201,6 +749,17 @@ export interface EvaluateOptions {
     eventSource?: string;
     interactiveProviders?: boolean;
 }
+export type PromptFunctionContext = {
+    vars: Record<string, string | object>;
+    provider: {
+        id: string;
+        label?: string;
+    };
+};
+export type PromptFunction = (context: {
+    vars: Record<string, string | object>;
+    provider?: ApiProvider;
+}) => Promise<string | object>;
 export interface Prompt {
     id?: string;
     raw: string;
@@ -209,10 +768,7 @@ export interface Prompt {
      */
     display?: string;
     label: string;
-    function?: (context: {
-        vars: Record<string, string | object>;
-        provider?: ApiProvider;
-    }) => Promise<string | object>;
+    function?: PromptFunction;
 }
 export type CompletedPrompt = Prompt & {
     provider: string;
@@ -303,7 +859,8 @@ export interface GradingResult {
     comment?: string;
 }
 export declare function isGradingResult(result: any): result is GradingResult;
-type BaseAssertionTypes = 'human' | 'equals' | 'contains' | 'icontains' | 'contains-all' | 'contains-any' | 'icontains-all' | 'icontains-any' | 'starts-with' | 'regex' | 'is-json' | 'is-sql' | 'contains-json' | 'contains-sql' | 'javascript' | 'python' | 'similar' | 'answer-relevance' | 'context-faithfulness' | 'context-recall' | 'context-relevance' | 'llm-rubric' | 'model-graded-closedqa' | 'factuality' | 'model-graded-factuality' | 'webhook' | 'rouge-n' | 'rouge-s' | 'rouge-l' | 'levenshtein' | 'is-valid-openai-function-call' | 'is-valid-openai-tools-call' | 'latency' | 'perplexity' | 'perplexity-score' | 'cost' | 'select-best' | 'moderation';
+export declare const BaseAssertionTypesSchema: z.ZodEnum<["answer-relevance", "contains-all", "contains-any", "contains-json", "contains-sql", "contains", "context-faithfulness", "context-recall", "context-relevance", "cost", "equals", "factuality", "human", "icontains-all", "icontains-any", "icontains", "is-json", "is-sql", "is-valid-openai-function-call", "is-valid-openai-tools-call", "javascript", "latency", "levenshtein", "llm-rubric", "model-graded-closedqa", "model-graded-factuality", "moderation", "perplexity-score", "perplexity", "python", "regex", "rouge-l", "rouge-n", "rouge-s", "select-best", "similar", "starts-with", "webhook"]>;
+export type BaseAssertionTypes = z.infer<typeof BaseAssertionTypesSchema>;
 type NotPrefixed<T extends string> = `not-${T}`;
 export type AssertionType = BaseAssertionTypes | NotPrefixed<BaseAssertionTypes>;
 export interface AssertionSet {
@@ -344,10 +901,268 @@ export interface TestCasesWithMetadata {
     count: number;
     prompts: TestCasesWithMetadataPrompt[];
 }
+export declare const ProviderSchema: z.ZodUnion<[z.ZodString, z.ZodObject<{
+    id: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
+    label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
+    config: z.ZodOptional<z.ZodAny>;
+    prompts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    transform: z.ZodOptional<z.ZodString>;
+    delay: z.ZodOptional<z.ZodNumber>;
+    env: z.ZodOptional<z.ZodObject<{
+        ANTHROPIC_API_KEY: z.ZodOptional<z.ZodString>;
+        BAM_API_KEY: z.ZodOptional<z.ZodString>;
+        BAM_API_HOST: z.ZodOptional<z.ZodString>;
+        AZURE_OPENAI_API_HOST: z.ZodOptional<z.ZodString>;
+        AZURE_OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
+        AZURE_OPENAI_API_BASE_URL: z.ZodOptional<z.ZodString>;
+        AZURE_OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
+        AWS_BEDROCK_REGION: z.ZodOptional<z.ZodString>;
+        COHERE_API_KEY: z.ZodOptional<z.ZodString>;
+        OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
+        OPENAI_API_HOST: z.ZodOptional<z.ZodString>;
+        OPENAI_API_BASE_URL: z.ZodOptional<z.ZodString>;
+        OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
+        OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
+        REPLICATE_API_KEY: z.ZodOptional<z.ZodString>;
+        REPLICATE_API_TOKEN: z.ZodOptional<z.ZodString>;
+        LOCALAI_BASE_URL: z.ZodOptional<z.ZodString>;
+        MISTRAL_API_HOST: z.ZodOptional<z.ZodString>;
+        MISTRAL_API_BASE_URL: z.ZodOptional<z.ZodString>;
+        PALM_API_KEY: z.ZodOptional<z.ZodString>;
+        PALM_API_HOST: z.ZodOptional<z.ZodString>;
+        GOOGLE_API_KEY: z.ZodOptional<z.ZodString>;
+        GOOGLE_API_HOST: z.ZodOptional<z.ZodString>;
+        VERTEX_API_KEY: z.ZodOptional<z.ZodString>;
+        VERTEX_API_HOST: z.ZodOptional<z.ZodString>;
+        VERTEX_PROJECT_ID: z.ZodOptional<z.ZodString>;
+        VERTEX_REGION: z.ZodOptional<z.ZodString>;
+        VERTEX_PUBLISHER: z.ZodOptional<z.ZodString>;
+        MISTRAL_API_KEY: z.ZodOptional<z.ZodString>;
+        CLOUDFLARE_API_KEY: z.ZodOptional<z.ZodString>;
+        CLOUDFLARE_ACCOUNT_ID: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    }, {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    }>>;
+}, "strict", z.ZodTypeAny, {
+    prompts?: string[] | undefined;
+    delay?: number | undefined;
+    config?: any;
+    id?: string | undefined;
+    label?: string | undefined;
+    transform?: string | undefined;
+    env?: {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    } | undefined;
+}, {
+    prompts?: string[] | undefined;
+    delay?: number | undefined;
+    config?: any;
+    id?: string | undefined;
+    label?: string | undefined;
+    transform?: string | undefined;
+    env?: {
+        ANTHROPIC_API_KEY?: string | undefined;
+        BAM_API_KEY?: string | undefined;
+        BAM_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_HOST?: string | undefined;
+        AZURE_OPENAI_API_KEY?: string | undefined;
+        AZURE_OPENAI_API_BASE_URL?: string | undefined;
+        AZURE_OPENAI_BASE_URL?: string | undefined;
+        AWS_BEDROCK_REGION?: string | undefined;
+        COHERE_API_KEY?: string | undefined;
+        OPENAI_API_KEY?: string | undefined;
+        OPENAI_API_HOST?: string | undefined;
+        OPENAI_API_BASE_URL?: string | undefined;
+        OPENAI_BASE_URL?: string | undefined;
+        OPENAI_ORGANIZATION?: string | undefined;
+        REPLICATE_API_KEY?: string | undefined;
+        REPLICATE_API_TOKEN?: string | undefined;
+        LOCALAI_BASE_URL?: string | undefined;
+        MISTRAL_API_HOST?: string | undefined;
+        MISTRAL_API_BASE_URL?: string | undefined;
+        PALM_API_KEY?: string | undefined;
+        PALM_API_HOST?: string | undefined;
+        GOOGLE_API_KEY?: string | undefined;
+        GOOGLE_API_HOST?: string | undefined;
+        VERTEX_API_KEY?: string | undefined;
+        VERTEX_API_HOST?: string | undefined;
+        VERTEX_PROJECT_ID?: string | undefined;
+        VERTEX_REGION?: string | undefined;
+        VERTEX_PUBLISHER?: string | undefined;
+        MISTRAL_API_KEY?: string | undefined;
+        CLOUDFLARE_API_KEY?: string | undefined;
+        CLOUDFLARE_ACCOUNT_ID?: string | undefined;
+    } | undefined;
+}>, z.ZodObject<{
+    id: z.ZodFunction<z.ZodTuple<[], z.ZodUnknown>, z.ZodString>;
+    callApi: z.ZodType<CallApiFunction, z.ZodTypeDef, CallApiFunction>;
+    callEmbeddingApi: z.ZodOptional<z.ZodFunction<z.ZodTuple<[z.ZodString], z.ZodUnknown>, z.ZodPromise<z.ZodType<{
+        error?: string | undefined;
+        tokenUsage?: {
+            cached?: number | undefined;
+            completion?: number | undefined;
+            prompt?: number | undefined;
+            total?: number | undefined;
+        } | undefined;
+        embedding?: number[] | undefined;
+    }, z.ZodTypeDef, {
+        error?: string | undefined;
+        tokenUsage?: {
+            cached?: number | undefined;
+            completion?: number | undefined;
+            prompt?: number | undefined;
+            total?: number | undefined;
+        } | undefined;
+        embedding?: number[] | undefined;
+    }>>>>;
+    callClassificationApi: z.ZodOptional<z.ZodFunction<z.ZodTuple<[z.ZodString], z.ZodUnknown>, z.ZodPromise<z.ZodType<{
+        error?: string | undefined;
+        classification?: Record<string, number> | undefined;
+    }, z.ZodTypeDef, {
+        error?: string | undefined;
+        classification?: Record<string, number> | undefined;
+    }>>>>;
+    label: z.ZodOptional<z.ZodType<string, z.ZodTypeDef, string>>;
+    transform: z.ZodOptional<z.ZodString>;
+    delay: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    id: (...args: unknown[]) => string;
+    callApi: CallApiFunction;
+    delay?: number | undefined;
+    label?: string | undefined;
+    transform?: string | undefined;
+    callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+        error?: string | undefined;
+        tokenUsage?: {
+            cached?: number | undefined;
+            completion?: number | undefined;
+            prompt?: number | undefined;
+            total?: number | undefined;
+        } | undefined;
+        embedding?: number[] | undefined;
+    }>) | undefined;
+    callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+        error?: string | undefined;
+        classification?: Record<string, number> | undefined;
+    }>) | undefined;
+}, {
+    id: (...args: unknown[]) => string;
+    callApi: CallApiFunction;
+    delay?: number | undefined;
+    label?: string | undefined;
+    transform?: string | undefined;
+    callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+        error?: string | undefined;
+        tokenUsage?: {
+            cached?: number | undefined;
+            completion?: number | undefined;
+            prompt?: number | undefined;
+            total?: number | undefined;
+        } | undefined;
+        embedding?: number[] | undefined;
+    }>) | undefined;
+    callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
+        error?: string | undefined;
+        classification?: Record<string, number> | undefined;
+    }>) | undefined;
+}>]>;
+export type Provider = z.infer<typeof ProviderSchema>;
 export interface TestCase<Vars = Record<string, string | string[] | object>> {
     description?: string;
     vars?: Vars;
-    provider?: string | ProviderOptions | ApiProvider;
+    provider?: Provider;
     providerOutput?: string | object;
     assert?: (AssertionSet | Assertion)[];
     options?: PromptConfig & OutputConfig & GradingConfig & {
@@ -413,9 +1228,6 @@ export interface EvalWithMetadata {
     results: EvaluateSummary;
     description?: string;
 }
-export type PromptFunction = (context: {
-    vars: Record<string, string | object>;
-}) => Promise<string | object>;
 export type EvaluateTestSuite = {
     prompts: (string | object | PromptFunction)[];
     writeLatestResults?: boolean;