promptfoo 0.73.9 → 0.74.0

package/dist/src/types/providers.d.ts

@@ -0,0 +1,126 @@
+import type { Prompt } from './prompts';
+import type { NunjucksFilterMap, TokenUsage } from './shared';
+export type ProviderId = string;
+export type ProviderLabel = string;
+export type ProviderFunction = ApiProvider['callApi'];
+export type ProviderOptionsMap = Record<ProviderId, ProviderOptions>;
+export type ProviderType = 'embedding' | 'classification' | 'text' | 'moderation';
+export type ProviderTypeMap = Partial<Record<ProviderType, string | ProviderOptions | ApiProvider>>;
+export interface ProviderModerationResponse {
+    error?: string;
+    flags?: ModerationFlag[];
+}
+export interface ModerationFlag {
+    code: string;
+    description: string;
+    confidence: number;
+}
+export interface ProviderOptions {
+    id?: ProviderId;
+    label?: ProviderLabel;
+    config?: any;
+    prompts?: string[];
+    transform?: string;
+    delay?: number;
+    env?: EnvOverrides;
+}
+export interface CallApiContextParams {
+    fetchWithCache?: any;
+    filters?: NunjucksFilterMap;
+    getCache?: any;
+    logger?: any;
+    originalProvider?: any;
+    prompt: Prompt;
+    vars: Record<string, string | object>;
+}
+export interface CallApiOptionsParams {
+    includeLogProbs?: boolean;
+}
+export interface ApiProvider {
+    id: () => string;
+    callApi: CallApiFunction;
+    callEmbeddingApi?: (input: string) => Promise<ProviderEmbeddingResponse>;
+    callClassificationApi?: (prompt: string) => Promise<ProviderClassificationResponse>;
+    label?: ProviderLabel;
+    transform?: string;
+    delay?: number;
+}
+export interface ApiEmbeddingProvider extends ApiProvider {
+    callEmbeddingApi: (input: string) => Promise<ProviderEmbeddingResponse>;
+}
+export interface ApiSimilarityProvider extends ApiProvider {
+    callSimilarityApi: (reference: string, input: string) => Promise<ProviderSimilarityResponse>;
+}
+export interface ApiClassificationProvider extends ApiProvider {
+    callClassificationApi: (prompt: string) => Promise<ProviderClassificationResponse>;
+}
+export interface ApiModerationProvider extends ApiProvider {
+    callModerationApi: (prompt: string, response: string) => Promise<ProviderModerationResponse>;
+}
+export interface ProviderResponse {
+    cached?: boolean;
+    cost?: number;
+    error?: string;
+    logProbs?: number[];
+    metadata?: {
+        redteamFinalPrompt?: string;
+        [key: string]: any;
+    };
+    output?: string | any;
+    tokenUsage?: TokenUsage;
+}
+export interface ProviderEmbeddingResponse {
+    error?: string;
+    embedding?: number[];
+    tokenUsage?: Partial<TokenUsage>;
+}
+export interface ProviderSimilarityResponse {
+    error?: string;
+    similarity?: number;
+    tokenUsage?: Partial<TokenUsage>;
+}
+export interface ProviderClassificationResponse {
+    error?: string;
+    classification?: Record<string, number>;
+}
+export type EnvOverrides = {
+    ANTHROPIC_API_KEY?: string;
+    BAM_API_KEY?: string;
+    BAM_API_HOST?: string;
+    AZURE_OPENAI_API_HOST?: string;
+    AZURE_OPENAI_API_KEY?: string;
+    AZURE_OPENAI_API_BASE_URL?: string;
+    AZURE_OPENAI_BASE_URL?: string;
+    AWS_BEDROCK_REGION?: string;
+    COHERE_API_KEY?: string;
+    OPENAI_API_KEY?: string;
+    OPENAI_API_HOST?: string;
+    OPENAI_API_BASE_URL?: string;
+    OPENAI_BASE_URL?: string;
+    OPENAI_ORGANIZATION?: string;
+    REPLICATE_API_KEY?: string;
+    REPLICATE_API_TOKEN?: string;
+    LOCALAI_BASE_URL?: string;
+    MISTRAL_API_HOST?: string;
+    MISTRAL_API_BASE_URL?: string;
+    PALM_API_KEY?: string;
+    PALM_API_HOST?: string;
+    GOOGLE_API_KEY?: string;
+    GOOGLE_API_HOST?: string;
+    VERTEX_API_KEY?: string;
+    VERTEX_API_HOST?: string;
+    VERTEX_PROJECT_ID?: string;
+    VERTEX_REGION?: string;
+    VERTEX_PUBLISHER?: string;
+    MISTRAL_API_KEY?: string;
+    CLOUDFLARE_API_KEY?: string;
+    CLOUDFLARE_ACCOUNT_ID?: string;
+};
+export type FilePath = string;
+export type CallApiFunction = {
+    (prompt: string, context?: CallApiContextParams, options?: CallApiOptionsParams): Promise<ProviderResponse>;
+    label?: string;
+};
+export declare function isApiProvider(provider: any): provider is ApiProvider;
+export declare function isProviderOptions(provider: any): provider is ProviderOptions;
+//# sourceMappingURL=providers.d.ts.map

package/dist/src/types/providers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../../../src/types/providers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE9D,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC;AAChC,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC;AACnC,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;AACtD,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;AAErE,MAAM,MAAM,YAAY,GAAG,WAAW,GAAG,gBAAgB,GAAG,MAAM,GAAG,YAAY,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,eAAe,GAAG,WAAW,CAAC,CAAC,CAAC;AAEpG,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,UAAU,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,YAAY,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,cAAc,CAAC,EAAE,GAAG,CAAC;IACrB,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,gBAAgB,CAAC,EAAE,GAAG,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,oBAAoB;IACnC,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,eAAe,CAAC;IACzB,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,yBAAyB,CAAC,CAAC;IACzE,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,8BAA8B,CAAC,CAAC;IACpF,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAqB,SAAQ,WAAW;IACvD,gBAAgB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,yBAAyB,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,qBAAsB,SAAQ,WAAW;IACxD,iBAAiB,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,0BAA0B,CAAC,CAAC;CAC9F;AAED,MAAM,WAAW,yBAA0B,SAAQ,WAAW;IAC5D,qBAAqB,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,8BAA8B,CAAC,CAAC;CACpF;AAED,MAAM,WAAW,qBAAsB,SAAQ,WAAW;IACxD,iBAAiB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,0BAA0B,CAAC,CAAC;CAC9F;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,CAAC,EAAE;QACT,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;KACpB,CAAC;IACF,MAAM,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC;IACtB,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,8BAA8B;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACzC;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAE9B,MAAM,MAAM,eAAe,GAAG;IAC5B,CACE,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,oBAAoB,EAC9B,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,wBAAgB,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,QAAQ,IAAI,WAAW,CAEpE;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,GAAG,GAAG,QAAQ,IAAI,eAAe,CAE5E"}

package/dist/src/types/providers.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isApiProvider = isApiProvider;
+exports.isProviderOptions = isProviderOptions;
+function isApiProvider(provider) {
+    return typeof provider === 'object' && 'id' in provider && typeof provider.id === 'function';
+}
+function isProviderOptions(provider) {
+    return !isApiProvider(provider) && typeof provider === 'object';
+}
+//# sourceMappingURL=providers.js.map

package/dist/src/types/providers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../../src/types/providers.ts"],"names":[],"mappings":";;AAoJA,sCAEC;AAED,8CAEC;AAND,SAAgB,aAAa,CAAC,QAAa;IACzC,OAAO,OAAO,QAAQ,KAAK,QAAQ,IAAI,IAAI,IAAI,QAAQ,IAAI,OAAO,QAAQ,CAAC,EAAE,KAAK,UAAU,CAAC;AAC/F,CAAC;AAED,SAAgB,iBAAiB,CAAC,QAAa;IAC7C,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,OAAO,QAAQ,KAAK,QAAQ,CAAC;AAClE,CAAC"}

package/dist/src/types/redteam.d.ts

@@ -0,0 +1,51 @@
+import type { ApiProvider, ProviderOptions } from '../types/providers';
+export type RedteamPluginObject = {
+    id: string;
+    numTests?: number;
+};
+export type RedteamPlugin = string | RedteamPluginObject;
+export type RedteamStrategyObject = {
+    id: string;
+};
+export type RedteamStrategy = string | RedteamStrategyObject;
+export type RedteamGenerateOptions = {
+    cache: boolean;
+    config?: string;
+    defaultConfig: Record<string, unknown>;
+    defaultConfigPath?: string;
+    envFile?: string;
+    injectVar?: string;
+    numTests: number;
+    output?: string;
+    plugins?: RedteamPluginObject[];
+    addPlugins?: string[];
+    addStrategies?: string[];
+    provider?: string;
+    purpose?: string;
+    strategies?: RedteamStrategy[];
+    write: boolean;
+};
+export type RedteamConfig = {
+    injectVar?: string;
+    purpose?: string;
+    provider?: string | ProviderOptions | ApiProvider;
+    plugins: RedteamPluginObject[];
+    strategies: RedteamStrategyObject[];
+};
+export type RedteamAssertionTypes = `promptfoo:redteam:${string}`;
+export interface SynthesizeOptions {
+    injectVar?: string;
+    numTests: number;
+    plugins: {
+        id: string;
+        numTests: number;
+        config?: Record<string, any>;
+    }[];
+    prompts: string[];
+    provider?: ApiProvider | ProviderOptions | string;
+    purpose?: string;
+    strategies: {
+        id: string;
+    }[];
+}
+//# sourceMappingURL=redteam.d.ts.map

package/dist/src/types/redteam.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/types/redteam.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAEvE,MAAM,MAAM,mBAAmB,GAAG;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,mBAAmB,CAAC;AAEzD,MAAM,MAAM,qBAAqB,GAAG;IAClC,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,qBAAqB,CAAC;AAE7D,MAAM,MAAM,sBAAsB,GAAG;IACnC,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAChC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,eAAe,EAAE,CAAC;IAC/B,KAAK,EAAE,OAAO,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,GAAG,eAAe,GAAG,WAAW,CAAC;IAClD,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,UAAU,EAAE,qBAAqB,EAAE,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG,qBAAqB,MAAM,EAAE,CAAC;AAElE,MAAM,WAAW,iBAAiB;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAAE,EAAE,CAAC;IAC1E,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,WAAW,GAAG,eAAe,GAAG,MAAM,CAAC;IAClD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC9B"}

package/dist/src/types/redteam.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=redteam.js.map

package/dist/src/types/redteam.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/types/redteam.ts"],"names":[],"mappings":""}

package/dist/src/types/shared.d.ts

@@ -0,0 +1,8 @@
+export interface TokenUsage {
+    cached?: number;
+    completion?: number;
+    prompt?: number;
+    total?: number;
+}
+export type NunjucksFilterMap = Record<string, (...args: any[]) => string>;
+//# sourceMappingURL=shared.d.ts.map

package/dist/src/types/shared.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../src/types/shared.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,MAAM,CAAC,CAAC"}

package/dist/src/types/shared.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=shared.js.map

package/dist/src/types/shared.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/types/shared.ts"],"names":[],"mappings":""}

package/dist/src/util/index.d.ts

@@ -55,45 +55,22 @@ export declare function getTestCases(limit?: number): Promise<{
         id: string;
         prompt: {
             provider: string;
-            label: string;
             raw: string;
+            label: string;
             function?: ((args_0: {
                 vars: Record<string, any>;
-                provider?: {
-                    id: (...args: unknown[]) => string;
-                    callApi: {
-                        (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                        label?: string;
-                    };
-                    delay?: number | undefined;
-                    label?: string | undefined;
-                    transform?: string | undefined;
-                    callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                        error?: string | undefined;
-                        embedding?: number[] | undefined;
-                        tokenUsage?: {
-                            cached?: number | undefined;
-                            completion?: number | undefined;
-                            prompt?: number | undefined;
-                            total?: number | undefined;
-                        } | undefined;
-                    }>) | undefined;
-                    callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                        error?: string | undefined;
-                        classification?: Record<string, number> | undefined;
-                    }>) | undefined;
-                } | undefined;
+                provider?: import("../types").ApiProvider | undefined;
             }, ...args_1: unknown[]) => Promise<any>) | undefined;
             id?: string | undefined;
             display?: string | undefined;
             metrics?: {
+                cost: number;
                 tokenUsage: {
                     cached?: number | undefined;
                     completion?: number | undefined;
                     prompt?: number | undefined;
                     total?: number | undefined;
                 };
-                cost: number;
                 score: number;
                 testPassCount: number;
                 testFailCount: number;
@@ -130,13 +107,14 @@ export declare function getTestCases(limit?: number): Promise<{
             disableVarExpansion?: boolean | undefined;
             disableConversationVar?: boolean | undefined;
         }) | undefined;
+        vars?: Record<string, string | any[] | string[] | {}> | undefined;
         provider?: string | {
             id?: string | undefined;
+            label?: string | undefined;
             config?: any;
             prompts?: string[] | undefined;
-            delay?: number | undefined;
-            label?: string | undefined;
             transform?: string | undefined;
+            delay?: number | undefined;
             env?: {
                 ANTHROPIC_API_KEY?: string | undefined;
                 BAM_API_KEY?: string | undefined;
@@ -171,30 +149,14 @@ export declare function getTestCases(limit?: number): Promise<{
                 CLOUDFLARE_ACCOUNT_ID?: string | undefined;
             } | undefined;
         } | {
+            callApi: import("../types").CallApiFunction;
             id: (...args: unknown[]) => string;
-            callApi: {
-                (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                label?: string;
-            };
-            delay?: number | undefined;
             label?: string | undefined;
             transform?: string | undefined;
-            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                error?: string | undefined;
-                embedding?: number[] | undefined;
-                tokenUsage?: {
-                    cached?: number | undefined;
-                    completion?: number | undefined;
-                    prompt?: number | undefined;
-                    total?: number | undefined;
-                } | undefined;
-            }>) | undefined;
-            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                error?: string | undefined;
-                classification?: Record<string, number> | undefined;
-            }>) | undefined;
+            delay?: number | undefined;
+            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
+            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
         } | undefined;
-        vars?: Record<string, string | any[] | string[] | {}> | undefined;
         metadata?: Record<string, any> | undefined;
         description?: string | undefined;
         providerOutput?: string | {} | undefined;
@@ -242,45 +204,22 @@ export declare function getDatasetFromHash(hash: string): Promise<{
         id: string;
         prompt: {
             provider: string;
-            label: string;
             raw: string;
+            label: string;
             function?: ((args_0: {
                 vars: Record<string, any>;
-                provider?: {
-                    id: (...args: unknown[]) => string;
-                    callApi: {
-                        (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                        label?: string;
-                    };
-                    delay?: number | undefined;
-                    label?: string | undefined;
-                    transform?: string | undefined;
-                    callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                        error?: string | undefined;
-                        embedding?: number[] | undefined;
-                        tokenUsage?: {
-                            cached?: number | undefined;
-                            completion?: number | undefined;
-                            prompt?: number | undefined;
-                            total?: number | undefined;
-                        } | undefined;
-                    }>) | undefined;
-                    callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                        error?: string | undefined;
-                        classification?: Record<string, number> | undefined;
-                    }>) | undefined;
-                } | undefined;
+                provider?: import("../types").ApiProvider | undefined;
             }, ...args_1: unknown[]) => Promise<any>) | undefined;
             id?: string | undefined;
             display?: string | undefined;
             metrics?: {
+                cost: number;
                 tokenUsage: {
                     cached?: number | undefined;
                     completion?: number | undefined;
                     prompt?: number | undefined;
                     total?: number | undefined;
                 };
-                cost: number;
                 score: number;
                 testPassCount: number;
                 testFailCount: number;
@@ -317,13 +256,14 @@ export declare function getDatasetFromHash(hash: string): Promise<{
             disableVarExpansion?: boolean | undefined;
             disableConversationVar?: boolean | undefined;
         }) | undefined;
+        vars?: Record<string, string | any[] | string[] | {}> | undefined;
         provider?: string | {
             id?: string | undefined;
+            label?: string | undefined;
             config?: any;
             prompts?: string[] | undefined;
-            delay?: number | undefined;
-            label?: string | undefined;
             transform?: string | undefined;
+            delay?: number | undefined;
             env?: {
                 ANTHROPIC_API_KEY?: string | undefined;
                 BAM_API_KEY?: string | undefined;
@@ -358,30 +298,14 @@ export declare function getDatasetFromHash(hash: string): Promise<{
                 CLOUDFLARE_ACCOUNT_ID?: string | undefined;
             } | undefined;
         } | {
+            callApi: import("../types").CallApiFunction;
             id: (...args: unknown[]) => string;
-            callApi: {
-                (prompt: string, context?: import("../types").CallApiContextParams, options?: import("../types").CallApiOptionsParams): Promise<import("../types").ProviderResponse>;
-                label?: string;
-            };
-            delay?: number | undefined;
             label?: string | undefined;
             transform?: string | undefined;
-            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                error?: string | undefined;
-                embedding?: number[] | undefined;
-                tokenUsage?: {
-                    cached?: number | undefined;
-                    completion?: number | undefined;
-                    prompt?: number | undefined;
-                    total?: number | undefined;
-                } | undefined;
-            }>) | undefined;
-            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<{
-                error?: string | undefined;
-                classification?: Record<string, number> | undefined;
-            }>) | undefined;
+            delay?: number | undefined;
+            callEmbeddingApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderEmbeddingResponse>) | undefined;
+            callClassificationApi?: ((args_0: string, ...args_1: unknown[]) => Promise<import("../types").ProviderClassificationResponse>) | undefined;
         } | undefined;
-        vars?: Record<string, string | any[] | string[] | {}> | undefined;
         metadata?: Record<string, any> | undefined;
         description?: string | undefined;
         providerOutput?: string | {} | undefined;

package/dist/src/util/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/util/index.ts"],"names":[],"mappings":"AAmBA,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,aAAa,EAElB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,qBAAqB,EAE1B,KAAK,aAAa,EAClB,KAAK,UAAU,EACf,KAAK,eAAe,EAEpB,KAAK,iBAAiB,EAIvB,MAAM,UAAU,CAAC;AAMlB,wBAAsB,WAAW,CAC/B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,iBAqF5B;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,MAAM,EAAE,EACrB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,iBAK5B;AAED,wBAAgB,MAAM,CAAC,GAAG,EAAE,MAAM,UAEjC;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CASxE;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,SAAS,CAAC,EAAE,IAAI,GACf,OAAO,CAAC,MAAM,CAAC,CA4FjB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,GAAE,MAA4B,EACnC,iBAAiB,CAAC,EAAE,MAAM,EAC1B,SAAS,CAAC,EAAE,MAAM,GACjB,iBAAiB,EAAE,CA4BrB;AAED;;GAEG;AACH,wBAAgB,sCAAsC,IAAI,MAAM,EAAE,CAYjE;AAID;;GAEG;AACH,wBAAgB,8BAA8B,IAAI;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,CAqB7F;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,QAqB9C;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,IAAI,UAExC;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,MAAM,GACX;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAgBlE;AAID,wBAAsB,sCAAsC,kBA0D3D;AAKD,wBAAgB,qBAAqB,CAAC,SAAS,SAAwB,QAKtE;AAED,wBAAsB,UAAU,CAC9B,EAAE,EAAE,MAAM,GACT,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAAC,CAsC3E;AAED,wBAAsB,YAAY,CAChC,EAAE,EAAE,MAAM,EACV,SAAS,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,EAClC,QAAQ,CAAC,EAAE,aAAa,GACvB,OAAO,CAAC,IAAI,CAAC,CAyCf;AAED,wBAAsB,iBAAiB,CACrC,iBAAiB,CAAC,EAAE,MAAM,GACzB,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAgClC;AAED,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAkE/B;AAED,wBAAgB,0BAA0B,CACxC,eAAe,EAAE,MAAM,EACvB,KAAK,GAAE,MAA4B,iCAOpC;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,QAAQ,EAAE,iCAI3D;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAuElC;AAED,wBAAgB,UAAU,CAAC,KAAK,GAAE,MAA4B,iCAE7D;AAED,wBAAsB,YAAY,CAAC,KAAK,GAAE,MAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAErE;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,MAAM,2CAQnD;AAED,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAQpD;AAED,wBAAsB,qBAAqB,CACzC,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAwC7B;AAED,wBAAsB,QAAQ,CAAC,KAAK,GAAE,MAA4B,+BAEjE;AAED,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,yCAQ/C;AAED,wBAAsB,UAAU,CAAC,MAAM,EAAE,MAAM,iBAa9C;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC/B,QAAQ,GAAE,MAAW,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAa5B;AAED,wBAAgB,WAAW,SAG1B;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,QAOnD;AAED,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAEF,wBAAgB,kBAAkB,CAAC,KAAK,GAAE,MAA4B,GAAG,cAAc,EAAE,CA+BxF;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,SAAS,CAQvF;AAED,wBAAgB,SAAS,CACvB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,EAC7D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,WAG9D;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAMvF;AAED,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAsBvF;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CA4BxD;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,GACjB;IACD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAuCA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/util/index.ts"],"names":[],"mappings":"AAmBA,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,aAAa,EAElB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,qBAAqB,EAE1B,KAAK,aAAa,EAClB,KAAK,UAAU,EACf,KAAK,eAAe,EAEpB,KAAK,iBAAiB,EAIvB,MAAM,UAAU,CAAC;AAMlB,wBAAsB,WAAW,CAC/B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,iBAqF5B;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,MAAM,EAAE,EACrB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,iBAK5B;AAED,wBAAgB,MAAM,CAAC,GAAG,EAAE,MAAM,UAEjC;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CASxE;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,SAAS,CAAC,EAAE,IAAI,GACf,OAAO,CAAC,MAAM,CAAC,CA4FjB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,GAAE,MAA4B,EACnC,iBAAiB,CAAC,EAAE,MAAM,EAC1B,SAAS,CAAC,EAAE,MAAM,GACjB,iBAAiB,EAAE,CAgCrB;AAED;;GAEG;AACH,wBAAgB,sCAAsC,IAAI,MAAM,EAAE,CAYjE;AAID;;GAEG;AACH,wBAAgB,8BAA8B,IAAI;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,CAqB7F;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,QAqB9C;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,IAAI,UAExC;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,MAAM,GACX;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAgBlE;AAID,wBAAsB,sCAAsC,kBA0D3D;AAKD,wBAAgB,qBAAqB,CAAC,SAAS,SAAwB,QAKtE;AAED,wBAAsB,UAAU,CAC9B,EAAE,EAAE,MAAM,GACT,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAAC,CAsC3E;AAED,wBAAsB,YAAY,CAChC,EAAE,EAAE,MAAM,EACV,SAAS,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,EAClC,QAAQ,CAAC,EAAE,aAAa,GACvB,OAAO,CAAC,IAAI,CAAC,CAyCf;AAED,wBAAsB,iBAAiB,CACrC,iBAAiB,CAAC,EAAE,MAAM,GACzB,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAgClC;AAED,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAkE/B;AAED,wBAAgB,0BAA0B,CACxC,eAAe,EAAE,MAAM,EACvB,KAAK,GAAE,MAA4B,iCAOpC;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,QAAQ,EAAE,iCAI3D;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAuElC;AAED,wBAAgB,UAAU,CAAC,KAAK,GAAE,MAA4B,iCAE7D;AAED,wBAAsB,YAAY,CAAC,KAAK,GAAE,MAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAErE;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,MAAM,2CAQnD;AAED,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAQpD;AAED,wBAAsB,qBAAqB,CACzC,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAwC7B;AAED,wBAAsB,QAAQ,CAAC,KAAK,GAAE,MAA4B,+BAEjE;AAED,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,yCAQ/C;AAED,wBAAsB,UAAU,CAAC,MAAM,EAAE,MAAM,iBAa9C;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC/B,QAAQ,GAAE,MAAW,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAa5B;AAED,wBAAgB,WAAW,SAG1B;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,QAOnD;AAED,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAEF,wBAAgB,kBAAkB,CAAC,KAAK,GAAE,MAA4B,GAAG,cAAc,EAAE,CA+BxF;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,SAAS,CASvF;AAED,wBAAgB,SAAS,CACvB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,EAC7D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,WAG9D;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAMvF;AAED,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAsBvF;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CA4BxD;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,GACjB;IACD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAuCA"}

package/dist/src/util/index.js

@@ -269,19 +269,23 @@ function listPreviousResults(limit = DEFAULT_QUERY_LIMIT, filterDescription, dat
         createdAt: operations_1.evals.createdAt,
         description: operations_1.evals.description,
         config: operations_1.evals.config,
+        results: operations_1.evals.results,
         datasetId: operations_1.evalsToDatasets.datasetId,
     })
         .from(operations_1.evals)
         .leftJoin(operations_1.evalsToDatasets, (0, drizzle_orm_1.eq)(operations_1.evals.id, operations_1.evalsToDatasets.evalId))
         .where((0, drizzle_orm_1.and)(datasetId ? (0, drizzle_orm_1.eq)(operations_1.evalsToDatasets.datasetId, datasetId) : undefined, filterDescription ? (0, drizzle_orm_1.like)(operations_1.evals.description, `%${filterDescription}%`) : undefined));
     const results = query.orderBy((0, drizzle_orm_1.desc)(operations_1.evals.createdAt)).limit(limit).all();
-    return results.map((result) => ({
-        evalId: result.evalId,
-        createdAt: result.createdAt,
-        description: result.description,
-        numTests: result.config.tests?.length || 0,
-        datasetId: result.datasetId,
-    }));
+    return results.map((result) => {
+        const numTests = result.results.table.body.length;
+        return {
+            evalId: result.evalId,
+            createdAt: result.createdAt,
+            description: result.description,
+            numTests,
+            datasetId: result.datasetId,
+        };
+    });
 }
 /**
  * @deprecated Used only for migration to sqlite
@@ -829,7 +833,10 @@ function providerToIdentifier(provider) {
     else if ((0, types_1.isProviderOptions)(provider)) {
         return provider.id;
     }
-    return provider;
+    else if (typeof provider === 'string') {
+        return provider;
+    }
+    return undefined;
 }
 function varsMatch(vars1, vars2) {
     return (0, fast_deep_equal_1.default)(vars1, vars2);