promptfoo 0.20.0 → 0.21.0

package/src/providers/openai.ts

@@ -1,337 +0,0 @@
-import logger from '../logger';
-import { fetchWithCache } from '../cache';
-import { REQUEST_TIMEOUT_MS, parseChatPrompt } from './shared';
-
-import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '../types.js';
-
-const DEFAULT_OPENAI_HOST = 'api.openai.com';
-
-interface OpenAiCompletionOptions {
-  temperature?: number;
-  max_tokens?: number;
-  top_p?: number;
-  frequency_penalty?: number;
-  presence_penalty?: number;
-  best_of?: number;
-  functions?: {
-    name: string;
-    description?: string;
-    parameters: any;
-  }[];
-  function_call?: 'none' | 'auto';
-  apiKey?: string;
-  organization?: string;
-}
-
-class OpenAiGenericProvider implements ApiProvider {
-  modelName: string;
-  apiKey?: string;
-  apiHost: string;
-
-  constructor(modelName: string, apiKey?: string) {
-    this.modelName = modelName;
-
-    this.apiKey = apiKey;
-
-    this.apiHost = process.env.OPENAI_API_HOST || DEFAULT_OPENAI_HOST;
-  }
-
-  id(): string {
-    return `openai:${this.modelName}`;
-  }
-
-  toString(): string {
-    return `[OpenAI Provider ${this.modelName}]`;
-  }
-
-  getOrganization(options?: OpenAiCompletionOptions): string | undefined {
-    return options?.organization || process.env.OPENAI_ORGANIZATION;
-  }
-
-  getApiKey(options?: OpenAiCompletionOptions): string | undefined {
-    return options?.apiKey || this.apiKey || process.env.OPENAI_API_KEY;
-  }
-
-  // @ts-ignore: Prompt is not used in this implementation
-  async callApi(prompt: string, options?: OpenAiCompletionOptions): Promise<ProviderResponse> {
-    throw new Error('Not implemented');
-  }
-}
-
-export class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
-  async callEmbeddingApi(text: string): Promise<ProviderEmbeddingResponse> {
-    if (!this.getApiKey()) {
-      throw new Error('OpenAI API key must be set for similarity comparison');
-    }
-
-    const body = {
-      input: text,
-      model: this.modelName,
-    };
-    let data,
-      cached = false;
-    try {
-      ({ data, cached } = (await fetchWithCache(
-        `https://${this.apiHost}/v1/embeddings`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            Authorization: `Bearer ${this.getApiKey()}`,
-            ...(this.getOrganization() ? { 'OpenAI-Organization': this.getOrganization() } : {}),
-          },
-          body: JSON.stringify(body),
-        },
-        REQUEST_TIMEOUT_MS,
-      )) as unknown as any);
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-        tokenUsage: {
-          total: 0,
-          prompt: 0,
-          completion: 0,
-        },
-      };
-    }
-    logger.debug(`\tOpenAI API response (embeddings): ${JSON.stringify(data)}`);
-
-    try {
-      const embedding = data?.data?.[0]?.embedding;
-      if (!embedding) {
-        throw new Error('No embedding returned');
-      }
-      const ret = {
-        embedding,
-        tokenUsage: cached
-          ? { cached: data.usage.total_tokens }
-          : {
-              total: data.usage.total_tokens,
-              prompt: data.usage.prompt_tokens,
-              completion: data.usage.completion_tokens,
-            },
-      };
-      return ret;
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
-        tokenUsage: {
-          total: data?.usage?.total_tokens,
-          prompt: data?.usage?.prompt_tokens,
-          completion: data?.usage?.completion_tokens,
-        },
-      };
-    }
-  }
-}
-
-export class OpenAiCompletionProvider extends OpenAiGenericProvider {
-  static OPENAI_COMPLETION_MODELS = [
-    'text-davinci-003',
-    'text-davinci-002',
-    'text-curie-001',
-    'text-babbage-001',
-    'text-ada-001',
-  ];
-
-  options: OpenAiCompletionOptions;
-
-  constructor(modelName: string, apiKey?: string, context?: OpenAiCompletionOptions, id?: string) {
-    if (!OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelName)) {
-      logger.warn(`Using unknown OpenAI completion model: ${modelName}`);
-    }
-    super(modelName, apiKey);
-    this.options = context || {};
-    this.id = id ? () => id : this.id;
-  }
-
-  async callApi(prompt: string, options?: OpenAiCompletionOptions): Promise<ProviderResponse> {
-    if (!this.getApiKey(options)) {
-      throw new Error(
-        'OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
-      );
-    }
-
-    let stop: string;
-    try {
-      stop = process.env.OPENAI_STOP
-        ? JSON.parse(process.env.OPENAI_STOP)
-        : ['<|im_end|>', '<|endoftext|>'];
-    } catch (err) {
-      throw new Error(`OPENAI_STOP is not a valid JSON string: ${err}`);
-    }
-    const body = {
-      model: this.modelName,
-      prompt,
-      max_tokens:
-        options?.max_tokens ??
-        this.options.max_tokens ??
-        parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
-      temperature:
-        options?.temperature ??
-        this.options.temperature ??
-        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
-      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
-      presence_penalty:
-        options?.presence_penalty ??
-        this.options.presence_penalty ??
-        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
-      frequency_penalty:
-        options?.frequency_penalty ??
-        this.options.frequency_penalty ??
-        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
-      best_of:
-        options?.best_of ?? this.options.best_of ?? parseInt(process.env.OPENAI_BEST_OF || '1'),
-      stop,
-    };
-    logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
-    let data,
-      cached = false;
-    try {
-      ({ data, cached } = (await fetchWithCache(
-        `https://${this.apiHost}/v1/completions`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            Authorization: `Bearer ${this.getApiKey(options)}`,
-            ...(this.getOrganization(options)
-              ? { 'OpenAI-Organization': this.getOrganization(options) }
-              : {}),
-          },
-          body: JSON.stringify(body),
-        },
-        REQUEST_TIMEOUT_MS,
-      )) as unknown as any);
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-    logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
-    try {
-      return {
-        output: data.choices[0].text,
-        tokenUsage: cached
-          ? { cached: data.usage.total_tokens }
-          : {
-              total: data.usage.total_tokens,
-              prompt: data.usage.prompt_tokens,
-              completion: data.usage.completion_tokens,
-            },
-      };
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
-      };
-    }
-  }
-}
-
-export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
-  static OPENAI_CHAT_MODELS = [
-    'gpt-4',
-    'gpt-4-0314',
-    'gpt-4-0613',
-    'gpt-4-32k',
-    'gpt-4-32k-0314',
-    'gpt-3.5-turbo',
-    'gpt-3.5-turbo-0301',
-    'gpt-3.5-turbo-0613',
-    'gpt-3.5-turbo-16k',
-    'gpt-3.5-turbo-16k-0613',
-  ];
-
-  options: OpenAiCompletionOptions;
-
-  constructor(modelName: string, apiKey?: string, context?: OpenAiCompletionOptions, id?: string) {
-    if (!OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelName)) {
-      logger.warn(`Using unknown OpenAI chat model: ${modelName}`);
-    }
-    super(modelName, apiKey);
-    this.options = context || {};
-    this.id = id ? () => id : this.id;
-  }
-
-  async callApi(prompt: string, options?: OpenAiCompletionOptions): Promise<ProviderResponse> {
-    if (!this.getApiKey(options)) {
-      throw new Error(
-        'OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
-      );
-    }
-
-    const messages = parseChatPrompt(prompt);
-    const body = {
-      model: this.modelName,
-      messages: messages,
-      max_tokens:
-        options?.max_tokens ??
-        this.options.max_tokens ??
-        parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
-      temperature:
-        options?.temperature ??
-        this.options.temperature ??
-        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
-      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
-      presence_penalty:
-        options?.presence_penalty ??
-        this.options.presence_penalty ??
-        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
-      frequency_penalty:
-        options?.frequency_penalty ??
-        this.options.frequency_penalty ??
-        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
-      functions: options?.functions || this.options.functions || undefined,
-      function_call: options?.function_call || this.options.function_call || undefined,
-    };
-    logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
-
-    let data,
-      cached = false;
-    try {
-      ({ data, cached } = (await fetchWithCache(
-        `https://${this.apiHost}/v1/chat/completions`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            Authorization: `Bearer ${this.getApiKey(options)}`,
-            ...(this.getOrganization(options)
-              ? { 'OpenAI-Organization': this.getOrganization(options) }
-              : {}),
-          },
-          body: JSON.stringify(body),
-        },
-        REQUEST_TIMEOUT_MS,
-      )) as unknown as any);
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-
-    logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
-    try {
-      const message = data.choices[0].message;
-      const output = message.content === null ? message.function_call : message.content;
-      return {
-        output,
-        tokenUsage: cached
-          ? { cached: data.usage.total_tokens }
-          : {
-              total: data.usage.total_tokens,
-              prompt: data.usage.prompt_tokens,
-              completion: data.usage.completion_tokens,
-            },
-      };
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
-      };
-    }
-  }
-}
-
-export const DefaultEmbeddingProvider = new OpenAiEmbeddingProvider('text-embedding-ada-002');
-export const DefaultGradingProvider = new OpenAiChatCompletionProvider('gpt-4-0613');
-export const DefaultSuggestionsProvider = new OpenAiChatCompletionProvider('gpt-4-0613');

package/src/providers/replicate.ts

@@ -1,99 +0,0 @@
-import Replicate from 'replicate';
-
-import fetch from 'node-fetch';
-import logger from '../logger';
-import { getCache, isCacheEnabled } from '../cache';
-
-import type { ApiProvider, ProviderResponse } from '../types.js';
-
-interface ReplicateCompletionOptions {
-  temperature?: number;
-  max_length?: number;
-  repetition_penalty?: number;
-}
-
-export class ReplicateProvider implements ApiProvider {
-  modelName: string;
-  apiKey?: string;
-  replicate: any;
-  options: ReplicateCompletionOptions;
-
-  constructor(modelName: string, apiKey?: string, options?: ReplicateCompletionOptions) {
-    this.modelName = modelName;
-    this.apiKey = apiKey || process.env.REPLICATE_API_TOKEN || process.env.REPLICATE_API_KEY;
-    this.options = options || {};
-  }
-
-  id(): string {
-    return `replicate:${this.modelName}`;
-  }
-
-  toString(): string {
-    return `[Replicate Provider ${this.modelName}]`;
-  }
-
-  async callApi(prompt: string): Promise<ProviderResponse> {
-    if (!this.apiKey) {
-      throw new Error(
-        'Replicate API key is not set. Set REPLICATE_API_TOKEN environment variable or pass it as an argument to the constructor.',
-      );
-    }
-
-    let cache;
-    let cacheKey;
-    if (isCacheEnabled()) {
-      cache = await getCache();
-      cacheKey = `replicate:${this.modelName}:${prompt}`;
-
-      // Try to get the cached response
-      const cachedResponse = await cache.get(cacheKey);
-
-      if (cachedResponse) {
-        logger.debug(`Returning cached response for ${prompt}: ${cachedResponse}`);
-        return JSON.parse(cachedResponse as string);
-      }
-    }
-
-    const replicate = new Replicate({
-      auth: this.apiKey,
-      fetch,
-    });
-
-    logger.debug(`Calling Replicate: ${prompt}`);
-    let response;
-    try {
-      const data = {
-        input: {
-          prompt,
-          max_length:
-            this.options.max_length || parseInt(process.env.REPLICATE_MAX_LENGTH || '2046', 10),
-          temperature:
-            this.options.temperature || parseFloat(process.env.REPLICATE_TEMPERATURE || '0.01'),
-          repetition_penalty:
-            this.options.repetition_penalty ||
-            parseFloat(process.env.REPLICATE_REPETITION_PENALTY || '1.0'),
-        },
-      };
-      response = await replicate.run(this.modelName as any, data);
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-    logger.debug(`\tReplicate API response: ${JSON.stringify(response)}`);
-    try {
-      const result = {
-        output: (response as string[]).join(''),
-        tokenUsage: {}, // TODO: add token usage once Replicate API supports it
-      };
-      if (cache && cacheKey) {
-        await cache.set(cacheKey, JSON.stringify(result));
-      }
-      return result;
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(response)}`,
-      };
-    }
-  }
-}

package/src/providers/scriptCompletion.ts

@@ -1,35 +0,0 @@
-import { exec } from 'child_process';
-
-import { ApiProvider, ProviderOptions, ProviderResponse } from '../types';
-
-const ANSI_ESCAPE = /\x1b(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g;
-
-function stripText(text: string) {
-  return text.replace(ANSI_ESCAPE, '');
-}
-
-export class ScriptCompletionProvider implements ApiProvider {
-  constructor(private scriptPath: string, private config?: ProviderOptions) {}
-
-  id() {
-    return `exec:${this.scriptPath}`;
-  }
-
-  async callApi(prompt: string) {
-    return new Promise((resolve, reject) => {
-      // TODO(ian): This config is real ugly.
-      const command = this.config?.config.basePath
-        ? `cd ${this.config.config.basePath} && ${this.scriptPath} "${prompt}"`
-        : `${this.scriptPath} "${prompt}"`;
-
-      // TODO(ian): Caching
-      exec(command, (error, stdout, stderr) => {
-        if (error) {
-          reject(error);
-        } else {
-          resolve({ output: stripText(stdout.trim()) });
-        }
-      });
-    }) as Promise<ProviderResponse>;
-  }
-}

package/src/providers/shared.ts

@@ -1,34 +0,0 @@
-import yaml from 'js-yaml';
-
-export const REQUEST_TIMEOUT_MS = process.env.REQUEST_TIMEOUT_MS
-  ? parseInt(process.env.REQUEST_TIMEOUT_MS, 10)
-  : 300_000;
-
-export function parseChatPrompt(
-  prompt: string,
-): { role: string; content: string; name?: string }[] {
-  const trimmedPrompt = prompt.trim();
-  if (trimmedPrompt.startsWith('- role:')) {
-    try {
-      // Try YAML
-      return yaml.load(prompt) as { role: string; content: string }[];
-    } catch (err) {
-      throw new Error(`Chat Completion prompt is not a valid YAML string: ${err}\n\n${prompt}`);
-    }
-  } else {
-    try {
-      // Try JSON
-      return JSON.parse(prompt) as { role: string; content: string }[];
-    } catch (err) {
-      if (
-        process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
-        trimmedPrompt.startsWith('{') ||
-        trimmedPrompt.startsWith('[')
-      ) {
-        throw new Error(`Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`);
-      }
-      // Fall back to wrapping the prompt in a user message
-      return [{ role: 'user', content: prompt }];
-    }
-  }
-}

package/src/providers.ts

@@ -1,192 +0,0 @@
-import path from 'path';
-
-import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './providers/openai';
-import { AnthropicCompletionProvider } from './providers/anthropic';
-import { ReplicateProvider } from './providers/replicate';
-import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai';
-import { LlamaProvider } from './providers/llama';
-import { OllamaProvider } from './providers/ollama';
-import { ScriptCompletionProvider } from './providers/scriptCompletion';
-import {
-  AzureOpenAiChatCompletionProvider,
-  AzureOpenAiCompletionProvider,
-} from './providers/azureopenai';
-
-import type {
-  ApiProvider,
-  ProviderOptions,
-  ProviderFunction,
-  ProviderId,
-  ProviderOptionsMap,
-} from './types';
-
-export async function loadApiProviders(
-  providerPaths:
-    | ProviderId
-    | ProviderId[]
-    | ProviderOptionsMap[]
-    | ProviderOptions[]
-    | ProviderFunction,
-  basePath?: string,
-): Promise<ApiProvider[]> {
-  if (typeof providerPaths === 'string') {
-    return [await loadApiProvider(providerPaths, undefined, basePath)];
-  } else if (typeof providerPaths === 'function') {
-    return [
-      {
-        id: () => 'custom-function',
-        callApi: providerPaths,
-      },
-    ];
-  } else if (Array.isArray(providerPaths)) {
-    return Promise.all(
-      providerPaths.map((provider, idx) => {
-        if (typeof provider === 'string') {
-          return loadApiProvider(provider, undefined, basePath);
-        } else if (typeof provider === 'function') {
-          return {
-            id: () => `custom-function-${idx}`,
-            callApi: provider,
-          };
-        } else if (provider.id) {
-          // List of ProviderConfig objects
-          return loadApiProvider((provider as ProviderOptions).id!, provider, basePath);
-        } else {
-          // List of { id: string, config: ProviderConfig } objects
-          const id = Object.keys(provider)[0];
-          const providerObject = (provider as ProviderOptionsMap)[id];
-          const context = { ...providerObject, id: providerObject.id || id };
-          return loadApiProvider(id, context, basePath);
-        }
-      }),
-    );
-  }
-  throw new Error('Invalid providers list');
-}
-
-export async function loadApiProvider(
-  providerPath: string,
-  context?: ProviderOptions,
-  basePath?: string,
-): Promise<ApiProvider> {
-  context = context || {};
-  if (providerPath?.startsWith('exec:')) {
-    // Load script module
-    const scriptPath = providerPath.split(':')[1];
-    return new ScriptCompletionProvider(scriptPath, {
-      id: `exec:${scriptPath}`,
-      config: { basePath },
-    });
-  } else if (providerPath?.startsWith('openai:')) {
-    // Load OpenAI module
-    const options = providerPath.split(':');
-    const modelType = options[1];
-    const modelName = options[2];
-
-    if (modelType === 'chat') {
-      return new OpenAiChatCompletionProvider(
-        modelName || 'gpt-3.5-turbo',
-        undefined,
-        context.config,
-      );
-    } else if (modelType === 'completion') {
-      return new OpenAiCompletionProvider(
-        modelName || 'text-davinci-003',
-        undefined,
-        context.config,
-      );
-    } else if (OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelType)) {
-      return new OpenAiChatCompletionProvider(modelType, undefined, context.config, context.id);
-    } else if (OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelType)) {
-      return new OpenAiCompletionProvider(modelType, undefined, context.config, context.id);
-    } else {
-      throw new Error(
-        `Unknown OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
-      );
-    }
-  } else if (providerPath?.startsWith('azureopenai:')) {
-    // Load Azure OpenAI module
-    const options = providerPath.split(':');
-    const modelType = options[1];
-    const deploymentName = options[2];
-
-    if (modelType === 'chat') {
-      return new AzureOpenAiChatCompletionProvider(
-        deploymentName,
-        undefined,
-        context.config,
-        context.id,
-      );
-    } else if (modelType === 'completion') {
-      return new AzureOpenAiCompletionProvider(
-        deploymentName,
-        undefined,
-        context.config,
-        context.id,
-      );
-    } else {
-      throw new Error(
-        `Unknown Azure OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
-      );
-    }
-  } else if (providerPath?.startsWith('anthropic:')) {
-    // Load Anthropic module
-    const options = providerPath.split(':');
-    const modelType = options[1];
-    const modelName = options[2];
-
-    if (modelType === 'completion') {
-      return new AnthropicCompletionProvider(
-        modelName || 'claude-instant-1',
-        undefined,
-        context.config,
-      );
-    } else if (AnthropicCompletionProvider.ANTHROPIC_COMPLETION_MODELS.includes(modelType)) {
-      return new AnthropicCompletionProvider(modelType, undefined, context.config);
-    } else {
-      throw new Error(
-        `Unknown Anthropic model type: ${modelType}. Use one of the following providers: anthropic:completion:<model name>`,
-      );
-    }
-  } else if (providerPath?.startsWith('replicate:')) {
-    // Load Replicate module
-    const options = providerPath.split(':');
-    const modelName = options.slice(1).join(':');
-
-    return new ReplicateProvider(modelName, undefined, context.config);
-  }
-
-  if (providerPath === 'llama' || providerPath.startsWith('llama:')) {
-    const modelName = providerPath.split(':')[1];
-    return new LlamaProvider(modelName, context.config);
-  } else if (providerPath.startsWith('ollama:')) {
-    const modelName = providerPath.split(':')[1];
-    return new OllamaProvider(modelName);
-  } else if (providerPath?.startsWith('localai:')) {
-    const options = providerPath.split(':');
-    const modelType = options[1];
-    const modelName = options[2];
-
-    if (modelType === 'chat') {
-      return new LocalAiChatProvider(modelName);
-    } else if (modelType === 'completion') {
-      return new LocalAiCompletionProvider(modelName);
-    } else {
-      return new LocalAiChatProvider(modelType);
-    }
-  }
-
-  // Load custom module
-  const CustomApiProvider = (await import(path.join(process.cwd(), providerPath))).default;
-  return new CustomApiProvider(context);
-}
-
-export default {
-  OpenAiCompletionProvider,
-  OpenAiChatCompletionProvider,
-  AnthropicCompletionProvider,
-  ReplicateProvider,
-  LocalAiCompletionProvider,
-  LocalAiChatProvider,
-  loadApiProvider,
-};