promptfoo 0.20.0 → 0.21.0

package/src/providers/anthropic.ts

@@ -1,88 +0,0 @@
-import Anthropic from '@anthropic-ai/sdk';
-import logger from '../logger';
-
-import type { ApiProvider, ProviderResponse } from '../types.js';
-
-interface AnthropicCompletionOptions {
-  temperature?: number;
-  top_p?: number;
-  top_k?: number;
-}
-
-export class AnthropicCompletionProvider implements ApiProvider {
-  static ANTHROPIC_COMPLETION_MODELS = [
-    'claude-1',
-    'claude-1-100k',
-    'claude-instant-1',
-    'claude-instant-1-100k',
-  ];
-
-  modelName: string;
-  apiKey?: string;
-  anthropic: Anthropic;
-  options: AnthropicCompletionOptions;
-
-  constructor(modelName: string, apiKey?: string, context?: AnthropicCompletionOptions) {
-    this.modelName = modelName;
-    this.apiKey = apiKey || process.env.ANTHROPIC_API_KEY;
-    this.anthropic = new Anthropic({ apiKey: this.apiKey });
-    this.options = context || {};
-  }
-
-  id(): string {
-    return `anthropic:${this.modelName}`;
-  }
-
-  toString(): string {
-    return `[Anthropic Provider ${this.modelName}]`;
-  }
-
-  async callApi(prompt: string, options?: AnthropicCompletionOptions): Promise<ProviderResponse> {
-    if (!this.apiKey) {
-      throw new Error(
-        'Anthropic API key is not set. Set ANTHROPIC_API_KEY environment variable or pass it as an argument to the constructor.',
-      );
-    }
-
-    let stop: string[];
-    try {
-      stop = process.env.ANTHROPIC_STOP
-        ? JSON.parse(process.env.ANTHROPIC_STOP)
-        : ['<|im_end|>', '<|endoftext|>'];
-    } catch (err) {
-      throw new Error(`ANTHROPIC_STOP is not a valid JSON string: ${err}`);
-    }
-
-    const params: Anthropic.CompletionCreateParams = {
-      model: this.modelName,
-      prompt: `${Anthropic.HUMAN_PROMPT} ${prompt} ${Anthropic.AI_PROMPT}`,
-      max_tokens_to_sample: parseInt(process.env.ANTHROPIC_MAX_TOKENS || '1024'),
-      temperature:
-        options?.temperature ??
-        this.options.temperature ??
-        parseFloat(process.env.ANTHROPIC_TEMPERATURE || '0'),
-      stop_sequences: stop,
-    };
-
-    logger.debug(`Calling Anthropic API: ${JSON.stringify(params)}`);
-    let response;
-    try {
-      response = await this.anthropic.completions.create(params);
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-    logger.debug(`\tAnthropic API response: ${JSON.stringify(response)}`);
-    try {
-      return {
-        output: response.completion,
-        tokenUsage: {}, // TODO: add token usage once Anthropic API supports it
-      };
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(response)}`,
-      };
-    }
-  }
-}

package/src/providers/azureopenai.ts

@@ -1,299 +0,0 @@
-import logger from '../logger';
-import { fetchWithCache } from '../cache';
-import { REQUEST_TIMEOUT_MS, parseChatPrompt } from './shared';
-
-import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '../types.js';
-
-interface AzureOpenAiCompletionOptions {
-  temperature?: number;
-  top_p?: number;
-  frequency_penalty?: number;
-  presence_penalty?: number;
-  best_of?: number;
-  functions?: {
-    name: string;
-    description?: string;
-    parameters: any;
-  }[];
-  function_call?: 'none' | 'auto';
-}
-
-class AzureOpenAiGenericProvider implements ApiProvider {
-  deploymentName: string;
-  apiKey?: string;
-  apiHost?: string;
-
-  constructor(deploymentName: string, apiKey?: string) {
-    this.deploymentName = deploymentName;
-
-    this.apiKey = apiKey || process.env.AZURE_OPENAI_API_KEY;
-
-    this.apiHost = process.env.AZURE_OPENAI_API_HOST;
-  }
-
-  id(): string {
-    return `azureopenai:${this.deploymentName}`;
-  }
-
-  toString(): string {
-    return `[Azure OpenAI Provider ${this.deploymentName}]`;
-  }
-
-  // @ts-ignore: Prompt is not used in this implementation
-  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
-    throw new Error('Not implemented');
-  }
-}
-
-export class AzureOpenAiEmbeddingProvider extends AzureOpenAiGenericProvider {
-  async callEmbeddingApi(text: string): Promise<ProviderEmbeddingResponse> {
-    if (!this.apiKey) {
-      throw new Error('Azure OpenAI API key must be set for similarity comparison');
-    }
-    if (!this.apiHost) {
-      throw new Error('Azure OpenAI API host must be set');
-    }
-
-    const body = {
-      input: text,
-      model: this.deploymentName,
-    };
-    let data,
-      cached = false;
-    try {
-      ({ data, cached } = (await fetchWithCache(
-        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/embeddings?api-version=2023-07-01-preview`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            'api-key': this.apiKey,
-          },
-          body: JSON.stringify(body),
-        },
-        REQUEST_TIMEOUT_MS,
-      )) as unknown as any);
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-        tokenUsage: {
-          total: 0,
-          prompt: 0,
-          completion: 0,
-        },
-      };
-    }
-    logger.debug(`\tAzure OpenAI API response (embeddings): ${JSON.stringify(data)}`);
-
-    try {
-      const embedding = data?.data?.[0]?.embedding;
-      if (!embedding) {
-        throw new Error('No embedding returned');
-      }
-      const ret = {
-        embedding,
-        tokenUsage: cached
-          ? { cached: data.usage.total_tokens }
-          : {
-              total: data.usage.total_tokens,
-              prompt: data.usage.prompt_tokens,
-              completion: data.usage.completion_tokens,
-            },
-      };
-      return ret;
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
-        tokenUsage: {
-          total: data?.usage?.total_tokens,
-          prompt: data?.usage?.prompt_tokens,
-          completion: data?.usage?.completion_tokens,
-        },
-      };
-    }
-  }
-}
-
-export class AzureOpenAiCompletionProvider extends AzureOpenAiGenericProvider {
-  options: AzureOpenAiCompletionOptions;
-
-  constructor(
-    deploymentName: string,
-    apiKey?: string,
-    context?: AzureOpenAiCompletionOptions,
-    id?: string,
-  ) {
-    super(deploymentName, apiKey);
-    this.options = context || {};
-    this.id = id ? () => id : this.id;
-  }
-
-  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
-    if (!this.apiKey) {
-      throw new Error(
-        'Azure OpenAI API key is not set. Set AZURE_OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
-      );
-    }
-    if (!this.apiHost) {
-      throw new Error('Azure OpenAI API host must be set');
-    }
-
-    let stop: string;
-    try {
-      stop = process.env.OPENAI_STOP
-        ? JSON.parse(process.env.OPENAI_STOP)
-        : ['<|im_end|>', '<|endoftext|>'];
-    } catch (err) {
-      throw new Error(`OPENAI_STOP is not a valid JSON string: ${err}`);
-    }
-    const body = {
-      model: this.deploymentName,
-      prompt,
-      max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
-      temperature:
-        options?.temperature ??
-        this.options.temperature ??
-        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
-      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
-      presence_penalty:
-        options?.presence_penalty ??
-        this.options.presence_penalty ??
-        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
-      frequency_penalty:
-        options?.frequency_penalty ??
-        this.options.frequency_penalty ??
-        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
-      best_of:
-        options?.best_of ?? this.options.best_of ?? parseInt(process.env.OPENAI_BEST_OF || '1'),
-      stop,
-    };
-    logger.debug(`Calling Azure OpenAI API: ${JSON.stringify(body)}`);
-    let data,
-      cached = false;
-    try {
-      ({ data, cached } = (await fetchWithCache(
-        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/completions?api-version=2023-07-01-preview`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            'api-key': this.apiKey,
-          },
-          body: JSON.stringify(body),
-        },
-        REQUEST_TIMEOUT_MS,
-      )) as unknown as any);
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-    logger.debug(`\tAzure OpenAI API response: ${JSON.stringify(data)}`);
-    try {
-      return {
-        output: data.choices[0].text,
-        tokenUsage: cached
-          ? { cached: data.usage.total_tokens }
-          : {
-              total: data.usage.total_tokens,
-              prompt: data.usage.prompt_tokens,
-              completion: data.usage.completion_tokens,
-            },
-      };
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
-      };
-    }
-  }
-}
-
-export class AzureOpenAiChatCompletionProvider extends AzureOpenAiGenericProvider {
-  options: AzureOpenAiCompletionOptions;
-
-  constructor(
-    deploymentName: string,
-    apiKey?: string,
-    context?: AzureOpenAiCompletionOptions,
-    id?: string,
-  ) {
-    super(deploymentName, apiKey);
-    this.options = context || {};
-    this.id = id ? () => id : this.id;
-  }
-
-  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
-    if (!this.apiKey) {
-      throw new Error(
-        'Azure OpenAI API key is not set. Set AZURE_OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
-      );
-    }
-    if (!this.apiHost) {
-      throw new Error('Azure OpenAI API host must be set');
-    }
-
-    const messages = parseChatPrompt(prompt);
-    const body = {
-      model: this.deploymentName,
-      messages: messages,
-      max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
-      temperature:
-        options?.temperature ??
-        this.options.temperature ??
-        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
-      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
-      presence_penalty:
-        options?.presence_penalty ??
-        this.options.presence_penalty ??
-        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
-      frequency_penalty:
-        options?.frequency_penalty ??
-        this.options.frequency_penalty ??
-        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
-      functions: options?.functions || this.options.functions || undefined,
-      function_call: options?.function_call || this.options.function_call || undefined,
-    };
-    logger.debug(`Calling Azure OpenAI API: ${JSON.stringify(body)}`);
-
-    let data,
-      cached = false;
-    try {
-      ({ data, cached } = (await fetchWithCache(
-        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/chat/completions?api-version=2023-07-01-preview`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            'api-key': this.apiKey,
-          },
-          body: JSON.stringify(body),
-        },
-        REQUEST_TIMEOUT_MS,
-      )) as unknown as any);
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-
-    logger.debug(`\tAzure OpenAI API response: ${JSON.stringify(data)}`);
-    try {
-      const message = data.choices[0].message;
-      const output = message.content == null ? message.function_call : message.content;
-      return {
-        output,
-        tokenUsage: cached
-          ? { cached: data.usage.total_tokens }
-          : {
-              total: data.usage.total_tokens,
-              prompt: data.usage.prompt_tokens,
-              completion: data.usage.completion_tokens,
-            },
-      };
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
-      };
-    }
-  }
-}

package/src/providers/llama.ts

@@ -1,95 +0,0 @@
-import { fetchWithCache } from '../cache';
-import { REQUEST_TIMEOUT_MS } from './shared';
-
-import type { ApiProvider, ProviderResponse } from '../types.js';
-
-interface LlamaCompletionOptions {
-  n_predict?: number;
-  temperature?: number;
-  top_k?: number;
-  top_p?: number;
-  n_keep?: number;
-  stop?: string[];
-  repeat_penalty?: number;
-  repeat_last_n?: number;
-  penalize_nl?: boolean;
-  presence_penalty?: number;
-  frequency_penalty?: number;
-  mirostat?: boolean;
-  mirostat_tau?: number;
-  mirostat_eta?: number;
-  seed?: number;
-  ignore_eos?: boolean;
-  logit_bias?: Record<string, number>;
-}
-
-export class LlamaProvider implements ApiProvider {
-  modelName: string;
-  options?: LlamaCompletionOptions;
-
-  constructor(modelName: string, options?: LlamaCompletionOptions) {
-    this.modelName = modelName;
-    this.options = options;
-  }
-
-  id(): string {
-    return `llama:${this.modelName}`;
-  }
-
-  toString(): string {
-    return `[Llama Provider ${this.modelName}]`;
-  }
-
-  async callApi(prompt: string, options?: LlamaCompletionOptions): Promise<ProviderResponse> {
-    options = Object.assign({}, this.options, options);
-    const body = {
-      prompt,
-      n_predict: options?.n_predict || 512,
-      temperature: options?.temperature,
-      top_k: options?.top_k,
-      top_p: options?.top_p,
-      n_keep: options?.n_keep,
-      stop: options?.stop,
-      repeat_penalty: options?.repeat_penalty,
-      repeat_last_n: options?.repeat_last_n,
-      penalize_nl: options?.penalize_nl,
-      presence_penalty: options?.presence_penalty,
-      frequency_penalty: options?.frequency_penalty,
-      mirostat: options?.mirostat,
-      mirostat_tau: options?.mirostat_tau,
-      mirostat_eta: options?.mirostat_eta,
-      seed: options?.seed,
-      ignore_eos: options?.ignore_eos,
-      logit_bias: options?.logit_bias,
-    };
-
-    let response;
-    try {
-      response = await fetchWithCache(
-        `${process.env.LLAMA_BASE_URL || 'http://localhost:8080'}/completion`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-          },
-          body: JSON.stringify(body),
-        },
-        REQUEST_TIMEOUT_MS,
-      );
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-
-    try {
-      return {
-        output: response.data.content,
-      };
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(response.data)}`,
-      };
-    }
-  }
-}

package/src/providers/localai.ts

@@ -1,111 +0,0 @@
-import logger from '../logger';
-import { fetchWithCache } from '../cache';
-import { REQUEST_TIMEOUT_MS, parseChatPrompt } from './shared';
-
-import type { ApiProvider, ProviderResponse } from '../types.js';
-
-class LocalAiGenericProvider implements ApiProvider {
-  modelName: string;
-  apiBaseUrl: string;
-
-  constructor(modelName: string) {
-    this.modelName = modelName;
-    this.apiBaseUrl = process.env.LOCALAI_BASE_URL || 'http://localhost:8080/v1';
-  }
-
-  id(): string {
-    return `localai:${this.modelName}`;
-  }
-
-  toString(): string {
-    return `[LocalAI Provider ${this.modelName}]`;
-  }
-
-  // @ts-ignore: Prompt is not used in this implementation
-  async callApi(prompt: string): Promise<ProviderResponse> {
-    throw new Error('Not implemented');
-  }
-}
-
-export class LocalAiChatProvider extends LocalAiGenericProvider {
-  async callApi(prompt: string): Promise<ProviderResponse> {
-    const messages = parseChatPrompt(prompt);
-    const body = {
-      model: this.modelName,
-      messages: messages,
-      temperature: process.env.LOCALAI_TEMPERATURE || 0.7,
-    };
-    logger.debug(`Calling LocalAI API: ${JSON.stringify(body)}`);
-
-    let data,
-      cached = false;
-    try {
-      ({ data, cached } = (await fetchWithCache(
-        `${this.apiBaseUrl}/chat/completions`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-          },
-          body: JSON.stringify(body),
-        },
-        REQUEST_TIMEOUT_MS,
-      )) as unknown as any);
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-    logger.debug(`	LocalAI API response: ${JSON.stringify(data)}`);
-    try {
-      return {
-        output: data.choices[0].text,
-      };
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
-      };
-    }
-  }
-}
-
-export class LocalAiCompletionProvider extends LocalAiGenericProvider {
-  async callApi(prompt: string): Promise<ProviderResponse> {
-    const body = {
-      model: this.modelName,
-      prompt,
-      temperature: process.env.LOCALAI_TEMPERATURE || 0.7,
-    };
-    logger.debug(`Calling LocalAI API: ${JSON.stringify(body)}`);
-
-    let data,
-      cached = false;
-    try {
-      ({ data, cached } = (await fetchWithCache(
-        `${this.apiBaseUrl}/completions`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-          },
-          body: JSON.stringify(body),
-        },
-        REQUEST_TIMEOUT_MS,
-      )) as unknown as any);
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-    logger.debug(`	LocalAI API response: ${JSON.stringify(data)}`);
-    try {
-      return {
-        output: data.choices[0].text,
-      };
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
-      };
-    }
-  }
-}

package/src/providers/ollama.ts

@@ -1,89 +0,0 @@
-import logger from '../logger';
-import { fetchWithCache } from '../cache';
-
-import type { ApiProvider, ProviderResponse } from '../types.js';
-import { REQUEST_TIMEOUT_MS } from './shared';
-
-interface OllamaJsonL {
-  model: string;
-  created_at: string;
-  response?: string;
-  done: boolean;
-  context?: number[];
-  total_duration?: number;
-  load_duration?: number;
-  sample_count?: number;
-  sample_duration?: number;
-  prompt_eval_count?: number;
-  prompt_eval_duration?: number;
-  eval_count?: number;
-  eval_duration?: number;
-}
-
-export class OllamaProvider implements ApiProvider {
-  modelName: string;
-
-  constructor(modelName: string) {
-    this.modelName = modelName;
-  }
-
-  id(): string {
-    return `ollama:${this.modelName}`;
-  }
-
-  toString(): string {
-    return `[Ollama Provider ${this.modelName}]`;
-  }
-
-  async callApi(prompt: string): Promise<ProviderResponse> {
-    const params = {
-      model: this.modelName,
-      prompt,
-    };
-
-    logger.debug(`Calling Ollama API: ${JSON.stringify(params)}`);
-    let response;
-    try {
-      response = await fetchWithCache(
-        `${process.env.OLLAMA_BASE_URL || 'http://localhost:11434'}/api/generate`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-          },
-          body: JSON.stringify(params),
-        },
-        REQUEST_TIMEOUT_MS,
-        'text',
-      );
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-    logger.debug(`\tOllama API response: ${response.data}`);
-
-    try {
-      const output = response.data
-        .split('\n')
-        .filter((line: string) => line.trim() !== '')
-        .map((line: string) => {
-          const parsed = JSON.parse(line) as OllamaJsonL;
-          if (parsed.response) {
-            return parsed.response;
-          }
-          return null;
-        })
-        .filter((s: string | null) => s !== null)
-        .join('');
-
-      return {
-        output,
-      };
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(response.data)}`,
-      };
-    }
-  }
-}