promptfoo 0.17.3 → 0.17.5

package/src/providers/azureopenai.ts

@@ -0,0 +1,280 @@
+import logger from '../logger';
+import { fetchJsonWithCache } from '../cache';
+import { REQUEST_TIMEOUT_MS } from './shared';
+
+import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '../types.js';
+
+interface AzureOpenAiCompletionOptions {
+  temperature?: number;
+  functions?: {
+    name: string;
+    description?: string;
+    parameters: any;
+  }[];
+  function_call?: 'none' | 'auto';
+}
+
+class AzureOpenAiGenericProvider implements ApiProvider {
+  deploymentName: string;
+  apiKey?: string;
+  apiHost?: string;
+
+  constructor(deploymentName: string, apiKey?: string) {
+    this.deploymentName = deploymentName;
+
+    this.apiKey = apiKey || process.env.AZURE_OPENAI_API_KEY;
+
+    this.apiHost = process.env.AZURE_OPENAI_API_HOST;
+  }
+
+  id(): string {
+    return `azureopenai:${this.deploymentName}`;
+  }
+
+  toString(): string {
+    return `[Azure OpenAI Provider ${this.deploymentName}]`;
+  }
+
+  // @ts-ignore: Prompt is not used in this implementation
+  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
+    throw new Error('Not implemented');
+  }
+}
+
+export class AzureOpenAiEmbeddingProvider extends AzureOpenAiGenericProvider {
+  async callEmbeddingApi(text: string): Promise<ProviderEmbeddingResponse> {
+    if (!this.apiKey) {
+      throw new Error('Azure OpenAI API key must be set for similarity comparison');
+    }
+    if (!this.apiHost) {
+      throw new Error('Azure OpenAI API host must be set');
+    }
+
+    const body = {
+      input: text,
+      model: this.deploymentName,
+    };
+    let data,
+      cached = false;
+    try {
+      ({ data, cached } = (await fetchJsonWithCache(
+        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/embeddings?api-version=2023-07-01-preview`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'api-key': this.apiKey,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      )) as unknown as any);
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+        tokenUsage: {
+          total: 0,
+          prompt: 0,
+          completion: 0,
+        },
+      };
+    }
+    logger.debug(`\tAzure OpenAI API response (embeddings): ${JSON.stringify(data)}`);
+
+    try {
+      const embedding = data?.data?.[0]?.embedding;
+      if (!embedding) {
+        throw new Error('No embedding returned');
+      }
+      const ret = {
+        embedding,
+        tokenUsage: cached
+          ? { cached: data.usage.total_tokens }
+          : {
+              total: data.usage.total_tokens,
+              prompt: data.usage.prompt_tokens,
+              completion: data.usage.completion_tokens,
+            },
+      };
+      return ret;
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+        tokenUsage: {
+          total: data?.usage?.total_tokens,
+          prompt: data?.usage?.prompt_tokens,
+          completion: data?.usage?.completion_tokens,
+        },
+      };
+    }
+  }
+}
+
+export class AzureOpenAiCompletionProvider extends AzureOpenAiGenericProvider {
+  options: AzureOpenAiCompletionOptions;
+
+  constructor(deploymentName: string, apiKey?: string, context?: AzureOpenAiCompletionOptions) {
+    super(deploymentName, apiKey);
+    this.options = context || {};
+  }
+
+  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
+    if (!this.apiKey) {
+      throw new Error(
+        'Azure OpenAI API key is not set. Set AZURE_OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
+      );
+    }
+    if (!this.apiHost) {
+      throw new Error('Azure OpenAI API host must be set');
+    }
+
+    let stop: string;
+    try {
+      stop = process.env.OPENAI_STOP
+        ? JSON.parse(process.env.OPENAI_STOP)
+        : ['<|im_end|>', '<|endoftext|>'];
+    } catch (err) {
+      throw new Error(`OPENAI_STOP is not a valid JSON string: ${err}`);
+    }
+    const body = {
+      model: this.deploymentName,
+      prompt,
+      max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      temperature:
+        options?.temperature ??
+        this.options.temperature ??
+        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      stop,
+    };
+    logger.debug(`Calling Azure OpenAI API: ${JSON.stringify(body)}`);
+    let data,
+      cached = false;
+    try {
+      ({ data, cached } = (await fetchJsonWithCache(
+        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/completions?api-version=2023-07-01-preview`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'api-key': this.apiKey,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      )) as unknown as any);
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+      };
+    }
+    logger.debug(`\tAzure OpenAI API response: ${JSON.stringify(data)}`);
+    try {
+      return {
+        output: data.choices[0].text,
+        tokenUsage: cached
+          ? { cached: data.usage.total_tokens }
+          : {
+              total: data.usage.total_tokens,
+              prompt: data.usage.prompt_tokens,
+              completion: data.usage.completion_tokens,
+            },
+      };
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+      };
+    }
+  }
+}
+
+export class AzureOpenAiChatCompletionProvider extends AzureOpenAiGenericProvider {
+  options: AzureOpenAiCompletionOptions;
+
+  constructor(deploymentName: string, apiKey?: string, context?: AzureOpenAiCompletionOptions) {
+    super(deploymentName, apiKey);
+    this.options = context || {};
+  }
+
+  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
+    if (!this.apiKey) {
+      throw new Error(
+        'Azure OpenAI API key is not set. Set AZURE_OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
+      );
+    }
+    if (!this.apiHost) {
+      throw new Error('Azure OpenAI API host must be set');
+    }
+
+    let messages: { role: string; content: string; name?: string }[];
+    try {
+      messages = JSON.parse(prompt) as { role: string; content: string }[];
+    } catch (err) {
+      const trimmedPrompt = prompt.trim();
+      if (
+        process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
+        trimmedPrompt.startsWith('{') ||
+        trimmedPrompt.startsWith('[')
+      ) {
+        throw new Error(
+          `Azure OpenAI Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`,
+        );
+      }
+      messages = [{ role: 'user', content: prompt }];
+    }
+
+    const body = {
+      model: this.deploymentName,
+      messages: messages,
+      max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      temperature:
+        options?.temperature ??
+        this.options.temperature ??
+        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      functions: options?.functions || this.options.functions || undefined,
+      function_call: options?.function_call || this.options.function_call || undefined,
+    };
+    logger.debug(`Calling Azure OpenAI API: ${JSON.stringify(body)}`);
+
+    let data,
+      cached = false;
+    try {
+      ({ data, cached } = (await fetchJsonWithCache(
+        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/chat/completions?api-version=2023-07-01-preview`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'api-key': this.apiKey,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      )) as unknown as any);
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+      };
+    }
+
+    logger.debug(`\tAzure OpenAI API response: ${JSON.stringify(data)}`);
+    try {
+      const message = data.choices[0].message;
+      const output =
+        message.content === null ? JSON.stringify(message.function_call) : message.content;
+      return {
+        output,
+        tokenUsage: cached
+          ? { cached: data.usage.total_tokens }
+          : {
+              total: data.usage.total_tokens,
+              prompt: data.usage.prompt_tokens,
+              completion: data.usage.completion_tokens,
+            },
+      };
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+      };
+    }
+  }
+}

package/src/providers/openai.ts

@@ -1,3 +1,5 @@
+import yaml from 'js-yaml';
+
 import logger from '../logger';
 import { fetchJsonWithCache } from '../cache';
 import { REQUEST_TIMEOUT_MS } from './shared';
@@ -226,20 +228,34 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     }
 
     let messages: { role: string; content: string; name?: string }[];
-    try {
-      messages = JSON.parse(prompt) as { role: string; content: string }[];
-    } catch (err) {
-      const trimmedPrompt = prompt.trim();
-      if (
-        process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
-        trimmedPrompt.startsWith('{') ||
-        trimmedPrompt.startsWith('[')
-      ) {
+    const trimmedPrompt = prompt.trim();
+    if (trimmedPrompt.startsWith('- role:')) {
+      try {
+        // Try YAML
+        messages = yaml.load(prompt) as { role: string; content: string }[];
+      } catch (err) {
         throw new Error(
-          `OpenAI Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`,
+          `OpenAI Chat Completion prompt is not a valid YAML string: ${err}\n\n${prompt}`,
         );
       }
-      messages = [{ role: 'user', content: prompt }];
+    } else {
+      try {
+        // Try JSON
+        messages = JSON.parse(prompt) as { role: string; content: string }[];
+      } catch (err) {
+        if (
+          process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
+          trimmedPrompt.startsWith('{') ||
+          trimmedPrompt.startsWith('[')
+        ) {
+          throw new Error(
+            `OpenAI Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`,
+          );
+        }
+
+        // Fall back to wrapping the prompt in a user message
+        messages = [{ role: 'user', content: prompt }];
+      }
     }
 
     const body = {

package/src/providers.ts

@@ -6,6 +6,10 @@ import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './provid
 import { AnthropicCompletionProvider } from './providers/anthropic';
 import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai';
 import { ScriptCompletionProvider } from './providers/scriptCompletion';
+import {
+  AzureOpenAiChatCompletionProvider,
+  AzureOpenAiCompletionProvider,
+} from './providers/azureopenai';
 
 export async function loadApiProviders(
   providerPaths: ProviderId | ProviderId[] | RawProviderConfig[],
@@ -68,6 +72,21 @@ export async function loadApiProvider(
         `Unknown OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
       );
     }
+  } else if (providerPath?.startsWith('azureopenai:')) {
+    // Load Azure OpenAI module
+    const options = providerPath.split(':');
+    const modelType = options[1];
+    const deploymentName = options[2];
+
+    if (modelType === 'chat') {
+      return new AzureOpenAiChatCompletionProvider(deploymentName, undefined, context?.config);
+    } else if (modelType === 'completion') {
+      return new AzureOpenAiCompletionProvider(deploymentName, undefined, context?.config);
+    } else {
+      throw new Error(
+        `Unknown Azure OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
+      );
+    }
   } else if (providerPath?.startsWith('anthropic:')) {
     // Load Anthropic module
     const options = providerPath.split(':');

package/src/telemetry.ts

@@ -1,5 +1,8 @@
+import chalk from 'chalk';
+
 import packageJson from '../package.json';
-import { fetchWithTimeout } from './util';
+import logger from './logger';
+import { fetchWithTimeout, maybeRecordFirstRun } from './util';
 
 type TelemetryEvent = {
   event: string;
@@ -30,6 +33,16 @@ export class Telemetry {
     }
   }
 
+  maybeShowNotice(): void {
+    if (maybeRecordFirstRun()) {
+      logger.info(
+        chalk.gray(
+          'Anonymous telemetry is enabled. For more info, see https://www.promptfoo.dev/docs/configuration/telemetry',
+        ),
+      );
+    }
+  }
+
   async send(): Promise<void> {
     if (!this.disabled && this.events.length > 0) {
       try {

package/src/types.ts

@@ -136,6 +136,7 @@ type BaseAssertionTypes =
   | 'icontains'
   | 'contains-all'
   | 'contains-any'
+  | 'starts-with'
   | 'regex'
   | 'is-json'
   | 'contains-json'
@@ -230,6 +231,9 @@ export interface TestSuiteConfig {
 
   // Path to write output. Writes to console/web viewer if not set.
   outputPath?: string;
+
+  // Determines whether or not sharing is enabled.
+  sharing?: boolean;
 }
 
 export type UnifiedConfig = TestSuiteConfig & {

package/src/updates.ts

@@ -17,7 +17,16 @@ export async function getLatestVersion(packageName: string) {
 }
 
 export async function checkForUpdates(): Promise<boolean> {
-  const latestVersion = await getLatestVersion('promptfoo');
+  if (process.env.PROMPTFOO_DISABLE_UPDATE) {
+    return false;
+  }
+
+  let latestVersion: string;
+  try {
+    latestVersion = await getLatestVersion('promptfoo');
+  } catch {
+    return false;
+  }
   if (semverGt(latestVersion, VERSION)) {
     const border = '='.repeat(process.stdout.columns - 10);
     logger.info(

package/src/util.ts

@@ -37,6 +37,46 @@ function parseJson(json: string): any | undefined {
   }
 }
 
+let globalConfigCache: any = null;
+
+export function resetGlobalConfig(): void {
+  globalConfigCache = null;
+}
+
+export function readGlobalConfig(): any {
+  if (!globalConfigCache) {
+    const configDir = getConfigDirectoryPath();
+    const configFilePath = path.join(configDir, 'promptfoo.yaml');
+
+    if (fs.existsSync(configFilePath)) {
+      globalConfigCache = yaml.load(fs.readFileSync(configFilePath, 'utf-8'));
+    } else {
+      if (!fs.existsSync(configDir)) {
+        fs.mkdirSync(configDir, { recursive: true });
+      }
+      globalConfigCache = { hasRun: false };
+      fs.writeFileSync(configFilePath, yaml.dump(globalConfigCache));
+    }
+  }
+
+  return globalConfigCache;
+}
+
+export function maybeRecordFirstRun(): boolean {
+  // Return true if first run
+  try {
+    const config = readGlobalConfig();
+    if (!config.hasRun) {
+      config.hasRun = true;
+      fs.writeFileSync(path.join(getConfigDirectoryPath(), 'promptfoo.yaml'), yaml.dump(config));
+      return true;
+    }
+    return false;
+  } catch (err) {
+    return false;
+  }
+}
+
 export async function maybeReadConfig(configPath: string): Promise<UnifiedConfig | undefined> {
   if (!fs.existsSync(configPath)) {
     return undefined;