promptfoo 0.17.4 → 0.17.5

package/src/main.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
 import { join as pathJoin, dirname } from 'path';
+import readline from 'readline';
 
 import chalk from 'chalk';
 import { Command } from 'commander';
@@ -56,7 +57,11 @@ function createDummyFiles(directory: string | null) {
   writeFileSync(pathJoin(process.cwd(), directory, 'README.md'), DEFAULT_README);
 
   if (directory === '.') {
-    logger.info(chalk.green.bold('Wrote prompts.txt and promptfooconfig.yaml. Open README.md to get started!'));
+    logger.info(
+      chalk.green.bold(
+        'Wrote prompts.txt and promptfooconfig.yaml. Open README.md to get started!',
+      ),
+    );
   } else {
     logger.info(chalk.green.bold(`Wrote prompts.txt and promptfooconfig.yaml to ./${directory}`));
     logger.info(chalk.green(`\`cd ${directory}\` and open README.md to get started!`));
@@ -72,20 +77,20 @@ async function main() {
     pathJoin(pwd, 'promptfooconfig.json'),
     pathJoin(pwd, 'promptfooconfig.yaml'),
   ];
-  let config: Partial<UnifiedConfig> = {};
+  let defaultConfig: Partial<UnifiedConfig> = {};
   for (const path of potentialPaths) {
     const maybeConfig = await maybeReadConfig(path);
     if (maybeConfig) {
-      config = maybeConfig;
+      defaultConfig = maybeConfig;
       break;
     }
   }
 
   let evaluateOptions: EvaluateOptions = {};
-  if (config.evaluateOptions) {
-    evaluateOptions.generateSuggestions = config.evaluateOptions.generateSuggestions;
-    evaluateOptions.maxConcurrency = config.evaluateOptions.maxConcurrency;
-    evaluateOptions.showProgressBar = config.evaluateOptions.showProgressBar;
+  if (defaultConfig.evaluateOptions) {
+    evaluateOptions.generateSuggestions = defaultConfig.evaluateOptions.generateSuggestions;
+    evaluateOptions.maxConcurrency = defaultConfig.evaluateOptions.maxConcurrency;
+    evaluateOptions.showProgressBar = defaultConfig.evaluateOptions.showProgressBar;
   }
 
   const program = new Command();
@@ -126,20 +131,46 @@ async function main() {
   program
     .command('share')
     .description('Share your most recent result')
-    .action(async (cmdObj: { port: number } & Command) => {
+    .option('-y, --yes', 'Skip confirmation')
+    .action(async (cmdObj: { yes: boolean } & Command) => {
       telemetry.maybeShowNotice();
       telemetry.record('command_used', {
         name: 'share',
       });
       await telemetry.send();
 
-      const latestResults = readLatestResults();
-      if (!latestResults) {
-        logger.error('Could not load results. Do you need to run `promptfoo eval` first?');
-        process.exit(1);
+      const createPublicUrl = async () => {
+        const latestResults = readLatestResults();
+        if (!latestResults) {
+          logger.error('Could not load results. Do you need to run `promptfoo eval` first?');
+          process.exit(1);
+        }
+        const url = await createShareableUrl(latestResults.results, latestResults.config);
+        logger.info(`View results: ${chalk.greenBright.bold(url)}`);
+      };
+
+      if (cmdObj.yes || process.env.PROMPTFOO_DISABLE_SHARE_WARNING) {
+        createPublicUrl();
+      } else {
+        const reader = readline.createInterface({
+          input: process.stdin,
+          output: process.stdout,
+        });
+
+        reader.question(
+          'Are you sure you want to create a public URL? [y/N] ',
+          async function (answer: string) {
+            if (answer.toLowerCase() !== 'yes' && answer.toLowerCase() !== 'y') {
+              logger.info('Did not create a public URL.');
+              reader.close();
+              return;
+            }
+            reader.close();
+
+            createPublicUrl();
+          },
+        );
       }
-      const url = await createShareableUrl(latestResults.results, latestResults.config);
-      logger.info(`View results: ${chalk.greenBright.bold(url)}`);
     });
 
   program
@@ -159,28 +190,32 @@ async function main() {
   program
     .command('eval')
     .description('Evaluate prompts')
-    .option('-p, --prompts <paths...>', 'Paths to prompt files (.txt)', config.prompts)
+    .option('-p, --prompts <paths...>', 'Paths to prompt files (.txt)')
     .option(
       '-r, --providers <name or path...>',
       'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module',
     )
     .option(
       '-c, --config <path>',
-      'Path to configuration file. Automatically loads promptfooconfig.js/json/yaml',
+      'Path to configuration file. Automatically loads promptfoodefaultConfig.js/json/yaml',
     )
     .option(
       // TODO(ian): Remove `vars` for v1
       '-v, --vars, -t, --tests <path>',
       'Path to CSV with test cases',
-      config?.commandLineOptions?.vars,
+      defaultConfig?.commandLineOptions?.vars,
+    )
+    .option('-t, --tests <path>', 'Path to CSV with test cases')
+    .option(
+      '-o, --output <path>',
+      'Path to output file (csv, json, yaml, html)',
+      defaultConfig.outputPath,
     )
-    .option('-t, --tests <path>', 'Path to CSV with test cases', config?.commandLineOptions?.tests)
-    .option('-o, --output <path>', 'Path to output file (csv, json, yaml, html)', config.outputPath)
     .option(
       '-j, --max-concurrency <number>',
       'Maximum number of concurrent API calls',
-      config.evaluateOptions?.maxConcurrency
-        ? String(config.evaluateOptions.maxConcurrency)
+      defaultConfig.evaluateOptions?.maxConcurrency
+        ? String(defaultConfig.evaluateOptions.maxConcurrency)
         : undefined,
     )
     .option(
@@ -195,28 +230,28 @@ async function main() {
     .option(
       '--prompt-prefix <path>',
       'This prefix is prepended to every prompt',
-      config.defaultTest?.options?.prefix,
+      defaultConfig.defaultTest?.options?.prefix,
     )
     .option(
       '--prompt-suffix <path>',
       'This suffix is append to every prompt',
-      config.defaultTest?.options?.suffix,
+      defaultConfig.defaultTest?.options?.suffix,
     )
     .option(
       '--no-write',
       'Do not write results to promptfoo directory',
-      config?.commandLineOptions?.write,
+      defaultConfig?.commandLineOptions?.write,
     )
     .option(
       '--no-cache',
       'Do not read or write results to disk cache',
-      config?.commandLineOptions?.cache,
+      defaultConfig?.commandLineOptions?.cache,
     )
     .option('--no-progress-bar', 'Do not show progress bar')
-    .option('--no-table', 'Do not output table in CLI', config?.commandLineOptions?.table)
-    .option('--share', 'Create a shareable URL', config?.commandLineOptions?.share)
-    .option('--grader', 'Model that will grade outputs', config?.commandLineOptions?.grader)
-    .option('--verbose', 'Show debug logs', config?.commandLineOptions?.verbose)
+    .option('--no-table', 'Do not output table in CLI', defaultConfig?.commandLineOptions?.table)
+    .option('--share', 'Create a shareable URL', defaultConfig?.commandLineOptions?.share)
+    .option('--grader', 'Model that will grade outputs', defaultConfig?.commandLineOptions?.grader)
+    .option('--verbose', 'Show debug logs', defaultConfig?.commandLineOptions?.verbose)
     .option('--view [port]', 'View in browser ui')
     .action(async (cmdObj: CommandLineOptions & Command) => {
       // Misc settings
@@ -229,15 +264,20 @@ async function main() {
 
       // Config parsing
       const maxConcurrency = parseInt(cmdObj.maxConcurrency || '', 10);
+      let fileConfig: Partial<UnifiedConfig> = {};
       const configPath = cmdObj.config;
       if (configPath) {
-        config = await readConfig(configPath);
+        fileConfig = await readConfig(configPath);
       }
-      config = {
-        prompts: cmdObj.prompts || config.prompts,
-        providers: cmdObj.providers || config.providers,
-        tests: cmdObj.tests || cmdObj.vars || config.tests,
-        defaultTest: config.defaultTest,
+      const config: Partial<UnifiedConfig> = {
+        prompts: cmdObj.prompts || fileConfig.prompts || defaultConfig.prompts,
+        providers: cmdObj.providers || fileConfig.providers || defaultConfig.providers,
+        tests: cmdObj.tests || cmdObj.vars || fileConfig.tests || defaultConfig.tests,
+        sharing:
+          process.env.PROMPTFOO_DISABLE_SHARING === '1'
+            ? false
+            : fileConfig.sharing ?? defaultConfig.sharing ?? true,
+        defaultTest: fileConfig.defaultTest,
       };
 
       // Validation
@@ -305,7 +345,8 @@ async function main() {
 
       const summary = await evaluate(testSuite, options);
 
-      const shareableUrl = cmdObj.share ? await createShareableUrl(summary, config) : null;
+      const shareableUrl =
+        cmdObj.share && config.sharing ? await createShareableUrl(summary, config) : null;
 
       if (cmdObj.output) {
         logger.info(chalk.yellow(`Writing output to ${cmdObj.output}`));

package/src/providers/anthropic.ts

@@ -25,7 +25,7 @@ export class AnthropicCompletionProvider implements ApiProvider {
   constructor(modelName: string, apiKey?: string, context?: AnthropicCompletionOptions) {
     this.modelName = modelName;
     this.apiKey = apiKey || process.env.ANTHROPIC_API_KEY;
-    this.anthropic = new Anthropic({apiKey: this.apiKey});
+    this.anthropic = new Anthropic({ apiKey: this.apiKey });
     this.options = context || {};
   }
 

package/src/providers/azureopenai.ts

@@ -0,0 +1,280 @@
+import logger from '../logger';
+import { fetchJsonWithCache } from '../cache';
+import { REQUEST_TIMEOUT_MS } from './shared';
+
+import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '../types.js';
+
+interface AzureOpenAiCompletionOptions {
+  temperature?: number;
+  functions?: {
+    name: string;
+    description?: string;
+    parameters: any;
+  }[];
+  function_call?: 'none' | 'auto';
+}
+
+class AzureOpenAiGenericProvider implements ApiProvider {
+  deploymentName: string;
+  apiKey?: string;
+  apiHost?: string;
+
+  constructor(deploymentName: string, apiKey?: string) {
+    this.deploymentName = deploymentName;
+
+    this.apiKey = apiKey || process.env.AZURE_OPENAI_API_KEY;
+
+    this.apiHost = process.env.AZURE_OPENAI_API_HOST;
+  }
+
+  id(): string {
+    return `azureopenai:${this.deploymentName}`;
+  }
+
+  toString(): string {
+    return `[Azure OpenAI Provider ${this.deploymentName}]`;
+  }
+
+  // @ts-ignore: Prompt is not used in this implementation
+  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
+    throw new Error('Not implemented');
+  }
+}
+
+export class AzureOpenAiEmbeddingProvider extends AzureOpenAiGenericProvider {
+  async callEmbeddingApi(text: string): Promise<ProviderEmbeddingResponse> {
+    if (!this.apiKey) {
+      throw new Error('Azure OpenAI API key must be set for similarity comparison');
+    }
+    if (!this.apiHost) {
+      throw new Error('Azure OpenAI API host must be set');
+    }
+
+    const body = {
+      input: text,
+      model: this.deploymentName,
+    };
+    let data,
+      cached = false;
+    try {
+      ({ data, cached } = (await fetchJsonWithCache(
+        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/embeddings?api-version=2023-07-01-preview`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'api-key': this.apiKey,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      )) as unknown as any);
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+        tokenUsage: {
+          total: 0,
+          prompt: 0,
+          completion: 0,
+        },
+      };
+    }
+    logger.debug(`\tAzure OpenAI API response (embeddings): ${JSON.stringify(data)}`);
+
+    try {
+      const embedding = data?.data?.[0]?.embedding;
+      if (!embedding) {
+        throw new Error('No embedding returned');
+      }
+      const ret = {
+        embedding,
+        tokenUsage: cached
+          ? { cached: data.usage.total_tokens }
+          : {
+              total: data.usage.total_tokens,
+              prompt: data.usage.prompt_tokens,
+              completion: data.usage.completion_tokens,
+            },
+      };
+      return ret;
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+        tokenUsage: {
+          total: data?.usage?.total_tokens,
+          prompt: data?.usage?.prompt_tokens,
+          completion: data?.usage?.completion_tokens,
+        },
+      };
+    }
+  }
+}
+
+export class AzureOpenAiCompletionProvider extends AzureOpenAiGenericProvider {
+  options: AzureOpenAiCompletionOptions;
+
+  constructor(deploymentName: string, apiKey?: string, context?: AzureOpenAiCompletionOptions) {
+    super(deploymentName, apiKey);
+    this.options = context || {};
+  }
+
+  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
+    if (!this.apiKey) {
+      throw new Error(
+        'Azure OpenAI API key is not set. Set AZURE_OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
+      );
+    }
+    if (!this.apiHost) {
+      throw new Error('Azure OpenAI API host must be set');
+    }
+
+    let stop: string;
+    try {
+      stop = process.env.OPENAI_STOP
+        ? JSON.parse(process.env.OPENAI_STOP)
+        : ['<|im_end|>', '<|endoftext|>'];
+    } catch (err) {
+      throw new Error(`OPENAI_STOP is not a valid JSON string: ${err}`);
+    }
+    const body = {
+      model: this.deploymentName,
+      prompt,
+      max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      temperature:
+        options?.temperature ??
+        this.options.temperature ??
+        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      stop,
+    };
+    logger.debug(`Calling Azure OpenAI API: ${JSON.stringify(body)}`);
+    let data,
+      cached = false;
+    try {
+      ({ data, cached } = (await fetchJsonWithCache(
+        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/completions?api-version=2023-07-01-preview`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'api-key': this.apiKey,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      )) as unknown as any);
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+      };
+    }
+    logger.debug(`\tAzure OpenAI API response: ${JSON.stringify(data)}`);
+    try {
+      return {
+        output: data.choices[0].text,
+        tokenUsage: cached
+          ? { cached: data.usage.total_tokens }
+          : {
+              total: data.usage.total_tokens,
+              prompt: data.usage.prompt_tokens,
+              completion: data.usage.completion_tokens,
+            },
+      };
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+      };
+    }
+  }
+}
+
+export class AzureOpenAiChatCompletionProvider extends AzureOpenAiGenericProvider {
+  options: AzureOpenAiCompletionOptions;
+
+  constructor(deploymentName: string, apiKey?: string, context?: AzureOpenAiCompletionOptions) {
+    super(deploymentName, apiKey);
+    this.options = context || {};
+  }
+
+  async callApi(prompt: string, options?: AzureOpenAiCompletionOptions): Promise<ProviderResponse> {
+    if (!this.apiKey) {
+      throw new Error(
+        'Azure OpenAI API key is not set. Set AZURE_OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
+      );
+    }
+    if (!this.apiHost) {
+      throw new Error('Azure OpenAI API host must be set');
+    }
+
+    let messages: { role: string; content: string; name?: string }[];
+    try {
+      messages = JSON.parse(prompt) as { role: string; content: string }[];
+    } catch (err) {
+      const trimmedPrompt = prompt.trim();
+      if (
+        process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
+        trimmedPrompt.startsWith('{') ||
+        trimmedPrompt.startsWith('[')
+      ) {
+        throw new Error(
+          `Azure OpenAI Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`,
+        );
+      }
+      messages = [{ role: 'user', content: prompt }];
+    }
+
+    const body = {
+      model: this.deploymentName,
+      messages: messages,
+      max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      temperature:
+        options?.temperature ??
+        this.options.temperature ??
+        parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      functions: options?.functions || this.options.functions || undefined,
+      function_call: options?.function_call || this.options.function_call || undefined,
+    };
+    logger.debug(`Calling Azure OpenAI API: ${JSON.stringify(body)}`);
+
+    let data,
+      cached = false;
+    try {
+      ({ data, cached } = (await fetchJsonWithCache(
+        `https://${this.apiHost}/openai/deployments/${this.deploymentName}/chat/completions?api-version=2023-07-01-preview`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'api-key': this.apiKey,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      )) as unknown as any);
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+      };
+    }
+
+    logger.debug(`\tAzure OpenAI API response: ${JSON.stringify(data)}`);
+    try {
+      const message = data.choices[0].message;
+      const output =
+        message.content === null ? JSON.stringify(message.function_call) : message.content;
+      return {
+        output,
+        tokenUsage: cached
+          ? { cached: data.usage.total_tokens }
+          : {
+              total: data.usage.total_tokens,
+              prompt: data.usage.prompt_tokens,
+              completion: data.usage.completion_tokens,
+            },
+      };
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+      };
+    }
+  }
+}

package/src/providers/openai.ts

@@ -1,3 +1,5 @@
+import yaml from 'js-yaml';
+
 import logger from '../logger';
 import { fetchJsonWithCache } from '../cache';
 import { REQUEST_TIMEOUT_MS } from './shared';
@@ -226,20 +228,34 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     }
 
     let messages: { role: string; content: string; name?: string }[];
-    try {
-      messages = JSON.parse(prompt) as { role: string; content: string }[];
-    } catch (err) {
-      const trimmedPrompt = prompt.trim();
-      if (
-        process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
-        trimmedPrompt.startsWith('{') ||
-        trimmedPrompt.startsWith('[')
-      ) {
+    const trimmedPrompt = prompt.trim();
+    if (trimmedPrompt.startsWith('- role:')) {
+      try {
+        // Try YAML
+        messages = yaml.load(prompt) as { role: string; content: string }[];
+      } catch (err) {
         throw new Error(
-          `OpenAI Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`,
+          `OpenAI Chat Completion prompt is not a valid YAML string: ${err}\n\n${prompt}`,
         );
       }
-      messages = [{ role: 'user', content: prompt }];
+    } else {
+      try {
+        // Try JSON
+        messages = JSON.parse(prompt) as { role: string; content: string }[];
+      } catch (err) {
+        if (
+          process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
+          trimmedPrompt.startsWith('{') ||
+          trimmedPrompt.startsWith('[')
+        ) {
+          throw new Error(
+            `OpenAI Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`,
+          );
+        }
+
+        // Fall back to wrapping the prompt in a user message
+        messages = [{ role: 'user', content: prompt }];
+      }
     }
 
     const body = {

package/src/providers.ts

@@ -6,6 +6,10 @@ import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './provid
 import { AnthropicCompletionProvider } from './providers/anthropic';
 import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai';
 import { ScriptCompletionProvider } from './providers/scriptCompletion';
+import {
+  AzureOpenAiChatCompletionProvider,
+  AzureOpenAiCompletionProvider,
+} from './providers/azureopenai';
 
 export async function loadApiProviders(
   providerPaths: ProviderId | ProviderId[] | RawProviderConfig[],
@@ -68,6 +72,21 @@ export async function loadApiProvider(
         `Unknown OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
       );
     }
+  } else if (providerPath?.startsWith('azureopenai:')) {
+    // Load Azure OpenAI module
+    const options = providerPath.split(':');
+    const modelType = options[1];
+    const deploymentName = options[2];
+
+    if (modelType === 'chat') {
+      return new AzureOpenAiChatCompletionProvider(deploymentName, undefined, context?.config);
+    } else if (modelType === 'completion') {
+      return new AzureOpenAiCompletionProvider(deploymentName, undefined, context?.config);
+    } else {
+      throw new Error(
+        `Unknown Azure OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
+      );
+    }
   } else if (providerPath?.startsWith('anthropic:')) {
     // Load Anthropic module
     const options = providerPath.split(':');

package/src/types.ts

@@ -136,6 +136,7 @@ type BaseAssertionTypes =
   | 'icontains'
   | 'contains-all'
   | 'contains-any'
+  | 'starts-with'
   | 'regex'
   | 'is-json'
   | 'contains-json'
@@ -230,6 +231,9 @@ export interface TestSuiteConfig {
 
   // Path to write output. Writes to console/web viewer if not set.
   outputPath?: string;
+
+  // Determines whether or not sharing is enabled.
+  sharing?: boolean;
 }
 
 export type UnifiedConfig = TestSuiteConfig & {

package/src/updates.ts

@@ -17,6 +17,10 @@ export async function getLatestVersion(packageName: string) {
 }
 
 export async function checkForUpdates(): Promise<boolean> {
+  if (process.env.PROMPTFOO_DISABLE_UPDATE) {
+    return false;
+  }
+
   let latestVersion: string;
   try {
     latestVersion = await getLatestVersion('promptfoo');

package/src/web/client/src/ResultsView.tsx

@@ -218,16 +218,18 @@ export default function ResultsView() {
                   </Button>
                 </Tooltip>
               )}
-              <Tooltip title="Generate a unique URL that others can access">
-                <Button
-                  color="primary"
-                  onClick={handleShareButtonClick}
-                  disabled={shareLoading}
-                  startIcon={shareLoading ? <CircularProgress size={16} /> : <ShareIcon />}
-                >
-                  Share
-                </Button>
-              </Tooltip>
+              {config?.sharing && (
+                <Tooltip title="Generate a unique URL that others can access">
+                  <Button
+                    color="primary"
+                    onClick={handleShareButtonClick}
+                    disabled={shareLoading}
+                    startIcon={shareLoading ? <CircularProgress size={16} /> : <ShareIcon />}
+                  >
+                    Share
+                  </Button>
+                </Tooltip>
+              )}
             </ResponsiveStack>
           </Box>
         </ResponsiveStack>