promptfoo 0.9.0 → 0.11.0

package/src/main.ts

@@ -6,9 +6,9 @@ import Table from 'cli-table3';
 import chalk from 'chalk';
 import { Command } from 'commander';
 
-import logger, { setLogLevel } from './logger.js';
-import { loadApiProvider, loadApiProviders } from './providers.js';
-import { evaluate } from './evaluator.js';
+import logger, { setLogLevel } from './logger';
+import { loadApiProvider, loadApiProviders } from './providers';
+import { evaluate } from './evaluator';
 import {
   maybeReadConfig,
   readConfig,
@@ -16,10 +16,10 @@ import {
   readTests,
   writeLatestResults,
   writeOutput,
-} from './util.js';
-import { getDirectory } from './esm.js';
-import { init } from './web/server.js';
-import { disableCache } from './cache.js';
+} from './util';
+import { getDirectory } from './esm';
+import { init } from './web/server';
+import { disableCache } from './cache';
 
 import type {
   CommandLineOptions,
@@ -27,8 +27,8 @@ import type {
   TestCase,
   TestSuite,
   UnifiedConfig,
-} from './types.js';
-import { DEFAULT_README, DEFAULT_YAML_CONFIG, DEFAULT_PROMPTS } from './onboarding.js';
+} from './types';
+import { DEFAULT_README, DEFAULT_YAML_CONFIG, DEFAULT_PROMPTS } from './onboarding';
 
 function createDummyFiles(directory: string | null) {
   if (directory) {
@@ -68,7 +68,7 @@ async function main() {
   ];
   let config: Partial<UnifiedConfig> = {};
   for (const path of potentialPaths) {
-    const maybeConfig = maybeReadConfig(path);
+    const maybeConfig = await maybeReadConfig(path);
     if (maybeConfig) {
       config = maybeConfig;
       break;
@@ -154,8 +154,16 @@ async function main() {
       'This suffix is append to every prompt',
       config.defaultTest?.options?.suffix,
     )
-    .option('--no-write', 'Do not write results to promptfoo directory')
-    .option('--no-cache', 'Do not read or write results to disk cache')
+    .option(
+      '--no-write',
+      'Do not write results to promptfoo directory',
+      config?.commandLineOptions?.write,
+    )
+    .option(
+      '--no-cache',
+      'Do not read or write results to disk cache',
+      config?.commandLineOptions?.cache,
+    )
     .option('--grader', 'Model that will grade outputs', config?.commandLineOptions?.grader)
     .option('--verbose', 'Show debug logs', config?.commandLineOptions?.verbose)
     .option('--view [port]', 'View in browser ui')
@@ -172,12 +180,13 @@ async function main() {
       const maxConcurrency = parseInt(cmdObj.maxConcurrency || '', 10);
       const configPath = cmdObj.config;
       if (configPath) {
-        config = readConfig(configPath);
+        config = await readConfig(configPath);
       } else {
         config = {
           prompts: cmdObj.prompts || config.prompts,
           providers: cmdObj.providers || config.providers,
           tests: cmdObj.tests || cmdObj.vars || config.tests,
+          defaultTest: config.defaultTest,
         };
       }
 
@@ -255,8 +264,9 @@ async function main() {
           },
         });
         // Skip first row (header) and add the rest. Color PASS/FAIL
-        for (const row of summary.table.body) {
+        for (const row of summary.table.body.slice(0, 25)) {
           table.push([
+            ...row.vars,
             ...row.outputs.map((col) => {
               const tableCellMaxLength = parseInt(cmdObj.tableCellMaxLength || '', 10);
               if (!isNaN(tableCellMaxLength) && col.length > tableCellMaxLength) {
@@ -274,11 +284,14 @@ async function main() {
               }
               return col;
             }),
-            ...row.vars,
           ]);
         }
 
         logger.info('\n' + table.toString());
+        if (summary.table.body.length > 25) {
+          const rowsLeft = summary.table.body.length - 25;
+          logger.info(`... ${rowsLeft} more row${rowsLeft === 1 ? '' : 's'} not shown ...\n`);
+        }
       }
       if (cmdObj.view || !cmdObj.write) {
         logger.info('Evaluation complete');

package/src/providers/localai.ts

@@ -1,6 +1,6 @@
-import logger from '../logger.js';
-import { fetchJsonWithCache } from '../cache.js';
-import { REQUEST_TIMEOUT_MS } from './shared.js';
+import logger from '../logger';
+import { fetchJsonWithCache } from '../cache';
+import { REQUEST_TIMEOUT_MS } from './shared';
 
 import type { ApiProvider, ProviderResponse } from '../types.js';
 

package/src/providers/openai.ts

@@ -1,6 +1,6 @@
-import logger from '../logger.js';
-import { fetchJsonWithCache } from '../cache.js';
-import { REQUEST_TIMEOUT_MS } from './shared.js';
+import logger from '../logger';
+import { fetchJsonWithCache } from '../cache';
+import { REQUEST_TIMEOUT_MS } from './shared';
 
 import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '../types.js';
 
@@ -126,12 +126,20 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
       );
     }
 
+    let stop: string;
+    try {
+      stop = process.env.OPENAI_STOP
+        ? JSON.parse(process.env.OPENAI_STOP)
+        : ['<|im_end|>', '<|endoftext|>'];
+    } catch (err) {
+      throw new Error(`OPENAI_STOP is not a valid JSON string: ${err}`);
+    }
     const body = {
       model: this.modelName,
       prompt,
-      max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
-      temperature: options?.temperature ?? (process.env.OPENAI_MAX_TEMPERATURE || 0),
-      stop: process.env.OPENAI_STOP ? JSON.parse(process.env.OPENAI_STOP) : undefined,
+      max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      temperature: options?.temperature ?? parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      stop,
     };
     logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
     let data,
@@ -210,8 +218,8 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     const body = {
       model: this.modelName,
       messages: messages,
-      max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
-      temperature: options?.temperature ?? (process.env.OPENAI_MAX_TEMPERATURE || 0),
+      max_tokens: parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      temperature: options?.temperature ?? parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
     };
     logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
 

package/src/providers.ts

@@ -1,9 +1,9 @@
 import path from 'node:path';
 
-import { ApiProvider } from './types.js';
+import { ApiProvider } from './types';
 
-import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './providers/openai.js';
-import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai.js';
+import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './providers/openai';
+import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai';
 
 export async function loadApiProviders(providerPaths: string | string[]): Promise<ApiProvider[]> {
   if (typeof providerPaths === 'string') {

package/src/types.ts

@@ -113,13 +113,34 @@ export interface GradingResult {
   tokensUsed?: TokenUsage;
 }
 
+type BaseAssertionTypes =
+  | 'equals'
+  | 'contains'
+  | 'icontains'
+  | 'contains-all'
+  | 'contains-any'
+  | 'regex'
+  | 'is-json'
+  | 'contains-json'
+  | 'javascript'
+  | 'similar'
+  | 'llm-rubric'
+  | 'webhook'
+  | 'rouge-n'
+  | 'rouge-s'
+  | 'rouge-l';
+
+type NotPrefixed<T extends string> = `not-${T}`;
+
+export type AssertionType = BaseAssertionTypes | NotPrefixed<BaseAssertionTypes>;
+
 // TODO(ian): maybe Assertion should support {type: config} to make the yaml cleaner
 export interface Assertion {
   // Type of assertion
-  type: 'equals' | 'is-json' | 'contains-json' | 'javascript' | 'similar' | 'llm-rubric';
+  type: AssertionType;
 
   // The expected value, if applicable
-  value?: string;
+  value?: string | string[];
 
   // The threshold value, only applicable for similarity (cosine distance)
   threshold?: number;
@@ -157,7 +178,7 @@ export interface TestSuite {
   providers: ApiProvider[];
 
   // One or more prompt strings
-  prompts: string[];
+  prompts: Prompt[];
 
   // Test cases
   tests?: TestCase[];

package/src/util.ts

@@ -2,6 +2,7 @@ import * as fs from 'fs';
 import * as path from 'node:path';
 import * as os from 'node:os';
 
+import $RefParser from '@apidevtools/json-schema-ref-parser';
 import fetch from 'node-fetch';
 import yaml from 'js-yaml';
 import nunjucks from 'nunjucks';
@@ -10,13 +11,13 @@ import { parse as parsePath } from 'path';
 import { parse as parseCsv } from 'csv-parse/sync';
 import { stringify } from 'csv-stringify/sync';
 
-import logger from './logger.js';
-import { getDirectory } from './esm.js';
+import logger from './logger';
+import { getDirectory } from './esm';
 
 import type { RequestInfo, RequestInit, Response } from 'node-fetch';
 
-import type { Assertion, CsvRow, EvaluateSummary, UnifiedConfig, TestCase } from './types.js';
-import { assertionFromString } from './assertions.js';
+import type { Assertion, CsvRow, EvaluateSummary, UnifiedConfig, TestCase, Prompt } from './types';
+import { assertionFromString } from './assertions';
 
 const PROMPT_DELIMITER = '---';
 
@@ -28,14 +29,14 @@ function parseJson(json: string): any | undefined {
   }
 }
 
-export function maybeReadConfig(configPath: string): UnifiedConfig | undefined {
+export async function maybeReadConfig(configPath: string): Promise<UnifiedConfig | undefined> {
   if (!fs.existsSync(configPath)) {
     return undefined;
   }
   return readConfig(configPath);
 }
 
-export function readConfig(configPath: string): UnifiedConfig {
+export async function readConfig(configPath: string): Promise<UnifiedConfig> {
   const ext = path.parse(configPath).ext;
   switch (ext) {
     case '.json':
@@ -44,17 +45,38 @@ export function readConfig(configPath: string): UnifiedConfig {
     case '.js':
       return require(configPath) as UnifiedConfig;
     case '.yaml':
-      return yaml.load(fs.readFileSync(configPath, 'utf-8')) as UnifiedConfig;
+    case '.yml':
+      let ret = yaml.load(fs.readFileSync(configPath, 'utf-8')) as UnifiedConfig;
+      ret = (await $RefParser.dereference(ret)) as UnifiedConfig;
+      return ret;
     default:
       throw new Error(`Unsupported configuration file format: ${ext}`);
   }
 }
 
-export function readPrompts(promptPathsOrGlobs: string | string[]): string[] {
-  promptPathsOrGlobs =
-    typeof promptPathsOrGlobs === 'string' ? [promptPathsOrGlobs] : promptPathsOrGlobs;
-  const promptPaths = promptPathsOrGlobs.flatMap((pathOrGlob) => globSync(pathOrGlob));
-  let promptContents: string[] = [];
+enum PromptInputType {
+  STRING = 1,
+  ARRAY = 2,
+  NAMED = 3,
+}
+
+export function readPrompts(
+  promptPathOrGlobs: string | string[] | Record<string, string>,
+): Prompt[] {
+  let promptPaths: string[] = [];
+  let promptContents: Prompt[] = [];
+
+  let inputType: PromptInputType | undefined;
+  if (typeof promptPathOrGlobs === 'string') {
+    promptPaths = [promptPathOrGlobs];
+    inputType = PromptInputType.STRING;
+  } else if (Array.isArray(promptPathOrGlobs)) {
+    promptPaths = promptPathOrGlobs.flatMap((pathOrGlob) => globSync(pathOrGlob));
+    inputType = PromptInputType.ARRAY;
+  } else if (typeof promptPathOrGlobs === 'object') {
+    promptPaths = Object.keys(promptPathOrGlobs);
+    inputType = PromptInputType.NAMED;
+  }
 
   for (const promptPath of promptPaths) {
     const stat = fs.statSync(promptPath);
@@ -63,18 +85,27 @@ export function readPrompts(promptPathsOrGlobs: string | string[]): string[] {
       const fileContents = filesInDirectory.map((fileName) =>
         fs.readFileSync(path.join(promptPath, fileName), 'utf-8'),
       );
-      promptContents.push(...fileContents);
+      promptContents.push(...fileContents.map((content) => ({ raw: content, display: content })));
     } else {
       const fileContent = fs.readFileSync(promptPath, 'utf-8');
-      promptContents.push(fileContent);
+      let display;
+      if (inputType === PromptInputType.NAMED) {
+        display = (promptPathOrGlobs as Record<string, string>)[promptPath];
+      } else {
+        display = fileContent.length > 200 ? promptPath : fileContent;
+      }
+      promptContents.push({ raw: fileContent, display });
     }
   }
 
-  if (promptContents.length === 1) {
-    promptContents = promptContents[0].split(PROMPT_DELIMITER).map((p) => p.trim());
+  if (promptContents.length === 1 && inputType !== PromptInputType.NAMED) {
+    const content = promptContents[0].raw;
+    promptContents = content
+      .split(PROMPT_DELIMITER)
+      .map((p) => ({ raw: p.trim(), display: p.trim() }));
   }
   if (promptContents.length === 0) {
-    throw new Error(`There are no prompts in ${promptPathsOrGlobs.join(', ')}`);
+    throw new Error(`There are no prompts in ${JSON.stringify(promptPathOrGlobs)}`);
   }
   return promptContents;
 }