promptfoo 0.18.1 → 0.18.3

package/src/types.ts

@@ -28,7 +28,7 @@ export interface CommandLineOptions {
 }
 
 export interface ProviderConfig {
-  id: ProviderId;
+  id?: ProviderId;
   config?: any;
   prompts?: string[]; // List of prompt display strings
 }
@@ -97,6 +97,7 @@ export interface EvaluateResult {
   success: boolean;
   score: number;
   latencyMs: number;
+  gradingResult?: GradingResult;
 }
 
 export interface EvaluateTableOutput {
@@ -106,6 +107,7 @@ export interface EvaluateTableOutput {
   prompt: string;
   latencyMs: number;
   tokenUsage?: Partial<TokenUsage>;
+  gradingResult?: GradingResult;
 }
 
 export interface EvaluateTable {
@@ -138,6 +140,8 @@ export interface GradingResult {
   score: number;
   reason: string;
   tokensUsed?: TokenUsage;
+  componentResults?: GradingResult[];
+  assertion: Assertion | null;
 }
 
 type BaseAssertionTypes =
@@ -244,7 +248,7 @@ export type ProviderId = string;
 
 export type ProviderFunction = (prompt: string) => Promise<ProviderResponse>;
 
-export type RawProviderConfig = Record<ProviderId, Omit<ProviderConfig, 'id'>>;
+export type RawProviderConfig = Record<ProviderId, ProviderConfig>;
 
 // TestSuiteConfig = Test Suite, but before everything is parsed and resolved.  Providers are just strings, prompts are filepaths, tests can be filepath or inline.
 export interface TestSuiteConfig {

package/src/util.ts

@@ -4,7 +4,6 @@ import * as os from 'os';
 
 import $RefParser from '@apidevtools/json-schema-ref-parser';
 import fetch from 'node-fetch';
-import invariant from 'tiny-invariant';
 import yaml from 'js-yaml';
 import nunjucks from 'nunjucks';
 import { globSync } from 'glob';
@@ -13,7 +12,6 @@ import { parse as parseCsv } from 'csv-parse/sync';
 import { stringify } from 'csv-stringify/sync';
 
 import logger from './logger';
-import { assertionFromString } from './assertions';
 import { getDirectory } from './esm';
 
 import type { RequestInfo, RequestInit, Response } from 'node-fetch';
@@ -45,20 +43,21 @@ export function readProviderPromptMap(
     allPrompts.push(prompt.display);
   }
 
-  invariant(
-    typeof config.providers !== 'string',
-    'In order to use a provider-prompt map, config.providers should be an array of objects, not a string',
-  );
-  invariant(
-    typeof config.providers !== 'function',
-    'In order to use a provider-prompt map, config.providers should be an array of objects, not a function',
-  );
+  if (typeof config.providers === 'string') {
+    return { [config.providers]: allPrompts };
+  }
+
+  if (typeof config.providers === 'function') {
+    return { 'Custom function': allPrompts };
+  }
 
   for (const provider of config.providers) {
     if (typeof provider === 'object') {
       const rawProvider = provider as RawProviderConfig;
-      const id = Object.keys(rawProvider)[0];
-      ret[id] = rawProvider[id].prompts || allPrompts;
+      const originalId = Object.keys(rawProvider)[0];
+      const providerObject = rawProvider[originalId];
+      const id = providerObject.id || originalId;
+      ret[id] = rawProvider[originalId].prompts || allPrompts;
     }
   }
 
@@ -368,7 +367,7 @@ export function writeOutput(
       [...results.table.head.prompts, ...results.table.head.vars],
       ...results.table.body.map((row) => [...row.outputs.map(outputToSimpleString), ...row.vars]),
     ];
-    const htmlOutput = nunjucks.renderString(template, {
+    const htmlOutput = getNunjucksEngine().renderString(template, {
       table,
       results: results.results,
     });
@@ -456,10 +455,12 @@ export function writeLatestResults(results: EvaluateSummary, config: Partial<Uni
         2,
       ),
     );
-    if (fs.existsSync(latestResultsPath) && fs.lstatSync(latestResultsPath).isSymbolicLink()) {
+
+    try {
       fs.unlinkSync(latestResultsPath);
-    }
+    } catch {}
     fs.symlinkSync(newResultsPath, latestResultsPath);
+
     cleanupOldResults();
   } catch (err) {
     logger.error(`Failed to write latest results to ${newResultsPath}:\n${err}`);
@@ -501,8 +502,7 @@ export function readResult(
 export function readLatestResults():
   | { results: EvaluateSummary; config: Partial<UnifiedConfig> }
   | undefined {
-  const latestResultsPath = getLatestResultsPath();
-  return readResult(latestResultsPath);
+  return JSON.parse(fs.readFileSync(getLatestResultsPath(), 'utf-8'));
 }
 
 export function cosineSimilarity(vecA: number[], vecB: number[]) {
@@ -521,6 +521,7 @@ export function testCaseFromCsvRow(row: CsvRow): TestCase {
   for (const [key, value] of Object.entries(row)) {
     if (key === '__expected') {
       if (value.trim() !== '') {
+        const { assertionFromString } = require('./assertions');
         asserts.push(assertionFromString(value));
       }
     } else {
@@ -533,3 +534,10 @@ export function testCaseFromCsvRow(row: CsvRow): TestCase {
     assert: asserts,
   };
 }
+
+export function getNunjucksEngine() {
+  nunjucks.configure({
+    autoescape: false,
+  });
+  return nunjucks;
+}