promptfoo 0.19.1 → 0.19.3

package/src/matchers.ts

@@ -0,0 +1,120 @@
+import { DefaultEmbeddingProvider, DefaultGradingProvider } from './providers/openai';
+import { cosineSimilarity, getNunjucksEngine } from './util';
+import { loadApiProvider } from './providers';
+import { DEFAULT_GRADING_PROMPT } from './prompts';
+
+import type { GradingConfig, GradingResult } from './types';
+
+const nunjucks = getNunjucksEngine();
+
+export async function matchesSimilarity(
+  expected: string,
+  output: string,
+  threshold: number,
+  inverse: boolean = false,
+): Promise<Omit<GradingResult, 'assertion'>> {
+  const expectedEmbedding = await DefaultEmbeddingProvider.callEmbeddingApi(expected);
+  const outputEmbedding = await DefaultEmbeddingProvider.callEmbeddingApi(output);
+
+  const tokensUsed = {
+    total: (expectedEmbedding.tokenUsage?.total || 0) + (outputEmbedding.tokenUsage?.total || 0),
+    prompt: (expectedEmbedding.tokenUsage?.prompt || 0) + (outputEmbedding.tokenUsage?.prompt || 0),
+    completion:
+      (expectedEmbedding.tokenUsage?.completion || 0) +
+      (outputEmbedding.tokenUsage?.completion || 0),
+  };
+
+  if (expectedEmbedding.error || outputEmbedding.error) {
+    return {
+      pass: false,
+      score: 0,
+      reason:
+        expectedEmbedding.error || outputEmbedding.error || 'Unknown error fetching embeddings',
+      tokensUsed,
+    };
+  }
+
+  if (!expectedEmbedding.embedding || !outputEmbedding.embedding) {
+    return {
+      pass: false,
+      score: 0,
+      reason: 'Embedding not found',
+      tokensUsed,
+    };
+  }
+
+  const similarity = cosineSimilarity(expectedEmbedding.embedding, outputEmbedding.embedding);
+  const pass = inverse ? similarity <= threshold : similarity >= threshold;
+  const greaterThanReason = `Similarity ${similarity} is greater than threshold ${threshold}`;
+  const lessThanReason = `Similarity ${similarity} is less than threshold ${threshold}`;
+  if (pass) {
+    return {
+      pass: true,
+      score: inverse ? 1 - similarity : similarity,
+      reason: inverse ? lessThanReason : greaterThanReason,
+      tokensUsed,
+    };
+  }
+  return {
+    pass: false,
+    score: inverse ? 1 - similarity : similarity,
+    reason: inverse ? greaterThanReason : lessThanReason,
+    tokensUsed,
+  };
+}
+
+export async function matchesLlmRubric(
+  expected: string,
+  output: string,
+  grading?: GradingConfig,
+): Promise<Omit<GradingResult, 'assertion'>> {
+  if (!grading) {
+    throw new Error(
+      'Cannot grade output without grading config. Specify --grader option or grading config.',
+    );
+  }
+
+  const prompt = nunjucks.renderString(grading.rubricPrompt || DEFAULT_GRADING_PROMPT, {
+    output: output.replace(/\n/g, '\\n').replace(/"/g, '\\"'),
+    rubric: expected.replace(/\n/g, '\\n').replace(/"/g, '\\"'),
+  });
+
+  let provider = grading.provider || DefaultGradingProvider;
+  if (typeof provider === 'string') {
+    provider = await loadApiProvider(provider);
+  }
+  const resp = await provider.callApi(prompt);
+  if (resp.error || !resp.output) {
+    return {
+      pass: false,
+      score: 0,
+      reason: resp.error || 'No output',
+      tokensUsed: {
+        total: resp.tokenUsage?.total || 0,
+        prompt: resp.tokenUsage?.prompt || 0,
+        completion: resp.tokenUsage?.completion || 0,
+      },
+    };
+  }
+
+  try {
+    const parsed = JSON.parse(resp.output) as Omit<GradingResult, 'score'>;
+    parsed.tokensUsed = {
+      total: resp.tokenUsage?.total || 0,
+      prompt: resp.tokenUsage?.prompt || 0,
+      completion: resp.tokenUsage?.completion || 0,
+    };
+    return { ...parsed, score: parsed.pass ? 1 : 0 };
+  } catch (err) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `Output is not valid JSON: ${resp.output}`,
+      tokensUsed: {
+        total: resp.tokenUsage?.total || 0,
+        prompt: resp.tokenUsage?.prompt || 0,
+        completion: resp.tokenUsage?.completion || 0,
+      },
+    };
+  }
+}

package/src/providers/azureopenai.ts

@@ -279,8 +279,7 @@ export class AzureOpenAiChatCompletionProvider extends AzureOpenAiGenericProvide
     logger.debug(`\tAzure OpenAI API response: ${JSON.stringify(data)}`);
     try {
       const message = data.choices[0].message;
-      const output =
-        message.content === null ? JSON.stringify(message.function_call) : message.content;
+      const output = message.content == null ? message.function_call : message.content;
       return {
         output,
         tokenUsage: cached

package/src/providers/openai.ts

@@ -78,9 +78,7 @@ export class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
           headers: {
             'Content-Type': 'application/json',
             Authorization: `Bearer ${this.getApiKey()}`,
-            ...(this.getOrganization()
-              ? { 'OpenAI-Organization': this.getOrganization() }
-              : {}),
+            ...(this.getOrganization() ? { 'OpenAI-Organization': this.getOrganization() } : {}),
           },
           body: JSON.stringify(body),
         },
@@ -315,8 +313,7 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
     try {
       const message = data.choices[0].message;
-      const output =
-        message.content === null ? JSON.stringify(message.function_call) : message.content;
+      const output = message.content === null ? message.function_call : message.content;
       return {
         output,
         tokenUsage: cached

package/src/providers/scriptCompletion.ts

@@ -1,6 +1,6 @@
 import { exec } from 'child_process';
 
-import { ApiProvider, ProviderConfig, ProviderResponse } from '../types';
+import { ApiProvider, ProviderOptions, ProviderResponse } from '../types';
 
 const ANSI_ESCAPE = /\x1b(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g;
 
@@ -9,7 +9,7 @@ function stripText(text: string) {
 }
 
 export class ScriptCompletionProvider implements ApiProvider {
-  constructor(private scriptPath: string, private config?: ProviderConfig) {}
+  constructor(private scriptPath: string, private config?: ProviderOptions) {}
 
   id() {
     return `exec:${this.scriptPath}`;

package/src/providers.ts

@@ -14,18 +14,18 @@ import {
 
 import type {
   ApiProvider,
-  ProviderConfig,
+  ProviderOptions,
   ProviderFunction,
   ProviderId,
-  RawProviderConfig,
+  ProviderOptionsMap,
 } from './types';
 
 export async function loadApiProviders(
   providerPaths:
     | ProviderId
     | ProviderId[]
-    | RawProviderConfig[]
-    | ProviderConfig[]
+    | ProviderOptionsMap[]
+    | ProviderOptions[]
     | ProviderFunction,
   basePath?: string,
 ): Promise<ApiProvider[]> {
@@ -50,11 +50,11 @@ export async function loadApiProviders(
           };
         } else if (provider.id) {
           // List of ProviderConfig objects
-          return loadApiProvider((provider as ProviderConfig).id!, provider, basePath);
+          return loadApiProvider((provider as ProviderOptions).id!, provider, basePath);
         } else {
           // List of { id: string, config: ProviderConfig } objects
           const id = Object.keys(provider)[0];
-          const providerObject = (provider as RawProviderConfig)[id];
+          const providerObject = (provider as ProviderOptionsMap)[id];
           const context = { ...providerObject, id: providerObject.id || id };
           return loadApiProvider(id, context, basePath);
         }
@@ -66,9 +66,10 @@ export async function loadApiProviders(
 
 export async function loadApiProvider(
   providerPath: string,
-  context?: ProviderConfig,
+  context?: ProviderOptions,
   basePath?: string,
 ): Promise<ApiProvider> {
+  context = context || {};
   if (providerPath?.startsWith('exec:')) {
     // Load script module
     const scriptPath = providerPath.split(':')[1];
@@ -86,18 +87,18 @@ export async function loadApiProvider(
       return new OpenAiChatCompletionProvider(
         modelName || 'gpt-3.5-turbo',
         undefined,
-        context?.config,
+        context.config,
       );
     } else if (modelType === 'completion') {
       return new OpenAiCompletionProvider(
         modelName || 'text-davinci-003',
         undefined,
-        context?.config,
+        context.config,
       );
     } else if (OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelType)) {
-      return new OpenAiChatCompletionProvider(modelType, undefined, context?.config, context?.id);
+      return new OpenAiChatCompletionProvider(modelType, undefined, context.config, context.id);
     } else if (OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelType)) {
-      return new OpenAiCompletionProvider(modelType, undefined, context?.config, context?.id);
+      return new OpenAiCompletionProvider(modelType, undefined, context.config, context.id);
     } else {
       throw new Error(
         `Unknown OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
@@ -113,15 +114,15 @@ export async function loadApiProvider(
       return new AzureOpenAiChatCompletionProvider(
         deploymentName,
         undefined,
-        context?.config,
-        context?.id,
+        context.config,
+        context.id,
       );
     } else if (modelType === 'completion') {
       return new AzureOpenAiCompletionProvider(
         deploymentName,
         undefined,
-        context?.config,
-        context?.id,
+        context.config,
+        context.id,
       );
     } else {
       throw new Error(
@@ -138,10 +139,10 @@ export async function loadApiProvider(
       return new AnthropicCompletionProvider(
         modelName || 'claude-instant-1',
         undefined,
-        context?.config,
+        context.config,
       );
     } else if (AnthropicCompletionProvider.ANTHROPIC_COMPLETION_MODELS.includes(modelType)) {
-      return new AnthropicCompletionProvider(modelType, undefined, context?.config);
+      return new AnthropicCompletionProvider(modelType, undefined, context.config);
     } else {
       throw new Error(
         `Unknown Anthropic model type: ${modelType}. Use one of the following providers: anthropic:completion:<model name>`,
@@ -152,12 +153,12 @@ export async function loadApiProvider(
     const options = providerPath.split(':');
     const modelName = options.slice(1).join(':');
 
-    return new ReplicateProvider(modelName, undefined, context?.config);
+    return new ReplicateProvider(modelName, undefined, context.config);
   }
 
   if (providerPath === 'llama' || providerPath.startsWith('llama:')) {
     const modelName = providerPath.split(':')[1];
-    return new LlamaProvider(modelName, context?.config);
+    return new LlamaProvider(modelName, context.config);
   } else if (providerPath.startsWith('ollama:')) {
     const modelName = providerPath.split(':')[1];
     return new OllamaProvider(modelName);

package/src/types.ts

@@ -27,7 +27,7 @@ export interface CommandLineOptions {
   promptSuffix?: string;
 }
 
-export interface ProviderConfig {
+export interface ProviderOptions {
   id?: ProviderId;
   config?: any;
   prompts?: string[]; // List of prompt display strings
@@ -177,6 +177,7 @@ export interface Assertion {
   value?:
     | string
     | string[]
+    | object
     | ((output: string, testCase: AtomicTestCase, assertion: Assertion) => Promise<GradingResult>);
 
   // The threshold value, only applicable for similarity (cosine distance)
@@ -186,7 +187,7 @@ export interface Assertion {
   weight?: number;
 
   // Some assertions (similarity, llm-rubric) require an LLM provider
-  provider?: ApiProvider;
+  provider?: GradingConfig['provider'];
 }
 
 // Each test case is graded pass/fail.  A test case represents a unique input to the LLM after substituting `vars` in the prompt.
@@ -249,7 +250,7 @@ export type ProviderId = string;
 
 export type ProviderFunction = (prompt: string) => Promise<ProviderResponse>;
 
-export type RawProviderConfig = Record<ProviderId, ProviderConfig>;
+export type ProviderOptionsMap = Record<ProviderId, ProviderOptions>;
 
 // TestSuiteConfig = Test Suite, but before everything is parsed and resolved.  Providers are just strings, prompts are filepaths, tests can be filepath or inline.
 export interface TestSuiteConfig {
@@ -257,7 +258,12 @@ export interface TestSuiteConfig {
   description?: string;
 
   // One or more LLM APIs to use, for example: openai:gpt-3.5-turbo, openai:gpt-4, localai:chat:vicuna
-  providers: ProviderId | ProviderId[] | RawProviderConfig[] | ProviderConfig[] | ProviderFunction;
+  providers:
+    | ProviderId
+    | ProviderId[]
+    | ProviderOptionsMap[]
+    | ProviderOptions[]
+    | ProviderFunction;
 
   // One or more prompt files to load
   prompts: string | string[];

package/src/util.ts

@@ -24,7 +24,7 @@ import type {
   UnifiedConfig,
   TestCase,
   Prompt,
-  RawProviderConfig,
+  ProviderOptionsMap,
   TestSuite,
 } from './types';
 
@@ -53,7 +53,7 @@ export function readProviderPromptMap(
 
   for (const provider of config.providers) {
     if (typeof provider === 'object') {
-      const rawProvider = provider as RawProviderConfig;
+      const rawProvider = provider as ProviderOptionsMap;
       const originalId = Object.keys(rawProvider)[0];
       const providerObject = rawProvider[originalId];
       const id = providerObject.id || originalId;

package/src/web/nextui/src/app/setup/ProviderConfigDialog.tsx

@@ -8,14 +8,14 @@ import {
   DialogActions,
   Button,
 } from '@mui/material';
-import { ProviderConfig } from '../../../../../types';
+import { ProviderOptions } from '../../../../../types';
 
 interface ProviderConfigDialogProps {
   open: boolean;
   providerId: string;
-  config: ProviderConfig['config'];
+  config: ProviderOptions['config'];
   onClose: () => void;
-  onSave: (config: ProviderConfig['config']) => void;
+  onSave: (config: ProviderOptions['config']) => void;
 }
 
 const ProviderConfigDialog: React.FC<ProviderConfigDialogProps> = ({

package/src/web/nextui/src/app/setup/ProviderSelector.tsx

@@ -1,9 +1,9 @@
 import React from 'react';
 import { Autocomplete, Box, Chip, TextField } from '@mui/material';
-import { ProviderConfig } from '../../../../../types';
+import { ProviderOptions } from '../../../../../types';
 import ProviderConfigDialog from './ProviderConfigDialog';
 
-const defaultProviders: ProviderConfig[] = [
+const defaultProviders: ProviderOptions[] = [
   {
     id: 'replicate:replicate/llama70b-v2-chat:e951f18578850b652510200860fc4ea62b3b16fac280f83ff32282f87bbd2e48',
     config: { temperature: 0.5 },
@@ -48,38 +48,38 @@ const defaultProviders: ProviderConfig[] = [
   .sort((a, b) => a.id.localeCompare(b.id));
 
 interface ProviderSelectorProps {
-  providers: ProviderConfig[];
-  onChange: (providers: ProviderConfig[]) => void;
+  providers: ProviderOptions[];
+  onChange: (providers: ProviderOptions[]) => void;
 }
 
 const ProviderSelector: React.FC<ProviderSelectorProps> = ({ providers, onChange }) => {
-  const [selectedProvider, setSelectedProvider] = React.useState<ProviderConfig | null>(null);
+  const [selectedProvider, setSelectedProvider] = React.useState<ProviderOptions | null>(null);
 
-  const getProviderLabel = (provider: string | ProviderConfig) => {
+  const getProviderLabel = (provider: string | ProviderOptions) => {
     if (typeof provider === 'string') {
       return provider;
     }
     return provider.id || 'Unknown provider';
   };
 
-  const getProviderKey = (provider: string | ProviderConfig, index: number) => {
+  const getProviderKey = (provider: string | ProviderOptions, index: number) => {
     if (typeof provider === 'string') {
       return provider;
     }
     return provider.id || index;
   };
 
-  const handleProviderClick = (provider: string | ProviderConfig) => {
+  const handleProviderClick = (provider: string | ProviderOptions) => {
     if (typeof provider === 'string') {
       alert('Cannot edit custom providers');
     } else if (!provider.config) {
       alert('There is no config for this provider');
     } else {
-      setSelectedProvider(provider as ProviderConfig);
+      setSelectedProvider(provider as ProviderOptions);
     }
   };
 
-  const handleSave = (config: ProviderConfig['config']) => {
+  const handleSave = (config: ProviderOptions['config']) => {
     if (selectedProvider) {
       const updatedProviders = providers.map((provider) =>
         provider.id === selectedProvider.id ? { ...provider, config } : provider,
@@ -96,7 +96,7 @@ const ProviderSelector: React.FC<ProviderSelectorProps> = ({ providers, onChange
         freeSolo
         options={defaultProviders}
         value={providers}
-        onChange={(event, newValue: (string | ProviderConfig)[]) => {
+        onChange={(event, newValue: (string | ProviderOptions)[]) => {
           onChange(newValue.map((value) => (typeof value === 'string' ? { id: value } : value)));
         }}
         getOptionLabel={(option) => {
@@ -106,7 +106,7 @@ const ProviderSelector: React.FC<ProviderSelectorProps> = ({ providers, onChange
           if (typeof option === 'string') {
             return option;
           }
-          return (option as ProviderConfig).id || 'Unknown provider';
+          return (option as ProviderOptions).id || 'Unknown provider';
         }}
         renderTags={(value, getTagProps) =>
           value.map((provider, index: number) => {

package/src/web/nextui/src/util/store.ts

@@ -1,18 +1,18 @@
 import { create } from 'zustand';
 import { persist } from 'zustand/middleware';
 
-import type { Assertion, ProviderConfig, TestCase } from '../../../../types';
+import type { Assertion, ProviderOptions, TestCase } from '../../../../types';
 
 export interface State {
   asserts: Assertion[];
   testCases: TestCase[];
   description: string;
-  providers: ProviderConfig[];
+  providers: ProviderOptions[];
   prompts: string[];
   setAsserts: (asserts: Assertion[]) => void;
   setTestCases: (testCases: TestCase[]) => void;
   setDescription: (description: string) => void;
-  setProviders: (providers: ProviderConfig[]) => void;
+  setProviders: (providers: ProviderOptions[]) => void;
   setPrompts: (prompts: string[]) => void;
 }