promptfoo 0.17.4 → 0.17.6

package/src/providers/localai.ts

@@ -1,6 +1,6 @@
 import logger from '../logger';
 import { fetchJsonWithCache } from '../cache';
-import { REQUEST_TIMEOUT_MS } from './shared';
+import { REQUEST_TIMEOUT_MS, parseChatPrompt } from './shared';
 
 import type { ApiProvider, ProviderResponse } from '../types.js';
 
@@ -29,9 +29,10 @@ class LocalAiGenericProvider implements ApiProvider {
 
 export class LocalAiChatProvider extends LocalAiGenericProvider {
   async callApi(prompt: string): Promise<ProviderResponse> {
+    const messages = parseChatPrompt(prompt);
     const body = {
       model: this.modelName,
-      prompt,
+      messages: messages,
       temperature: process.env.LOCALAI_TEMPERATURE || 0.7,
     };
     logger.debug(`Calling LocalAI API: ${JSON.stringify(body)}`);

package/src/providers/openai.ts

@@ -1,6 +1,7 @@
+
 import logger from '../logger';
 import { fetchJsonWithCache } from '../cache';
-import { REQUEST_TIMEOUT_MS } from './shared';
+import { REQUEST_TIMEOUT_MS, parseChatPrompt } from './shared';
 
 import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '../types.js';
 
@@ -225,23 +226,7 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
       );
     }
 
-    let messages: { role: string; content: string; name?: string }[];
-    try {
-      messages = JSON.parse(prompt) as { role: string; content: string }[];
-    } catch (err) {
-      const trimmedPrompt = prompt.trim();
-      if (
-        process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
-        trimmedPrompt.startsWith('{') ||
-        trimmedPrompt.startsWith('[')
-      ) {
-        throw new Error(
-          `OpenAI Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`,
-        );
-      }
-      messages = [{ role: 'user', content: prompt }];
-    }
-
+    const messages = parseChatPrompt(prompt);
     const body = {
       model: this.modelName,
       messages: messages,

package/src/providers/replicate.ts

@@ -0,0 +1,86 @@
+import Replicate from 'replicate';
+
+import fetch from 'node-fetch';
+import logger from '../logger';
+import { getCache, isCacheEnabled } from '../cache';
+
+import type { ApiProvider, ProviderResponse } from '../types.js';
+
+export class ReplicateProvider implements ApiProvider {
+  modelName: string;
+  apiKey?: string;
+  replicate: any;
+
+  constructor(modelName: string, apiKey?: string) {
+    this.modelName = modelName;
+    this.apiKey = apiKey || process.env.REPLICATE_API_TOKEN || process.env.REPLICATE_API_KEY;
+  }
+
+  id(): string {
+    return `replicate:${this.modelName}`;
+  }
+
+  toString(): string {
+    return `[Replicate Provider ${this.modelName}]`;
+  }
+
+    async callApi(prompt: string): Promise<ProviderResponse> {
+    if (!this.apiKey) {
+      throw new Error(
+        'Replicate API key is not set. Set REPLICATE_API_TOKEN environment variable or pass it as an argument to the constructor.',
+      );
+    }
+
+    let cache;
+    let cacheKey;
+    if (isCacheEnabled()) {
+      cache = await getCache();
+      cacheKey = `replicate:${this.modelName}:${prompt}`;
+
+      // Try to get the cached response
+      const cachedResponse = await cache.get(cacheKey);
+
+      if (cachedResponse) {
+        logger.debug(`Returning cached response for ${prompt}: ${cachedResponse}`);
+        return JSON.parse(cachedResponse as string);
+      }
+    }
+
+    const replicate = new Replicate({
+      auth: this.apiKey,
+      fetch,
+    });
+
+    logger.debug(`Calling Replicate: ${prompt}`);
+    let response;
+    try {
+      response = await replicate.run(this.modelName as any, {
+        input: {
+          prompt,
+          max_length: process.env.REPLICATE_MAX_LENGTH || 2046,
+          temperature: process.env.REPLICATE_TEMPERATURE || 0.5,
+          repetition_penalty: process.env.REPLICATE_REPETITION_PENALTY || 1.0,
+        },
+      });
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+      };
+    }
+    logger.debug(`\tReplicate API response: ${JSON.stringify(response)}`);
+    try {
+      const result = {
+        output: (response as string[]).join(''),
+        tokenUsage: {}, // TODO: add token usage once Replicate API supports it
+      };
+      if (cache && cacheKey) {
+        await cache.set(cacheKey, JSON.stringify(result));
+      }
+      return result;
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(response)}`,
+      };
+    }
+  }
+}

package/src/providers/shared.ts

@@ -1,3 +1,32 @@
+import yaml from 'js-yaml';
+
 export const REQUEST_TIMEOUT_MS = process.env.REQUEST_TIMEOUT_MS
   ? parseInt(process.env.REQUEST_TIMEOUT_MS, 10)
   : 300_000;
+
+export function parseChatPrompt(prompt: string): { role: string; content: string; name?: string }[] {
+  const trimmedPrompt = prompt.trim();
+  if (trimmedPrompt.startsWith('- role:')) {
+    try {
+      // Try YAML
+      return yaml.load(prompt) as { role: string; content: string }[];
+    } catch (err) {
+      throw new Error(`Chat Completion prompt is not a valid YAML string: ${err}\n\n${prompt}`);
+    }
+  } else {
+    try {
+      // Try JSON
+      return JSON.parse(prompt) as { role: string; content: string }[];
+    } catch (err) {
+      if (
+        process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
+        trimmedPrompt.startsWith('{') ||
+        trimmedPrompt.startsWith('[')
+      ) {
+        throw new Error(`Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`);
+      }
+      // Fall back to wrapping the prompt in a user message
+      return [{ role: 'user', content: prompt }];
+    }
+  }
+}

package/src/providers.ts

@@ -4,8 +4,13 @@ import { ApiProvider, ProviderConfig, ProviderId, RawProviderConfig } from './ty
 
 import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './providers/openai';
 import { AnthropicCompletionProvider } from './providers/anthropic';
+import { ReplicateProvider } from './providers/replicate';
 import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai';
 import { ScriptCompletionProvider } from './providers/scriptCompletion';
+import {
+  AzureOpenAiChatCompletionProvider,
+  AzureOpenAiCompletionProvider,
+} from './providers/azureopenai';
 
 export async function loadApiProviders(
   providerPaths: ProviderId | ProviderId[] | RawProviderConfig[],
@@ -68,6 +73,21 @@ export async function loadApiProvider(
         `Unknown OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
       );
     }
+  } else if (providerPath?.startsWith('azureopenai:')) {
+    // Load Azure OpenAI module
+    const options = providerPath.split(':');
+    const modelType = options[1];
+    const deploymentName = options[2];
+
+    if (modelType === 'chat') {
+      return new AzureOpenAiChatCompletionProvider(deploymentName, undefined, context?.config);
+    } else if (modelType === 'completion') {
+      return new AzureOpenAiCompletionProvider(deploymentName, undefined, context?.config);
+    } else {
+      throw new Error(
+        `Unknown Azure OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
+      );
+    }
   } else if (providerPath?.startsWith('anthropic:')) {
     // Load Anthropic module
     const options = providerPath.split(':');
@@ -87,6 +107,12 @@ export async function loadApiProvider(
         `Unknown Anthropic model type: ${modelType}. Use one of the following providers: anthropic:completion:<model name>`,
       );
     }
+  } else if (providerPath?.startsWith('replicate:')) {
+    // Load Replicate module
+    const options = providerPath.split(':');
+    const modelName = options.slice(1).join(':');
+
+    return new ReplicateProvider(modelName, undefined);
   }
 
   if (providerPath?.startsWith('localai:')) {
@@ -112,6 +138,7 @@ export default {
   OpenAiCompletionProvider,
   OpenAiChatCompletionProvider,
   AnthropicCompletionProvider,
+  ReplicateProvider,
   LocalAiCompletionProvider,
   LocalAiChatProvider,
   loadApiProvider,

package/src/types.ts

@@ -6,6 +6,7 @@ export interface CommandLineOptions {
 
   // Shared with EvaluateOptions
   maxConcurrency: string;
+  repeat: string;
 
   // Command line only
   vars?: string;
@@ -75,6 +76,7 @@ export interface EvaluateOptions {
   maxConcurrency?: number;
   showProgressBar?: boolean;
   generateSuggestions?: boolean;
+  repeat?: number;
 }
 
 export interface Prompt {
@@ -136,6 +138,7 @@ type BaseAssertionTypes =
   | 'icontains'
   | 'contains-all'
   | 'contains-any'
+  | 'starts-with'
   | 'regex'
   | 'is-json'
   | 'contains-json'
@@ -230,6 +233,9 @@ export interface TestSuiteConfig {
 
   // Path to write output. Writes to console/web viewer if not set.
   outputPath?: string;
+
+  // Determines whether or not sharing is enabled.
+  sharing?: boolean;
 }
 
 export type UnifiedConfig = TestSuiteConfig & {

package/src/updates.ts

@@ -17,6 +17,10 @@ export async function getLatestVersion(packageName: string) {
 }
 
 export async function checkForUpdates(): Promise<boolean> {
+  if (process.env.PROMPTFOO_DISABLE_UPDATE) {
+    return false;
+  }
+
   let latestVersion: string;
   try {
     latestVersion = await getLatestVersion('promptfoo');

package/src/web/client/src/App.tsx

@@ -37,6 +37,12 @@ function App() {
     }
   };
 
+  React.useEffect(() => {
+    if (prefersDarkMode) {
+      document.documentElement.setAttribute('data-theme', 'dark');
+    }
+  }, [prefersDarkMode]);
+
   React.useEffect(() => {
     const fetchEvalData = async (id: string) => {
       if (loadedFromApi.current) {

package/src/web/client/src/EvalOutputPromptDialog.tsx

@@ -37,7 +37,7 @@ export default function EvalOutputPromptDialog({
     <Dialog open={open} onClose={onClose} fullWidth maxWidth="lg">
       <DialogTitle>Prompt</DialogTitle>
       <DialogContent>
-        <TextareaAutosize readOnly value={prompt} style={{ width: '100%' }} />
+        <TextareaAutosize readOnly value={prompt} style={{ width: '100%', padding: '0.75rem' }} />
         <IconButton
           onClick={() => copyToClipboard(prompt)}
           style={{ position: 'absolute', right: '10px', top: '10px' }}
@@ -49,7 +49,11 @@ export default function EvalOutputPromptDialog({
         <>
           <DialogTitle>Output</DialogTitle>
           <DialogContent>
-            <TextareaAutosize readOnly value={output} style={{ width: '100%' }} />
+            <TextareaAutosize
+              readOnly
+              value={output}
+              style={{ width: '100%', padding: '0.75rem' }}
+            />
           </DialogContent>
         </>
       )}

package/src/web/client/src/ResultsTable.tsx

@@ -334,6 +334,11 @@ export default function ResultsTable({
           return failureFilter[columnId] && isFail;
         });
       });
+    } else if (filterMode === 'different') {
+      return body.filter((row) => {
+        // TODO(ian): This works for strings, but not objects.
+        return !row.outputs.every((output) => output.text === row.outputs[0].text);
+      });
     }
     return body;
   }, [body, failureFilter, filterMode]);

package/src/web/client/src/ResultsView.tsx

@@ -181,7 +181,8 @@ export default function ResultsView() {
                 label="Filter"
               >
                 <MenuItem value="all">Show all results</MenuItem>
-                <MenuItem value="failures">Show only failures</MenuItem>
+                <MenuItem value="failures">Show failures only</MenuItem>
+                <MenuItem value="different">Show different only</MenuItem>
               </Select>
             </FormControl>
           </Box>
@@ -218,16 +219,18 @@ export default function ResultsView() {
                   </Button>
                 </Tooltip>
               )}
-              <Tooltip title="Generate a unique URL that others can access">
-                <Button
-                  color="primary"
-                  onClick={handleShareButtonClick}
-                  disabled={shareLoading}
-                  startIcon={shareLoading ? <CircularProgress size={16} /> : <ShareIcon />}
-                >
-                  Share
-                </Button>
-              </Tooltip>
+              {config?.sharing && (
+                <Tooltip title="Generate a unique URL that others can access">
+                  <Button
+                    color="primary"
+                    onClick={handleShareButtonClick}
+                    disabled={shareLoading}
+                    startIcon={shareLoading ? <CircularProgress size={16} /> : <ShareIcon />}
+                  >
+                    Share
+                  </Button>
+                </Tooltip>
+              )}
             </ResponsiveStack>
           </Box>
         </ResponsiveStack>

package/src/web/client/src/index.css

@@ -21,19 +21,8 @@
 }
 
 /* Dark mode colors */
-@media (prefers-color-scheme: dark) {
-  :root {
-    --background-color: #1a1a1a;
-    --text-color: #f0f0f0;
-    --border-color: #444444;
-    --table-border-color: #444444;
-    --pass-color: #4caf50;
-    --fail-color: #f44336;
-    --smalltext-color: #888888;
-  }
-}
-
 [data-theme='dark'] {
+  /* Keep synced with prefers-color-scheme above */
   --background-color: #1a1a1a;
   --text-color: #f0f0f0;
   --border-color: #444444;

package/src/web/client/src/types.ts

@@ -25,6 +25,6 @@ export type EvalTable = {
   body: EvalRow[];
 };
 
-export type FilterMode = 'all' | 'failures';
+export type FilterMode = 'all' | 'failures' | 'different';
 
 export type { UnifiedConfig } from '../../../types';

package/dist/src/web/client/assets/index-b82d0138.css

@@ -1 +0,0 @@
-:root{font-family:system-ui,Avenir,Helvetica,Arial,sans-serif;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #ffffff;--text-color: #404040;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--smalltext-color: gray;--success-background-color: #d1ffd7;--variable-background-color: #f7f7f7;--header-background-color: #fffdf7}@media (prefers-color-scheme: dark){:root{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888}}[data-theme=dark]{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888;--success-background-color: #216d2b;--variable-background-color: #333;--header-background-color: #333}html{font-size:calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)))}*{box-sizing:border-box}html{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol;font-size:16px;background-color:var(--background-color);color:var(--text-color)}table,.divTable{border:1px solid var(--table-border-color);border-collapse:collapse;width:100%;margin:1rem 0;box-shadow:0 2px 4px #0000001a}.tr{display:flex}tr,.tr{width:fit-content}tr:hover,.tr:hover{background-color:#0000000d}th,.th,td,.td{position:relative;box-shadow:inset 0 0 0 1px var(--border-color);vertical-align:top;padding:1.5rem}th.variable,.th.variable,td.variable,.td.variable{background-color:var(--variable-background-color)}tr.header{background-color:var(--header-background-color)}th,.th{padding:1rem;position:relative;text-align:center;vertical-align:bottom}th .action{cursor:pointer;margin-left:.5rem}tr .cell-actions{display:flex;gap:.5rem;visibility:hidden;position:absolute;bottom:1.25rem;right:0;line-height:0;font-size:1.75rem}tr:hover .cell-actions{visibility:visible}tr .cell-actions .action{cursor:pointer}th .smalltext{visibility:hidden;font-weight:400;font-size:.75rem;color:var(--smalltext-color)}th:hover .smalltext{visibility:visible}th .summary{font-weight:400;font-size:.8rem;padding:.25rem}th .summary.highlight{background-color:var(--success-background-color)}td .status{margin-bottom:.5rem;font-weight:700}td .score{font-weight:400}td .pass{color:var(--pass-color)}td .fail{color:var(--fail-color)}.first-prompt-col{border-left:2px solid #888}.first-prompt-row{border-top:2px solid #888}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.logo{display:flex;align-items:center;gap:4px}.logo img{width:30px}.logo span{margin-bottom:6px;color:var(--text-color)}[data-theme=dark] .logo img{filter:invert(1)}nav{display:flex;justify-content:space-between;align-items:center;margin-bottom:1rem;color:var(--text-color)}.dark-mode-toggle{background-color:transparent;border:none;color:var(--text-color);cursor:pointer;font-size:16px;padding:8px;transition:color .3s}.dark-mode-toggle:hover{color:var(--pass-color)}body{background-color:var(--background-color);color:var(--text-color)}