promptfoo 0.18.2 → 0.18.3

package/src/providers/ollama.ts

@@ -0,0 +1,88 @@
+import logger from '../logger';
+import { fetchWithCache } from '../cache';
+
+import type { ApiProvider, ProviderResponse } from '../types.js';
+import { REQUEST_TIMEOUT_MS } from './shared';
+
+interface OllamaJsonL {
+  model: string;
+  created_at: string;
+  response?: string;
+  done: boolean;
+  context?: number[];
+  total_duration?: number;
+  load_duration?: number;
+  sample_count?: number;
+  sample_duration?: number;
+  prompt_eval_count?: number;
+  prompt_eval_duration?: number;
+  eval_count?: number;
+  eval_duration?: number;
+}
+
+export class OllamaProvider implements ApiProvider {
+  modelName: string;
+
+  constructor(modelName: string) {
+    this.modelName = modelName;
+  }
+
+  id(): string {
+    return `ollama:${this.modelName}`;
+  }
+
+  toString(): string {
+    return `[Ollama Provider ${this.modelName}]`;
+  }
+
+  async callApi(prompt: string): Promise<ProviderResponse> {
+    const params = {
+      model: this.modelName,
+      prompt,
+    };
+
+    logger.debug(`Calling Ollama API: ${JSON.stringify(params)}`);
+    let response;
+    try {
+      response = await fetchWithCache(
+        `${process.env.OLLAMA_BASE_URL || 'http://localhost:11434'}/api/generate`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify(params),
+        },
+        REQUEST_TIMEOUT_MS,
+        'text',
+      );
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+      };
+    }
+    logger.debug(`\tOllama API response: ${response.data}`);
+
+    try {
+      const output = response.data
+        .split('\n')
+        .map((line: string) => {
+          const parsed = JSON.parse(line) as OllamaJsonL;
+          if (parsed.response) {
+            return parsed.response;
+          }
+          return null;
+        })
+        .filter((s: string | null) => s !== null)
+        .join('');
+
+      return {
+        output,
+      };
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(response.data)}`,
+      };
+    }
+  }
+}

package/src/providers/openai.ts

@@ -1,5 +1,5 @@
 import logger from '../logger';
-import { fetchJsonWithCache } from '../cache';
+import { fetchWithCache } from '../cache';
 import { REQUEST_TIMEOUT_MS, parseChatPrompt } from './shared';
 
 import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '../types.js';
@@ -61,7 +61,7 @@ export class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
     let data,
       cached = false;
     try {
-      ({ data, cached } = (await fetchJsonWithCache(
+      ({ data, cached } = (await fetchWithCache(
         `https://${this.apiHost}/v1/embeddings`,
         {
           method: 'POST',
@@ -177,7 +177,7 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
     let data,
       cached = false;
     try {
-      ({ data, cached } = (await fetchJsonWithCache(
+      ({ data, cached } = (await fetchWithCache(
         `https://${this.apiHost}/v1/completions`,
         {
           method: 'POST',
@@ -275,7 +275,7 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     let data,
       cached = false;
     try {
-      ({ data, cached } = (await fetchJsonWithCache(
+      ({ data, cached } = (await fetchWithCache(
         `https://${this.apiHost}/v1/chat/completions`,
         {
           method: 'POST',

package/src/providers.ts

@@ -5,6 +5,7 @@ import { AnthropicCompletionProvider } from './providers/anthropic';
 import { ReplicateProvider } from './providers/replicate';
 import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai';
 import { LlamaProvider } from './providers/llama';
+import { OllamaProvider } from './providers/ollama';
 import { ScriptCompletionProvider } from './providers/scriptCompletion';
 import {
   AzureOpenAiChatCompletionProvider,
@@ -100,9 +101,19 @@ export async function loadApiProvider(
     const deploymentName = options[2];
 
     if (modelType === 'chat') {
-      return new AzureOpenAiChatCompletionProvider(deploymentName, undefined, context?.config, context?.id);
+      return new AzureOpenAiChatCompletionProvider(
+        deploymentName,
+        undefined,
+        context?.config,
+        context?.id,
+      );
     } else if (modelType === 'completion') {
-      return new AzureOpenAiCompletionProvider(deploymentName, undefined, context?.config, context?.id);
+      return new AzureOpenAiCompletionProvider(
+        deploymentName,
+        undefined,
+        context?.config,
+        context?.id,
+      );
     } else {
       throw new Error(
         `Unknown Azure OpenAI model type: ${modelType}. Use one of the following providers: openai:chat:<model name>, openai:completion:<model name>`,
@@ -138,6 +149,9 @@ export async function loadApiProvider(
   if (providerPath === 'llama' || providerPath.startsWith('llama:')) {
     const modelName = providerPath.split(':')[1];
     return new LlamaProvider(modelName, context?.config);
+  } else if (providerPath.startsWith('ollama:')) {
+    const modelName = providerPath.split(':')[1];
+    return new OllamaProvider(modelName);
   } else if (providerPath?.startsWith('localai:')) {
     const options = providerPath.split(':');
     const modelType = options[1];

package/src/util.ts

@@ -12,7 +12,6 @@ import { parse as parseCsv } from 'csv-parse/sync';
 import { stringify } from 'csv-stringify/sync';
 
 import logger from './logger';
-import { assertionFromString } from './assertions';
 import { getDirectory } from './esm';
 
 import type { RequestInfo, RequestInit, Response } from 'node-fetch';
@@ -368,7 +367,7 @@ export function writeOutput(
       [...results.table.head.prompts, ...results.table.head.vars],
       ...results.table.body.map((row) => [...row.outputs.map(outputToSimpleString), ...row.vars]),
     ];
-    const htmlOutput = nunjucks.renderString(template, {
+    const htmlOutput = getNunjucksEngine().renderString(template, {
       table,
       results: results.results,
     });
@@ -456,10 +455,12 @@ export function writeLatestResults(results: EvaluateSummary, config: Partial<Uni
         2,
       ),
     );
-    if (fs.existsSync(latestResultsPath) && fs.lstatSync(latestResultsPath).isSymbolicLink()) {
+
+    try {
       fs.unlinkSync(latestResultsPath);
-    }
+    } catch {}
     fs.symlinkSync(newResultsPath, latestResultsPath);
+
     cleanupOldResults();
   } catch (err) {
     logger.error(`Failed to write latest results to ${newResultsPath}:\n${err}`);
@@ -520,6 +521,7 @@ export function testCaseFromCsvRow(row: CsvRow): TestCase {
   for (const [key, value] of Object.entries(row)) {
     if (key === '__expected') {
       if (value.trim() !== '') {
+        const { assertionFromString } = require('./assertions');
         asserts.push(assertionFromString(value));
       }
     } else {
@@ -532,3 +534,10 @@ export function testCaseFromCsvRow(row: CsvRow): TestCase {
     assert: asserts,
   };
 }
+
+export function getNunjucksEngine() {
+  nunjucks.configure({
+    autoescape: false,
+  });
+  return nunjucks;
+}

package/src/web/client/src/ResultsView.tsx

@@ -86,7 +86,7 @@ export default function ResultsView({ recentFiles, onRecentFileSelected }: Resul
     setFailureFilter(newFailureFilter);
   };
 
-  const [wordBreak, setWordBreak] = React.useState<'break-word' | 'break-all'>('break-all');
+  const [wordBreak, setWordBreak] = React.useState<'break-word' | 'break-all'>('break-word');
   const handleWordBreakChange = (event: React.ChangeEvent<HTMLInputElement>) => {
     setWordBreak(event.target.checked ? 'break-all' : 'break-word');
   };