promptfoo 0.18.4 → 0.19.1

package/src/main.ts

@@ -24,7 +24,7 @@ import {
 import { DEFAULT_README, DEFAULT_YAML_CONFIG, DEFAULT_PROMPTS } from './onboarding';
 import { disableCache, clearCache } from './cache';
 import { getDirectory } from './esm';
-import { init } from './web/server';
+import { startServer } from './web/server';
 import { checkForUpdates } from './updates';
 
 import type {
@@ -127,7 +127,7 @@ async function main() {
         name: 'view',
       });
       await telemetry.send();
-      init(cmdObj.port);
+      startServer(cmdObj.port);
     });
 
   program
@@ -424,7 +424,7 @@ async function main() {
       logger.info('Done.');
 
       if (cmdObj.view) {
-        init(parseInt(cmdObj.view, 10) || 15500);
+        startServer(parseInt(cmdObj.view, 10) || 15500);
       }
     });
 

package/src/providers/openai.ts

@@ -19,6 +19,8 @@ interface OpenAiCompletionOptions {
     parameters: any;
   }[];
   function_call?: 'none' | 'auto';
+  apiKey?: string;
+  organization?: string;
 }
 
 class OpenAiGenericProvider implements ApiProvider {
@@ -29,7 +31,7 @@ class OpenAiGenericProvider implements ApiProvider {
   constructor(modelName: string, apiKey?: string) {
     this.modelName = modelName;
 
-    this.apiKey = apiKey || process.env.OPENAI_API_KEY;
+    this.apiKey = apiKey;
 
     this.apiHost = process.env.OPENAI_API_HOST || DEFAULT_OPENAI_HOST;
   }
@@ -42,6 +44,14 @@ class OpenAiGenericProvider implements ApiProvider {
     return `[OpenAI Provider ${this.modelName}]`;
   }
 
+  getOrganization(options?: OpenAiCompletionOptions): string | undefined {
+    return options?.organization || process.env.OPENAI_ORGANIZATION;
+  }
+
+  getApiKey(options?: OpenAiCompletionOptions): string | undefined {
+    return options?.apiKey || this.apiKey || process.env.OPENAI_API_KEY;
+  }
+
   // @ts-ignore: Prompt is not used in this implementation
   async callApi(prompt: string, options?: OpenAiCompletionOptions): Promise<ProviderResponse> {
     throw new Error('Not implemented');
@@ -50,7 +60,7 @@ class OpenAiGenericProvider implements ApiProvider {
 
 export class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
   async callEmbeddingApi(text: string): Promise<ProviderEmbeddingResponse> {
-    if (!this.apiKey) {
+    if (!this.getApiKey()) {
       throw new Error('OpenAI API key must be set for similarity comparison');
     }
 
@@ -67,7 +77,10 @@ export class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
           method: 'POST',
           headers: {
             'Content-Type': 'application/json',
-            Authorization: `Bearer ${this.apiKey}`,
+            Authorization: `Bearer ${this.getApiKey()}`,
+            ...(this.getOrganization()
+              ? { 'OpenAI-Organization': this.getOrganization() }
+              : {}),
           },
           body: JSON.stringify(body),
         },
@@ -135,7 +148,7 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
   }
 
   async callApi(prompt: string, options?: OpenAiCompletionOptions): Promise<ProviderResponse> {
-    if (!this.apiKey) {
+    if (!this.getApiKey(options)) {
       throw new Error(
         'OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
       );
@@ -183,7 +196,10 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
           method: 'POST',
           headers: {
             'Content-Type': 'application/json',
-            Authorization: `Bearer ${this.apiKey}`,
+            Authorization: `Bearer ${this.getApiKey(options)}`,
+            ...(this.getOrganization(options)
+              ? { 'OpenAI-Organization': this.getOrganization(options) }
+              : {}),
           },
           body: JSON.stringify(body),
         },
@@ -240,7 +256,7 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
   }
 
   async callApi(prompt: string, options?: OpenAiCompletionOptions): Promise<ProviderResponse> {
-    if (!this.apiKey) {
+    if (!this.getApiKey(options)) {
       throw new Error(
         'OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
       );
@@ -281,7 +297,10 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
           method: 'POST',
           headers: {
             'Content-Type': 'application/json',
-            Authorization: `Bearer ${this.apiKey}`,
+            Authorization: `Bearer ${this.getApiKey(options)}`,
+            ...(this.getOrganization(options)
+              ? { 'OpenAI-Organization': this.getOrganization(options) }
+              : {}),
           },
           body: JSON.stringify(body),
         },

package/src/providers.ts

@@ -21,7 +21,12 @@ import type {
 } from './types';
 
 export async function loadApiProviders(
-  providerPaths: ProviderId | ProviderId[] | RawProviderConfig[] | ProviderFunction,
+  providerPaths:
+    | ProviderId
+    | ProviderId[]
+    | RawProviderConfig[]
+    | ProviderConfig[]
+    | ProviderFunction,
   basePath?: string,
 ): Promise<ApiProvider[]> {
   if (typeof providerPaths === 'string') {
@@ -43,9 +48,13 @@ export async function loadApiProviders(
             id: () => `custom-function-${idx}`,
             callApi: provider,
           };
+        } else if (provider.id) {
+          // List of ProviderConfig objects
+          return loadApiProvider((provider as ProviderConfig).id!, provider, basePath);
         } else {
+          // List of { id: string, config: ProviderConfig } objects
           const id = Object.keys(provider)[0];
-          const providerObject = provider[id];
+          const providerObject = (provider as RawProviderConfig)[id];
           const context = { ...providerObject, id: providerObject.id || id };
           return loadApiProvider(id, context, basePath);
         }

package/src/share.ts

@@ -1,23 +1,25 @@
 import fetch from 'node-fetch';
 
-import type { EvaluateSummary, UnifiedConfig } from './types';
+import type { EvaluateSummary, SharedResults, UnifiedConfig } from './types';
 
 export async function createShareableUrl(
   results: EvaluateSummary,
   config: Partial<UnifiedConfig>,
 ): Promise<string> {
+  const sharedResults: SharedResults = {
+    data: {
+      version: 1,
+      results,
+      config,
+    },
+  };
+
   const response = await fetch('https://api.promptfoo.dev/eval', {
     method: 'POST',
     headers: {
       'Content-Type': 'application/json',
     },
-    body: JSON.stringify({
-      data: {
-        version: 1,
-        results,
-        config,
-      },
-    }),
+    body: JSON.stringify(sharedResults),
   });
 
   const { id } = (await response.json()) as { id: string };

package/src/types.ts

@@ -80,6 +80,7 @@ export interface OutputConfig {
 export interface EvaluateOptions {
   maxConcurrency?: number;
   showProgressBar?: boolean;
+  progressCallback?: (progress: number, total: number) => void;
   generateSuggestions?: boolean;
   repeat?: number;
 }
@@ -256,7 +257,7 @@ export interface TestSuiteConfig {
   description?: string;
 
   // One or more LLM APIs to use, for example: openai:gpt-3.5-turbo, openai:gpt-4, localai:chat:vicuna
-  providers: ProviderId | ProviderId[] | RawProviderConfig[] | ProviderFunction;
+  providers: ProviderId | ProviderId[] | RawProviderConfig[] | ProviderConfig[] | ProviderFunction;
 
   // One or more prompt files to load
   prompts: string | string[];
@@ -281,3 +282,11 @@ export type UnifiedConfig = TestSuiteConfig & {
   evaluateOptions: EvaluateOptions;
   commandLineOptions: Partial<CommandLineOptions>;
 };
+
+export interface SharedResults {
+  data: {
+    version: number;
+    results: EvaluateSummary;
+    config: Partial<UnifiedConfig>;
+  };
+}

package/src/web/nextui/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

package/src/web/nextui/next.config.js

@@ -0,0 +1,14 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  output: process.env.USING_VERCEL ? 'standalone' : 'export',
+  trailingSlash: true,
+  webpack: (config, { buildId, dev, isServer, defaultLoaders, webpack }) => {
+    config.externals.push({
+      'utf-8-validate': 'commonjs utf-8-validate',
+      bufferutil: 'commonjs bufferutil',
+    });
+    return config;
+  },
+};
+
+module.exports = nextConfig;