promptfoo 0.18.4 → 0.19.0

package/src/providers.ts

@@ -21,7 +21,12 @@ import type {
 } from './types';
 
 export async function loadApiProviders(
-  providerPaths: ProviderId | ProviderId[] | RawProviderConfig[] | ProviderFunction,
+  providerPaths:
+    | ProviderId
+    | ProviderId[]
+    | RawProviderConfig[]
+    | ProviderConfig[]
+    | ProviderFunction,
   basePath?: string,
 ): Promise<ApiProvider[]> {
   if (typeof providerPaths === 'string') {
@@ -43,9 +48,13 @@ export async function loadApiProviders(
             id: () => `custom-function-${idx}`,
             callApi: provider,
           };
+        } else if (provider.id) {
+          // List of ProviderConfig objects
+          return loadApiProvider((provider as ProviderConfig).id!, provider, basePath);
         } else {
+          // List of { id: string, config: ProviderConfig } objects
           const id = Object.keys(provider)[0];
-          const providerObject = provider[id];
+          const providerObject = (provider as RawProviderConfig)[id];
           const context = { ...providerObject, id: providerObject.id || id };
           return loadApiProvider(id, context, basePath);
         }

package/src/share.ts

@@ -1,23 +1,25 @@
 import fetch from 'node-fetch';
 
-import type { EvaluateSummary, UnifiedConfig } from './types';
+import type { EvaluateSummary, SharedResults, UnifiedConfig } from './types';
 
 export async function createShareableUrl(
   results: EvaluateSummary,
   config: Partial<UnifiedConfig>,
 ): Promise<string> {
+  const sharedResults: SharedResults = {
+    data: {
+      version: 1,
+      results,
+      config,
+    },
+  };
+
   const response = await fetch('https://api.promptfoo.dev/eval', {
     method: 'POST',
     headers: {
       'Content-Type': 'application/json',
     },
-    body: JSON.stringify({
-      data: {
-        version: 1,
-        results,
-        config,
-      },
-    }),
+    body: JSON.stringify(sharedResults),
   });
 
   const { id } = (await response.json()) as { id: string };

package/src/types.ts

@@ -80,6 +80,7 @@ export interface OutputConfig {
 export interface EvaluateOptions {
   maxConcurrency?: number;
   showProgressBar?: boolean;
+  progressCallback?: (progress: number, total: number) => void;
   generateSuggestions?: boolean;
   repeat?: number;
 }
@@ -256,7 +257,7 @@ export interface TestSuiteConfig {
   description?: string;
 
   // One or more LLM APIs to use, for example: openai:gpt-3.5-turbo, openai:gpt-4, localai:chat:vicuna
-  providers: ProviderId | ProviderId[] | RawProviderConfig[] | ProviderFunction;
+  providers: ProviderId | ProviderId[] | RawProviderConfig[] | ProviderConfig[] | ProviderFunction;
 
   // One or more prompt files to load
   prompts: string | string[];
@@ -281,3 +282,11 @@ export type UnifiedConfig = TestSuiteConfig & {
   evaluateOptions: EvaluateOptions;
   commandLineOptions: Partial<CommandLineOptions>;
 };
+
+export interface SharedResults {
+  data: {
+    version: number;
+    results: EvaluateSummary;
+    config: Partial<UnifiedConfig>;
+  };
+}

package/src/web/nextui/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

package/src/web/nextui/next.config.js

@@ -0,0 +1,14 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  output: process.env.USING_VERCEL ? 'standalone' : 'export',
+  trailingSlash: true,
+  webpack: (config, { buildId, dev, isServer, defaultLoaders, webpack }) => {
+    config.externals.push({
+      'utf-8-validate': 'commonjs utf-8-validate',
+      bufferutil: 'commonjs bufferutil',
+    });
+    return config;
+  },
+};
+
+module.exports = nextConfig;