npm - promptfoo - Versions diffs - 0.20.0 → 0.21.0 - Mend

promptfoo 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (184) hide show

package/README.md +1 -1
package/dist/package.json +4 -4
package/dist/src/assertions.d.ts.map +1 -1
package/dist/src/assertions.js +5 -0
package/dist/src/assertions.js.map +1 -1
package/dist/src/evaluator.js +1 -1
package/dist/src/evaluator.js.map +1 -1
package/dist/src/index.d.ts +1 -5
package/dist/src/index.d.ts.map +1 -1
package/dist/src/index.js +1 -1
package/dist/src/index.js.map +1 -1
package/dist/src/matchers.d.ts +3 -2
package/dist/src/matchers.d.ts.map +1 -1
package/dist/src/matchers.js +37 -9
package/dist/src/matchers.js.map +1 -1
package/dist/src/providers/anthropic.d.ts +5 -3
package/dist/src/providers/anthropic.d.ts.map +1 -1
package/dist/src/providers/anthropic.js +8 -10
package/dist/src/providers/anthropic.js.map +1 -1
package/dist/src/providers/azureopenai.d.ts +9 -8
package/dist/src/providers/azureopenai.d.ts.map +1 -1
package/dist/src/providers/azureopenai.js +33 -36
package/dist/src/providers/azureopenai.js.map +1 -1
package/dist/src/providers/openai.d.ts +12 -12
package/dist/src/providers/openai.d.ts.map +1 -1
package/dist/src/providers/openai.js +54 -65
package/dist/src/providers/openai.js.map +1 -1
package/dist/src/providers/replicate.d.ts +4 -2
package/dist/src/providers/replicate.d.ts.map +1 -1
package/dist/src/providers/replicate.js +10 -8
package/dist/src/providers/replicate.js.map +1 -1
package/dist/src/providers/webhook.d.ts +9 -0
package/dist/src/providers/webhook.d.ts.map +1 -0
package/dist/src/providers/webhook.js +54 -0
package/dist/src/providers/webhook.js.map +1 -0
package/dist/src/providers.d.ts +1 -1
package/dist/src/providers.d.ts.map +1 -1
package/dist/src/providers.js +36 -28
package/dist/src/providers.js.map +1 -1
package/dist/src/suggestions.d.ts.map +1 -1
package/dist/src/suggestions.js +1 -3
package/dist/src/suggestions.js.map +1 -1
package/dist/src/types.d.ts +7 -1
package/dist/src/types.d.ts.map +1 -1
package/dist/src/util.js +1 -1
package/dist/src/util.js.map +1 -1
package/dist/src/web/nextui/404/index.html +1 -1
package/dist/src/web/nextui/404.html +1 -1
package/dist/src/web/nextui/_next/static/Bl3o5lF4ON7Fjki46lPhr/_buildManifest.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/226-7bbb6c98a19542fd.js +37 -0
package/dist/src/web/nextui/_next/static/chunks/249-ea9c0f034888ccff.js +125 -0
package/dist/src/web/nextui/_next/static/chunks/339-501c32916b785ef1.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/365-e426ea5bc7e815fc.js +8 -0
package/dist/src/web/nextui/_next/static/chunks/396-0a51429a01e24cdd.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/596-297f7ff4a0436e87.js +25 -0
package/dist/src/web/nextui/_next/static/chunks/613-572c22424de64659.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/706-ae1d3352d28419e9.js +9 -0
package/dist/src/web/nextui/_next/static/chunks/891-7035926a62c1c4e0.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/app/eval/[id]/not-found-366629541fd598e9.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/app/eval/[id]/page-319d2ee38d37574e.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/app/eval/page-a6b1ff91723b7beb.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/app/layout-024c4adc71c9feb0.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/app/page-1ae60660130041b2.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/app/setup/page-6ef16148040bf4f4.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/{ca377847-cb6ae6a6a073aebb.js → ca377847-26b462611379a4f7.js} +3 -3
package/dist/src/web/nextui/_next/static/chunks/{fd9d1056-ac777be631f5a9e9.js → fd9d1056-fba4b53a2f01213b.js} +1 -1
package/dist/src/web/nextui/_next/static/chunks/framework-8883d1e9be70c3da.js +25 -0
package/dist/src/web/nextui/_next/static/chunks/main-8ea85465d428ecfe.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/main-app-581ccf0003955b21.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/pages/_app-52924524f99094ab.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/pages/_error-c92d5c4bb2b49926.js +1 -0
package/dist/src/web/nextui/_next/static/chunks/webpack-55c264ce2fd85eb7.js +1 -0
package/dist/src/web/nextui/_next/static/css/4d399fceacd06992.css +1 -0
package/dist/src/web/nextui/eval/index.html +1 -1
package/dist/src/web/nextui/eval/index.txt +6 -6
package/dist/src/web/nextui/index.html +1 -1
package/dist/src/web/nextui/index.txt +5 -5
package/dist/src/web/nextui/setup/index.html +27 -1
package/dist/src/web/nextui/setup/index.txt +9 -9
package/dist/src/web/server.d.ts.map +1 -1
package/dist/src/web/server.js +9 -5
package/dist/src/web/server.js.map +1 -1
package/package.json +4 -4
package/dist/src/web/nextui/_next/static/US6gOx8LHTX_Hzm9aYNrC/_buildManifest.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/339-4fc8a80fa840e771.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/373-8a280796c0f2d1af.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/583-125d32af505e9bc4.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/596-07e4a23a5c6cdf04.js +0 -25
package/dist/src/web/nextui/_next/static/chunks/658-a62210d07dc4dcb6.js +0 -15
package/dist/src/web/nextui/_next/static/chunks/707-699cbd84b259c37b.js +0 -37
package/dist/src/web/nextui/_next/static/chunks/858-ceb6fa22e614492b.js +0 -125
package/dist/src/web/nextui/_next/static/chunks/891-3000ea7c0a292558.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/app/eval/[id]/not-found-50e40614fa05600e.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/app/eval/[id]/page-c19c44ed1b2dfb58.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/app/eval/page-d4a1813b2f8c4532.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/app/layout-664a8d716d2d24b1.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/app/page-1f8ef6a00a2355f0.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/app/setup/page-182018a3c6397345.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/framework-43665103d101a22d.js +0 -25
package/dist/src/web/nextui/_next/static/chunks/main-50cc0a98559591ce.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/main-app-c9dc13756d166550.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/pages/_app-6b79a29ad0d63b21.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/pages/_error-9aeb3e4d490fe4b8.js +0 -1
package/dist/src/web/nextui/_next/static/chunks/webpack-6e474e42be502dd7.js +0 -1
package/dist/src/web/nextui/_next/static/css/a35c840ac696f161.css +0 -1
package/dist/src/web/nextui/api +0 -1
package/src/__mocks__/esm.ts +0 -3
package/src/assertions.ts +0 -580
package/src/cache.ts +0 -109
package/src/esm.ts +0 -13
package/src/evaluator.ts +0 -500
package/src/index.ts +0 -52
package/src/logger.ts +0 -46
package/src/main.ts +0 -442
package/src/matchers.ts +0 -120
package/src/onboarding.ts +0 -69
package/src/prompts.ts +0 -39
package/src/providers/anthropic.ts +0 -88
package/src/providers/azureopenai.ts +0 -299
package/src/providers/llama.ts +0 -95
package/src/providers/localai.ts +0 -111
package/src/providers/ollama.ts +0 -89
package/src/providers/openai.ts +0 -337
package/src/providers/replicate.ts +0 -99
package/src/providers/scriptCompletion.ts +0 -35
package/src/providers/shared.ts +0 -34
package/src/providers.ts +0 -192
package/src/share.ts +0 -27
package/src/suggestions.ts +0 -63
package/src/table.ts +0 -43
package/src/tableOutput.html +0 -52
package/src/telemetry.ts +0 -70
package/src/types.ts +0 -299
package/src/updates.ts +0 -46
package/src/util.ts +0 -543
package/src/web/nextui/.eslintrc.json +0 -3
package/src/web/nextui/next.config.js +0 -14
package/src/web/nextui/package-lock.json +0 -4644
package/src/web/nextui/package.json +0 -47
package/src/web/nextui/public/favicon.ico +0 -0
package/src/web/nextui/public/logo.svg +0 -30
package/src/web/nextui/src/app/Home.css +0 -3
package/src/web/nextui/src/app/api/route.ts +0 -6
package/src/web/nextui/src/app/components/DarkMode.css +0 -22
package/src/web/nextui/src/app/components/DarkMode.tsx +0 -17
package/src/web/nextui/src/app/components/Logo.css +0 -32
package/src/web/nextui/src/app/components/Logo.tsx +0 -11
package/src/web/nextui/src/app/components/PageShell.css +0 -33
package/src/web/nextui/src/app/components/PageShell.tsx +0 -87
package/src/web/nextui/src/app/eval/ConfigModal.tsx +0 -84
package/src/web/nextui/src/app/eval/Eval.css +0 -13
package/src/web/nextui/src/app/eval/Eval.tsx +0 -79
package/src/web/nextui/src/app/eval/EvalOutputPromptDialog.tsx +0 -127
package/src/web/nextui/src/app/eval/ResultsCharts.tsx +0 -355
package/src/web/nextui/src/app/eval/ResultsTable.css +0 -179
package/src/web/nextui/src/app/eval/ResultsTable.tsx +0 -503
package/src/web/nextui/src/app/eval/ResultsView.tsx +0 -301
package/src/web/nextui/src/app/eval/ShareModal.tsx +0 -70
package/src/web/nextui/src/app/eval/[id]/not-found.tsx +0 -5
package/src/web/nextui/src/app/eval/[id]/page.css +0 -9
package/src/web/nextui/src/app/eval/[id]/page.tsx +0 -20
package/src/web/nextui/src/app/eval/index.css +0 -0
package/src/web/nextui/src/app/eval/page.tsx +0 -8
package/src/web/nextui/src/app/eval/store.ts +0 -18
package/src/web/nextui/src/app/eval/types.ts +0 -20
package/src/web/nextui/src/app/globals.css +0 -58
package/src/web/nextui/src/app/layout.tsx +0 -25
package/src/web/nextui/src/app/page.tsx +0 -7
package/src/web/nextui/src/app/setup/AssertsForm.tsx +0 -118
package/src/web/nextui/src/app/setup/PromptDialog.tsx +0 -77
package/src/web/nextui/src/app/setup/PromptsSection.tsx +0 -190
package/src/web/nextui/src/app/setup/ProviderConfigDialog.tsx +0 -99
package/src/web/nextui/src/app/setup/ProviderSelector.tsx +0 -149
package/src/web/nextui/src/app/setup/RunTestSuiteButton.tsx +0 -88
package/src/web/nextui/src/app/setup/TestCaseDialog.tsx +0 -108
package/src/web/nextui/src/app/setup/TestCasesSection.tsx +0 -154
package/src/web/nextui/src/app/setup/VarsForm.tsx +0 -57
package/src/web/nextui/src/app/setup/page.css +0 -3
package/src/web/nextui/src/app/setup/page.tsx +0 -160
package/src/web/nextui/src/util/api.ts +0 -1
package/src/web/nextui/src/util/store.ts +0 -53
package/src/web/nextui/tsconfig.json +0 -28
package/src/web/server.ts +0 -151
/package/dist/src/web/nextui/_next/static/{US6gOx8LHTX_Hzm9aYNrC → Bl3o5lF4ON7Fjki46lPhr}/_ssgManifest.js +0 -0

package/src/share.ts DELETED Viewed

@@ -1,27 +0,0 @@
-import fetch from 'node-fetch';
-import type { EvaluateSummary, SharedResults, UnifiedConfig } from './types';
-export async function createShareableUrl(
-  results: EvaluateSummary,
-  config: Partial<UnifiedConfig>,
-): Promise<string> {
-  const sharedResults: SharedResults = {
-    data: {
-      version: 1,
-      results,
-      config,
-    },
-  };
-  const response = await fetch('https://api.promptfoo.dev/eval', {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify(sharedResults),
-  });
-  const { id } = (await response.json()) as { id: string };
-  return `https://app.promptfoo.dev/eval/${id}`;
-}

package/src/suggestions.ts DELETED Viewed

@@ -1,63 +0,0 @@
-import { SUGGEST_PROMPTS_SYSTEM_MESSAGE } from './prompts';
-import { DefaultSuggestionsProvider } from './providers/openai';
-import type { TokenUsage } from './types';
-const DEFAULT_TEMPERATURE = 0.9;
-interface GeneratePromptsOutput {
-  prompts?: string[];
-  error?: string;
-  tokensUsed: TokenUsage;
-}
-export async function generatePrompts(prompt: string, num: number): Promise<GeneratePromptsOutput> {
-  const provider = DefaultSuggestionsProvider;
-  const resp = await provider.callApi(
-    JSON.stringify([
-      SUGGEST_PROMPTS_SYSTEM_MESSAGE,
-      {
-        role: 'user',
-        content: 'Generate a variant for the following prompt:',
-      },
-      {
-        role: 'user',
-        content: prompt,
-      },
-    ]),
-    {
-      temperature: DEFAULT_TEMPERATURE,
-    },
-  );
-  if (resp.error || !resp.output) {
-    return {
-      error: resp.error || 'Unknown error',
-      tokensUsed: {
-        total: resp.tokenUsage?.total || 0,
-        prompt: resp.tokenUsage?.prompt || 0,
-        completion: resp.tokenUsage?.completion || 0,
-      },
-    };
-  }
-  try {
-    return {
-      prompts: [resp.output],
-      tokensUsed: {
-        total: resp.tokenUsage?.total || 0,
-        prompt: resp.tokenUsage?.prompt || 0,
-        completion: resp.tokenUsage?.completion || 0,
-      },
-    };
-  } catch (err) {
-    return {
-      error: `Output is not valid JSON: ${resp.output}`,
-      tokensUsed: {
-        total: resp.tokenUsage?.total || 0,
-        prompt: resp.tokenUsage?.prompt || 0,
-        completion: resp.tokenUsage?.completion || 0,
-      },
-    };
-  }
-}

package/src/table.ts DELETED Viewed

@@ -1,43 +0,0 @@
-import Table from 'cli-table3';
-import chalk from 'chalk';
-import type { EvaluateSummary } from './types';
-export function generateTable(summary: EvaluateSummary, tableCellMaxLength = 250, maxRows = 25) {
-  const maxWidth = process.stdout.columns ? process.stdout.columns - 10 : 120;
-  const head = summary.table.head;
-  const headLength = head.prompts.length + head.vars.length;
-  const table = new Table({
-    head: [...head.vars, ...head.prompts.map((prompt) => prompt.display)],
-    colWidths: Array(headLength).fill(Math.floor(maxWidth / headLength)),
-    wordWrap: true,
-    wrapOnWordBoundary: false,
-    style: {
-      head: ['blue', 'bold'],
-    },
-  });
-  // Skip first row (header) and add the rest. Color PASS/FAIL
-  for (const row of summary.table.body.slice(0, maxRows)) {
-    table.push([
-      ...row.vars,
-      ...row.outputs.map(({ pass, score, text }) => {
-        if (text.length > tableCellMaxLength) {
-          text = text.slice(0, tableCellMaxLength) + '...';
-        }
-        if (pass) {
-          return chalk.green('[PASS] ') + text;
-        } else if (!pass) {
-          // color everything red up until '---'
-          return (
-            chalk.red('[FAIL] ') +
-            text
-              .split('---')
-              .map((c, idx) => (idx === 0 ? chalk.red.bold(c) : c))
-              .join('---')
-          );
-        }
-        return text;
-      }),
-    ]);
-  }
-  return table;
-}

package/src/tableOutput.html DELETED Viewed

@@ -1,52 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width" />
-    <title>Table Output</title>
-    <style>
-      body {
-        font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Helvetica, Arial,
-          sans-serif;
-      }
-      table,
-      th,
-      td {
-        border: 1px solid black;
-        border-collapse: collapse;
-        text-align: left;
-        word-break: break-all;
-      }
-      th,
-      td {
-        padding: 5px;
-        min-width: 200px;
-      }
-      tr > td[data-content^='[PASS]'] {
-        color: green;
-      }
-      tr > td[data-content^='[FAIL]'] {
-        color: #ad0000;
-      }
-    </style>
-  </head>
-  <body>
-    <table>
-      <thead>
-        {% for header in table[0] %}
-        <th>{{ header }}</th>
-        {% endfor %}
-      </thead>
-      <tbody>
-        {% for row in table.slice(1) %}
-        <tr>
-          {% for cell in row %}
-          <td data-content="{{cell}}">{{ cell }}</td>
-          {% endfor %}
-        </tr>
-        {% endfor %}
-      </tbody>
-    </table>
-  </body>
-</html>

package/src/telemetry.ts DELETED Viewed

@@ -1,70 +0,0 @@
-import chalk from 'chalk';
-import packageJson from '../package.json';
-import logger from './logger';
-import { fetchWithTimeout, maybeRecordFirstRun } from './util';
-type TelemetryEvent = {
-  event: string;
-  packageVersion: string;
-  properties: Record<string, string | number>;
-};
-type TelemetryEventTypes = 'eval_ran' | 'assertion_used' | 'command_used';
-const TELEMETRY_ENDPOINT = 'https://api.promptfoo.dev/telemetry';
-const TELEMETRY_TIMEOUT_MS = 1000;
-export class Telemetry {
-  private events: TelemetryEvent[] = [];
-  get disabled() {
-    return process.env.PROMPTFOO_DISABLE_TELEMETRY === '1';
-  }
-  record(eventName: TelemetryEventTypes, properties: Record<string, string | number>): void {
-    if (!this.disabled) {
-      this.events.push({
-        event: eventName,
-        packageVersion: packageJson.version,
-        properties,
-      });
-    }
-  }
-  maybeShowNotice(): void {
-    if (maybeRecordFirstRun()) {
-      logger.info(
-        chalk.gray(
-          'Anonymous telemetry is enabled. For more info, see https://www.promptfoo.dev/docs/configuration/telemetry',
-        ),
-      );
-    }
-  }
-  async send(): Promise<void> {
-    if (!this.disabled && this.events.length > 0) {
-      try {
-        const response = await fetchWithTimeout(
-          TELEMETRY_ENDPOINT,
-          {
-            method: 'POST',
-            headers: {
-              'Content-Type': 'application/json',
-            },
-            body: JSON.stringify(this.events),
-          },
-          TELEMETRY_TIMEOUT_MS,
-        );
-        if (response.ok) {
-          this.events = [];
-        }
-      } catch (err) {}
-    }
-  }
-}
-const telemetry = new Telemetry();
-export default telemetry;

package/src/types.ts DELETED Viewed

@@ -1,299 +0,0 @@
-export interface CommandLineOptions {
-  // Shared with TestSuite
-  prompts: string[];
-  providers: string[];
-  output: string;
-  // Shared with EvaluateOptions
-  maxConcurrency: string;
-  repeat: string;
-  // Command line only
-  vars?: string;
-  tests?: string;
-  config?: string;
-  verbose?: boolean;
-  grader?: string;
-  view?: string;
-  tableCellMaxLength?: string;
-  write?: boolean;
-  cache?: boolean;
-  table?: boolean;
-  share?: boolean;
-  progressBar?: boolean;
-  generateSuggestions?: boolean;
-  promptPrefix?: string;
-  promptSuffix?: string;
-}
-export interface ProviderOptions {
-  id?: ProviderId;
-  config?: any;
-  prompts?: string[]; // List of prompt display strings
-}
-export interface ApiProvider {
-  id: () => string;
-  callApi: (prompt: string) => Promise<ProviderResponse>;
-}
-export interface TokenUsage {
-  total: number;
-  prompt: number;
-  completion: number;
-  cached?: number;
-}
-export interface ProviderResponse {
-  error?: string;
-  output?: string;
-  tokenUsage?: Partial<TokenUsage>;
-}
-export interface ProviderEmbeddingResponse {
-  error?: string;
-  embedding?: number[];
-  tokenUsage?: Partial<TokenUsage>;
-}
-export interface CsvRow {
-  [key: string]: string;
-}
-export type VarMapping = Record<string, string>;
-export interface GradingConfig {
-  rubricPrompt?: string;
-  provider?: string | ApiProvider;
-}
-export interface PromptConfig {
-  prefix?: string;
-  suffix?: string;
-}
-export interface OutputConfig {
-  postprocess?: string;
-}
-export interface EvaluateOptions {
-  maxConcurrency?: number;
-  showProgressBar?: boolean;
-  progressCallback?: (progress: number, total: number) => void;
-  generateSuggestions?: boolean;
-  repeat?: number;
-}
-export interface Prompt {
-  raw: string;
-  display: string;
-}
-export interface EvaluateResult {
-  prompt: Prompt;
-  vars: Record<string, string | object>;
-  response?: ProviderResponse;
-  error?: string;
-  success: boolean;
-  score: number;
-  latencyMs: number;
-  gradingResult?: GradingResult;
-}
-export interface EvaluateTableOutput {
-  pass: boolean;
-  score: number;
-  text: string;
-  prompt: string;
-  latencyMs: number;
-  tokenUsage?: Partial<TokenUsage>;
-  gradingResult?: GradingResult;
-}
-export interface EvaluateTable {
-  head: {
-    prompts: Prompt[];
-    vars: string[];
-  };
-  body: {
-    outputs: EvaluateTableOutput[];
-    vars: string[];
-  }[];
-}
-export interface EvaluateStats {
-  successes: number;
-  failures: number;
-  tokenUsage: Required<TokenUsage>;
-}
-export interface EvaluateSummary {
-  version: number;
-  results: EvaluateResult[];
-  table: EvaluateTable;
-  stats: EvaluateStats;
-}
-export interface GradingResult {
-  pass: boolean;
-  score: number;
-  reason: string;
-  tokensUsed?: TokenUsage;
-  componentResults?: GradingResult[];
-  assertion: Assertion | null;
-}
-type BaseAssertionTypes =
-  | 'equals'
-  | 'contains'
-  | 'icontains'
-  | 'contains-all'
-  | 'contains-any'
-  | 'starts-with'
-  | 'regex'
-  | 'is-json'
-  | 'contains-json'
-  | 'javascript'
-  | 'python'
-  | 'similar'
-  | 'llm-rubric'
-  | 'webhook'
-  | 'rouge-n'
-  | 'rouge-s'
-  | 'rouge-l'
-  | 'levenshtein';
-type NotPrefixed<T extends string> = `not-${T}`;
-export type AssertionType = BaseAssertionTypes | NotPrefixed<BaseAssertionTypes>;
-// TODO(ian): maybe Assertion should support {type: config} to make the yaml cleaner
-export interface Assertion {
-  // Type of assertion
-  type: AssertionType;
-  // The expected value, if applicable
-  value?:
-    | string
-    | string[]
-    | object
-    | ((output: string, testCase: AtomicTestCase, assertion: Assertion) => Promise<GradingResult>);
-  // The threshold value, only applicable for similarity (cosine distance)
-  threshold?: number;
-  // The weight of this assertion compared to other assertions in the test case. Defaults to 1.
-  weight?: number;
-  // Some assertions (similarity, llm-rubric) require an LLM provider
-  provider?: GradingConfig['provider'];
-}
-// Each test case is graded pass/fail.  A test case represents a unique input to the LLM after substituting `vars` in the prompt.
-export interface TestCase {
-  // Optional description of what you're testing
-  description?: string;
-  // Key-value pairs to substitute in the prompt
-  vars?: Record<string, string | string[] | object>;
-  // Optional list of automatic checks to run on the LLM output
-  assert?: Assertion[];
-  // Additional configuration settings for the prompt
-  options?: PromptConfig & OutputConfig & GradingConfig;
-}
-export interface Scenario {
-  // Optional description of what you're testing
-  description?: string;
-  // Default test case config
-  config: Partial<TestCase>[];
-  // Optional list of automatic checks to run on the LLM output
-  tests: TestCase[];
-}
-// Same as a TestCase, except the `vars` object has been flattened into its final form.
-export interface AtomicTestCase extends TestCase {
-  vars?: Record<string, string | object>;
-}
-// The test suite defines the "knobs" that we are tuning in prompt engineering: providers and prompts
-export interface TestSuite {
-  // Optional description of what your LLM is trying to do
-  description?: string;
-  // One or more LLM APIs to use
-  providers: ApiProvider[];
-  // One or more prompt strings
-  prompts: Prompt[];
-  // Optional mapping of provider to prompt display strings.  If not provided,
-  // all prompts are used for all providers.
-  providerPromptMap?: Record<string, string[]>;
-  // Test cases
-  tests?: TestCase[];
-  // scenarios
-  scenarios?: Scenario[];
-  // Default test case config
-  defaultTest?: Partial<TestCase>;
-}
-export type ProviderId = string;
-export type ProviderFunction = (prompt: string) => Promise<ProviderResponse>;
-export type ProviderOptionsMap = Record<ProviderId, ProviderOptions>;
-// TestSuiteConfig = Test Suite, but before everything is parsed and resolved.  Providers are just strings, prompts are filepaths, tests can be filepath or inline.
-export interface TestSuiteConfig {
-  // Optional description of what your LLM is trying to do
-  description?: string;
-  // One or more LLM APIs to use, for example: openai:gpt-3.5-turbo, openai:gpt-4, localai:chat:vicuna
-  providers:
-    | ProviderId
-    | ProviderId[]
-    | ProviderOptionsMap[]
-    | ProviderOptions[]
-    | ProviderFunction;
-  // One or more prompt files to load
-  prompts: string | string[];
-  // Path to a test file, OR list of LLM prompt variations (aka "test case")
-  tests: string | string[] | TestCase[];
-  // Scenarios, groupings of data and tests to be evaluated
-  scenarios?: Scenario[];
-  // Sets the default properties for each test case. Useful for setting an assertion, on all test cases, for example.
-  defaultTest?: Omit<TestCase, 'description'>;
-  // Path to write output. Writes to console/web viewer if not set.
-  outputPath?: string;
-  // Determines whether or not sharing is enabled.
-  sharing?: boolean;
-}
-export type UnifiedConfig = TestSuiteConfig & {
-  evaluateOptions: EvaluateOptions;
-  commandLineOptions: Partial<CommandLineOptions>;
-};
-export interface SharedResults {
-  data: {
-    version: number;
-    results: EvaluateSummary;
-    config: Partial<UnifiedConfig>;
-  };
-}

package/src/updates.ts DELETED Viewed

@@ -1,46 +0,0 @@
-import chalk from 'chalk';
-import semverGt from 'semver/functions/gt';
-import logger from './logger';
-import { fetchWithTimeout } from './util';
-import packageJson from '../package.json';
-const VERSION = packageJson.version;
-export async function getLatestVersion(packageName: string) {
-  const response = await fetchWithTimeout(`https://registry.npmjs.org/${packageName}`, {}, 1000);
-  if (!response.ok) {
-    throw new Error(`Failed to fetch package information for ${packageName}`);
-  }
-  const data = await response.json();
-  return data['dist-tags'].latest;
-}
-export async function checkForUpdates(): Promise<boolean> {
-  if (process.env.PROMPTFOO_DISABLE_UPDATE) {
-    return false;
-  }
-  let latestVersion: string;
-  try {
-    latestVersion = await getLatestVersion('promptfoo');
-  } catch {
-    return false;
-  }
-  if (semverGt(latestVersion, VERSION)) {
-    const border = '='.repeat(process.stdout.columns - 10);
-    logger.info(
-      `\n${border}
-${chalk.yellow('⚠️')} The current version of promptfoo ${chalk.yellow(
-        VERSION,
-      )} is lower than the latest available version ${chalk.green(latestVersion)}.
-Please run ${chalk.green('npx promptfoo@latest')} or ${chalk.green(
-        'npm install -g promptfoo@latest',
-      )} to update.
-${border}\n`,
-    );
-    return true;
-  }
-  return false;
-}