promptfoo 0.20.0 → 0.21.0

package/src/assertions.ts

@@ -1,580 +0,0 @@
-import rouge from 'rouge';
-import invariant from 'tiny-invariant';
-import Ajv from 'ajv';
-import { distance as levenshtein } from 'fastest-levenshtein';
-
-import telemetry from './telemetry';
-import { fetchWithRetries, getNunjucksEngine } from './util';
-import { matchesSimilarity, matchesLlmRubric } from './matchers';
-
-import type { Assertion, AssertionType, GradingResult, AtomicTestCase } from './types';
-
-const DEFAULT_SEMANTIC_SIMILARITY_THRESHOLD = 0.8;
-
-const ajv = new Ajv();
-const nunjucks = getNunjucksEngine();
-
-function handleRougeScore(
-  baseType: 'rouge-n',
-  assertion: Assertion,
-  expected: string | string[],
-  output: string,
-  inverted: boolean,
-): GradingResult {
-  const fnName = baseType[baseType.length - 1] as 'n' | 'l' | 's';
-  const rougeMethod = rouge[fnName];
-  const score = rougeMethod(output, expected);
-  const pass = score >= (assertion.threshold || 0.75) != inverted;
-
-  return {
-    pass,
-    score: inverted ? 1 - score : score,
-    reason: pass
-      ? `${baseType.toUpperCase()} score ${score} is greater than or equal to threshold ${
-          assertion.threshold || 0.75
-        }`
-      : `${baseType.toUpperCase()} score ${score} is less than threshold ${
-          assertion.threshold || 0.75
-        }`,
-    assertion,
-  };
-}
-
-export async function runAssertions(test: AtomicTestCase, output: string): Promise<GradingResult> {
-  const tokensUsed = {
-    total: 0,
-    prompt: 0,
-    completion: 0,
-  };
-
-  if (!test.assert || test.assert.length < 1) {
-    return { pass: true, score: 1, reason: 'No assertions', tokensUsed, assertion: null };
-  }
-
-  let totalScore = 0;
-  let totalWeight = 0;
-  let allPass = true;
-  let failedReason = '';
-  const componentResults: GradingResult[] = [];
-
-  for (const assertion of test.assert) {
-    const weight = assertion.weight || 1;
-    totalWeight += weight;
-
-    const result = await runAssertion(assertion, test, output);
-    totalScore += result.score * weight;
-    componentResults.push(result);
-
-    if (result.tokensUsed) {
-      tokensUsed.total += result.tokensUsed.total;
-      tokensUsed.prompt += result.tokensUsed.prompt;
-      tokensUsed.completion += result.tokensUsed.completion;
-    }
-
-    if (!result.pass) {
-      allPass = false;
-      failedReason = result.reason;
-      if (process.env.PROMPTFOO_SHORT_CIRCUIT_TEST_FAILURES) {
-        return result;
-      }
-    }
-  }
-
-  return {
-    pass: allPass,
-    score: totalScore / totalWeight,
-    reason: allPass ? 'All assertions passed' : failedReason,
-    tokensUsed,
-    componentResults,
-    assertion: null,
-  };
-}
-
-export async function runAssertion(
-  assertion: Assertion,
-  test: AtomicTestCase,
-  output: string,
-): Promise<GradingResult> {
-  let pass: boolean = false;
-  let score: number = 0.0;
-
-  invariant(assertion.type, `Assertion must have a type: ${JSON.stringify(assertion)}`);
-
-  const inverse = assertion.type.startsWith('not-');
-  const baseType = inverse ? assertion.type.slice(4) : assertion.type;
-
-  telemetry.record('assertion_used', {
-    type: baseType,
-  });
-
-  //render assertion values
-  let renderedValue = assertion.value;
-  // renderString for assertion values
-  if (renderedValue && typeof renderedValue === 'string') {
-    renderedValue = nunjucks.renderString(renderedValue, test.vars || {});
-  } else if (renderedValue && Array.isArray(renderedValue)) {
-    renderedValue = renderedValue.map((v) => nunjucks.renderString(v, test.vars || {}));
-  }
-
-  if (baseType === 'equals') {
-    pass = renderedValue === output;
-    return {
-      pass,
-      score: pass ? 1 : 0,
-      reason: pass ? 'Assertion passed' : `Expected output "${renderedValue}"`,
-      assertion,
-    };
-  }
-
-  if (baseType === 'is-json') {
-    let parsedJson;
-    try {
-      parsedJson = JSON.parse(output);
-      pass = !inverse;
-    } catch (err) {
-      pass = inverse;
-    }
-
-    if (pass && renderedValue) {
-      invariant(typeof renderedValue === 'object', 'is-json assertion must have an object value');
-      const validate = ajv.compile(renderedValue);
-      pass = validate(parsedJson);
-      if (!pass) {
-        return {
-          pass,
-          score: 0,
-          reason: `JSON does not conform to the provided schema. Errors: ${ajv.errorsText(
-            validate.errors,
-          )}`,
-          assertion,
-        };
-      }
-    }
-
-    return {
-      pass,
-      score: pass ? 1 : 0,
-      reason: pass ? 'Assertion passed' : 'Expected output to be valid JSON',
-      assertion,
-    };
-  }
-
-  if (baseType === 'contains') {
-    invariant(renderedValue, '"contains" assertion type must have a string or number value');
-    invariant(
-      typeof renderedValue === 'string' || typeof renderedValue === 'number',
-      '"contains" assertion type must have a string or number value',
-    );
-    pass = output.includes(String(renderedValue)) !== inverse;
-    return {
-      pass,
-      score: pass ? 1 : 0,
-      reason: pass
-        ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
-      assertion,
-    };
-  }
-
-  if (baseType === 'contains-any') {
-    invariant(renderedValue, '"contains-any" assertion type must have a value');
-    invariant(
-      Array.isArray(renderedValue),
-      '"contains-any" assertion type must have an array value',
-    );
-    pass = renderedValue.some((value) => output.includes(value)) !== inverse;
-    return {
-      pass,
-      score: pass ? 1 : 0,
-      reason: pass
-        ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain one of "${renderedValue.join(', ')}"`,
-      assertion,
-    };
-  }
-
-  if (baseType === 'contains-all') {
-    invariant(renderedValue, '"contains-all" assertion type must have a value');
-    invariant(
-      Array.isArray(renderedValue),
-      '"contains-all" assertion type must have an array value',
-    );
-    pass = renderedValue.every((value) => output.includes(value)) !== inverse;
-    return {
-      pass,
-      score: pass ? 1 : 0,
-      reason: pass
-        ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain all of "${renderedValue.join(', ')}"`,
-      assertion,
-    };
-  }
-
-  if (baseType === 'regex') {
-    invariant(renderedValue, '"regex" assertion type must have a string value');
-    invariant(
-      typeof renderedValue === 'string',
-      '"contains" assertion type must have a string value',
-    );
-    const regex = new RegExp(renderedValue);
-    pass = regex.test(output) !== inverse;
-    return {
-      pass,
-      score: pass ? 1 : 0,
-      reason: pass
-        ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}match regex "${renderedValue}"`,
-      assertion,
-    };
-  }
-
-  if (baseType === 'icontains') {
-    invariant(renderedValue, '"icontains" assertion type must have a string or number value');
-    invariant(
-      typeof renderedValue === 'string' || typeof renderedValue === 'number',
-      '"icontains" assertion type must have a string or number value',
-    );
-    pass = output.toLowerCase().includes(String(renderedValue).toLowerCase()) !== inverse;
-    return {
-      pass,
-      score: pass ? 1 : 0,
-      reason: pass
-        ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
-      assertion,
-    };
-  }
-
-  if (baseType === 'starts-with') {
-    invariant(renderedValue, '"starts-with" assertion type must have a string value');
-    invariant(
-      typeof renderedValue === 'string',
-      '"starts-with" assertion type must have a string value',
-    );
-    pass = output.startsWith(String(renderedValue)) !== inverse;
-    return {
-      pass,
-      score: pass ? 1 : 0,
-      reason: pass
-        ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}start with "${renderedValue}"`,
-      assertion,
-    };
-  }
-
-  if (baseType === 'contains-json') {
-    const jsonMatch = containsJSON(output);
-    pass = jsonMatch !== inverse;
-
-    if (pass && renderedValue) {
-      invariant(
-        typeof renderedValue === 'object',
-        'contains-json assertion must have an object value',
-      );
-      const validate = ajv.compile(renderedValue);
-      pass = validate(jsonMatch);
-      if (!pass) {
-        return {
-          pass,
-          score: 0,
-          reason: `JSON does not conform to the provided schema. Errors: ${ajv.errorsText(
-            validate.errors,
-          )}`,
-          assertion,
-        };
-      }
-    }
-
-    return {
-      pass,
-      score: pass ? 1 : 0,
-      reason: pass ? 'Assertion passed' : 'Expected output to contain valid JSON',
-      assertion,
-    };
-  }
-
-  const context = {
-    vars: test.vars || {},
-  };
-
-  if (baseType === 'javascript') {
-    try {
-      if (typeof assertion.value === 'function') {
-        return assertion.value(output, test, assertion);
-      }
-      invariant(typeof renderedValue === 'string', 'javascript assertion must have a string value');
-      const functionBody = renderedValue.includes('\n') ? renderedValue : `return ${renderedValue}`;
-      const customFunction = new Function('output', 'context', functionBody);
-      const result = customFunction(output, context) as any;
-      if (typeof result === 'boolean') {
-        pass = result !== inverse;
-        score = 1.0;
-      } else if (typeof result === 'number') {
-        pass = true;
-        score = result;
-      } else if (typeof result === 'object') {
-        return result;
-      } else {
-        throw new Error('Custom function must return a boolean or number');
-      }
-    } catch (err) {
-      return {
-        pass: false,
-        score: 0,
-        reason: `Custom function threw error: ${(err as Error).message}
-${renderedValue}`,
-        assertion,
-      };
-    }
-    return {
-      pass,
-      score,
-      reason: pass
-        ? 'Assertion passed'
-        : `Custom function returned ${inverse ? 'true' : 'false'}
-${renderedValue}`,
-      assertion,
-    };
-  }
-
-  if (baseType === 'python') {
-    try {
-      const { execSync } = require('child_process');
-      const escapedOutput = output.replace(/'/g, "\\'").replace(/"/g, '\\"');
-      const escapedContext = JSON.stringify(context).replace(/'/g, "\\'").replace(/"/g, '\\"');
-      const result = execSync(
-        `python -c "import json; import math; import os; import sys; import re; import datetime; import random; import collections; output='${escapedOutput}'; context='${escapedContext}'; print(json.dumps(${assertion.value}))"`,
-      )
-        .toString()
-        .trim();
-      if (result === 'true') {
-        pass = true;
-        score = 1.0;
-      } else if (result === 'false') {
-        pass = false;
-        score = 0.0;
-      } else if (result.startsWith('{')) {
-        return JSON.parse(result);
-      } else {
-        pass = true;
-        score = parseFloat(result);
-        if (isNaN(score)) {
-          throw new Error(
-            'Python code must return a boolean, number, or {pass, score, reason} object',
-          );
-        }
-      }
-    } catch (err) {
-      return {
-        pass: false,
-        score: 0,
-        reason: `Python code execution failed: ${(err as Error).message}`,
-        assertion,
-      };
-    }
-    return {
-      pass,
-      score,
-      reason: pass
-        ? 'Assertion passed'
-        : `Python code returned ${pass ? 'true' : 'false'}
-${assertion.value}`,
-      assertion,
-    };
-  }
-
-  if (baseType === 'similar') {
-    invariant(renderedValue, 'Similarity assertion must have a string value');
-    invariant(
-      typeof renderedValue === 'string',
-      '"contains" assertion type must have a string value',
-    );
-    return {
-      assertion,
-      ...(await matchesSimilarity(renderedValue, output, assertion.threshold || 0.75, inverse)),
-    };
-  }
-
-  if (baseType === 'llm-rubric') {
-    invariant(renderedValue, 'Similarity assertion must have a string value');
-    invariant(
-      typeof renderedValue === 'string',
-      '"contains" assertion type must have a string value',
-    );
-
-    // Assertion provider overrides test provider
-    test.options = test.options || {};
-    test.options.provider = assertion.provider || test.options.provider;
-
-    return {
-      assertion,
-      ...(await matchesLlmRubric(renderedValue, output, test.options)),
-    };
-  }
-
-  if (baseType === 'webhook') {
-    invariant(renderedValue, '"webhook" assertion type must have a URL value');
-    invariant(typeof renderedValue === 'string', '"webhook" assertion type must have a URL value');
-
-    try {
-      const context = {
-        vars: test.vars || {},
-      };
-      const response = await fetchWithRetries(
-        renderedValue,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-          },
-          body: JSON.stringify({ output, context }),
-        },
-        process.env.WEBHOOK_TIMEOUT ? parseInt(process.env.WEBHOOK_TIMEOUT, 10) : 5000,
-      );
-
-      if (!response.ok) {
-        throw new Error(`Webhook response status: ${response.status}`);
-      }
-
-      const jsonResponse = await response.json();
-      pass = jsonResponse.pass !== inverse;
-      score =
-        typeof jsonResponse.score === 'undefined'
-          ? pass
-            ? 1
-            : 0
-          : inverse
-          ? 1 - jsonResponse.score
-          : jsonResponse.score;
-    } catch (err) {
-      return {
-        pass: false,
-        score: 0,
-        reason: `Webhook error: ${(err as Error).message}`,
-        assertion,
-      };
-    }
-
-    return {
-      pass,
-      score,
-      reason: pass ? 'Assertion passed' : `Webhook returned ${inverse ? 'true' : 'false'}`,
-      assertion,
-    };
-  }
-
-  if (baseType === 'rouge-n') {
-    invariant(
-      typeof renderedValue === 'string' || Array.isArray(renderedValue),
-      '"rouge" assertion type must be a value (string or string array)',
-    );
-    return handleRougeScore(baseType, assertion, renderedValue, output, inverse);
-  }
-
-  if (baseType === 'levenshtein') {
-    invariant(
-      typeof renderedValue === 'string',
-      '"levenshtein" assertion type must have a string value',
-    );
-    const levDistance = levenshtein(output, renderedValue);
-    pass = levDistance <= (assertion.threshold || 5);
-    return {
-      pass,
-      score: pass ? 1 : 0,
-      reason: pass
-        ? 'Assertion passed'
-        : `Levenshtein distance ${levDistance} is greater than threshold ${
-            assertion.threshold || 5
-          }`,
-      assertion,
-    };
-  }
-
-  throw new Error('Unknown assertion type: ' + assertion.type);
-}
-
-function containsJSON(str: string): boolean {
-  // Regular expression to check for JSON-like pattern
-  const jsonPattern = /({[\s\S]*}|\[[\s\S]*])/;
-
-  const match = str.match(jsonPattern);
-
-  if (!match) {
-    return false;
-  }
-
-  try {
-    return JSON.parse(match[0]);
-  } catch (error) {
-    return false;
-  }
-}
-
-export function assertionFromString(expected: string): Assertion {
-  // Legacy options
-  if (expected.startsWith('fn:') || expected.startsWith('eval:')) {
-    // TODO(1.0): delete eval: legacy option
-    const sliceLength = expected.startsWith('fn:') ? 'fn:'.length : 'eval:'.length;
-    const functionBody = expected.slice(sliceLength);
-    return {
-      type: 'javascript',
-      value: functionBody,
-    };
-  }
-  if (expected.startsWith('grade:')) {
-    return {
-      type: 'llm-rubric',
-      value: expected.slice(6),
-    };
-  }
-
-  // New options
-  const assertionRegex =
-    /^(not-)?(equals|contains-any|contains-all|contains-json|is-json|regex|icontains|contains|webhook|rouge-n|similar|starts-with|levenshtein)(?:\((\d+(?:\.\d+)?)\))?(?::(.*))?$/;
-  const regexMatch = expected.match(assertionRegex);
-
-  if (regexMatch) {
-    const [_, notPrefix, type, thresholdStr, value] = regexMatch;
-    const fullType = notPrefix ? `not-${type}` : type;
-    const threshold = parseFloat(thresholdStr);
-
-    if (type === 'contains-any' || type === 'contains-all') {
-      return {
-        type: fullType as AssertionType,
-        value: value.split(',').map((s) => s.trim()),
-      };
-    } else if (type === 'contains-json' || type === 'is-json') {
-      return {
-        type: fullType as AssertionType,
-      };
-    } else if (
-      type === 'rouge-n' ||
-      type === 'similar' ||
-      type === 'starts-with' ||
-      type === 'levenshtein'
-    ) {
-      return {
-        type: fullType as AssertionType,
-        value,
-        threshold: threshold || (type === 'similar' ? DEFAULT_SEMANTIC_SIMILARITY_THRESHOLD : 0.75),
-      };
-    } else {
-      return {
-        type: fullType as AssertionType,
-        value,
-      };
-    }
-  }
-
-  // Default to equality
-  return {
-    type: 'equals',
-    value: expected,
-  };
-}
-
-// These exports are used by the node.js package (index.ts)
-export default {
-  matchesSimilarity,
-  matchesLlmRubric,
-};

package/src/cache.ts

@@ -1,109 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-
-import cacheManager from 'cache-manager';
-import fsStore from 'cache-manager-fs-hash';
-
-import logger from './logger';
-import { getConfigDirectoryPath, fetchWithRetries } from './util';
-
-import type { Cache } from 'cache-manager';
-import type { RequestInfo, RequestInit } from 'node-fetch';
-
-let cacheInstance: Cache | undefined;
-
-let enabled =
-  typeof process.env.PROMPTFOO_CACHE_ENABLED === 'undefined'
-    ? true
-    : Boolean(process.env.PROMPTFOO_CACHE_ENABLED);
-
-const cacheType =
-  process.env.PROMPTFOO_CACHE_TYPE || (process.env.NODE_ENV === 'test' ? 'memory' : 'disk');
-
-export function getCache() {
-  if (!cacheInstance) {
-    const cachePath =
-      process.env.PROMPTFOO_CACHE_PATH || path.join(getConfigDirectoryPath(), 'cache');
-    if (!fs.existsSync(cachePath)) {
-      logger.info(`Creating cache folder at ${cachePath}.`);
-      fs.mkdirSync(cachePath, { recursive: true });
-    }
-    cacheInstance = cacheManager.caching({
-      store: cacheType === 'disk' ? fsStore : 'memory',
-      options: {
-        max: process.env.PROMPTFOO_CACHE_MAX_FILE_COUNT || 10_000, // number of files
-        path: cachePath,
-        ttl: process.env.PROMPTFOO_CACHE_TTL || 60 * 60 * 24 * 14, // in seconds, 14 days
-        maxsize: process.env.PROMPTFOO_CACHE_MAX_SIZE || 1e7, // in bytes, 10mb
-        //zip: true, // whether to use gzip compression
-      },
-    });
-  }
-  return cacheInstance;
-}
-
-export async function fetchWithCache(
-  url: RequestInfo,
-  options: RequestInit = {},
-  timeout: number,
-  format: 'json' | 'text' = 'json',
-): Promise<{ data: any; cached: boolean }> {
-  if (!enabled) {
-    const resp = await fetchWithRetries(url, options, timeout);
-    return {
-      cached: false,
-      data: await resp.json(),
-    };
-  }
-
-  const cache = await getCache();
-
-  const copy = Object.assign({}, options);
-  delete copy.headers;
-  const cacheKey = `fetch:${url}:${JSON.stringify(copy)}`;
-
-  // Try to get the cached response
-  const cachedResponse = await cache.get(cacheKey);
-
-  if (cachedResponse) {
-    logger.debug(`Returning cached response for ${url}: ${cachedResponse}`);
-    return {
-      cached: true,
-      data: JSON.parse(cachedResponse as string),
-    };
-  }
-
-  // Fetch the actual data and store it in the cache
-  const response = await fetchWithRetries(url, options, timeout);
-  try {
-    const data = format === 'json' ? await response.json() : await response.text();
-    if (response.ok) {
-      logger.debug(`Storing ${url} response in cache: ${JSON.stringify(data)}`);
-      await cache.set(cacheKey, JSON.stringify(data));
-    }
-    return {
-      cached: false,
-      data,
-    };
-  } catch (err) {
-    throw new Error(`Error parsing response from ${url}: ${err}`);
-  }
-}
-
-export function enableCache() {
-  enabled = true;
-}
-
-export function disableCache() {
-  logger.info('Cache is disabled.');
-  enabled = false;
-}
-
-export async function clearCache() {
-  logger.info('Clearing cache...');
-  return getCache().reset();
-}
-
-export function isCacheEnabled() {
-  return enabled;
-}

package/src/esm.ts

@@ -1,13 +0,0 @@
-// esm-specific crap that needs to get mocked out in tests
-
-//import path from 'path';
-//import { fileURLToPath } from 'url';
-
-export function getDirectory(): string {
-  /*
-  // @ts-ignore: Jest chokes on this
-  const __filename = fileURLToPath(import.meta.url);
-  return path.dirname(__filename);
- */
-  return __dirname;
-}