promptfoo 0.121.3 → 0.121.5

package/dist/src/eval-7aEqoMs3.js

@@ -1,15 +0,0 @@
-#!/usr/bin/env node
-import "./logger-BcJBzSSA.js";
-import "./fetch-DoVRJZhJ.js";
-import "./accounts-CFLK3mnD.js";
-import "./cloud-z8KZpUoa.js";
-import "./types-CIhFeUC4.js";
-import "./utils-DEuL4VNB.js";
-import "./blobs-B1JriOyi.js";
-import "./tables-DmzvLbeZ.js";
-import "./extractor-CAZ2G3Kh.js";
-import "./store-P8OKm19S.js";
-import "./signal-C3ZTsUgi.js";
-import { t as Eval } from "./eval-BTqTn7lb.js";
-import "./evalResult-DUDShQrm.js";
-export { Eval as default };

package/dist/src/evalResult-CYNHkk5A.js

@@ -1,12 +0,0 @@
-#!/usr/bin/env node
-import "./logger-BcJBzSSA.js";
-import "./fetch-DoVRJZhJ.js";
-import "./accounts-CFLK3mnD.js";
-import "./cloud-z8KZpUoa.js";
-import "./types-CIhFeUC4.js";
-import "./utils-DEuL4VNB.js";
-import "./blobs-B1JriOyi.js";
-import "./tables-DmzvLbeZ.js";
-import "./extractor-CAZ2G3Kh.js";
-import { t as EvalResult } from "./evalResult-DUDShQrm.js";
-export { EvalResult as default };

package/dist/src/evalResult-CuvJeNiM.js

@@ -1,10 +0,0 @@
-import "./logger-BnkjG2jt.js";
-import "./fetch-BiYv2BZc.js";
-import "./accounts-Xatc0RYb.js";
-import "./tables-BEIFz2tM.js";
-import "./types-Cd3ygw8W.js";
-import "./utils-DOjD4dTC.js";
-import "./blobs-BUWmKWzo.js";
-import "./extractor-C8XwivI9.js";
-import { t as EvalResult } from "./server/index.js";
-export { EvalResult as default };

package/dist/src/evalResult-tGdilrWt.cjs

@@ -1,10 +0,0 @@
-require("./logger-D5iKBpu_.cjs");
-require("./types-D8cGDZbL.cjs");
-require("./fetch-BnR9wSnm.cjs");
-require("./utils-DKw8mrgr.cjs");
-require("./accounts-BPyfpSeU.cjs");
-require("./blobs-C6j0bvFz.cjs");
-require("./tables-BdZQEpRz.cjs");
-require("./extractor-DG3sSfXE.cjs");
-const require_evalResult = require("./evalResult-DpARzUCb.cjs");
-exports.default = require_evalResult.EvalResult;

package/dist/src/evaluator-BBUqRhz1.js

@@ -1,36 +0,0 @@
-#!/usr/bin/env node
-import "./logger-BcJBzSSA.js";
-import "./fetch-DoVRJZhJ.js";
-import "./accounts-CFLK3mnD.js";
-import "./cloud-z8KZpUoa.js";
-import "./telemetry-BnH5VJAU.js";
-import "./types-CIhFeUC4.js";
-import "./server-DZ9MtCn0.js";
-import "./providers-Ch6Mr0gn.js";
-import "./cache-DSqR6ezl.js";
-import "./util-Bm3E9jpK.js";
-import "./esm-7UIl0pPM.js";
-import "./pythonUtils-wIqk7zAf.js";
-import "./transform-ljLYHEPh.js";
-import "./graders-RjHF8VfG.js";
-import "./utils-DEuL4VNB.js";
-import "./genaiTracer-C1rxGO8Q.js";
-import "./chat-BEwdgGEg.js";
-import "./transform-DrleutM3.js";
-import "./messages-DJNo37Ko.js";
-import "./util-DMFeUvLz.js";
-import "./responses-BKP_WYis.js";
-import "./openai-BMcwgD5C.js";
-import "./util-DM2rTn_6.js";
-import "./completion-DoYy49ti.js";
-import "./blobs-B1JriOyi.js";
-import "./tables-DmzvLbeZ.js";
-import "./extractor-CAZ2G3Kh.js";
-import "./store-P8OKm19S.js";
-import "./base-CKjwebIH.js";
-import "./image-PoF6DN3x.js";
-import "./providerRegistry-B9lh-_tx.js";
-import "./rubyUtils-CiVfln3g.js";
-import { n as evaluate } from "./evaluator-BcvOGaam.js";
-import "./signal-C3ZTsUgi.js";
-export { evaluate };

package/dist/src/fetch-UWU706qb.js

@@ -1,5 +0,0 @@
-#!/usr/bin/env node
-import "./logger-BcJBzSSA.js";
-import { n as fetchWithProxy } from "./fetch-DoVRJZhJ.js";
-import "./cloud-z8KZpUoa.js";
-export { fetchWithProxy };

package/dist/src/graders-BxfEguVY.js

@@ -1,32 +0,0 @@
-import "./logger-BnkjG2jt.js";
-import "./fetch-BiYv2BZc.js";
-import "./accounts-Xatc0RYb.js";
-import "./server-Cns05F1j.js";
-import "./tables-BEIFz2tM.js";
-import "./esm-CKWP3u_P.js";
-import "./types-Cd3ygw8W.js";
-import "./utils-DOjD4dTC.js";
-import "./store-VB0GP46K.js";
-import "./cache-HP0NP4k3.js";
-import "./blobs-BUWmKWzo.js";
-import "./extractor-C8XwivI9.js";
-import "./providers-DvddrgxL.js";
-import "./telemetry-DPXLd7UE.js";
-import "./genaiTracer-70Z8BIuV.js";
-import "./pythonUtils-Bzwbgpbg.js";
-import "./util-BLvy9qfE.js";
-import "./chat-B0iaWhoh.js";
-import "./transform-B2-jIv68.js";
-import "./messages-biC_ex-p.js";
-import "./util-DbVG-yZU.js";
-import { n as getGraderById } from "./graders-BXAJ0sbS.js";
-import "./responses-CgNyTPsY.js";
-import "./openai-D6wITiVn.js";
-import "./util-BtoGs5Cb.js";
-import "./completion-BCimtq-h.js";
-import "./transform-BzK09Q_9.js";
-import "./base-Cz2ZC_iA.js";
-import "./image-B8b6f36E.js";
-import "./providerRegistry-BkzVH5Ba.js";
-import "./rubyUtils-DECSbsfY.js";
-export { getGraderById };

package/dist/src/graders-CzVMbEnv.js

@@ -1,34 +0,0 @@
-#!/usr/bin/env node
-import "./logger-BcJBzSSA.js";
-import "./fetch-DoVRJZhJ.js";
-import "./accounts-CFLK3mnD.js";
-import "./cloud-z8KZpUoa.js";
-import "./telemetry-BnH5VJAU.js";
-import "./types-CIhFeUC4.js";
-import "./server-DZ9MtCn0.js";
-import "./providers-Ch6Mr0gn.js";
-import "./cache-DSqR6ezl.js";
-import "./util-Bm3E9jpK.js";
-import "./esm-7UIl0pPM.js";
-import "./pythonUtils-wIqk7zAf.js";
-import "./transform-ljLYHEPh.js";
-import { n as getGraderById } from "./graders-RjHF8VfG.js";
-import "./utils-DEuL4VNB.js";
-import "./genaiTracer-C1rxGO8Q.js";
-import "./chat-BEwdgGEg.js";
-import "./transform-DrleutM3.js";
-import "./messages-DJNo37Ko.js";
-import "./util-DMFeUvLz.js";
-import "./responses-BKP_WYis.js";
-import "./openai-BMcwgD5C.js";
-import "./util-DM2rTn_6.js";
-import "./completion-DoYy49ti.js";
-import "./blobs-B1JriOyi.js";
-import "./tables-DmzvLbeZ.js";
-import "./extractor-CAZ2G3Kh.js";
-import "./store-P8OKm19S.js";
-import "./base-CKjwebIH.js";
-import "./image-PoF6DN3x.js";
-import "./providerRegistry-B9lh-_tx.js";
-import "./rubyUtils-CiVfln3g.js";
-export { getGraderById };

package/dist/src/graders-DjCXfj0l.cjs

@@ -1,32 +0,0 @@
-require("./logger-D5iKBpu_.cjs");
-require("./esm-CipptfDu.cjs");
-require("./pythonUtils-dAVigVK-.cjs");
-require("./transform-ZrG2dvlo.cjs");
-const require_graders = require("./graders-BElhu9ZY.cjs");
-require("./types-D8cGDZbL.cjs");
-require("./util-CuLo2pMR.cjs");
-require("./fetch-BnR9wSnm.cjs");
-require("./cache-C5yFZ4gC.cjs");
-require("./providers-CScd1wN6.cjs");
-require("./utils-DKw8mrgr.cjs");
-require("./genaiTracer-BfxrvSUb.cjs");
-require("./chat-CM8qWR3_.cjs");
-require("./transform-0BwoBsvO.cjs");
-require("./messages-HJsyEh4o.cjs");
-require("./util--9u9UVCt.cjs");
-require("./responses-mo0KQDbu.cjs");
-require("./openai-CoxGAQwn.cjs");
-require("./util-CFj4YKIn.cjs");
-require("./completion-DlXUhj5c.cjs");
-require("./accounts-BPyfpSeU.cjs");
-require("./server-BtoCXeXI.cjs");
-require("./blobs-C6j0bvFz.cjs");
-require("./tables-BdZQEpRz.cjs");
-require("./extractor-DG3sSfXE.cjs");
-require("./telemetry-re627Lre.cjs");
-require("./store-CLyU7AtI.cjs");
-require("./base-BboXIF_0.cjs");
-require("./image--F58eEIn.cjs");
-require("./providerRegistry-BTDgfV5h.cjs");
-require("./rubyUtils-CGeUtCfW.cjs");
-exports.getGraderById = require_graders.getGraderById;

package/dist/src/graders-kHzIWOKu.js

@@ -1,32 +0,0 @@
-import "./logger-DO8_zM18.js";
-import "./esm-SUNIX1x3.js";
-import "./pythonUtils-Cpo0Ez1p.js";
-import "./transform-BqPkNPYm.js";
-import { n as getGraderById } from "./graders-DG7mhg-b.js";
-import "./types-q8GXGF65.js";
-import "./util-CMMkIxfU.js";
-import "./fetch-CVAtKnI3.js";
-import "./cache-CaT5tPgo.js";
-import "./providers-Cn73d5sr.js";
-import "./utils-CFxO9KGo.js";
-import "./genaiTracer-D3fD9dNV.js";
-import "./chat-pxmiVpWe.js";
-import "./transform-DyDAwEpE.js";
-import "./messages-Dy9QecMs.js";
-import "./util-vNmDL5DT.js";
-import "./responses-CQb1Tj69.js";
-import "./openai-BcB5KlTk.js";
-import "./util-CgDCK4KI.js";
-import "./completion-DCjv7RZ3.js";
-import "./accounts-B2XmGjty.js";
-import "./server-CP9qKM40.js";
-import "./blobs-DXTl6J3H.js";
-import "./tables-kC7R5kiK.js";
-import "./extractor-D_wd8jxt.js";
-import "./telemetry-BugWqKiu.js";
-import "./store-Cj258DgL.js";
-import "./base-CqzQ4K8j.js";
-import "./image-CoxZp9PZ.js";
-import "./providerRegistry-CUWki5mQ.js";
-import "./rubyUtils-PgU-gHmx.js";
-export { getGraderById };

package/dist/src/messages-DJNo37Ko.js

@@ -1,246 +0,0 @@
-#!/usr/bin/env node
-import { E as getEnvString, T as getEnvInt, s as logger, w as getEnvFloat } from "./logger-BcJBzSSA.js";
-import { S as transformTools, x as transformToolChoice } from "./fetch-DoVRJZhJ.js";
-import { o as getCache, s as isCacheEnabled } from "./cache-DSqR6ezl.js";
-import { g as maybeLoadToolsFromExternalFile, h as maybeLoadResponseFormatFromExternalFile } from "./util-Bm3E9jpK.js";
-import { n as withGenAISpan } from "./genaiTracer-C1rxGO8Q.js";
-import { i as normalizeFinishReason, n as MCPClient } from "./chat-BEwdgGEg.js";
-import { a as createEmptyTokenUsage } from "./tokenUsageUtils-DflFMjS0.js";
-import { n as transformMCPToolsToAnthropic } from "./transform-DrleutM3.js";
-import { a as parseMessages, i as outputFromMessage, n as calculateAnthropicCost, o as processAnthropicTools, r as getTokenUsage, t as ANTHROPIC_MODELS } from "./util-DMFeUvLz.js";
-import Anthropic, { APIError } from "@anthropic-ai/sdk";
-//#region src/providers/anthropic/generic.ts
-/**
-* Generic provider class for Anthropic APIs
-* Serves as a base class with shared functionality for all Anthropic providers
-*/
-var AnthropicGenericProvider = class {
-	modelName;
-	config;
-	env;
-	apiKey;
-	anthropic;
-	constructor(modelName, options = {}) {
-		const { config, id, env } = options;
-		this.env = env;
-		this.modelName = modelName;
-		this.config = config || {};
-		this.apiKey = this.getApiKey();
-		this.anthropic = new Anthropic({
-			apiKey: this.apiKey,
-			baseURL: this.getApiBaseUrl()
-		});
-		this.id = id ? () => id : this.id;
-	}
-	id() {
-		return `anthropic:${this.modelName}`;
-	}
-	toString() {
-		return `[Anthropic Provider ${this.modelName}]`;
-	}
-	requiresApiKey() {
-		return true;
-	}
-	getApiKey() {
-		return this.config?.apiKey || this.env?.ANTHROPIC_API_KEY || getEnvString("ANTHROPIC_API_KEY");
-	}
-	getApiBaseUrl() {
-		return this.config?.apiBaseUrl || this.env?.ANTHROPIC_BASE_URL || getEnvString("ANTHROPIC_BASE_URL");
-	}
-	/**
-	* Base implementation - should be overridden by specific provider implementations
-	*/
-	async callApi(_prompt, _context) {
-		throw new Error("Not implemented: callApi must be implemented by subclasses");
-	}
-};
-//#endregion
-//#region src/providers/anthropic/messages.ts
-function parseEnvFloat(value) {
-	if (value === void 0) return;
-	const parsed = Number.parseFloat(value);
-	return Number.isNaN(parsed) ? void 0 : parsed;
-}
-var AnthropicMessagesProvider = class AnthropicMessagesProvider extends AnthropicGenericProvider {
-	mcpClient = null;
-	initializationPromise = null;
-	static ANTHROPIC_MODELS = ANTHROPIC_MODELS;
-	static ANTHROPIC_MODELS_NAMES = ANTHROPIC_MODELS.map((model) => model.id);
-	constructor(modelName, options = {}) {
-		if (!AnthropicMessagesProvider.ANTHROPIC_MODELS_NAMES.includes(modelName)) logger.warn(`Using unknown Anthropic model: ${modelName}`);
-		super(modelName, options);
-		const { id } = options;
-		this.id = id ? () => id : this.id;
-		if (this.config.mcp?.enabled) this.initializationPromise = this.initializeMCP();
-	}
-	async initializeMCP() {
-		this.mcpClient = new MCPClient(this.config.mcp);
-		await this.mcpClient.initialize();
-	}
-	async cleanup() {
-		if (this.mcpClient) {
-			await this.initializationPromise;
-			await this.mcpClient.cleanup();
-			this.mcpClient = null;
-		}
-	}
-	toString() {
-		if (!this.modelName) throw new Error("Anthropic model name is not set. Please provide a valid model name.");
-		return `[Anthropic Messages Provider ${this.modelName}]`;
-	}
-	async callApi(prompt, context) {
-		if (this.initializationPromise != null) await this.initializationPromise;
-		if (!this.apiKey) throw new Error("Anthropic API key is not set. Set the ANTHROPIC_API_KEY environment variable or add `apiKey` to the provider config.");
-		if (!this.modelName) throw new Error("Anthropic model name is not set. Please provide a valid model name.");
-		const spanContext = {
-			system: "anthropic",
-			operationName: "chat",
-			model: this.modelName,
-			providerId: this.id(),
-			maxTokens: this.config.max_tokens,
-			temperature: this.config.temperature,
-			testIndex: context?.test?.vars?.__testIdx,
-			promptLabel: context?.prompt?.label,
-			traceparent: context?.traceparent,
-			requestBody: prompt
-		};
-		const resultExtractor = (response) => {
-			const result = {};
-			if (response.tokenUsage) result.tokenUsage = {
-				prompt: response.tokenUsage.prompt,
-				completion: response.tokenUsage.completion,
-				total: response.tokenUsage.total,
-				cached: response.tokenUsage.cached
-			};
-			if (response.finishReason) result.finishReasons = [response.finishReason];
-			if (response.cached !== void 0) result.cacheHit = response.cached;
-			if (response.output !== void 0) result.responseBody = typeof response.output === "string" ? response.output : JSON.stringify(response.output);
-			return result;
-		};
-		return withGenAISpan(spanContext, () => this.callApiInternal(prompt, context), resultExtractor);
-	}
-	/**
-	* Internal implementation of callApi without tracing wrapper.
-	*/
-	async callApiInternal(prompt, context) {
-		const config = {
-			...this.config,
-			...context?.prompt?.config
-		};
-		const { system, extractedMessages, thinking } = parseMessages(prompt);
-		let mcpTools = [];
-		if (this.mcpClient) mcpTools = transformMCPToolsToAnthropic(this.mcpClient.getAllTools());
-		const { processedTools: processedConfigTools, requiredBetaFeatures } = processAnthropicTools(transformTools(await maybeLoadToolsFromExternalFile(config.tools, context?.vars) || [], "anthropic"));
-		const allTools = [...mcpTools, ...processedConfigTools];
-		const processedOutputFormat = maybeLoadResponseFormatFromExternalFile(config.output_format, context?.vars);
-		const shouldStream = config.stream ?? false;
-		const params = {
-			model: this.modelName,
-			...system ? { system } : {},
-			max_tokens: config?.max_tokens || getEnvInt("ANTHROPIC_MAX_TOKENS", config.thinking || thinking ? 2048 : 1024),
-			messages: extractedMessages,
-			stream: shouldStream,
-			temperature: config.thinking || thinking ? config.temperature : config.temperature ?? parseEnvFloat(this.env?.ANTHROPIC_TEMPERATURE) ?? getEnvFloat("ANTHROPIC_TEMPERATURE", 0),
-			...allTools.length > 0 ? { tools: allTools } : {},
-			...config.tool_choice ? { tool_choice: transformToolChoice(config.tool_choice, "anthropic") } : {},
-			...config.thinking || thinking ? { thinking: config.thinking || thinking } : {},
-			...processedOutputFormat || config.effort ? { output_config: {
-				...processedOutputFormat ? { format: processedOutputFormat } : {},
-				...config.effort ? { effort: config.effort } : {}
-			} } : {},
-			...typeof config?.extra_body === "object" && config.extra_body ? config.extra_body : {}
-		};
-		logger.debug("Calling Anthropic Messages API", { params });
-		const headers = { ...config.headers || {} };
-		let allBetaFeatures = [...config.beta || [], ...requiredBetaFeatures];
-		if (processedOutputFormat && !allBetaFeatures.includes("structured-outputs-2025-11-13")) allBetaFeatures.push("structured-outputs-2025-11-13");
-		allBetaFeatures = [...new Set(allBetaFeatures)];
-		if (allBetaFeatures.length > 0) headers["anthropic-beta"] = allBetaFeatures.join(",");
-		const cache = await getCache();
-		const cacheKey = `anthropic:${JSON.stringify(params)}`;
-		if (isCacheEnabled()) {
-			const cachedResponse = await cache.get(cacheKey);
-			if (cachedResponse) {
-				logger.debug(`Returning cached response for ${prompt}: ${cachedResponse}`);
-				try {
-					const parsedCachedResponse = JSON.parse(cachedResponse);
-					const finishReason = normalizeFinishReason(parsedCachedResponse.stop_reason);
-					let output = outputFromMessage(parsedCachedResponse, config.showThinking ?? true);
-					if (processedOutputFormat?.type === "json_schema" && typeof output === "string") try {
-						output = JSON.parse(output);
-					} catch (error) {
-						logger.error(`Failed to parse JSON output from structured outputs: ${error}`);
-					}
-					return {
-						output,
-						tokenUsage: getTokenUsage(parsedCachedResponse, true),
-						...finishReason && { finishReason },
-						cost: calculateAnthropicCost(this.modelName, config, parsedCachedResponse.usage?.input_tokens, parsedCachedResponse.usage?.output_tokens),
-						cached: true
-					};
-				} catch {
-					return {
-						output: cachedResponse,
-						tokenUsage: createEmptyTokenUsage()
-					};
-				}
-			}
-		}
-		try {
-			if (shouldStream) {
-				const finalMessage = await (await this.anthropic.messages.stream(params, { ...typeof headers === "object" && Object.keys(headers).length > 0 ? { headers } : {} })).finalMessage();
-				logger.debug(`Anthropic Messages API streaming complete`, { finalMessage });
-				if (isCacheEnabled()) try {
-					await cache.set(cacheKey, JSON.stringify(finalMessage));
-				} catch (err) {
-					logger.error(`Failed to cache response: ${String(err)}`);
-				}
-				const finishReason = normalizeFinishReason(finalMessage.stop_reason);
-				let output = outputFromMessage(finalMessage, config.showThinking ?? true);
-				if (processedOutputFormat?.type === "json_schema" && typeof output === "string") try {
-					output = JSON.parse(output);
-				} catch (error) {
-					logger.error(`Failed to parse JSON output from structured outputs: ${error}`);
-				}
-				return {
-					output,
-					tokenUsage: getTokenUsage(finalMessage, false),
-					...finishReason && { finishReason },
-					cost: calculateAnthropicCost(this.modelName, config, finalMessage.usage?.input_tokens, finalMessage.usage?.output_tokens)
-				};
-			} else {
-				const response = await this.anthropic.messages.create(params, { ...typeof headers === "object" && Object.keys(headers).length > 0 ? { headers } : {} });
-				logger.debug(`Anthropic Messages API response`, { response });
-				if (isCacheEnabled()) try {
-					await cache.set(cacheKey, JSON.stringify(response));
-				} catch (err) {
-					logger.error(`Failed to cache response: ${String(err)}`);
-				}
-				const finishReason = normalizeFinishReason(response.stop_reason);
-				let output = outputFromMessage(response, config.showThinking ?? true);
-				if (processedOutputFormat?.type === "json_schema" && typeof output === "string") try {
-					output = JSON.parse(output);
-				} catch (error) {
-					logger.error(`Failed to parse JSON output from structured outputs: ${error}`);
-				}
-				return {
-					output,
-					tokenUsage: getTokenUsage(response, false),
-					...finishReason && { finishReason },
-					cost: calculateAnthropicCost(this.modelName, config, response.usage?.input_tokens, response.usage?.output_tokens)
-				};
-			}
-		} catch (err) {
-			logger.error(`Anthropic Messages API call error: ${err instanceof Error ? err.message : String(err)}`);
-			if (err instanceof APIError && err.error) {
-				const errorDetails = err.error;
-				return { error: `API call error: ${errorDetails.error.message}, status ${err.status}, type ${errorDetails.error.type}` };
-			}
-			return { error: `API call error: ${err instanceof Error ? err.message : String(err)}` };
-		}
-	}
-};
-//#endregion
-export { AnthropicGenericProvider as n, AnthropicMessagesProvider as t };
-
-//# sourceMappingURL=messages-DJNo37Ko.js.map