promptfoo 0.125.0 → 0.125.1

package/dist/src/litellm-BBB5DQdt.cjs

@@ -0,0 +1,146 @@
+const require_env = require("./env-PwgLIVUT.cjs");
+const require_util = require("./util-CbYgIWBp.cjs");
+const require_chat = require("./chat-CwrurR57.cjs");
+const require_embedding = require("./embedding-DTrSDt7k.cjs");
+const require_completion = require("./completion-Cf2G9OlM.cjs");
+const require_creator = require("./creator-BFroofDZ.cjs");
+//#region src/providers/litellm.ts
+/**
+* Base class for LiteLLM providers that maintains LiteLLM identity
+*/
+var LiteLLMProviderWrapper = class {
+	provider;
+	providerType;
+	constructor(provider, providerType, id) {
+		this.provider = provider;
+		this.providerType = providerType;
+		if (id !== void 0) this.id = () => id;
+	}
+	get modelName() {
+		return this.provider.modelName;
+	}
+	get config() {
+		return this.provider.config;
+	}
+	id() {
+		return `litellm${this.providerType === "chat" ? "" : `:${this.providerType}`}:${this.modelName}`;
+	}
+	toString() {
+		return `[LiteLLM Provider${this.providerType === "chat" ? "" : ` ${this.providerType}`} ${this.modelName}]`;
+	}
+	toJSON() {
+		return {
+			provider: "litellm",
+			model: this.modelName,
+			type: this.providerType,
+			config: {
+				...this.config,
+				...this.getApiKey && this.getApiKey() && { apiKey: void 0 }
+			}
+		};
+	}
+	async callApi(prompt, context, options) {
+		const headers = this.providerType === "chat" ? context?.prompt?.config?.headers ?? this.config.headers : this.config.headers;
+		return this.withAuthHint(await this.provider.callApi(prompt, context, options), headers);
+	}
+	withAuthHint(response, headers = this.config.headers) {
+		if (!response.error || this.getApiKey?.() || require_util.hasOpenAiGatewayCredentials(headers, this.provider.getApiUrl()) || !/\b(?:401|unauthorized|authentication error|auth_error|invalid_api_key)\b/i.test(response.error.split("\n", 1)[0])) return response;
+		return {
+			...response,
+			error: `${response.error}\nNo LiteLLM API key was configured. Set LITELLM_API_KEY or the provider's apiKey or apiKeyEnvar. OPENAI_API_KEY is not used by default.`
+		};
+	}
+	getApiKey;
+};
+/**
+* LiteLLM Chat Provider
+*/
+var LiteLLMChatProvider = class extends LiteLLMProviderWrapper {
+	constructor(modelName, options) {
+		const provider = new require_chat.OpenAiChatCompletionProvider(modelName, options);
+		super(provider, "chat", options.id);
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+};
+/**
+* LiteLLM Completion Provider
+*/
+var LiteLLMCompletionProvider = class extends LiteLLMProviderWrapper {
+	constructor(modelName, options) {
+		const provider = new require_completion.OpenAiCompletionProvider(modelName, options);
+		super(provider, "completion", options.id);
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+};
+/**
+* LiteLLM Embedding Provider
+*/
+var LiteLLMEmbeddingProvider = class extends LiteLLMProviderWrapper {
+	supportsEmbeddingCancellation = true;
+	embeddingProvider;
+	constructor(modelName, options) {
+		const provider = new require_embedding.OpenAiEmbeddingProvider(modelName, options);
+		super(provider, "embedding", options.id);
+		this.embeddingProvider = provider;
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+	async callEmbeddingApi(text, context, options) {
+		return this.withAuthHint(await this.embeddingProvider.callEmbeddingApi(text, context, options));
+	}
+};
+var LiteLLMProvider = class extends LiteLLMChatProvider {};
+/**
+* Creates a LiteLLM provider using OpenAI-compatible endpoints
+*
+* LiteLLM supports chat, completion, and embedding models through its proxy server.
+* All parameters are automatically passed through to the LiteLLM API.
+*
+* @example
+* // Chat model (default)
+* createLiteLLMProvider('litellm:gpt-4')
+* createLiteLLMProvider('litellm:chat:gpt-4')
+*
+* // Completion model
+* createLiteLLMProvider('litellm:completion:gpt-3.5-turbo-instruct')
+*
+* // Embedding model
+* createLiteLLMProvider('litellm:embedding:text-embedding-3-large')
+*/
+function createLiteLLMProvider(providerPath, options = {}) {
+	const providerOptions = require_creator.resolveProviderCreatorInput({
+		...options,
+		id: options.config?.id ?? options.id
+	});
+	const splits = providerPath.split(":");
+	const providerType = splits[1];
+	const modelName = [
+		"chat",
+		"completion",
+		"embedding",
+		"embeddings"
+	].includes(providerType) ? splits.slice(2).join(":") : splits.slice(1).join(":");
+	const config = providerOptions.config || {};
+	const mergedConfig = {
+		apiKeyEnvar: "LITELLM_API_KEY",
+		apiKeyRequired: false,
+		apiBaseUrl: config.apiBaseUrl || require_env.resolveProviderEnv(providerOptions.env, ["LITELLM_API_BASE"])?.value || "http://0.0.0.0:4000",
+		omitDefaults: true
+	};
+	Object.keys(config).forEach((key) => {
+		if (config[key] !== void 0 && config[key] !== null) mergedConfig[key] = config[key];
+	});
+	const litellmConfig = {
+		...providerOptions,
+		config: mergedConfig
+	};
+	switch (providerType) {
+		case "completion": return new LiteLLMCompletionProvider(modelName, litellmConfig);
+		case "embedding":
+		case "embeddings": return new LiteLLMEmbeddingProvider(modelName, litellmConfig);
+		default: return new LiteLLMProvider(modelName, litellmConfig);
+	}
+}
+//#endregion
+exports.createLiteLLMProvider = createLiteLLMProvider;
+
+//# sourceMappingURL=litellm-BBB5DQdt.cjs.map

package/dist/src/litellm-Bo43ow7r.js

@@ -0,0 +1,145 @@
+import { t as resolveProviderEnv } from "./env-zrgUcH6g.js";
+import { j as hasOpenAiGatewayCredentials } from "./util-CsTD_ROg.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-ClTFM3qq.js";
+import { A as OpenAiEmbeddingProvider, p as OpenAiCompletionProvider } from "./server/index.js";
+import { t as resolveProviderCreatorInput } from "./creator--OvtY8Pn.js";
+//#region src/providers/litellm.ts
+/**
+* Base class for LiteLLM providers that maintains LiteLLM identity
+*/
+var LiteLLMProviderWrapper = class {
+	provider;
+	providerType;
+	constructor(provider, providerType, id) {
+		this.provider = provider;
+		this.providerType = providerType;
+		if (id !== void 0) this.id = () => id;
+	}
+	get modelName() {
+		return this.provider.modelName;
+	}
+	get config() {
+		return this.provider.config;
+	}
+	id() {
+		return `litellm${this.providerType === "chat" ? "" : `:${this.providerType}`}:${this.modelName}`;
+	}
+	toString() {
+		return `[LiteLLM Provider${this.providerType === "chat" ? "" : ` ${this.providerType}`} ${this.modelName}]`;
+	}
+	toJSON() {
+		return {
+			provider: "litellm",
+			model: this.modelName,
+			type: this.providerType,
+			config: {
+				...this.config,
+				...this.getApiKey && this.getApiKey() && { apiKey: void 0 }
+			}
+		};
+	}
+	async callApi(prompt, context, options) {
+		const headers = this.providerType === "chat" ? context?.prompt?.config?.headers ?? this.config.headers : this.config.headers;
+		return this.withAuthHint(await this.provider.callApi(prompt, context, options), headers);
+	}
+	withAuthHint(response, headers = this.config.headers) {
+		if (!response.error || this.getApiKey?.() || hasOpenAiGatewayCredentials(headers, this.provider.getApiUrl()) || !/\b(?:401|unauthorized|authentication error|auth_error|invalid_api_key)\b/i.test(response.error.split("\n", 1)[0])) return response;
+		return {
+			...response,
+			error: `${response.error}\nNo LiteLLM API key was configured. Set LITELLM_API_KEY or the provider's apiKey or apiKeyEnvar. OPENAI_API_KEY is not used by default.`
+		};
+	}
+	getApiKey;
+};
+/**
+* LiteLLM Chat Provider
+*/
+var LiteLLMChatProvider = class extends LiteLLMProviderWrapper {
+	constructor(modelName, options) {
+		const provider = new OpenAiChatCompletionProvider(modelName, options);
+		super(provider, "chat", options.id);
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+};
+/**
+* LiteLLM Completion Provider
+*/
+var LiteLLMCompletionProvider = class extends LiteLLMProviderWrapper {
+	constructor(modelName, options) {
+		const provider = new OpenAiCompletionProvider(modelName, options);
+		super(provider, "completion", options.id);
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+};
+/**
+* LiteLLM Embedding Provider
+*/
+var LiteLLMEmbeddingProvider = class extends LiteLLMProviderWrapper {
+	supportsEmbeddingCancellation = true;
+	embeddingProvider;
+	constructor(modelName, options) {
+		const provider = new OpenAiEmbeddingProvider(modelName, options);
+		super(provider, "embedding", options.id);
+		this.embeddingProvider = provider;
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+	async callEmbeddingApi(text, context, options) {
+		return this.withAuthHint(await this.embeddingProvider.callEmbeddingApi(text, context, options));
+	}
+};
+var LiteLLMProvider = class extends LiteLLMChatProvider {};
+/**
+* Creates a LiteLLM provider using OpenAI-compatible endpoints
+*
+* LiteLLM supports chat, completion, and embedding models through its proxy server.
+* All parameters are automatically passed through to the LiteLLM API.
+*
+* @example
+* // Chat model (default)
+* createLiteLLMProvider('litellm:gpt-4')
+* createLiteLLMProvider('litellm:chat:gpt-4')
+*
+* // Completion model
+* createLiteLLMProvider('litellm:completion:gpt-3.5-turbo-instruct')
+*
+* // Embedding model
+* createLiteLLMProvider('litellm:embedding:text-embedding-3-large')
+*/
+function createLiteLLMProvider(providerPath, options = {}) {
+	const providerOptions = resolveProviderCreatorInput({
+		...options,
+		id: options.config?.id ?? options.id
+	});
+	const splits = providerPath.split(":");
+	const providerType = splits[1];
+	const modelName = [
+		"chat",
+		"completion",
+		"embedding",
+		"embeddings"
+	].includes(providerType) ? splits.slice(2).join(":") : splits.slice(1).join(":");
+	const config = providerOptions.config || {};
+	const mergedConfig = {
+		apiKeyEnvar: "LITELLM_API_KEY",
+		apiKeyRequired: false,
+		apiBaseUrl: config.apiBaseUrl || resolveProviderEnv(providerOptions.env, ["LITELLM_API_BASE"])?.value || "http://0.0.0.0:4000",
+		omitDefaults: true
+	};
+	Object.keys(config).forEach((key) => {
+		if (config[key] !== void 0 && config[key] !== null) mergedConfig[key] = config[key];
+	});
+	const litellmConfig = {
+		...providerOptions,
+		config: mergedConfig
+	};
+	switch (providerType) {
+		case "completion": return new LiteLLMCompletionProvider(modelName, litellmConfig);
+		case "embedding":
+		case "embeddings": return new LiteLLMEmbeddingProvider(modelName, litellmConfig);
+		default: return new LiteLLMProvider(modelName, litellmConfig);
+	}
+}
+//#endregion
+export { createLiteLLMProvider };
+
+//# sourceMappingURL=litellm-Bo43ow7r.js.map

package/dist/src/litellm-CaU8CMLS.js

@@ -0,0 +1,146 @@
+import { t as resolveProviderEnv } from "./env-DG25rpzi.js";
+import { j as hasOpenAiGatewayCredentials } from "./util-CeGgoYs1.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-zCWuBKAj.js";
+import { t as OpenAiEmbeddingProvider } from "./embedding-j2ORNl5H.js";
+import { t as OpenAiCompletionProvider } from "./completion-Bub2N48b.js";
+import { t as resolveProviderCreatorInput } from "./creator--OvtY8Pn.js";
+//#region src/providers/litellm.ts
+/**
+* Base class for LiteLLM providers that maintains LiteLLM identity
+*/
+var LiteLLMProviderWrapper = class {
+	provider;
+	providerType;
+	constructor(provider, providerType, id) {
+		this.provider = provider;
+		this.providerType = providerType;
+		if (id !== void 0) this.id = () => id;
+	}
+	get modelName() {
+		return this.provider.modelName;
+	}
+	get config() {
+		return this.provider.config;
+	}
+	id() {
+		return `litellm${this.providerType === "chat" ? "" : `:${this.providerType}`}:${this.modelName}`;
+	}
+	toString() {
+		return `[LiteLLM Provider${this.providerType === "chat" ? "" : ` ${this.providerType}`} ${this.modelName}]`;
+	}
+	toJSON() {
+		return {
+			provider: "litellm",
+			model: this.modelName,
+			type: this.providerType,
+			config: {
+				...this.config,
+				...this.getApiKey && this.getApiKey() && { apiKey: void 0 }
+			}
+		};
+	}
+	async callApi(prompt, context, options) {
+		const headers = this.providerType === "chat" ? context?.prompt?.config?.headers ?? this.config.headers : this.config.headers;
+		return this.withAuthHint(await this.provider.callApi(prompt, context, options), headers);
+	}
+	withAuthHint(response, headers = this.config.headers) {
+		if (!response.error || this.getApiKey?.() || hasOpenAiGatewayCredentials(headers, this.provider.getApiUrl()) || !/\b(?:401|unauthorized|authentication error|auth_error|invalid_api_key)\b/i.test(response.error.split("\n", 1)[0])) return response;
+		return {
+			...response,
+			error: `${response.error}\nNo LiteLLM API key was configured. Set LITELLM_API_KEY or the provider's apiKey or apiKeyEnvar. OPENAI_API_KEY is not used by default.`
+		};
+	}
+	getApiKey;
+};
+/**
+* LiteLLM Chat Provider
+*/
+var LiteLLMChatProvider = class extends LiteLLMProviderWrapper {
+	constructor(modelName, options) {
+		const provider = new OpenAiChatCompletionProvider(modelName, options);
+		super(provider, "chat", options.id);
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+};
+/**
+* LiteLLM Completion Provider
+*/
+var LiteLLMCompletionProvider = class extends LiteLLMProviderWrapper {
+	constructor(modelName, options) {
+		const provider = new OpenAiCompletionProvider(modelName, options);
+		super(provider, "completion", options.id);
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+};
+/**
+* LiteLLM Embedding Provider
+*/
+var LiteLLMEmbeddingProvider = class extends LiteLLMProviderWrapper {
+	supportsEmbeddingCancellation = true;
+	embeddingProvider;
+	constructor(modelName, options) {
+		const provider = new OpenAiEmbeddingProvider(modelName, options);
+		super(provider, "embedding", options.id);
+		this.embeddingProvider = provider;
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+	async callEmbeddingApi(text, context, options) {
+		return this.withAuthHint(await this.embeddingProvider.callEmbeddingApi(text, context, options));
+	}
+};
+var LiteLLMProvider = class extends LiteLLMChatProvider {};
+/**
+* Creates a LiteLLM provider using OpenAI-compatible endpoints
+*
+* LiteLLM supports chat, completion, and embedding models through its proxy server.
+* All parameters are automatically passed through to the LiteLLM API.
+*
+* @example
+* // Chat model (default)
+* createLiteLLMProvider('litellm:gpt-4')
+* createLiteLLMProvider('litellm:chat:gpt-4')
+*
+* // Completion model
+* createLiteLLMProvider('litellm:completion:gpt-3.5-turbo-instruct')
+*
+* // Embedding model
+* createLiteLLMProvider('litellm:embedding:text-embedding-3-large')
+*/
+function createLiteLLMProvider(providerPath, options = {}) {
+	const providerOptions = resolveProviderCreatorInput({
+		...options,
+		id: options.config?.id ?? options.id
+	});
+	const splits = providerPath.split(":");
+	const providerType = splits[1];
+	const modelName = [
+		"chat",
+		"completion",
+		"embedding",
+		"embeddings"
+	].includes(providerType) ? splits.slice(2).join(":") : splits.slice(1).join(":");
+	const config = providerOptions.config || {};
+	const mergedConfig = {
+		apiKeyEnvar: "LITELLM_API_KEY",
+		apiKeyRequired: false,
+		apiBaseUrl: config.apiBaseUrl || resolveProviderEnv(providerOptions.env, ["LITELLM_API_BASE"])?.value || "http://0.0.0.0:4000",
+		omitDefaults: true
+	};
+	Object.keys(config).forEach((key) => {
+		if (config[key] !== void 0 && config[key] !== null) mergedConfig[key] = config[key];
+	});
+	const litellmConfig = {
+		...providerOptions,
+		config: mergedConfig
+	};
+	switch (providerType) {
+		case "completion": return new LiteLLMCompletionProvider(modelName, litellmConfig);
+		case "embedding":
+		case "embeddings": return new LiteLLMEmbeddingProvider(modelName, litellmConfig);
+		default: return new LiteLLMProvider(modelName, litellmConfig);
+	}
+}
+//#endregion
+export { createLiteLLMProvider };
+
+//# sourceMappingURL=litellm-CaU8CMLS.js.map

package/dist/src/litellm-wY7Unh8o.js

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+import { t as resolveProviderEnv } from "./env-BJ9OfjdV.js";
+import { j as hasOpenAiGatewayCredentials } from "./util-BOuXO5w-.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-DwP5kVnl.js";
+import { t as OpenAiEmbeddingProvider } from "./embedding-0UbzkTk9.js";
+import { t as OpenAiCompletionProvider } from "./completion-Bn7COFMk.js";
+import { t as resolveProviderCreatorInput } from "./creator-RI6vUEZK.js";
+//#region src/providers/litellm.ts
+/**
+* Base class for LiteLLM providers that maintains LiteLLM identity
+*/
+var LiteLLMProviderWrapper = class {
+	provider;
+	providerType;
+	constructor(provider, providerType, id) {
+		this.provider = provider;
+		this.providerType = providerType;
+		if (id !== void 0) this.id = () => id;
+	}
+	get modelName() {
+		return this.provider.modelName;
+	}
+	get config() {
+		return this.provider.config;
+	}
+	id() {
+		return `litellm${this.providerType === "chat" ? "" : `:${this.providerType}`}:${this.modelName}`;
+	}
+	toString() {
+		return `[LiteLLM Provider${this.providerType === "chat" ? "" : ` ${this.providerType}`} ${this.modelName}]`;
+	}
+	toJSON() {
+		return {
+			provider: "litellm",
+			model: this.modelName,
+			type: this.providerType,
+			config: {
+				...this.config,
+				...this.getApiKey && this.getApiKey() && { apiKey: void 0 }
+			}
+		};
+	}
+	async callApi(prompt, context, options) {
+		const headers = this.providerType === "chat" ? context?.prompt?.config?.headers ?? this.config.headers : this.config.headers;
+		return this.withAuthHint(await this.provider.callApi(prompt, context, options), headers);
+	}
+	withAuthHint(response, headers = this.config.headers) {
+		if (!response.error || this.getApiKey?.() || hasOpenAiGatewayCredentials(headers, this.provider.getApiUrl()) || !/\b(?:401|unauthorized|authentication error|auth_error|invalid_api_key)\b/i.test(response.error.split("\n", 1)[0])) return response;
+		return {
+			...response,
+			error: `${response.error}\nNo LiteLLM API key was configured. Set LITELLM_API_KEY or the provider's apiKey or apiKeyEnvar. OPENAI_API_KEY is not used by default.`
+		};
+	}
+	getApiKey;
+};
+/**
+* LiteLLM Chat Provider
+*/
+var LiteLLMChatProvider = class extends LiteLLMProviderWrapper {
+	constructor(modelName, options) {
+		const provider = new OpenAiChatCompletionProvider(modelName, options);
+		super(provider, "chat", options.id);
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+};
+/**
+* LiteLLM Completion Provider
+*/
+var LiteLLMCompletionProvider = class extends LiteLLMProviderWrapper {
+	constructor(modelName, options) {
+		const provider = new OpenAiCompletionProvider(modelName, options);
+		super(provider, "completion", options.id);
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+};
+/**
+* LiteLLM Embedding Provider
+*/
+var LiteLLMEmbeddingProvider = class extends LiteLLMProviderWrapper {
+	supportsEmbeddingCancellation = true;
+	embeddingProvider;
+	constructor(modelName, options) {
+		const provider = new OpenAiEmbeddingProvider(modelName, options);
+		super(provider, "embedding", options.id);
+		this.embeddingProvider = provider;
+		if (provider.getApiKey) this.getApiKey = provider.getApiKey.bind(provider);
+	}
+	async callEmbeddingApi(text, context, options) {
+		return this.withAuthHint(await this.embeddingProvider.callEmbeddingApi(text, context, options));
+	}
+};
+var LiteLLMProvider = class extends LiteLLMChatProvider {};
+/**
+* Creates a LiteLLM provider using OpenAI-compatible endpoints
+*
+* LiteLLM supports chat, completion, and embedding models through its proxy server.
+* All parameters are automatically passed through to the LiteLLM API.
+*
+* @example
+* // Chat model (default)
+* createLiteLLMProvider('litellm:gpt-4')
+* createLiteLLMProvider('litellm:chat:gpt-4')
+*
+* // Completion model
+* createLiteLLMProvider('litellm:completion:gpt-3.5-turbo-instruct')
+*
+* // Embedding model
+* createLiteLLMProvider('litellm:embedding:text-embedding-3-large')
+*/
+function createLiteLLMProvider(providerPath, options = {}) {
+	const providerOptions = resolveProviderCreatorInput({
+		...options,
+		id: options.config?.id ?? options.id
+	});
+	const splits = providerPath.split(":");
+	const providerType = splits[1];
+	const modelName = [
+		"chat",
+		"completion",
+		"embedding",
+		"embeddings"
+	].includes(providerType) ? splits.slice(2).join(":") : splits.slice(1).join(":");
+	const config = providerOptions.config || {};
+	const mergedConfig = {
+		apiKeyEnvar: "LITELLM_API_KEY",
+		apiKeyRequired: false,
+		apiBaseUrl: config.apiBaseUrl || resolveProviderEnv(providerOptions.env, ["LITELLM_API_BASE"])?.value || "http://0.0.0.0:4000",
+		omitDefaults: true
+	};
+	Object.keys(config).forEach((key) => {
+		if (config[key] !== void 0 && config[key] !== null) mergedConfig[key] = config[key];
+	});
+	const litellmConfig = {
+		...providerOptions,
+		config: mergedConfig
+	};
+	switch (providerType) {
+		case "completion": return new LiteLLMCompletionProvider(modelName, litellmConfig);
+		case "embedding":
+		case "embeddings": return new LiteLLMEmbeddingProvider(modelName, litellmConfig);
+		default: return new LiteLLMProvider(modelName, litellmConfig);
+	}
+}
+//#endregion
+export { createLiteLLMProvider };
+
+//# sourceMappingURL=litellm-wY7Unh8o.js.map