promptfoo 0.125.0 → 0.125.1

package/dist/src/typesafe-DYM-SFNH.js

@@ -1,253 +0,0 @@
-import { Z as getProviderEnvString, o as logger, q as getEnvString } from "./logger-CNHLbG5Y.js";
-import { C as formatRateLimitErrorMessage, o as rateLimitTimingFromHeaders, y as HttpRateLimitError } from "./fetch-5HtnNhHg.js";
-import { a as getRequestTimeoutMs } from "./shared-BnN4RWgk.js";
-import { i as fetchWithCache } from "./cache-Cbw1zH6q.js";
-import { t as ellipsize } from "./text-Fd7EOhdv.js";
-//#region src/providers/typesafe.ts
-/**
-* TypeSafe provider for Jev, a "System One" model. Jev does not generate text: it answers
-* typed questions (Noul, Score, Choice) about a `state` with calibrated probabilities.
-*
-* - As an `llm-rubric` grader, the rubric is asked about the graded output as a yes/no
-*   Noul question, or as a Score question when `levels` is configured.
-* - As a `classifier` provider, `instructions` and `labels` form a Choice question.
-* - As a regular provider, the prompt is the state, `questions` is sent as configured, and
-*   the output is Jev's answers.
-*
-* API reference: https://docs.typesafe.ai/api
-*/
-const DEFAULT_API_BASE_URL = "https://api.typesafe.ai";
-const DEFAULT_THRESHOLD = .5;
-const DEFAULT_RETRY_AFTER_MS = 2e3;
-const UNSUPPORTED_GRADER_LABELS = ["agent-rubric", "trajectory:goal-success"];
-const INPUT_COST_PER_TOKEN = { "jev-1.13.0": .042 / 1e6 };
-function isPlainObject(value) {
-	return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-function isFiniteNumber(value) {
-	return typeof value === "number" && Number.isFinite(value);
-}
-/**
-* Jev rejects content that is not a string, object, or array. Text that holds a JSON
-* object or array is sent as structure, which Jev can address by field name.
-*/
-function toEntry(value) {
-	if (typeof value === "string") {
-		try {
-			const parsed = JSON.parse(value);
-			if (typeof parsed === "object" && parsed !== null) return parsed;
-		} catch {}
-		return value;
-	}
-	if (typeof value === "object" && value !== null) return value;
-	return value === void 0 ? "" : JSON.stringify(value);
-}
-/**
-* TypeSafe reports errors as `{detail}`, where `detail` is a string, an
-* `{error_type, message}` object, or a list of field validation errors.
-*/
-function formatErrorDetail(data) {
-	const detail = isPlainObject(data) && data.detail != null ? data.detail : data;
-	if (typeof detail === "string") return detail;
-	if (Array.isArray(detail)) return detail.map((item) => isPlainObject(item) && Array.isArray(item.loc) && typeof item.msg === "string" ? `${item.loc.join(".")}: ${item.msg}` : JSON.stringify(item)).join("; ");
-	if (isPlainObject(detail)) {
-		const text = [detail.message, detail.error_type].find((value) => typeof value === "string");
-		if (text) return text;
-	}
-	return JSON.stringify(detail) ?? "";
-}
-/** A failure the scheduler should back off from and retry, unless it is a hard quota. */
-function rateLimited(error, http, retryAfterMs, rateLimitKind = "rate_limit") {
-	const delay = String(Math.ceil(retryAfterMs ?? DEFAULT_RETRY_AFTER_MS));
-	return {
-		error,
-		metadata: {
-			rateLimitKind,
-			http: {
-				...http,
-				headers: { "retry-after-ms": delay }
-			}
-		}
-	};
-}
-var TypeSafeProvider = class {
-	modelName;
-	config;
-	env;
-	constructor(modelName, options = {}) {
-		const { config, id, env } = options;
-		this.modelName = modelName;
-		this.config = config || {};
-		this.env = env;
-		this.id = id ? () => id : this.id;
-	}
-	id() {
-		return `typesafe:${this.modelName}`;
-	}
-	toString() {
-		return `[TypeSafe Provider ${this.modelName}]`;
-	}
-	requiresApiKey() {
-		return true;
-	}
-	getApiKey() {
-		return this.config.apiKey || getProviderEnvString(this.env, "TYPESAFE_API_KEY") || getEnvString("TYPESAFE_API_KEY");
-	}
-	async callApi(prompt, context, options) {
-		const config = {
-			...this.config,
-			...context?.prompt?.config
-		};
-		const bustCache = context?.bustCache ?? context?.debug;
-		const abortSignal = options?.abortSignal;
-		const label = context?.prompt?.label;
-		if (label === "llm-rubric" && context?.vars?.rubric !== void 0) return this.grade(context.vars.rubric, context.vars.output, config, bustCache, abortSignal);
-		if (label && UNSUPPORTED_GRADER_LABELS.includes(label)) return { error: `TypeSafe provider ${this.id()} cannot grade \`${label}\` assertions. Jev grades only \`llm-rubric\` and \`classifier\`.` };
-		const { questions } = config;
-		if (!isPlainObject(questions) || Object.keys(questions).length === 0) return { error: `TypeSafe provider ${this.id()} needs \`questions\` in its config to answer prompts. To grade with Jev instead, use it as the provider for \`llm-rubric\` or \`classifier\` assertions.` };
-		const result = await this.ask(toEntry(prompt), questions, bustCache, abortSignal);
-		if ("error" in result) return result;
-		return {
-			...result.response,
-			output: result.answers
-		};
-	}
-	async callClassificationApi(prompt) {
-		const { instructions, labels } = this.config;
-		if (!instructions || !labels) return { error: `TypeSafe provider ${this.id()} needs \`instructions\` and \`labels\` in its config to classify, for example \`instructions: Which team should handle this?\` and \`labels: [billing, technical, sales]\`.` };
-		const criteria = Array.isArray(labels) ? Object.fromEntries(labels.map((label) => [label, null])) : labels;
-		const result = await this.ask(toEntry(prompt), { classification: {
-			type: "choice",
-			instructions,
-			criteria
-		} });
-		if ("error" in result) return result;
-		const { probabilities } = result.answers.classification;
-		if (!isPlainObject(probabilities) || !Object.values(probabilities).every(isFiniteNumber)) return result.malformed();
-		return { classification: probabilities };
-	}
-	async grade(rubric, output, config, bustCache, abortSignal) {
-		const { levels } = config;
-		const threshold = config.threshold ?? DEFAULT_THRESHOLD;
-		if (typeof threshold !== "number" || !(threshold >= 0 && threshold <= 1)) return { error: `TypeSafe \`threshold\` must be a number from 0 to 1, got ${JSON.stringify(threshold)}.` };
-		if (levels !== void 0 && (!Array.isArray(levels) || levels.length < 2)) return { error: "TypeSafe `levels` must list at least two Score levels, ordered low to high." };
-		const instructions = toEntry(rubric);
-		const question = levels ? {
-			type: "score",
-			instructions,
-			criteria: levels.map(toEntry)
-		} : {
-			type: "noul",
-			instructions
-		};
-		const result = await this.ask(toEntry(output), { grade: question }, bustCache, abortSignal);
-		if ("error" in result) return result;
-		const answer = result.answers.grade;
-		const raw = levels ? answer.score : answer.noul;
-		const top = levels ? levels.length - 1 : 1;
-		if (!isFiniteNumber(raw) || raw < 0 || raw > top) return result.malformed();
-		const score = Number((raw / top).toFixed(6));
-		const pass = score >= threshold;
-		const comparison = `${pass ? ">=" : "<"} threshold ${threshold}`;
-		const reason = levels ? `Jev Score ${raw} on levels 0–${top} (${score} normalized) ${comparison}; nearest level: ${ellipsize(JSON.stringify(levels[Math.round(raw)]) ?? "", 200)}` : `Jev Noul probability ${raw} ${comparison}`;
-		return {
-			...result.response,
-			output: {
-				pass,
-				score,
-				reason
-			},
-			metadata: { typesafe: {
-				...result.response.metadata.typesafe,
-				answer
-			} }
-		};
-	}
-	/** Ask Jev `questions` about `state`. */
-	async ask(state, questions, bustCache, abortSignal) {
-		const apiKey = this.getApiKey();
-		if (!apiKey) return { error: "TypeSafe API key is not set. Set the TYPESAFE_API_KEY environment variable or add `apiKey` to the provider config." };
-		logger.debug("[TypeSafe] Calling Jev", {
-			model: this.modelName,
-			questions: Object.keys(questions)
-		});
-		let fetched;
-		try {
-			fetched = await fetchWithCache(`${this.config.apiBaseUrl || DEFAULT_API_BASE_URL}/v1/systemone`, {
-				method: "POST",
-				headers: {
-					"Content-Type": "application/json",
-					Authorization: `Bearer ${apiKey}`
-				},
-				body: JSON.stringify({
-					state,
-					model: this.modelName,
-					questions
-				}),
-				...abortSignal ? { signal: abortSignal } : {}
-			}, getRequestTimeoutMs(), "text", bustCache);
-		} catch (err) {
-			abortSignal?.throwIfAborted();
-			if (err instanceof HttpRateLimitError) {
-				const detail = ellipsize(formatErrorDetail(err.body), 300);
-				return rateLimited(`TypeSafe API error: ${formatRateLimitErrorMessage(err, detail)}`, {
-					status: err.status,
-					statusText: err.statusText
-				}, err.retryAfterMs, err.kind);
-			}
-			return { error: `TypeSafe API call error: ${String(err)}` };
-		}
-		const { status, statusText, headers = {}, cached, latencyMs, deleteFromCache } = fetched;
-		const requestId = headers["x-typesafe-request-id"];
-		const requestIdText = requestId ? ` (request id ${requestId})` : "";
-		let data = fetched.data;
-		try {
-			data = JSON.parse(fetched.data);
-		} catch {}
-		if (status < 200 || status >= 300) {
-			const detail = ellipsize(formatErrorDetail(data), 1e3);
-			const error = `TypeSafe API error: ${[status, statusText].filter(Boolean).join(" ")}${requestIdText}${detail && `\n${detail}`}`;
-			return status === 529 ? rateLimited(error, {
-				status,
-				statusText
-			}, rateLimitTimingFromHeaders(headers).retryAfterMs) : { error };
-		}
-		const malformed = async () => {
-			await deleteFromCache?.();
-			return { error: `TypeSafe API returned an unexpected response${requestIdText}: ${ellipsize(String(fetched.data), 1e3)}` };
-		};
-		const body = isPlainObject(data) ? data : void 0;
-		const answers = body?.answers;
-		const isAnswered = (id) => Object.prototype.hasOwnProperty.call(answers, id) && isPlainObject(answers?.[id]);
-		if (!body || !isPlainObject(answers) || !Object.keys(questions).every(isAnswered)) return malformed();
-		const { input_tokens: prompt = 0, output_tokens: completion = 0 } = body.usage ?? {};
-		const total = prompt + completion;
-		const inputCost = INPUT_COST_PER_TOKEN[body.model ?? this.modelName];
-		return {
-			answers,
-			malformed,
-			response: {
-				cached,
-				latencyMs,
-				tokenUsage: cached ? {
-					cached: total,
-					total
-				} : {
-					total,
-					prompt,
-					completion,
-					numRequests: 1
-				},
-				cost: inputCost === void 0 ? void 0 : prompt * inputCost,
-				metadata: { typesafe: {
-					model: body.model,
-					...requestId && { requestId }
-				} }
-			}
-		};
-	}
-};
-//#endregion
-export { TypeSafeProvider };
-
-//# sourceMappingURL=typesafe-DYM-SFNH.js.map

package/dist/src/typesafe-LXmSbw8F.js

@@ -1,254 +0,0 @@
-#!/usr/bin/env node
-import { Q as getEnvString, c as logger, nt as getProviderEnvString } from "./logger-pCNnmVuz.js";
-import { C as HttpRateLimitError, D as formatRateLimitErrorMessage, u as rateLimitTimingFromHeaders } from "./fetch-BhfUnKen.js";
-import { a as getRequestTimeoutMs } from "./shared-DS_sDrmK.js";
-import { i as fetchWithCache } from "./cache-CdaZNb75.js";
-import { t as ellipsize } from "./text-CV7MmVb_.js";
-//#region src/providers/typesafe.ts
-/**
-* TypeSafe provider for Jev, a "System One" model. Jev does not generate text: it answers
-* typed questions (Noul, Score, Choice) about a `state` with calibrated probabilities.
-*
-* - As an `llm-rubric` grader, the rubric is asked about the graded output as a yes/no
-*   Noul question, or as a Score question when `levels` is configured.
-* - As a `classifier` provider, `instructions` and `labels` form a Choice question.
-* - As a regular provider, the prompt is the state, `questions` is sent as configured, and
-*   the output is Jev's answers.
-*
-* API reference: https://docs.typesafe.ai/api
-*/
-const DEFAULT_API_BASE_URL = "https://api.typesafe.ai";
-const DEFAULT_THRESHOLD = .5;
-const DEFAULT_RETRY_AFTER_MS = 2e3;
-const UNSUPPORTED_GRADER_LABELS = ["agent-rubric", "trajectory:goal-success"];
-const INPUT_COST_PER_TOKEN = { "jev-1.13.0": .042 / 1e6 };
-function isPlainObject(value) {
-	return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-function isFiniteNumber(value) {
-	return typeof value === "number" && Number.isFinite(value);
-}
-/**
-* Jev rejects content that is not a string, object, or array. Text that holds a JSON
-* object or array is sent as structure, which Jev can address by field name.
-*/
-function toEntry(value) {
-	if (typeof value === "string") {
-		try {
-			const parsed = JSON.parse(value);
-			if (typeof parsed === "object" && parsed !== null) return parsed;
-		} catch {}
-		return value;
-	}
-	if (typeof value === "object" && value !== null) return value;
-	return value === void 0 ? "" : JSON.stringify(value);
-}
-/**
-* TypeSafe reports errors as `{detail}`, where `detail` is a string, an
-* `{error_type, message}` object, or a list of field validation errors.
-*/
-function formatErrorDetail(data) {
-	const detail = isPlainObject(data) && data.detail != null ? data.detail : data;
-	if (typeof detail === "string") return detail;
-	if (Array.isArray(detail)) return detail.map((item) => isPlainObject(item) && Array.isArray(item.loc) && typeof item.msg === "string" ? `${item.loc.join(".")}: ${item.msg}` : JSON.stringify(item)).join("; ");
-	if (isPlainObject(detail)) {
-		const text = [detail.message, detail.error_type].find((value) => typeof value === "string");
-		if (text) return text;
-	}
-	return JSON.stringify(detail) ?? "";
-}
-/** A failure the scheduler should back off from and retry, unless it is a hard quota. */
-function rateLimited(error, http, retryAfterMs, rateLimitKind = "rate_limit") {
-	const delay = String(Math.ceil(retryAfterMs ?? DEFAULT_RETRY_AFTER_MS));
-	return {
-		error,
-		metadata: {
-			rateLimitKind,
-			http: {
-				...http,
-				headers: { "retry-after-ms": delay }
-			}
-		}
-	};
-}
-var TypeSafeProvider = class {
-	modelName;
-	config;
-	env;
-	constructor(modelName, options = {}) {
-		const { config, id, env } = options;
-		this.modelName = modelName;
-		this.config = config || {};
-		this.env = env;
-		this.id = id ? () => id : this.id;
-	}
-	id() {
-		return `typesafe:${this.modelName}`;
-	}
-	toString() {
-		return `[TypeSafe Provider ${this.modelName}]`;
-	}
-	requiresApiKey() {
-		return true;
-	}
-	getApiKey() {
-		return this.config.apiKey || getProviderEnvString(this.env, "TYPESAFE_API_KEY") || getEnvString("TYPESAFE_API_KEY");
-	}
-	async callApi(prompt, context, options) {
-		const config = {
-			...this.config,
-			...context?.prompt?.config
-		};
-		const bustCache = context?.bustCache ?? context?.debug;
-		const abortSignal = options?.abortSignal;
-		const label = context?.prompt?.label;
-		if (label === "llm-rubric" && context?.vars?.rubric !== void 0) return this.grade(context.vars.rubric, context.vars.output, config, bustCache, abortSignal);
-		if (label && UNSUPPORTED_GRADER_LABELS.includes(label)) return { error: `TypeSafe provider ${this.id()} cannot grade \`${label}\` assertions. Jev grades only \`llm-rubric\` and \`classifier\`.` };
-		const { questions } = config;
-		if (!isPlainObject(questions) || Object.keys(questions).length === 0) return { error: `TypeSafe provider ${this.id()} needs \`questions\` in its config to answer prompts. To grade with Jev instead, use it as the provider for \`llm-rubric\` or \`classifier\` assertions.` };
-		const result = await this.ask(toEntry(prompt), questions, bustCache, abortSignal);
-		if ("error" in result) return result;
-		return {
-			...result.response,
-			output: result.answers
-		};
-	}
-	async callClassificationApi(prompt) {
-		const { instructions, labels } = this.config;
-		if (!instructions || !labels) return { error: `TypeSafe provider ${this.id()} needs \`instructions\` and \`labels\` in its config to classify, for example \`instructions: Which team should handle this?\` and \`labels: [billing, technical, sales]\`.` };
-		const criteria = Array.isArray(labels) ? Object.fromEntries(labels.map((label) => [label, null])) : labels;
-		const result = await this.ask(toEntry(prompt), { classification: {
-			type: "choice",
-			instructions,
-			criteria
-		} });
-		if ("error" in result) return result;
-		const { probabilities } = result.answers.classification;
-		if (!isPlainObject(probabilities) || !Object.values(probabilities).every(isFiniteNumber)) return result.malformed();
-		return { classification: probabilities };
-	}
-	async grade(rubric, output, config, bustCache, abortSignal) {
-		const { levels } = config;
-		const threshold = config.threshold ?? DEFAULT_THRESHOLD;
-		if (typeof threshold !== "number" || !(threshold >= 0 && threshold <= 1)) return { error: `TypeSafe \`threshold\` must be a number from 0 to 1, got ${JSON.stringify(threshold)}.` };
-		if (levels !== void 0 && (!Array.isArray(levels) || levels.length < 2)) return { error: "TypeSafe `levels` must list at least two Score levels, ordered low to high." };
-		const instructions = toEntry(rubric);
-		const question = levels ? {
-			type: "score",
-			instructions,
-			criteria: levels.map(toEntry)
-		} : {
-			type: "noul",
-			instructions
-		};
-		const result = await this.ask(toEntry(output), { grade: question }, bustCache, abortSignal);
-		if ("error" in result) return result;
-		const answer = result.answers.grade;
-		const raw = levels ? answer.score : answer.noul;
-		const top = levels ? levels.length - 1 : 1;
-		if (!isFiniteNumber(raw) || raw < 0 || raw > top) return result.malformed();
-		const score = Number((raw / top).toFixed(6));
-		const pass = score >= threshold;
-		const comparison = `${pass ? ">=" : "<"} threshold ${threshold}`;
-		const reason = levels ? `Jev Score ${raw} on levels 0–${top} (${score} normalized) ${comparison}; nearest level: ${ellipsize(JSON.stringify(levels[Math.round(raw)]) ?? "", 200)}` : `Jev Noul probability ${raw} ${comparison}`;
-		return {
-			...result.response,
-			output: {
-				pass,
-				score,
-				reason
-			},
-			metadata: { typesafe: {
-				...result.response.metadata.typesafe,
-				answer
-			} }
-		};
-	}
-	/** Ask Jev `questions` about `state`. */
-	async ask(state, questions, bustCache, abortSignal) {
-		const apiKey = this.getApiKey();
-		if (!apiKey) return { error: "TypeSafe API key is not set. Set the TYPESAFE_API_KEY environment variable or add `apiKey` to the provider config." };
-		logger.debug("[TypeSafe] Calling Jev", {
-			model: this.modelName,
-			questions: Object.keys(questions)
-		});
-		let fetched;
-		try {
-			fetched = await fetchWithCache(`${this.config.apiBaseUrl || DEFAULT_API_BASE_URL}/v1/systemone`, {
-				method: "POST",
-				headers: {
-					"Content-Type": "application/json",
-					Authorization: `Bearer ${apiKey}`
-				},
-				body: JSON.stringify({
-					state,
-					model: this.modelName,
-					questions
-				}),
-				...abortSignal ? { signal: abortSignal } : {}
-			}, getRequestTimeoutMs(), "text", bustCache);
-		} catch (err) {
-			abortSignal?.throwIfAborted();
-			if (err instanceof HttpRateLimitError) {
-				const detail = ellipsize(formatErrorDetail(err.body), 300);
-				return rateLimited(`TypeSafe API error: ${formatRateLimitErrorMessage(err, detail)}`, {
-					status: err.status,
-					statusText: err.statusText
-				}, err.retryAfterMs, err.kind);
-			}
-			return { error: `TypeSafe API call error: ${String(err)}` };
-		}
-		const { status, statusText, headers = {}, cached, latencyMs, deleteFromCache } = fetched;
-		const requestId = headers["x-typesafe-request-id"];
-		const requestIdText = requestId ? ` (request id ${requestId})` : "";
-		let data = fetched.data;
-		try {
-			data = JSON.parse(fetched.data);
-		} catch {}
-		if (status < 200 || status >= 300) {
-			const detail = ellipsize(formatErrorDetail(data), 1e3);
-			const error = `TypeSafe API error: ${[status, statusText].filter(Boolean).join(" ")}${requestIdText}${detail && `\n${detail}`}`;
-			return status === 529 ? rateLimited(error, {
-				status,
-				statusText
-			}, rateLimitTimingFromHeaders(headers).retryAfterMs) : { error };
-		}
-		const malformed = async () => {
-			await deleteFromCache?.();
-			return { error: `TypeSafe API returned an unexpected response${requestIdText}: ${ellipsize(String(fetched.data), 1e3)}` };
-		};
-		const body = isPlainObject(data) ? data : void 0;
-		const answers = body?.answers;
-		const isAnswered = (id) => Object.prototype.hasOwnProperty.call(answers, id) && isPlainObject(answers?.[id]);
-		if (!body || !isPlainObject(answers) || !Object.keys(questions).every(isAnswered)) return malformed();
-		const { input_tokens: prompt = 0, output_tokens: completion = 0 } = body.usage ?? {};
-		const total = prompt + completion;
-		const inputCost = INPUT_COST_PER_TOKEN[body.model ?? this.modelName];
-		return {
-			answers,
-			malformed,
-			response: {
-				cached,
-				latencyMs,
-				tokenUsage: cached ? {
-					cached: total,
-					total
-				} : {
-					total,
-					prompt,
-					completion,
-					numRequests: 1
-				},
-				cost: inputCost === void 0 ? void 0 : prompt * inputCost,
-				metadata: { typesafe: {
-					model: body.model,
-					...requestId && { requestId }
-				} }
-			}
-		};
-	}
-};
-//#endregion
-export { TypeSafeProvider };
-
-//# sourceMappingURL=typesafe-LXmSbw8F.js.map