promptfoo 0.125.0 → 0.125.1

package/dist/src/agentWorkspace-DxUqd-XK.js

@@ -1,592 +0,0 @@
-import { G as state, o as logger } from "./logger-Gff7RMNF.js";
-import { o as resolveAgenticWorkingDir } from "./agentic-utils-CyCLuLoj.js";
-import { n as renderVarsInObject } from "./render-DuOHcYot.js";
-import { constants, lstatSync, realpathSync, rmSync } from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { promisify } from "node:util";
-import fs$1 from "node:fs/promises";
-import { execFile } from "node:child_process";
-import { onExit } from "signal-exit";
-//#region src/providers/agentWorkspace.ts
-/**
-* Isolated workspaces for agentic providers (`copy_working_dir`).
-*
-* For each eval step, `promptfoo eval` creates a fresh workspace from the provider's
-* `working_dir`, passes it to the provider as that call's `working_dir`, and removes it once
-* the step's assertions have run. Clean Git repositories are cloned when checkout preserves
-* their files; otherwise the directory is copied. Clones leave the source untouched and record
-* the agent's changes as a diff.
-*
-* A workspace keeps one call from affecting another; it is not a security sandbox.
-*/
-const execFileAsync = promisify(execFile);
-const MAX_DIFF_LENGTH = 1e5;
-const MAX_GIT_BUFFER = 67108864;
-const MAX_SHARED_INDEX_FILES = 64;
-const GIT_TIMEOUT_MS = 3e4;
-const REPOSITORY_ENV_VARS = [
-	"GIT_ALTERNATE_OBJECT_DIRECTORIES",
-	"GIT_ATTR_SOURCE",
-	"GIT_COMMON_DIR",
-	"GIT_CONFIG",
-	"GIT_CONFIG_COUNT",
-	"GIT_CONFIG_PARAMETERS",
-	"GIT_DIR",
-	"GIT_GRAFT_FILE",
-	"GIT_IMPLICIT_WORK_TREE",
-	"GIT_INDEX_FILE",
-	"GIT_INTERNAL_SUPER_PREFIX",
-	"GIT_NO_REPLACE_OBJECTS",
-	"GIT_OBJECT_DIRECTORY",
-	"GIT_PREFIX",
-	"GIT_REPLACE_REF_BASE",
-	"GIT_SHALLOW_FILE",
-	"GIT_WORK_TREE"
-];
-const liveWorkspaces = /* @__PURE__ */ new Set();
-const cleanupRoots = /* @__PURE__ */ new Set();
-let exitCleanupRegistered = false;
-/** Clear Git repository selectors from a per-call subprocess environment. */
-function clearRepositoryEnv(env) {
-	for (const name of Object.keys(env)) {
-		const canonicalName = process.platform === "win32" ? name.toUpperCase() : name;
-		if (REPOSITORY_ENV_VARS.includes(canonicalName)) delete env[name];
-	}
-}
-/** Parse `copy_working_dir`: `true` picks git or copy automatically. */
-function getCopyWorkingDirMode(value) {
-	if (value === void 0 || value === false) return;
-	if (value === true) return "auto";
-	if (value === "git" || value === "copy") return value;
-	throw new Error("copy_working_dir must be true, false, 'git', or 'copy'");
-}
-/** Whether `dir` is registered. Throws if a registered workspace was deleted or replaced. */
-function isAgentWorkspace(dir) {
-	const resolved = path.resolve(dir);
-	if (!liveWorkspaces.has(resolved)) return false;
-	try {
-		if (lstatSync(resolved).isDirectory() && realpathSync(resolved) === resolved) return true;
-	} catch {}
-	throw new Error(`copy_working_dir workspace is no longer available or was replaced: ${resolved}`);
-}
-/**
-* Check a provider call made with `copy_working_dir`. Returns true when the call runs in a
-* workspace, so the provider can skip its response cache. Throws when the call would run in
-* the original `working_dir`, which happens when something other than an eval step calls the
-* provider, such as a multi-turn redteam strategy or a script.
-*/
-function assertIsolatedWorkingDir(config) {
-	const mode = getCopyWorkingDirMode(config.copy_working_dir);
-	if (typeof config.working_dir === "string" && isAgentWorkspace(config.working_dir)) return true;
-	if (!mode) return false;
-	throw new Error("copy_working_dir runs each eval step in a fresh copy of working_dir, which promptfoo eval creates for the step. This call was not made by an eval step, so it would run in working_dir itself and was not made.");
-}
-async function git(args, { cwd, env, input, signal } = {}) {
-	signal?.throwIfAborted();
-	const baseEnv = { ...process.env };
-	clearRepositoryEnv(baseEnv);
-	const command = execFileAsync("git", cwd ? [
-		"-C",
-		cwd,
-		...args
-	] : args, {
-		env: {
-			...baseEnv,
-			...env
-		},
-		maxBuffer: MAX_GIT_BUFFER,
-		timeout: GIT_TIMEOUT_MS,
-		killSignal: "SIGKILL",
-		signal
-	});
-	if (input !== void 0) {
-		command.child.stdin?.on("error", () => {});
-		command.child.stdin?.end(input);
-	}
-	const { stdout } = await command;
-	return stdout;
-}
-var UnsupportedGitAttributesError = class extends Error {};
-var UnsupportedGitSubmoduleError = class extends Error {};
-/**
-* The repository to clone when `source` is the root of a git repository whose working tree
-* matches its current commit. `allowIgnored` also accepts ignored files, which a clone leaves
-* out.
-*/
-async function getCloneableRepository(source, allowIgnored, signal) {
-	try {
-		const [topLevel, objectsDir, infoAttributes, objectFormat, head] = (await git([
-			"rev-parse",
-			"--show-toplevel",
-			"--path-format=absolute",
-			"--git-path",
-			"objects",
-			"--git-path",
-			"info/attributes",
-			"--show-object-format",
-			"HEAD"
-		], {
-			cwd: source,
-			signal
-		})).trim().split("\n");
-		if (await fs$1.realpath(topLevel) !== source) return;
-		await git([
-			"--attr-source=" + head,
-			"check-attr",
-			"-z",
-			"diff",
-			"--",
-			"."
-		], {
-			cwd: source,
-			signal
-		}).catch((error) => {
-			if (error instanceof Error && "code" in error && error.code === 129) throw new UnsupportedGitAttributesError("copy_working_dir: 'git' requires Git with --attr-source support (2.41 or newer); upgrade Git or use 'copy' or true.");
-			throw error;
-		});
-		const trackedEntries = (await git([
-			"ls-files",
-			"--stage",
-			"-z"
-		], {
-			cwd: source,
-			signal
-		})).split("\0");
-		if (trackedEntries.some((entry) => entry.startsWith("160000 "))) throw new UnsupportedGitSubmoduleError(`copy_working_dir does not support git submodules yet: ${source}`);
-		if (!allowIgnored) {
-			if ((await git([
-				"config",
-				"--name-only",
-				"--list"
-			], {
-				cwd: source,
-				signal
-			})).split("\n").includes("core.attributesfile") || await fs$1.lstat(infoAttributes).catch(() => void 0)) return;
-		}
-		const attributes = (await git([
-			"check-attr",
-			"-z",
-			"--stdin",
-			"filter",
-			"ident",
-			"working-tree-encoding"
-		], {
-			cwd: source,
-			input: trackedEntries.map((entry) => entry.slice(entry.indexOf("	") + 1)).join("\0"),
-			signal
-		})).split("\0");
-		for (let index = 0; index + 2 < attributes.length; index += 3) {
-			const attribute = attributes[index + 1];
-			const value = attributes[index + 2];
-			if (value === "unspecified" || value === "unset") continue;
-			if (attribute === "filter") throw new UnsupportedGitAttributesError("copy_working_dir: 'git' does not support tracked Git filter attributes; use 'copy' or true to preserve materialized files.");
-			if (!allowIgnored) return;
-		}
-		if (!allowIgnored) {
-			if ((await git([
-				"ls-files",
-				"--eol",
-				"-z"
-			], {
-				cwd: source,
-				signal
-			})).split("\0").some((entry) => {
-				const match = /^i\/(\S+)\s+w\/(\S+)\s+attr\/(.*?)\t/.exec(entry);
-				if (!match) return false;
-				const [, index, working, attributes] = match;
-				return index !== working || attributes.includes("eol=crlf") && (working === "lf" || working === "mixed");
-			})) return;
-		}
-		if ((await git([
-			"--no-optional-locks",
-			...process.platform === "win32" ? [] : ["-c", "core.filemode=true"],
-			"-c",
-			"core.symlinks=true",
-			"status",
-			"--porcelain",
-			"--untracked-files=all",
-			...allowIgnored ? [] : ["--ignored"]
-		], {
-			cwd: source,
-			signal
-		})).trim()) return;
-		const files = await git([
-			"--no-optional-locks",
-			"ls-files",
-			"-v"
-		], {
-			cwd: source,
-			signal
-		});
-		return /^[a-zS]/m.test(files) ? void 0 : {
-			head,
-			objectsDir,
-			objectFormat
-		};
-	} catch (error) {
-		signal?.throwIfAborted();
-		if (error instanceof Error && "killed" in error && error.killed) throw error;
-		if (error instanceof UnsupportedGitSubmoduleError || allowIgnored && error instanceof UnsupportedGitAttributesError) throw error;
-		return;
-	}
-}
-function isInside(root, target) {
-	const relative = path.relative(root, target);
-	return relative === "" || relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
-}
-async function assertCopyable(entry, root, signal, destinationRoot = root) {
-	signal?.throwIfAborted();
-	const stat = await fs$1.lstat(entry);
-	const destination = path.join(destinationRoot, path.relative(root, entry));
-	if (stat.isSymbolicLink()) {
-		if (path.basename(entry) === ".git") throw new Error(`copy_working_dir cannot copy ${entry}: symbolic .git metadata can share another repository`);
-		const target = await fs$1.readlink(entry);
-		if (path.isAbsolute(target) || !isInside(root, path.resolve(path.dirname(entry), target)) || !isInside(destinationRoot, path.resolve(path.dirname(destination), target)) || !isInside(root, await fs$1.realpath(entry))) throw new Error(`copy_working_dir cannot copy ${entry}: it links outside working_dir`);
-	} else if (stat.isDirectory() && path.basename(entry) === ".git") {
-		const worktree = (await git([
-			"--git-dir",
-			entry,
-			"config",
-			"--includes",
-			"--path",
-			"--default",
-			"",
-			"--get",
-			"core.worktree"
-		], { signal })).trim();
-		if (worktree && (path.isAbsolute(worktree) || !isInside(root, path.resolve(entry, worktree)) || !isInside(destinationRoot, path.resolve(destination, worktree)))) throw new Error(`copy_working_dir cannot copy ${entry}: core.worktree would point outside the copy. Commit the changes so working_dir is cloned instead.`);
-		if (await fs$1.lstat(path.join(entry, "commondir")).catch(() => void 0)) throw new Error(`copy_working_dir cannot copy ${entry}: it shares a git common directory`);
-		if (await fs$1.lstat(path.join(entry, "worktrees")).catch(() => void 0)) throw new Error(`copy_working_dir cannot copy ${entry}: it contains linked worktree metadata; use a clean clone instead`);
-	} else if (stat.isFile() && path.basename(entry) === ".git") throw new Error(`copy_working_dir cannot copy ${entry}: it belongs to a git worktree or submodule, whose repository a copy would share. Commit the changes so working_dir is cloned instead.`);
-	else if (!stat.isFile() && !stat.isDirectory()) throw new Error(`copy_working_dir cannot copy ${entry}: it is not a regular file or directory`);
-	return true;
-}
-function registerExitCleanup() {
-	if (exitCleanupRegistered) return;
-	exitCleanupRegistered = true;
-	const signals = ["SIGINT", "SIGTERM"];
-	const existingListeners = new Set(signals.flatMap((signal) => process.rawListeners(signal)));
-	onExit(() => {
-		for (const root of cleanupRoots) try {
-			rmSync(root, {
-				recursive: true,
-				force: true
-			});
-		} catch {}
-	});
-	for (const signal of signals) for (const listener of process.rawListeners(signal)) if (!existingListeners.has(listener)) {
-		process.removeListener(signal, listener);
-		process.prependListener(signal, listener);
-	}
-}
-async function copyIndexFile(indexPath, destination, maxBytes, signal) {
-	signal?.throwIfAborted();
-	const flags = constants.O_RDONLY | (process.platform === "win32" ? 0 : constants.O_NONBLOCK | constants.O_NOFOLLOW);
-	if (process.platform === "win32" && (await fs$1.lstat(indexPath).catch(() => void 0))?.isSymbolicLink()) throw new Error("workspace Git index must not be a symbolic link");
-	const index = await fs$1.open(indexPath, flags).catch((error) => {
-		if (error.code === "ENOENT") return;
-		throw error;
-	});
-	if (!index) return;
-	try {
-		const stat = await index.stat();
-		if (!stat.isFile() || stat.size > maxBytes) throw new Error(`workspace Git indexes must be regular files totaling at most ${MAX_GIT_BUFFER} bytes`);
-		const contents = Buffer.alloc(stat.size);
-		let offset = 0;
-		while (offset < contents.length) {
-			signal?.throwIfAborted();
-			const { bytesRead } = await index.read(contents, offset, contents.length - offset, offset);
-			if (bytesRead === 0) break;
-			offset += bytesRead;
-		}
-		await fs$1.writeFile(destination, contents.subarray(0, offset));
-		return offset;
-	} finally {
-		await index.close();
-	}
-}
-async function copyWorkspaceIndex(dir, destination, signal) {
-	const gitDir = path.join(dir, ".git");
-	const indexSize = await copyIndexFile(path.join(gitDir, "index"), destination, MAX_GIT_BUFFER, signal);
-	if (indexSize === void 0) return false;
-	let remainingBytes = MAX_GIT_BUFFER - indexSize;
-	let sharedCount = 0;
-	for await (const entry of await fs$1.opendir(gitDir)) {
-		signal?.throwIfAborted();
-		if (!/^sharedindex\.(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(entry.name)) continue;
-		if (++sharedCount > MAX_SHARED_INDEX_FILES) throw new Error(`workspace Git index snapshot exceeds ${MAX_SHARED_INDEX_FILES} shared files`);
-		remainingBytes -= await copyIndexFile(path.join(gitDir, entry.name), path.join(path.dirname(destination), entry.name), remainingBytes, signal) ?? 0;
-	}
-	return true;
-}
-/**
-* Diff against the cloned commit using a scratch repository. The agent's Git configuration
-* is never loaded, and its index is copied through a checked file handle before Git reads it.
-* User/system filters are disabled; attributes and ignore rules come from the cloned commit.
-*/
-async function getWorkspaceDiff(dir, repo, signal) {
-	signal?.throwIfAborted();
-	if (!isAgentWorkspace(dir)) throw new Error("the workspace directory was replaced by a link");
-	const scratch = await fs$1.mkdtemp(path.join(path.dirname(dir), "diff-"));
-	try {
-		const gitDir = path.join(scratch, "git");
-		const isolatedConfig = {
-			GIT_CONFIG_GLOBAL: "/dev/null",
-			GIT_CONFIG_NOSYSTEM: "1",
-			GIT_TEMPLATE_DIR: ""
-		};
-		await git([
-			"init",
-			"--quiet",
-			"--bare",
-			`--object-format=${repo.objectFormat}`,
-			gitDir
-		], {
-			env: isolatedConfig,
-			signal
-		});
-		await fs$1.writeFile(path.join(gitDir, "objects", "info", "alternates"), `${repo.objectsDir}\n`);
-		const env = {
-			...isolatedConfig,
-			GIT_DIR: gitDir,
-			GIT_WORK_TREE: dir,
-			GIT_ATTR_SOURCE: repo.head
-		};
-		await git(["read-tree", repo.head], {
-			env,
-			signal
-		});
-		const ignoreDir = path.join(scratch, "ignore");
-		await fs$1.mkdir(ignoreDir);
-		const ignoreFiles = await git([
-			"ls-files",
-			"-z",
-			"--",
-			".gitignore",
-			":(glob)**/.gitignore"
-		], {
-			env,
-			signal
-		});
-		if (ignoreFiles) await git([
-			"checkout-index",
-			"-z",
-			"--stdin",
-			`--prefix=${ignoreDir}${path.sep}`
-		], {
-			env,
-			input: ignoreFiles,
-			signal
-		});
-		const ignoreEnv = {
-			...env,
-			GIT_WORK_TREE: ignoreDir
-		};
-		const workspaceIndex = path.join(scratch, "workspace-index");
-		const ignoredTracked = await copyWorkspaceIndex(dir, workspaceIndex, signal) ? await git([
-			"ls-files",
-			"-z",
-			"--cached",
-			"--ignored",
-			"--exclude-standard"
-		], {
-			env: {
-				...ignoreEnv,
-				GIT_INDEX_FILE: workspaceIndex
-			},
-			signal
-		}) : "";
-		const newFiles = /* @__PURE__ */ new Set();
-		for (const file of ignoredTracked.split("\0").filter(Boolean)) {
-			const fullPath = path.resolve(dir, file);
-			if (isInside(dir, fullPath) && await fs$1.lstat(fullPath).catch(() => void 0)) newFiles.add(file);
-		}
-		const untracked = await git([
-			"ls-files",
-			"-z",
-			"--others"
-		], {
-			env,
-			signal
-		});
-		if (untracked) {
-			const ignored = await git([
-				"check-ignore",
-				"--no-index",
-				"-z",
-				"--stdin"
-			], {
-				env: ignoreEnv,
-				input: untracked,
-				signal
-			}).catch((error) => {
-				if (error instanceof Error && "code" in error && error.code === 1) return "";
-				throw error;
-			});
-			const ignoredFiles = new Set(ignored.split("\0"));
-			for (const file of untracked.split("\0").filter(Boolean)) if (!ignoredFiles.has(file)) newFiles.add(file);
-		}
-		if (newFiles.size > 0) await git([
-			"--literal-pathspecs",
-			"add",
-			"--force",
-			"--pathspec-from-file=-",
-			"--pathspec-file-nul"
-		], {
-			env,
-			input: `${[...newFiles].join("\0")}\0`,
-			signal
-		});
-		await git([
-			"-c",
-			"core.fsmonitor=false",
-			"add",
-			"--update"
-		], {
-			env,
-			signal
-		});
-		const diff = await git([
-			"-c",
-			"core.fsmonitor=false",
-			"diff",
-			"--cached",
-			"--no-color",
-			"--no-ext-diff",
-			"--no-textconv",
-			repo.head
-		], {
-			env,
-			signal
-		});
-		return diff.length > MAX_DIFF_LENGTH ? `${diff.slice(0, MAX_DIFF_LENGTH)}\n[diff truncated after ${MAX_DIFF_LENGTH} characters]` : diff;
-	} finally {
-		await fs$1.rm(scratch, {
-			recursive: true,
-			force: true,
-			maxRetries: 3
-		});
-	}
-}
-/** Create a workspace from `source`, an absolute directory. */
-async function createAgentWorkspace(source, mode = "auto", signal) {
-	signal?.throwIfAborted();
-	const realSource = await fs$1.realpath(source).catch(() => {
-		throw new Error(`copy_working_dir: working_dir does not exist: ${source}`);
-	});
-	if (!(await fs$1.stat(realSource)).isDirectory()) throw new Error(`copy_working_dir requires working_dir to be a directory: ${source}`);
-	const repo = mode === "copy" ? void 0 : await getCloneableRepository(realSource, mode === "git", signal);
-	if (mode === "git" && !repo) throw new Error(`copy_working_dir: 'git' requires working_dir to be the root of a git repository whose files all match its current commit, apart from ignored files: ${source}`);
-	if (repo && await fs$1.stat(path.join(realSource, ".gitmodules")).catch(() => void 0)) throw new Error(`copy_working_dir does not support git submodules yet: ${source}`);
-	const tempDir = await fs$1.realpath(os.tmpdir());
-	const tempParent = isInside(realSource, tempDir) ? path.dirname(realSource) : tempDir;
-	if (isInside(realSource, tempParent)) throw new Error(`copy_working_dir cannot create a workspace outside working_dir: ${source}`);
-	const root = await fs$1.realpath(await fs$1.mkdtemp(path.join(tempParent, "promptfoo-workspace-")));
-	const dir = path.join(root, "workspace");
-	liveWorkspaces.add(dir);
-	cleanupRoots.add(root);
-	registerExitCleanup();
-	const remove = async () => {
-		liveWorkspaces.delete(dir);
-		try {
-			await fs$1.rm(root, {
-				recursive: true,
-				force: true,
-				maxRetries: 3
-			});
-			cleanupRoots.delete(root);
-		} catch (error) {
-			logger.warn(`[copy_working_dir] Could not remove workspace ${root}: ${error}`);
-		}
-	};
-	try {
-		if (repo) {
-			await git([
-				"clone",
-				"--quiet",
-				"--shared",
-				"--no-checkout",
-				"--origin",
-				"origin",
-				realSource,
-				dir
-			], { signal });
-			await git([
-				"-c",
-				"core.autocrlf=false",
-				"-c",
-				"core.eol=lf",
-				"-c",
-				"core.symlinks=true",
-				"checkout",
-				"--quiet",
-				"--detach",
-				repo.head
-			], {
-				cwd: dir,
-				signal
-			});
-			await git([
-				"remote",
-				"remove",
-				"origin"
-			], {
-				cwd: dir,
-				signal
-			});
-			const trackedFiles = await git([
-				"ls-files",
-				"--stage",
-				"-z"
-			], {
-				cwd: dir,
-				signal
-			});
-			for (const entry of trackedFiles.split("\0")) if (entry.startsWith("120000 ")) await assertCopyable(path.join(dir, entry.slice(entry.indexOf("	") + 1)), dir, signal);
-		} else await fs$1.cp(realSource, dir, {
-			recursive: true,
-			verbatimSymlinks: true,
-			errorOnExist: true,
-			force: false,
-			filter: (entry) => assertCopyable(entry, realSource, signal, dir)
-		});
-	} catch (error) {
-		await remove();
-		throw error;
-	}
-	return {
-		dir,
-		strategy: repo ? "git" : "copy",
-		remove,
-		async metadata() {
-			if (!repo) return { workingDir: dir };
-			try {
-				return {
-					workingDir: dir,
-					workspaceDiff: await getWorkspaceDiff(dir, repo, signal)
-				};
-			} catch (error) {
-				signal?.throwIfAborted();
-				if (error instanceof Error && "killed" in error && error.killed) throw error;
-				logger.warn(`[copy_working_dir] Could not compute the workspace diff: ${error}`);
-				return { workingDir: dir };
-			}
-		}
-	};
-}
-/**
-* Create the workspace for one eval step when `config`, a provider config merged with its
-* prompt config, sets `copy_working_dir`.
-*/
-async function createAgentWorkspaceForConfig(config, vars, signal) {
-	const mode = getCopyWorkingDirMode(config?.copy_working_dir);
-	if (!mode) return;
-	const workingDir = renderVarsInObject(config?.working_dir, vars);
-	if (typeof workingDir !== "string" || !workingDir) throw new Error("copy_working_dir requires working_dir");
-	return createAgentWorkspace(resolveAgenticWorkingDir(workingDir, state.basePath), mode, signal);
-}
-//#endregion
-export { isAgentWorkspace as i, clearRepositoryEnv as n, createAgentWorkspaceForConfig as r, assertIsolatedWorkingDir as t };
-
-//# sourceMappingURL=agentWorkspace-DxUqd-XK.js.map