pi-shorthand 0.4.0 → 0.6.0

package/README.md

@@ -20,22 +20,17 @@ xcode-select --install
 curl -fsSL https://agentfs.ai/install | bash
 ```
 
-On Linux, install bubblewrap 0.9 or later. For Debian and Ubuntu:
+On Linux, install bubblewrap 0.11 or later, a C compiler, and `lsof`. For Debian and Ubuntu, check the available bubblewrap version before installing:
 
 ```sh
-sudo apt install bubblewrap
+apt-cache policy bubblewrap
+sudo apt install bubblewrap build-essential lsof
 ```
 
 ## Technical choices
 
 pi-shorthand gives Pi a programming environment instead of a patch format. This makes multi-file and structural edits possible in one call, but does not guarantee that Pi will find it easier or more reliable than its built-in edit tool. Which works better depends on the model and the task.
 
-Programs edit an isolated snapshot, with host files outside the repository kept read-only. By default, a failure keeps completed files and rolls back files involved in failed or interrupted edits; changes are applied only if their destination files have not changed.
+Programs edit a private workspace, with host files outside the repository kept read-only. By default, a failure keeps completed files and rolls back failed or interrupted edits. Concurrent edits can cause a run to be rejected; conflict detection is best-effort, not an atomic commit.
 
-## Diagnosing slow calls
-
-Calls taking longer than their configured program timeout (two seconds by default) show a muted timing footer. The timeout limits the editing program, not the entire transaction. Live progress also names the active setup or cleanup step.
-
-The footer separates parent-observed startup/IPC, runner work, and response/exit overhead. Snapshot detail includes inventory, copy, verification, attempt count, entry count, and logical file bytes. Nested measurements overlap and must not be added to their containing phase totals; logical bytes are not measured disk I/O. Formatting and edit execution include separate subprocess-wait and cleanup measurements.
-
-Infrastructure failures retain completed measurements and identify the failed phase. If the runner exits before reporting completion, its execution interval is marked as observed/incomplete. Structured diagnostics are attached to tool result details; source contents and individual filenames are not recorded in diagnostic events.
+The `code` tool uses Pi's working directory by default. Pass `cwd` to target another checkout; relative paths are resolved from Pi's working directory. For example, when Pi starts in a bare worktree container, `cwd: "child"` targets its `child` worktree. The chosen directory must be inside a Git worktree.

package/agentfs-database.ts

@@ -0,0 +1,141 @@
+/** Reuse only pristine, never-mounted AgentFS databases; each run gets independent copies. */
+import { createHash } from "node:crypto";
+import * as fs from "node:fs/promises";
+import { constants } from "node:fs";
+import { homedir } from "node:os";
+import * as path from "node:path";
+import { $ } from "bun";
+import { Database } from "bun:sqlite";
+
+async function ownedDirectory(directory: string): Promise<void> {
+	await fs.mkdir(directory, { recursive: true, mode: 0o700 });
+	const stat = await fs.lstat(directory);
+	if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== process.getuid?.() || stat.mode & 0o077)
+		throw new Error(`Unsafe AgentFS template directory: ${directory}`);
+}
+
+async function templateKey(agentfs: string, repo: string): Promise<string> {
+	const executable = await fs.stat(agentfs, { bigint: true });
+	const repository = await fs.stat(repo, { bigint: true });
+	// Do not tie the template to checkout contents: it contains schema and the base
+	// path, not source files. Replacing either the executable or root invalidates it.
+	return createHash("sha256")
+		.update(
+			JSON.stringify(
+				[
+					1,
+					process.getuid?.(),
+					process.getgid?.(),
+					agentfs,
+					executable.dev,
+					executable.ino,
+					executable.size,
+					executable.mtimeNs,
+					executable.ctimeNs,
+					repo,
+					repository.dev,
+					repository.ino,
+				],
+				(_, value) => (typeof value === "bigint" ? String(value) : value),
+			),
+		)
+		.digest("hex");
+}
+
+async function copyTemplate(template: string, destination: string): Promise<void> {
+	const directory = await fs.lstat(template);
+	if (
+		!directory.isDirectory() ||
+		directory.isSymbolicLink() ||
+		directory.uid !== process.getuid?.() ||
+		directory.mode & 0o077
+	)
+		throw new Error("Unsafe AgentFS database template");
+	const names = await fs.readdir(template);
+	if (!names.includes("run.db")) throw new Error("Incomplete AgentFS database template");
+	await fs.mkdir(destination, { recursive: true, mode: 0o700 });
+	for (const name of names) {
+		const source = path.join(template, name);
+		const stat = await fs.lstat(source);
+		if (
+			!(name === "run.db" || name === "run.db-wal") ||
+			!stat.isFile() ||
+			stat.uid !== process.getuid?.() ||
+			stat.mode & 0o077 ||
+			stat.nlink !== 1
+		)
+			throw new Error("Unsafe AgentFS database template entry");
+		// No hardlinks: serving a transaction must never mutate the template.
+		await fs.copyFile(source, path.join(destination, name), constants.COPYFILE_EXCL);
+	}
+}
+
+/** Inspect the private copy, so SQLite never opens or modifies the shared template. */
+function validateEmptyDatabase(file: string, repo: string): void {
+	const db = new Database(file, { readonly: true, strict: true });
+	try {
+		const base = db.query("SELECT value FROM fs_overlay_config WHERE key = 'base_path'").get() as
+			| { value: string }
+			| undefined;
+		const root = db.query("SELECT mode, uid, gid FROM fs_inode WHERE ino = 1").get() as
+			| { mode: number; uid: number; gid: number }
+			| undefined;
+		if (
+			base?.value !== repo ||
+			!root ||
+			(root.mode & 0o170000) !== 0o040000 ||
+			root.uid !== process.getuid?.() ||
+			root.gid !== process.getgid?.()
+		)
+			throw new Error("AgentFS template has the wrong base or root identity");
+		for (const table of ["fs_dentry", "fs_whiteout", "fs_origin", "fs_data", "fs_symlink", "kv_store", "tool_calls"]) {
+			const count = db.query(`SELECT COUNT(*) AS count FROM ${table}`).get() as { count: number };
+			if (count.count !== 0) throw new Error(`AgentFS template contains private ${table} state`);
+		}
+		const inodes = db.query("SELECT COUNT(*) AS count FROM fs_inode").get() as { count: number };
+		if (inodes.count !== 1) throw new Error("AgentFS template contains private inode state");
+	} finally {
+		db.close();
+	}
+}
+
+export async function createAgentFsDatabase(
+	agentfs: string,
+	base: string,
+	tempDir: string,
+	cache = path.join(homedir(), ".cache", "pi-shorthand", "agentfs-templates"),
+): Promise<string> {
+	const executable = await fs.realpath(Bun.which(agentfs) ?? agentfs);
+	const repo = await fs.realpath(base);
+	await ownedDirectory(cache);
+	const key = await templateKey(executable, repo);
+	const template = path.join(cache, key);
+	const existing = await fs.lstat(template).catch((error: NodeJS.ErrnoException) => {
+		if (error.code !== "ENOENT") throw error;
+		return null;
+	});
+	if (!existing) {
+		const staging = await fs.mkdtemp(path.join(cache, ".initializing-"));
+		try {
+			await $`${executable} init run --base ${repo}`.cwd(staging).quiet();
+			if (key !== (await templateKey(executable, repo)))
+				throw new Error("AgentFS or repository changed during initialization");
+			const database = path.join(staging, ".agentfs");
+			await fs.chmod(database, 0o700);
+			for (const name of await fs.readdir(database)) await fs.chmod(path.join(database, name), 0o600);
+			// Publish only after the initializing process has exited, including its
+			// database sidecars. Concurrent creators can safely use the winner.
+			await fs.rename(database, template).catch((error: NodeJS.ErrnoException) => {
+				if (error.code !== "EEXIST" && error.code !== "ENOTEMPTY") throw error;
+			});
+		} finally {
+			await fs.rm(staging, { recursive: true, force: true });
+		}
+	}
+	const destination = path.join(tempDir, ".agentfs");
+	await copyTemplate(template, destination);
+	validateEmptyDatabase(path.join(destination, "run.db"), repo);
+	if (key !== (await templateKey(executable, repo)))
+		throw new Error("AgentFS or repository changed while preparing the transaction database");
+	return path.join(destination, "run.db");
+}

package/display.ts

@@ -21,8 +21,15 @@ const INLINE_DIFF_LINES = 40; // a longer diff collapses to a list of its files
 const LISTED_FILES = 8; // …showing this many, then "and N more files"
 const EXPANDED_DIFF_LINES = 2000; // even expanded, a diff of hundreds of files stops here
 
-export function callLine(args: { title?: string; timeout?: number; rollback?: string }, theme: Theme): string {
-	const settings = [args.rollback === "all" && "rollback all", args.timeout && `program timeout ${args.timeout}s`];
+export function callLine(
+	args: { title?: string; cwd?: string; timeout?: number; rollback?: string },
+	theme: Theme,
+): string {
+	const settings = [
+		args.cwd && `cwd ${args.cwd}`,
+		args.rollback === "all" && "rollback all",
+		args.timeout && `program timeout ${args.timeout}s`,
+	];
 	const suffix = settings.filter(Boolean).join(", ");
 	return `${theme.fg("toolTitle", theme.bold("code"))} ${args.title ?? ""}${suffix ? theme.fg("muted", ` (${suffix})`) : ""}`;
 }

package/file-outcomes.ts

@@ -11,6 +11,7 @@ export type FileOutcomeEvent =
 
 const descriptor = process.env.PI_SHORTHAND_OUTCOMES_FD;
 const root = process.env.PI_SHORTHAND_EXECUTION_ROOT;
+export { root as executionRoot };
 const forceInspectionFailure = process.env.PI_SHORTHAND_INSPECTION_FAILURE === "1";
 // Subprocesses do not inherit descriptor 3 by default, so do not advertise it to them.
 delete process.env.PI_SHORTHAND_OUTCOMES_FD;

package/index.ts

@@ -35,7 +35,7 @@ class RunnerError extends Error {
 // Runs typically take well under a second. Longer transformations can request more time.
 const DEFAULT_TIMEOUT_SECONDS = 2;
 
-const DESCRIPTION = `Edit repository files with a TypeScript program run by Bun. Top-level await and ordinary Bun/Node APIs work. Use repository-relative paths. The program runs in an isolated workspace; changes apply on successful exit by default and the tool reports the diff. Run tests, type-checks and builds separately afterward with the shell tool.
+const DESCRIPTION = `Edit repository files with a TypeScript program run by Bun. Top-level await and ordinary Bun/Node APIs work. Use repository-relative paths. Set cwd to a checkout path when Pi's working directory is outside the repository, such as a child worktree in a bare worktree container. The program runs in an isolated workspace; changes apply on successful exit by default and the tool reports the diff. Run tests, type-checks and builds separately afterward with the shell tool.
 
 Common operations:
 - edit({ path, oldText, newText }) replaces exactly one literal occurrence; missing or ambiguous text is an error. Use text edits for known source, structural matching when it saves enumerating occurrences or preserves varying syntax.
@@ -70,6 +70,12 @@ export default function (pi: ExtensionAPI) {
 		parameters: Type.Object({
 			title: Type.String({ description: "A few words describing the change, shown to the user" }),
 			program: Type.String({ description: "TypeScript program run with Bun (top-level await allowed)" }),
+			cwd: Type.Optional(
+				Type.String({
+					description:
+						"Working directory for the program; relative to Pi's working directory, or an absolute path. Defaults to Pi's working directory. Must be inside a git worktree.",
+				}),
+			),
 			rollback: Type.Optional(
 				StringEnum(["all", "file"] as const, {
 					description:
@@ -95,7 +101,7 @@ export default function (pi: ExtensionAPI) {
 			try {
 				result = await runWithBun(
 					{
-						cwd: ctx.cwd,
+						cwd: path.resolve(ctx.cwd, params.cwd ?? "."),
 						program: params.program,
 						timeoutMs: (params.timeout ?? DEFAULT_TIMEOUT_SECONDS) * 1000,
 						rollback: params.rollback ?? "file",

package/linux-observation.ts

@@ -0,0 +1,162 @@
+/** Host-side journal broker for the native Linux observer. */
+import { createHash, randomBytes } from "node:crypto";
+import * as fs from "node:fs/promises";
+import { createServer, type Socket } from "node:net";
+import { once } from "node:events";
+import { homedir } from "node:os";
+import * as path from "node:path";
+import { $ } from "bun";
+import { TransactionJournal } from "./transaction-journal.ts";
+import { diagnosticCounter, measure } from "./diagnostics.ts";
+
+type InvocationState = "expected" | "active" | "finished";
+
+const FRAME_HEADER_SIZE = 9;
+const MAX_FIELD_SIZE = 4096;
+const MAX_FRAME_SIZE = FRAME_HEADER_SIZE + 2 * MAX_FIELD_SIZE;
+
+function decodeFrame(buffer: Buffer): { kind: string; first: string; second: string } | undefined {
+	if (buffer.length > MAX_FRAME_SIZE) throw new Error("Oversized observation request");
+	if (buffer.length < FRAME_HEADER_SIZE) return undefined;
+	const firstSize = buffer.readUInt32BE(1);
+	const secondSize = buffer.readUInt32BE(5);
+	if (firstSize > MAX_FIELD_SIZE || secondSize > MAX_FIELD_SIZE) throw new Error("Oversized observation path");
+	const frameSize = FRAME_HEADER_SIZE + firstSize + secondSize;
+	if (buffer.length < frameSize) return undefined;
+	if (buffer.length !== frameSize) throw new Error("Unexpected observation framing");
+
+	const firstBytes = buffer.subarray(FRAME_HEADER_SIZE, FRAME_HEADER_SIZE + firstSize);
+	const secondBytes = buffer.subarray(FRAME_HEADER_SIZE + firstSize);
+	const first = firstBytes.toString();
+	const second = secondBytes.toString();
+	if (!Buffer.from(first).equals(firstBytes) || !Buffer.from(second).equals(secondBytes))
+		throw new Error("Non-UTF8 observation path");
+	return { kind: String.fromCharCode(buffer[0]), first, second };
+}
+
+export async function openLinuxObservation(repo: string, temporary: string) {
+	const helper = await measure("preparing native observer", observerHelper);
+	const socketPath = path.join(temporary, "observer.sock");
+	const invocations = new Map<string, InvocationState>();
+	const journal = new TransactionJournal(repo);
+	const connections = new Set<Socket>();
+	const completedConnections = new Set<Socket>();
+	const pending = new Set<Promise<void>>();
+	let closing = false;
+	const server = createServer((socket) => {
+		connections.add(socket);
+		const operation = (async () => {
+			let authenticated = false;
+			let token: string | undefined;
+			let finished = false;
+			let buffer: Buffer = Buffer.alloc(0);
+			try {
+				for await (const data of socket) {
+					buffer = Buffer.concat([buffer, Buffer.from(data)]);
+					const frame = decodeFrame(buffer);
+					if (!frame) continue;
+					if (finished) throw new Error("Unexpected observation framing");
+					const { kind, first, second } = frame;
+					buffer = Buffer.alloc(0);
+					if (!authenticated) {
+						if (closing || kind !== "H" || invocations.get(first) !== "expected" || second)
+							throw new Error("Unauthenticated observer");
+						authenticated = true;
+						token = first;
+						invocations.set(token, "active");
+					} else if (kind === "F" && !first && !second) {
+						finished = true;
+						invocations.set(token!, "finished");
+						completedConnections.add(socket);
+					} else if (kind === "X") throw new Error(first);
+					else if (kind === "R") await journal.observeRename(first, second);
+					else if (second) throw new Error("Unexpected second observation path");
+					else if (kind === "T") await journal.observeTree(first);
+					else if (kind === "D") await journal.observeDirectory(first);
+					else if (kind === "M") await journal.observe(first, "metadata");
+					else if (kind === "C") await journal.observe(first, "contents");
+					else throw new Error(`Unknown observation kind ${kind}`);
+					socket.write(Buffer.from([1]));
+				}
+				if (!authenticated || !finished || buffer.length) throw new Error("Observer disconnected before completion");
+			} catch (error) {
+				journal.invalidate(error instanceof Error ? error.message : String(error));
+				socket.destroy();
+			} finally {
+				connections.delete(socket);
+				completedConnections.delete(socket);
+			}
+		})();
+		pending.add(operation);
+		void operation.finally(() => pending.delete(operation));
+	});
+	server.listen(socketPath);
+	try {
+		await once(server, "listening");
+		await fs.chmod(socketPath, 0o600);
+	} catch (error) {
+		for (const socket of connections) socket.destroy();
+		if (server.listening) await new Promise<void>((resolve) => server.close(() => resolve()));
+		throw error;
+	}
+	return {
+		journal,
+		wrap: (command: string[]) => {
+			const secret = randomBytes(32).toString("hex");
+			invocations.set(secret, "expected");
+			return [helper, socketPath, secret, repo, ...command];
+		},
+		async finish() {
+			closing = true;
+			const stopped = new Promise<void>((resolve) => server.close(() => resolve()));
+			if ([...connections].some((socket) => !completedConnections.has(socket))) {
+				journal.invalidate("Observer remained active after execution stopped");
+			}
+			for (const socket of connections) {
+				if (completedConnections.has(socket)) socket.end();
+				else socket.destroy();
+			}
+			await Promise.all([stopped, ...pending]);
+			if (!invocations.size || [...invocations.values()].some((state) => state !== "finished"))
+				journal.invalidate("Not every expected observer completed");
+			diagnosticCounter("observed entries", journal.entryCount);
+			diagnosticCounter("content captures", journal.contentCaptureCount);
+			await journal.seal();
+			return await measure("validating observed dependencies", () => journal.conflicts());
+		},
+	};
+}
+
+async function observerHelper(): Promise<string> {
+	const source = path.join(import.meta.dir, "linux-observer.c");
+	const digest = createHash("sha256")
+		.update(await fs.readFile(source))
+		.digest("hex")
+		.slice(0, 16);
+	const directory = path.join(homedir(), ".cache", "pi-shorthand", "native");
+	await fs.mkdir(directory, { recursive: true, mode: 0o700 });
+	const info = await fs.lstat(directory);
+	if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== process.getuid?.())
+		throw new Error("Unsafe native helper directory");
+	await fs.chmod(directory, 0o700);
+	const helper = path.join(directory, `linux-observer-${digest}`);
+	try {
+		const existing = await fs.lstat(helper);
+		if (!existing.isFile() || existing.isSymbolicLink() || existing.uid !== info.uid || existing.mode & 0o022)
+			throw new Error("Unsafe native observer helper");
+		return helper;
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+	}
+	const compiler = Bun.which("cc") ?? Bun.which("gcc");
+	if (!compiler) throw new Error("The Linux code tool needs a C compiler to build its cached observation helper.");
+	const temporary = `${helper}.${randomBytes(8).toString("hex")}.tmp`;
+	try {
+		await $`${compiler} -O2 -Wall -Wextra ${source} -o ${temporary}`.quiet();
+		await fs.chmod(temporary, 0o700);
+		await fs.rename(temporary, helper);
+	} finally {
+		await fs.rm(temporary, { force: true });
+	}
+	return helper;
+}