pi-shorthand 0.2.0 → 0.3.0

package/README.md

@@ -2,12 +2,12 @@
 
 ![A code call in Pi: the verdict, then the diff it applied, then the program's output](https://raw.githubusercontent.com/sebinsua/pi-shorthand/main/docs/screenshot.png?v=2)
 
-A [Pi](https://github.com/earendil-works/pi) tool for token-efficient writes. The model writes a whole
-change as one small Bun program, in shorthand, instead of calling `read`, `edit` and `bash` over
-and over: fewer tokens, and fewer round trips.
+A [Pi](https://github.com/earendil-works/pi) tool for editing repositories with Bun programs.
+The model can combine ordinary JavaScript, text edits and structural transformations in one call.
 
 The program sees your repo as normal, but its writes are held back. By default, they're applied only
 if the program succeeds and destination files are unchanged, and the model gets the diff.
+The program performs the edit; tests, type-checks and other verification run separately afterward.
 
 ## Install
 
@@ -28,7 +28,7 @@ You also need Bun, git, and either [bubblewrap](https://github.com/containers/bu
 Anything in Bun or Node, plus these globals (no imports):
 
 ```ts
-await $`bun test src/api.test.ts`; // Bun's shell (the only async one)
+await $`git ls-files`.text(); // Bun's shell (the only async one)
 glob("src/**/*.ts"); // → ["src/a.ts", …]
 grep("oldApi(", "src"); // → [{ file, line, text }, …]
 sg.find("oldApi($$$ARGS)", "src"); // ast-grep search
@@ -40,6 +40,16 @@ sg.parse(sg.Lang.TypeScript, source); // ast-grep's own API (or import from "@as
 grit("`console.log($x)` => `logger.info($x)`", "src");
 ```
 
+`sg.find`, `sg.one`, `sg.rewrite` and `grit` also accept `sg.file("src/app.ts")` directly, including
+in arrays mixed with paths. Search reads the file's current contents. Missing targets are valid insertion
+destinations but cannot be searched. An explicit target can select an ignored file inside the workspace;
+the tool still only applies Git-visible changes.
+
+[api.d.ts](api.d.ts) exposes the actual injected helper types for editor completion and external TypeScript
+checking of editing programs. Include it in the program's TypeScript project (or reference
+`pi-shorthand/api` via `compilerOptions.types` when installed as a package). This supplies types, not runtime
+globals. Bun execution does not automatically type-check programs or run application verification.
+
 ## Options
 
 - `title`: a short description shown with the call (required).
@@ -51,6 +61,9 @@ grit("`console.log($x)` => `logger.info($x)`", "src");
 ## Good to know
 
 - Only files git tracks, or would track, are applied.
+- Successful edits are formatted with detected installed project tools (Prettier, oxfmt, Biome, Ruff,
+  Black, gofmt or rustfmt) before the final diff. Ambiguous setups are skipped; formatter failures warn
+  without discarding completed edits. Set `PI_SHORTHAND_FORMAT=0` to disable. No project config is required.
 - Each run snapshots the checkout first; reflinks make that cheap where supported, while other
   filesystems copy its contents and use corresponding temporary space. On macOS the program runs at
   a private AgentFS mount, so use paths relative to its working directory for repository files.
@@ -73,6 +86,8 @@ hook runs it; `npm run format` fixes formatting.
 `npm test` runs the tests against real overlays (it needs AgentFS on macOS, bubblewrap on Linux).
 `test/linux.sh` runs them on Linux in Docker.
 
-`bun e2e/run.ts --repo <path or git URL> --task "…" --setup baseline|code|read-code --check "…"` runs Pi
-with a real model on a fresh copy of a repo and summarises what it did (time, turns, tool calls,
-tokens, whether the check passed).
+`bun e2e/suite.ts` previews the local benchmark suite without calling a model. Add `--execute` to run it.
+The [comparison harness](e2e/README.md) supports stock, optional and replacement editing tools, controlled
+documentation/skills, independent task checks, saved final changes and session reports.
+`bun e2e/run.ts --repo <path or git URL> --task "…" --setups baseline,replace,code --check "…"` runs Pi
+with a real model on fresh copies of your own repository.

package/api.d.ts

@@ -0,0 +1,14 @@
+/** Types for programs executed by the code tool. Type-only: this does not install runtime globals. */
+import type { ShorthandGlobals } from "./prelude.ts";
+
+declare global {
+	const $: ShorthandGlobals["$"];
+	const edit: ShorthandGlobals["edit"];
+	const glob: ShorthandGlobals["glob"];
+	const grep: ShorthandGlobals["grep"];
+	const sg: ShorthandGlobals["sg"];
+	const grit: ShorthandGlobals["grit"];
+}
+
+export type { RewriteResult } from "./prelude.ts";
+export type { ShorthandGlobals };

package/format.ts

@@ -0,0 +1,148 @@
+/** Best-effort formatting using existing project tools. Runs inside the editing workspace. */
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, extname, join, relative, resolve } from "node:path";
+
+type Command = { name: string; executable: string; args: string[]; cwd: string };
+
+function text(file: string): string {
+	try {
+		return readFileSync(file, "utf8");
+	} catch {
+		return "";
+	}
+}
+
+function directories(start: string, root: string): string[] {
+	const result = [];
+	for (let dir = start; ; dir = dirname(dir)) {
+		result.push(dir);
+		if (dir === root || dirname(dir) === dir) return result;
+	}
+}
+
+export function formatterFor(file: string, root: string): Command | null {
+	root = resolve(root);
+	const extension = extname(file);
+	const js = /\.(?:[cm]?[jt]sx?|jsonc?|css|scss|less|html|vue|svelte|mdx?|ya?ml|graphql)$/i.test(extension);
+	for (const cwd of directories(dirname(resolve(root, file)), root)) {
+		const has = (...names: string[]) => names.some((name) => existsSync(join(cwd, name)));
+		let name: string | undefined;
+		let args: string[] = [];
+		if (js) {
+			let pkg: {
+				scripts?: Record<string, string>;
+				dependencies?: Record<string, string>;
+				devDependencies?: Record<string, string>;
+				prettier?: unknown;
+			};
+			try {
+				pkg = JSON.parse(text(join(cwd, "package.json")) || "{}");
+			} catch {
+				return null;
+			}
+			const deps = { ...pkg.dependencies, ...pkg.devDependencies };
+			const choices = [
+				...(deps.prettier ||
+				pkg.prettier !== undefined ||
+				has(
+					".prettierrc",
+					".prettierrc.json",
+					".prettierrc.yaml",
+					".prettierrc.yml",
+					".prettierrc.js",
+					".prettierrc.cjs",
+					".prettierrc.mjs",
+					"prettier.config.js",
+					"prettier.config.cjs",
+					"prettier.config.mjs",
+					"prettier.config.ts",
+				)
+					? ["prettier"]
+					: []),
+				...(deps.oxfmt || has(".oxfmtrc.json", ".oxfmtrc.jsonc") ? ["oxfmt"] : []),
+				...(deps["@biomejs/biome"] || has("biome.json", "biome.jsonc") ? ["biome"] : []),
+			];
+			// Recognize the executable, never execute an arbitrary package script.
+			const script = pkg.scripts?.format;
+			const scripted = script?.match(/^(prettier|oxfmt|biome)(?:\s|$)/)?.[1];
+			// Custom wrappers/options may encode conventions we cannot reproduce. Leave them alone.
+			if (
+				script &&
+				(!scripted ||
+					/[;&|`$<>]/.test(script) ||
+					(script.match(/(?<!\S)--?[\w-]+(?:=\S+)?/g) ?? []).some(
+						(flag) => !["--write", "--check", "--ignore-unknown"].includes(flag),
+					))
+			)
+				return null;
+			if (!scripted && choices.length > 1) return null;
+			name = scripted ?? choices[0];
+			args =
+				name === "prettier"
+					? ["--write", "--ignore-unknown"]
+					: name === "biome"
+						? ["format", "--write", "--files-ignore-unknown=true"]
+						: [];
+		} else if (extension === ".py" || extension === ".pyi") {
+			const config = text(join(cwd, "pyproject.toml"));
+			const ruff = has("ruff.toml", ".ruff.toml") || /\[tool\.ruff(?:\.|\])/.test(config);
+			const black = /\[tool\.black\]/.test(config);
+			if (ruff && black) return null;
+			name = ruff ? "ruff" : black ? "black" : undefined;
+			args = name === "ruff" ? ["format"] : [];
+		} else if (extension === ".go" && has("go.mod", "go.work")) {
+			name = "gofmt";
+			args = ["-w"];
+		} else if (extension === ".rs" && has("Cargo.toml")) {
+			const cargo = text(join(cwd, "Cargo.toml"));
+			if (/edition\s*\.\s*workspace\s*=/.test(cargo)) return null;
+			name = "rustfmt";
+			args = ["--edition", cargo.match(/^\s*edition\s*=\s*["'](\d+)["']/m)?.[1] ?? "2015"];
+		}
+		if (!name) continue;
+		const search = directories(cwd, root);
+		const candidates = search.map((dir) => join(dir, js ? "node_modules/.bin" : ".venv/bin", name!));
+		// JS formatters must belong to this project, not the extension's dependencies.
+		const executable = candidates.find((candidate) => Bun.which(candidate)) ?? (!js ? Bun.which(name) : null);
+		return executable ? { name, executable, args, cwd } : null;
+	}
+	return null;
+}
+
+export async function formatChanged(
+	files: string[],
+	root: string,
+): Promise<{ messages: string[]; warnings: string[] }> {
+	const groups = new Map<string, { command: Command; files: string[] }>();
+	const result = { messages: [] as string[], warnings: [] as string[] };
+	for (const file of files) {
+		const command = formatterFor(file, root);
+		if (!command) continue;
+		const key = JSON.stringify(command);
+		const group = groups.get(key) ?? { command, files: [] };
+		group.files.push("./" + relative(command.cwd, resolve(root, file)));
+		groups.set(key, group);
+	}
+	for (const { command, files: targets } of groups.values()) {
+		try {
+			const child = Bun.spawn([command.executable, ...command.args, ...targets], {
+				cwd: command.cwd,
+				stdout: "pipe",
+				stderr: "pipe",
+			});
+			const [code, stdout, stderr] = await Promise.all([
+				child.exited,
+				new Response(child.stdout).text(),
+				new Response(child.stderr).text(),
+			]);
+			if (code !== 0)
+				result.warnings.push(
+					`${command.name} formatting failed: ${(stderr || stdout).trim().slice(-2000) || `exit ${code}`}`,
+				);
+			else result.messages.push(`Formatted ${targets.length} file(s) with ${command.name}.`);
+		} catch (error) {
+			result.warnings.push(`${command.name} formatting failed: ${String(error)}`);
+		}
+	}
+	return result;
+}

package/index.ts

@@ -15,28 +15,19 @@ import { callLine, countLines, fileMetadataSummary, resultLines, unstructuredRes
 import { RUN_HISTORY_FILE } from "./history.ts";
 import type { FileChange, RunOptions, RunResult } from "./runner.ts";
 
-// Runs typically take well under a second. Programs that run tests or builds pass a longer timeout.
+// Runs typically take well under a second. Longer transformations can request more time.
 const DEFAULT_TIMEOUT_SECONDS = 2;
 
-const DESCRIPTION = `Make a repository change with one TypeScript program, run by Bun as a transaction: its writes are applied only if it exits successfully, and you get the diff. Put the checks that prove the change worked (type-check, targeted tests, no leftover matches) in the same program, and throw if they fail. Work out what to change inside the program (e.g. with grep) rather than copying lists from earlier output.
+const DESCRIPTION = `Edit repository files with a TypeScript program run by Bun. Top-level await and ordinary Bun/Node APIs work. Use repository-relative paths. The program runs in an isolated workspace; changes apply on successful exit by default and the tool reports the diff. Run tests, type-checks and builds separately afterward with the shell tool.
 
-Use it when a change takes several deterministic steps (reads, searches, multi-file edits, structural rewrites, checks) and you already know what to do with each intermediate result. If seeing an intermediate result could change your plan, look first with a normal tool call.
+Common operations:
+- edit({ path, oldText, newText }) replaces exactly one literal occurrence; missing or ambiguous text is an error. Use text edits for known source, structural matching when it saves enumerating occurrences or preserves varying syntax.
+- await Bun.file(path).text(); await Bun.write(path, text)
+- sg.rewrite(pattern, replacement, files?) discovers and rewrites matching code; omit files for the working directory. $X captures one node; $$$X captures a sequence.
+- sg.one(pattern, files?) selects exactly one match; sg.find returns an array. sg.rewrite also accepts a selected match or array without a file scope.
+- A rewrite callback receives a match and returns text, a native node.replace(text) edit, or null to skip. Return native edits to apply them. Pass selected arrays together for independent edits; select again after changing their file.
 
-The program runs in an isolated copy of the working directory. Use relative paths for repository files. On Linux, host paths outside the repository are read-only and $TMPDIR is private to the run; on macOS the real checkout's absolute path is intentionally inaccessible. Top-level await works, and so do ordinary Bun and Node APIs. These globals are synchronous, and see the files git sees (not node_modules or ignored files):
-- glob(pattern, dir?) → string[]
-- grep(stringOrRegExp, paths?) → {file, line, text}[]. A string matches literally.
-- sg.find(pattern, files?) → {file, line, text, vars}[]. ast-grep pattern: $X is one node, $$$X is zero or more. files is a file, directory, glob or a list of them (JS/TS).
-- sg.rewrite(pattern, templateOrFunction, files?) → number rewritten. A template can use $X and $$$X; a function gets the match (its captures are on it: m.X) and returns the new text, or null to leave it.
-- sg.one(pattern, files?) requires exactly one match. sg.file(path) selects a JS/TS file root (also works for new files).
-- sg.insert(text, destination), sg.move(match, destination, transform?), sg.remove(match). destination is exactly one of {before: match}, {after: match}, {startOf: container}, {endOf: container}. JS/TS statements/declarations only; containers are file roots or matched statement blocks. Rematch after editing a file. move's optional function transforms its text; insert(match.text, destination) copies.
-- sg also has ast-grep's own API (sg.parse, sg.Lang, sg.findInFiles, …), and import "@ast-grep/napi" works too.
-- grit(gritqlPattern, paths?, {lang?, dryRun?}) → {file, matches}[], e.g. grit("\`a($x)\` => \`b($x)\`", "src")
-Bun's shell $ needs await: await $\`bun test src/foo.test.ts\`. You can also run the ast-grep, grit and git CLIs with it. For how to write these programs, see the shorthand skill.
-
-Throw or exit non-zero to fail. rollback decides what a failure undoes:
-- "all" (default): nothing is applied; you get the error and the candidate diff.
-- "file": on timeout, files not open for writing are applied if writer inspection succeeds. If inspection fails, or on another failure, nothing is applied.
-The default timeout is 2 seconds; pass a longer timeout when the program runs tests or builds. Only files git sees (tracked, or untracked and not ignored) are diffed and applied; writes to .git are blocked. Print what you need to know (counts, assertions), not whole files.`;
+See the shorthand skill for common writes. For extraction, complex rewrites or other languages, read its advanced-refactors.md guide. The default timeout is two seconds; request more for longer programs.`;
 
 export default function (pi: ExtensionAPI) {
 	// A failed run is an error, both for the model and for how Pi shows it. (execute() returns its details
@@ -50,12 +41,8 @@ export default function (pi: ExtensionAPI) {
 		name: "code",
 		label: "Code",
 		description: DESCRIPTION,
-		promptSnippet:
-			"Make a change with one Bun program, run as a transaction: its edits are kept only if it exits 0, so put your checks inside it",
-		promptGuidelines: [
-			"Use code when several related reads, searches, edits or checks can be done without looking at intermediate results: put that logic in one program rather than many read/edit/bash calls.",
-			"Don't use code to explore when you need to see results before deciding what to do.",
-		],
+		promptSnippet: "Make a change with one transactional Bun editing program; run verification separately afterward",
+
 		parameters: Type.Object({
 			title: Type.String({ description: "A few words describing the change, shown to the user" }),
 			program: Type.String({ description: "TypeScript program run with Bun (top-level await allowed)" }),

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "pi-shorthand",
-	"version": "0.2.0",
-	"description": "Pi tool for token-efficient writes: the model makes a whole change with one isolated Bun program.",
+	"version": "0.3.0",
+	"description": "Pi tool for editing repositories with Bun programs, text edits and structural transformations.",
 	"keywords": [
 		"ast-grep",
 		"bun",
@@ -33,6 +33,7 @@
 		"lint": "oxlint --deny-warnings",
 		"format": "oxfmt",
 		"check": "tsc -p . && oxlint --deny-warnings && oxfmt --check",
+		"postinstall": "grit init --global",
 		"prepare": "git config core.hooksPath .githooks 2>/dev/null || true"
 	},
 	"dependencies": {

package/placement.ts

@@ -9,6 +9,15 @@ export interface Match {
 	node: SgNode;
 }
 
+const fileTarget = Symbol("shorthand.file");
+export interface FileTarget extends Match {
+	readonly [fileTarget]: true;
+}
+
+export function isFileTarget(value: unknown): value is FileTarget {
+	return typeof value === "object" && value !== null && fileTarget in value && snapshots.has(value as unknown as Match);
+}
+
 export type Destination =
 	| { before: Match; after?: never; startOf?: never; endOf?: never }
 	| { after: Match; before?: never; startOf?: never; endOf?: never }
@@ -41,27 +50,45 @@ export function remember<T extends Match>(match: T, source: string, existed = tr
 }
 
 /** A file root, including a not-yet-created file. Merely selecting it performs no writes. */
-export function file(filename: string): Match {
+export function file(filename: string): FileTarget {
 	const lang = languages[filename.split(".").pop()!];
 	if (!lang) throw new Error("sg.file requires a JS/TS filename");
 	const existed = existsSync(filename);
 	const source = existed ? readFileSync(filename, "utf8") : "";
-	return remember({ file: filename, text: source, node: parse(lang, source).root() }, source, existed);
+	return remember(
+		{ file: filename, text: source, node: parse(lang, source).root(), [fileTarget]: true as const },
+		source,
+		existed,
+	);
 }
 
-function snapshot(match: Match): Snapshot {
+export function getMatchSnapshot(
+	match: Match,
+	sources = new Map<string, string | null>(),
+	staleAdvice = "match the file again after editing it",
+): Snapshot {
 	const saved = snapshots.get(match);
 	if (!saved) throw new Error("Expected a file-backed match from sg.find, sg.one, or sg.file");
-	if (!languages[saved.file.split(".").pop()!]) throw new Error("Placement currently supports JS/TS only");
 	if (
-		existsSync(saved.file) !== saved.existed ||
-		(saved.existed && readFileSync(saved.file, "utf8") !== saved.source)
-	) {
-		throw new Error(`Stale match in ${match.file}; match the file again after editing it`);
+		match.node !== saved.node ||
+		(existsSync(match.file) ? realpathSync(match.file) : resolve(match.file)) !== saved.file
+	)
+		throw new Error("File-backed match identity was changed; select it again");
+	if (!sources.has(saved.file))
+		sources.set(saved.file, existsSync(saved.file) ? readFileSync(saved.file, "utf8") : null);
+	const source = sources.get(saved.file);
+	if ((source !== null) !== saved.existed || (saved.existed && source !== saved.source)) {
+		throw new Error(`Stale match in ${match.file}; ${staleAdvice}`);
 	}
 	return saved;
 }
 
+function snapshot(match: Match): Snapshot {
+	const saved = getMatchSnapshot(match);
+	if (!languages[saved.file.split(".").pop()!]) throw new Error("Placement currently supports JS/TS only");
+	return saved;
+}
+
 function container(node: SgNode): boolean {
 	return node.kind() === "program" || node.kind() === "statement_block";
 }