pi-hashline-edit-pro 0.16.15 → 0.17.0

package/README.md

@@ -94,9 +94,11 @@ Replaces using the `HASH│content` anchors from `read` output to target lines p
 
 ### Stable hashing across edits
 
-Hashes are now computed with a persistent store (`~/.config/pi-hashline-edit-pro/hash-store.json`) that preserves hashes for unchanged lines across edits. When you replace lines in a file, the runtime maps the old content against the new content and copies hashes for unchanged lines to their new positions. This means editing one part of a file does not change the hashes of unrelated lines elsewhere — the model can keep using previously seen anchors for untouched regions.
+Hashes are now computed with a persistent store (`~/.config/pi-hashline-edit-pro/hash-store.sqlite`) that preserves hashes for unchanged lines across edits. When you replace lines in a file, the runtime maps the old content against the new content and copies hashes for unchanged lines to their new positions. This means editing one part of a file does not change the hashes of unrelated lines elsewhere — the model can keep using previously seen anchors for untouched regions.
 
-The store contains per-file snapshots: the last known content and hashes for each file. On read, if the file content matches the snapshot, the saved hashes are returned immediately. Stale snapshots (for files that no longer exist) are pruned on session start.
+The store is a SQLite database (WAL journal mode) keyed by canonical file path. Each snapshot stores a 64-bit content checksum (`xxhash64`) plus the per-line hashes, not the full text, so a cache hit is a single keyed lookup and a one-row write. Reads, replaces, undo, and pruning all share one transactional store, so concurrent Pi sessions editing different files never silently clobber each other's snapshots (per-path writers serialize via `BEGIN IMMEDIATE`; same-path concurrent edits still fail safe — stale anchors are rejected by content matching). Stale snapshots (for files that no longer exist) are pruned on session start.
+
+On first run after upgrading, a one-time migration imports the previous `hash-store.json` into the database and renames the old file to `hash-store.json.bak`; the old JSON store is otherwise discarded.
 
 ### Chained edits
 
@@ -145,7 +147,7 @@ The file is created automatically when any setting is toggled. Both fields are i
 - **Per-file mutation queue.** Edits queue by the canonical write target, so concurrent edits through different symlink paths still serialize onto the same underlying file.
 - **Boundary duplication auto-fix.** When the last line of a replacement matches the next surviving line (or the first line matches the preceding one), the runtime automatically strips the duplicate from `content_lines` before applying the edit. This catches a common LLM pattern where closing delimiters like `}`, `});`, or `} else {` are accidentally duplicated. The auto-fix is completely silent — the model sees a normal successful edit. The duplicate never reaches the file. Raw line comparison (not trimmed) avoids false positives when indentation differs.
 - **Flat mode normalization.** When flat mode is active, the tool's `execute` function wraps the top-level `hash_range_inclusive` and `content_lines` into a single-element `changes` array internally, then runs the same pipeline as bulk mode. The `normReq` function in `replace-normalize.ts` also handles flat format directly, so any code path that normalizes input (e.g. `compPreview`) works with both formats.
-- **Persistent hash store.** `lineHashes` is async and uses a persistent store to preserve hashes for unchanged lines across edits. The store is at `~/.config/pi-hashline-edit-pro/hash-store.json` and is auto-created on first use. It contains per-file snapshots (last known content+hashes). When called from the replace pipeline, it maps old vs new content and copies hashes for unchanged lines. When called from read, it returns saved hashes if the content matches, otherwise computes fresh hashes via `_lineHashesPure`. Stale snapshots are pruned on session start. This ensures that editing one part of a file does not cascade to change hashes of unrelated lines.
+- **Persistent hash store.** `lineHashes` is async and uses a persistent store to preserve hashes for unchanged lines across edits. The store is a SQLite database at `~/.config/pi-hashline-edit-pro/hash-store.sqlite` (per-path snapshots keyed by resolved path storing a 64-bit content checksum + line hashes; auto-created on first use). When called from the replace pipeline, it maps old vs new content and copies hashes for unchanged lines. When called from read, it returns saved hashes if the content's checksum matches, otherwise computes fresh hashes via `_lineHashesPure`. Stale snapshots are pruned on session start. This ensures that editing one part of a file does not cascade to change hashes of unrelated lines. Per-operation work scales with the target file, not cumulative history.
 ## Hashing
 
 Hashes are computed with [xxhash-wasm](https://github.com/jungomi/xxhash-wasm) (xxHash32 via WebAssembly), then mapped to a 3-character string from the URL-safe base64 alphabet `A-Za-z0-9-_`. That's 64 distinct characters, 6 bits per position, 18 bits of entropy per anchor.

package/index.ts

@@ -10,7 +10,7 @@ import {
   toggleReplaceMode,
   toggleAutoRead,
 } from "./src/config";
-import { loadHashStore, pruneHashStore } from "./src/hash-store";
+import { loadHashStore, pruneMissing } from "./src/hash-store";
 import { readNormFile } from "./src/file-reader";
 
 export default function (pi: ExtensionAPI): void {
@@ -36,7 +36,7 @@ function registerReplaceTool(pi: ExtensionAPI, mode: string, autoRead?: boolean)
     await initHasher();
     try {
       const store = await loadHashStore();
-      await pruneHashStore(store);
+      await pruneMissing(store);
     } catch (err) {
       console.error("Failed to load or prune hash store:", err);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-hashline-edit-pro",
-  "version": "0.16.15",
+  "version": "0.17.0",
   "description": "Strict hashline read/replace tool for pi-coding-agent with hash-anchored edits (3-char, 18-bit, perfect hashing)",
   "main": "index.ts",
   "repository": {
@@ -31,6 +31,7 @@
     ]
   },
   "dependencies": {
+    "better-sqlite3": "^13.0.1",
     "diff": "^8.0.2",
     "file-type": "^21.3.0",
     "xxhash-wasm": "^1.1.0"
@@ -46,8 +47,12 @@
   },
   "devDependencies": {
     "@earendil-works/pi-coding-agent": "^0.74.0",
+    "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^22.0.0",
-    "vitest": "^4.1.8",
-    "typescript": "^5.8.0"
+    "typescript": "^5.8.0",
+    "vitest": "^4.1.8"
+  },
+  "allowScripts": {
+    "better-sqlite3@13.0.1": true
   }
 }

package/src/constants.ts

@@ -5,6 +5,9 @@ export const MAX_BYTES = 100 * 1024 * 1024;
 export const MAX_HASH_LINES = 1_000_000;
 export const MAX_HASH_RETRIES = 262_144;
 
+export const HASH_STORE_VERSION = 2;
+export const HASH_STORE_BUSY_TIMEOUT = 1000;
+
 export const CONTENT_LINES_NOT_STRING_MSG =
   `[E_BAD_SHAPE] "content_lines" must be a native JSON array of strings, not a JSON string.`
   + ` Do not serialize the array (e.g. '["line1", "line2"]') — pass it as a proper JSON array: ["line1", "line2"].`;

package/src/hash-store.ts

@@ -1,72 +1,191 @@
-import { readFile, mkdir } from "fs/promises";
-import { stat } from "fs/promises";
-import { hashStorePath, hashStoreDir } from "./paths";
+import Database from "better-sqlite3";
+import type { Database as DatabaseType, Statement } from "better-sqlite3";
+import { existsSync } from "fs";
+import { readFile, rename, mkdir, stat } from "fs/promises";
+import { hashStorePath, hashStoreDir, legacyHashStorePath } from "./paths";
 import { errCode } from "./utils";
-import { writeAtomic } from "./fs-write";
-function isValidSnapshot(value: unknown): value is FileSnapshot {
-  if (typeof value !== "object" || value === null) return false;
-  const v = value as Record<string, unknown>;
-  if (typeof v.content !== "string") return false;
-  if (!Array.isArray(v.hashes)) return false;
-  for (const h of v.hashes) {
-    if (typeof h !== "string") return false;
-  }
-  return true;
-}
-export interface FileSnapshot {
-  content: string;
-  hashes: string[];
+import { initHasher, contentChecksum } from "./hashline/hasher";
+import { HASH_STORE_VERSION, HASH_STORE_BUSY_TIMEOUT } from "./constants";
+
+interface Prepared {
+	get: Statement<[string, string, number], { hashes: string }>;
+	allPaths: Statement<[], { path: string }>;
+	deleteOne: Statement<[string]>;
+	upsert: Statement<[string, string, number, string, number]>;
 }
 
 export interface HashStore {
-  version: 1;
-  snapshots: Record<string, FileSnapshot>;
+	readonly db: DatabaseType;
+	readonly stmts: Prepared;
+}
+
+interface LegacySnapshot {
+	content: string;
+	hashes: string[];
+}
+
+function isValidSnapshot(value: unknown): value is LegacySnapshot {
+	if (typeof value !== "object" || value === null) return false;
+	const v = value as Record<string, unknown>;
+	if (typeof v.content !== "string") return false;
+	if (!Array.isArray(v.hashes)) return false;
+	for (const h of v.hashes) {
+		if (typeof h !== "string") return false;
+	}
+	return true;
+}
+
+let cachedStore: { path: string; store: HashStore } | null = null;
+
+function openDatabase(storePath: string): HashStore {
+	const db = new Database(storePath);
+	db.pragma("journal_mode = WAL");
+	db.pragma("synchronous = NORMAL");
+	db.pragma(`busy_timeout = ${HASH_STORE_BUSY_TIMEOUT}`);
+	db.exec(`
+		CREATE TABLE IF NOT EXISTS snapshots (
+			path       TEXT PRIMARY KEY,
+			checksum   TEXT NOT NULL,
+			line_count INTEGER NOT NULL,
+			hashes     TEXT NOT NULL,
+			updated_at INTEGER NOT NULL
+		);
+	`);
+	const stmts: Prepared = {
+		get: db.prepare<[string, string, number], { hashes: string }>(
+			"SELECT hashes FROM snapshots WHERE path = ? AND checksum = ? AND line_count = ?",
+		),
+		allPaths: db.prepare<[], { path: string }>("SELECT path FROM snapshots"),
+		deleteOne: db.prepare<[string]>("DELETE FROM snapshots WHERE path = ?"),
+		upsert: db.prepare<[string, string, number, string, number]>(
+			"INSERT INTO snapshots (path, checksum, line_count, hashes, updated_at) VALUES (?, ?, ?, ?, ?) " +
+				"ON CONFLICT(path) DO UPDATE SET checksum = excluded.checksum, line_count = excluded.line_count, hashes = excluded.hashes, updated_at = excluded.updated_at",
+		),
+	};
+	return { db, stmts };
+}
+
+async function migrateLegacy(store: HashStore): Promise<void> {
+	const legacyPath = legacyHashStorePath();
+	let content: string;
+	try {
+		content = await readFile(legacyPath, "utf-8");
+	} catch (error: unknown) {
+		if (errCode(error) === "ENOENT") return;
+		console.error("Failed to read legacy hash store for migration:", error);
+		return;
+	}
+
+	let parsed: { snapshots?: Record<string, unknown> };
+	try {
+		parsed = JSON.parse(content) as typeof parsed;
+	} catch (error) {
+		console.error("Failed to parse legacy hash store, skipping migration:", error);
+		return;
+	}
+
+	const raw = parsed.snapshots;
+	if (!raw || typeof raw !== "object" || Array.isArray(raw)) return;
+
+	const insert = store.db.prepare(
+		"INSERT OR REPLACE INTO snapshots (path, checksum, line_count, hashes, updated_at) VALUES (?, ?, ?, ?, ?)",
+	);
+	const rows: [string, string, number, string, number][] = [];
+	for (const [key, value] of Object.entries(raw)) {
+		if (!isValidSnapshot(value)) continue;
+		rows.push([
+			key,
+			contentChecksum(value.content),
+			value.content.split("\n").length,
+			JSON.stringify(value.hashes),
+			Date.now(),
+		]);
+	}
+
+	if (rows.length > 0) {
+		store.db.transaction(() => {
+			for (const row of rows) insert.run(...row);
+		}).immediate();
+	}
+
+	try {
+		await rename(legacyPath, `${legacyPath}.bak`);
+	} catch (error) {
+		console.error("Failed to rename legacy hash store after migration:", error);
+	}
 }
 
 export async function loadHashStore(): Promise<HashStore> {
-  try {
-    const content = await readFile(hashStorePath(), "utf-8");
-    const parsed = JSON.parse(content) as Partial<HashStore>;
-    const raw = parsed.snapshots;
-    const snapshots: Record<string, FileSnapshot> = {};
-    if (raw && typeof raw === "object" && !Array.isArray(raw)) {
-      for (const [key, value] of Object.entries(raw)) {
-        if (isValidSnapshot(value)) {
-          snapshots[key] = value;
-        }
-      }
-    }
-    return { version: 1, snapshots };
-  } catch (error: unknown) {
-    if (errCode(error) !== "ENOENT") {
-      console.error("Hash store corrupted, creating fresh store:", error);
-    }
-    await mkdir(hashStoreDir(), { recursive: true });
-    const defaultStore: HashStore = {
-      version: 1,
-      snapshots: {},
-    };
-    await writeAtomic(hashStorePath(), JSON.stringify(defaultStore));
-    return defaultStore;
-  }
+	const storePath = hashStorePath();
+	if (cachedStore && cachedStore.path === storePath) {
+		return cachedStore.store;
+	}
+
+	shutdownHashStore();
+
+	await initHasher();
+	await mkdir(hashStoreDir(), { recursive: true });
+
+	const existed = existsSync(storePath);
+	const store = openDatabase(storePath);
+	if (!existed) {
+		await migrateLegacy(store);
+	}
+	cachedStore = { path: storePath, store };
+	return store;
 }
 
-export async function saveHashStore(store: HashStore): Promise<void> {
-  await mkdir(hashStoreDir(), { recursive: true });
-  await writeAtomic(hashStorePath(), JSON.stringify(store, null, 2));
+export function shutdownHashStore(): void {
+	if (cachedStore) {
+		cachedStore.store.db.close();
+		cachedStore = null;
+	}
 }
 
-export async function pruneHashStore(store: HashStore): Promise<void> {
-  let changed = false;
-  for (const filePath of Object.keys(store.snapshots)) {
-    try {
-      await stat(filePath);
-    } catch {
-      delete store.snapshots[filePath];
-      changed = true;
-    }
-  }
-  if (changed) {
-    await saveHashStore(store);
-  }
+export function getSnapshot(
+	store: HashStore,
+	path: string,
+	content: string,
+): string[] | undefined {
+	const checksum = contentChecksum(content);
+	const lineCount = content.split("\n").length;
+	const row = store.stmts.get.get(path, checksum, lineCount);
+	return row ? (JSON.parse(row.hashes) as string[]) : undefined;
 }
+
+export function upsertSnapshot(
+	store: HashStore,
+	path: string,
+	checksum: string,
+	lineCount: number,
+	hashes: string[],
+): void {
+	const hashesJson = JSON.stringify(hashes);
+	store.db.transaction(() => {
+		store.stmts.upsert.run(path, checksum, lineCount, hashesJson, Date.now());
+	}).immediate();
+}
+
+export function deleteSnapshot(store: HashStore, path: string): void {
+	store.db.transaction(() => {
+		store.stmts.deleteOne.run(path);
+	}).immediate();
+}
+
+export async function pruneMissing(store: HashStore): Promise<void> {
+	const rows = store.stmts.allPaths.all();
+	const missing: string[] = [];
+	for (const row of rows) {
+		try {
+			await stat(row.path);
+		} catch {
+			missing.push(row.path);
+		}
+	}
+	if (missing.length === 0) return;
+	store.db.transaction(() => {
+		for (const path of missing) store.stmts.deleteOne.run(path);
+	}).immediate();
+}
+
+export { HASH_STORE_VERSION };

package/src/hashline/apply.ts

@@ -51,7 +51,7 @@ type RESpan = {
 function assertNotEmpty(originalContent: string, result: string): void {
 	if (originalContent.length > 0 && result.length === 0) {
 		throw new Error(
-			"[E_WOULD_EMPTY] Cannot empty a non-empty file via edit."
+			"[E_WOULD_EMPTY] Cannot empty a non-empty file via edit. Use `write` if you need to clear the file."
 		);
 	}
 }

package/src/hashline/hash.ts

@@ -1,7 +1,13 @@
-import xxhash from "xxhash-wasm";
-import * as Diff from "diff";
-import { loadHashStore, saveHashStore, type HashStore } from "../hash-store";
 import { MAX_HASH_RETRIES } from "../constants";
+import {
+  loadHashStore,
+  type HashStore,
+  getSnapshot,
+  upsertSnapshot,
+} from "../hash-store";
+import { xxh32, contentChecksum, initHasher } from "./hasher";
+
+export { initHasher };
 
 export const HASH_LEN = 3;
 export const ANCHOR_LEN = HASH_LEN;
@@ -41,30 +47,6 @@ export const DIFF_MINUS_RE = /^-\s*\d+\s{4}/;
 
 export const HL_BARE_PREFIX_RE = new RegExp(`^\\s*(${HASH_CLASS})│`);
 
-type Hasher = { h32(input: string, seed?: number): number };
-let hasherP: Promise<Hasher> | null = null;
-let hasher: Hasher | null = null;
-
-function getH(): Hasher {
-	if (hasher) return hasher;
-	throw new Error("xxhash-wasm not initialized yet. This should not happen.");
-}
-
-hasherP = xxhash().then((h) => {
-	hasher = h;
-	return h;
-}).catch((err) => {
-	console.error("xxhash-wasm initialization failed:", err);
-	throw err;
-});
-
-export function initHasher(): Promise<Hasher> {
-	return hasherP!
-}
-
-function xxh32(input: string, seed = 0): number {
-	return getH().h32(input, seed) >>> 0;
-}
 
 function canon(line: string): string {
 	return line.replace(/\r/g, "").trimEnd();
@@ -99,6 +81,7 @@ export async function lineHashes(
   path?: string,
   previous?: { content: string; hashes: string[]; removedHashes?: Set<string> },
   store?: HashStore,
+  persist?: boolean,
 ): Promise<string[]> {
   if (!path) {
     return _lineHashesPure(content);
@@ -112,19 +95,21 @@ export async function lineHashes(
       content,
       previous.removedHashes,
     );
-    hashStore.snapshots[path] = { content, hashes: newHashes };
-    await saveHashStore(hashStore);
+    if (persist !== false) {
+      upsertSnapshot(hashStore, path, contentChecksum(content), content.split("\n").length, newHashes);
+    }
     return newHashes;
   }
 
-  const snapshot = hashStore.snapshots[path];
-  if (snapshot && snapshot.content === content) {
-    return snapshot.hashes;
+  const cached = getSnapshot(hashStore, path, content);
+  if (cached) {
+    return cached;
   }
 
   const newHashes = _lineHashesPure(content);
-  hashStore.snapshots[path] = { content, hashes: newHashes };
-  await saveHashStore(hashStore);
+  if (persist !== false) {
+    upsertSnapshot(hashStore, path, contentChecksum(content), content.split("\n").length, newHashes);
+  }
   return newHashes;
 }
 

package/src/hashline/hasher.ts

@@ -0,0 +1,33 @@
+import xxhash from "xxhash-wasm";
+
+export type Hasher = {
+	h32(input: string, seed?: number): number;
+	h64ToString(input: string, seed?: bigint): string;
+};
+
+let hasher: Hasher | null = null;
+
+export function getH(): Hasher {
+	if (hasher) return hasher;
+	throw new Error("xxhash-wasm not initialized yet. This should not happen.");
+}
+
+const hasherP: Promise<Hasher> = xxhash().then((h) => {
+	hasher = h as unknown as Hasher;
+	return hasher;
+}).catch((err: unknown) => {
+	console.error("xxhash-wasm initialization failed:", err);
+	throw err;
+});
+
+export function initHasher(): Promise<Hasher> {
+	return hasherP;
+}
+
+export function xxh32(input: string, seed = 0): number {
+	return getH().h32(input, seed) >>> 0;
+}
+
+export function contentChecksum(content: string): string {
+	return getH().h64ToString(content);
+}

package/src/paths.ts

@@ -2,8 +2,13 @@ import { homedir } from "os";
 import { isAbsolute, resolve as resolvePath, join, dirname } from "path";
 
 
+function homeBase(): string {
+  const envHome = process.env.HOME;
+  return envHome && envHome.length > 0 ? envHome : homedir();
+}
+
 export function configDir(): string {
-  return join(homedir(), ".config", "pi-hashline-edit-pro");
+  return join(homeBase(), ".config", "pi-hashline-edit-pro");
 }
 
 export function configPath(): string {
@@ -11,6 +16,10 @@ export function configPath(): string {
 }
 
 export function hashStorePath(): string {
+  return join(configDir(), "hash-store.sqlite");
+}
+
+export function legacyHashStorePath(): string {
   return join(configDir(), "hash-store.json");
 }
 
@@ -19,8 +28,9 @@ export function hashStoreDir(): string {
 }
 
 function expand(filePath: string): string {
-  if (filePath === "~") return homedir();
-  if (filePath.startsWith("~/")) return homedir() + filePath.slice(1);
+  const home = homeBase();
+  if (filePath === "~") return home;
+  if (filePath.startsWith("~/")) return home + filePath.slice(1);
   return filePath;
 }
 

package/src/replace-undo.ts

@@ -2,7 +2,8 @@ import { readFile } from "fs/promises";
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { withFileMutationQueue } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
-import { loadHashStore, saveHashStore } from "./hash-store";
+import { loadHashStore, upsertSnapshot } from "./hash-store";
+import { contentChecksum } from "./hashline/hasher";
 import { resolveTarget, writeAtomic } from "./fs-write";
 import { toCwd } from "./paths";
 import { toLF, stripBOM, genDiff, restoreEndings } from "./replace-diff";
@@ -82,11 +83,7 @@ export function regReplaceUndo(pi: ExtensionAPI): void {
         );
 
         const store = await loadHashStore();
-        store.snapshots[mutationTargetPath] = {
-          content: undo.content,
-          hashes: undo.hashes,
-        };
-        await saveHashStore(store);
+        upsertSnapshot(store, mutationTargetPath, contentChecksum(undo.content), undo.content.split("\n").length, undo.hashes);
 
         clearUndo(mutationTargetPath);
 

package/src/replace.ts

@@ -154,6 +154,7 @@ export async function execPipeline(
   accessMode: number,
   signal?: AbortSignal,
   store?: HashStore,
+  noPersist?: boolean,
 ): Promise<PipelineResult> {
 
   const path = params.path;
@@ -199,7 +200,7 @@ export async function execPipeline(
     content: originalNormalized,
     hashes: originalHashes,
     removedHashes,
-  }, hashStore);
+  }, hashStore, noPersist !== true);
 
   const warnings = [...(anchorResult.warnings ?? [])];
 
@@ -248,6 +249,9 @@ export async function compPreview(
       normalized,
       cwd,
       constants.R_OK,
+      undefined,
+      undefined,
+      true,
     );
 
     if (originalNormalized === result) {