pi-delta-db 0.1.0 → 0.2.0

package/CHANGELOG.md

@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.0] - 2026-11-01
+
+### Added
+
+- Durable outbox: records are fsynced under `~/.pi/agent/delta-db/outbox/` before Delta is
+  contacted, and a failed append is retried on the next message and the next session start.
+- Thoughts, tool calls, and tool results are recorded, not only message text.
+- `/delta off` and `/delta on` are remembered for the session.
+- Bindings whose session file is gone and that have not been touched for a week are pruned.
+- Tests for record shapes, sequence assignment, outbox retry, and pruning (`npm test`).
+- CI runs `npm run check` and `npm test` on every push.
+- npm publish workflow: a published release publishes to npmjs.org when `NPM_TOKEN` is set.
+- GitHub Packages mirror: `.github/workflows/publish-github-packages.yml` publishes
+  `@morizkay/pi-delta-db` on every published release, populating the repository's Packages tab.
+  npmjs.org (`pi-delta-db`) remains the installable channel.
+
 ## [0.1.0] - 2026-11-01
 
 ### Added
@@ -32,5 +48,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `session_info_update` re-titled the thread on every continuation. The title is now written
   once and remembered in the session binding.
 
-[Unreleased]: https://github.com/morizkay/pi-delta-db/compare/v0.1.0...HEAD
+[Unreleased]: https://github.com/morizkay/pi-delta-db/compare/v0.2.0...HEAD
+[0.2.0]: https://github.com/morizkay/pi-delta-db/releases/tag/v0.2.0
 [0.1.0]: https://github.com/morizkay/pi-delta-db/releases/tag/v0.1.0

package/README.md

@@ -12,17 +12,23 @@ in the Delta app and can be read back with or without Pi.
 
 ## Auto-recording
 
-Every finalized user and assistant message is appended to this session's Delta thread
-automatically, in order. The thread is created on the first message and titled with the first
-prompt; continuing a session (`pi -c`) keeps appending to the same thread.
+Every finalized user message, assistant message, thought, tool call, and tool result is
+appended to this session's Delta thread, in order. The thread is created on the first message
+and titled with the first prompt; continuing a session (`pi -c`) keeps appending to the same thread.
+
+Records are written to `~/.pi/agent/delta-db/outbox/` and fsynced before Delta is contacted.
+If Delta is closed, offline, or rejects the write, the turn stays queued and is retried on the
+next message and the next session start. A retry cannot duplicate or erase earlier turns:
+each message id is unique, and Delta replaces a record that reuses one.
 
 | Control | Effect |
 | --- | --- |
-| `/delta` | Show recording state, thread id, title, share URL |
-| `/delta off` / `/delta on` | Toggle recording for this session |
-| `PI_DELTA_RECORD=0` | Start Pi with recording off |
+| `/delta` | Show recording state, thread id, queued count, title, share URL |
+| `/delta off` / `/delta on` | Toggle recording for this session, remembered next start |
+| `PI_DELTA_RECORD=0` | Start Pi with recording off for this process |
 
-Only text is recorded — thinking blocks and tool calls are skipped.
+Bindings whose Pi session file is gone and that have not been touched for a week are deleted
+on session start.
 
 ## Tools
 
@@ -63,13 +69,30 @@ For local development of this repository, link it into the agent directory inste
 ln -sfn "$(pwd)" ~/.pi/agent/extensions/pi-delta-db
 ```
 
+### GitHub Packages mirror
+
+Every release is also mirrored to GitHub Packages as `@morizkay/pi-delta-db` by
+[`.github/workflows/publish-github-packages.yml`](./.github/workflows/publish-github-packages.yml),
+which is why the repository's Packages tab lists it. Prefer npm: GitHub Packages requires a token
+with `read:packages` even for public packages.
+
+```bash
+# ~/.npmrc: //npm.pkg.github.com/:_authToken=${GITHUB_TOKEN}
+pi install npm:@morizkay/pi-delta-db
+```
+
 ## Development
 
 ```bash
 npm install    # host packages are peer dependencies, dev-only here
 npm run check  # tsc --noEmit
+npm test       # record shapes, outbox retry, and prune, against a fake delta
 ```
 
+CI runs both on every push. Publishing a GitHub release also mirrors the package to GitHub
+Packages, and publishes to npmjs.org when the `NPM_TOKEN` secret is set
+(`gh secret set NPM_TOKEN`; use an automation token so OTP is not required).
+
 ## Gotchas found against delta 0.23.0
 
 - `messageId` must be globally unique: a record whose `messageId` matches an earlier one

package/index.ts

@@ -4,6 +4,10 @@
  * Delta's SQLite store (`delta db ...`) is internal and read-only from outside,
  * so writes go through Delta's supported path: a private thread bound to this
  * session (`--source pi --external-id <session>`) recorded with schema-1 JSONL.
+ *
+ * Records are staged to a local outbox before any network call. A failed append
+ * stays on disk and is retried; resending is safe because each messageId is unique
+ * and Delta replaces a record that reuses one.
  */
 
 import { spawn } from "node:child_process";
@@ -16,9 +20,23 @@ import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-a
 import { Type } from "typebox";
 
 const SOURCE = "pi";
-const STATE_DIR = path.join(os.homedir(), ".pi", "agent", "delta-db");
+const STATE_DIR = process.env.PI_DELTA_STATE || path.join(os.homedir(), ".pi", "agent", "delta-db");
+const SESSIONS_DIR = path.join(os.homedir(), ".pi", "agent", "sessions");
 const MAX_CHARS = 12_000;
 const TIMEOUT_MS = 120_000;
+const PRUNE_AFTER_MS = 7 * 24 * 60 * 60 * 1000;
+
+export interface Update {
+	sessionUpdate: string;
+	messageId?: string;
+	title?: string;
+	content?: { type: string; text: string }[];
+	toolCallId?: string;
+	name?: string;
+	status?: string;
+	rawInput?: unknown;
+	rawOutput?: unknown;
+}
 
 interface Org {
 	id: string;
@@ -51,22 +69,23 @@ interface Stats {
 interface ContentBlock {
 	type?: string;
 	text?: string;
+	thinking?: string;
+	id?: string;
+	name?: string;
+	arguments?: unknown;
 }
 interface FinalMessage {
 	role?: string;
 	content?: string | ContentBlock[];
+	toolCallId?: string;
+	toolName?: string;
+	isError?: boolean;
 }
-interface Update {
-	sessionUpdate: string;
-	messageId?: string;
-	title?: string;
-	content?: { type: string; text: string }[];
-}
-
 /** A session's Delta thread binding. `titled` keeps a continued session from renaming it. */
 interface Binding {
-	threadId: string;
+	threadId?: string;
 	titled?: boolean;
+	enabled?: boolean;
 }
 
 type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
@@ -130,10 +149,14 @@ function bindingFile(key: string): string {
 	return path.join(STATE_DIR, `${key}.json`);
 }
 
+function outboxFile(key: string): string {
+	return path.join(STATE_DIR, "outbox", `${key}.jsonl`);
+}
+
 function readBinding(key: string): Binding | undefined {
 	try {
 		const saved = parseJson(fs.readFileSync(bindingFile(key), "utf8")) as Binding | undefined;
-		return saved?.threadId ? saved : undefined;
+		return saved && typeof saved === "object" && !Array.isArray(saved) ? saved : undefined;
 	} catch {
 		// No binding file yet: this session has never written to Delta.
 		return undefined;
@@ -145,9 +168,19 @@ function writeBinding(key: string, binding: Binding): void {
 	fs.writeFileSync(bindingFile(key), JSON.stringify(binding, null, 2));
 }
 
+/** `PI_DELTA_RECORD=0` forces off for this process; `/delta off` persists in the binding. */
+export function recordingEnabled(key: string): boolean {
+	if (process.env.PI_DELTA_RECORD === "0") return false;
+	return readBinding(key)?.enabled !== false;
+}
+
+export function setRecording(key: string, enabled: boolean): void {
+	writeBinding(key, { ...readBinding(key), enabled });
+}
+
 async function ensureThread(key: string): Promise<Binding> {
 	const bound = readBinding(key);
-	if (bound) return bound;
+	if (bound?.threadId) return bound;
 	const { organizations } = await json<{ organizations?: Org[] }>(["org", "list", "--json"]);
 	const org = organizations?.find((item) => item.current) ?? organizations?.[0];
 	if (!org) throw new Error("No Delta organization. Run `delta auth login`, then retry.");
@@ -164,7 +197,7 @@ async function ensureThread(key: string): Promise<Binding> {
 		org.id,
 		"--json",
 	]);
-	const binding: Binding = { threadId: created.thread_id };
+	const binding: Binding = { ...bound, threadId: created.thread_id };
 	writeBinding(key, binding);
 	return binding;
 }
@@ -173,27 +206,27 @@ function statusOf(key: string, threadId: string): Promise<ThreadStatus> {
 	return json<ThreadStatus>(["thread", "status", "--source", SOURCE, "--external-id", key, "--json", threadId]);
 }
 
+/** Schema-1 JSONL continuing at `first`. Exported so tests can lock the envelope. */
+export function assignSequences(first: number, updates: Update[]): string {
+	return updates
+		.map((update, index) => JSON.stringify({ schemaVersion: 1, sequence: first + index, update }))
+		.join("\n");
+}
+
 /** Append schema-1 records, continuing this producer's sequence. */
 async function append(key: string, threadId: string, updates: Update[]) {
 	const status = await statusOf(key, threadId);
 	const first = status.producer?.next_sequence;
 	if (first === undefined || !Number.isSafeInteger(first))
 		throw new Error("Delta did not report the next sequence for this thread");
-	let sequence = first;
-	const lines = updates
-		.map((update) => JSON.stringify({ schemaVersion: 1, sequence: sequence++, update }))
-		.join("\n");
 	await run(
 		["thread", "append", "--source", SOURCE, "--external-id", key, "--acknowledge", "local", "--json", threadId],
-		`${lines}\n`,
+		`${assignSequences(first, updates)}\n`,
 	);
-	return { count: updates.length, nextSequence: sequence, url: status.url };
+	return { count: updates.length, nextSequence: first + updates.length, url: status.url };
 }
 
-/** Text of a finalized message; only text blocks are recorded. */
-// shortcut: thinking and tool calls are not recorded, add them when the transcript needs them
-function messageText(message: FinalMessage): string {
-	const content = message?.content;
+function textOf(content: string | ContentBlock[] | undefined): string {
 	if (typeof content === "string") return content.trim();
 	if (!Array.isArray(content)) return "";
 	return content
@@ -203,11 +236,162 @@ function messageText(message: FinalMessage): string {
 		.trim();
 }
 
+function textUpdate(role: string | undefined, id: string, text: string): Update {
+	return {
+		sessionUpdate: role === "user" ? "user_message" : "agent_message",
+		messageId: id,
+		content: [{ type: "text", text: clip(text) }],
+	};
+}
+
+/**
+ * Turn one finalized Pi message into Delta records.
+ * `id` must be unique per message: Delta replaces a record that reuses a messageId.
+ */
+export function updatesFromMessage(message: FinalMessage, id: string): Update[] {
+	const updates: Update[] = [];
+	if (message.role === "toolResult" && message.toolCallId) {
+		updates.push({
+			sessionUpdate: "tool_call_update",
+			toolCallId: message.toolCallId,
+			name: message.toolName || "tool",
+			status: message.isError ? "failed" : "completed",
+			rawOutput: clip(textOf(message.content) || (message.isError ? "failed" : "")),
+		});
+		return updates;
+	}
+	const content = message.content;
+	if (typeof content === "string") {
+		if (content.trim()) updates.push(textUpdate(message.role, `${id}:t0`, content.trim()));
+		return updates;
+	}
+	if (!Array.isArray(content)) return updates;
+	const texts: string[] = [];
+	let part = 0;
+	const flushText = () => {
+		const text = texts.join("\n").trim();
+		texts.length = 0;
+		if (!text) return;
+		updates.push(textUpdate(message.role, `${id}:t${part++}`, text));
+	};
+	for (const [index, block] of content.entries()) {
+		if (block?.type === "thinking" && block.thinking?.trim()) {
+			flushText();
+			updates.push({
+				sessionUpdate: "agent_thought",
+				messageId: `${id}:thought:${index}`,
+				content: [{ type: "text", text: clip(block.thinking.trim()) }],
+			});
+		} else if (block?.type === "toolCall" && block.id) {
+			flushText();
+			updates.push({
+				sessionUpdate: "tool_call_update",
+				toolCallId: block.id,
+				name: block.name || "tool",
+				status: "in_progress",
+				rawInput: block.arguments ?? {},
+			});
+		} else if (block?.type === "text" && block.text?.trim()) {
+			texts.push(block.text);
+		}
+	}
+	flushText();
+	return updates;
+}
+
 function firstLine(text: string): string {
 	const line = text.split("\n").find((part) => part.trim()) ?? "";
 	return line.trim().slice(0, 60);
 }
 
+/** Append records to the outbox and fsync. A crash after this still retries. */
+export function stageUpdates(key: string, updates: Update[]): void {
+	if (!updates.length) return;
+	const file = outboxFile(key);
+	fs.mkdirSync(path.dirname(file), { recursive: true });
+	const fd = fs.openSync(file, "a");
+	try {
+		fs.writeSync(fd, `${updates.map((update) => JSON.stringify(update)).join("\n")}\n`);
+		fs.fsyncSync(fd);
+	} finally {
+		fs.closeSync(fd);
+	}
+}
+
+export function pendingUpdates(key: string): Update[] {
+	let text: string;
+	try {
+		text = fs.readFileSync(outboxFile(key), "utf8");
+	} catch {
+		return [];
+	}
+	if (!text.trim()) return [];
+	return text
+		.split("\n")
+		.filter(Boolean)
+		.map((line) => {
+			try {
+				return JSON.parse(line) as Update;
+			} catch {
+				throw new Error(`corrupt outbox record for ${key}; sharing paused without dropping it`);
+			}
+		});
+}
+
+/** Send the outbox, then drop it. On failure the file stays for the next attempt. */
+export async function flushPending(key: string): Promise<{ count: number; threadId?: string; url?: string; nextSequence?: number }> {
+	const updates = pendingUpdates(key);
+	if (!updates.length) return { count: 0 };
+	const binding = await ensureThread(key);
+	if (!binding.threadId) throw new Error("Delta thread was not created");
+	const result = await append(key, binding.threadId, updates);
+	fs.rmSync(outboxFile(key), { force: true });
+	return { count: result.count, threadId: binding.threadId, url: result.url, nextSequence: result.nextSequence };
+}
+
+function sessionFileExists(sessionsDir: string, key: string): boolean {
+	const stack = [sessionsDir];
+	while (stack.length) {
+		const dir = stack.pop();
+		if (!dir) break;
+		let entries: fs.Dirent[];
+		try {
+			entries = fs.readdirSync(dir, { withFileTypes: true });
+		} catch {
+			continue;
+		}
+		for (const entry of entries) {
+			if (entry.isDirectory()) stack.push(path.join(dir, entry.name));
+			else if (entry.name.endsWith(`_${key}.jsonl`)) return true;
+		}
+	}
+	return false;
+}
+
+/** Drop bindings whose session file is gone and that have not been touched for a week. */
+export function pruneBindings(currentKey: string, sessionsDir: string, now = Date.now()): string[] {
+	if (!fs.existsSync(sessionsDir) || !fs.existsSync(STATE_DIR)) return [];
+	let names: string[];
+	try {
+		names = fs.readdirSync(STATE_DIR);
+	} catch {
+		return [];
+	}
+	const removed: string[] = [];
+	for (const name of names) {
+		if (!name.endsWith(".json")) continue;
+		const key = name.slice(0, -".json".length);
+		if (key === currentKey || !/^[\w-]+$/.test(key)) continue;
+		const file = path.join(STATE_DIR, name);
+		if (now - fs.statSync(file).mtimeMs < PRUNE_AFTER_MS) continue;
+		if (sessionFileExists(sessionsDir, key)) continue;
+		fs.rmSync(file, { force: true });
+		fs.rmSync(outboxFile(key), { force: true });
+		removed.push(key);
+	}
+	return removed;
+}
+
 function summarizeList(threads: ThreadListItem[]): string {
 	if (!threads.length) return "No Delta threads yet.";
 	return threads
@@ -219,6 +403,22 @@ function summarizeList(threads: ThreadListItem[]): string {
 		.join("\n");
 }
 
+/** Stage a message, including the thread title the first time a user prompt arrives. */
+function stageMessage(key: string, message: FinalMessage, id: string): number {
+	const updates = updatesFromMessage(message, id);
+	if (!updates.length) return 0;
+	const binding = readBinding(key);
+	if (message.role === "user" && !binding?.titled) {
+		const title = firstLine(textOf(message.content));
+		if (title) {
+			updates.unshift({ sessionUpdate: "session_info_update", title });
+			writeBinding(key, { ...binding, titled: true });
+		}
+	}
+	stageUpdates(key, updates);
+	return updates.length;
+}
+
 export default function (pi: ExtensionAPI) {
 	// One writer at a time: sequence numbers come from `thread status`, so overlapping
 	// appends would reuse a sequence and Delta would reject them.
@@ -230,49 +430,49 @@ export default function (pi: ExtensionAPI) {
 	}
 
 	let rec: { key: string; enabled: boolean; warned: boolean } | null = null;
-
 	const sessionOf = (ctx: ExtensionContext) =>
-		(rec ??= { key: sessionKey(ctx), enabled: process.env.PI_DELTA_RECORD !== "0", warned: false });
+		(rec ??= { key: sessionKey(ctx), enabled: recordingEnabled(sessionKey(ctx)), warned: false });
+
+	async function record(key: string, message: FinalMessage, id: string): Promise<void> {
+		stageMessage(key, message, id);
+		await flushPending(key);
+	}
 
 	pi.on("session_start", (_event, ctx) => {
-		rec = { key: sessionKey(ctx), enabled: process.env.PI_DELTA_RECORD !== "0", warned: false };
+		const key = sessionKey(ctx);
+		rec = { key, enabled: recordingEnabled(key), warned: false };
+		pruneBindings(key, SESSIONS_DIR);
+		enqueue(() => flushPending(key)).catch((error: Error) => {
+			if (rec?.warned) return;
+			if (rec) rec.warned = true;
+			ctx.ui.notify(`Delta unreachable; queued records will retry. ${error.message}`, "warning");
+		});
 	});
 
-	// Auto-record every finalized user and assistant message into this session's thread.
 	pi.on("message_end", (event, ctx) => {
 		const current = sessionOf(ctx);
 		if (!current.enabled) return;
 		const message = event.message as FinalMessage;
-		if (message?.role !== "user" && message?.role !== "assistant") return;
-		const text = messageText(message);
-		if (!text) return;
-		const wantsTitle = message.role === "user" ? firstLine(text) : undefined;
+		if (message?.role !== "user" && message?.role !== "assistant" && message?.role !== "toolResult") return;
+		const id = randomUUID();
 		enqueue(async () => {
 			try {
-				const binding = await ensureThread(current.key);
-				const updates: Update[] = [];
-				// A messageId must be globally unique: Delta replaces an existing record with the same id.
-				if (wantsTitle && !binding.titled) {
-					updates.push({ sessionUpdate: "session_info_update", title: wantsTitle });
-					binding.titled = true;
-					writeBinding(current.key, binding);
+				await record(current.key, message, id);
+				if (current.warned) {
+					current.warned = false;
+					ctx.ui.notify("Delta recording resumed", "info");
 				}
-				updates.push({
-					sessionUpdate: message.role === "user" ? "user_message" : "agent_message",
-					messageId: `${current.key}:${randomUUID()}`,
-					content: [{ type: "text", text }],
-				});
-				await append(current.key, binding.threadId, updates);
 			} catch (error) {
 				if (current.warned) return;
 				current.warned = true;
-				ctx.ui.notify(`Delta recording stopped: ${(error as Error).message}`, "warning");
+				ctx.ui.notify(`Delta unreachable; this turn is queued and will retry. ${(error as Error).message}`, "warning");
 			}
 		});
 	});
 
 	pi.on("session_shutdown", async () => {
-		// Let the tail of the queue reach Delta before the process exits.
+		// Let the tail of the queue reach Delta before the process exits. What doesn't
+		// make it stays in the outbox for the next start.
 		await Promise.race([queue, new Promise((resolve) => setTimeout(resolve, 8000))]);
 	});
 
@@ -291,19 +491,16 @@ export default function (pi: ExtensionAPI) {
 		async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
 			const key = sessionOf(ctx).key;
 			const result = await enqueue(async () => {
-				const binding = await ensureThread(key);
+				const binding = readBinding(key);
 				const updates: Update[] = [];
-				if (params.title && !binding.titled) {
+				if (params.title && !binding?.titled) {
 					updates.push({ sessionUpdate: "session_info_update", title: params.title });
-					binding.titled = true;
-					writeBinding(key, binding);
+					writeBinding(key, { ...binding, titled: true });
 				}
-				updates.push({
-					sessionUpdate: params.role === "user" ? "user_message" : "agent_message",
-					messageId: `${key}:${randomUUID()}`,
-					content: [{ type: "text", text: params.text }],
-				});
-				return { ...(await append(key, binding.threadId, updates)), threadId: binding.threadId };
+				updates.push(textUpdate(params.role, `${key}:${randomUUID()}`, params.text));
+				stageUpdates(key, updates);
+				const flushed = await flushPending(key);
+				return { ...flushed, threadId: flushed.threadId };
 			});
 			const where = result.url ? ` ${result.url}` : "";
 			return {
@@ -401,15 +598,18 @@ export default function (pi: ExtensionAPI) {
 			const arg = args.trim().toLowerCase();
 			if (arg === "on" || arg === "off") {
 				current.enabled = arg === "on";
+				setRecording(current.key, current.enabled);
 				ctx.ui.notify(`Delta recording ${arg}`, "info");
 				return;
 			}
 			const binding = readBinding(current.key);
+			const queued = pendingUpdates(current.key).length;
 			const lines = [
-				`Recording: ${current.enabled ? "on" : "off"} (/delta on|off)`,
-				binding ? `Thread: ${binding.threadId}` : "Thread: none yet",
+				`Recording: ${current.enabled ? "on" : "off"} (/delta on|off, remembered)`,
+				binding?.threadId ? `Thread: ${binding.threadId}` : "Thread: none yet",
+				queued ? `Queued: ${queued}` : "Queued: 0",
 			];
-			if (binding) {
+			if (binding?.threadId) {
 				const status = await statusOf(current.key, binding.threadId);
 				if (status.title) lines.push(`Title: ${status.title}`);
 				if (status.url) lines.push(status.url);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-delta-db",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Pi extension that stores and reads Pi session data through the Delta CLI: auto-recorded Delta threads plus read-only inspection of Delta's local store",
   "license": "MIT",
   "type": "module",
@@ -45,7 +45,8 @@
     "typescript": "^5.9.3"
   },
   "scripts": {
-    "check": "tsc --noEmit"
+    "check": "tsc --noEmit",
+    "test": "node --test test/*.test.mjs"
   },
   "repository": {
     "type": "git",