pi-session-memory 0.1.3 → 0.1.4

package/README.md

@@ -8,7 +8,8 @@ A local-first Pi extension that saves completed conversations to SQLite and give
 ## Features
 
 - Persists completed Pi conversations in `~/.pi/agent/memory.db`.
-- Imports historical session records from Pi, Claude Code, and Codex with `/memory-backfill`.
+- Automatically imports only new or changed Pi, Claude Code, and Codex session files when Pi starts.
+- Supports a forced full rescan with `/memory-backfill`.
 - Exposes `recall_memory`, allowing Pi to retrieve relevant prior discussions when users explicitly refer to earlier work.
 - Uses stable native user-message IDs and `INSERT OR IGNORE`, making live persistence and backfill idempotent.
 - Searches literal substrings with escaped SQLite `LIKE` patterns, including technical terms containing `%`, `_`, or `\\`.
@@ -17,20 +18,25 @@ A local-first Pi extension that saves completed conversations to SQLite and give
 ## Installation
 
 ```bash
-pi install npm:pi-session-memory@0.1.3
+pi install npm:pi-session-memory@0.1.4
 ```
 
 To try the package without installing it permanently:
 
 ```bash
-pi -e npm:pi-session-memory@0.1.3
+pi -e npm:pi-session-memory@0.1.4
 ```
 
 ## Usage
 
 ### Import existing history
 
-Run this once after installation, and again whenever you want to scan newly available historical session files:
+At Pi startup, the extension automatically scans the three source roots. It compares
+per-file size and modification time to saved sync state; only new or changed JSONL
+files are read and hashed with SHA-256 before import. Unchanged files are skipped.
+
+Use this command when you intentionally want to force a full rescan of every
+historical JSONL file:
 
 ```text
 /memory-backfill
@@ -61,10 +67,12 @@ The tool searches prior user prompts and assistant responses, ranks matches by e
 ## How it works
 
 ```text
-completed Pi agent run ───► SQLite memory.db ◄─── /memory-backfill
-                                  ▲                     │
-                                  │                     ▼
-                           recall_memory         Pi / Claude / Codex
+Pi startup ──► incremental source sync ──► SQLite memory.db ◄── completed Pi agent run
+                                                  ▲
+                                                  │
+                                           recall_memory
+                                                  │
+                                      Pi / Claude / Codex
 ```
 
 After a Pi agent run settles, the extension captures the latest user message and the subsequent assistant replies/tool names from the active session branch. Historical imports normalize each supported source into the same session/turn schema.

package/extensions/index.ts

@@ -2,10 +2,20 @@ import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-a
 import { Type } from "typebox";
 import { writeTurn } from "../src/writer.ts";
 import { recallTurns, formatRecallResults } from "../src/retriever.ts";
-import { backfillAll } from "../src/backfill.ts";
+import { backfillAll, syncChangedHistory } from "../src/backfill.ts";
 
 export default function (pi: ExtensionAPI) {
 
+  pi.on("session_start", async (_event, ctx) => {
+    const stats = syncChangedHistory();
+    if (stats.scannedFiles > 0) {
+      ctx.ui.notify(
+        `[session-memory] synced ${stats.turns} turns from ${stats.scannedFiles} changed session files`,
+        "info",
+      );
+    }
+  });
+
   // ── Write: persist each completed agent run to SQLite ────────────────────
   pi.on("agent_settled", async (_event, ctx) => {
     try {
@@ -20,7 +30,7 @@ export default function (pi: ExtensionAPI) {
     handler: async (_args, ctx) => {
       const stats = backfillAll();
       ctx.ui.notify(
-        `[session-memory] imported ${stats.turns} turns from ${stats.pi} Pi, ${stats.claude} Claude, ${stats.codex} Codex sessions`,
+        `[session-memory] imported ${stats.turns} turns from ${stats.scannedFiles} files: ${stats.pi} Pi, ${stats.claude} Claude, ${stats.codex} Codex sessions`,
         "info",
       );
     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-session-memory",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "Persistent, local-first cross-session memory for Pi, with SQLite-backed recall across Pi, Claude Code, and Codex conversations",
   "keywords": [
     "pi-package"

package/spec.md

@@ -14,12 +14,14 @@ pi turn_end event ────────────────────
                                  Writer (src/writer.ts)
                                  - writes current pi turns
                                              │
-/backfill command ────────┐                 │
-                           ▼                 ▼
-                    Source adapters → SQLite ~/.pi/agent/memory.db
-                    - Pi JSONL        sessions + turns
-                    - Claude JSONL
-                    - Codex JSONL
+session_start ───────────┐                 │
+                          ▼                 ▼
+                 Incremental source sync → SQLite ~/.pi/agent/memory.db
+                 - per-file size + mtime quick check
+                 - SHA-256 only for changed candidates
+                 - Pi / Claude / Codex JSONL adapters
+
+/memory-backfill forces a full rescan.
                                              │
                                              ▼
                                  recall_memory tool
@@ -72,6 +74,15 @@ Steps:
 
 ## Retrieval Path (recall_memory tool)
 
+### source_files
+| column | type | note |
+|---|---|---|
+| jsonl_path | TEXT PK | absolute source file path |
+| source | TEXT | `pi`, `claude`, or `codex` |
+| size | INTEGER | file size at last sync |
+| mtime_ms | REAL | modification time at last sync |
+| sha256 | TEXT | content hash for changed candidates |
+
 ### Tool Invocation Policy
 
 - **Direct recall:** Call `recall_memory` immediately when the user explicitly
@@ -101,8 +112,15 @@ escaping, so it does not lose substring matches through tokenization.
 
 ## Backfill
 
-`/memory-backfill` scans and imports all historical records. All writes use
-`INSERT OR IGNORE`, so it is safe and idempotent to run repeatedly.
+At `session_start`, `syncChangedHistory()` recursively enumerates the three
+source roots. It records one row per source JSONL file in `source_files`:
+`jsonl_path`, source, size, `mtime_ms`, and SHA-256. Unchanged size/mtime files
+are skipped without reading; changed candidates are SHA-256 checked, and only
+new content is parsed and imported.
+
+`/memory-backfill` forces a full reparse of all historical records. All writes
+use `INSERT OR IGNORE`, so both automatic sync and forced backfill are safe and
+idempotent to run repeatedly.
 
 | source | scan root | accepted user/assistant records | excluded records |
 |---|---|---|---|

package/src/backfill.ts

@@ -1,7 +1,8 @@
-import { readdirSync, readFileSync } from "node:fs";
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { createHash } from "node:crypto";
 import { homedir } from "node:os";
 import { join } from "node:path";
-import { insertTurn, upsertSession } from "./db.ts";
+import { getSourceFile, insertTurn, upsertSession, upsertSourceFile } from "./db.ts";
 
 type Source = "pi" | "claude" | "codex";
 type Role = "user" | "assistant";
@@ -29,35 +30,52 @@ export interface BackfillStats {
   claude: number;
   codex: number;
   turns: number;
+  scannedFiles: number;
+  skippedFiles: number;
 }
 
+const SOURCES: Array<{ source: Source; root: string; parse: (path: string) => ImportedSession | undefined }> = [
+  { source: "pi", root: join(homedir(), ".pi", "agent", "sessions"), parse: _parsePi },
+  { source: "claude", root: join(homedir(), ".claude", "projects"), parse: _parseClaude },
+  { source: "codex", root: join(homedir(), ".codex", "sessions"), parse: _parseCodex },
+];
+
+/** Reparse every known source file. Intended for the explicit /memory-backfill command. */
 export function backfillAll(): BackfillStats {
-  const stats: BackfillStats = { pi: 0, claude: 0, codex: 0, turns: 0 };
+  return _syncHistory(true);
+}
 
-  for (const jsonlPath of _jsonlFiles(join(homedir(), ".pi", "agent", "sessions"))) {
-    const session = _parsePi(jsonlPath);
-    if (session) {
-      stats.pi++;
-      stats.turns += _persist(session);
-    }
-  }
+/** Parse only source files that are new or whose metadata/content changed. */
+export function syncChangedHistory(): BackfillStats {
+  return _syncHistory(false);
+}
 
-  for (const jsonlPath of _jsonlFiles(join(homedir(), ".claude", "projects"))) {
-    const session = _parseClaude(jsonlPath);
-    if (session) {
-      stats.claude++;
-      stats.turns += _persist(session);
-    }
-  }
+function _syncHistory(force: boolean): BackfillStats {
+  const stats: BackfillStats = { pi: 0, claude: 0, codex: 0, turns: 0, scannedFiles: 0, skippedFiles: 0 };
+  for (const definition of SOURCES) {
+    for (const jsonlPath of _jsonlFiles(definition.root)) {
+      const metadata = statSync(jsonlPath);
+      const known = getSourceFile(jsonlPath);
+      if (!force && known?.size === metadata.size && known.mtime_ms === metadata.mtimeMs) {
+        stats.skippedFiles++;
+        continue;
+      }
+
+      const sha256 = _sha256(jsonlPath);
+      if (!force && known?.sha256 === sha256) {
+        upsertSourceFile({ jsonl_path: jsonlPath, source: definition.source, size: metadata.size, mtime_ms: metadata.mtimeMs, sha256 });
+        stats.skippedFiles++;
+        continue;
+      }
 
-  for (const jsonlPath of _jsonlFiles(join(homedir(), ".codex", "sessions"))) {
-    const session = _parseCodex(jsonlPath);
-    if (session) {
-      stats.codex++;
+      const session = definition.parse(jsonlPath);
+      upsertSourceFile({ jsonl_path: jsonlPath, source: definition.source, size: metadata.size, mtime_ms: metadata.mtimeMs, sha256 });
+      stats.scannedFiles++;
+      if (!session) continue;
+      stats[definition.source]++;
       stats.turns += _persist(session);
     }
   }
-
   return stats;
 }
 
@@ -244,6 +262,10 @@ function _isCodexInjectedContext(text: string): boolean {
     || text.startsWith("<image name=");
 }
 
+function _sha256(jsonlPath: string): string {
+  return createHash("sha256").update(readFileSync(jsonlPath)).digest("hex");
+}
+
 function _readJsonl(jsonlPath: string): any[] {
   return readFileSync(jsonlPath, "utf8")
     .split("\n")

package/src/db.ts

@@ -30,6 +30,14 @@ CREATE TABLE IF NOT EXISTS turns (
 
 CREATE INDEX IF NOT EXISTS idx_turns_session ON turns(session_id, turn_index);
 CREATE INDEX IF NOT EXISTS idx_turns_ts      ON turns(ts DESC);
+
+CREATE TABLE IF NOT EXISTS source_files (
+  jsonl_path TEXT PRIMARY KEY,
+  source     TEXT NOT NULL,
+  size       INTEGER NOT NULL,
+  mtime_ms   REAL NOT NULL,
+  sha256     TEXT NOT NULL
+);
 `;
 
 let _db: DatabaseSync | undefined;
@@ -79,6 +87,34 @@ export interface TurnRow {
   user_message_id: string;
 }
 
+export interface SourceFileRow {
+  jsonl_path: string;
+  source: "pi" | "claude" | "codex";
+  size: number;
+  mtime_ms: number;
+  sha256: string;
+}
+
+export function getSourceFile(jsonlPath: string): SourceFileRow | undefined {
+  return getDb().prepare(`
+    SELECT jsonl_path, source, size, mtime_ms, sha256
+    FROM source_files
+    WHERE jsonl_path = ?
+  `).get(jsonlPath) as SourceFileRow | undefined;
+}
+
+export function upsertSourceFile(row: SourceFileRow): void {
+  getDb().prepare(`
+    INSERT INTO source_files (jsonl_path, source, size, mtime_ms, sha256)
+    VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(jsonl_path) DO UPDATE SET
+      source = excluded.source,
+      size = excluded.size,
+      mtime_ms = excluded.mtime_ms,
+      sha256 = excluded.sha256
+  `).run(row.jsonl_path, row.source, row.size, row.mtime_ms, row.sha256);
+}
+
 export function upsertSession(row: SessionRow): void {
   getDb().prepare(`
     INSERT OR IGNORE INTO sessions (session_id, source, cwd, started_at, model_id, jsonl_path)