pi-hermes-memory 0.7.18 → 0.7.20

package/README.md

@@ -283,7 +283,7 @@ Search behavior notes:
 - Exact phrases can be requested with quotes, for example `"memory search"`.
 - Advanced FTS queries with operators like `OR` still work when you need them.
 
-Session history is indexed automatically on session shutdown. To bulk-import existing sessions:
+Session history is indexed automatically during the active session and on session shutdown. Startup also runs a bounded incremental backfill for missed sessions: it compares stored file metadata and only parses files without matching metadata, capped per startup. To bulk-import existing sessions manually:
 
 ```
 /memory-index-sessions

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-hermes-memory",
-  "version": "0.7.18",
+  "version": "0.7.20",
   "description": "🧠 Persistent memory + 🔍 session search + 🛡️ secret scanning for Pi. Token-aware policy-only memory by default, SQLite FTS5 search, auto-consolidation, procedural skills. 368 tests. Ported from Hermes agent.",
   "type": "module",
   "main": "src/index.ts",

package/src/handlers/session-backfill.ts

@@ -1,12 +1,13 @@
 import type { DatabaseManager } from '../store/db.js';
 import {
-  indexAllSessions,
+  indexChangedSessions,
   needsBackfill,
   touchBackfillTimestamp,
   type BulkIndexResult,
 } from '../store/session-indexer.js';
 
 export const SESSION_BACKFILL_SHUTDOWN_TIMEOUT_MS = 5000;
+export const SESSION_BACKFILL_MAX_FILES = 50;
 
 type NotifyLevel = 'info' | 'warning' | 'error';
 type NotifyFn = (message: string, level: NotifyLevel) => void;
@@ -28,13 +29,15 @@ export interface ScheduleSessionBackfillOptions {
   state?: SessionBackfillState;
   setTimeoutFn?: SetTimeoutFn;
   needsBackfillFn?: typeof needsBackfill;
-  indexAllSessionsFn?: typeof indexAllSessions;
+  indexSessionsFn?: typeof indexChangedSessions;
+  maxFilesToIndex?: number;
   touchBackfillTimestampFn?: typeof touchBackfillTimestamp;
 }
 
 function formatBackfillResult(result: BulkIndexResult): string {
   const errorSuffix = result.errors.length > 0 ? ` (${result.errors.length} file error${result.errors.length === 1 ? '' : 's'})` : '';
-  return `🧠 Session backfill complete: ${result.sessionsIndexed} indexed, ${result.sessionsSkipped} skipped, ${result.messagesIndexed} messages${errorSuffix}.`;
+  const limitSuffix = result.reachedLimit ? ' (startup limit reached)' : '';
+  return `🧠 Session backfill complete: ${result.sessionsIndexed} indexed, ${result.sessionsSkipped} skipped, ${result.messagesIndexed} messages${errorSuffix}${limitSuffix}.`;
 }
 
 function notifyBestEffort(notify: NotifyFn | undefined, message: string, level: NotifyLevel): void {
@@ -46,11 +49,11 @@ function notifyBestEffort(notify: NotifyFn | undefined, message: string, level:
 }
 
 /**
- * Schedule a best-effort, non-blocking backfill of unindexed Pi sessions.
+ * Schedule a best-effort, bounded incremental backfill of unindexed Pi sessions.
  *
- * The expensive indexAllSessions() pass is always deferred with setTimeout(0)
- * so session_start can resolve before disk parsing/indexing begins. A shared
- * state guard prevents concurrent backfills within this extension instance.
+ * The JSONL parsing work is deferred with setTimeout(0) so session_start can
+ * resolve first. The scheduled pass only parses files without matching stored
+ * metadata and caps the number of files parsed per startup.
  *
  * @returns true when a backfill task was scheduled; false when it was skipped.
  */
@@ -62,7 +65,8 @@ export function scheduleSessionBackfill(
   const state = options.state ?? sessionBackfillState;
   const setTimeoutFn = options.setTimeoutFn ?? setTimeout;
   const needsBackfillFn = options.needsBackfillFn ?? needsBackfill;
-  const indexAllSessionsFn = options.indexAllSessionsFn ?? indexAllSessions;
+  const indexSessionsFn = options.indexSessionsFn ?? indexChangedSessions;
+  const maxFilesToIndex = options.maxFilesToIndex ?? SESSION_BACKFILL_MAX_FILES;
   const touchBackfillTimestampFn = options.touchBackfillTimestampFn ?? touchBackfillTimestamp;
 
   if (state.inProgress) {
@@ -86,9 +90,9 @@ export function scheduleSessionBackfill(
   state.promise = new Promise<void>((resolve) => {
     setTimeoutFn(() => {
       try {
-        const result = indexAllSessionsFn(dbManager, sessionsDir);
-        touchBackfillTimestampFn(dbManager);
-        notifyBestEffort(options.notify, formatBackfillResult(result), result.errors.length > 0 ? 'warning' : 'info');
+        const result = indexSessionsFn(dbManager, sessionsDir, { maxFilesToIndex });
+        if (!result.reachedLimit) touchBackfillTimestampFn(dbManager);
+        notifyBestEffort(options.notify, formatBackfillResult(result), result.errors.length > 0 || result.reachedLimit ? 'warning' : 'info');
       } catch (err) {
         notifyBestEffort(
           options.notify,

package/src/handlers/session-live-index.ts

@@ -52,7 +52,9 @@ export function scheduleLiveSessionIndex(
   state.promise = new Promise<void>((resolve) => {
     setTimeoutFn(() => {
       try {
-        indexLiveSessionFn(dbManager, sessionManager);
+        dbManager.withCorruptionRecovery(() => {
+          indexLiveSessionFn(dbManager, sessionManager);
+        });
       } catch (err) {
         try { options.onError?.(err); } catch { /* best effort */ }
       } finally {

package/src/index.ts

@@ -27,7 +27,7 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { MemoryStore } from "./store/memory-store.js";
 import { SkillStore } from "./store/skill-store.js";
 import { DatabaseManager } from "./store/db.js";
-import { indexSession } from "./store/session-indexer.js";
+import { indexSession, upsertSessionFileMetadata } from "./store/session-indexer.js";
 import { scheduleSessionBackfill, waitForSessionBackfill, SESSION_BACKFILL_SHUTDOWN_TIMEOUT_MS } from "./handlers/session-backfill.js";
 import { scheduleLiveSessionIndex, waitForLiveSessionIndex, SESSION_LIVE_INDEX_SHUTDOWN_TIMEOUT_MS } from "./handlers/session-live-index.js";
 import { parseSessionFile } from "./store/session-parser.js";
@@ -246,7 +246,14 @@ export default function (pi: ExtensionAPI) {
       if (sessionFile && require("node:fs").existsSync(sessionFile)) {
         const sessionData = parseSessionFile(sessionFile);
         if (sessionData) {
-          indexSession(dbManager, sessionData);
+          dbManager.withCorruptionRecovery(() => {
+            indexSession(dbManager, sessionData);
+            // Keep session_files metadata in sync with the final on-disk state.
+            // Pi appends the closing session entry on shutdown after the last
+            // message_end, so without this upsert the stored size/mtime would be
+            // stale and the next startup would re-parse this file unnecessarily.
+            upsertSessionFileMetadata(dbManager, sessionFile, sessionData.id);
+          });
         }
       }
     } catch {

package/src/store/db.ts

@@ -7,6 +7,7 @@ type StatementLike = {
   run: (...args: any[]) => any;
   get: (...args: any[]) => any;
   all: (...args: any[]) => any;
+  iterate?: (...args: any[]) => Iterable<Record<string, unknown>>;
 };
 
 type DatabaseLike = {
@@ -25,6 +26,39 @@ type BunDatabaseInstance = {
   transaction?: (fn: any) => any;
 };
 
+type DatabaseFileSuffix = '' | '-wal' | '-shm';
+
+type MovedDatabaseFile = {
+  original: string;
+  backup: string;
+};
+
+export interface DatabaseRecoveryResult {
+  strategy: 'rebuilt' | 'recreated-empty';
+  backupPaths: string[];
+  recoveredRows?: Record<string, number>;
+  error?: string;
+}
+
+class DatabaseCorruptionError extends Error {
+  code = 'SQLITE_CORRUPT';
+
+  constructor(message: string) {
+    super(message);
+    this.name = 'DatabaseCorruptionError';
+  }
+}
+
+export const SQLITE_WAL_AUTOCHECKPOINT_PAGES = 1000;
+
+const DATABASE_FILE_SUFFIXES: readonly DatabaseFileSuffix[] = ['', '-wal', '-shm'];
+const MEMORY_TARGETS = new Set(['memory', 'user', 'failure']);
+const MEMORY_CATEGORIES = new Set(['failure', 'correction', 'insight', 'preference', 'convention', 'tool-quirk']);
+
+function quoteIdentifier(identifier: string): string {
+  return `"${identifier.replace(/"/g, '""')}"`;
+}
+
 function createBunCompatDatabaseCtor(require: NodeRequire): DatabaseCtor {
   const bunSqlite = require('bun:sqlite') as { Database: new (dbPath: string) => BunDatabaseInstance };
 
@@ -77,11 +111,37 @@ const Database = loadDatabaseCtor();
 export class DatabaseManager {
   private db: DatabaseLike | null = null;
   private readonly dbPath: string;
+  private lastRecovery: DatabaseRecoveryResult | null = null;
 
   constructor(memoryDir: string) {
     this.dbPath = path.join(memoryDir, 'sessions.db');
   }
 
+  /**
+   * True when an error indicates SQLite file/page corruption rather than a
+   * normal constraint, migration, or query failure.
+   */
+  static isCorruptionError(err: unknown): boolean {
+    if (!err) return false;
+
+    const code = typeof err === 'object' && 'code' in err ? String((err as { code?: unknown }).code) : '';
+    if (code === 'SQLITE_CORRUPT' || code === 'SQLITE_NOTADB') return true;
+
+    const message = DatabaseManager.errorMessage(err).toLowerCase();
+    return message.includes('database disk image is malformed')
+      || message.includes('file is not a database')
+      || message.includes('database schema is corrupt')
+      || message.includes('malformed database schema')
+      || message.includes('btreeinitpage')
+      || message.includes('sqlite_corrupt')
+      || message.includes('sqlite_notadb');
+  }
+
+  private static errorMessage(err: unknown): string {
+    if (err instanceof Error) return err.message;
+    return String(err);
+  }
+
   /**
    * Get the database instance. Creates/opens on first call.
    */
@@ -92,24 +152,95 @@ export class DatabaseManager {
     return this.db;
   }
 
+  /**
+   * Last self-heal performed by this manager, if any. Exposed for diagnostics
+   * and tests; normal callers do not need it.
+   */
+  getLastRecovery(): DatabaseRecoveryResult | null {
+    return this.lastRecovery;
+  }
+
+  /**
+   * Retry a DB operation once after quarantining/rebuilding a corrupt DB.
+   */
+  withCorruptionRecovery<T>(operation: () => T): T {
+    try {
+      return operation();
+    } catch (err) {
+      if (!DatabaseManager.isCorruptionError(err)) {
+        throw err;
+      }
+      this.recoverFromCorruption(err);
+      return operation();
+    }
+  }
+
+  /**
+   * Close any open handle, rebuild/quarantine the DB file set, and let the next
+   * getDb() reopen a clean database.
+   */
+  recoverFromCorruption(cause?: unknown): DatabaseRecoveryResult {
+    this.close();
+    const recovery = this.recoverDatabaseFile(cause);
+    this.lastRecovery = recovery;
+    return recovery;
+  }
+
   /**
    * Open the database and initialize schema.
    */
   private open(): DatabaseLike {
-    // Ensure directory exists
     const dir = path.dirname(this.dbPath);
     if (!fs.existsSync(dir)) {
       fs.mkdirSync(dir, { recursive: true });
     }
 
+    try {
+      return this.openUnchecked();
+    } catch (err) {
+      if (!DatabaseManager.isCorruptionError(err)) {
+        throw err;
+      }
+
+      const recovery = this.recoverDatabaseFile(err);
+      this.lastRecovery = recovery;
+      return this.openUnchecked();
+    }
+  }
+
+  private openUnchecked(): DatabaseLike {
+    const existed = this.hasExistingMainDatabaseFile();
     const db = new Database(this.dbPath);
+    let ok = false;
 
-    // Enable WAL mode + FK enforcement for each connection.
+    try {
+      if (existed) {
+        this.assertIntegrityOk(db, 'quick_check', 'before schema initialization');
+      }
+
+      this.configureConnection(db);
+      this.initializeSchema(db);
+      this.assertIntegrityOk(db, 'quick_check', 'after schema initialization');
+      ok = true;
+      return db;
+    } finally {
+      if (!ok) {
+        this.safeClose(db);
+      }
+    }
+  }
+
+  private configureConnection(db: DatabaseLike): void {
+    // Enable WAL mode + FK enforcement for each connection. Keep SQLite's
+    // default WAL autocheckpoint size; a very aggressive checkpoint cadence
+    // increases the chance that abrupt VM/host shutdown catches a checkpoint.
     db.exec('PRAGMA journal_mode = WAL');
-    db.exec('PRAGMA wal_autocheckpoint = 100');
+    db.exec(`PRAGMA wal_autocheckpoint = ${SQLITE_WAL_AUTOCHECKPOINT_PAGES}`);
     db.exec('PRAGMA journal_size_limit = 5242880');
     db.exec('PRAGMA foreign_keys = ON');
+  }
 
+  private initializeSchema(db: DatabaseLike): void {
     // Create tables and triggers
     try {
       db.exec(SCHEMA_SQL);
@@ -129,8 +260,334 @@ export class DatabaseManager {
     this.ensureMemoriesColumns(db);
     this.migrateLegacyMemoriesTargetConstraint(db);
     this.rebuildMemoryFts(db);
+  }
+
+  private hasExistingMainDatabaseFile(): boolean {
+    try {
+      return fs.existsSync(this.dbPath) && fs.statSync(this.dbPath).size > 0;
+    } catch {
+      return false;
+    }
+  }
+
+  private databaseFileSetExists(): boolean {
+    return DATABASE_FILE_SUFFIXES.some((suffix) => fs.existsSync(`${this.dbPath}${suffix}`));
+  }
+
+  private assertIntegrityOk(
+    db: DatabaseLike,
+    check: 'quick_check' | 'integrity_check' = 'quick_check',
+    context = '',
+  ): void {
+    const rows = db.prepare(`PRAGMA ${check}`).all() as Record<string, unknown>[];
+    const messages = rows.map((row) => String(Object.values(row)[0] ?? ''));
+    const failures = messages.filter((message) => message.toLowerCase() !== 'ok');
+
+    if (rows.length === 0 || failures.length > 0) {
+      const detail = failures.length > 0 ? failures.slice(0, 5).join('\n') : 'no result rows';
+      const suffix = context ? ` ${context}` : '';
+      throw new DatabaseCorruptionError(`SQLite ${check} failed${suffix}: ${detail}`);
+    }
+  }
+
+  private assertForeignKeysOk(db: DatabaseLike): void {
+    const rows = db.prepare('PRAGMA foreign_key_check').all() as Record<string, unknown>[];
+    if (rows.length > 0) {
+      throw new Error(`SQLite foreign_key_check failed after rebuild (${rows.length} violation${rows.length === 1 ? '' : 's'})`);
+    }
+  }
+
+  private recoverDatabaseFile(cause?: unknown): DatabaseRecoveryResult {
+    const backupBase = this.corruptBackupBase();
+    let rebuildError: unknown;
+
+    if (this.databaseFileSetExists()) {
+      try {
+        return this.rebuildDatabaseFromReadableRows(backupBase);
+      } catch (err) {
+        rebuildError = err;
+      }
+    }
+
+    const moved = this.moveDatabaseFilesToBackup(backupBase);
+    return {
+      strategy: 'recreated-empty',
+      backupPaths: moved.map((file) => file.backup),
+      error: DatabaseManager.errorMessage(rebuildError ?? cause ?? 'unknown corruption'),
+    };
+  }
+
+  private rebuildDatabaseFromReadableRows(backupBase: string): DatabaseRecoveryResult {
+    const tempPath = this.rebuildTempPath();
+    this.removeDatabaseFileSet(tempPath);
+
+    let source: DatabaseLike | null = null;
+    let target: DatabaseLike | null = null;
+    let recoveredRows: Record<string, number> | undefined;
+    let rebuildOk = false;
+
+    try {
+      source = new Database(this.dbPath);
+      target = new Database(tempPath);
+      target.exec('PRAGMA journal_mode = DELETE');
+      target.exec('PRAGMA foreign_keys = OFF');
+      target.exec(SCHEMA_SQL);
+
+      recoveredRows = this.copyRecoverableRows(source, target);
+      this.rebuildFtsTables(target);
+      this.assertForeignKeysOk(target);
+      this.assertIntegrityOk(target, 'quick_check', 'after corruption rebuild');
+      rebuildOk = true;
+    } finally {
+      if (source) this.safeClose(source);
+      if (target) this.safeClose(target);
+      if (!rebuildOk) this.removeDatabaseFileSet(tempPath);
+    }
+
+    const moved = this.swapRebuiltDatabase(tempPath, backupBase);
+    this.removeDatabaseFileSet(tempPath);
+
+    return {
+      strategy: 'rebuilt',
+      backupPaths: moved.map((file) => file.backup),
+      recoveredRows,
+    };
+  }
+
+  private copyRecoverableRows(source: DatabaseLike, target: DatabaseLike): Record<string, number> {
+    return {
+      extension_metadata: this.copyExtensionMetadata(source, target),
+      sessions: this.copySessions(source, target),
+      messages: this.copyMessages(source, target),
+      session_files: this.copySessionFiles(source, target),
+      memories: this.copyMemories(source, target),
+    };
+  }
+
+  private copyExtensionMetadata(source: DatabaseLike, target: DatabaseLike): number {
+    const insert = target.prepare('INSERT OR REPLACE INTO extension_metadata (key, value) VALUES (?, ?)');
+    let copied = 0;
+
+    for (const row of this.readTableRows(source, 'extension_metadata', ['key', 'value'])) {
+      if (typeof row.key !== 'string' || typeof row.value !== 'string') continue;
+      insert.run(row.key, row.value);
+      copied++;
+    }
 
-    return db;
+    return copied;
+  }
+
+  private copySessions(source: DatabaseLike, target: DatabaseLike): number {
+    const insert = target.prepare(`
+      INSERT OR IGNORE INTO sessions (id, project, cwd, started_at, ended_at, message_count)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `);
+    let copied = 0;
+
+    for (const row of this.readTableRows(source, 'sessions', ['id', 'project', 'cwd', 'started_at', 'ended_at', 'message_count'])) {
+      if (typeof row.id !== 'string' || typeof row.cwd !== 'string' || typeof row.started_at !== 'string') continue;
+      const project = typeof row.project === 'string' && row.project ? row.project : (path.basename(row.cwd) || 'unknown');
+      insert.run(
+        row.id,
+        project,
+        row.cwd,
+        row.started_at,
+        this.nullableString(row.ended_at),
+        this.integerOr(row.message_count, 0),
+      );
+      copied++;
+    }
+
+    return copied;
+  }
+
+  private copyMessages(source: DatabaseLike, target: DatabaseLike): number {
+    const insert = target.prepare(`
+      INSERT OR IGNORE INTO messages (id, session_id, role, content, timestamp, tool_calls)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `);
+    let copied = 0;
+
+    for (const row of this.readTableRows(source, 'messages', ['id', 'session_id', 'role', 'content', 'timestamp', 'tool_calls'])) {
+      if (
+        typeof row.id !== 'string'
+        || typeof row.session_id !== 'string'
+        || (row.role !== 'user' && row.role !== 'assistant' && row.role !== 'system')
+        || typeof row.content !== 'string'
+        || typeof row.timestamp !== 'string'
+      ) {
+        continue;
+      }
+
+      insert.run(row.id, row.session_id, row.role, row.content, row.timestamp, this.nullableString(row.tool_calls));
+      copied++;
+    }
+
+    return copied;
+  }
+
+  private copySessionFiles(source: DatabaseLike, target: DatabaseLike): number {
+    const insert = target.prepare(`
+      INSERT OR IGNORE INTO session_files (path, session_id, size, mtime_ms, indexed_at)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+    let copied = 0;
+
+    for (const row of this.readTableRows(source, 'session_files', ['path', 'session_id', 'size', 'mtime_ms', 'indexed_at'])) {
+      if (typeof row.path !== 'string' || typeof row.session_id !== 'string') continue;
+      insert.run(
+        row.path,
+        row.session_id,
+        this.integerOr(row.size, 0),
+        this.integerOr(row.mtime_ms, 0),
+        typeof row.indexed_at === 'string' ? row.indexed_at : new Date(0).toISOString(),
+      );
+      copied++;
+    }
+
+    return copied;
+  }
+
+  private copyMemories(source: DatabaseLike, target: DatabaseLike): number {
+    const insert = target.prepare(`
+      INSERT OR IGNORE INTO memories (id, project, target, category, content, failure_reason, tool_state, corrected_to, created, last_referenced)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    let copied = 0;
+
+    for (const row of this.readTableRows(source, 'memories', [
+      'id',
+      'project',
+      'target',
+      'category',
+      'content',
+      'failure_reason',
+      'tool_state',
+      'corrected_to',
+      'created',
+      'last_referenced',
+    ])) {
+      const id = this.integerOr(row.id, NaN);
+      if (!Number.isFinite(id) || typeof row.content !== 'string') continue;
+
+      const targetName = typeof row.target === 'string' && MEMORY_TARGETS.has(row.target) ? row.target : 'memory';
+      const category = typeof row.category === 'string' && MEMORY_CATEGORIES.has(row.category) ? row.category : null;
+      const created = typeof row.created === 'string' ? row.created : new Date(0).toISOString();
+      const lastReferenced = typeof row.last_referenced === 'string' ? row.last_referenced : created;
+
+      insert.run(
+        id,
+        this.nullableString(row.project),
+        targetName,
+        category,
+        row.content,
+        this.nullableString(row.failure_reason),
+        this.nullableString(row.tool_state),
+        this.nullableString(row.corrected_to),
+        created,
+        lastReferenced,
+      );
+      copied++;
+    }
+
+    return copied;
+  }
+
+  private readTableRows(source: DatabaseLike, table: string, desiredColumns: string[]): Iterable<Record<string, unknown>> {
+    const columns = this.getColumnNames(source, table);
+    const selected = desiredColumns.filter((column) => columns.has(column));
+    if (selected.length === 0) return [];
+
+    const sql = `SELECT ${selected.map(quoteIdentifier).join(', ')} FROM ${quoteIdentifier(table)} NOT INDEXED`;
+    const statement = source.prepare(sql);
+    if (statement.iterate) {
+      return statement.iterate() as Iterable<Record<string, unknown>>;
+    }
+    return statement.all() as Record<string, unknown>[];
+  }
+
+  private getColumnNames(db: DatabaseLike, table: string): Set<string> {
+    const rows = db.prepare(`PRAGMA table_info(${quoteIdentifier(table)})`).all() as { name?: unknown }[];
+    return new Set(rows.map((row) => row.name).filter((name): name is string => typeof name === 'string'));
+  }
+
+  private nullableString(value: unknown): string | null {
+    return typeof value === 'string' ? value : null;
+  }
+
+  private integerOr(value: unknown, fallback: number): number {
+    if (typeof value === 'number' && Number.isFinite(value)) return Math.trunc(value);
+    if (typeof value === 'bigint') return Number(value);
+    if (typeof value === 'string' && value.trim()) {
+      const parsed = Number.parseInt(value, 10);
+      if (Number.isFinite(parsed)) return parsed;
+    }
+    return fallback;
+  }
+
+  private rebuildFtsTables(db: DatabaseLike): void {
+    db.exec("INSERT INTO message_fts(message_fts) VALUES('rebuild')");
+    db.exec("INSERT INTO memory_fts(memory_fts) VALUES('rebuild')");
+  }
+
+  private corruptBackupBase(): string {
+    const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const nonce = Math.random().toString(16).slice(2, 8);
+    return `${this.dbPath}.corrupt-${stamp}-${process.pid}-${nonce}`;
+  }
+
+  private rebuildTempPath(): string {
+    const stamp = Date.now();
+    const nonce = Math.random().toString(16).slice(2, 8);
+    return `${this.dbPath}.rebuild-${process.pid}-${stamp}-${nonce}.tmp`;
+  }
+
+  private swapRebuiltDatabase(tempPath: string, backupBase: string): MovedDatabaseFile[] {
+    const moved = this.moveDatabaseFilesToBackup(backupBase);
+    try {
+      fs.renameSync(tempPath, this.dbPath);
+      return moved;
+    } catch (err) {
+      this.restoreMovedDatabaseFiles(moved);
+      this.removeDatabaseFileSet(tempPath);
+      throw err;
+    }
+  }
+
+  private moveDatabaseFilesToBackup(backupBase: string): MovedDatabaseFile[] {
+    const moved: MovedDatabaseFile[] = [];
+    for (const suffix of DATABASE_FILE_SUFFIXES) {
+      const original = `${this.dbPath}${suffix}`;
+      if (!fs.existsSync(original)) continue;
+
+      const backup = `${backupBase}${suffix}`;
+      fs.rmSync(backup, { force: true });
+      fs.renameSync(original, backup);
+      moved.push({ original, backup });
+    }
+    return moved;
+  }
+
+  private restoreMovedDatabaseFiles(moved: MovedDatabaseFile[]): void {
+    for (const file of [...moved].reverse()) {
+      try {
+        if (!fs.existsSync(file.backup)) continue;
+        fs.rmSync(file.original, { force: true });
+        fs.renameSync(file.backup, file.original);
+      } catch {
+        // Best effort. The backup path remains available if restoration fails.
+      }
+    }
+  }
+
+  private removeDatabaseFileSet(basePath: string): void {
+    for (const suffix of DATABASE_FILE_SUFFIXES) {
+      fs.rmSync(`${basePath}${suffix}`, { force: true });
+    }
+  }
+
+  private safeClose(db: DatabaseLike): void {
+    try { db.close(); } catch { /* best effort */ }
   }
 
   private isLegacyMemoriesCategoryError(err: unknown): boolean {
@@ -254,7 +711,7 @@ export class DatabaseManager {
   close(): void {
     if (this.db) {
       try { this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); } catch { /* best effort */ }
-      this.db.close();
+      try { this.db.close(); } catch { /* best effort — close may throw on a corrupt handle */ }
       this.db = null;
     }
   }

package/src/store/schema.ts

@@ -3,6 +3,7 @@
  *
  * Tables:
  * - sessions — Pi session metadata
+ * - session_files — indexed JSONL metadata for incremental backfill
  * - messages — all conversation messages
  * - message_fts — FTS5 index for full-text search across messages
  * - memories — extended memory entries (unlimited, searchable)
@@ -26,6 +27,15 @@ export const SCHEMA_SQL = `
     message_count INTEGER DEFAULT 0
   );
 
+  -- Indexed session file metadata for cheap incremental backfill
+  CREATE TABLE IF NOT EXISTS session_files (
+    path TEXT PRIMARY KEY,
+    session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
+    size INTEGER NOT NULL,
+    mtime_ms INTEGER NOT NULL,
+    indexed_at TEXT NOT NULL
+  );
+
   -- All messages from all sessions
   CREATE TABLE IF NOT EXISTS messages (
     id TEXT PRIMARY KEY,
@@ -102,4 +112,5 @@ export const SCHEMA_SQL = `
   CREATE INDEX IF NOT EXISTS idx_memories_category ON memories(category);
   CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project);
   CREATE INDEX IF NOT EXISTS idx_sessions_started_at ON sessions(started_at);
+  CREATE INDEX IF NOT EXISTS idx_session_files_session_id ON session_files(session_id);
 `;

package/src/store/session-indexer.ts

@@ -23,6 +23,18 @@ export interface BulkIndexResult {
   sessionsSkipped: number;
   messagesIndexed: number;
   errors: string[];
+  reachedLimit?: boolean;
+}
+
+interface SessionFileMetadata {
+  path: string;
+  size: number;
+  mtimeMs: number;
+}
+
+export interface IncrementalIndexOptions {
+  projectDir?: string;
+  maxFilesToIndex?: number;
 }
 
 /**
@@ -31,6 +43,10 @@ export interface BulkIndexResult {
  * @returns IndexResult with count of messages indexed
  */
 export function indexSession(dbManager: DatabaseManager, session: ParsedSession): IndexResult {
+  return dbManager.withCorruptionRecovery(() => indexSessionOnce(dbManager, session));
+}
+
+function indexSessionOnce(dbManager: DatabaseManager, session: ParsedSession): IndexResult {
   const db = dbManager.getDb();
 
   const existingSession = db.prepare('SELECT id FROM sessions WHERE id = ?').get(session.id) as { id: string } | undefined;
@@ -198,15 +214,85 @@ export function indexCurrentSession(dbManager: DatabaseManager, sessionManager:
 }
 
 export function indexLiveSession(dbManager: DatabaseManager, sessionManager: SessionManagerSnapshot): IndexResult | null {
+  return dbManager.withCorruptionRecovery(() => indexLiveSessionOnce(dbManager, sessionManager));
+}
+
+function indexLiveSessionOnce(dbManager: DatabaseManager, sessionManager: SessionManagerSnapshot): IndexResult | null {
   const sessionFile = sessionManager.getSessionFile?.();
   if (sessionFile && fs.existsSync(sessionFile)) {
     const session = parseSessionFile(sessionFile);
-    if (session) return indexSession(dbManager, session);
+    if (session) {
+      const result = indexSession(dbManager, session);
+      upsertSessionFileMetadata(dbManager, sessionFile, session.id);
+      return result;
+    }
   }
 
   return indexCurrentSession(dbManager, sessionManager);
 }
 
+function getSessionFileMetadata(filePath: string): SessionFileMetadata {
+  const stat = fs.statSync(filePath);
+  return { path: filePath, size: stat.size, mtimeMs: Math.trunc(stat.mtimeMs) };
+}
+
+function getStoredSessionFileMetadata(dbManager: DatabaseManager, filePath: string): { size: number; mtime_ms: number } | undefined {
+  return dbManager.getDb().prepare('SELECT size, mtime_ms FROM session_files WHERE path = ?').get(filePath) as { size: number; mtime_ms: number } | undefined;
+}
+
+function storedSessionFileMatches(dbManager: DatabaseManager, metadata: SessionFileMetadata): boolean {
+  const row = getStoredSessionFileMetadata(dbManager, metadata.path);
+  return Boolean(row && row.size === metadata.size && row.mtime_ms === metadata.mtimeMs);
+}
+
+export function upsertSessionFileMetadata(
+  dbManager: DatabaseManager,
+  filePath: string,
+  sessionId: string,
+  metadata = getSessionFileMetadata(filePath),
+  indexedAt = new Date(),
+): void {
+  const db = dbManager.getDb();
+  db.prepare(`
+    INSERT INTO session_files (path, session_id, size, mtime_ms, indexed_at)
+    VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(path) DO UPDATE SET
+      session_id = excluded.session_id,
+      size = excluded.size,
+      mtime_ms = excluded.mtime_ms,
+      indexed_at = excluded.indexed_at
+  `).run(metadata.path, sessionId, metadata.size, metadata.mtimeMs, indexedAt.toISOString());
+}
+
+function emptyBulkIndexResult(): BulkIndexResult {
+  return {
+    sessionsProcessed: 0,
+    sessionsIndexed: 0,
+    sessionsSkipped: 0,
+    messagesIndexed: 0,
+    errors: [],
+  };
+}
+
+function indexSessionFile(dbManager: DatabaseManager, file: string, result: BulkIndexResult): void {
+  result.sessionsProcessed++;
+
+  const session = parseSessionFile(file);
+  if (!session) {
+    result.errors.push(`Failed to parse: ${file}`);
+    return;
+  }
+
+  const indexResult = indexSession(dbManager, session);
+  upsertSessionFileMetadata(dbManager, file, session.id);
+  if (indexResult.skipped) {
+    result.sessionsSkipped++;
+  } else {
+    result.sessionsIndexed++;
+    result.messagesIndexed += indexResult.messagesIndexed;
+  }
+}
+
 /**
  * Index all sessions from disk.
  *
@@ -221,36 +307,69 @@ export function indexAllSessions(
   projectDir?: string
 ): BulkIndexResult {
   const files = getSessionFiles(sessionsDir, projectDir);
-  const result: BulkIndexResult = {
-    sessionsProcessed: 0,
-    sessionsIndexed: 0,
-    sessionsSkipped: 0,
-    messagesIndexed: 0,
-    errors: [],
-  };
+  const result = emptyBulkIndexResult();
 
   for (const file of files) {
-    result.sessionsProcessed++;
-
     try {
-      const session = parseSessionFile(file);
-      if (!session) {
-        result.errors.push(`Failed to parse: ${file}`);
-        continue;
-      }
+      indexSessionFile(dbManager, file, result);
+    } catch (err) {
+      result.errors.push(`Error indexing ${file}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
 
-      const indexResult = indexSession(dbManager, session);
-      if (indexResult.skipped) {
+  return result;
+}
+
+/**
+ * Incrementally index session JSONL files without matching stored metadata.
+ *
+ * This is intentionally cheaper than indexAllSessions() for startup backfill:
+ * files with matching stored size/mtime metadata are skipped, and all other
+ * files are parsed under the startup cap.
+ */
+export function indexChangedSessions(
+  dbManager: DatabaseManager,
+  sessionsDir: string,
+  options: IncrementalIndexOptions = {},
+): BulkIndexResult {
+  const files = getSessionFiles(sessionsDir, options.projectDir);
+  const maxFilesToIndex = options.maxFilesToIndex ?? 50;
+  const result = emptyBulkIndexResult();
+
+  // Gather the changed set first, then sort newest-first before applying the
+  // cap. Crash recovery is the primary value of startup backfill (the live
+  // message_end path missed the session's final state), and crashed sessions
+  // are the most recently modified files. Sorting newest-first ensures they
+  // are indexed on the very next startup instead of waiting behind old
+  // historical files that fill the per-startup cap in filesystem order.
+  const changed: SessionFileMetadata[] = [];
+  for (const file of files) {
+    try {
+      const metadata = getSessionFileMetadata(file);
+      if (storedSessionFileMatches(dbManager, metadata)) {
         result.sessionsSkipped++;
-      } else {
-        result.sessionsIndexed++;
-        result.messagesIndexed += indexResult.messagesIndexed;
+        continue;
       }
+      changed.push(metadata);
     } catch (err) {
       result.errors.push(`Error indexing ${file}: ${err instanceof Error ? err.message : String(err)}`);
     }
   }
 
+  changed.sort((a, b) => b.mtimeMs - a.mtimeMs);
+
+  for (const metadata of changed) {
+    if (result.sessionsProcessed >= maxFilesToIndex) {
+      result.reachedLimit = true;
+      break;
+    }
+    try {
+      indexSessionFile(dbManager, metadata.path, result);
+    } catch (err) {
+      result.errors.push(`Error indexing ${metadata.path}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
   return result;
 }
 
@@ -277,20 +396,28 @@ function isRecentBackfillTimestamp(value: string | null, nowMs: number): boolean
 /**
  * Determine whether a background session backfill should run.
  *
- * A backfill is needed when the number of JSONL session files differs from
- * the indexed session count, or when no successful backfill has completed in
- * the last 24 hours. The count check catches crashed/abnormal sessions; the
- * timestamp check periodically repairs parse errors or manual DB edits.
+ * The check stays cheap: it compares file counts and stored file size/mtime
+ * metadata. Full JSONL parsing is left to the scheduled incremental backfill.
  */
 export function needsBackfill(dbManager: DatabaseManager, sessionsDir: string, now = new Date()): boolean {
   const db = dbManager.getDb();
-  const fileCount = countSessionFiles(sessionsDir);
+  const files = getSessionFiles(sessionsDir);
   const indexed = db.prepare('SELECT COUNT(*) as count FROM sessions').get() as { count: number };
 
-  if (fileCount !== indexed.count) {
+  if (files.length > indexed.count) {
     return true;
   }
 
+  for (const file of files) {
+    try {
+      const metadata = getSessionFileMetadata(file);
+      if (storedSessionFileMatches(dbManager, metadata)) continue;
+      return true;
+    } catch {
+      return true;
+    }
+  }
+
   return !isRecentBackfillTimestamp(getLastBackfillTimestamp(dbManager), now.getTime());
 }