pi-hermes-memory 0.7.18 → 0.7.19

package/README.md

@@ -283,7 +283,7 @@ Search behavior notes:
 - Exact phrases can be requested with quotes, for example `"memory search"`.
 - Advanced FTS queries with operators like `OR` still work when you need them.
 
-Session history is indexed automatically on session shutdown. To bulk-import existing sessions:
+Session history is indexed automatically during the active session and on session shutdown. Startup also runs a bounded incremental backfill for missed sessions: it compares stored file metadata and only parses files without matching metadata, capped per startup. To bulk-import existing sessions manually:
 
 ```
 /memory-index-sessions

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-hermes-memory",
-  "version": "0.7.18",
+  "version": "0.7.19",
   "description": "🧠 Persistent memory + 🔍 session search + 🛡️ secret scanning for Pi. Token-aware policy-only memory by default, SQLite FTS5 search, auto-consolidation, procedural skills. 368 tests. Ported from Hermes agent.",
   "type": "module",
   "main": "src/index.ts",

package/src/handlers/session-backfill.ts

@@ -1,12 +1,13 @@
 import type { DatabaseManager } from '../store/db.js';
 import {
-  indexAllSessions,
+  indexChangedSessions,
   needsBackfill,
   touchBackfillTimestamp,
   type BulkIndexResult,
 } from '../store/session-indexer.js';
 
 export const SESSION_BACKFILL_SHUTDOWN_TIMEOUT_MS = 5000;
+export const SESSION_BACKFILL_MAX_FILES = 50;
 
 type NotifyLevel = 'info' | 'warning' | 'error';
 type NotifyFn = (message: string, level: NotifyLevel) => void;
@@ -28,13 +29,15 @@ export interface ScheduleSessionBackfillOptions {
   state?: SessionBackfillState;
   setTimeoutFn?: SetTimeoutFn;
   needsBackfillFn?: typeof needsBackfill;
-  indexAllSessionsFn?: typeof indexAllSessions;
+  indexSessionsFn?: typeof indexChangedSessions;
+  maxFilesToIndex?: number;
   touchBackfillTimestampFn?: typeof touchBackfillTimestamp;
 }
 
 function formatBackfillResult(result: BulkIndexResult): string {
   const errorSuffix = result.errors.length > 0 ? ` (${result.errors.length} file error${result.errors.length === 1 ? '' : 's'})` : '';
-  return `🧠 Session backfill complete: ${result.sessionsIndexed} indexed, ${result.sessionsSkipped} skipped, ${result.messagesIndexed} messages${errorSuffix}.`;
+  const limitSuffix = result.reachedLimit ? ' (startup limit reached)' : '';
+  return `🧠 Session backfill complete: ${result.sessionsIndexed} indexed, ${result.sessionsSkipped} skipped, ${result.messagesIndexed} messages${errorSuffix}${limitSuffix}.`;
 }
 
 function notifyBestEffort(notify: NotifyFn | undefined, message: string, level: NotifyLevel): void {
@@ -46,11 +49,11 @@ function notifyBestEffort(notify: NotifyFn | undefined, message: string, level:
 }
 
 /**
- * Schedule a best-effort, non-blocking backfill of unindexed Pi sessions.
+ * Schedule a best-effort, bounded incremental backfill of unindexed Pi sessions.
  *
- * The expensive indexAllSessions() pass is always deferred with setTimeout(0)
- * so session_start can resolve before disk parsing/indexing begins. A shared
- * state guard prevents concurrent backfills within this extension instance.
+ * The JSONL parsing work is deferred with setTimeout(0) so session_start can
+ * resolve first. The scheduled pass only parses files without matching stored
+ * metadata and caps the number of files parsed per startup.
  *
  * @returns true when a backfill task was scheduled; false when it was skipped.
  */
@@ -62,7 +65,8 @@ export function scheduleSessionBackfill(
   const state = options.state ?? sessionBackfillState;
   const setTimeoutFn = options.setTimeoutFn ?? setTimeout;
   const needsBackfillFn = options.needsBackfillFn ?? needsBackfill;
-  const indexAllSessionsFn = options.indexAllSessionsFn ?? indexAllSessions;
+  const indexSessionsFn = options.indexSessionsFn ?? indexChangedSessions;
+  const maxFilesToIndex = options.maxFilesToIndex ?? SESSION_BACKFILL_MAX_FILES;
   const touchBackfillTimestampFn = options.touchBackfillTimestampFn ?? touchBackfillTimestamp;
 
   if (state.inProgress) {
@@ -86,9 +90,9 @@ export function scheduleSessionBackfill(
   state.promise = new Promise<void>((resolve) => {
     setTimeoutFn(() => {
       try {
-        const result = indexAllSessionsFn(dbManager, sessionsDir);
-        touchBackfillTimestampFn(dbManager);
-        notifyBestEffort(options.notify, formatBackfillResult(result), result.errors.length > 0 ? 'warning' : 'info');
+        const result = indexSessionsFn(dbManager, sessionsDir, { maxFilesToIndex });
+        if (!result.reachedLimit) touchBackfillTimestampFn(dbManager);
+        notifyBestEffort(options.notify, formatBackfillResult(result), result.errors.length > 0 || result.reachedLimit ? 'warning' : 'info');
       } catch (err) {
         notifyBestEffort(
           options.notify,

package/src/index.ts

@@ -27,7 +27,7 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { MemoryStore } from "./store/memory-store.js";
 import { SkillStore } from "./store/skill-store.js";
 import { DatabaseManager } from "./store/db.js";
-import { indexSession } from "./store/session-indexer.js";
+import { indexSession, upsertSessionFileMetadata } from "./store/session-indexer.js";
 import { scheduleSessionBackfill, waitForSessionBackfill, SESSION_BACKFILL_SHUTDOWN_TIMEOUT_MS } from "./handlers/session-backfill.js";
 import { scheduleLiveSessionIndex, waitForLiveSessionIndex, SESSION_LIVE_INDEX_SHUTDOWN_TIMEOUT_MS } from "./handlers/session-live-index.js";
 import { parseSessionFile } from "./store/session-parser.js";
@@ -247,6 +247,11 @@ export default function (pi: ExtensionAPI) {
         const sessionData = parseSessionFile(sessionFile);
         if (sessionData) {
           indexSession(dbManager, sessionData);
+          // Keep session_files metadata in sync with the final on-disk state.
+          // Pi appends the closing session entry on shutdown after the last
+          // message_end, so without this upsert the stored size/mtime would be
+          // stale and the next startup would re-parse this file unnecessarily.
+          upsertSessionFileMetadata(dbManager, sessionFile, sessionData.id);
         }
       }
     } catch {

package/src/store/schema.ts

@@ -3,6 +3,7 @@
  *
  * Tables:
  * - sessions — Pi session metadata
+ * - session_files — indexed JSONL metadata for incremental backfill
  * - messages — all conversation messages
  * - message_fts — FTS5 index for full-text search across messages
  * - memories — extended memory entries (unlimited, searchable)
@@ -26,6 +27,15 @@ export const SCHEMA_SQL = `
     message_count INTEGER DEFAULT 0
   );
 
+  -- Indexed session file metadata for cheap incremental backfill
+  CREATE TABLE IF NOT EXISTS session_files (
+    path TEXT PRIMARY KEY,
+    session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
+    size INTEGER NOT NULL,
+    mtime_ms INTEGER NOT NULL,
+    indexed_at TEXT NOT NULL
+  );
+
   -- All messages from all sessions
   CREATE TABLE IF NOT EXISTS messages (
     id TEXT PRIMARY KEY,
@@ -102,4 +112,5 @@ export const SCHEMA_SQL = `
   CREATE INDEX IF NOT EXISTS idx_memories_category ON memories(category);
   CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project);
   CREATE INDEX IF NOT EXISTS idx_sessions_started_at ON sessions(started_at);
+  CREATE INDEX IF NOT EXISTS idx_session_files_session_id ON session_files(session_id);
 `;

package/src/store/session-indexer.ts

@@ -23,6 +23,18 @@ export interface BulkIndexResult {
   sessionsSkipped: number;
   messagesIndexed: number;
   errors: string[];
+  reachedLimit?: boolean;
+}
+
+interface SessionFileMetadata {
+  path: string;
+  size: number;
+  mtimeMs: number;
+}
+
+export interface IncrementalIndexOptions {
+  projectDir?: string;
+  maxFilesToIndex?: number;
 }
 
 /**
@@ -201,12 +213,78 @@ export function indexLiveSession(dbManager: DatabaseManager, sessionManager: Ses
   const sessionFile = sessionManager.getSessionFile?.();
   if (sessionFile && fs.existsSync(sessionFile)) {
     const session = parseSessionFile(sessionFile);
-    if (session) return indexSession(dbManager, session);
+    if (session) {
+      const result = indexSession(dbManager, session);
+      upsertSessionFileMetadata(dbManager, sessionFile, session.id);
+      return result;
+    }
   }
 
   return indexCurrentSession(dbManager, sessionManager);
 }
 
+function getSessionFileMetadata(filePath: string): SessionFileMetadata {
+  const stat = fs.statSync(filePath);
+  return { path: filePath, size: stat.size, mtimeMs: Math.trunc(stat.mtimeMs) };
+}
+
+function getStoredSessionFileMetadata(dbManager: DatabaseManager, filePath: string): { size: number; mtime_ms: number } | undefined {
+  return dbManager.getDb().prepare('SELECT size, mtime_ms FROM session_files WHERE path = ?').get(filePath) as { size: number; mtime_ms: number } | undefined;
+}
+
+function storedSessionFileMatches(dbManager: DatabaseManager, metadata: SessionFileMetadata): boolean {
+  const row = getStoredSessionFileMetadata(dbManager, metadata.path);
+  return Boolean(row && row.size === metadata.size && row.mtime_ms === metadata.mtimeMs);
+}
+
+export function upsertSessionFileMetadata(
+  dbManager: DatabaseManager,
+  filePath: string,
+  sessionId: string,
+  metadata = getSessionFileMetadata(filePath),
+  indexedAt = new Date(),
+): void {
+  const db = dbManager.getDb();
+  db.prepare(`
+    INSERT INTO session_files (path, session_id, size, mtime_ms, indexed_at)
+    VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(path) DO UPDATE SET
+      session_id = excluded.session_id,
+      size = excluded.size,
+      mtime_ms = excluded.mtime_ms,
+      indexed_at = excluded.indexed_at
+  `).run(metadata.path, sessionId, metadata.size, metadata.mtimeMs, indexedAt.toISOString());
+}
+
+function emptyBulkIndexResult(): BulkIndexResult {
+  return {
+    sessionsProcessed: 0,
+    sessionsIndexed: 0,
+    sessionsSkipped: 0,
+    messagesIndexed: 0,
+    errors: [],
+  };
+}
+
+function indexSessionFile(dbManager: DatabaseManager, file: string, result: BulkIndexResult): void {
+  result.sessionsProcessed++;
+
+  const session = parseSessionFile(file);
+  if (!session) {
+    result.errors.push(`Failed to parse: ${file}`);
+    return;
+  }
+
+  const indexResult = indexSession(dbManager, session);
+  upsertSessionFileMetadata(dbManager, file, session.id);
+  if (indexResult.skipped) {
+    result.sessionsSkipped++;
+  } else {
+    result.sessionsIndexed++;
+    result.messagesIndexed += indexResult.messagesIndexed;
+  }
+}
+
 /**
  * Index all sessions from disk.
  *
@@ -221,36 +299,69 @@ export function indexAllSessions(
   projectDir?: string
 ): BulkIndexResult {
   const files = getSessionFiles(sessionsDir, projectDir);
-  const result: BulkIndexResult = {
-    sessionsProcessed: 0,
-    sessionsIndexed: 0,
-    sessionsSkipped: 0,
-    messagesIndexed: 0,
-    errors: [],
-  };
+  const result = emptyBulkIndexResult();
 
   for (const file of files) {
-    result.sessionsProcessed++;
-
     try {
-      const session = parseSessionFile(file);
-      if (!session) {
-        result.errors.push(`Failed to parse: ${file}`);
-        continue;
-      }
+      indexSessionFile(dbManager, file, result);
+    } catch (err) {
+      result.errors.push(`Error indexing ${file}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  return result;
+}
 
-      const indexResult = indexSession(dbManager, session);
-      if (indexResult.skipped) {
+/**
+ * Incrementally index session JSONL files without matching stored metadata.
+ *
+ * This is intentionally cheaper than indexAllSessions() for startup backfill:
+ * files with matching stored size/mtime metadata are skipped, and all other
+ * files are parsed under the startup cap.
+ */
+export function indexChangedSessions(
+  dbManager: DatabaseManager,
+  sessionsDir: string,
+  options: IncrementalIndexOptions = {},
+): BulkIndexResult {
+  const files = getSessionFiles(sessionsDir, options.projectDir);
+  const maxFilesToIndex = options.maxFilesToIndex ?? 50;
+  const result = emptyBulkIndexResult();
+
+  // Gather the changed set first, then sort newest-first before applying the
+  // cap. Crash recovery is the primary value of startup backfill (the live
+  // message_end path missed the session's final state), and crashed sessions
+  // are the most recently modified files. Sorting newest-first ensures they
+  // are indexed on the very next startup instead of waiting behind old
+  // historical files that fill the per-startup cap in filesystem order.
+  const changed: SessionFileMetadata[] = [];
+  for (const file of files) {
+    try {
+      const metadata = getSessionFileMetadata(file);
+      if (storedSessionFileMatches(dbManager, metadata)) {
         result.sessionsSkipped++;
-      } else {
-        result.sessionsIndexed++;
-        result.messagesIndexed += indexResult.messagesIndexed;
+        continue;
       }
+      changed.push(metadata);
     } catch (err) {
       result.errors.push(`Error indexing ${file}: ${err instanceof Error ? err.message : String(err)}`);
     }
   }
 
+  changed.sort((a, b) => b.mtimeMs - a.mtimeMs);
+
+  for (const metadata of changed) {
+    if (result.sessionsProcessed >= maxFilesToIndex) {
+      result.reachedLimit = true;
+      break;
+    }
+    try {
+      indexSessionFile(dbManager, metadata.path, result);
+    } catch (err) {
+      result.errors.push(`Error indexing ${metadata.path}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
   return result;
 }
 
@@ -277,20 +388,28 @@ function isRecentBackfillTimestamp(value: string | null, nowMs: number): boolean
 /**
  * Determine whether a background session backfill should run.
  *
- * A backfill is needed when the number of JSONL session files differs from
- * the indexed session count, or when no successful backfill has completed in
- * the last 24 hours. The count check catches crashed/abnormal sessions; the
- * timestamp check periodically repairs parse errors or manual DB edits.
+ * The check stays cheap: it compares file counts and stored file size/mtime
+ * metadata. Full JSONL parsing is left to the scheduled incremental backfill.
  */
 export function needsBackfill(dbManager: DatabaseManager, sessionsDir: string, now = new Date()): boolean {
   const db = dbManager.getDb();
-  const fileCount = countSessionFiles(sessionsDir);
+  const files = getSessionFiles(sessionsDir);
   const indexed = db.prepare('SELECT COUNT(*) as count FROM sessions').get() as { count: number };
 
-  if (fileCount !== indexed.count) {
+  if (files.length > indexed.count) {
     return true;
   }
 
+  for (const file of files) {
+    try {
+      const metadata = getSessionFileMetadata(file);
+      if (storedSessionFileMatches(dbManager, metadata)) continue;
+      return true;
+    } catch {
+      return true;
+    }
+  }
+
   return !isRecentBackfillTimestamp(getLastBackfillTimestamp(dbManager), now.getTime());
 }