pi-hermes-memory 0.7.15 → 0.7.18

package/src/index.ts

@@ -28,6 +28,8 @@ import { MemoryStore } from "./store/memory-store.js";
 import { SkillStore } from "./store/skill-store.js";
 import { DatabaseManager } from "./store/db.js";
 import { indexSession } from "./store/session-indexer.js";
+import { scheduleSessionBackfill, waitForSessionBackfill, SESSION_BACKFILL_SHUTDOWN_TIMEOUT_MS } from "./handlers/session-backfill.js";
+import { scheduleLiveSessionIndex, waitForLiveSessionIndex, SESSION_LIVE_INDEX_SHUTDOWN_TIMEOUT_MS } from "./handlers/session-live-index.js";
 import { parseSessionFile } from "./store/session-parser.js";
 import { registerMemoryTool } from "./tools/memory-tool.js";
 import { registerSkillTool } from "./tools/skill-tool.js";
@@ -107,6 +109,7 @@ export default function (pi: ExtensionAPI) {
     migrationSentinelPath: path.join(globalDir, ".skills-migrated-to-extension-storage"),
   });
   const dbManager = new DatabaseManager(globalDir);
+  const sessionsDir = path.join(agentRoot, "sessions");
 
   const refreshSkillProjectContext = (cwd?: string) => {
     const resource = resolveProjectSkillDiscovery(skillStore, config.projectsMemoryDir, cwd);
@@ -150,6 +153,19 @@ export default function (pi: ExtensionAPI) {
     await skillStore.ensureDiscoveredRoots();
     await store.loadFromDisk();
     if (projectStore) await projectStore.loadFromDisk();
+
+    scheduleSessionBackfill(dbManager, sessionsDir, {
+      notify: (message, level) => {
+        const ui = (ctx as { ui?: { notify?: (message: string, level?: string) => void } }).ui;
+        if (ui?.notify) {
+          ui.notify(message, level);
+        } else if (level === "error" || level === "warning") {
+          console.warn(message);
+        } else {
+          console.info(message);
+        }
+      },
+    });
   });
 
   registerProjectSkillDiscoveryHandler(pi, skillStore, config.projectsMemoryDir);
@@ -201,12 +217,29 @@ export default function (pi: ExtensionAPI) {
   registerSyncMarkdownMemoriesCommand(pi, dbManager, globalDir, config.projectsMemoryDir, agentRoot);
   registerPreviewContextCommand(pi, store, projectStore, projectName, config);
 
-  // ── 10. SQLite session search + extended memory ──
+  // ── 10. Live session indexing ──
+  pi.on("message_end", async (_event, ctx) => {
+    scheduleLiveSessionIndex(dbManager, ctx.sessionManager, {
+      onError: (err) => console.warn(`⚠️ Live session indexing failed: ${err instanceof Error ? err.message : String(err)}`),
+    });
+  });
+
+  // ── 11. SQLite session search + extended memory ──
   registerSessionSearchTool(pi, dbManager, config.sessionSearch ?? { variant: "legacy" });
   registerMemorySearchTool(pi, dbManager);
   registerIndexSessionsCommand(pi);
 
-  // ── 11. Auto-index session on shutdown ──
+  // ── 12. Auto-index session on shutdown ──
+  // Registered last, so this runs after the session-flush shutdown handler and
+  // is the final DB activity. Closing here truncates the WAL via
+  // PRAGMA wal_checkpoint(TRUNCATE); without it the WAL only grows to its
+  // high-water mark and is never reclaimed across sessions.
+  //
+  // Ordering is safe: Pi's ExtensionRunner.emit() runs same-extension handlers
+  // sequentially in registration order and awaits each one, so the flush above
+  // fully completes before close() runs. WARNING: do not register another
+  // DB-writing session_shutdown handler after this block — it would run after
+  // close() and silently no-op.
   pi.on("session_shutdown", async (_event, ctx) => {
     try {
       const sessionFile = ctx.sessionManager.getSessionFile();
@@ -218,6 +251,16 @@ export default function (pi: ExtensionAPI) {
       }
     } catch {
       // Silent fail — don't block shutdown
+    } finally {
+      try {
+        await Promise.all([
+          waitForSessionBackfill(SESSION_BACKFILL_SHUTDOWN_TIMEOUT_MS),
+          waitForLiveSessionIndex(SESSION_LIVE_INDEX_SHUTDOWN_TIMEOUT_MS),
+        ]);
+      } catch {
+        // Best effort only — shutdown should not be held up by indexing errors.
+      }
+      try { dbManager.close(); } catch { /* best effort — never block shutdown */ }
     }
   });
 }

package/src/paths.ts

@@ -2,7 +2,12 @@ import * as os from "node:os";
 import * as path from "node:path";
 import { DEFAULT_PROJECTS_MEMORY_DIR } from "./constants.js";
 
-export const AGENT_ROOT = path.join(os.homedir(), ".pi", "agent");
+export const AGENT_ROOT = resolveAgentRoot();
+
+export function resolveAgentRoot(env: Record<string, string | undefined> = process.env): string {
+  const configured = env.PI_CODING_AGENT_DIR?.trim();
+  return configured ? path.resolve(expandHome(configured)) : path.join(os.homedir(), ".pi", "agent");
+}
 
 export function expandHome(input: string): string {
   if (input === "~") return os.homedir();

package/src/store/fts-query.ts

@@ -2,6 +2,10 @@ const FTS5_OPERATOR_PATTERN = /\b(OR|AND|NOT|NEAR)\b/;
 const FTS5_TOKEN_PATTERN = /"([^"]*)"|(\S+)/g;
 const NATURAL_LANGUAGE_CONNECTORS = new Set(['and', 'or', 'not', 'near']);
 
+export function hasExplicitFts5Operator(query: string): boolean {
+  return FTS5_OPERATOR_PATTERN.test(query.trim());
+}
+
 function collectNaturalLanguageTerms(query: string): string[] {
   const terms: string[] = [];
 
@@ -29,7 +33,7 @@ export function normalizeFts5Query(query: string): string {
   const trimmed = query.trim();
   if (trimmed.length === 0) return '';
 
-  if (FTS5_OPERATOR_PATTERN.test(trimmed)) {
+  if (hasExplicitFts5Operator(trimmed)) {
     return trimmed;
   }
 
@@ -45,7 +49,7 @@ export function normalizeFts5Query(query: string): string {
  */
 export function buildFallbackFts5Query(query: string): string | null {
   const trimmed = query.trim();
-  if (trimmed.length === 0 || FTS5_OPERATOR_PATTERN.test(trimmed)) {
+  if (trimmed.length === 0 || hasExplicitFts5Operator(trimmed)) {
     return null;
   }
 

package/src/store/memory-store.ts

@@ -13,7 +13,6 @@
 
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
-import * as os from "node:os";
 import { scanContent } from "./content-scanner.js";
 import { normalizeMemoryLookupText } from "./memory-lookup.js";
 import {
@@ -26,6 +25,7 @@ import {
   USER_FILE,
 } from "../constants.js";
 import type { MemoryConfig, MemoryResult, MemorySnapshot, ConsolidationResult, MemoryCategory, MemoryOverflowStrategy } from "../types.js";
+import { AGENT_ROOT } from "../paths.js";
 
 export class MemoryStore {
   private memoryEntries: string[] = [];
@@ -47,7 +47,7 @@ export class MemoryStore {
   // ─── Path helpers ───
 
   private get memoryDir(): string {
-    return this.config.memoryDir ?? path.join(os.homedir(), ".pi", "agent", "pi-hermes-memory");
+    return this.config.memoryDir ?? path.join(AGENT_ROOT, "pi-hermes-memory");
   }
 
   private pathFor(target: "memory" | "user" | "failure"): string {
@@ -338,6 +338,15 @@ export class MemoryStore {
     return block ? this.fenceBlock(block) : "";
   }
 
+  /**
+   * All failure entries (no age filter), metadata stripped.
+   * Used by consolidation, which must consider the full file size —
+   * unlike getFailureEntries(), which filters by age for injection.
+   */
+  getAllFailureEntries(): string[] {
+    return this.failureEntries.map((e) => this.stripMetadata(e));
+  }
+
   getMemoryEntries(): string[] {
     return this.memoryEntries.map((e) => this.stripMetadata(e));
   }

package/src/store/schema.ts

@@ -10,6 +10,12 @@
  */
 
 export const SCHEMA_SQL = `
+  -- Extension key/value metadata
+  CREATE TABLE IF NOT EXISTS extension_metadata (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+  );
+
   -- Session metadata
   CREATE TABLE IF NOT EXISTS sessions (
     id TEXT PRIMARY KEY,

package/src/store/session-indexer.ts

@@ -1,13 +1,17 @@
+import fs from 'node:fs';
 import { DatabaseManager } from './db.js';
 import { parseSessionFile, getSessionFiles, type ParsedSession } from './session-parser.js';
 
+export const LAST_SESSION_BACKFILL_KEY = 'last_session_backfill';
+export const SESSION_BACKFILL_INTERVAL_MS = 24 * 60 * 60 * 1000;
+
 /**
  * Index result for a single session.
  */
 export interface IndexResult {
   sessionId: string;
   messagesIndexed: number;
-  skipped: boolean; // true if already indexed
+  skipped: boolean; // true if the session already existed and no new messages were indexed
 }
 
 /**
@@ -29,33 +33,39 @@ export interface BulkIndexResult {
 export function indexSession(dbManager: DatabaseManager, session: ParsedSession): IndexResult {
   const db = dbManager.getDb();
 
-  // Check if already indexed
-  const existing = db.prepare('SELECT id FROM sessions WHERE id = ?').get(session.id) as { id: string } | undefined;
-  if (existing) {
-    return { sessionId: session.id, messagesIndexed: 0, skipped: true };
-  }
+  const existingSession = db.prepare('SELECT id FROM sessions WHERE id = ?').get(session.id) as { id: string } | undefined;
+  const before = db.prepare('SELECT COUNT(*) as count FROM messages WHERE session_id = ?').get(session.id) as { count: number };
 
-  // Insert session
-  db.prepare(`
-    INSERT INTO sessions (id, project, cwd, started_at, ended_at, message_count)
+  const insertSession = db.prepare(`
+    INSERT OR IGNORE INTO sessions (id, project, cwd, started_at, ended_at, message_count)
     VALUES (?, ?, ?, ?, ?, ?)
-  `).run(
-    session.id,
-    session.project,
-    session.cwd,
-    session.startedAt,
-    session.endedAt,
-    session.messages.length
-  );
-
-  // Insert messages in a transaction for performance
+  `);
+
   const insertMsg = db.prepare(`
-    INSERT INTO messages (id, session_id, role, content, timestamp, tool_calls)
+    INSERT OR IGNORE INTO messages (id, session_id, role, content, timestamp, tool_calls)
     VALUES (?, ?, ?, ?, ?, ?)
   `);
 
-  const writeMessages = (messages: ParsedSession['messages']) => {
-    for (const msg of messages) {
+  const updateSession = db.prepare(`
+    UPDATE sessions
+    SET project = ?,
+        cwd = ?,
+        ended_at = COALESCE(?, ended_at),
+        message_count = (SELECT COUNT(*) FROM messages WHERE session_id = ?)
+    WHERE id = ?
+  `);
+
+  const writeSession = () => {
+    insertSession.run(
+      session.id,
+      session.project,
+      session.cwd,
+      session.startedAt,
+      session.endedAt,
+      session.messages.length
+    );
+
+    for (const msg of session.messages) {
       insertMsg.run(
         msg.id,
         session.id,
@@ -65,16 +75,136 @@ export function indexSession(dbManager: DatabaseManager, session: ParsedSession)
         msg.toolCalls ? JSON.stringify(msg.toolCalls) : null
       );
     }
+
+    updateSession.run(session.project, session.cwd, session.endedAt, session.id, session.id);
   };
 
   if (db.transaction) {
-    const insertMany = db.transaction(writeMessages);
-    insertMany(session.messages);
+    const tx = db.transaction(writeSession);
+    tx();
   } else {
-    writeMessages(session.messages);
+    writeSession();
+  }
+
+  const after = db.prepare('SELECT COUNT(*) as count FROM messages WHERE session_id = ?').get(session.id) as { count: number };
+  const messagesIndexed = after.count - before.count;
+
+  return { sessionId: session.id, messagesIndexed, skipped: Boolean(existingSession) && messagesIndexed === 0 };
+}
+
+type SessionManagerSnapshot = {
+  getHeader: () => { id: string; timestamp: string; cwd: string } | null;
+  getEntries: () => unknown[];
+  getSessionFile?: () => string | undefined;
+};
+
+type SessionMessageEntryLike = {
+  type?: unknown;
+  id?: unknown;
+  timestamp?: unknown;
+  message?: {
+    role?: unknown;
+    content?: unknown;
+  };
+};
+
+function extractTextContent(content: unknown): string {
+  if (typeof content === 'string') return content;
+  if (!Array.isArray(content)) return '';
+
+  const parts: string[] = [];
+  for (const block of content) {
+    if (!block || typeof block !== 'object') continue;
+    const b = block as Record<string, unknown>;
+
+    switch (b.type) {
+      case 'text':
+        if (typeof b.text === 'string') parts.push(b.text);
+        break;
+      case 'tool_result':
+        if (typeof b.content === 'string') {
+          parts.push(b.content);
+        } else if (Array.isArray(b.content)) {
+          for (const item of b.content) {
+            if (item && typeof item === 'object' && (item as Record<string, unknown>).type === 'text') {
+              const text = (item as Record<string, unknown>).text;
+              if (typeof text === 'string') parts.push(text);
+            }
+          }
+        }
+        break;
+    }
+  }
+
+  return parts.join('\n').trim();
+}
+
+function extractToolCalls(content: unknown): string[] | undefined {
+  if (!Array.isArray(content)) return undefined;
+
+  const toolNames: string[] = [];
+  for (const block of content) {
+    if (!block || typeof block !== 'object') continue;
+    const b = block as Record<string, unknown>;
+    if ((b.type === 'toolCall' || b.type === 'tool_use') && typeof b.name === 'string') {
+      toolNames.push(b.name);
+    }
+  }
+  return toolNames.length > 0 ? toolNames : undefined;
+}
+
+function parseMessageEntry(entry: unknown): ParsedSession['messages'][number] | null {
+  if (!entry || typeof entry !== 'object') return null;
+  const e = entry as SessionMessageEntryLike;
+  if (e.type !== 'message' || typeof e.id !== 'string' || typeof e.timestamp !== 'string' || !e.message) return null;
+
+  const role = e.message.role;
+  if (role !== 'user' && role !== 'assistant' && role !== 'system') return null;
+
+  const content = extractTextContent(e.message.content);
+  if (!content) return null;
+
+  return {
+    id: e.id,
+    role,
+    content,
+    timestamp: e.timestamp,
+    toolCalls: role === 'assistant' ? extractToolCalls(e.message.content) : undefined,
+  };
+}
+
+export function parseSessionManagerSnapshot(sessionManager: SessionManagerSnapshot): ParsedSession | null {
+  const header = sessionManager.getHeader();
+  if (!header?.id || !header.cwd || !header.timestamp) return null;
+
+  const messages = sessionManager.getEntries()
+    .map(parseMessageEntry)
+    .filter((msg): msg is ParsedSession['messages'][number] => msg !== null);
+
+  return {
+    id: header.id,
+    project: header.cwd.split('/').pop() ?? header.cwd,
+    cwd: header.cwd,
+    startedAt: header.timestamp,
+    endedAt: null,
+    messages,
+  };
+}
+
+export function indexCurrentSession(dbManager: DatabaseManager, sessionManager: SessionManagerSnapshot): IndexResult | null {
+  const session = parseSessionManagerSnapshot(sessionManager);
+  if (!session) return null;
+  return indexSession(dbManager, session);
+}
+
+export function indexLiveSession(dbManager: DatabaseManager, sessionManager: SessionManagerSnapshot): IndexResult | null {
+  const sessionFile = sessionManager.getSessionFile?.();
+  if (sessionFile && fs.existsSync(sessionFile)) {
+    const session = parseSessionFile(sessionFile);
+    if (session) return indexSession(dbManager, session);
   }
 
-  return { sessionId: session.id, messagesIndexed: session.messages.length, skipped: false };
+  return indexCurrentSession(dbManager, sessionManager);
 }
 
 /**
@@ -124,6 +254,58 @@ export function indexAllSessions(
   return result;
 }
 
+/**
+ * Cheaply count session JSONL files in the same scope indexAllSessions scans.
+ */
+export function countSessionFiles(sessionsDir: string): number {
+  return getSessionFiles(sessionsDir).length;
+}
+
+function getLastBackfillTimestamp(dbManager: DatabaseManager): string | null {
+  const db = dbManager.getDb();
+  const row = db.prepare('SELECT value FROM extension_metadata WHERE key = ?').get(LAST_SESSION_BACKFILL_KEY) as { value: string } | undefined;
+  return row?.value ?? null;
+}
+
+function isRecentBackfillTimestamp(value: string | null, nowMs: number): boolean {
+  if (!value) return false;
+  const parsed = Date.parse(value);
+  if (!Number.isFinite(parsed)) return false;
+  return nowMs - parsed < SESSION_BACKFILL_INTERVAL_MS;
+}
+
+/**
+ * Determine whether a background session backfill should run.
+ *
+ * A backfill is needed when the number of JSONL session files differs from
+ * the indexed session count, or when no successful backfill has completed in
+ * the last 24 hours. The count check catches crashed/abnormal sessions; the
+ * timestamp check periodically repairs parse errors or manual DB edits.
+ */
+export function needsBackfill(dbManager: DatabaseManager, sessionsDir: string, now = new Date()): boolean {
+  const db = dbManager.getDb();
+  const fileCount = countSessionFiles(sessionsDir);
+  const indexed = db.prepare('SELECT COUNT(*) as count FROM sessions').get() as { count: number };
+
+  if (fileCount !== indexed.count) {
+    return true;
+  }
+
+  return !isRecentBackfillTimestamp(getLastBackfillTimestamp(dbManager), now.getTime());
+}
+
+/**
+ * Record a successful session backfill completion timestamp.
+ */
+export function touchBackfillTimestamp(dbManager: DatabaseManager, timestamp = new Date()): void {
+  const db = dbManager.getDb();
+  db.prepare(`
+    INSERT INTO extension_metadata (key, value)
+    VALUES (?, ?)
+    ON CONFLICT(key) DO UPDATE SET value = excluded.value
+  `).run(LAST_SESSION_BACKFILL_KEY, timestamp.toISOString());
+}
+
 /**
  * Get statistics about indexed sessions.
  */

package/src/store/session-parser.ts

@@ -1,4 +1,5 @@
 import fs from 'node:fs';
+import path from 'node:path';
 
 /**
  * Parsed session data from a JSONL file.
@@ -90,7 +91,7 @@ function extractToolCalls(content: unknown): string[] | undefined {
   for (const block of content) {
     if (!block || typeof block !== 'object') continue;
     const b = block as Record<string, unknown>;
-    if (b.type === 'tool_use' && typeof b.name === 'string') {
+    if ((b.type === 'tool_use' || b.type === 'toolCall') && typeof b.name === 'string') {
       toolNames.push(b.name);
     }
   }
@@ -179,23 +180,29 @@ export function parseSessionFile(filePath: string): ParsedSession | null {
  */
 export function getSessionFiles(sessionsDir: string, projectDir?: string): string[] {
   if (projectDir) {
-    const dir = `${sessionsDir}/${projectDir}`;
+    const dir = path.join(sessionsDir, projectDir);
     if (!fs.existsSync(dir)) return [];
     return fs.readdirSync(dir)
       .filter(f => f.endsWith('.jsonl'))
-      .map(f => `${dir}/${f}`);
+      .map(f => path.join(dir, f));
   }
 
   // All projects
   if (!fs.existsSync(sessionsDir)) return [];
   const files: string[] = [];
-  for (const dir of fs.readdirSync(sessionsDir)) {
-    const dirPath = `${sessionsDir}/${dir}`;
-    if (!fs.statSync(dirPath).isDirectory()) continue;
-    for (const f of fs.readdirSync(dirPath)) {
-      if (f.endsWith('.jsonl')) {
-        files.push(`${dirPath}/${f}`);
+  for (const entry of fs.readdirSync(sessionsDir)) {
+    const entryPath = path.join(sessionsDir, entry);
+    const stat = fs.statSync(entryPath);
+    if (stat.isDirectory()) {
+      // Scan .jsonl files inside project subdirectories
+      for (const f of fs.readdirSync(entryPath)) {
+        if (f.endsWith('.jsonl')) {
+          files.push(path.join(entryPath, f));
+        }
       }
+    } else if (stat.isFile() && entry.endsWith('.jsonl')) {
+      // Also pick up root-level .jsonl files
+      files.push(entryPath);
     }
   }
   return files;