pi-hermes-memory 0.3.3 → 0.4.0

package/src/index.ts

@@ -27,8 +27,13 @@ import * as os from "node:os";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { MemoryStore } from "./store/memory-store.js";
 import { SkillStore } from "./store/skill-store.js";
+import { DatabaseManager } from "./store/db.js";
+import { indexSession } from "./store/session-indexer.js";
+import { parseSessionFile } from "./store/session-parser.js";
 import { registerMemoryTool } from "./tools/memory-tool.js";
 import { registerSkillTool } from "./tools/skill-tool.js";
+import { registerSessionSearchTool } from "./tools/session-search-tool.js";
+import { registerMemorySearchTool } from "./tools/memory-search-tool.js";
 import { setupBackgroundReview } from "./handlers/background-review.js";
 import { setupSessionFlush } from "./handlers/session-flush.js";
 import { registerInsightsCommand } from "./handlers/insights.js";
@@ -38,6 +43,7 @@ import { setupSkillAutoTrigger } from "./handlers/skill-auto-trigger.js";
 import { registerSkillsCommand } from "./handlers/skills-command.js";
 import { registerInterviewCommand } from "./handlers/interview.js";
 import { registerSwitchProjectCommand } from "./handlers/switch-project.js";
+import { registerIndexSessionsCommand } from "./handlers/index-sessions.js";
 import { loadConfig } from "./config.js";
 import { detectProject } from "./project.js";
 
@@ -47,6 +53,7 @@ export default function (pi: ExtensionAPI) {
   const globalDir = config.memoryDir ?? path.join(os.homedir(), ".pi", "agent", "memory");
   const store = new MemoryStore(config);
   const skillStore = new SkillStore(path.join(globalDir, "skills"));
+  const dbManager = new DatabaseManager(globalDir);
 
   // Detect project from cwd using shared helper
   const project = detectProject();
@@ -111,4 +118,39 @@ export default function (pi: ExtensionAPI) {
   registerSkillsCommand(pi, skillStore);
   registerInterviewCommand(pi, store);
   registerSwitchProjectCommand(pi);
+
+  // ── 11. SQLite session search + extended memory ──
+  registerSessionSearchTool(pi, dbManager);
+  registerMemorySearchTool(pi, dbManager);
+  registerIndexSessionsCommand(pi);
+
+  // ── 12. Auto-index session on shutdown ──
+  let currentSessionId: string | null = null;
+  let currentSessionCwd: string | null = null;
+
+  pi.on("session_start", async (event, _ctx) => {
+    // Capture session metadata for indexing
+    currentSessionId = (event as Record<string, unknown>).sessionId as string ?? null;
+    currentSessionCwd = process.cwd();
+  });
+
+  pi.on("session_shutdown", async (_event, _ctx) => {
+    // Index the current session to SQLite
+    if (currentSessionId && currentSessionCwd) {
+      try {
+        const sessionsDir = path.join(os.homedir(), ".pi", "agent", "sessions");
+        const encodedCwd = currentSessionCwd.replace(/\//g, "-");
+        const sessionFiles = require("node:fs").readdirSync(path.join(sessionsDir, encodedCwd))
+          .filter((f: string) => f.includes(currentSessionId!) && f.endsWith(".jsonl"));
+        if (sessionFiles.length > 0) {
+          const sessionData = parseSessionFile(path.join(sessionsDir, encodedCwd, sessionFiles[0]));
+          if (sessionData) {
+            indexSession(dbManager, sessionData);
+          }
+        }
+      } catch {
+        // Silent fail — don't block shutdown
+      }
+    }
+  });
 }

package/src/skills/learn-memory-tool/SKILL.md

@@ -0,0 +1,125 @@
+---
+name: learn-memory-tool
+description: Learn how to use the pi-hermes-memory extension effectively — when to save memories, how to search, and best practices for persistent memory.
+version: 1
+created: 2026-05-03
+updated: 2026-05-03
+---
+
+## When to Use
+
+When a user asks about the memory system, how to use it, or when they seem confused about what gets remembered. Also useful for onboarding new users to the extension.
+
+## Overview
+
+Pi Hermes Memory gives your AI agent persistent memory across sessions. Here's what it does:
+
+### What Gets Saved
+
+| Type | File | What Goes Here | Limit |
+|---|---|---|---|
+| **Memory** | `MEMORY.md` | Facts — env details, project conventions, tool quirks | 5,000 chars |
+| **User Profile** | `USER.md` | Who you are — name, preferences, communication style | 5,000 chars |
+| **Skills** | `skills/*.md` | Procedures — *how* to debug, deploy, test | Unlimited |
+| **Extended Memory** | `sessions.db` | Searchable memories beyond the core limit | Unlimited |
+
+### The `memory` Tool
+
+The agent has a `memory` tool with these actions:
+
+| Action | Target | What It Does |
+|---|---|---|
+| `add` | `memory` or `user` | Append a new entry |
+| `replace` | `memory` or `user` | Update an existing entry (matched by substring) |
+| `remove` | `memory` or `user` | Delete an entry (matched by substring) |
+
+### The `skill` Tool
+
+For saving reusable procedures:
+
+| Action | What It Does |
+|---|---|
+| `create` | Save a new skill |
+| `view` | Read a skill or list all skills |
+| `patch` | Update one section of a skill |
+| `edit` | Replace description and/or body |
+| `delete` | Remove a skill |
+
+### Search Tools
+
+| Tool | What It Does |
+|---|---|
+| `session_search` | Search past conversations across all sessions |
+| `memory_search` | Search extended memory store (unlimited capacity) |
+
+### Commands
+
+| Command | What It Does |
+|---|---|
+| `/memory-insights` | Shows everything stored in memory and user profile |
+| `/memory-skills` | Lists all agent-created skills |
+| `/memory-consolidate` | Manually trigger memory consolidation |
+| `/memory-interview` | Answer questions to pre-fill your user profile |
+| `/memory-switch-project` | List all project memories |
+| `/memory-index-sessions` | Import past sessions for search |
+
+## Best Practices
+
+### What TO Save
+
+- **User preferences**: "prefers pnpm over npm", "uses vim", "likes concise answers"
+- **Environment facts**: "macOS M1", "Node 20", "project uses Prisma"
+- **Corrections**: "don't use npm — use pnpm", "always run tests first"
+- **Project conventions**: "monorepo with turborepo", "conventional commits"
+- **Tool quirks**: "CI needs `--frozen-lockfile`", "deploy script is in scripts/deploy.sh"
+
+### What NOT to Save
+
+- **Task progress**: "finished implementing auth" — this is temporary
+- **Session outcomes**: "PR #42 was merged" — this belongs in git history
+- **Temporary state**: "currently debugging the test failure" — will be irrelevant soon
+- **Large code blocks**: Use skills instead for procedures
+
+### How Memory Flows
+
+1. **Session starts**: Core memory (MEMORY.md + USER.md) is injected into the system prompt
+2. **During conversation**: Agent saves memories via the `memory` tool
+3. **Every 10 turns or 15 tool calls**: Background review saves anything noteworthy
+4. **When you correct the agent**: Immediate save — no waiting
+5. **When memory is full**: Auto-consolidation merges and prunes entries
+6. **Session ends**: One last flush before shutdown
+
+### Two-Tier Architecture
+
+- **Global memory** (`~/.pi/agent/memory/`): Always injected — your name, preferences, tools
+- **Project memory** (`~/.pi/agent/<project>/`): Injected when cwd matches — project-specific facts
+
+### Memory Aging
+
+Entries carry timestamps. When consolidating, the agent knows which entries are stale (created long ago, never referenced) and which are fresh.
+
+### Context Fencing
+
+Memory is wrapped in `<memory-context>` XML tags so the LLM never treats stored facts as user instructions. This prevents injection attacks through stored memory.
+
+## Troubleshooting
+
+### "Memory is full"
+Run `/memory-consolidate` to manually merge entries. Or let auto-consolidation handle it.
+
+### "I can't find what I'm looking for"
+Use `memory_search` to search the extended store, or `session_search` to search past conversations.
+
+### "The agent forgot something"
+Check `/memory-insights` to see what's stored. If it's not there, the agent may not have saved it yet. You can tell the agent: "remember that X".
+
+### "I want to edit memory manually"
+Memory files are plain markdown at `~/.pi/agent/memory/MEMORY.md` and `USER.md`. Edit them directly if you want.
+
+## Verification
+
+After reading this skill, the user should understand:
+1. What the memory tool does and when to use it
+2. The difference between memory, user profile, skills, and extended memory
+3. How to search across sessions and extended memory
+4. Best practices for what to save and what not to save

package/src/store/db.ts

@@ -0,0 +1,84 @@
+import Database from 'better-sqlite3';
+import path from 'node:path';
+import fs from 'node:fs';
+import { SCHEMA_SQL } from './schema.js';
+
+export class DatabaseManager {
+  private db: Database.Database | null = null;
+  private readonly dbPath: string;
+
+  constructor(memoryDir: string) {
+    this.dbPath = path.join(memoryDir, 'sessions.db');
+  }
+
+  /**
+   * Get the database instance. Creates/opens on first call.
+   */
+  getDb(): Database.Database {
+    if (!this.db) {
+      this.db = this.open();
+    }
+    return this.db;
+  }
+
+  /**
+   * Open the database and initialize schema.
+   */
+  private open(): Database.Database {
+    // Ensure directory exists
+    const dir = path.dirname(this.dbPath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+
+    const db = new Database(this.dbPath);
+
+    // Enable WAL mode for concurrent reads
+    db.pragma('journal_mode = WAL');
+    db.pragma('foreign_keys = ON');
+
+    // Create tables and triggers
+    db.exec(SCHEMA_SQL);
+
+    return db;
+  }
+
+  /**
+   * Close the database connection.
+   */
+  close(): void {
+    if (this.db) {
+      this.db.close();
+      this.db = null;
+    }
+  }
+
+  /**
+   * Get the database file path.
+   */
+  getPath(): string {
+    return this.dbPath;
+  }
+
+  /**
+   * Check if the database file exists.
+   */
+  exists(): boolean {
+    return fs.existsSync(this.dbPath);
+  }
+
+  /**
+   * Get stats about the database.
+   */
+  getStats(): { sessions: number; messages: number; memories: number } {
+    const db = this.getDb();
+    const sessions = db.prepare('SELECT COUNT(*) as count FROM sessions').get() as { count: number };
+    const messages = db.prepare('SELECT COUNT(*) as count FROM messages').get() as { count: number };
+    const memories = db.prepare('SELECT COUNT(*) as count FROM memories').get() as { count: number };
+    return {
+      sessions: sessions.count,
+      messages: messages.count,
+      memories: memories.count,
+    };
+  }
+}

package/src/store/schema.ts

@@ -0,0 +1,94 @@
+/**
+ * SQLite schema for pi-hermes-memory v0.4
+ *
+ * Tables:
+ * - sessions — Pi session metadata
+ * - messages — all conversation messages
+ * - message_fts — FTS5 index for full-text search across messages
+ * - memories — extended memory entries (unlimited, searchable)
+ * - memory_fts — FTS5 index for memory search
+ */
+
+export const SCHEMA_SQL = `
+  -- Session metadata
+  CREATE TABLE IF NOT EXISTS sessions (
+    id TEXT PRIMARY KEY,
+    project TEXT NOT NULL,
+    cwd TEXT NOT NULL,
+    started_at TEXT NOT NULL,
+    ended_at TEXT,
+    message_count INTEGER DEFAULT 0
+  );
+
+  -- All messages from all sessions
+  CREATE TABLE IF NOT EXISTS messages (
+    id TEXT PRIMARY KEY,
+    session_id TEXT NOT NULL REFERENCES sessions(id),
+    role TEXT NOT NULL CHECK (role IN ('user', 'assistant', 'system')),
+    content TEXT NOT NULL,
+    timestamp TEXT NOT NULL,
+    tool_calls TEXT
+  );
+
+  -- FTS5 index for full-text search across messages
+  -- content='messages' + content_rowid='rowid' keeps FTS in sync with the content table
+  CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts5(
+    content,
+    content='messages',
+    content_rowid='rowid'
+  );
+
+  -- Triggers to keep message_fts in sync with messages table
+  CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN
+    INSERT INTO message_fts(rowid, content) VALUES (new.rowid, new.content);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN
+    INSERT INTO message_fts(message_fts, rowid, content) VALUES ('delete', old.rowid, old.content);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN
+    INSERT INTO message_fts(message_fts, rowid, content) VALUES ('delete', old.rowid, old.content);
+    INSERT INTO message_fts(rowid, content) VALUES (new.rowid, new.content);
+  END;
+
+  -- Extended memory entries (beyond MEMORY.md limit)
+  CREATE TABLE IF NOT EXISTS memories (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    project TEXT,
+    target TEXT NOT NULL CHECK (target IN ('memory', 'user')),
+    content TEXT NOT NULL,
+    created DATE NOT NULL,
+    last_referenced DATE NOT NULL
+  );
+
+  -- FTS5 index for memory search
+  -- content='memories' + content_rowid='id' keeps FTS in sync
+  CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(
+    content,
+    content='memories',
+    content_rowid='id'
+  );
+
+  -- Triggers to keep memory_fts in sync with memories table
+  CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
+    INSERT INTO memory_fts(rowid, content) VALUES (new.id, new.content);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
+    INSERT INTO memory_fts(memory_fts, rowid, content) VALUES ('delete', old.id, old.content);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
+    INSERT INTO memory_fts(memory_fts, rowid, content) VALUES ('delete', old.id, old.content);
+    INSERT INTO memory_fts(rowid, content) VALUES (new.id, new.content);
+  END;
+
+  -- Indexes for common queries
+  CREATE INDEX IF NOT EXISTS idx_messages_session_id ON messages(session_id);
+  CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp);
+  CREATE INDEX IF NOT EXISTS idx_memories_project ON memories(project);
+  CREATE INDEX IF NOT EXISTS idx_memories_target ON memories(target);
+  CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project);
+  CREATE INDEX IF NOT EXISTS idx_sessions_started_at ON sessions(started_at);
+`;

package/src/store/session-indexer.ts

@@ -0,0 +1,153 @@
+import { DatabaseManager } from './db.js';
+import { parseSessionFile, getSessionFiles, type ParsedSession } from './session-parser.js';
+
+/**
+ * Index result for a single session.
+ */
+export interface IndexResult {
+  sessionId: string;
+  messagesIndexed: number;
+  skipped: boolean; // true if already indexed
+}
+
+/**
+ * Bulk index result.
+ */
+export interface BulkIndexResult {
+  sessionsProcessed: number;
+  sessionsIndexed: number;
+  sessionsSkipped: number;
+  messagesIndexed: number;
+  errors: string[];
+}
+
+/**
+ * Index a single session into the database.
+ *
+ * @returns IndexResult with count of messages indexed
+ */
+export function indexSession(dbManager: DatabaseManager, session: ParsedSession): IndexResult {
+  const db = dbManager.getDb();
+
+  // Check if already indexed
+  const existing = db.prepare('SELECT id FROM sessions WHERE id = ?').get(session.id) as { id: string } | undefined;
+  if (existing) {
+    return { sessionId: session.id, messagesIndexed: 0, skipped: true };
+  }
+
+  // Insert session
+  db.prepare(`
+    INSERT INTO sessions (id, project, cwd, started_at, ended_at, message_count)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `).run(
+    session.id,
+    session.project,
+    session.cwd,
+    session.startedAt,
+    session.endedAt,
+    session.messages.length
+  );
+
+  // Insert messages in a transaction for performance
+  const insertMsg = db.prepare(`
+    INSERT INTO messages (id, session_id, role, content, timestamp, tool_calls)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `);
+
+  const insertMany = db.transaction((messages: ParsedSession['messages']) => {
+    for (const msg of messages) {
+      insertMsg.run(
+        msg.id,
+        session.id,
+        msg.role,
+        msg.content,
+        msg.timestamp,
+        msg.toolCalls ? JSON.stringify(msg.toolCalls) : null
+      );
+    }
+  });
+
+  insertMany(session.messages);
+
+  return { sessionId: session.id, messagesIndexed: session.messages.length, skipped: false };
+}
+
+/**
+ * Index all sessions from disk.
+ *
+ * @param dbManager — Database manager instance
+ * @param sessionsDir — Path to ~/.pi/agent/sessions/
+ * @param projectDir — Optional: specific project directory to index
+ * @returns Bulk index result
+ */
+export function indexAllSessions(
+  dbManager: DatabaseManager,
+  sessionsDir: string,
+  projectDir?: string
+): BulkIndexResult {
+  const files = getSessionFiles(sessionsDir, projectDir);
+  const result: BulkIndexResult = {
+    sessionsProcessed: 0,
+    sessionsIndexed: 0,
+    sessionsSkipped: 0,
+    messagesIndexed: 0,
+    errors: [],
+  };
+
+  for (const file of files) {
+    result.sessionsProcessed++;
+
+    try {
+      const session = parseSessionFile(file);
+      if (!session) {
+        result.errors.push(`Failed to parse: ${file}`);
+        continue;
+      }
+
+      const indexResult = indexSession(dbManager, session);
+      if (indexResult.skipped) {
+        result.sessionsSkipped++;
+      } else {
+        result.sessionsIndexed++;
+        result.messagesIndexed += indexResult.messagesIndexed;
+      }
+    } catch (err) {
+      result.errors.push(`Error indexing ${file}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Get statistics about indexed sessions.
+ */
+export function getSessionStats(dbManager: DatabaseManager): {
+  totalSessions: number;
+  totalMessages: number;
+  projects: { project: string; sessions: number; messages: number }[];
+} {
+  const db = dbManager.getDb();
+
+  const totals = db.prepare(`
+    SELECT
+      (SELECT COUNT(*) FROM sessions) as sessions,
+      (SELECT COUNT(*) FROM messages) as messages
+  `).get() as { sessions: number; messages: number };
+
+  const projects = db.prepare(`
+    SELECT
+      project,
+      COUNT(*) as sessions,
+      (SELECT COUNT(*) FROM messages m WHERE m.session_id IN (SELECT id FROM sessions s2 WHERE s2.project = s.project)) as messages
+    FROM sessions s
+    GROUP BY project
+    ORDER BY sessions DESC
+  `).all() as { project: string; sessions: number; messages: number }[];
+
+  return {
+    totalSessions: totals.sessions,
+    totalMessages: totals.messages,
+    projects,
+  };
+}