pi-hermes-memory 0.3.3 → 0.4.1

package/src/skills/learn-memory-tool/SKILL.md

@@ -0,0 +1,125 @@
+---
+name: learn-memory-tool
+description: Learn how to use the pi-hermes-memory extension effectively — when to save memories, how to search, and best practices for persistent memory.
+version: 1
+created: 2026-05-03
+updated: 2026-05-03
+---
+
+## When to Use
+
+When a user asks about the memory system, how to use it, or when they seem confused about what gets remembered. Also useful for onboarding new users to the extension.
+
+## Overview
+
+Pi Hermes Memory gives your AI agent persistent memory across sessions. Here's what it does:
+
+### What Gets Saved
+
+| Type | File | What Goes Here | Limit |
+|---|---|---|---|
+| **Memory** | `MEMORY.md` | Facts — env details, project conventions, tool quirks | 5,000 chars |
+| **User Profile** | `USER.md` | Who you are — name, preferences, communication style | 5,000 chars |
+| **Skills** | `skills/*.md` | Procedures — *how* to debug, deploy, test | Unlimited |
+| **Extended Memory** | `sessions.db` | Searchable memories beyond the core limit | Unlimited |
+
+### The `memory` Tool
+
+The agent has a `memory` tool with these actions:
+
+| Action | Target | What It Does |
+|---|---|---|
+| `add` | `memory` or `user` | Append a new entry |
+| `replace` | `memory` or `user` | Update an existing entry (matched by substring) |
+| `remove` | `memory` or `user` | Delete an entry (matched by substring) |
+
+### The `skill` Tool
+
+For saving reusable procedures:
+
+| Action | What It Does |
+|---|---|
+| `create` | Save a new skill |
+| `view` | Read a skill or list all skills |
+| `patch` | Update one section of a skill |
+| `edit` | Replace description and/or body |
+| `delete` | Remove a skill |
+
+### Search Tools
+
+| Tool | What It Does |
+|---|---|
+| `session_search` | Search past conversations across all sessions |
+| `memory_search` | Search extended memory store (unlimited capacity) |
+
+### Commands
+
+| Command | What It Does |
+|---|---|
+| `/memory-insights` | Shows everything stored in memory and user profile |
+| `/memory-skills` | Lists all agent-created skills |
+| `/memory-consolidate` | Manually trigger memory consolidation |
+| `/memory-interview` | Answer questions to pre-fill your user profile |
+| `/memory-switch-project` | List all project memories |
+| `/memory-index-sessions` | Import past sessions for search |
+
+## Best Practices
+
+### What TO Save
+
+- **User preferences**: "prefers pnpm over npm", "uses vim", "likes concise answers"
+- **Environment facts**: "macOS M1", "Node 20", "project uses Prisma"
+- **Corrections**: "don't use npm — use pnpm", "always run tests first"
+- **Project conventions**: "monorepo with turborepo", "conventional commits"
+- **Tool quirks**: "CI needs `--frozen-lockfile`", "deploy script is in scripts/deploy.sh"
+
+### What NOT to Save
+
+- **Task progress**: "finished implementing auth" — this is temporary
+- **Session outcomes**: "PR #42 was merged" — this belongs in git history
+- **Temporary state**: "currently debugging the test failure" — will be irrelevant soon
+- **Large code blocks**: Use skills instead for procedures
+
+### How Memory Flows
+
+1. **Session starts**: Core memory (MEMORY.md + USER.md) is injected into the system prompt
+2. **During conversation**: Agent saves memories via the `memory` tool
+3. **Every 10 turns or 15 tool calls**: Background review saves anything noteworthy
+4. **When you correct the agent**: Immediate save — no waiting
+5. **When memory is full**: Auto-consolidation merges and prunes entries
+6. **Session ends**: One last flush before shutdown
+
+### Two-Tier Architecture
+
+- **Global memory** (`~/.pi/agent/memory/`): Always injected — your name, preferences, tools
+- **Project memory** (`~/.pi/agent/<project>/`): Injected when cwd matches — project-specific facts
+
+### Memory Aging
+
+Entries carry timestamps. When consolidating, the agent knows which entries are stale (created long ago, never referenced) and which are fresh.
+
+### Context Fencing
+
+Memory is wrapped in `<memory-context>` XML tags so the LLM never treats stored facts as user instructions. This prevents injection attacks through stored memory.
+
+## Troubleshooting
+
+### "Memory is full"
+Run `/memory-consolidate` to manually merge entries. Or let auto-consolidation handle it.
+
+### "I can't find what I'm looking for"
+Use `memory_search` to search the extended store, or `session_search` to search past conversations.
+
+### "The agent forgot something"
+Check `/memory-insights` to see what's stored. If it's not there, the agent may not have saved it yet. You can tell the agent: "remember that X".
+
+### "I want to edit memory manually"
+Memory files are plain markdown at `~/.pi/agent/memory/MEMORY.md` and `USER.md`. Edit them directly if you want.
+
+## Verification
+
+After reading this skill, the user should understand:
+1. What the memory tool does and when to use it
+2. The difference between memory, user profile, skills, and extended memory
+3. How to search across sessions and extended memory
+4. Best practices for what to save and what not to save

package/src/store/db.ts

@@ -0,0 +1,84 @@
+import Database from 'better-sqlite3';
+import path from 'node:path';
+import fs from 'node:fs';
+import { SCHEMA_SQL } from './schema.js';
+
+export class DatabaseManager {
+  private db: Database.Database | null = null;
+  private readonly dbPath: string;
+
+  constructor(memoryDir: string) {
+    this.dbPath = path.join(memoryDir, 'sessions.db');
+  }
+
+  /**
+   * Get the database instance. Creates/opens on first call.
+   */
+  getDb(): Database.Database {
+    if (!this.db) {
+      this.db = this.open();
+    }
+    return this.db;
+  }
+
+  /**
+   * Open the database and initialize schema.
+   */
+  private open(): Database.Database {
+    // Ensure directory exists
+    const dir = path.dirname(this.dbPath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+
+    const db = new Database(this.dbPath);
+
+    // Enable WAL mode for concurrent reads
+    db.pragma('journal_mode = WAL');
+    db.pragma('foreign_keys = ON');
+
+    // Create tables and triggers
+    db.exec(SCHEMA_SQL);
+
+    return db;
+  }
+
+  /**
+   * Close the database connection.
+   */
+  close(): void {
+    if (this.db) {
+      this.db.close();
+      this.db = null;
+    }
+  }
+
+  /**
+   * Get the database file path.
+   */
+  getPath(): string {
+    return this.dbPath;
+  }
+
+  /**
+   * Check if the database file exists.
+   */
+  exists(): boolean {
+    return fs.existsSync(this.dbPath);
+  }
+
+  /**
+   * Get stats about the database.
+   */
+  getStats(): { sessions: number; messages: number; memories: number } {
+    const db = this.getDb();
+    const sessions = db.prepare('SELECT COUNT(*) as count FROM sessions').get() as { count: number };
+    const messages = db.prepare('SELECT COUNT(*) as count FROM messages').get() as { count: number };
+    const memories = db.prepare('SELECT COUNT(*) as count FROM memories').get() as { count: number };
+    return {
+      sessions: sessions.count,
+      messages: messages.count,
+      memories: memories.count,
+    };
+  }
+}

package/src/store/schema.ts

@@ -0,0 +1,94 @@
+/**
+ * SQLite schema for pi-hermes-memory v0.4
+ *
+ * Tables:
+ * - sessions — Pi session metadata
+ * - messages — all conversation messages
+ * - message_fts — FTS5 index for full-text search across messages
+ * - memories — extended memory entries (unlimited, searchable)
+ * - memory_fts — FTS5 index for memory search
+ */
+
+export const SCHEMA_SQL = `
+  -- Session metadata
+  CREATE TABLE IF NOT EXISTS sessions (
+    id TEXT PRIMARY KEY,
+    project TEXT NOT NULL,
+    cwd TEXT NOT NULL,
+    started_at TEXT NOT NULL,
+    ended_at TEXT,
+    message_count INTEGER DEFAULT 0
+  );
+
+  -- All messages from all sessions
+  CREATE TABLE IF NOT EXISTS messages (
+    id TEXT PRIMARY KEY,
+    session_id TEXT NOT NULL REFERENCES sessions(id),
+    role TEXT NOT NULL CHECK (role IN ('user', 'assistant', 'system')),
+    content TEXT NOT NULL,
+    timestamp TEXT NOT NULL,
+    tool_calls TEXT
+  );
+
+  -- FTS5 index for full-text search across messages
+  -- content='messages' + content_rowid='rowid' keeps FTS in sync with the content table
+  CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts5(
+    content,
+    content='messages',
+    content_rowid='rowid'
+  );
+
+  -- Triggers to keep message_fts in sync with messages table
+  CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN
+    INSERT INTO message_fts(rowid, content) VALUES (new.rowid, new.content);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN
+    INSERT INTO message_fts(message_fts, rowid, content) VALUES ('delete', old.rowid, old.content);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN
+    INSERT INTO message_fts(message_fts, rowid, content) VALUES ('delete', old.rowid, old.content);
+    INSERT INTO message_fts(rowid, content) VALUES (new.rowid, new.content);
+  END;
+
+  -- Extended memory entries (beyond MEMORY.md limit)
+  CREATE TABLE IF NOT EXISTS memories (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    project TEXT,
+    target TEXT NOT NULL CHECK (target IN ('memory', 'user')),
+    content TEXT NOT NULL,
+    created DATE NOT NULL,
+    last_referenced DATE NOT NULL
+  );
+
+  -- FTS5 index for memory search
+  -- content='memories' + content_rowid='id' keeps FTS in sync
+  CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(
+    content,
+    content='memories',
+    content_rowid='id'
+  );
+
+  -- Triggers to keep memory_fts in sync with memories table
+  CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
+    INSERT INTO memory_fts(rowid, content) VALUES (new.id, new.content);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
+    INSERT INTO memory_fts(memory_fts, rowid, content) VALUES ('delete', old.id, old.content);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
+    INSERT INTO memory_fts(memory_fts, rowid, content) VALUES ('delete', old.id, old.content);
+    INSERT INTO memory_fts(rowid, content) VALUES (new.id, new.content);
+  END;
+
+  -- Indexes for common queries
+  CREATE INDEX IF NOT EXISTS idx_messages_session_id ON messages(session_id);
+  CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp);
+  CREATE INDEX IF NOT EXISTS idx_memories_project ON memories(project);
+  CREATE INDEX IF NOT EXISTS idx_memories_target ON memories(target);
+  CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project);
+  CREATE INDEX IF NOT EXISTS idx_sessions_started_at ON sessions(started_at);
+`;

package/src/store/session-indexer.ts

@@ -0,0 +1,153 @@
+import { DatabaseManager } from './db.js';
+import { parseSessionFile, getSessionFiles, type ParsedSession } from './session-parser.js';
+
+/**
+ * Index result for a single session.
+ */
+export interface IndexResult {
+  sessionId: string;
+  messagesIndexed: number;
+  skipped: boolean; // true if already indexed
+}
+
+/**
+ * Bulk index result.
+ */
+export interface BulkIndexResult {
+  sessionsProcessed: number;
+  sessionsIndexed: number;
+  sessionsSkipped: number;
+  messagesIndexed: number;
+  errors: string[];
+}
+
+/**
+ * Index a single session into the database.
+ *
+ * @returns IndexResult with count of messages indexed
+ */
+export function indexSession(dbManager: DatabaseManager, session: ParsedSession): IndexResult {
+  const db = dbManager.getDb();
+
+  // Check if already indexed
+  const existing = db.prepare('SELECT id FROM sessions WHERE id = ?').get(session.id) as { id: string } | undefined;
+  if (existing) {
+    return { sessionId: session.id, messagesIndexed: 0, skipped: true };
+  }
+
+  // Insert session
+  db.prepare(`
+    INSERT INTO sessions (id, project, cwd, started_at, ended_at, message_count)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `).run(
+    session.id,
+    session.project,
+    session.cwd,
+    session.startedAt,
+    session.endedAt,
+    session.messages.length
+  );
+
+  // Insert messages in a transaction for performance
+  const insertMsg = db.prepare(`
+    INSERT INTO messages (id, session_id, role, content, timestamp, tool_calls)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `);
+
+  const insertMany = db.transaction((messages: ParsedSession['messages']) => {
+    for (const msg of messages) {
+      insertMsg.run(
+        msg.id,
+        session.id,
+        msg.role,
+        msg.content,
+        msg.timestamp,
+        msg.toolCalls ? JSON.stringify(msg.toolCalls) : null
+      );
+    }
+  });
+
+  insertMany(session.messages);
+
+  return { sessionId: session.id, messagesIndexed: session.messages.length, skipped: false };
+}
+
+/**
+ * Index all sessions from disk.
+ *
+ * @param dbManager — Database manager instance
+ * @param sessionsDir — Path to ~/.pi/agent/sessions/
+ * @param projectDir — Optional: specific project directory to index
+ * @returns Bulk index result
+ */
+export function indexAllSessions(
+  dbManager: DatabaseManager,
+  sessionsDir: string,
+  projectDir?: string
+): BulkIndexResult {
+  const files = getSessionFiles(sessionsDir, projectDir);
+  const result: BulkIndexResult = {
+    sessionsProcessed: 0,
+    sessionsIndexed: 0,
+    sessionsSkipped: 0,
+    messagesIndexed: 0,
+    errors: [],
+  };
+
+  for (const file of files) {
+    result.sessionsProcessed++;
+
+    try {
+      const session = parseSessionFile(file);
+      if (!session) {
+        result.errors.push(`Failed to parse: ${file}`);
+        continue;
+      }
+
+      const indexResult = indexSession(dbManager, session);
+      if (indexResult.skipped) {
+        result.sessionsSkipped++;
+      } else {
+        result.sessionsIndexed++;
+        result.messagesIndexed += indexResult.messagesIndexed;
+      }
+    } catch (err) {
+      result.errors.push(`Error indexing ${file}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Get statistics about indexed sessions.
+ */
+export function getSessionStats(dbManager: DatabaseManager): {
+  totalSessions: number;
+  totalMessages: number;
+  projects: { project: string; sessions: number; messages: number }[];
+} {
+  const db = dbManager.getDb();
+
+  const totals = db.prepare(`
+    SELECT
+      (SELECT COUNT(*) FROM sessions) as sessions,
+      (SELECT COUNT(*) FROM messages) as messages
+  `).get() as { sessions: number; messages: number };
+
+  const projects = db.prepare(`
+    SELECT
+      project,
+      COUNT(*) as sessions,
+      (SELECT COUNT(*) FROM messages m WHERE m.session_id IN (SELECT id FROM sessions s2 WHERE s2.project = s.project)) as messages
+    FROM sessions s
+    GROUP BY project
+    ORDER BY sessions DESC
+  `).all() as { project: string; sessions: number; messages: number }[];
+
+  return {
+    totalSessions: totals.sessions,
+    totalMessages: totals.messages,
+    projects,
+  };
+}

package/src/store/session-parser.ts

@@ -0,0 +1,214 @@
+import fs from 'node:fs';
+
+/**
+ * Parsed session data from a JSONL file.
+ */
+export interface ParsedSession {
+  id: string;
+  project: string;
+  cwd: string;
+  startedAt: string;
+  endedAt: string | null;
+  messages: ParsedMessage[];
+}
+
+/**
+ * A single parsed message from a session.
+ */
+export interface ParsedMessage {
+  id: string;
+  role: 'user' | 'assistant' | 'system';
+  content: string;
+  timestamp: string;
+  toolCalls?: string[];
+}
+
+/**
+ * Raw JSONL entry types.
+ */
+interface JsonlEntry {
+  type: string;
+  id?: string;
+  parentId?: string | null;
+  timestamp?: string;
+  cwd?: string;
+  message?: {
+    role?: string;
+    content?: unknown;
+    timestamp?: number;
+  };
+  customType?: string;
+  [key: string]: unknown;
+}
+
+/**
+ * Extract text content from a message's content array.
+ */
+function extractTextContent(content: unknown): string {
+  if (typeof content === 'string') return content;
+  if (!Array.isArray(content)) return '';
+
+  const parts: string[] = [];
+  for (const block of content) {
+    if (!block || typeof block !== 'object') continue;
+    const b = block as Record<string, unknown>;
+
+    switch (b.type) {
+      case 'text':
+        if (typeof b.text === 'string') parts.push(b.text);
+        break;
+      case 'thinking':
+        // Skip thinking blocks — they're internal reasoning
+        break;
+      case 'tool_use':
+        // Skip tool_use blocks — we track tool calls separately
+        break;
+      case 'tool_result':
+        // Include tool result text if present
+        if (typeof b.content === 'string') {
+          parts.push(b.content);
+        } else if (Array.isArray(b.content)) {
+          for (const item of b.content) {
+            if (item && typeof item === 'object' && (item as Record<string, unknown>).type === 'text') {
+              parts.push((item as Record<string, unknown>).text as string);
+            }
+          }
+        }
+        break;
+    }
+  }
+  return parts.join('\n').trim();
+}
+
+/**
+ * Extract tool call names from a message's content array.
+ */
+function extractToolCalls(content: unknown): string[] | undefined {
+  if (!Array.isArray(content)) return undefined;
+
+  const toolNames: string[] = [];
+  for (const block of content) {
+    if (!block || typeof block !== 'object') continue;
+    const b = block as Record<string, unknown>;
+    if (b.type === 'tool_use' && typeof b.name === 'string') {
+      toolNames.push(b.name);
+    }
+  }
+  return toolNames.length > 0 ? toolNames : undefined;
+}
+
+/**
+ * Parse a Pi session JSONL file.
+ *
+ * @param filePath — Path to the .jsonl file
+ * @returns Parsed session data, or null if the file is invalid
+ */
+export function parseSessionFile(filePath: string): ParsedSession | null {
+  const content = fs.readFileSync(filePath, 'utf-8');
+  const lines = content.split('\n').filter(line => line.trim());
+
+  if (lines.length === 0) return null;
+
+  let sessionId: string | null = null;
+  let sessionCwd: string | null = null;
+  let sessionTimestamp: string | null = null;
+  const messages: ParsedMessage[] = [];
+
+  for (const line of lines) {
+    let entry: JsonlEntry;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue; // Skip malformed lines
+    }
+
+    switch (entry.type) {
+      case 'session':
+        sessionId = entry.id ?? null;
+        sessionCwd = entry.cwd ?? null;
+        sessionTimestamp = entry.timestamp ?? null;
+        break;
+
+      case 'message': {
+        if (!entry.message || !entry.id || !entry.timestamp) break;
+
+        const role = entry.message.role;
+        if (role !== 'user' && role !== 'assistant' && role !== 'system') break;
+
+        const textContent = extractTextContent(entry.message.content);
+        if (!textContent) break; // Skip empty messages
+
+        const toolCalls = role === 'assistant' ? extractToolCalls(entry.message.content) : undefined;
+
+        messages.push({
+          id: entry.id,
+          role,
+          content: textContent,
+          timestamp: entry.timestamp,
+          toolCalls,
+        });
+        break;
+      }
+      // Skip other entry types (model_change, thinking_level_change, custom, etc.)
+    }
+  }
+
+  if (!sessionId || !sessionCwd || !sessionTimestamp) return null;
+
+  // Decode project name from cwd-encoded directory name
+  // The directory is named like "--Users-chandrateja-Documents-pi-hermes-memory--"
+  // We extract the last segment as the project name
+  const project = sessionCwd.split('/').pop() ?? sessionCwd;
+
+  return {
+    id: sessionId,
+    project,
+    cwd: sessionCwd,
+    startedAt: sessionTimestamp,
+    endedAt: null, // We don't know when it ended from the JSONL
+    messages,
+  };
+}
+
+/**
+ * Get all session JSONL files for a project (or all projects).
+ *
+ * @param sessionsDir — Path to ~/.pi/agent/sessions/
+ * @param projectDir — Optional: specific project directory name (e.g., "--Users-...--")
+ * @returns Array of file paths
+ */
+export function getSessionFiles(sessionsDir: string, projectDir?: string): string[] {
+  if (projectDir) {
+    const dir = `${sessionsDir}/${projectDir}`;
+    if (!fs.existsSync(dir)) return [];
+    return fs.readdirSync(dir)
+      .filter(f => f.endsWith('.jsonl'))
+      .map(f => `${dir}/${f}`);
+  }
+
+  // All projects
+  if (!fs.existsSync(sessionsDir)) return [];
+  const files: string[] = [];
+  for (const dir of fs.readdirSync(sessionsDir)) {
+    const dirPath = `${sessionsDir}/${dir}`;
+    if (!fs.statSync(dirPath).isDirectory()) continue;
+    for (const f of fs.readdirSync(dirPath)) {
+      if (f.endsWith('.jsonl')) {
+        files.push(`${dirPath}/${f}`);
+      }
+    }
+  }
+  return files;
+}
+
+/**
+ * Decode a project directory name to a human-readable project name.
+ * "--Users-chandrateja-Documents-pi-hermes-memory--" → "pi-hermes-memory"
+ */
+export function decodeProjectDir(dirName: string): string {
+  // Remove leading/trailing dashes
+  const cleaned = dirName.replace(/^-+|-+$/g, '');
+  // Split by dash and take the last segment (project name)
+  const segments = cleaned.split('-');
+  return segments[segments.length - 1] ?? cleaned;
+}