pi-memory-stone 0.1.0

package/src/db/schema.ts

@@ -0,0 +1,161 @@
+/**
+ * Database schema for pi-memory-stone.
+ * Versioned migrations applied sequentially.
+ */
+
+export const SCHEMA_VERSION = 1;
+
+export interface Migration {
+  version: number;
+  name: string;
+  sql: string;
+}
+
+export const MIGRATIONS: Migration[] = [
+  {
+    version: 1,
+    name: "initial-schema",
+    sql: `
+      -- Track indexed sessions
+      CREATE TABLE IF NOT EXISTS sessions (
+        id TEXT PRIMARY KEY,
+        session_file TEXT UNIQUE,
+        cwd TEXT,
+        repo_root TEXT,
+        project_id TEXT,
+        session_name TEXT,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL,
+        source_status TEXT NOT NULL DEFAULT 'active',
+        file_mtime INTEGER,
+        file_size INTEGER,
+        schema_version INTEGER NOT NULL DEFAULT 1
+      );
+
+      -- Structured memory records
+      CREATE TABLE IF NOT EXISTS records (
+        id TEXT PRIMARY KEY,
+        kind TEXT NOT NULL,
+        scope TEXT NOT NULL DEFAULT 'project',
+        project_id TEXT,
+        session_id TEXT,
+        session_file TEXT,
+        branch_leaf_id TEXT,
+        entry_id_start TEXT,
+        entry_id_end TEXT,
+        text TEXT NOT NULL,
+        tags TEXT,
+        status TEXT NOT NULL DEFAULT 'active',
+        confidence REAL NOT NULL DEFAULT 1.0,
+        importance REAL NOT NULL DEFAULT 0.5,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL,
+        superseded_by TEXT,
+        derived_from_memory_refs TEXT
+      );
+
+      -- Full-text search index (contentless — manually synced)
+      CREATE VIRTUAL TABLE IF NOT EXISTS record_fts USING fts5(
+        text,
+        tags
+      );
+
+      -- File activity tracking
+      CREATE TABLE IF NOT EXISTS file_activity (
+        id TEXT PRIMARY KEY,
+        record_id TEXT,
+        project_id TEXT,
+        path TEXT NOT NULL,
+        action TEXT NOT NULL,
+        entry_id TEXT
+      );
+
+      -- Injection audit log
+      CREATE TABLE IF NOT EXISTS injections (
+        id TEXT PRIMARY KEY,
+        session_id TEXT,
+        turn_entry_id TEXT,
+        prompt_hash TEXT,
+        injected_refs TEXT,
+        packet TEXT,
+        reasons TEXT,
+        created_at INTEGER NOT NULL
+      );
+
+      -- Per-session-file indexing progress
+      CREATE TABLE IF NOT EXISTS index_state (
+        session_file TEXT PRIMARY KEY,
+        session_id TEXT,
+        last_indexed_entry_id TEXT,
+        last_indexed_entry_timestamp TEXT,
+        file_mtime INTEGER,
+        file_size INTEGER,
+        branch_leaf_id TEXT,
+        schema_version INTEGER NOT NULL DEFAULT 1
+      );
+
+      -- Background job queue
+      CREATE TABLE IF NOT EXISTS jobs (
+        id TEXT PRIMARY KEY,
+        type TEXT NOT NULL,
+        payload TEXT,
+        status TEXT NOT NULL DEFAULT 'pending',
+        attempts INTEGER NOT NULL DEFAULT 0,
+        last_error TEXT,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL
+      );
+
+      -- Schema migration tracking
+      CREATE TABLE IF NOT EXISTS schema_migrations (
+        version INTEGER PRIMARY KEY,
+        applied_at INTEGER NOT NULL,
+        name TEXT NOT NULL
+      );
+
+      -- Indexes
+      CREATE INDEX IF NOT EXISTS idx_records_project ON records(project_id);
+      CREATE INDEX IF NOT EXISTS idx_records_kind ON records(kind);
+      CREATE INDEX IF NOT EXISTS idx_records_scope ON records(scope);
+      CREATE INDEX IF NOT EXISTS idx_records_session ON records(session_id);
+      CREATE INDEX IF NOT EXISTS idx_records_status ON records(status);
+      CREATE INDEX IF NOT EXISTS idx_file_activity_record ON file_activity(record_id);
+      CREATE INDEX IF NOT EXISTS idx_file_activity_path ON file_activity(path);
+      CREATE INDEX IF NOT EXISTS idx_file_activity_project ON file_activity(project_id);
+      CREATE INDEX IF NOT EXISTS idx_injections_session ON injections(session_id);
+    `,
+  },
+];
+
+/** Record kinds */
+export const RECORD_KINDS = [
+  "session_summary",
+  "turn_summary",
+  "decision",
+  "preference",
+  "task",
+  "error_resolution",
+  "file_activity",
+] as const;
+
+export type RecordKind = (typeof RECORD_KINDS)[number];
+
+/** Record scopes */
+export const RECORD_SCOPES = ["project", "global"] as const;
+export type RecordScope = (typeof RECORD_SCOPES)[number];
+
+/** Record statuses */
+export const RECORD_STATUSES = ["active", "soft_forgotten", "hard_forgotten", "superseded"] as const;
+export type RecordStatus = (typeof RECORD_STATUSES)[number];
+
+/** Source statuses */
+export const SOURCE_STATUSES = ["active", "missing", "archived"] as const;
+export type SourceStatus = (typeof SOURCE_STATUSES)[number];
+
+/** Job statuses */
+export const JOB_STATUSES = ["pending", "running", "done", "failed"] as const;
+export type JobStatus = (typeof JOB_STATUSES)[number];
+
+/** File action types */
+export const FILE_ACTIONS = ["read", "write", "edit", "bash", "delete"] as const;
+export type FileAction = (typeof FILE_ACTIONS)[number];

package/src/index.ts

@@ -0,0 +1,197 @@
+/**
+ * pi-memory-stone — Global pi extension for session memory.
+ *
+ * Preserves and retrieves useful memory across pi sessions.
+ * Builds a searchable SQLite+FTS5 index with backreferences to session entries.
+ *
+ * Vertical slice MVP:
+ *  - SQLite schema + migrations
+ *  - Deterministic turn_summary and file_activity capture on agent_end
+ *  - FTS5 search
+ *  - /memory-status, /memory-search, /memory-last commands
+ *  - memory_search, memory_open, memory_remember, memory_forget tools
+ *  - Conservative same-project before_agent_start injection
+ */
+
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { registerCommands } from "./commands/index.js";
+import { registerTools } from "./tools/index.js";
+import { indexSessionOnAgentEnd } from "./indexing/index.js";
+import { retrieve, buildInjectionPacket, formatInjectionForLlm } from "./retrieval/index.js";
+import { getProjectId, getConfig, clearProjectCache } from "./config/index.js";
+import { closeDb } from "./db/index.js";
+import { createHash } from "node:crypto";
+
+// ─── Session-scoped state ───────────────────────────────────────────
+
+/** Track injected refs per turn to prevent feedback loops */
+const injectedRefsThisSession: Set<string> = new Set();
+
+/** Whether memory injection is temporarily disabled for this session */
+let sessionEnabled = true;
+
+// ─── Extension entry point ─────────────────────────────────────────
+
+export default function (pi: ExtensionAPI) {
+  // ── Register commands ──────────────────────────────────────────
+
+  registerCommands(pi);
+
+  // ── Register tools ─────────────────────────────────────────────
+
+  registerTools(pi);
+
+  // ── agent_end: index session turn ──────────────────────────────
+
+  pi.on("agent_end", async (event, ctx) => {
+    try {
+      const { recordsCreated, errors } = await indexSessionOnAgentEnd(ctx, event);
+
+      if (errors.length > 0 && ctx.hasUI) {
+        // Log errors but don't flood UI
+        for (const err of errors.slice(0, 2)) {
+          console.error("[pi-memory-stone] indexing error:", err);
+        }
+      }
+
+      // Optionally notify on first indexing
+      if (recordsCreated > 0 && ctx.hasUI) {
+        // Silent by default; uncomment for verbose mode:
+        // ctx.ui.setStatus("memory-stone", `Indexed ${recordsCreated} records`);
+      }
+    } catch (err) {
+      console.error("[pi-memory-stone] agent_end handler error:", err);
+    }
+  });
+
+  // ── before_agent_start: inject relevant memories ───────────────
+
+  pi.on("before_agent_start", async (event, ctx) => {
+    try {
+      // Check if memory is enabled
+      const config = getConfig(ctx.cwd);
+      if (!config.enabled) return;
+
+      // Check for session toggle entries before honoring the cached toggle so
+      // /memory-on can re-enable injection after /memory-off in the same session.
+      for (const entry of ctx.sessionManager.getBranch()) {
+        if (
+          entry.type === "custom" &&
+          entry.customType === "memory-stone:session-toggle"
+        ) {
+          const data = entry.data as { enabled?: boolean } | undefined;
+          if (typeof data?.enabled === "boolean") {
+            sessionEnabled = data.enabled;
+          }
+        }
+      }
+
+      if (!sessionEnabled) return;
+
+      const prompt = event.prompt || "";
+      if (!prompt.trim()) return;
+
+      // Hash prompt to detect repeated injections
+      const promptHash = createHash("sha256").update(prompt).digest("hex").slice(0, 12);
+
+      const projectId = getProjectId(ctx.cwd);
+
+      // Build search query
+      const results = retrieve(prompt, projectId, [], {
+        limit: config.maxInjectedRecords,
+        crossProjectEnabled: config.crossProjectEnabled,
+      });
+
+      // Filter: skip already-injected refs
+      const newResults = results.filter((r) => !injectedRefsThisSession.has(r.record.id));
+
+      // Score threshold
+      const thresholdResults = newResults.filter((r) => r.score >= config.scoreThreshold);
+
+      if (thresholdResults.length === 0) return;
+
+      // Build and format injection packet
+      const packet = buildInjectionPacket(thresholdResults);
+      const formatted = formatInjectionForLlm(packet, config.maxInjectedTokens);
+
+      // Track injected refs (prevent feedback loop)
+      for (const r of thresholdResults) {
+        injectedRefsThisSession.add(r.record.id);
+      }
+
+      // Log injection to DB
+      const { insertInjection } = await import("./db/index.js");
+      insertInjection({
+        session_id: ctx.sessionManager.getSessionId(),
+        turn_entry_id: ctx.sessionManager.getLeafId() ?? undefined,
+        prompt_hash: promptHash,
+        injected_refs: thresholdResults.map((r) => r.record.id).join(","),
+        packet: formatted,
+        reasons: thresholdResults.map((r) => r.reasons.join(";")).join(" | "),
+      });
+
+      // Inject as a non-context audit custom entry (separate from LLM context)
+      // but also as a system prompt addition for the LLM
+      const systemPromptAddition = [
+        "",
+        "--- Memory Stone Context ---",
+        formatted,
+        "--- End Memory Stone Context ---",
+      ].join("\n");
+
+      return {
+        systemPrompt: (event.systemPrompt || "") + systemPromptAddition,
+      };
+    } catch (err) {
+      console.error("[pi-memory-stone] before_agent_start handler error:", err);
+    }
+  });
+
+  // ── session_start: restore state ───────────────────────────────
+
+  pi.on("session_start", async (_event, ctx) => {
+    try {
+      // Clear session-scoped state
+      injectedRefsThisSession.clear();
+      sessionEnabled = true;
+
+      // Restore session toggle from branch
+      for (const entry of ctx.sessionManager.getBranch()) {
+        if (
+          entry.type === "custom" &&
+          entry.customType === "memory-stone:session-toggle"
+        ) {
+          const data = entry.data as { enabled?: boolean } | undefined;
+          if (typeof data?.enabled === "boolean") {
+            sessionEnabled = data.enabled;
+          }
+        }
+      }
+
+      // Clear project ID cache on session change
+      clearProjectCache();
+    } catch (err) {
+      console.error("[pi-memory-stone] session_start handler error:", err);
+    }
+  });
+
+  // ── session_shutdown: cleanup ──────────────────────────────────
+
+  pi.on("session_shutdown", async (_event, _ctx) => {
+    try {
+      // Best-effort flush — DB is WAL mode so writes are already durable.
+      // Cleanup any pending state.
+      injectedRefsThisSession.clear();
+    } catch (err) {
+      console.error("[pi-memory-stone] session_shutdown handler error:", err);
+    }
+  });
+
+  // ── Cleanup when extension is unloaded ─────────────────────────
+
+  // Note: pi doesn't have an explicit unload hook, but session_shutdown
+  // fires before reload. For process exit, Node handles file descriptor cleanup.
+  process.on("exit", () => {
+    closeDb();
+  });
+}

package/src/indexing/index.ts

@@ -0,0 +1,207 @@
+/**
+ * Indexing engine: processes session entries on agent_end and stores structured records.
+ * Deterministic-only for MVP; LLM extraction deferred.
+ */
+
+import { statSync } from "node:fs";
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import {
+  upsertSession,
+  upsertRecord,
+  insertFileActivity,
+  getIndexState,
+  upsertIndexState,
+  contentHash,
+} from "../db/index.js";
+import { parseEntries, turnsToRecords, type ParsedFileActivity } from "./parser.js";
+import { getProjectId } from "../config/index.js";
+
+// ─── Types for agent_end data ───────────────────────────────────────
+
+type AgentEndEvent = unknown;
+
+// ─── Index a session from agent_end ─────────────────────────────────
+
+export async function indexSessionOnAgentEnd(
+  ctx: ExtensionContext,
+  _event: AgentEndEvent,
+): Promise<{ recordsCreated: number; errors: string[] }> {
+  const errors: string[] = [];
+  let recordsCreated = 0;
+
+  try {
+    const sessionManager = ctx.sessionManager;
+    const sessionFile = sessionManager.getSessionFile();
+
+    // Non-persisted sessions: skip indexing (plan: do not index ephemeral)
+    if (!sessionFile) {
+      return { recordsCreated: 0, errors: [] };
+    }
+
+    const sessionId = sessionManager.getSessionId();
+    const projectId = getProjectId(ctx.cwd);
+    const cwd = ctx.cwd;
+    const branch = sessionManager.getBranch() as unknown as Parameters<typeof parseEntries>[0];
+    const leafId = sessionManager.getLeafId();
+
+    // Get file info
+    let fileMtime: number | null = null;
+    let fileSize: number | null = null;
+    try {
+      const stat = statSync(sessionFile);
+      fileMtime = stat.mtimeMs;
+      fileSize = stat.size;
+    } catch {
+      // File may not exist (in-memory session), that's fine
+    }
+
+    // Check index state
+    const prevState = getIndexState(sessionFile);
+    const lastIndexedId = prevState?.last_indexed_entry_id ?? null;
+
+    // Find new entries since last index. Prefer timestamps so this stays correct
+    // even if SessionManager branch ordering changes; fall back to root-to-leaf slicing.
+    const lastIndexedTimestamp = prevState?.last_indexed_entry_timestamp
+      ? Date.parse(prevState.last_indexed_entry_timestamp)
+      : NaN;
+    const lastIndexedIndex = lastIndexedId
+      ? branch.findIndex((entry) => entry.id === lastIndexedId)
+      : -1;
+    let newEntries = branch;
+    if (lastIndexedId) {
+      if (Number.isFinite(lastIndexedTimestamp)) {
+        const timestampEntries = branch.filter((entry) => {
+          const ts = entry.timestamp ? Date.parse(entry.timestamp) : NaN;
+          return Number.isFinite(ts) && ts > lastIndexedTimestamp;
+        });
+        newEntries = timestampEntries.length > 0 || lastIndexedIndex < 0
+          ? timestampEntries
+          : branch.slice(lastIndexedIndex + 1);
+      } else {
+        newEntries = lastIndexedIndex >= 0 ? branch.slice(lastIndexedIndex + 1) : branch;
+      }
+    }
+
+    if (newEntries.length === 0) {
+      // Update session metadata even if no new entries
+      upsertSession({
+        id: sessionId,
+        session_file: sessionFile,
+        cwd,
+        project_id: projectId,
+        file_mtime: fileMtime,
+        file_size: fileSize,
+      });
+      upsertIndexState({
+        session_file: sessionFile,
+        session_id: sessionId,
+        last_indexed_entry_id: prevState?.last_indexed_entry_id ?? undefined,
+        last_indexed_entry_timestamp: new Date().toISOString(),
+        file_mtime: fileMtime,
+        file_size: fileSize,
+        branch_leaf_id: leafId ?? undefined,
+      });
+      return { recordsCreated: 0, errors: [] };
+    }
+
+    // Parse new entries
+    const { turns, fileActivities } = parseEntries(newEntries);
+
+    // Convert to records
+    const recordPayloads = turnsToRecords(turns, projectId, sessionId, sessionFile);
+
+    // Upsert session
+    upsertSession({
+      id: sessionId,
+      session_file: sessionFile,
+      cwd,
+      project_id: projectId,
+      file_mtime: fileMtime,
+      file_size: fileSize,
+    });
+
+    // Store records
+    for (const payload of recordPayloads) {
+      try {
+        const recordId = upsertRecord({
+          kind: payload.kind,
+          scope: payload.scope,
+          project_id: projectId,
+          session_id: sessionId,
+          session_file: sessionFile,
+          branch_leaf_id: leafId ?? undefined,
+          entry_id_start: payload.entryIdStart,
+          entry_id_end: payload.entryIdEnd,
+          text: payload.text,
+          tags: payload.tags,
+        });
+        recordsCreated++;
+      } catch (err) {
+        errors.push(`Failed to store record: ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+
+    // Store file activity
+    for (const fa of fileActivities) {
+      try {
+        insertFileActivity({
+          record_id: contentHash(
+            `file:${fa.path}:${fa.action}`,
+            "file_activity",
+          ),
+          project_id: projectId,
+          path: fa.path,
+          action: fa.action,
+          entry_id: fa.entryId,
+        });
+      } catch (err) {
+        // Non-critical: file activity insertion failure is logged but not fatal
+        errors.push(`Failed to store file activity: ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+
+    // Update index state
+    const lastEntry = newEntries.reduce((latest, entry) => {
+      const latestTs = latest?.timestamp ? Date.parse(latest.timestamp) : NaN;
+      const entryTs = entry.timestamp ? Date.parse(entry.timestamp) : NaN;
+      if (!Number.isFinite(latestTs) && !Number.isFinite(entryTs)) return entry;
+      if (!Number.isFinite(latestTs)) return entry;
+      if (!Number.isFinite(entryTs)) return latest;
+      return entryTs > latestTs ? entry : latest;
+    }, newEntries[0]);
+    upsertIndexState({
+      session_file: sessionFile,
+      session_id: sessionId,
+      last_indexed_entry_id: lastEntry?.id ?? prevState?.last_indexed_entry_id ?? undefined,
+      last_indexed_entry_timestamp: lastEntry?.timestamp ?? new Date().toISOString(),
+      file_mtime: fileMtime,
+      file_size: fileSize,
+      branch_leaf_id: leafId ?? undefined,
+    });
+
+  } catch (err) {
+    errors.push(`Indexing failed: ${err instanceof Error ? err.message : String(err)}`);
+  }
+
+  return { recordsCreated, errors };
+}
+
+// ─── Backfill (for /memory-backfill command) ────────────────────────
+
+export async function backfillSession(
+  sessionFile: string,
+  ctx: ExtensionContext,
+): Promise<{ recordsCreated: number; errors: string[] }> {
+  const errors: string[] = [];
+  let recordsCreated = 0;
+
+  try {
+    // For backfill we need raw JSONL access. We use SessionManager.open for that.
+    // But in the MVP, we process via agent_end. Backfill is deferred.
+    // This is a placeholder for future implementation.
+  } catch (err) {
+    errors.push(`Backfill failed: ${err instanceof Error ? err.message : String(err)}`);
+  }
+
+  return { recordsCreated, errors };
+}