pi-memory-stone 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 pi-memory-stone
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,262 @@
+# pi-memory-stone
+
+A global pi extension that preserves and retrieves useful memory across pi sessions. Raw pi session JSONL files remain the source of truth; the extension builds a searchable, privacy-safe SQLite+FTS5 index with backreferences to exact session entries.
+
+## Status
+
+**MVP vertical slice** — deterministic indexing, FTS5 search, conservative injection, commands, tools, and tests are complete. LLM extraction, embeddings, and historical backfill are deferred.
+
+## Quick Start
+
+Install globally as a pi package:
+
+```bash
+pi install git:github.com/nikolasp/pi-memory-stone
+```
+
+Or install manually in pi's global extension directory:
+
+```bash
+git clone https://github.com/nikolasp/pi-memory-stone ~/.pi/agent/extensions/pi-memory-stone
+```
+
+Then restart pi or run `/reload`, and verify it is active:
+
+```bash
+/memory-status
+```
+
+> Scope note: `pi install -l ...` / `pi install --local ...` writes to the current project's `.pi/settings.json` and only loads there. For all projects, run `pi install git:github.com/nikolasp/pi-memory-stone` without `--local` (user settings) or use `~/.pi/agent/extensions/`.
+
+## Architecture
+
+```
+~/.pi/agent/memory/memory.db      # SQLite + FTS5 (WAL mode)
+~/.pi/agent/extensions/pi-memory-stone/
+├── src/
+│   ├── index.ts                  # Entry point: hooks, lifecycle
+│   ├── db/                       # SQLite connection, migrations, CRUD
+│   ├── indexing/                 # Deterministic JSONL parser, agent_end handler
+│   ├── retrieval/                # FTS search, hybrid ranking, injection builder
+│   ├── commands/                 # /memory-* slash commands
+│   ├── tools/                    # LLM-callable tools
+│   ├── privacy/                  # Secret redaction, sensitive path filtering
+│   └── config/                   # Project identity, settings
+└── test/                         # 44 tests across 4 suites
+```
+
+## How It Works
+
+### 1. Indexing (agent_end)
+
+Every time the agent finishes a response, the extension:
+
+- Reads new session entries since the last indexed position
+- Parses turns deterministically (no LLM): extracts user prompts, assistant responses, tool calls, errors
+- Redacts secrets (API keys, tokens, passwords, private keys) before storage
+- Skips sensitive files (`.env`, keys, certs, `node_modules`, `.git`)
+- Stores structured `turn_summary` and `error_resolution` records in SQLite
+- Maintains `index_state` per session file so indexing is incremental
+
+### 2. Retrieval (before_agent_start)
+
+Before each agent turn, the extension:
+
+- Builds a focused query from the user's prompt
+- Searches records via FTS5
+- Ranks by hybrid score: FTS match + same-project boost + recency decay + kind weight + confidence
+- Injects top results (max 5, threshold-limited) as system prompt context
+- Tracks injected refs to prevent feedback loops
+- Logs every injection for audit via `/memory-last`
+
+### 3. Storage Model
+
+**Records:**
+| Kind | Description |
+|---|---|
+| `turn_summary` | Concatenated user prompt + assistant response + tools used |
+| `error_resolution` | Tool errors with context |
+| `decision` | Explicitly remembered decisions (LLM/tool) |
+| `preference` | User preferences (LLM/tool) |
+| `task` | Tracked tasks (LLM/tool) |
+| `session_summary` | Session-level summary (future) |
+
+**Scopes:**
+| Scope | Visibility |
+|---|---|
+| `project` | Visible within the same project (git repo root) |
+| `global` | Visible across all projects (explicit opt-in only) |
+
+**Statuses:** `active`, `soft_forgotten`, `hard_forgotten`, `superseded`
+
+## Commands
+
+| Command | Alias | Description |
+|---|---|---|
+| `/memory-status` | `/stone-status` | Show index statistics, record counts by kind, config |
+| `/memory-status --verbose` | | Include per-kind record breakdown |
+| `/memory-search <query>` | `/stone-search` | Search memory for relevant records |
+| `/memory-last` | `/stone-last` | Show the last memory injection packet |
+| `/memory-forget <id>` | `/stone-forget` | Soft-forget a record (hide from searches) |
+| `/memory-forget <id> --hard` | | Permanently delete (with confirmation) |
+| `/memory-on` | | Enable memory injection for this session |
+| `/memory-off` | | Disable memory injection for this session |
+
+## Tools
+
+### `memory_search`
+
+Search memory stone for relevant records. Use before making decisions to recall past context.
+
+```
+Parameters:
+  query   Search query text
+  kind?   Filter: decision | preference | task | error_resolution | turn_summary | session_summary
+  scope?  Filter: project | global
+  limit?  Max results (default 5)
+```
+
+### `memory_open`
+
+Open a specific memory record by its reference ID.
+
+```
+Parameters:
+  ref     Memory record reference ID (from search results or injection packets)
+```
+
+### `memory_remember`
+
+Explicitly store a memory record. Only use when the user explicitly asks to remember something.
+
+```
+Parameters:
+  kind         Record kind: decision | preference | task | error_resolution | turn_summary | session_summary
+  text         Memory text to store
+  scope?       project (default) | global
+  tags?        Comma-separated tags
+  importance?  0-1 (default 0.5)
+```
+
+### `memory_forget`
+
+Soft-forget a memory record by its reference ID. Hard deletion requires explicit user confirmation via command.
+
+```
+Parameters:
+  ref     Memory record reference ID
+  hard?   Request permanent deletion (requires confirmation)
+```
+
+## Privacy & Safety
+
+### Secret Redaction
+
+All text is redacted before storage. Patterns covered:
+
+- OpenAI API keys (`sk-...`)
+- GitHub tokens (`ghp_...`, `ghs_...`)
+- AWS access keys (`AKIA...`)
+- JWT tokens
+- Generic API keys and secrets in `key=value` assignments
+- Private keys (PEM format)
+- Password/secrets in assignments
+- Connection strings (credentials removed)
+
+### Path Filtering
+
+Files at these paths are never indexed:
+
+- `.env*`, `.envrc`
+- Keys and certificates (`.pem`, `.key`, `.crt`)
+- SSH keys (`id_rsa`, `id_ed25519`, `.ssh/`)
+- AWS credentials (`~/.aws/`)
+- GPG keys (`~/.gnupg/`)
+- Dependency dirs (`node_modules`, `.git`, `dist`, `build`, `.next`, etc.)
+
+### Cross-Project Safety
+
+- Records default to project scope
+- Cross-project memory requires explicit global flag
+- Global promotion refuses sensitive patterns (paths, secrets, hostnames, implementation details)
+- `memory_open` never sends raw excerpts to LLM automatically
+
+## Ranking
+
+Results are ranked by a hybrid score:
+
+| Factor | Effect |
+|---|---|
+| FTS5 match quality | Base score (normalized reciprocal rank) |
+| Same project | ×1.5 boost |
+| Global scope | ×1.2 boost |
+| Kind weight | Decision ×1.5, Preference ×1.3, Error ×1.4 |
+| Recency | Exponential decay, half-life 7 days |
+| Confidence | Direct multiplier |
+| Importance | 0.5–1.5x multiplier |
+
+## DB Schema
+
+```sql
+sessions         — Indexed session metadata
+records          — Structured memory records (with FTS5 index)
+record_fts       — Full-text search index (contentless)
+file_activity    — File read/write/edit/bash tracking
+injections       — Audit log of memory injections
+index_state      — Per-session indexing progress
+jobs             — Background job queue
+schema_migrations— Versioned migration tracking
+```
+
+Storage: `~/.pi/agent/memory/memory.db` (SQLite, WAL mode, busy timeout 5s).
+
+## Tests
+
+```bash
+cd ~/.pi/agent/extensions/pi-memory-stone
+npm test
+npm run typecheck
+```
+
+These scripts are also runnable from a pi package clone installed from git; the required script runners are regular dependencies because pi package installs omit `devDependencies`.
+
+44 tests across 4 suites:
+
+| Suite | Tests | Focus |
+|---|---|---|
+| `indexing.test.ts` | 1 | Incremental session indexing |
+| `privacy.test.ts` | 17 | Secret redaction, sensitive path filtering |
+| `parser.test.ts` | 10 | Turn parsing, file activity detection, error extraction |
+| `ranking.test.ts` | 16 | Hybrid ranking, cross-project filtering, injection formatting |
+
+## Configuration
+
+Project settings in `.pi/settings.json`:
+
+```json
+{
+  "memory": {
+    "enabled": true,
+    "maxInjectedRecords": 5,
+    "maxInjectedTokens": 1000,
+    "scoreThreshold": 0.3,
+    "crossProjectEnabled": false
+  }
+}
+```
+
+## Deferred (Future Slices)
+
+- LLM extraction (decisions, preferences, tasks)
+- Historical backfill (`/memory-backfill`)
+- Embedding-based semantic search
+- `/memory-edit`, `/memory-supersede`
+- Rich TUI memory browser
+- Multi-machine sync
+- Daemon-mode background worker for LLM extraction
+- `/memory-prune-missing`
+- `.pi/memoryignore` and `~/.pi/agent/memoryignore`
+
+## License
+
+MIT — see [LICENSE](./LICENSE) for details.

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "pi-memory-stone",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Global pi extension: preserves and retrieves useful memory across pi sessions",
+  "license": "MIT",
+  "pi": {
+    "extensions": ["./src/index.ts"]
+  },
+  "scripts": {
+    "test": "NODE_OPTIONS='--experimental-sqlite' tsx --test test/*.test.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "tsx": "^4.22.3",
+    "typescript": "^5.9.3"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-coding-agent": "*",
+    "typebox": "*"
+  }
+}

package/src/commands/index.ts

@@ -0,0 +1,234 @@
+/**
+ * Commands: /memory-status, /memory-search, /memory-last
+ */
+
+import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
+import { getStats, getLastInjection } from "../db/index.js";
+import { retrieve } from "../retrieval/index.js";
+import { getProjectId, getConfig } from "../config/index.js";
+
+export function registerCommands(pi: ExtensionAPI): void {
+  // ── /memory-status ──────────────────────────────────────────────
+
+  pi.registerCommand("memory-status", {
+    description: "Show memory stone index statistics",
+    handler: async (args, ctx) => {
+      await handleMemoryStatus(args, ctx);
+    },
+  });
+
+  pi.registerCommand("stone-status", {
+    description: "Alias for /memory-status",
+    handler: async (args, ctx) => {
+      await handleMemoryStatus(args, ctx);
+    },
+  });
+
+  // ── /memory-search ──────────────────────────────────────────────
+
+  pi.registerCommand("memory-search", {
+    description: "Search memory stone for relevant records",
+    handler: async (args, ctx) => {
+      await handleMemorySearch(args, ctx);
+    },
+  });
+
+  pi.registerCommand("stone-search", {
+    description: "Alias for /memory-search",
+    handler: async (args, ctx) => {
+      await handleMemorySearch(args, ctx);
+    },
+  });
+
+  // ── /memory-last ────────────────────────────────────────────────
+
+  pi.registerCommand("memory-last", {
+    description: "Show the last memory injection packet",
+    handler: async (_args, ctx) => {
+      await handleMemoryLast(ctx);
+    },
+  });
+
+  pi.registerCommand("stone-last", {
+    description: "Alias for /memory-last",
+    handler: async (_args, ctx) => {
+      await handleMemoryLast(ctx);
+    },
+  });
+
+  // ── /memory-forget ──────────────────────────────────────────────
+
+  pi.registerCommand("memory-forget", {
+    description: "Soft-forget a memory record by ID. Use --hard for permanent deletion.",
+    handler: async (args, ctx) => {
+      await handleMemoryForget(args, ctx);
+    },
+  });
+
+  pi.registerCommand("stone-forget", {
+    description: "Alias for /memory-forget",
+    handler: async (args, ctx) => {
+      await handleMemoryForget(args, ctx);
+    },
+  });
+
+  // ── /memory-on / /memory-off ────────────────────────────────────
+
+  pi.registerCommand("memory-on", {
+    description: "Enable memory injection for this session",
+    handler: async (_args, ctx) => {
+      ctx.ui.notify("Memory injection enabled for this session", "info");
+      // Session toggle — stored in extension state
+      pi.appendEntry("memory-stone:session-toggle", { enabled: true });
+    },
+  });
+
+  pi.registerCommand("memory-off", {
+    description: "Disable memory injection for this session",
+    handler: async (_args, ctx) => {
+      ctx.ui.notify("Memory injection disabled for this session", "info");
+      pi.appendEntry("memory-stone:session-toggle", { enabled: false });
+    },
+  });
+}
+
+// ─── Handler implementations ────────────────────────────────────────
+
+async function handleMemoryStatus(args: string, ctx: ExtensionCommandContext): Promise<void> {
+  const verbose = args.includes("--verbose") || args.includes("-v");
+  const stats = getStats();
+
+  const lines: string[] = [];
+  lines.push("📊 Memory Stone Status");
+  lines.push("");
+  lines.push(`  Total active records: ${stats.totalRecords}`);
+  lines.push(`  Indexed sessions: ${stats.totalSessions}`);
+  lines.push(`  File activity entries: ${stats.totalFileActivity}`);
+  lines.push("");
+
+  if (verbose && Object.keys(stats.recordsByKind).length > 0) {
+    lines.push("  Records by kind:");
+    for (const [kind, count] of Object.entries(stats.recordsByKind)) {
+      lines.push(`    ${kind}: ${count}`);
+    }
+    lines.push("");
+  }
+
+  const config = getConfig(ctx.cwd);
+  lines.push("  Config:");
+  lines.push(`    enabled: ${config.enabled}`);
+  lines.push(`    maxInjectedRecords: ${config.maxInjectedRecords}`);
+  lines.push(`    maxInjectedTokens: ${config.maxInjectedTokens}`);
+  lines.push(`    crossProjectEnabled: ${config.crossProjectEnabled}`);
+
+  if (ctx.hasUI) {
+    ctx.ui.notify(lines.join("\n"), "info");
+  }
+}
+
+async function handleMemorySearch(args: string, ctx: ExtensionCommandContext): Promise<void> {
+  const query = args?.trim();
+
+  if (!query) {
+    ctx.ui.notify("Usage: /memory-search <query>", "warning");
+    return;
+  }
+
+  const projectId = getProjectId(ctx.cwd);
+  const config = getConfig(ctx.cwd);
+
+  const results = retrieve(query, projectId, [], {
+    limit: 20,
+    crossProjectEnabled: config.crossProjectEnabled,
+  });
+
+  if (results.length === 0) {
+    ctx.ui.notify("No matching memories found.", "info");
+    return;
+  }
+
+  const lines: string[] = [];
+  lines.push(`🔍 Memory search results for: "${query}"`);
+  lines.push("");
+
+  for (const [i, r] of results.entries()) {
+    const age = Date.now() - r.record.created_at;
+    const ageStr = age < 3600000
+      ? `${Math.floor(age / 60000)}m ago`
+      : age < 86400000
+        ? `${Math.floor(age / 3600000)}h ago`
+        : `${Math.floor(age / 86400000)}d ago`;
+
+    lines.push(`  ${i + 1}. [${r.record.kind}] ${r.record.id} (${ageStr}, score: ${r.score.toFixed(2)})`);
+    lines.push(`     ${r.record.text.slice(0, 120)}`);
+    lines.push("");
+  }
+
+  if (ctx.hasUI) {
+    ctx.ui.notify(lines.join("\n"), "info");
+  }
+}
+
+async function handleMemoryLast(ctx: ExtensionCommandContext): Promise<void> {
+  const sessionId = ctx.sessionManager.getSessionId();
+  const last = getLastInjection(sessionId);
+
+  if (!last) {
+    ctx.ui.notify("No memory injections in this session yet.", "info");
+    return;
+  }
+
+  const lines: string[] = [];
+  lines.push("📋 Last Memory Injection");
+  lines.push("");
+
+  if (last.packet) {
+    lines.push(last.packet);
+  } else {
+    lines.push(`  Injected refs: ${last.injected_refs ?? "none"}`);
+    lines.push(`  Reasons: ${last.reasons ?? "none"}`);
+    lines.push(`  Time: ${new Date(last.created_at).toISOString()}`);
+  }
+
+  if (ctx.hasUI) {
+    ctx.ui.notify(lines.join("\n"), "info");
+  }
+}
+
+async function handleMemoryForget(args: string, ctx: ExtensionCommandContext): Promise<void> {
+  const { softForgetRecord, hardDeleteRecord, getRecord } = await import("../db/index.js");
+
+  const parts = args?.trim().split(/\s+/) ?? [];
+  const hardFlag = parts.includes("--hard");
+  const refIds = parts.filter((p) => !p.startsWith("--"));
+
+  if (refIds.length === 0) {
+    ctx.ui.notify("Usage: /memory-forget <ref-id> [--hard]", "warning");
+    return;
+  }
+
+  for (const refId of refIds) {
+    const record = getRecord(refId);
+    if (!record) {
+      ctx.ui.notify(`Record ${refId} not found.`, "warning");
+      continue;
+    }
+
+    if (hardFlag) {
+      const confirmed = ctx.hasUI
+        ? await ctx.ui.confirm(
+            "Permanent deletion",
+            `Permanently delete memory record?\n\nKind: ${record.kind}\nText: ${record.text.slice(0, 100)}`,
+          )
+        : false;
+
+      if (confirmed) {
+        hardDeleteRecord(refId);
+        ctx.ui.notify(`Permanently deleted record ${refId}`, "info");
+      }
+    } else {
+      softForgetRecord(refId);
+      ctx.ui.notify(`Soft-forgotten record ${refId}`, "info");
+    }
+  }
+}

package/src/config/index.ts

@@ -0,0 +1,108 @@
+/**
+ * Configuration module.
+ * Project identity defaults to git repo root.
+ * Reads .pi/settings.json for memory.* config overrides.
+ */
+
+import { execSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+
+// ─── Project identity ───────────────────────────────────────────────
+
+let _gitRootCache: Map<string, string | null> = new Map();
+
+export function getProjectId(cwd: string): string | null {
+  // Use git repo root as project identity
+  if (_gitRootCache.has(cwd)) {
+    return _gitRootCache.get(cwd) ?? null;
+  }
+
+  try {
+    const root = execSync("git rev-parse --show-toplevel", {
+      cwd,
+      encoding: "utf8",
+      stdio: ["pipe", "pipe", "pipe"],
+    }).trim();
+    _gitRootCache.set(cwd, root);
+    return root;
+  } catch {
+    // Not a git repo; use cwd as fallback
+    _gitRootCache.set(cwd, cwd);
+    return cwd;
+  }
+}
+
+export function clearProjectCache(): void {
+  _gitRootCache.clear();
+}
+
+// ─── Config ─────────────────────────────────────────────────────────
+
+export interface MemoryConfig {
+  /** Whether memory injection is enabled */
+  enabled: boolean;
+  /** Max records to inject per turn */
+  maxInjectedRecords: number;
+  /** Max tokens for injected packet */
+  maxInjectedTokens: number;
+  /** Minimum score threshold for injection */
+  scoreThreshold: number;
+  /** Whether cross-project injection is enabled */
+  crossProjectEnabled: boolean;
+  /** Extra ignore patterns */
+  ignorePatterns: string[];
+}
+
+const DEFAULT_CONFIG: MemoryConfig = {
+  enabled: true,
+  maxInjectedRecords: 5,
+  maxInjectedTokens: 1000,
+  scoreThreshold: 0.3,
+  crossProjectEnabled: false,
+  ignorePatterns: [],
+};
+
+const _configCache: Map<string, MemoryConfig> = new Map();
+
+export function getConfig(cwd: string = process.cwd()): MemoryConfig {
+  const cacheKey = cwd;
+  const cached = _configCache.get(cacheKey);
+  if (cached) return cached;
+
+  // Try loading from project .pi/settings.json
+  const projectSettings = join(cwd, ".pi", "settings.json");
+
+  if (existsSync(projectSettings)) {
+    try {
+      const raw = readFileSync(projectSettings, "utf8");
+      const settings = JSON.parse(raw);
+      if (settings.memory) {
+        const config = { ...DEFAULT_CONFIG, ...settings.memory };
+        _configCache.set(cacheKey, config);
+        return config;
+      }
+    } catch {
+      // Invalid JSON, use defaults
+    }
+  }
+
+  const config = { ...DEFAULT_CONFIG };
+  _configCache.set(cacheKey, config);
+  return config;
+}
+
+export function reloadConfig(): void {
+  _configCache.clear();
+}
+
+// ─── Environment ────────────────────────────────────────────────────
+
+export function getHomeDir(): string {
+  return homedir() || "/tmp";
+}
+
+export function getMemoryDir(): string {
+  return `${getHomeDir()}/.pi/agent/memory`;
+}