pi-memory-stone 0.1.3 → 0.1.5

package/README.md

@@ -46,9 +46,10 @@ pi-memory-stone package source
 │   ├── retrieval/                # FTS search, hybrid ranking, injection builder
 │   ├── commands/                 # /memory-* slash commands
 │   ├── tools/                    # LLM-callable tools
+│   ├── vault/                    # Optional Obsidian-compatible markdown vaults
 │   ├── privacy/                  # Secret redaction, sensitive path filtering
 │   └── config/                   # Project identity, settings
-└── test/                         # 51 tests across 6 test files
+└── test/                         # tests across memory, privacy, portable, and vault helpers
 ```
 
 ## How It Works
@@ -116,6 +117,10 @@ Before each agent turn, the extension:
 | `/memory-import <memory-export.json> --preserve-project` | | Import records while preserving exported project IDs |
 | `/memory-import <memory-export.json> --global` | | Import all records as global memories |
 | `/memory-backup [path]` | `/stone-backup` | Copy the SQLite memory database to a timestamped backup file |
+| `/memory-vault-init [--project\|--personal]` | `/stone-vault-init` | Initialize an Obsidian-compatible markdown vault |
+| `/memory-vault-sync [--project\|--personal]` | `/stone-vault-sync` | Generate vault pages from active memory records |
+| `/memory-vault-status [--project\|--personal]` | `/stone-vault-status` | Show vault path, page counts, registry, and last sync |
+| `/memory-vault-capture-url <url> [--project\|--personal]` | `/stone-vault-capture-url` | Capture a web page into the vault as a source page |
 | `/memory-on` | | Enable memory injection for this session |
 | `/memory-off` | | Disable memory injection for this session |
 
@@ -188,6 +193,52 @@ Use a database backup before pruning or hard deletion:
 
 Import defaults are intentionally practical for machine moves: project-scoped records are remapped to the current project. Use `--preserve-project` to keep exported project IDs, or `--global` to import everything as global memory.
 
+## Knowledge Vaults
+
+Memory vaults are optional Obsidian-compatible markdown projections of active memory records. SQLite remains the source of truth; generated pages may be overwritten by `/memory-vault-sync`.
+
+```bash
+# Project-local vault, written only after explicit init or capture request
+/memory-vault-init --project
+/memory-vault-sync --project
+/memory-vault-status --project
+
+# Capture a web page source into the vault
+/memory-vault-capture-url https://example.com/article --project
+
+# Capture resolves known source formats before generic HTML extraction.
+# Examples: GitHub Gist pages are fetched from gist.githubusercontent.com/raw,
+# GitHub blob URLs are fetched from raw.githubusercontent.com, and raw Markdown
+# is preserved as Markdown.
+
+# Natural-language capture also works from normal prompts:
+# "Capture this article into vault https://example.com/article"
+# "Add page to personal vault https://example.com/article"
+
+# Private personal vault for global memories
+/memory-vault-init --personal
+/memory-vault-sync --personal
+```
+
+Default locations:
+
+```txt
+<repo>/.memory-stone/vault/              # project vault
+~/.pi/agent/memory/vaults/personal/      # personal vault
+```
+
+Initial layout:
+
+```txt
+index.md
+WIKI_SCHEMA.md
+records/{decisions,preferences,tasks,error-resolutions,turn-summaries,session-summaries}/
+sources/                       # captured web source pages
+meta/registry.json
+```
+
+URL capture writes curated source notes into `sources/`. Raw provenance packets (manifest, metadata, fetch attempts, original artifact, extracted markdown) are stored outside the Obsidian vault under `.memory-stone/source-packets/` for project vaults or `~/.pi/agent/memory/source-packets/personal/` for personal vaults. Capture extracts HTML articles with Mozilla Readability, converts to Markdown, redacts secrets, and marks extraction quality as `good` or `weak` with warnings.
+
 ## Privacy & Safety
 
 ### Secret Redaction
@@ -260,16 +311,18 @@ npm run typecheck
 
 These scripts are also runnable from a pi package clone installed from git; the required script runners are regular dependencies because pi package installs omit `devDependencies`.
 
-51 tests across 6 test files:
+69 tests across 8 test files:
 
 | Suite | Tests | Focus |
 |---|---|---|
 | `indexing.test.ts` | 1 | Incremental session indexing |
 | `parser.test.ts` | 10 | Turn parsing, file activity detection, error extraction |
-| `portable.test.ts` | 3 | JSON/Markdown export, JSON import, SQLite backup |
-| `privacy.test.ts` | 17 | Secret redaction, sensitive path filtering |
-| `ranking.test.ts` | 16 | Hybrid ranking, cross-project filtering, injection formatting |
-| `session-state.test.ts` | 4 | Injection mode config, session ref selection, manual-only injection |
+| `portable.test.ts` | 5 | JSON/Markdown export, JSON import, SQLite backup |
+| `privacy.test.ts` | 21 | Secret redaction, sensitive path filtering |
+| `ranking.test.ts` | 18 | Hybrid ranking, cross-project filtering, injection formatting |
+| `session-state.test.ts` | 5 | Injection mode config, session ref selection, manual-only injection |
+| `tools.test.ts` | 2 | Tool visibility and forgetting safety |
+| `vault.test.ts` | 7 | Vault path resolution, initialization, sync, URL capture, registry generation |
 
 ## Configuration
 
@@ -302,6 +355,7 @@ For manual-only memory, keep `enabled: true` and set `injectionMode: "manual"`.
 - LLM extraction (decisions, preferences, tasks)
 - Historical backfill (`/memory-backfill`)
 - Embedding-based semantic search
+- Vault backlinks/lint/search/capture commands
 - `/memory-edit`, `/memory-supersede`
 - Rich TUI memory browser
 - Multi-machine sync

package/package.json

@@ -1,24 +1,37 @@
 {
   "name": "pi-memory-stone",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "type": "module",
   "description": "Global pi extension: preserves and retrieves useful memory across pi sessions",
   "license": "MIT",
-  "keywords": ["pi-package"],
+  "keywords": [
+    "pi-package"
+  ],
   "files": [
     "src",
+    "skills/**/*",
     "README.md",
     "LICENSE"
   ],
   "pi": {
-    "extensions": ["./src/index.ts"]
+    "extensions": [
+      "./src/index.ts"
+    ],
+    "skills": [
+      "./skills"
+    ]
   },
   "scripts": {
     "test": "NODE_OPTIONS='--experimental-sqlite' tsx --test test/*.test.ts",
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@mozilla/readability": "^0.6.0",
+    "@types/jsdom": "^28.0.3",
+    "@types/turndown": "^5.0.6",
+    "jsdom": "^29.1.1",
     "tsx": "^4.22.3",
+    "turndown": "^7.2.4",
     "typescript": "^5.9.3"
   },
   "peerDependencies": {

package/skills/pi-memory-stone/SKILL.md

@@ -0,0 +1,101 @@
+---
+name: pi-memory-stone
+description: Persistent cross-session memory for pi via SQLite+FTS5. Remembers past decisions, preferences, turns, and error resolutions across sessions. Use when recalling context from prior sessions, using memory_search/memory_open/memory_remember/memory_forget tools, troubleshooting memory injection, or when user asks about pi-memory-stone, memory stone, session memory, or remembering context.
+---
+
+# Pi Memory Stone
+
+pi-memory-stone indexes session turns into a searchable SQLite+FTS5 database (`~/.pi/agent/memory/memory.db`) and auto-injects relevant past memories into the system prompt before each agent turn.
+
+**Quick check:** `/memory-status` and `/memory-search <query>`.
+
+## Tools
+
+### `memory_search(query, kind?, scope?, limit?)` — find past context
+
+Use **before** answering questions that may have been addressed in past sessions. FTS5-based: include concrete keywords, not abstract concepts. Use `kind` filter for precision: `"decision"`, `"error_resolution"`, `"preference"`, `"task"`, `"turn_summary"`, `"session_summary"`. Use `scope: "global"` for cross-project recall.
+
+**Examples:** `memory_search("database schema user authentication")`, `memory_search("Next.js build error", kind: "error_resolution")`, `memory_search("deployment workflow", scope: "global")`.
+
+### `memory_open(ref)` — full record by ID
+
+Use when an injection packet or search result references a record ID (e.g., `ref=abc123`). Returns full text and metadata. Only shows records visible in current project.
+
+### `memory_remember(kind, text, scope?, tags?, importance?)` — explicit storage
+
+Use **only when the user explicitly asks** to remember something. Defaults to `scope: "project"`. Use `scope: "global"` only when the user says "for all projects". Auto-downgrades to project scope if text appears to contain secrets, hostnames, or internal details.
+
+### `memory_forget(ref, hard?)` — remove stale/wrong memories
+
+Soft-forget hides from future searches. Hard delete (`hard: true`) requires user confirmation via the `/memory-forget --hard` command.
+
+## Ranking (craft better queries)
+
+Hybrid score: FTS match × kind boost × recency decay × confidence × importance, then sorted descending.
+
+| Factor | Boost | Implication |
+|---|---|---|
+| Same-project | ×1.5 | Current project memories rank highest |
+| Kind: decision | ×1.5 | Decisions and errors surface prominently |
+| Kind: error_resolution | ×1.4 | |
+| Kind: preference | ×1.3 | |
+| Recency | Half-life 7 days | Older memories fade; add time hints to queries |
+| Confidence × Importance | 0.5–1.5× | Explicitly remembered items outrank auto-indexed turns |
+
+Queries include the user's first ~200 chars + basenames of recently touched files. Include filenames when searching for file-specific context.
+
+## Injection modes
+
+| Mode | Behavior |
+|---|---|
+| `auto` (default) | FTS5 search every turn + manual refs. Score threshold: 0.3. Auto-injected refs not re-injected (anti-feedback-loop). |
+| `manual` | Only `/memory-inject` refs. Inject every turn until `/memory-clear-injected`. |
+
+Session override (`/memory-mode`) takes precedence over config. Check: `/memory-status`.
+
+## Session state (per-session, persists across turns)
+
+- **enabled** — toggle via `/memory-on` / `/memory-off`
+- **injectionMode** — `auto` or `manual`, via `/memory-mode`, falls back to config
+- **manualRefs** — via `/memory-inject <ref>`, cleared by `/memory-clear-injected`
+
+## Commands quick reference
+
+| Command | Purpose |
+|---|---|
+| `/memory-status [-v]` | Stats, config, effective mode |
+| `/memory-search <q>` | Search (top 20) |
+| `/memory-open <ref>` | Full record |
+| `/memory-inject <ref>...` | Add manual refs for session |
+| `/memory-clear-injected` | Clear manual refs |
+| `/memory-mode <auto\|manual>` | Override injection mode |
+| `/memory-last` | Last injection packet |
+| `/memory-forget <ref> [--hard]` | Soft/hard delete |
+| `/memory-export [path] [--format json\|md] [--all]` | Portable export |
+| `/memory-import <file> [--preserve-project\|--global]` | Import (default: remap to current project) |
+| `/memory-backup [path]` | Backup SQLite DB |
+| `/memory-on` / `/memory-off` | Enable/disable injection |
+
+## Troubleshooting
+
+| Problem | Likely cause | Fix |
+|---|---|---|
+| No memories injected | Score < 0.3 threshold, manual mode, or `/memory-off` | Lower threshold, `/memory-mode auto`, or `/memory-on` |
+| Cross-project memories missing | `crossProjectEnabled: false` in config | Enable in `.pi/settings.json` or `/memory-inject` manually |
+| Search returns nothing | Keywords don't match FTS5, or record soft_forgotten | Use concrete terms; try `/memory-export --all` to find |
+| Global `memory_remember` refused | Text matched as sensitive | The tool auto-downgrades safely; this is expected |
+
+## Config
+
+`.pi/settings.json` in project root (all fields optional, shown with defaults):
+
+```json
+{ "memory": { "enabled": true, "maxInjectedRecords": 5, "maxInjectedTokens": 1000, "scoreThreshold": 0.3, "crossProjectEnabled": false, "injectionMode": "auto" } }
+```
+
+## Privacy guarantees
+
+- All text redacted before storage: API keys, tokens, passwords, connection strings
+- Sensitive paths never indexed: `.env`, keys, certs, `node_modules`, `.git`, `.ssh`, `.aws`
+- Global scope refused for text containing secret/hostname/internal patterns
+- `memory_open` only shows records visible in current project

package/src/commands/index.ts

@@ -1,6 +1,6 @@
 /**
  * Commands: /memory-status, /memory-search, /memory-open, /memory-inject, /memory-mode, /memory-last,
- * /memory-export, /memory-import, /memory-backup
+ * /memory-export, /memory-import, /memory-backup, /memory-vault-*
  */
 
 import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
@@ -24,6 +24,15 @@ import {
   isRecordVisibleInProject,
   parseRefArgs,
 } from "../session-state/index.js";
+import {
+  getVaultStatus,
+  initVault,
+  parseVaultScope,
+  syncVault,
+  type VaultScope,
+} from "../vault/index.js";
+import { captureUrlToVault } from "../vault/capture.js";
+import { extractFirstUrl } from "../vault/intent.js";
 
 export function registerCommands(pi: ExtensionAPI): void {
   // ── /memory-status ──────────────────────────────────────────────
@@ -198,6 +207,64 @@ export function registerCommands(pi: ExtensionAPI): void {
     },
   });
 
+  // ── /memory-vault-* ─────────────────────────────────────────────
+
+  pi.registerCommand("memory-vault-init", {
+    description: "Initialize an Obsidian-compatible memory vault",
+    handler: async (args, ctx) => {
+      await handleMemoryVaultInit(args, ctx);
+    },
+  });
+
+  pi.registerCommand("stone-vault-init", {
+    description: "Alias for /memory-vault-init",
+    handler: async (args, ctx) => {
+      await handleMemoryVaultInit(args, ctx);
+    },
+  });
+
+  pi.registerCommand("memory-vault-sync", {
+    description: "Sync active memory records into the initialized markdown vault",
+    handler: async (args, ctx) => {
+      await handleMemoryVaultSync(args, ctx);
+    },
+  });
+
+  pi.registerCommand("stone-vault-sync", {
+    description: "Alias for /memory-vault-sync",
+    handler: async (args, ctx) => {
+      await handleMemoryVaultSync(args, ctx);
+    },
+  });
+
+  pi.registerCommand("memory-vault-status", {
+    description: "Show memory vault path, initialization, and sync status",
+    handler: async (args, ctx) => {
+      await handleMemoryVaultStatus(args, ctx);
+    },
+  });
+
+  pi.registerCommand("memory-vault-capture-url", {
+    description: "Capture a web page URL into the memory vault",
+    handler: async (args, ctx) => {
+      await handleMemoryVaultCaptureUrl(args, ctx);
+    },
+  });
+
+  pi.registerCommand("stone-vault-status", {
+    description: "Alias for /memory-vault-status",
+    handler: async (args, ctx) => {
+      await handleMemoryVaultStatus(args, ctx);
+    },
+  });
+
+  pi.registerCommand("stone-vault-capture-url", {
+    description: "Alias for /memory-vault-capture-url",
+    handler: async (args, ctx) => {
+      await handleMemoryVaultCaptureUrl(args, ctx);
+    },
+  });
+
   // ── /memory-on / /memory-off ────────────────────────────────────
 
   pi.registerCommand("memory-on", {
@@ -481,6 +548,98 @@ async function handleMemoryBackup(args: string, ctx: ExtensionCommandContext): P
   }
 }
 
+async function handleMemoryVaultInit(args: string, ctx: ExtensionCommandContext): Promise<void> {
+  const parsed = parseCommandArgs(args);
+  const scope = parseVaultScopeOrNotify(parsed, ctx);
+  if (!scope) return;
+
+  try {
+    const projectId = getProjectId(ctx.cwd);
+    const result = initVault(scope, projectId, ctx.cwd);
+    ctx.ui.notify(
+      result.created
+        ? `Initialized ${scope} memory vault at ${result.path}`
+        : `${scope} memory vault already initialized at ${result.path}`,
+      "info",
+    );
+  } catch (err) {
+    ctx.ui.notify(`Memory vault init failed: ${err instanceof Error ? err.message : String(err)}`, "warning");
+  }
+}
+
+async function handleMemoryVaultSync(args: string, ctx: ExtensionCommandContext): Promise<void> {
+  const parsed = parseCommandArgs(args);
+  const scope = parseVaultScopeOrNotify(parsed, ctx);
+  if (!scope) return;
+
+  try {
+    const projectId = getProjectId(ctx.cwd);
+    const result = syncVault(scope, projectId, ctx.cwd);
+    ctx.ui.notify(
+      `Synced ${result.records} memory records to ${result.path} (${result.pagesWritten} page${result.pagesWritten === 1 ? "" : "s"} written). Registry: ${result.registryPath}`,
+      "info",
+    );
+  } catch (err) {
+    ctx.ui.notify(`Memory vault sync failed: ${err instanceof Error ? err.message : String(err)}`, "warning");
+  }
+}
+
+async function handleMemoryVaultStatus(args: string, ctx: ExtensionCommandContext): Promise<void> {
+  const parsed = parseCommandArgs(args);
+  const scope = parseVaultScopeOrNotify(parsed, ctx);
+  if (!scope) return;
+
+  const projectId = getProjectId(ctx.cwd);
+  const status = getVaultStatus(scope, projectId, ctx.cwd);
+  const lines: string[] = [];
+  lines.push("📚 Memory Vault Status");
+  lines.push("");
+  lines.push(`  scope: ${scope}`);
+  lines.push(`  path: ${status.path}`);
+  lines.push(`  initialized: ${status.initialized}`);
+  lines.push(`  registry: ${status.registryExists ? "present" : "missing"}`);
+  lines.push(`  markdown pages: ${status.pageCount}`);
+  lines.push(`  synced record pages: ${status.recordPageCount}`);
+  lines.push(`  last sync: ${status.lastSyncedAt ?? "never"}`);
+  ctx.ui.notify(lines.join("\n"), "info");
+}
+
+async function handleMemoryVaultCaptureUrl(args: string, ctx: ExtensionCommandContext): Promise<void> {
+  const parsed = parseCommandArgs(args);
+  const scope = parseVaultScopeOrNotify(parsed, ctx);
+  if (!scope) return;
+
+  const url = extractFirstUrl(args);
+  if (!url) {
+    ctx.ui.notify("Usage: /memory-vault-capture-url <http-url> [--project|--personal]", "warning");
+    return;
+  }
+
+  try {
+    const projectId = getProjectId(ctx.cwd);
+    const result = await captureUrlToVault(scope, projectId, ctx.cwd, url);
+    const warnings = result.warnings.length > 0 ? `\nWarnings: ${result.warnings.join("; ")}` : "";
+    ctx.ui.notify(
+      `Captured ${result.title} into ${scope} memory vault${result.initialized ? " (initialized vault)" : ""}\nQuality: ${result.quality} (${result.qualityScore})${warnings}\nPage: ${result.pagePath}`,
+      "info",
+    );
+  } catch (err) {
+    ctx.ui.notify(`Memory vault URL capture failed: ${err instanceof Error ? err.message : String(err)}`, "warning");
+  }
+}
+
+function parseVaultScopeOrNotify(
+  parsed: ReturnType<typeof parseCommandArgs>,
+  ctx: ExtensionCommandContext,
+): VaultScope | null {
+  const scope = parseVaultScope(parsed);
+  if (!scope) {
+    ctx.ui.notify("Usage: /memory-vault-<init|sync|status> [--project|--personal|--scope project|personal]", "warning");
+    return null;
+  }
+  return scope;
+}
+
 async function handleMemoryForget(args: string, ctx: ExtensionCommandContext): Promise<void> {
   const { softForgetRecord, hardDeleteRecord, getRecord } = await import("../db/index.js");
 
@@ -500,6 +659,12 @@ async function handleMemoryForget(args: string, ctx: ExtensionCommandContext): P
       continue;
     }
 
+    const currentProjectId = getProjectId(ctx.cwd);
+    if (record.status !== "active" || !isRecordVisibleInProject(record, currentProjectId)) {
+      ctx.ui.notify(`Memory record ${refId} is not available.`, "warning");
+      continue;
+    }
+
     if (hardFlag) {
       const confirmed = ctx.hasUI
         ? await ctx.ui.confirm(
@@ -544,7 +709,7 @@ function parseCommandArgs(args: string): {
     }
 
     const next = parts[i + 1];
-    if (next && !next.startsWith("--") && ["format"].includes(withoutPrefix)) {
+    if (next && !next.startsWith("--") && ["format", "scope"].includes(withoutPrefix)) {
       options.set(withoutPrefix, next);
       i += 1;
     } else {

package/src/db/index.ts

@@ -4,7 +4,7 @@
  */
 
 import { DatabaseSync } from "node:sqlite";
-import { existsSync, mkdirSync } from "node:fs";
+import { chmodSync, existsSync, mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 import { createHash, randomUUID } from "node:crypto";
 import { redactSecrets } from "../privacy/index.js";
@@ -41,17 +41,38 @@ export function getDb(): DatabaseSync {
   if (!_db) {
     const dbDir = getDbDir();
     if (!existsSync(dbDir)) {
-      mkdirSync(dbDir, { recursive: true });
+      mkdirSync(dbDir, { recursive: true, mode: 0o700 });
     }
+    hardenPathPermissions(dbDir, 0o700);
     _db = new DatabaseSync(getDbPath());
+    hardenDbFilePermissions();
     _db.exec("PRAGMA journal_mode = WAL");
     _db.exec("PRAGMA busy_timeout = 5000");
     _db.exec("PRAGMA foreign_keys = ON");
     runMigrations(_db);
+    hardenDbFilePermissions();
   }
   return _db;
 }
 
+function hardenPathPermissions(path: string, mode: number): void {
+  try {
+    chmodSync(path, mode);
+  } catch {
+    // Best-effort only: do not make memory unavailable on filesystems that
+    // do not support POSIX permissions.
+  }
+}
+
+export function hardenDbFilePermissions(): void {
+  const dbPath = getDbPath();
+  for (const path of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]) {
+    if (existsSync(path)) {
+      hardenPathPermissions(path, 0o600);
+    }
+  }
+}
+
 export function closeDb(): void {
   if (_db) {
     try {
@@ -253,6 +274,9 @@ export function upsertRecord(record: {
   const updatedAt = record.updated_at ?? now;
   const scope = record.scope ?? "project";
   const projectId = scope === "global" ? null : (record.project_id ?? null);
+  if (scope === "project" && !projectId) {
+    throw new Error("Project-scoped memory records require a project_id");
+  }
   const redactedText = redactSecrets(record.text);
   const redactedTags = record.tags ? redactSecrets(record.tags) : null;
   const id = recordIdentityHash(redactedText, record.kind, scope, projectId);
@@ -363,6 +387,8 @@ export function searchRecordsFts(
 
   if (!terms) return [];
 
+  const safeLimit = Math.max(1, Math.min(200, Number.isFinite(limit) ? Math.floor(limit) : 20));
+
   const results = db
     .prepare(
       `SELECT r.*, fts.rank as rank
@@ -372,7 +398,7 @@ export function searchRecordsFts(
        ORDER BY rank
        LIMIT ?`
     )
-    .all(terms, limit) as unknown as (RecordRow & { rank: number })[];
+    .all(terms, safeLimit) as unknown as (RecordRow & { rank: number })[];
 
   // Apply post-filters
   return results.filter((r) => {