persyst-mcp 2.1.1 → 2.1.2

package/hooks/persyst-hook.js

@@ -1,38 +1,66 @@
 #!/usr/bin/env node
 
 /**
- * persyst-hook.js — Claude Code Hook for Persyst Memory
+ * persyst-hook.js — Claude Code Hook for Persyst Memory (PAMP-Enhanced)
  * 
  * Automatically injects relevant memories into Claude Code's context
- * on SessionStart and UserPromptSubmit events.
+ * on SessionStart and UserPromptSubmit events, and queues conversation
+ * turns for async background extraction on Stop events.
+ * 
+ * PAMP Integration (Persyst Auto-Memory Pipeline):
+ *   - Tier 1: Agent-explicit add_memory calls (existing, unchanged)
+ *   - Tier 2: Heuristic regex extraction on UserPromptSubmit (sync, zero-cost)
+ *   - Tier 3: Async LLM extraction via background worker (spawned on Stop)
  * 
  * How it works:
- *   1. Claude Code sends a JSON payload on stdin with hook_event_name, session_id, cwd, etc.
+ *   1. Claude Code sends a JSON payload on stdin with hook_event_name, session_id, etc.
  *   2. This script connects to the Persyst MCP server via StdioClientTransport.
  *   3. It calls get_optimized_context or search_memories to retrieve relevant memories.
  *   4. It returns a JSON response on stdout with additionalContext for Claude Code to inject.
+ *   5. On Stop: queues the conversation text for background LLM extraction.
  * 
  * Installation:
  *   npx persyst-mcp setup
  * 
  * Manual registration in ~/.claude/settings.json:
- *   { "hooks": { "SessionStart": [...], "UserPromptSubmit": [...] } }
+ *   { "hooks": { "SessionStart": [...], "UserPromptSubmit": [...], "Stop": [...] } }
  */
 
 import { Client } from '@modelcontextprotocol/sdk/client/index.js';
 import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
 import { fileURLToPath } from 'url';
-import { dirname, resolve } from 'path';
+import { dirname, resolve, join } from 'path';
+import { spawn } from 'child_process';
+import { writeFileSync, readdirSync, mkdirSync, existsSync } from 'fs';
+import { homedir } from 'os';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
+// ============================================================
+// CONFIGURATION
+// ============================================================
+
 // Minimum prompt length to trigger memory search (skip "y", "ok", "/run", etc.)
 const MIN_PROMPT_LENGTH = 15;
 
 // Maximum time to wait for Persyst MCP connection (ms)
 const CONNECTION_TIMEOUT = 8000;
 
+// Hard timeout for the entire hook execution (ms)
+// Claude Code will kill the hook if it exceeds this
+const MAX_HOOK_LATENCY_MS = 500;
+
+// Maximum active queue jobs before skipping worker spawn
+const MAX_QUEUE_JOBS = 20;
+
+// Queue directory for background extraction jobs
+const QUEUE_DIR = join(homedir(), '.persyst', 'queue');
+
+// ============================================================
+// STDIN READER
+// ============================================================
+
 /**
  * Read the full JSON payload from stdin.
  * Claude Code sends the hook context as a single JSON object.
@@ -53,13 +81,20 @@ function readStdin() {
   });
 }
 
+// ============================================================
+// MCP CLIENT CONNECTION
+// ============================================================
+
 /**
  * Connect to the Persyst MCP server as a client.
  * Uses StdioClientTransport to spawn and communicate with the server.
  */
 async function connectToPersyst() {
-  // Resolve the path to Persyst's index.js relative to this hook file
-  const persystPath = resolve(__dirname, '..', 'index.js');
+  // Resolve the path to Persyst's index.js
+  let persystPath = '{{PERSYST_INDEX_PATH}}';
+  if (persystPath.startsWith('{{')) {
+    persystPath = resolve(__dirname, '..', 'index.js');
+  }
 
   const transport = new StdioClientTransport({
     command: 'node',
@@ -93,6 +128,89 @@ async function callTool(client, toolName, args) {
   return null;
 }
 
+// ============================================================
+// PAMP: QUEUE MANAGEMENT
+// ============================================================
+
+/**
+ * Count active job files in the queue directory.
+ * Used for worker pool protection — don't spawn if overloaded.
+ * @returns {number}
+ */
+function countQueueJobs() {
+  try {
+    if (!existsSync(QUEUE_DIR)) return 0;
+    return readdirSync(QUEUE_DIR).filter(f => f.endsWith('.json')).length;
+  } catch (_) {
+    return 0;
+  }
+}
+
+/**
+ * Write a conversation turn to the extraction queue.
+ * @param {string} text - The conversation text to extract from
+ * @param {Object} meta - Metadata (session_id, agent_id, etc.)
+ */
+function enqueueJob(text, meta = {}) {
+  try {
+    mkdirSync(QUEUE_DIR, { recursive: true });
+
+    const jobId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    const jobFile = join(QUEUE_DIR, `${jobId}.json`);
+
+    writeFileSync(jobFile, JSON.stringify({
+      text,
+      session_id: meta.session_id || null,
+      agent_id: meta.agent_id || 'claude-code',
+      namespace: meta.namespace || 'shared',
+      cwd: meta.cwd || null,
+      queued_at: new Date().toISOString(),
+      _retries: 0
+    }, null, 2));
+
+    return jobId;
+  } catch (err) {
+    // Non-critical — log and continue
+    process.stderr.write(`[persyst-hook] Queue write failed: ${err.message}\n`);
+    return null;
+  }
+}
+
+/**
+ * Spawn the background extraction worker as a detached process.
+ * The worker runs independently — hook doesn't wait for it.
+ */
+function spawnWorker() {
+  // Check queue depth first
+  const queueDepth = countQueueJobs();
+  if (queueDepth > MAX_QUEUE_JOBS) {
+    process.stderr.write(`[persyst-hook] Queue overloaded (${queueDepth} jobs), skipping worker spawn.\n`);
+    return;
+  }
+
+  try {
+    let workerPath = '{{PERSYST_WORKER_PATH}}';
+    if (workerPath.startsWith('{{')) {
+      workerPath = resolve(__dirname, '..', 'bin', 'extract-worker.js');
+    }
+
+    const child = spawn('node', [workerPath], {
+      detached: true,
+      stdio: 'ignore',
+      env: { ...process.env }
+    });
+
+    // Unref so the hook can exit without waiting for the worker
+    child.unref();
+  } catch (err) {
+    process.stderr.write(`[persyst-hook] Worker spawn failed: ${err.message}\n`);
+  }
+}
+
+// ============================================================
+// EVENT HANDLERS
+// ============================================================
+
 /**
  * Handle SessionStart: load project-wide context and ingest git history.
  */
@@ -152,6 +270,7 @@ async function handleSessionStart(client, input) {
 
 /**
  * Handle UserPromptSubmit: search for memories relevant to the user's prompt.
+ * Also runs Tier 2 heuristic extraction inline (zero-cost).
  */
 async function handleUserPromptSubmit(client, input) {
   const prompt = input.prompt || '';
@@ -161,6 +280,25 @@ async function handleUserPromptSubmit(client, input) {
     return {};
   }
 
+  // --- Tier 2: Run heuristic extraction inline (sync, zero-cost) ---
+  // We don't store results here — we queue them alongside the LLM job.
+  // This just detects if there's extractable signal in the prompt.
+  let heuristicFacts = [];
+  try {
+    const { extractHeuristic } = await import('../src/extractor-heuristic.js');
+    heuristicFacts = extractHeuristic(prompt);
+  } catch (_) {
+    // Heuristic module not available — Tier 3 will handle it
+  }
+
+  // Queue the prompt for Tier 3 background extraction (non-blocking)
+  enqueueJob(prompt, {
+    session_id: input.session_id,
+    agent_id: 'claude-code',
+    cwd: input.cwd
+  });
+
+  // --- Memory Retrieval (existing behavior) ---
   // Use search_memories for speed on per-prompt lookups (faster than get_optimized_context)
   const searchResult = await callTool(client, 'search_memories', {
     query: prompt.slice(0, 200), // Truncate very long prompts for search efficiency
@@ -178,6 +316,13 @@ async function handleUserPromptSubmit(client, input) {
   for (const mem of searchResult.results) {
     contextLines.push(`• [Memory #${mem.id}] ${mem.content}`);
   }
+
+  // Add heuristic extraction notice if any facts were detected
+  if (heuristicFacts.length > 0) {
+    contextLines.push('');
+    contextLines.push(`[PAMP: ${heuristicFacts.length} fact signal(s) detected, queued for extraction]`);
+  }
+
   contextLines.push('=== END MEMORY ===');
 
   return {
@@ -189,8 +334,31 @@ async function handleUserPromptSubmit(client, input) {
 }
 
 /**
- * Main entry point.
+ * Handle Stop: queue the final conversation turn for background extraction
+ * and spawn the worker to process the queue.
  */
+async function handleStop(input) {
+  // The Stop event may include conversation_turns or transcript data
+  const transcript = input.transcript || input.conversation || '';
+
+  if (transcript && typeof transcript === 'string' && transcript.length > MIN_PROMPT_LENGTH) {
+    enqueueJob(transcript, {
+      session_id: input.session_id,
+      agent_id: 'claude-code',
+      cwd: input.cwd
+    });
+  }
+
+  // Spawn background worker to process all queued jobs
+  spawnWorker();
+
+  return {};
+}
+
+// ============================================================
+// MAIN ENTRY POINT
+// ============================================================
+
 async function main() {
   let client = null;
 
@@ -198,20 +366,37 @@ async function main() {
     const input = await readStdin();
     const eventName = input.hook_event_name;
 
+    // Handle Stop event without MCP connection (just queue + spawn)
+    if (eventName === 'Stop') {
+      const response = await handleStop(input);
+      console.log(JSON.stringify(response));
+      return;
+    }
+
     // Only handle events we care about
     if (eventName !== 'SessionStart' && eventName !== 'UserPromptSubmit') {
       console.log(JSON.stringify({}));
       return;
     }
 
-    // Connect to Persyst
+    // Connect to Persyst with hard timeout
+    const hookStart = Date.now();
     client = await connectToPersyst();
 
     let response;
     if (eventName === 'SessionStart') {
       response = await handleSessionStart(client, input);
     } else if (eventName === 'UserPromptSubmit') {
-      response = await handleUserPromptSubmit(client, input);
+      // Apply hard timeout for prompt-time hook execution
+      response = await Promise.race([
+        handleUserPromptSubmit(client, input),
+        new Promise((resolve) =>
+          setTimeout(() => {
+            process.stderr.write(`[persyst-hook] UserPromptSubmit hit ${MAX_HOOK_LATENCY_MS}ms timeout, returning partial.\n`);
+            resolve({});
+          }, MAX_HOOK_LATENCY_MS - (Date.now() - hookStart))
+        )
+      ]);
     } else {
       response = {};
     }

package/index.js

@@ -32,6 +32,13 @@ if (subcommand === 'setup') {
   // Shift 'ingest' from process.argv so ingest.js gets the correct arguments
   process.argv.splice(2, 1);
   await import('./bin/ingest.js');
+} else if (subcommand === 'extract') {
+  // Shift 'extract' from process.argv so extract.js gets the correct arguments
+  process.argv.splice(2, 1);
+  await import('./bin/extract.js');
+} else if (subcommand === 'worker') {
+  // Run the background extraction worker directly
+  await import('./bin/extract-worker.js');
 } else {
   // Default: start the MCP server
   const { startServer } = await import('./src/server.js');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "persyst-mcp",
-  "version": "2.1.1",
+  "version": "2.1.2",
   "description": "Local-first MCP memory server with hybrid keyword + semantic search for coding agents",
   "type": "module",
   "main": "index.js",
@@ -9,7 +9,9 @@
     "persyst-setup": "bin/setup.js",
     "persyst-aider": "bin/aider.js",
     "persyst-init": "bin/init.js",
-    "persyst-ingest": "bin/ingest.js"
+    "persyst-ingest": "bin/ingest.js",
+    "persyst-extract": "bin/extract.js",
+    "persyst-worker": "bin/extract-worker.js"
   },
   "engines": {
     "node": ">=18.0.0"
@@ -25,7 +27,9 @@
   "scripts": {
     "start": "node index.js",
     "test": "node test/smoke.js",
-    "test:heavy": "cross-env NODE_ENV=test node --test test/test_*.js"
+    "test:heavy": "cross-env NODE_ENV=test node --test test/test_*.js",
+    "worker": "node bin/extract-worker.js",
+    "extract": "node bin/extract.js"
   },
   "keywords": [
     "mcp",

package/src/database.js

@@ -72,6 +72,16 @@ try {
   db.exec('ALTER TABLE memories ADD COLUMN assertion_time INTEGER DEFAULT (unixepoch())');
 } catch (e) { /* Column already exists */ }
 
+// --- Migration: add namespace column for per-agent isolation ---
+try {
+  db.exec("ALTER TABLE memories ADD COLUMN namespace TEXT DEFAULT 'shared'");
+} catch (e) { /* Column already exists */ }
+
+// --- Index on namespace for fast filtered queries ---
+try {
+  db.exec('CREATE INDEX IF NOT EXISTS idx_memories_namespace ON memories (namespace)');
+} catch (e) { /* Index already exists */ }
+
 // --- Contradictions table ---
 db.exec(`
   CREATE TABLE IF NOT EXISTS contradictions (
@@ -208,7 +218,7 @@ console.error('[persyst] Schema initialized ✓');
 const stmts = {
   // -- Insert --
   insertMemory: db.prepare(
-    'INSERT INTO memories (content, importance_score) VALUES (?, ?)'
+    'INSERT INTO memories (content, importance_score, namespace) VALUES (?, ?, ?)'
   ),
   insertVec: db.prepare(
     'INSERT INTO memories_vec (rowid, embedding) VALUES (?, ?)'
@@ -246,15 +256,24 @@ const stmts = {
   getById: db.prepare(
     'SELECT * FROM memories WHERE id = ? AND valid_until IS NULL'
   ),
+  getByIdNs: db.prepare(
+    "SELECT * FROM memories WHERE id = ? AND (namespace = ? OR namespace = 'shared') AND valid_until IS NULL"
+  ),
   getAnyById: db.prepare(
     'SELECT * FROM memories WHERE id = ?'
   ),
   getRecent: db.prepare(
     'SELECT * FROM memories WHERE valid_until IS NULL ORDER BY created_at DESC LIMIT ?'
   ),
+  getRecentNs: db.prepare(
+    "SELECT * FROM memories WHERE (namespace = ? OR namespace = 'shared') AND valid_until IS NULL ORDER BY created_at DESC LIMIT ?"
+  ),
   getImportant: db.prepare(
     'SELECT * FROM memories WHERE valid_until IS NULL ORDER BY importance_score DESC LIMIT ?'
   ),
+  getImportantNs: db.prepare(
+    "SELECT * FROM memories WHERE (namespace = ? OR namespace = 'shared') AND valid_until IS NULL ORDER BY importance_score DESC LIMIT ?"
+  ),
   getProvenance: db.prepare(
     'SELECT * FROM provenance WHERE memory_id = ?'
   ),
@@ -353,6 +372,9 @@ const stmts = {
   findMemoryByContent: db.prepare(
     'SELECT id FROM memories WHERE content = ? AND valid_until IS NULL LIMIT 1'
   ),
+  findMemoryByContentNs: db.prepare(
+    "SELECT id FROM memories WHERE content = ? AND (namespace = ? OR namespace = 'shared') AND valid_until IS NULL LIMIT 1"
+  ),
 
   // -- Hash-prefix lookup for git dedup (Bug 1 fix) --
   findMemoryByHashPrefix: db.prepare(
@@ -363,6 +385,14 @@ const stmts = {
   getActiveMemoryCount: db.prepare(
     'SELECT COUNT(*) as count FROM memories WHERE valid_until IS NULL'
   ),
+  getActiveMemoryCountNs: db.prepare(
+    "SELECT COUNT(*) as count FROM memories WHERE (namespace = ? OR namespace = 'shared') AND valid_until IS NULL"
+  ),
+
+  // -- Namespace stats --
+  getNamespaceStats: db.prepare(
+    'SELECT namespace, COUNT(*) as count FROM memories WHERE valid_until IS NULL GROUP BY namespace ORDER BY count DESC'
+  ),
 
   // -- Memory History Chain (Feature 6: prepared statements) --
   getContradictionAncestors: db.prepare(
@@ -380,10 +410,14 @@ const stmts = {
 
 /**
  * Insert a new memory into the memories table and log its provenance.
+ * @param {string} content - Memory content
+ * @param {number} importance - Importance score (0-1)
+ * @param {Object} provenanceInfo - Provenance metadata
+ * @param {string} namespace - Namespace for agent isolation (default: 'shared')
  * @returns {number} The new memory's ID
  */
-export function insertMemory(content, importance = 1.0, provenanceInfo = null) {
-  const result = stmts.insertMemory.run(content, importance);
+export function insertMemory(content, importance = 1.0, provenanceInfo = null, namespace = 'shared') {
+  const result = stmts.insertMemory.run(content, importance, namespace || 'shared');
   const id = Number(result.lastInsertRowid);
 
   // Provenance Info handling
@@ -412,13 +446,16 @@ export function insertVector(id, embedding) {
 
 /**
  * Get a memory by ID. Boosts its importance on access.
+ * @param {number} id - Memory ID
+ * @param {string|null} namespace - Namespace filter (null = no filter)
  * @returns {object|null} The memory row, or null if not found
  */
-export function getMemory(id) {
-  const memory = stmts.getById.get(id);
+export function getMemory(id, namespace = null) {
+  const memory = namespace
+    ? stmts.getByIdNs.get(id, namespace)
+    : stmts.getById.get(id);
   if (memory) {
     boostMemory(id);
-    // Fetch and link provenance info
     const prov = getProvenance(id);
     memory.provenance = prov;
   }
@@ -439,10 +476,14 @@ export function getAnyMemoryById(id) {
 
 /**
  * Get a memory by ID WITHOUT boosting. Used internally for search results.
+ * @param {number} id - Memory ID
+ * @param {string|null} namespace - Namespace filter (null = no filter)
  * @returns {object|null} The memory row, or null if not found
  */
-export function getMemoryById(id) {
-  const memory = stmts.getById.get(id);
+export function getMemoryById(id, namespace = null) {
+  const memory = namespace
+    ? stmts.getByIdNs.get(id, namespace)
+    : stmts.getById.get(id);
   if (memory) {
     memory.provenance = getProvenance(id);
   }
@@ -480,9 +521,13 @@ export function deleteMemory(id) {
 
 /**
  * Get the N most recently created memories.
+ * @param {number} limit - Max results
+ * @param {string|null} namespace - Namespace filter (null = all)
  */
-export function getRecentMemories(limit = 10) {
-  const rows = stmts.getRecent.all(limit);
+export function getRecentMemories(limit = 10, namespace = null) {
+  const rows = namespace
+    ? stmts.getRecentNs.all(namespace, limit)
+    : stmts.getRecent.all(limit);
   rows.forEach(r => {
     r.provenance = getProvenance(r.id);
   });
@@ -491,9 +536,13 @@ export function getRecentMemories(limit = 10) {
 
 /**
  * Get the N most important memories (by importance_score).
+ * @param {number} limit - Max results
+ * @param {string|null} namespace - Namespace filter (null = all)
  */
-export function getImportantMemories(limit = 10) {
-  const rows = stmts.getImportant.all(limit);
+export function getImportantMemories(limit = 10, namespace = null) {
+  const rows = namespace
+    ? stmts.getImportantNs.all(namespace, limit)
+    : stmts.getImportant.all(limit);
   rows.forEach(r => {
     r.provenance = getProvenance(r.id);
   });
@@ -620,9 +669,13 @@ export function getMemoriesByEntity(entityId) {
  * Check if a memory with exact content already exists.
  * Used for deduplication.
  * @param {string} content - Exact content to match
+ * @param {string|null} namespace - Namespace filter (null = global dedup)
  * @returns {boolean}
  */
-export function memoryExists(content) {
+export function memoryExists(content, namespace = null) {
+  if (namespace) {
+    return stmts.findMemoryByContentNs.get(content, namespace) !== undefined;
+  }
   return stmts.findMemoryByContent.get(content) !== undefined;
 }
 
@@ -638,12 +691,24 @@ export function memoryExistsByHashPrefix(pattern) {
 
 /**
  * Get count of active (non-archived) memories.
+ * @param {string|null} namespace - Namespace filter (null = all)
  * @returns {number}
  */
-export function getActiveMemoryCount() {
+export function getActiveMemoryCount(namespace = null) {
+  if (namespace) {
+    return stmts.getActiveMemoryCountNs.get(namespace).count;
+  }
   return stmts.getActiveMemoryCount.get().count;
 }
 
+/**
+ * Get namespace breakdown stats.
+ * @returns {Array<{namespace: string, count: number}>}
+ */
+export function getNamespaceStats() {
+  return stmts.getNamespaceStats.all();
+}
+
 // ============================================================
 // DEDUPLICATION BY EXACT CONTENT
 // ============================================================
@@ -651,10 +716,13 @@ export function getActiveMemoryCount() {
 /**
  * Find memory by exact content.
  * @param {string} content
+ * @param {string|null} namespace - Namespace filter (null = global)
  * @returns {object|null} The memory row, or null if not found
  */
-export function getMemoryByContent(content) {
-  const row = stmts.findMemoryByContent.get(content);
+export function getMemoryByContent(content, namespace = null) {
+  const row = namespace
+    ? stmts.findMemoryByContentNs.get(content, namespace)
+    : stmts.findMemoryByContent.get(content);
   return row ? getMemoryById(row.id) : null;
 }