persyst-mcp 2.1.3 → 2.2.0

package/src/server.js

@@ -1,67 +1,72 @@
-/**
- * server.js — MCP Server Setup
- * 
- * Creates the MCP server, registers all tools, and connects
- * via stdio transport (the standard MCP communication method).
- * Sets up hourly temporal decay and daily consolidation background tasks.
- * 
- * IMPORTANT: Never write to stdout — it's reserved for MCP protocol.
- * All logging goes to stderr via console.error().
- */
-
-import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
-import { registerTools, cleanupWatchers } from './tools.js';
-import { applyTemporalDecay, closeDatabase } from './database.js';
-import { consolidateMemories } from './search.js';
-
-/**
- * Start the Persyst MCP server.
- * This is called from index.js (the entry point).
- */
-export async function startServer() {
-  // --- Create MCP server ---
-  const server = new McpServer({
-    name: 'persyst',
-    version: '2.1.3'
-  });
-
-  // --- Register all tools ---
-  const registeredCount = registerTools(server);
-  console.error(`[persyst] ${registeredCount} tools registered ✓`);
-
-  // --- Start temporal decay timer ---
-  // Runs every hour: reduces importance of memories not accessed in 7+ days
-  const decayTimer = setInterval(applyTemporalDecay, 3600000);
-
-  // --- Start daily consolidation sweep ---
-  // Runs every 24 hours: merges similar memories (similarity > 0.85)
-  const consolidationTimer = setInterval(async () => {
-    console.error('[persyst] Running scheduled daily memory consolidation sweep...');
-    try {
-      const report = await consolidateMemories();
-      console.error(`[persyst] Consolidation sweep completed: consolidated ${report.consolidated_groups} duplicate groups.`);
-    } catch (err) {
-      console.error('[persyst] Daily consolidation sweep failed:', err.message);
-    }
-  }, 86400000);
-
-  // --- Graceful shutdown (Bug 3 fix: also cleans up git watchers) ---
-  const shutdown = () => {
-    console.error('[persyst] Shutting down...');
-    clearInterval(decayTimer);
-    clearInterval(consolidationTimer);
-    cleanupWatchers();  // Bug 3 fix: stop all git repo watchers
-    closeDatabase();
-    process.exit(0);
-  };
-  process.on('SIGINT', shutdown);
-  process.on('SIGTERM', shutdown);
-
-  // --- Connect via stdio ---
-  const transport = new StdioServerTransport();
-  await server.connect(transport);
-
-  console.error('[persyst] MCP server running on stdio ✓');
-  console.error('[persyst] Ready to receive tool calls');
-}
+/**
+ * server.js — MCP Server Setup
+ * 
+ * Creates the MCP server, registers all tools, and connects
+ * via stdio transport (the standard MCP communication method).
+ * Sets up hourly temporal decay and daily consolidation background tasks.
+ * 
+ * IMPORTANT: Never write to stdout — it's reserved for MCP protocol.
+ * All logging goes to stderr via console.error().
+ */
+
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { registerTools, cleanupWatchers } from './tools.js';
+import { applyTemporalDecay, closeDatabase } from './database.js';
+import { consolidateMemories } from './search.js';
+import { startWatcher, stopWatcher } from './watcher.js';
+
+/**
+ * Start the Persyst MCP server.
+ * This is called from index.js (the entry point).
+ */
+export async function startServer() {
+  // --- Create MCP server ---
+  const server = new McpServer({
+    name: 'persyst',
+    version: '2.1.3'
+  });
+
+  // --- Register all tools ---
+  const registeredCount = registerTools(server);
+  console.error(`[persyst] ${registeredCount} tools registered ✓`);
+
+  // --- Start background log watcher daemon ---
+  startWatcher();
+
+  // --- Start temporal decay timer ---
+  // Runs every hour: reduces importance of memories not accessed in 7+ days
+  const decayTimer = setInterval(applyTemporalDecay, 3600000);
+
+  // --- Start daily consolidation sweep ---
+  // Runs every 24 hours: merges similar memories (similarity > 0.85)
+  const consolidationTimer = setInterval(async () => {
+    console.error('[persyst] Running scheduled daily memory consolidation sweep...');
+    try {
+      const report = await consolidateMemories();
+      console.error(`[persyst] Consolidation sweep completed: consolidated ${report.consolidated_groups} duplicate groups.`);
+    } catch (err) {
+      console.error('[persyst] Daily consolidation sweep failed:', err.message);
+    }
+  }, 86400000);
+
+  // --- Graceful shutdown (Bug 3 fix: also cleans up git watchers) ---
+  const shutdown = () => {
+    console.error('[persyst] Shutting down...');
+    clearInterval(decayTimer);
+    clearInterval(consolidationTimer);
+    stopWatcher();      // Stop background log watcher
+    cleanupWatchers();  // Bug 3 fix: stop all git repo watchers
+    closeDatabase();
+    process.exit(0);
+  };
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+
+  // --- Connect via stdio ---
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+
+  console.error('[persyst] MCP server running on stdio ✓');
+  console.error('[persyst] Ready to receive tool calls');
+}

package/src/tools.js

@@ -13,7 +13,7 @@
 
 import { z } from 'zod';
 import { generateEmbedding } from './embeddings.js';
-import {
+import db, {
   insertMemory,
   insertVector,
   getMemory,
@@ -32,6 +32,7 @@ import {
   getMemoryByContent,
   boostMemory,
   logContradiction,
+  getProvenance,
   getAllAgentStats,
   getAttestationsByDateRange,
   getMemoryHistoryChain,
@@ -174,22 +175,54 @@ export function registerTools(server) {
               const existingMemory = getMemoryById(hitId, namespace);
               if (!existingMemory) continue;
 
-              // Check if content is substantially different (Jaccard distance > 0.5)
               const jaccard = jaccardDistance(content, existingMemory.content);
-              if (jaccard > 0.5) {
-                // This is a contradiction: similar topic, different content
-                logContradiction(hitId, id, `Auto-detected contradiction (similarity: ${sim.toFixed(3)}, content_diff: ${jaccard.toFixed(3)})`);
-                contradictions.push({
-                  old_memory_id: hitId,
-                  old_content_preview: existingMemory.content.slice(0, 100),
-                  similarity: sim.toFixed(4),
-                  content_difference: jaccard.toFixed(4)
-                });
+              // Contradiction: similar topic (high similarity), but differing key terms
+              if (jaccard > 0 && jaccard < 0.5) {
+                // Fetch provenances for trust calculation
+                const oldProv = getProvenance(hitId);
+                let oldReputation = 1.0;
+                if (oldProv && oldProv.source_type === 'agent' && oldProv.source_id) {
+                  const agentRow = db.prepare('SELECT reputation_score FROM agent_stats WHERE agent_id = ?').get(oldProv.source_id);
+                  if (agentRow) oldReputation = agentRow.reputation_score;
+                }
+
+                let newReputation = 1.0;
+                if (agent_id) {
+                  const agentRow = db.prepare('SELECT reputation_score FROM agent_stats WHERE agent_id = ?').get(agent_id);
+                  if (agentRow) newReputation = agentRow.reputation_score;
+                }
+
+                const trustOld = (oldProv ? oldProv.confidence : 1.0) * oldReputation;
+                const trustNew = 1.0 * newReputation; // New confidence is 1.0
+
+                const isSelfUpdate = oldProv && oldProv.source_type === 'agent' && oldProv.source_id === agent_id;
+
+                if (isSelfUpdate || trustNew > trustOld) {
+                  // New is preferred
+                  logContradiction(hitId, id, `Auto-detected contradiction: new memory is more trustworthy (similarity: ${sim.toFixed(3)}, content_diff: ${jaccard.toFixed(3)})`);
+                  contradictions.push({
+                    old_memory_id: hitId,
+                    old_content_preview: existingMemory.content.slice(0, 100),
+                    similarity: sim.toFixed(4),
+                    content_difference: jaccard.toFixed(4),
+                    resolution: 'replaced_old'
+                  });
+                } else {
+                  // Old is preferred
+                  logContradiction(id, hitId, `Auto-detected contradiction: existing memory is more trustworthy (similarity: ${sim.toFixed(3)}, content_diff: ${jaccard.toFixed(3)})`);
+                  contradictions.push({
+                    old_memory_id: hitId,
+                    old_content_preview: existingMemory.content.slice(0, 100),
+                    similarity: sim.toFixed(4),
+                    content_difference: jaccard.toFixed(4),
+                    resolution: 'kept_old'
+                  });
+                  break; // New memory was archived, stop contradiction check
+                }
               }
             }
           }
         } catch (e) {
-          // Contradiction detection is best-effort, don't fail the memory insertion
           console.error(`[persyst] Contradiction detection error: ${e.message}`);
         }
 
@@ -499,7 +532,19 @@ export function registerTools(server) {
     },
     async ({ query }) => {
       try {
-        const hits = searchAllMemoriesFts(query, 5);
+        let hits = [];
+        const queryAsId = Number(query);
+        if (!isNaN(queryAsId) && Number.isInteger(queryAsId)) {
+          const mem = getAnyMemoryById(queryAsId);
+          if (mem) {
+            hits.push({ id: mem.id });
+          }
+        }
+
+        if (hits.length === 0) {
+          hits = searchAllMemoriesFts(query, 5);
+        }
+
         if (hits.length === 0) {
           return text({ message: 'No memories matching query found.' });
         }
@@ -507,6 +552,14 @@ export function registerTools(server) {
         const histories = {};
         for (const hit of hits) {
           const chain = getMemoryHistoryChain(hit.id);
+          // Decorate chain versions with semantic diffs from the previous version
+          for (let idx = 0; idx < chain.length; idx++) {
+            if (idx > 0) {
+              chain[idx].diff_from_previous = diffWords(chain[idx - 1].content, chain[idx].content);
+            } else {
+              chain[idx].diff_from_previous = null;
+            }
+          }
           histories[hit.id] = chain;
         }
 
@@ -716,3 +769,61 @@ function jaccardDistance(a, b) {
   if (union === 0) return 0;
   return 1 - (intersection / union);
 }
+
+/**
+ * Compute word-level diff between two text strings using dynamic programming.
+ * Highlights additions as [+added+] and deletions as [-deleted-].
+ * @param {string} oldStr - Original text
+ * @param {string} newStr - New version of text
+ * @returns {string} Diff string
+ */
+function diffWords(oldStr, newStr) {
+  const oldWords = oldStr.split(/(\s+)/);
+  const newWords = newStr.split(/(\s+)/);
+  
+  const dp = Array(oldWords.length + 1).fill(0).map(() => Array(newWords.length + 1).fill(0));
+  
+  for (let i = 1; i <= oldWords.length; i++) {
+    for (let j = 1; j <= newWords.length; j++) {
+      if (oldWords[i-1] === newWords[j-1]) {
+        dp[i][j] = dp[i-1][j-1] + 1;
+      } else {
+        dp[i][j] = Math.max(dp[i-1][j], dp[i][j-1]);
+      }
+    }
+  }
+  
+  let i = oldWords.length;
+  let j = newWords.length;
+  const result = [];
+  
+  while (i > 0 || j > 0) {
+    if (i > 0 && j > 0 && oldWords[i-1] === newWords[j-1]) {
+      result.unshift({ type: 'common', value: oldWords[i-1] });
+      i--;
+      j--;
+    } else if (j > 0 && (i === 0 || dp[i][j-1] >= dp[i-1][j])) {
+      result.unshift({ type: 'added', value: newWords[j-1] });
+      j--;
+    } else {
+      result.unshift({ type: 'removed', value: oldWords[i-1] });
+      i--;
+    }
+  }
+
+  // Combine consecutive items of the same type
+  const combined = [];
+  for (const part of result) {
+    if (combined.length > 0 && combined[combined.length - 1].type === part.type) {
+      combined[combined.length - 1].value += part.value;
+    } else {
+      combined.push({ ...part });
+    }
+  }
+
+  return combined.map(part => {
+    if (part.type === 'added') return `[+${part.value}+]`;
+    if (part.type === 'removed') return `[-${part.value}-]`;
+    return part.value;
+  }).join('');
+}

package/src/watcher.js

@@ -0,0 +1,306 @@
+/**
+ * watcher.js — Persyst Automatic Log Watcher Daemon
+ * 
+ * Periodically scans configured log directories for coding agents (Antigravity, Roo-Code, etc.),
+ * reads new appends/messages from transcripts, runs heuristics to extract high-value memories,
+ * and stores them in the local database.
+ */
+
+import { join, resolve } from 'path';
+import { homedir } from 'os';
+import { existsSync, readFileSync, writeFileSync, readdirSync, statSync } from 'fs';
+import {
+  getWatchPosition,
+  upsertWatchPosition,
+  insertMemory,
+  insertVector,
+  memoryExists
+} from './database.js';
+import { generateEmbedding } from './embeddings.js';
+import { extractHeuristic } from './extractor-heuristic.js';
+import { searchHybrid } from './search.js';
+import { searchCache } from './cache.js';
+
+// Config path: ~/.persyst/config.json
+const CONFIG_FILE = join(homedir(), '.persyst', 'config.json');
+
+let intervalId = null;
+const DEDUP_THRESHOLD = 0.80;
+
+/**
+ * Load configured directories to watch. Generates a default config if missing.
+ * @returns {Array<string>} List of directory paths to scan
+ */
+export function loadWatchedDirs() {
+  const defaultDirs = [
+    join(homedir(), '.gemini', 'antigravity-ide', 'brain').replace(/\\/g, '/')
+  ];
+
+  // Also probe standard paths for Cline / Roo Code
+  const platform = process.platform;
+  if (platform === 'win32') {
+    const appData = process.env.APPDATA || join(homedir(), 'AppData', 'Roaming');
+    const rooPath = join(appData, 'Roo-Code', 'tasks').replace(/\\/g, '/');
+    if (existsSync(rooPath)) defaultDirs.push(rooPath);
+    const clinePath = join(appData, 'Code', 'User', 'globalStorage', 'saoudrizwan.claude-dev', 'tasks').replace(/\\/g, '/');
+    if (existsSync(clinePath)) defaultDirs.push(clinePath);
+  } else if (platform === 'darwin') {
+    const rooPath = join(homedir(), 'Library', 'Application Support', 'Roo-Code', 'tasks');
+    if (existsSync(rooPath)) defaultDirs.push(rooPath);
+    const clinePath = join(homedir(), 'Library', 'Application Support', 'Code', 'User', 'globalStorage', 'saoudrizwan.claude-dev', 'tasks');
+    if (existsSync(clinePath)) defaultDirs.push(clinePath);
+  } else {
+    const rooPath = join(homedir(), '.config', 'Roo-Code', 'tasks');
+    if (existsSync(rooPath)) defaultDirs.push(rooPath);
+    const clinePath = join(homedir(), '.config', 'Code', 'User', 'globalStorage', 'saoudrizwan.claude-dev', 'tasks');
+    if (existsSync(clinePath)) defaultDirs.push(clinePath);
+  }
+
+  try {
+    if (existsSync(CONFIG_FILE)) {
+      const config = JSON.parse(readFileSync(CONFIG_FILE, 'utf8'));
+      if (Array.isArray(config.watch_dirs)) {
+        return config.watch_dirs;
+      }
+    }
+  } catch (_) {
+    // Fail-safe: rewrite or fallback
+  }
+
+  // Create default config file if it does not exist
+  try {
+    writeFileSync(CONFIG_FILE, JSON.stringify({ watch_dirs: defaultDirs }, null, 2));
+  } catch (_) {}
+
+  return defaultDirs;
+}
+
+/**
+ * Scan a single transcript file in JSONL format (Antigravity).
+ * @param {string} filePath 
+ */
+async function processJsonlFile(filePath) {
+  try {
+    const stat = statSync(filePath);
+    const lastPos = getWatchPosition(filePath);
+
+    if (stat.size <= lastPos) return;
+
+    // Read only new content appended to the file
+    const fileBuffer = readFileSync(filePath);
+    const newContentBuffer = fileBuffer.subarray(lastPos, stat.size);
+    const text = newContentBuffer.toString('utf8');
+
+    const lines = text.split('\n');
+    let addedCount = 0;
+
+    for (const line of lines) {
+      if (!line.trim()) continue;
+
+      let record;
+      try {
+        record = JSON.parse(line);
+      } catch (_) {
+        // Line might be incomplete/partially written — skip and parse next time
+        continue;
+      }
+
+      // Check if it's user prompt or assistant response
+      if (
+        record.content &&
+        (record.type === 'USER_INPUT' || record.type === 'PLANNER_RESPONSE' || record.source === 'MODEL')
+      ) {
+        // Strip XML/markdown wrapper tags (like <USER_REQUEST> or <ADDITIONAL_METADATA>)
+        const cleanText = record.content.replace(/<[^>]+>[\s\S]*?<\/[^>]+>/g, '').trim();
+        if (cleanText.length < 15) continue;
+
+        const facts = extractHeuristic(cleanText);
+        for (const fact of facts) {
+          // Verify against exact duplicate
+          if (memoryExists(fact.content)) continue;
+
+          // Verify against semantic similarity
+          const similar = await searchHybrid(fact.content, 1);
+          if (similar.length > 0 && parseFloat(similar[0].similarity) >= DEDUP_THRESHOLD) {
+            continue;
+          }
+
+          // Insert memory with provenance
+          const id = insertMemory(fact.content, fact.confidence, {
+            source_type: 'agent',
+            source_id: record.source === 'MODEL' ? 'antigravity-worker' : 'user-dialogue',
+            confidence: fact.confidence
+          });
+
+          const embedding = await generateEmbedding(fact.content);
+          insertVector(id, embedding);
+          addedCount++;
+          console.error(`[persyst-watcher] Auto-extracted fact: "${fact.content}" (Memory #${id})`);
+        }
+      }
+    }
+
+    if (addedCount > 0) {
+      searchCache.invalidate();
+    }
+
+    // Persist new byte offset position
+    upsertWatchPosition(filePath, stat.size);
+  } catch (err) {
+    console.error(`[persyst-watcher] Failed to process JSONL file ${filePath}: ${err.message}`);
+  }
+}
+
+/**
+ * Scan a single task file in JSON format (Roo Code / Cline).
+ * @param {string} filePath 
+ */
+async function processJsonFile(filePath) {
+  try {
+    const lastMsgCount = getWatchPosition(filePath);
+
+    // Read full JSON (JSON objects are written entirely, not appended)
+    const contentText = readFileSync(filePath, 'utf8');
+    let task;
+    try {
+      task = JSON.parse(contentText);
+    } catch (_) {
+      return; // incomplete JSON, try again later
+    }
+
+    const history = task.history;
+    if (!Array.isArray(history) || history.length <= lastMsgCount) return;
+
+    let addedCount = 0;
+    // Process only newly added messages
+    for (let i = lastMsgCount; i < history.length; i++) {
+      const msg = history[i];
+      if (!msg.content || typeof msg.content !== 'string') continue;
+
+      // Filter out system message structures
+      if (msg.role === 'user' || msg.role === 'assistant') {
+        const facts = extractHeuristic(msg.content);
+        for (const fact of facts) {
+          if (memoryExists(fact.content)) continue;
+
+          const similar = await searchHybrid(fact.content, 1);
+          if (similar.length > 0 && parseFloat(similar[0].similarity) >= DEDUP_THRESHOLD) {
+            continue;
+          }
+
+          const id = insertMemory(fact.content, fact.confidence, {
+            source_type: 'agent',
+            source_id: msg.role === 'assistant' ? 'roo-worker' : 'user-dialogue',
+            confidence: fact.confidence
+          });
+
+          const embedding = await generateEmbedding(fact.content);
+          insertVector(id, embedding);
+          addedCount++;
+          console.error(`[persyst-watcher] Auto-extracted fact: "${fact.content}" (Memory #${id})`);
+        }
+      }
+    }
+
+    if (addedCount > 0) {
+      searchCache.invalidate();
+    }
+
+    // Persist message count index
+    upsertWatchPosition(filePath, history.length);
+  } catch (err) {
+    console.error(`[persyst-watcher] Failed to process JSON file ${filePath}: ${err.message}`);
+  }
+}
+
+/**
+ * Find files ending with a given extension recursively in a folder up to a certain depth.
+ * @param {string} dir 
+ * @param {string} ext 
+ * @param {number} depth 
+ * @returns {Array<string>}
+ */
+function findFiles(dir, ext, depth = 3) {
+  const results = [];
+  if (depth < 0) return results;
+
+  try {
+    if (!existsSync(dir)) return results;
+    const items = readdirSync(dir);
+    for (const item of items) {
+      const path = join(dir, item);
+      let stat;
+      try { stat = statSync(path); } catch (_) { continue; }
+
+      if (stat.isDirectory()) {
+        results.push(...findFiles(path, ext, depth - 1));
+      } else if (item.endsWith(ext)) {
+        results.push(path);
+      }
+    }
+  } catch (_) {}
+
+  return results;
+}
+
+/**
+ * Perform a single scan of watched directories.
+ */
+export async function scanDirectories() {
+  const watchDirs = loadWatchedDirs();
+
+  for (const dir of watchDirs) {
+    if (!existsSync(dir)) continue;
+
+    // Scan for JSONL (Antigravity transcripts)
+    const jsonlFiles = findFiles(dir, 'transcript.jsonl', 3);
+    for (const file of jsonlFiles) {
+      await processJsonlFile(file);
+    }
+
+    // Scan for JSON (Roo Code / Cline task files)
+    const jsonFiles = findFiles(dir, '.json', 2);
+    for (const file of jsonFiles) {
+      // Avoid processing general configurations/settings files
+      if (file.includes('tasks')) {
+        await processJsonFile(file);
+      }
+    }
+  }
+}
+
+/**
+ * Start the background log watcher daemon.
+ */
+export function startWatcher() {
+  if (intervalId) return;
+
+  console.error('[persyst-watcher] Starting background log watcher daemon...');
+  // Warm up config/paths
+  loadWatchedDirs();
+
+  // Run initial scan
+  scanDirectories().catch(err => {
+    console.error(`[persyst-watcher] Initial scan failed: ${err.message}`);
+  });
+
+  // Polling directory scan every 5 seconds
+  intervalId = setInterval(async () => {
+    try {
+      await scanDirectories();
+    } catch (err) {
+      console.error(`[persyst-watcher] Folder scan failed: ${err.message}`);
+    }
+  }, 5000);
+}
+
+/**
+ * Stop the background log watcher daemon.
+ */
+export function stopWatcher() {
+  if (intervalId) {
+    clearInterval(intervalId);
+    intervalId = null;
+    console.error('[persyst-watcher] Background log watcher daemon stopped.');
+  }
+}