persyst-mcp 2.2.0 → 2.2.2

package/bin/extract-worker.js

@@ -30,6 +30,8 @@ import {
 } from 'fs';
 import { fileURLToPath } from 'url';
 import { dirname } from 'path';
+import http from 'http';
+import https from 'https';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -58,6 +60,331 @@ const DEDUP_SIMILARITY_THRESHOLD = 0.80;
 const RECENT_MEMORY_WINDOW_S = 60; // Check last 60 seconds for agent race
 const MIN_CONFIDENCE = 0.65;
 
+// ============================================================
+// LLM EXTRACTION PIPELINE (Tiers 1 & 2)
+// ============================================================
+
+const EXTRACTION_SYSTEM_PROMPT = `You are a precise developer memory extraction assistant.
+Analyze the following developer conversation turn or transcript and extract any:
+1. Explicit user preferences (e.g. "I prefer HSL colors", "I like clean UI")
+2. Architectural decisions (e.g. "We decided to expose a gateway server on port 4321")
+3. Project stack choices (e.g. "Using Node.js for backend")
+4. Coding rules/styles (e.g. "Always use camelCase for variables")
+5. Project config/settings (e.g. "Port is set to 4321")
+
+Do not extract temporary tasks, questions, or vague conversational statements.
+Do not invent facts. Only extract facts that are clearly stated or implied by the developer.
+
+You MUST respond with a valid JSON array of objects, and absolutely NOTHING else. No markdown formatting, no explanation.
+Each object must have the following fields:
+- "content": A clean, concise statement of the preference/decision (e.g., "Preference: Use HSL tailwind colors"). Start the content with the prefix indicating the category: "Preference: ...", "Decision: ...", "Stack: ...", "Rule: ...", "Config: ...".
+- "category": One of "preference", "decision", "stack", "naming", "architecture", "rule", "config".
+- "confidence": A float value between 0.65 and 1.0 representing your confidence.
+
+Example output:
+[
+  {
+    "content": "Preference: Always use vanilla CSS for maximum control.",
+    "category": "preference",
+    "confidence": 0.95
+  }
+]`;
+
+function parseJsonArray(text) {
+  try {
+    let clean = text.trim();
+    if (clean.startsWith('```json')) {
+      clean = clean.slice(7);
+    } else if (clean.startsWith('```')) {
+      clean = clean.slice(3);
+    }
+    if (clean.endsWith('```')) {
+      clean = clean.slice(0, -3);
+    }
+    clean = clean.trim();
+    const arr = JSON.parse(clean);
+    if (Array.isArray(arr)) return arr;
+    return [];
+  } catch (err) {
+    try {
+      const start = text.indexOf('[');
+      const end = text.lastIndexOf(']');
+      if (start !== -1 && end !== -1) {
+        const substring = text.slice(start, end + 1);
+        const arr = JSON.parse(substring);
+        if (Array.isArray(arr)) return arr;
+      }
+    } catch (_) {}
+    return [];
+  }
+}
+
+function extractAnthropic(text, apiKey) {
+  return new Promise((resolve, reject) => {
+    const payload = JSON.stringify({
+      model: 'claude-3-5-sonnet-20241022',
+      max_tokens: 1000,
+      system: EXTRACTION_SYSTEM_PROMPT,
+      messages: [
+        { role: 'user', content: `Extract facts from this transcript:\n\n${text}` }
+      ]
+    });
+
+    const req = https.request({
+      hostname: 'api.anthropic.com',
+      port: 443,
+      path: '/v1/messages',
+      method: 'POST',
+      headers: {
+        'x-api-key': apiKey,
+        'anthropic-version': '2023-06-01',
+        'content-type': 'application/json',
+        'content-length': Buffer.byteLength(payload)
+      },
+      timeout: 8000
+    });
+
+    req.on('response', (res) => {
+      let data = '';
+      res.on('data', chunk => { data += chunk; });
+      res.on('end', () => {
+        try {
+          const parsed = JSON.parse(data);
+          if (res.statusCode !== 200) {
+            reject(new Error(`Anthropic error: ${parsed.error?.message || data}`));
+            return;
+          }
+          const contentText = parsed.content?.[0]?.text || '';
+          resolve(parseJsonArray(contentText));
+        } catch (e) {
+          reject(e);
+        }
+      });
+    });
+
+    req.on('error', reject);
+    req.on('timeout', () => { req.destroy(); reject(new Error('Anthropic timeout')); });
+    req.write(payload);
+    req.end();
+  });
+}
+
+function extractOpenAI(text, apiKey) {
+  return new Promise((resolve, reject) => {
+    const payload = JSON.stringify({
+      model: 'gpt-4o-mini',
+      messages: [
+        { role: 'system', content: EXTRACTION_SYSTEM_PROMPT },
+        { role: 'user', content: `Extract facts from this transcript:\n\n${text}` }
+      ],
+      response_format: { type: 'json_object' }
+    });
+
+    const req = https.request({
+      hostname: 'api.openai.com',
+      port: 443,
+      path: '/v1/chat/completions',
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${apiKey}`,
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(payload)
+      },
+      timeout: 8000
+    });
+
+    req.on('response', (res) => {
+      let data = '';
+      res.on('data', chunk => { data += chunk; });
+      res.on('end', () => {
+        try {
+          const parsed = JSON.parse(data);
+          if (res.statusCode !== 200) {
+            reject(new Error(`OpenAI error: ${parsed.error?.message || data}`));
+            return;
+          }
+          const contentText = parsed.choices?.[0]?.message?.content || '';
+          let obj = JSON.parse(contentText);
+          if (Array.isArray(obj)) {
+            resolve(obj);
+          } else if (obj && typeof obj === 'object') {
+            const key = Object.keys(obj).find(k => Array.isArray(obj[k]));
+            if (key) {
+              resolve(obj[key]);
+            } else {
+              resolve([]);
+            }
+          } else {
+            resolve([]);
+          }
+        } catch (e) {
+          reject(e);
+        }
+      });
+    });
+
+    req.on('error', reject);
+    req.on('timeout', () => { req.destroy(); reject(new Error('OpenAI timeout')); });
+    req.write(payload);
+    req.end();
+  });
+}
+
+function extractOllama(text, model) {
+  return new Promise((resolve, reject) => {
+    const payload = JSON.stringify({
+      model: model,
+      messages: [
+        { role: 'system', content: EXTRACTION_SYSTEM_PROMPT },
+        { role: 'user', content: `Extract facts from this transcript:\n\n${text}` }
+      ],
+      stream: false,
+      format: 'json'
+    });
+
+    const req = http.request({
+      hostname: '127.0.0.1',
+      port: 11434,
+      path: '/api/chat',
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(payload)
+      },
+      timeout: 12000
+    });
+
+    req.on('response', (res) => {
+      let data = '';
+      res.on('data', chunk => { data += chunk; });
+      res.on('end', () => {
+        try {
+          if (res.statusCode !== 200) {
+            reject(new Error(`Ollama error: status ${res.statusCode}`));
+            return;
+          }
+          const parsed = JSON.parse(data);
+          const contentText = parsed.message?.content || '';
+          let obj = JSON.parse(contentText);
+          if (Array.isArray(obj)) {
+            resolve(obj);
+          } else if (obj && typeof obj === 'object') {
+            const key = Object.keys(obj).find(k => Array.isArray(obj[k]));
+            if (key) {
+              resolve(obj[key]);
+            } else {
+              resolve([]);
+            }
+          } else {
+            resolve([]);
+          }
+        } catch (e) {
+          reject(e);
+        }
+      });
+    });
+
+    req.on('error', reject);
+    req.on('timeout', () => { req.destroy(); reject(new Error('Ollama timeout')); });
+    req.write(payload);
+    req.end();
+  });
+}
+
+function checkOllamaAlive() {
+  return new Promise((resolve) => {
+    const req = http.request({
+      hostname: '127.0.0.1',
+      port: 11434,
+      path: '/api/tags',
+      method: 'GET',
+      timeout: 1000
+    });
+
+    req.on('response', (res) => {
+      if (res.statusCode === 200) {
+        let data = '';
+        res.on('data', chunk => { data += chunk; });
+        res.on('end', () => {
+          try {
+            const parsed = JSON.parse(data);
+            const models = parsed.models || [];
+            resolve({ alive: true, models: models.map(m => m.name) });
+          } catch (_) {
+            resolve({ alive: true, models: [] });
+          }
+        });
+      } else {
+        resolve({ alive: false });
+      }
+    });
+
+    req.on('error', () => {
+      resolve({ alive: false });
+    });
+
+    req.on('timeout', () => {
+      req.destroy();
+      resolve({ alive: false });
+    });
+
+    req.end();
+  });
+}
+
+async function extractFacts(text) {
+  // Try Anthropic first
+  if (process.env.ANTHROPIC_API_KEY) {
+    try {
+      log('INFO', 'Attempting Anthropic fact extraction...');
+      const facts = await extractAnthropic(text, process.env.ANTHROPIC_API_KEY);
+      log('INFO', `Anthropic extraction succeeded: extracted ${facts.length} facts.`);
+      return { facts, tier: 'anthropic' };
+    } catch (err) {
+      log('WARN', `Anthropic extraction failed: ${err.message}`);
+    }
+  }
+
+  // Try OpenAI second
+  if (process.env.OPENAI_API_KEY) {
+    try {
+      log('INFO', 'Attempting OpenAI fact extraction...');
+      const facts = await extractOpenAI(text, process.env.OPENAI_API_KEY);
+      log('INFO', `OpenAI extraction succeeded: extracted ${facts.length} facts.`);
+      return { facts, tier: 'openai' };
+    } catch (err) {
+      log('WARN', `OpenAI extraction failed: ${err.message}`);
+    }
+  }
+
+  // Try Ollama third
+  try {
+    const ollamaStatus = await checkOllamaAlive();
+    if (ollamaStatus.alive && ollamaStatus.models && ollamaStatus.models.length > 0) {
+      const targetModel = ollamaStatus.models.find(m => m.includes('qwen') || m.includes('coder') || m.includes('llama')) || ollamaStatus.models[0];
+      if (targetModel) {
+        log('INFO', `Attempting Ollama extraction using model: ${targetModel}...`);
+        const facts = await extractOllama(text, targetModel);
+        log('INFO', `Ollama extraction succeeded: extracted ${facts.length} facts.`);
+        return { facts, tier: \`ollama:\${targetModel}\` };
+      }
+    }
+  } catch (err) {
+    log('WARN', `Ollama extraction failed: ${err.message}`);
+  }
+
+  // Fallback to Heuristic
+  log('INFO', 'All LLM extraction options unavailable/failed. Falling back to Heuristic extraction.');
+  try {
+    const { extractHeuristic } = await import('../src/extractor-heuristic.js');
+    const heuristicFacts = extractHeuristic(text);
+    return { facts: heuristicFacts, tier: 'heuristic' };
+  } catch (err) {
+    log('ERROR', `Heuristic extraction fallback failed: ${err.message}`);
+    return { facts: [], tier: 'failed' };
+  }
+}
+
 
 
 // ============================================================
@@ -285,21 +612,10 @@ async function main() {
       try {
         log('INFO', `Processing: ${filename} (retry: ${retryCount})`);
 
-        const facts = [];
-        let heuristicFacts = [];
-
-        // 1. Run Tier 2 Heuristic Extraction (always safe, zero cost)
-        try {
-          const { extractHeuristic } = await import('../src/extractor-heuristic.js');
-          heuristicFacts = extractHeuristic(data.text);
-          for (const f of heuristicFacts) {
-            facts.push({ ...f, tier: 'heuristic' });
-          }
-        } catch (heurErr) {
-          log('ERROR', `Heuristic extraction failed: ${heurErr.message}`);
-        }
+        const { facts: extractedFacts, tier } = await extractFacts(data.text);
+        const facts = extractedFacts.map(f => ({ ...f, tier }));
 
-        log('INFO', `Extracted ${facts.length} heuristic fact(s)`);
+        log('INFO', `Extracted ${facts.length} fact(s) using tier: ${tier}`);
 
         // Deduplicate facts within this run
         const uniqueFacts = [];

package/bin/setup.js

@@ -61,6 +61,12 @@ const HOOK_CONFIG = {
       matcher: '',
       hooks: [{ ...HOOK_ENTRY, timeout: 10 }]
     }
+  ],
+  Stop: [
+    {
+      matcher: '',
+      hooks: [{ ...HOOK_ENTRY }]
+    }
   ]
 };
 

package/hooks/persyst-hook.js

@@ -33,6 +33,7 @@ import { dirname, resolve, join } from 'path';
 import { spawn } from 'child_process';
 import { writeFileSync, readdirSync, mkdirSync, existsSync } from 'fs';
 import { homedir } from 'os';
+import http from 'http';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -85,6 +86,59 @@ function readStdin() {
 // MCP CLIENT CONNECTION
 // ============================================================
 
+let isHttpAvailable = true;
+let stdioClient = null;
+
+async function getStdioClient() {
+  if (stdioClient) return stdioClient;
+  stdioClient = await connectToPersyst();
+  return stdioClient;
+}
+
+function callToolViaHttp(toolName, args, timeoutMs = 150) {
+  return new Promise((resolve, reject) => {
+    const payload = JSON.stringify({ name: toolName, arguments: args });
+    const req = http.request({
+      hostname: '127.0.0.1',
+      port: 4321,
+      path: '/tool',
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(payload)
+      }
+    });
+
+    req.setTimeout(timeoutMs, () => {
+      req.destroy();
+      reject(new Error('HTTP timeout'));
+    });
+
+    req.on('response', (res) => {
+      if (res.statusCode !== 200) {
+        reject(new Error(`HTTP status ${res.statusCode}`));
+        return;
+      }
+      let data = '';
+      res.on('data', chunk => { data += chunk; });
+      res.on('end', () => {
+        try {
+          resolve(JSON.parse(data));
+        } catch (e) {
+          reject(e);
+        }
+      });
+    });
+
+    req.on('error', (err) => {
+      reject(err);
+    });
+
+    req.write(payload);
+    req.end();
+  });
+}
+
 /**
  * Connect to the Persyst MCP server as a client.
  * Uses StdioClientTransport to spawn and communicate with the server.
@@ -121,7 +175,20 @@ async function connectToPersyst() {
  * Call a Persyst MCP tool and parse the JSON result.
  */
 async function callTool(client, toolName, args) {
-  const result = await client.callTool({ name: toolName, arguments: args });
+  if (isHttpAvailable) {
+    try {
+      const httpResult = await callToolViaHttp(toolName, args, 150);
+      if (httpResult && httpResult.content && httpResult.content[0] && httpResult.content[0].text) {
+        return JSON.parse(httpResult.content[0].text);
+      }
+    } catch (err) {
+      isHttpAvailable = false;
+      process.stderr.write(`[persyst-hook] HTTP Gateway call failed (${err.message}). Falling back to stdio client.\n`);
+    }
+  }
+
+  const activeClient = await getStdioClient();
+  const result = await activeClient.callTool({ name: toolName, arguments: args });
   if (result.content && result.content[0] && result.content[0].text) {
     return JSON.parse(result.content[0].text);
   }
@@ -360,8 +427,6 @@ async function handleStop(input) {
 // ============================================================
 
 async function main() {
-  let client = null;
-
   try {
     const input = await readStdin();
     const eventName = input.hook_event_name;
@@ -379,17 +444,15 @@ async function main() {
       return;
     }
 
-    // Connect to Persyst with hard timeout
     const hookStart = Date.now();
-    client = await connectToPersyst();
 
     let response;
     if (eventName === 'SessionStart') {
-      response = await handleSessionStart(client, input);
+      response = await handleSessionStart(null, input);
     } else if (eventName === 'UserPromptSubmit') {
       // Apply hard timeout for prompt-time hook execution
       response = await Promise.race([
-        handleUserPromptSubmit(client, input),
+        handleUserPromptSubmit(null, input),
         new Promise((resolve) =>
           setTimeout(() => {
             process.stderr.write(`[persyst-hook] UserPromptSubmit hit ${MAX_HOOK_LATENCY_MS}ms timeout, returning partial.\n`);
@@ -409,8 +472,8 @@ async function main() {
     console.log(JSON.stringify({}));
   } finally {
     // Clean up MCP connection
-    if (client) {
-      try { await client.close(); } catch (_) {}
+    if (stdioClient) {
+      try { await stdioClient.close(); } catch (_) {}
     }
     process.exit(0);
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "persyst-mcp",
-  "version": "2.2.0",
+  "version": "2.2.2",
   "description": "Local-first MCP memory server with hybrid keyword + semantic search for coding agents",
   "type": "module",
   "main": "index.js",
@@ -27,7 +27,7 @@
   "scripts": {
     "start": "node index.js",
     "test": "cross-env NODE_ENV=test node test/smoke.js",
-    "test:heavy": "cross-env NODE_ENV=test node --test test/test_*.js",
+    "test:heavy": "cross-env NODE_ENV=test node --test --test-concurrency=1 test/test_*.js",
     "worker": "node bin/extract-worker.js",
     "extract": "node bin/extract.js"
   },

package/src/attestation.js

@@ -60,10 +60,16 @@ export function createAttestation(query, memories, agentId = null, sessionId = n
   // Map memories to {id, content_hash, score}
   const memoriesRetrieved = memories.map(m => {
     const contentHash = crypto.createHash('sha256').update(m.content).digest('hex');
+    let scoreVal = 0;
+    if (m.hybrid_score !== undefined && m.hybrid_score !== null) {
+      scoreVal = m.hybrid_score;
+    } else if (m.score !== undefined && m.score !== null) {
+      scoreVal = m.score;
+    }
     return {
       id: m.id,
       content_hash: contentHash,
-      score: parseFloat(m.hybrid_score || 0)
+      score: parseFloat(scoreVal)
     };
   });