shieldcortex 4.15.0 → 4.17.0

package/scripts/pre-compact-hook.mjs

@@ -21,6 +21,12 @@ import { readTranscriptText } from './lib/transcript-reader.mjs';
 import { getAutoMemoryConfig } from './lib/auto-memory-config.mjs';
 import { recordHookInvocation } from './lib/telemetry.mjs';
 import { deriveProjectKey } from './lib/project-key.mjs';
+import { recordSessionEvent } from './lib/session-capture.mjs';
+import {
+  extractMemorableSegments,
+  processSegments,
+  PRE_COMPACT_CATEGORY_THRESHOLDS,
+} from './lib/extract-memorable-segments.mjs';
 
 // Database paths (with legacy fallback)
 const NEW_DB_DIR = join(homedir(), '.shieldcortex');
@@ -42,31 +48,13 @@ const { dir: DB_DIR, path: DB_PATH } = getDbPath();
 const MAX_SHORT_TERM_MEMORIES = 100;
 const MAX_LONG_TERM_MEMORIES = 1000;
 
-// Base salience threshold (will be adjusted dynamically)
-// Lowered from 0.45 to capture more content
-const BASE_THRESHOLD = 0.35;
-
-// Category-specific extraction thresholds (lower = easier to extract).
-// Raised +0.1 across the board in v4.11.0 after fleet evidence showed the
-// previous permissive thresholds produced ~5% signal and flooded recall with
-// noise. Prefer missing a marginal memory to saving a noisy one.
-const CATEGORY_EXTRACTION_THRESHOLDS = {
-  architecture: 0.38,
-  error: 0.40,
-  context: 0.42,
-  learning: 0.42,
-  pattern: 0.45,
-  preference: 0.48,
-  note: 0.52,
-  todo: 0.50,
-  relationship: 0.45,
-  custom: 0.45,
-};
-
 // Maximum memories to auto-create per compaction.
 // Dropped 5 → 2 in v4.11.0 for the same reason thresholds were raised.
 const MAX_AUTO_MEMORIES = 2;
 
+// Pre-compact uses tighter category thresholds (raised +0.10 in v4.11.0)
+// — see PRE_COMPACT_CATEGORY_THRESHOLDS in scripts/lib/extract-memorable-segments.mjs.
+
 // ==================== DYNAMIC THRESHOLD CALCULATION ====================
 
 /**
@@ -103,357 +91,10 @@ function getDynamicThreshold(memoryCount, maxMemories) {
   return 0.25;                       // Very sparse - accept most valuable items
 }
 
-/**
- * Get extraction threshold for a specific category
- * Combines dynamic threshold with category-specific adjustments
- */
-function getExtractionThreshold(category, dynamicThreshold) {
-  const categoryThreshold = CATEGORY_EXTRACTION_THRESHOLDS[category] || BASE_THRESHOLD;
-  // Use whichever is lower (more permissive for valuable categories when memory is sparse)
-  return Math.min(categoryThreshold, dynamicThreshold);
-}
-
-// ==================== SALIENCE DETECTION (Mirrors src/memory/salience.ts) ====================
-
-const ARCHITECTURE_KEYWORDS = [
-  'architecture', 'design', 'pattern', 'structure', 'system',
-  'database', 'api', 'schema', 'model', 'framework', 'stack',
-  'microservice', 'monolith', 'serverless', 'infrastructure'
-];
-
-const ERROR_KEYWORDS = [
-  'error', 'bug', 'fix', 'issue', 'problem', 'crash', 'fail',
-  'exception', 'debug', 'resolve', 'solution', 'workaround'
-];
-
-const PREFERENCE_KEYWORDS = [
-  'prefer', 'always', 'never', 'style', 'convention', 'standard',
-  'like', 'want', 'should', 'must', 'require'
-];
-
-const PATTERN_KEYWORDS = [
-  'pattern', 'practice', 'approach', 'method', 'technique',
-  'implementation', 'strategy', 'algorithm', 'workflow'
-];
-
-const DECISION_KEYWORDS = [
-  'decided', 'decision', 'chose', 'chosen', 'selected', 'going with',
-  'will use', 'opted for', 'settled on', 'agreed'
-];
-
-const LEARNING_KEYWORDS = [
-  'learned', 'discovered', 'realized', 'found out', 'turns out',
-  'TIL', 'now know', 'understand now', 'figured out'
-];
-
-const EMOTIONAL_MARKERS = [
-  'important', 'critical', 'crucial', 'essential', 'key',
-  'finally', 'breakthrough', 'eureka', 'aha', 'got it',
-  'frustrating', 'annoying', 'tricky', 'remember'
-];
-
-const CODE_REFERENCE_PATTERNS = [
-  /\b[A-Z][a-zA-Z]*\.[a-zA-Z]+\b/,
-  /\b[a-z_][a-zA-Z0-9_]*\.(ts|js|py|go|rs)\b/,
-  /`[^`]+`/,
-  /\b(function|class|interface|type|const|let|var)\s+\w+/i,
-  /\bline\s*\d+\b/i,
-  /\b(src|lib|app|components?)\/\S+/,
-];
-
-function detectKeywords(text, keywords) {
-  const lower = text.toLowerCase();
-  return keywords.some(keyword => lower.includes(keyword.toLowerCase()));
-}
-
-function detectCodeReferences(content) {
-  return CODE_REFERENCE_PATTERNS.some(pattern => pattern.test(content));
-}
-
-function detectExplicitRequest(text) {
-  const patterns = [
-    /\bremember\s+(this|that)\b/i,
-    /\bdon'?t\s+forget\b/i,
-    /\bkeep\s+(in\s+)?mind\b/i,
-    /\bnote\s+(this|that)\b/i,
-    /\bsave\s+(this|that)\b/i,
-    /\bimportant[:\s]/i,
-    /\bfor\s+future\s+reference\b/i,
-  ];
-  return patterns.some(pattern => pattern.test(text));
-}
-
-function calculateSalience(text) {
-  let score = 0.25; // Base score
-
-  if (detectExplicitRequest(text)) score += 0.5;
-  if (detectKeywords(text, ARCHITECTURE_KEYWORDS)) score += 0.4;
-  if (detectKeywords(text, ERROR_KEYWORDS)) score += 0.35;
-  if (detectKeywords(text, DECISION_KEYWORDS)) score += 0.35;
-  if (detectKeywords(text, LEARNING_KEYWORDS)) score += 0.3;
-  if (detectKeywords(text, PATTERN_KEYWORDS)) score += 0.25;
-  if (detectKeywords(text, PREFERENCE_KEYWORDS)) score += 0.25;
-  if (detectCodeReferences(text)) score += 0.15;
-  if (detectKeywords(text, EMOTIONAL_MARKERS)) score += 0.2;
-
-  return Math.min(1.0, score);
-}
-
-function suggestCategory(text) {
-  const lower = text.toLowerCase();
-
-  if (detectKeywords(lower, ARCHITECTURE_KEYWORDS)) return 'architecture';
-  if (detectKeywords(lower, ERROR_KEYWORDS)) return 'error';
-  if (detectKeywords(lower, DECISION_KEYWORDS)) return 'context';
-  if (detectKeywords(lower, LEARNING_KEYWORDS)) return 'learning';
-  if (detectKeywords(lower, PREFERENCE_KEYWORDS)) return 'preference';
-  if (detectKeywords(lower, PATTERN_KEYWORDS)) return 'pattern';
-  if (/\b(todo|fixme|hack|xxx)\b/i.test(lower)) return 'todo';
-
-  return 'note';
-}
-
-function extractTags(text, extractorName = null) {
-  const tags = new Set();
-
-  // Extract hashtags
-  const hashtagMatches = text.match(/#[a-zA-Z][a-zA-Z0-9_-]*/g);
-  if (hashtagMatches) {
-    hashtagMatches.forEach(tag => tags.add(tag.slice(1).toLowerCase()));
-  }
-
-  // Extract common tech terms
-  const techTerms = [
-    'react', 'vue', 'angular', 'node', 'python', 'typescript', 'javascript',
-    'api', 'database', 'sql', 'mongodb', 'postgresql', 'mysql',
-    'docker', 'kubernetes', 'aws', 'git', 'testing', 'auth', 'security'
-  ];
-
-  const lowerText = text.toLowerCase();
-  techTerms.forEach(term => {
-    if (lowerText.includes(term)) tags.add(term);
-  });
-
-  // Add auto-extracted tag
-  tags.add('auto-extracted');
-
-  // Add source extractor tag for tracking
-  if (extractorName) {
-    tags.add(`source:${extractorName}`);
-  }
-
-  return Array.from(tags).slice(0, 12);
-}
-
-/**
- * Calculate frequency boost based on how often key terms appear
- * across all extracted segments. Repeated topics are more important.
- */
-function calculateFrequencyBoost(segment, allSegments) {
-  // Extract key terms (words > 5 chars that aren't common)
-  const commonWords = new Set([
-    'about', 'after', 'before', 'being', 'between', 'could', 'during',
-    'every', 'found', 'through', 'would', 'should', 'which', 'where',
-    'there', 'these', 'their', 'other', 'using', 'because', 'without'
-  ]);
-
-  const words = segment.content.toLowerCase().split(/\s+/);
-  const keyTerms = words.filter(w =>
-    w.length > 5 &&
-    !commonWords.has(w) &&
-    /^[a-z]+$/.test(w)
-  );
-
-  let boost = 0;
-  const seenTerms = new Set();
-
-  for (const term of keyTerms) {
-    if (seenTerms.has(term)) continue;
-    seenTerms.add(term);
-
-    // Count how many other segments mention this term
-    const mentions = allSegments.filter(s =>
-      s !== segment &&
-      s.content.toLowerCase().includes(term)
-    ).length;
-
-    // Boost for repeated topics (cap at 5 mentions)
-    if (mentions > 1) {
-      boost += 0.03 * Math.min(mentions, 5);
-    }
-  }
-
-  // Cap total frequency boost at 0.15
-  return Math.min(0.15, boost);
-}
-
-// ==================== CONTENT EXTRACTION ====================
-
-/**
- * Extract meaningful segments from conversation text
- * Looks for decisions, learnings, fixes, patterns, etc.
- */
-function extractMemorableSegments(conversationText) {
-  const segments = [];
-
-  // Pattern matchers for different types of important content
-  // Expanded patterns with lower minimum lengths for better capture
-  const extractors = [
-    {
-      name: 'decision',
-      patterns: [
-        /(?:we\s+)?decided\s+(?:to\s+)?(.{15,200})/gi,
-        /(?:going|went)\s+with\s+(.{15,150})/gi,
-        /(?:chose|chosen|selected)\s+(.{15,150})/gi,
-        /the\s+(?:approach|solution|fix)\s+(?:is|was)\s+(.{15,200})/gi,
-        // New patterns
-        /(?:using|will\s+use)\s+(.{15,150})/gi,
-        /(?:opted\s+for|settled\s+on)\s+(.{15,150})/gi,
-      ],
-      titlePrefix: 'Decision: ',
-    },
-    {
-      name: 'error-fix',
-      patterns: [
-        /(?:fixed|solved|resolved)\s+(?:by\s+)?(.{15,200})/gi,
-        /the\s+(?:fix|solution|workaround)\s+(?:is|was)\s+(.{15,200})/gi,
-        /(?:root\s+cause|issue)\s+(?:is|was)\s+(.{15,200})/gi,
-        /(?:error|bug)\s+(?:was\s+)?caused\s+by\s+(.{15,200})/gi,
-        // New patterns
-        /(?:problem|issue)\s+was\s+(.{15,150})/gi,
-        /(?:the\s+)?bug\s+(?:is|was)\s+(.{15,150})/gi,
-        /(?:debugging|debugged)\s+(.{15,150})/gi,
-      ],
-      titlePrefix: 'Fix: ',
-    },
-    {
-      name: 'learning',
-      patterns: [
-        /(?:learned|discovered|realized|found\s+out)\s+(?:that\s+)?(.{15,200})/gi,
-        /turns\s+out\s+(?:that\s+)?(.{15,200})/gi,
-        /(?:TIL|today\s+I\s+learned)[:\s]+(.{15,200})/gi,
-        // New patterns
-        /(?:now\s+)?(?:understand|know)\s+(?:that\s+)?(.{15,150})/gi,
-        /(?:figured\s+out|worked\s+out)\s+(.{15,150})/gi,
-      ],
-      titlePrefix: 'Learned: ',
-    },
-    {
-      name: 'architecture',
-      patterns: [
-        /the\s+architecture\s+(?:is|uses|consists\s+of)\s+(.{15,200})/gi,
-        /(?:design|pattern)\s+(?:is|uses)\s+(.{15,200})/gi,
-        /(?:system|api|database)\s+(?:structure|design)\s+(?:is|uses)\s+(.{15,200})/gi,
-        // New patterns
-        /(?:created|added|implemented|built)\s+(?:a\s+)?(.{15,200})/gi,
-        /(?:refactored|updated|changed)\s+(?:the\s+)?(.{15,150})/gi,
-      ],
-      titlePrefix: 'Architecture: ',
-    },
-    {
-      name: 'preference',
-      patterns: [
-        /(?:always|never)\s+(.{10,150})/gi,
-        /(?:prefer|want)\s+to\s+(.{10,150})/gi,
-        /(?:should|must)\s+(?:always\s+)?(.{10,150})/gi,
-      ],
-      titlePrefix: 'Preference: ',
-    },
-    {
-      name: 'important-note',
-      patterns: [
-        /important[:\s]+(.{15,200})/gi,
-        /(?:note|remember)[:\s]+(.{15,200})/gi,
-        /(?:key|critical)\s+(?:point|thing)[:\s]+(.{15,200})/gi,
-        // New patterns
-        /(?:this\s+is\s+)?(?:crucial|essential)[:\s]+(.{15,150})/gi,
-        /(?:don't\s+forget|keep\s+in\s+mind)[:\s]+(.{15,150})/gi,
-      ],
-      titlePrefix: 'Note: ',
-    },
-  ];
-
-  for (const extractor of extractors) {
-    for (const pattern of extractor.patterns) {
-      let match;
-      while ((match = pattern.exec(conversationText)) !== null) {
-        const content = match[1].trim();
-        if (content.length >= 20) {
-          // Generate a title from first ~50 chars
-          const titleContent = content.slice(0, 50).replace(/\s+/g, ' ').trim();
-          const title = extractor.titlePrefix + (titleContent.length < 50 ? titleContent : titleContent + '...');
-
-          segments.push({
-            title,
-            content: content.slice(0, 500), // Cap content length
-            extractorType: extractor.name,
-          });
-        }
-      }
-    }
-  }
-
-  return segments;
-}
-
-/**
- * Deduplicate and score segments
- * @param {Array} segments - Raw extracted segments
- * @param {number} dynamicThreshold - Dynamic threshold based on memory fullness
- */
-function processSegments(segments, dynamicThreshold = BASE_THRESHOLD) {
-  // Remove near-duplicates (segments with >80% overlap)
-  const unique = [];
-  for (const seg of segments) {
-    const isDupe = unique.some(existing => {
-      const overlap = calculateOverlap(existing.content, seg.content);
-      return overlap > 0.8;
-    });
-    if (!isDupe) {
-      const text = seg.title + ' ' + seg.content;
-      const baseSalience = calculateSalience(text);
-      const category = suggestCategory(text);
-
-      unique.push({
-        ...seg,
-        baseSalience,
-        category,
-        tags: extractTags(text, seg.extractorType),
-      });
-    }
-  }
-
-  // Calculate frequency boost after we have all unique segments
-  for (const seg of unique) {
-    const frequencyBoost = calculateFrequencyBoost(seg, unique);
-    seg.salience = Math.min(1.0, seg.baseSalience + frequencyBoost);
-    seg.frequencyBoost = frequencyBoost;
-  }
-
-  // Sort by salience (highest first)
-  unique.sort((a, b) => b.salience - a.salience);
-
-  // Filter by category-specific threshold (combined with dynamic threshold)
-  const filtered = unique.filter(seg => {
-    const threshold = getExtractionThreshold(seg.category, dynamicThreshold);
-    return seg.salience >= threshold;
-  });
-
-  return filtered.slice(0, MAX_AUTO_MEMORIES);
-}
-
-/**
- * Simple overlap calculation (Jaccard similarity on words)
- */
-function calculateOverlap(text1, text2) {
-  const words1 = new Set(text1.toLowerCase().split(/\s+/));
-  const words2 = new Set(text2.toLowerCase().split(/\s+/));
-
-  const intersection = new Set([...words1].filter(w => words2.has(w)));
-  const union = new Set([...words1, ...words2]);
+// getExtractionThreshold lives in the shared chunker module.
 
-  return intersection.size / union.size;
-}
+// Salience detection, content extraction, and segment processing now
+// live in scripts/lib/extract-memorable-segments.mjs (imported above).
 
 // ==================== DATABASE OPERATIONS ====================
 
@@ -461,7 +102,7 @@ function calculateOverlap(text1, text2) {
 // write lives in scripts/lib/save-memory.mjs so pre-compact and
 // session-end share one code path (and one regression test).
 function saveMemory(db, memory, project) {
-  saveAutoExtractedMemory(db, memory, project);
+  return saveAutoExtractedMemory(db, memory, project, { source: 'pre-compact-hook' });
 }
 
 
@@ -477,7 +118,7 @@ process.stdin.on('readable', () => {
   }
 });
 
-process.stdin.on('end', () => {
+process.stdin.on('end', async () => {
   const startedAt = Date.now();
   let db = null;
   let autoExtractedCount = 0;
@@ -514,6 +155,24 @@ process.stdin.on('end', () => {
     // timeout: 5000ms prevents hook from hanging if DB is locked
     db = new Database(DB_PATH, { timeout: 5000 });
 
+    // ── Session capture (v4.17): mark the compaction boundary in the
+    //    replay timeline. Reuses the already-open write connection so
+    //    no extra better-sqlite3 handle is created.
+    {
+      const sessionId = hookData.session_id || hookData.sessionId || null;
+      if (sessionId) {
+        try {
+          recordSessionEvent(db, {
+            session_id: sessionId,
+            ts: new Date().toISOString(),
+            kind: 'hook_fire',
+            payload: { hook: 'pre-compact', trigger, transcript_path: hookData.transcript_path ?? null },
+            project: project || null,
+          });
+        } catch { /* best-effort */ }
+      }
+    }
+
     // Get current memory stats for dynamic threshold calculation
     const stats = getMemoryStats(db);
     const totalMemories = stats.shortTerm + stats.longTerm;
@@ -527,12 +186,16 @@ process.stdin.on('end', () => {
     if (conversationText && conversationText.length > 100) {
       // Extract memorable segments
       const segments = extractMemorableSegments(conversationText);
-      const processedSegments = processSegments(segments, dynamicThreshold);
+      const processedSegments = processSegments(segments, dynamicThreshold, {
+        maxMemories: MAX_AUTO_MEMORIES,
+        categoryThresholds: PRE_COMPACT_CATEGORY_THRESHOLDS,
+        conversationText,
+      });
 
       // Save auto-extracted memories
       for (const memory of processedSegments) {
         try {
-          saveMemory(db, memory, project);
+          await saveMemory(db, memory, project);
           autoExtractedCount++;
           const boostInfo = memory.frequencyBoost > 0 ? ` +${memory.frequencyBoost.toFixed(2)} boost` : '';
           console.error(`[auto-extract] Saved: ${memory.title} (salience: ${memory.salience.toFixed(2)}${boostInfo}, category: ${memory.category})`);

package/scripts/prompt-recall-hook.mjs

@@ -15,6 +15,7 @@ import { join } from 'path';
 import { homedir } from 'os';
 import { deriveProjectKey } from './lib/project-key.mjs';
 import { sanitisePromptForRecall } from './lib/prompt-sanitiser.mjs';
+import { recordSessionEvent } from './lib/session-capture.mjs';
 
 // ==================== CONFIG ====================
 
@@ -153,15 +154,40 @@ process.stdin.on('readable', () => {
 process.stdin.on('end', () => {
   try {
     const config = loadConfig();
+    const hookData = JSON.parse(input || '{}');
+    const rawPrompt = hookData.prompt || '';
+    const cwd = hookData.cwd || process.cwd();
+    const sessionId = hookData.session_id || hookData.sessionId || null;
+
+    // ── Session capture (v4.17): record the user prompt regardless of
+    //    proactiveRecall config so the dashboard replay timeline gets a
+    //    complete event stream. Opt-out via captureEvents=false. Failures
+    //    are swallowed — capture must never block the user prompt.
+    if (config.captureEvents !== false && sessionId && rawPrompt) {
+      try {
+        const project = deriveProjectKey(cwd);
+        const dbPath = getDbPath();
+        if (existsSync(dbPath)) {
+          const captureDb = new Database(dbPath, { timeout: 1000 });
+          recordSessionEvent(captureDb, {
+            session_id: sessionId,
+            ts: new Date().toISOString(),
+            kind: 'prompt',
+            payload: { text: rawPrompt },
+            project: project || null,
+            actor: 'user',
+          });
+          captureDb.close();
+        }
+      } catch {
+        // Capture is best-effort.
+      }
+    }
 
     if (config.proactiveRecall !== true) {
       process.exit(0);
     }
 
-    const hookData = JSON.parse(input || '{}');
-    const rawPrompt = hookData.prompt || '';
-    const cwd = hookData.cwd || process.cwd();
-
     // Strip OpenClaw / framework metadata wrappers (e.g. Telegram channel
     // headers) before any prompt-based decisions. Without this, the FTS5
     // query is built from "Conversation info untrusted metadata json" tokens