shieldcortex 4.15.0 → 4.16.0

package/scripts/lib/save-memory.mjs

@@ -1,27 +1,84 @@
 import { randomUUID } from 'crypto';
+import { dirname, resolve } from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
 
 /**
- * Insert an auto-extracted memory into the SC database.
+ * Insert an auto-extracted memory into the SC database, routed through the
+ * full defence pipeline.
  *
- * Single source of truth for hook-side memory writes. Both the pre-compact
- * and session-end hooks call this — keeping them in sync prevents the
- * "one hook works, the other silently fails" bug class (e.g. v4.12.4
- * post-fix where pre-compact and session-end would have drifted).
+ * Single source of truth for hook-side memory writes (session-end,
+ * pre-compact, and stop hooks all converge here). Every byte that lands in
+ * `memories` must have passed `runDefencePipeline()` first — that guarantees:
+ *   - a defence_audit row exists for every capture (good or bad)
+ *   - injection-shaped content lands in `quarantine`, not in `memories`
+ *   - a hard BLOCK is dropped with an audit trail (no silent loss)
  *
- * Schema invariant: `memories.uuid` is `TEXT NOT NULL UNIQUE` with no
- * default. Every write path MUST supply a UUID.
+ * If the pipeline cannot be loaded (no dist build, e.g. dev workspace before
+ * `npm run build`) the call is fail-closed: nothing is written to memories
+ * and a stderr warning is printed. A fallback defence_audit row is logged.
  *
- * @param {import('better-sqlite3').Database} db - Open DB handle
+ * Async: the pipeline lives in dist/ as ESM and is loaded via dynamic import.
+ *
+ * @param {import('better-sqlite3').Database} db
  * @param {{ title: string, content: string, category: string, salience: number, tags: string[] }} memory
- * @param {string|null} [project] - Optional project scope
+ * @param {string|null} [project]
+ * @param {{ source?: string }} [opts] — `source` identifies the calling hook
+ *   ('session-end-hook' | 'pre-compact-hook' | 'stop-hook' | 'hook').
  */
-export function saveAutoExtractedMemory(db, memory, project) {
+export async function saveAutoExtractedMemory(db, memory, project, opts = {}) {
+  const sourceIdentifier = opts.source ?? 'hook';
+  const source = { type: 'hook', identifier: sourceIdentifier };
+
+  const defence = await loadDefenceModules(db);
+
+  if (!defence) {
+    writeFallbackAudit(db, memory, project, sourceIdentifier, 'defence_pipeline_unavailable: dist build missing');
+    process.stderr.write(`[shieldcortex save-memory] dropped (defence pipeline unavailable): ${memory.title}\n`);
+    return;
+  }
+
+  let result;
+  try {
+    result = defence.runDefencePipeline(memory.content, memory.title, source, undefined, project ?? undefined);
+  } catch (err) {
+    const msg = err && typeof err === 'object' && 'message' in err ? String(err.message) : String(err);
+    writeFallbackAudit(db, memory, project, sourceIdentifier, `pipeline_error: ${msg}`);
+    process.stderr.write(`[shieldcortex save-memory] dropped (pipeline error): ${memory.title} — ${msg}\n`);
+    return;
+  }
+
+  const decision = result.firewall.result;
+
+  if (decision === 'ALLOW') {
+    insertMemoryRow(db, memory, project);
+    return;
+  }
+
+  if (decision === 'QUARANTINE') {
+    // Route quarantine writes through the singleton's connection so the
+    // audit_id FK reference resolves against the same connection that
+    // wrote the audit row a few lines up in pipeline.ts.
+    const quarantineDb = (typeof _getDatabase === 'function' && defence.isDatabaseInitialized && defence.isDatabaseInitialized())
+      ? _getDatabase()
+      : db;
+    insertQuarantineRow(quarantineDb, memory, project, source, result);
+    process.stderr.write(`[shieldcortex save-memory] quarantined: ${memory.title} — ${result.firewall.reason}\n`);
+    return;
+  }
+
+  // BLOCK — defence_audit row already written by the pipeline. Drop with
+  // a single stderr line for operator visibility.
+  process.stderr.write(`[shieldcortex save-memory] blocked: ${memory.title} — ${result.firewall.reason}\n`);
+}
+
+// ==================== Internal: writes ====================
+
+function insertMemoryRow(db, memory, project) {
   const timestamp = new Date().toISOString();
-  const stmt = db.prepare(`
+  db.prepare(`
     INSERT INTO memories (uuid, title, content, type, category, salience, tags, project, created_at, last_accessed)
     VALUES (?, ?, ?, 'short_term', ?, ?, ?, ?, ?, ?)
-  `);
-  stmt.run(
+  `).run(
     randomUUID(),
     memory.title,
     memory.content,
@@ -33,3 +90,131 @@ export function saveAutoExtractedMemory(db, memory, project) {
     timestamp,
   );
 }
+
+function insertQuarantineRow(db, memory, project, source, result) {
+  // Mirrors the canonical SQL used by src/memory/store.ts:quarantineMemory
+  // so QUARANTINE decisions from any path produce schema-identical rows.
+  const firewallResult = result.firewall.result === 'ALLOW' ? 'BLOCK' : result.firewall.result;
+  db.prepare(`
+    INSERT INTO quarantine (
+      original_title, original_content, project,
+      source_type, source_identifier, reason,
+      threat_indicators, anomaly_score, firewall_result, audit_id, status
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending')
+  `).run(
+    memory.title,
+    memory.content,
+    project || null,
+    source.type,
+    source.identifier,
+    result.firewall.reason,
+    JSON.stringify(result.firewall.threatIndicators),
+    result.firewall.anomalyScore,
+    firewallResult,
+    result.auditId ?? null,
+  );
+}
+
+function writeFallbackAudit(db, memory, project, sourceIdentifier, reason) {
+  // Synthetic audit row for cases where the pipeline could not run.
+  try {
+    db.prepare(`
+      INSERT INTO defence_audit (
+        memory_id, project, timestamp,
+        source_type, source_identifier,
+        trust_score, sensitivity_level, firewall_result,
+        anomaly_score, threat_indicators, blocked_patterns,
+        reason, fragmentation_score, pipeline_duration_ms
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, 'BLOCK', ?, ?, ?, ?, ?, ?)
+    `).run(
+      null,
+      project || null,
+      new Date().toISOString(),
+      'hook',
+      sourceIdentifier,
+      0,
+      'INTERNAL',
+      0,
+      '[]',
+      '[]',
+      reason,
+      null,
+      0,
+    );
+  } catch {
+    // Schema may be older than the audit columns. Better silent here than
+    // raising into the hook — the stderr line above carries the signal.
+  }
+}
+
+// ==================== Internal: lazy dist loader ====================
+
+let _defenceCache = null;
+let _defenceCacheKey = null;
+let _getDatabase = null;
+
+async function loadDefenceModules(db) {
+  // Resolve the dist build relative to this file's location. save-memory.mjs
+  // lives at scripts/lib/, so dist is two directories up.
+  const here = dirname(fileURLToPath(import.meta.url));
+  const distRoot = resolve(here, '..', '..', 'dist');
+
+  if (_defenceCache && _defenceCacheKey === distRoot) {
+    ensureDatabaseSingleton(db, _defenceCache);
+    return _defenceCache;
+  }
+
+  try {
+    const pipelineUrl = pathToFileURL(resolve(distRoot, 'defence', 'pipeline.js')).href;
+    const initUrl = pathToFileURL(resolve(distRoot, 'database', 'init.js')).href;
+
+    const [pipelineMod, initMod] = await Promise.all([
+      import(pipelineUrl),
+      import(initUrl),
+    ]);
+
+    if (typeof pipelineMod.runDefencePipeline !== 'function') return null;
+    if (typeof initMod.initDatabase !== 'function') return null;
+
+    _defenceCache = {
+      runDefencePipeline: pipelineMod.runDefencePipeline,
+      initDatabase: initMod.initDatabase,
+      isDatabaseInitialized: initMod.isDatabaseInitialized,
+      getDatabase: initMod.getDatabase,
+      closeDatabase: initMod.closeDatabase,
+    };
+    _defenceCacheKey = distRoot;
+    _getDatabase = initMod.getDatabase;
+
+    ensureDatabaseSingleton(db, _defenceCache);
+    return _defenceCache;
+  } catch {
+    return null;
+  }
+}
+
+function ensureDatabaseSingleton(db, defence) {
+  // The pipeline's audit + custom-rules layers use getDatabase() (singleton).
+  // Initialise it against the same path the hook is writing to so audit
+  // rows are visible across connections (and to the dashboard).
+  const targetPath = db && db.name ? db.name : null;
+  if (!targetPath || targetPath === ':memory:') return;
+
+  if (defence.isDatabaseInitialized && defence.isDatabaseInitialized()) {
+    try {
+      const current = defence.getDatabase();
+      if (current && current.name === targetPath) return; // already pointed here
+      // Path mismatch (typical in tests using per-case temp DBs) — re-init.
+      if (defence.closeDatabase) defence.closeDatabase();
+    } catch {
+      // If anything throws, fall through and try to (re-)init.
+    }
+  }
+
+  try {
+    defence.initDatabase(targetPath);
+  } catch {
+    // Recoverable: pipeline.ts skips audit + custom-rules gracefully when
+    // the singleton isn't initialised.
+  }
+}

package/scripts/pre-compact-hook.mjs

@@ -21,6 +21,11 @@ import { readTranscriptText } from './lib/transcript-reader.mjs';
 import { getAutoMemoryConfig } from './lib/auto-memory-config.mjs';
 import { recordHookInvocation } from './lib/telemetry.mjs';
 import { deriveProjectKey } from './lib/project-key.mjs';
+import {
+  extractMemorableSegments,
+  processSegments,
+  PRE_COMPACT_CATEGORY_THRESHOLDS,
+} from './lib/extract-memorable-segments.mjs';
 
 // Database paths (with legacy fallback)
 const NEW_DB_DIR = join(homedir(), '.shieldcortex');
@@ -42,31 +47,13 @@ const { dir: DB_DIR, path: DB_PATH } = getDbPath();
 const MAX_SHORT_TERM_MEMORIES = 100;
 const MAX_LONG_TERM_MEMORIES = 1000;
 
-// Base salience threshold (will be adjusted dynamically)
-// Lowered from 0.45 to capture more content
-const BASE_THRESHOLD = 0.35;
-
-// Category-specific extraction thresholds (lower = easier to extract).
-// Raised +0.1 across the board in v4.11.0 after fleet evidence showed the
-// previous permissive thresholds produced ~5% signal and flooded recall with
-// noise. Prefer missing a marginal memory to saving a noisy one.
-const CATEGORY_EXTRACTION_THRESHOLDS = {
-  architecture: 0.38,
-  error: 0.40,
-  context: 0.42,
-  learning: 0.42,
-  pattern: 0.45,
-  preference: 0.48,
-  note: 0.52,
-  todo: 0.50,
-  relationship: 0.45,
-  custom: 0.45,
-};
-
 // Maximum memories to auto-create per compaction.
 // Dropped 5 → 2 in v4.11.0 for the same reason thresholds were raised.
 const MAX_AUTO_MEMORIES = 2;
 
+// Pre-compact uses tighter category thresholds (raised +0.10 in v4.11.0)
+// — see PRE_COMPACT_CATEGORY_THRESHOLDS in scripts/lib/extract-memorable-segments.mjs.
+
 // ==================== DYNAMIC THRESHOLD CALCULATION ====================
 
 /**
@@ -103,357 +90,10 @@ function getDynamicThreshold(memoryCount, maxMemories) {
   return 0.25;                       // Very sparse - accept most valuable items
 }
 
-/**
- * Get extraction threshold for a specific category
- * Combines dynamic threshold with category-specific adjustments
- */
-function getExtractionThreshold(category, dynamicThreshold) {
-  const categoryThreshold = CATEGORY_EXTRACTION_THRESHOLDS[category] || BASE_THRESHOLD;
-  // Use whichever is lower (more permissive for valuable categories when memory is sparse)
-  return Math.min(categoryThreshold, dynamicThreshold);
-}
-
-// ==================== SALIENCE DETECTION (Mirrors src/memory/salience.ts) ====================
-
-const ARCHITECTURE_KEYWORDS = [
-  'architecture', 'design', 'pattern', 'structure', 'system',
-  'database', 'api', 'schema', 'model', 'framework', 'stack',
-  'microservice', 'monolith', 'serverless', 'infrastructure'
-];
-
-const ERROR_KEYWORDS = [
-  'error', 'bug', 'fix', 'issue', 'problem', 'crash', 'fail',
-  'exception', 'debug', 'resolve', 'solution', 'workaround'
-];
-
-const PREFERENCE_KEYWORDS = [
-  'prefer', 'always', 'never', 'style', 'convention', 'standard',
-  'like', 'want', 'should', 'must', 'require'
-];
-
-const PATTERN_KEYWORDS = [
-  'pattern', 'practice', 'approach', 'method', 'technique',
-  'implementation', 'strategy', 'algorithm', 'workflow'
-];
-
-const DECISION_KEYWORDS = [
-  'decided', 'decision', 'chose', 'chosen', 'selected', 'going with',
-  'will use', 'opted for', 'settled on', 'agreed'
-];
-
-const LEARNING_KEYWORDS = [
-  'learned', 'discovered', 'realized', 'found out', 'turns out',
-  'TIL', 'now know', 'understand now', 'figured out'
-];
-
-const EMOTIONAL_MARKERS = [
-  'important', 'critical', 'crucial', 'essential', 'key',
-  'finally', 'breakthrough', 'eureka', 'aha', 'got it',
-  'frustrating', 'annoying', 'tricky', 'remember'
-];
-
-const CODE_REFERENCE_PATTERNS = [
-  /\b[A-Z][a-zA-Z]*\.[a-zA-Z]+\b/,
-  /\b[a-z_][a-zA-Z0-9_]*\.(ts|js|py|go|rs)\b/,
-  /`[^`]+`/,
-  /\b(function|class|interface|type|const|let|var)\s+\w+/i,
-  /\bline\s*\d+\b/i,
-  /\b(src|lib|app|components?)\/\S+/,
-];
-
-function detectKeywords(text, keywords) {
-  const lower = text.toLowerCase();
-  return keywords.some(keyword => lower.includes(keyword.toLowerCase()));
-}
-
-function detectCodeReferences(content) {
-  return CODE_REFERENCE_PATTERNS.some(pattern => pattern.test(content));
-}
-
-function detectExplicitRequest(text) {
-  const patterns = [
-    /\bremember\s+(this|that)\b/i,
-    /\bdon'?t\s+forget\b/i,
-    /\bkeep\s+(in\s+)?mind\b/i,
-    /\bnote\s+(this|that)\b/i,
-    /\bsave\s+(this|that)\b/i,
-    /\bimportant[:\s]/i,
-    /\bfor\s+future\s+reference\b/i,
-  ];
-  return patterns.some(pattern => pattern.test(text));
-}
-
-function calculateSalience(text) {
-  let score = 0.25; // Base score
-
-  if (detectExplicitRequest(text)) score += 0.5;
-  if (detectKeywords(text, ARCHITECTURE_KEYWORDS)) score += 0.4;
-  if (detectKeywords(text, ERROR_KEYWORDS)) score += 0.35;
-  if (detectKeywords(text, DECISION_KEYWORDS)) score += 0.35;
-  if (detectKeywords(text, LEARNING_KEYWORDS)) score += 0.3;
-  if (detectKeywords(text, PATTERN_KEYWORDS)) score += 0.25;
-  if (detectKeywords(text, PREFERENCE_KEYWORDS)) score += 0.25;
-  if (detectCodeReferences(text)) score += 0.15;
-  if (detectKeywords(text, EMOTIONAL_MARKERS)) score += 0.2;
-
-  return Math.min(1.0, score);
-}
-
-function suggestCategory(text) {
-  const lower = text.toLowerCase();
-
-  if (detectKeywords(lower, ARCHITECTURE_KEYWORDS)) return 'architecture';
-  if (detectKeywords(lower, ERROR_KEYWORDS)) return 'error';
-  if (detectKeywords(lower, DECISION_KEYWORDS)) return 'context';
-  if (detectKeywords(lower, LEARNING_KEYWORDS)) return 'learning';
-  if (detectKeywords(lower, PREFERENCE_KEYWORDS)) return 'preference';
-  if (detectKeywords(lower, PATTERN_KEYWORDS)) return 'pattern';
-  if (/\b(todo|fixme|hack|xxx)\b/i.test(lower)) return 'todo';
-
-  return 'note';
-}
-
-function extractTags(text, extractorName = null) {
-  const tags = new Set();
-
-  // Extract hashtags
-  const hashtagMatches = text.match(/#[a-zA-Z][a-zA-Z0-9_-]*/g);
-  if (hashtagMatches) {
-    hashtagMatches.forEach(tag => tags.add(tag.slice(1).toLowerCase()));
-  }
-
-  // Extract common tech terms
-  const techTerms = [
-    'react', 'vue', 'angular', 'node', 'python', 'typescript', 'javascript',
-    'api', 'database', 'sql', 'mongodb', 'postgresql', 'mysql',
-    'docker', 'kubernetes', 'aws', 'git', 'testing', 'auth', 'security'
-  ];
-
-  const lowerText = text.toLowerCase();
-  techTerms.forEach(term => {
-    if (lowerText.includes(term)) tags.add(term);
-  });
-
-  // Add auto-extracted tag
-  tags.add('auto-extracted');
-
-  // Add source extractor tag for tracking
-  if (extractorName) {
-    tags.add(`source:${extractorName}`);
-  }
-
-  return Array.from(tags).slice(0, 12);
-}
-
-/**
- * Calculate frequency boost based on how often key terms appear
- * across all extracted segments. Repeated topics are more important.
- */
-function calculateFrequencyBoost(segment, allSegments) {
-  // Extract key terms (words > 5 chars that aren't common)
-  const commonWords = new Set([
-    'about', 'after', 'before', 'being', 'between', 'could', 'during',
-    'every', 'found', 'through', 'would', 'should', 'which', 'where',
-    'there', 'these', 'their', 'other', 'using', 'because', 'without'
-  ]);
-
-  const words = segment.content.toLowerCase().split(/\s+/);
-  const keyTerms = words.filter(w =>
-    w.length > 5 &&
-    !commonWords.has(w) &&
-    /^[a-z]+$/.test(w)
-  );
-
-  let boost = 0;
-  const seenTerms = new Set();
-
-  for (const term of keyTerms) {
-    if (seenTerms.has(term)) continue;
-    seenTerms.add(term);
-
-    // Count how many other segments mention this term
-    const mentions = allSegments.filter(s =>
-      s !== segment &&
-      s.content.toLowerCase().includes(term)
-    ).length;
-
-    // Boost for repeated topics (cap at 5 mentions)
-    if (mentions > 1) {
-      boost += 0.03 * Math.min(mentions, 5);
-    }
-  }
-
-  // Cap total frequency boost at 0.15
-  return Math.min(0.15, boost);
-}
-
-// ==================== CONTENT EXTRACTION ====================
-
-/**
- * Extract meaningful segments from conversation text
- * Looks for decisions, learnings, fixes, patterns, etc.
- */
-function extractMemorableSegments(conversationText) {
-  const segments = [];
-
-  // Pattern matchers for different types of important content
-  // Expanded patterns with lower minimum lengths for better capture
-  const extractors = [
-    {
-      name: 'decision',
-      patterns: [
-        /(?:we\s+)?decided\s+(?:to\s+)?(.{15,200})/gi,
-        /(?:going|went)\s+with\s+(.{15,150})/gi,
-        /(?:chose|chosen|selected)\s+(.{15,150})/gi,
-        /the\s+(?:approach|solution|fix)\s+(?:is|was)\s+(.{15,200})/gi,
-        // New patterns
-        /(?:using|will\s+use)\s+(.{15,150})/gi,
-        /(?:opted\s+for|settled\s+on)\s+(.{15,150})/gi,
-      ],
-      titlePrefix: 'Decision: ',
-    },
-    {
-      name: 'error-fix',
-      patterns: [
-        /(?:fixed|solved|resolved)\s+(?:by\s+)?(.{15,200})/gi,
-        /the\s+(?:fix|solution|workaround)\s+(?:is|was)\s+(.{15,200})/gi,
-        /(?:root\s+cause|issue)\s+(?:is|was)\s+(.{15,200})/gi,
-        /(?:error|bug)\s+(?:was\s+)?caused\s+by\s+(.{15,200})/gi,
-        // New patterns
-        /(?:problem|issue)\s+was\s+(.{15,150})/gi,
-        /(?:the\s+)?bug\s+(?:is|was)\s+(.{15,150})/gi,
-        /(?:debugging|debugged)\s+(.{15,150})/gi,
-      ],
-      titlePrefix: 'Fix: ',
-    },
-    {
-      name: 'learning',
-      patterns: [
-        /(?:learned|discovered|realized|found\s+out)\s+(?:that\s+)?(.{15,200})/gi,
-        /turns\s+out\s+(?:that\s+)?(.{15,200})/gi,
-        /(?:TIL|today\s+I\s+learned)[:\s]+(.{15,200})/gi,
-        // New patterns
-        /(?:now\s+)?(?:understand|know)\s+(?:that\s+)?(.{15,150})/gi,
-        /(?:figured\s+out|worked\s+out)\s+(.{15,150})/gi,
-      ],
-      titlePrefix: 'Learned: ',
-    },
-    {
-      name: 'architecture',
-      patterns: [
-        /the\s+architecture\s+(?:is|uses|consists\s+of)\s+(.{15,200})/gi,
-        /(?:design|pattern)\s+(?:is|uses)\s+(.{15,200})/gi,
-        /(?:system|api|database)\s+(?:structure|design)\s+(?:is|uses)\s+(.{15,200})/gi,
-        // New patterns
-        /(?:created|added|implemented|built)\s+(?:a\s+)?(.{15,200})/gi,
-        /(?:refactored|updated|changed)\s+(?:the\s+)?(.{15,150})/gi,
-      ],
-      titlePrefix: 'Architecture: ',
-    },
-    {
-      name: 'preference',
-      patterns: [
-        /(?:always|never)\s+(.{10,150})/gi,
-        /(?:prefer|want)\s+to\s+(.{10,150})/gi,
-        /(?:should|must)\s+(?:always\s+)?(.{10,150})/gi,
-      ],
-      titlePrefix: 'Preference: ',
-    },
-    {
-      name: 'important-note',
-      patterns: [
-        /important[:\s]+(.{15,200})/gi,
-        /(?:note|remember)[:\s]+(.{15,200})/gi,
-        /(?:key|critical)\s+(?:point|thing)[:\s]+(.{15,200})/gi,
-        // New patterns
-        /(?:this\s+is\s+)?(?:crucial|essential)[:\s]+(.{15,150})/gi,
-        /(?:don't\s+forget|keep\s+in\s+mind)[:\s]+(.{15,150})/gi,
-      ],
-      titlePrefix: 'Note: ',
-    },
-  ];
-
-  for (const extractor of extractors) {
-    for (const pattern of extractor.patterns) {
-      let match;
-      while ((match = pattern.exec(conversationText)) !== null) {
-        const content = match[1].trim();
-        if (content.length >= 20) {
-          // Generate a title from first ~50 chars
-          const titleContent = content.slice(0, 50).replace(/\s+/g, ' ').trim();
-          const title = extractor.titlePrefix + (titleContent.length < 50 ? titleContent : titleContent + '...');
-
-          segments.push({
-            title,
-            content: content.slice(0, 500), // Cap content length
-            extractorType: extractor.name,
-          });
-        }
-      }
-    }
-  }
-
-  return segments;
-}
-
-/**
- * Deduplicate and score segments
- * @param {Array} segments - Raw extracted segments
- * @param {number} dynamicThreshold - Dynamic threshold based on memory fullness
- */
-function processSegments(segments, dynamicThreshold = BASE_THRESHOLD) {
-  // Remove near-duplicates (segments with >80% overlap)
-  const unique = [];
-  for (const seg of segments) {
-    const isDupe = unique.some(existing => {
-      const overlap = calculateOverlap(existing.content, seg.content);
-      return overlap > 0.8;
-    });
-    if (!isDupe) {
-      const text = seg.title + ' ' + seg.content;
-      const baseSalience = calculateSalience(text);
-      const category = suggestCategory(text);
-
-      unique.push({
-        ...seg,
-        baseSalience,
-        category,
-        tags: extractTags(text, seg.extractorType),
-      });
-    }
-  }
+// getExtractionThreshold lives in the shared chunker module.
 
-  // Calculate frequency boost after we have all unique segments
-  for (const seg of unique) {
-    const frequencyBoost = calculateFrequencyBoost(seg, unique);
-    seg.salience = Math.min(1.0, seg.baseSalience + frequencyBoost);
-    seg.frequencyBoost = frequencyBoost;
-  }
-
-  // Sort by salience (highest first)
-  unique.sort((a, b) => b.salience - a.salience);
-
-  // Filter by category-specific threshold (combined with dynamic threshold)
-  const filtered = unique.filter(seg => {
-    const threshold = getExtractionThreshold(seg.category, dynamicThreshold);
-    return seg.salience >= threshold;
-  });
-
-  return filtered.slice(0, MAX_AUTO_MEMORIES);
-}
-
-/**
- * Simple overlap calculation (Jaccard similarity on words)
- */
-function calculateOverlap(text1, text2) {
-  const words1 = new Set(text1.toLowerCase().split(/\s+/));
-  const words2 = new Set(text2.toLowerCase().split(/\s+/));
-
-  const intersection = new Set([...words1].filter(w => words2.has(w)));
-  const union = new Set([...words1, ...words2]);
-
-  return intersection.size / union.size;
-}
+// Salience detection, content extraction, and segment processing now
+// live in scripts/lib/extract-memorable-segments.mjs (imported above).
 
 // ==================== DATABASE OPERATIONS ====================
 
@@ -461,7 +101,7 @@ function calculateOverlap(text1, text2) {
 // write lives in scripts/lib/save-memory.mjs so pre-compact and
 // session-end share one code path (and one regression test).
 function saveMemory(db, memory, project) {
-  saveAutoExtractedMemory(db, memory, project);
+  return saveAutoExtractedMemory(db, memory, project, { source: 'pre-compact-hook' });
 }
 
 
@@ -477,7 +117,7 @@ process.stdin.on('readable', () => {
   }
 });
 
-process.stdin.on('end', () => {
+process.stdin.on('end', async () => {
   const startedAt = Date.now();
   let db = null;
   let autoExtractedCount = 0;
@@ -527,12 +167,16 @@ process.stdin.on('end', () => {
     if (conversationText && conversationText.length > 100) {
       // Extract memorable segments
       const segments = extractMemorableSegments(conversationText);
-      const processedSegments = processSegments(segments, dynamicThreshold);
+      const processedSegments = processSegments(segments, dynamicThreshold, {
+        maxMemories: MAX_AUTO_MEMORIES,
+        categoryThresholds: PRE_COMPACT_CATEGORY_THRESHOLDS,
+        conversationText,
+      });
 
       // Save auto-extracted memories
       for (const memory of processedSegments) {
         try {
-          saveMemory(db, memory, project);
+          await saveMemory(db, memory, project);
           autoExtractedCount++;
           const boostInfo = memory.frequencyBoost > 0 ? ` +${memory.frequencyBoost.toFixed(2)} boost` : '';
           console.error(`[auto-extract] Saved: ${memory.title} (salience: ${memory.salience.toFixed(2)}${boostInfo}, category: ${memory.category})`);