squish-memory 1.1.5 → 1.2.1

package/core/security/encrypt.ts

@@ -0,0 +1,51 @@
+import { randomBytes, pbkdf2Sync, createCipheriv, createDecipheriv } from 'crypto';
+import { config } from '../../config.js';
+
+const ITERATIONS = 100_000;
+const KEY_LEN = 32; // 256-bit
+const DIGEST = 'sha256';
+const ALGO = 'aes-256-gcm';
+const NONCE_LEN = 12;
+
+/**
+ * Derive a symmetric key from the passphrase and a salt.
+ * The salt is stored in `.squish/salt` (plain text) and is generated once.
+ */
+function getSalt(): Buffer {
+  const fs = require('fs');
+  const path = require('path');
+  const saltPath = path.join(config.dataDir, 'salt');
+  if (!fs.existsSync(saltPath)) {
+    const salt = randomBytes(16).toString('hex');
+    fs.writeFileSync(saltPath, salt);
+    return Buffer.from(salt, 'hex');
+  }
+  return Buffer.from(fs.readFileSync(saltPath, 'utf-8'), 'hex');
+}
+
+function deriveKey(passphrase: string): Buffer {
+  const salt = getSalt();
+  return pbkdf2Sync(passphrase, salt, ITERATIONS, KEY_LEN, DIGEST);
+}
+
+export function encrypt(plain: string, passphrase?: string): { ciphertext: string; nonce: string } {
+  const key = deriveKey(passphrase || config.encryptionPassphrase);
+  const nonce = randomBytes(NONCE_LEN);
+  const cipher = createCipheriv(ALGO, key, nonce);
+  const encrypted = Buffer.concat([cipher.update(plain, 'utf8'), cipher.final()]);
+  const tag = cipher.getAuthTag();
+  const ciphertext = Buffer.concat([encrypted, tag]).toString('base64');
+  return { ciphertext, nonce: nonce.toString('base64') };
+}
+
+export function decrypt(ciphertext: string, nonceB64: string, passphrase?: string): string {
+  const key = deriveKey(passphrase || config.encryptionPassphrase);
+  const nonce = Buffer.from(nonceB64, 'base64');
+  const data = Buffer.from(ciphertext, 'base64');
+  const tag = data.slice(data.length - 16);
+  const encrypted = data.slice(0, data.length - 16);
+  const decipher = createDecipheriv(ALGO, key, nonce);
+  decipher.setAuthTag(tag);
+  const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]);
+  return decrypted.toString('utf8');
+}

package/core/security/governance.ts

@@ -0,0 +1,102 @@
+/**
+ * Memory Governance
+ * Implements protection, pinning, and immutability rules
+ */
+
+import { eq } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { config } from '../../config.js';
+import { performMemoryOperation } from '../utils/memory-operations.js';
+import { logger } from '../logger.js';
+import { createDatabaseClient } from '../storage/database.js';
+
+/**
+ * Mark a memory as protected (cannot be evicted)
+ */
+export async function protectMemory(memoryId: string, reason: string): Promise<void> {
+  await performMemoryOperation(memoryId, {
+    name: 'protecting memory',
+    updates: {
+      isProtected: true,
+      metadata: { protectionReason: reason, protectedAt: new Date().toISOString() },
+    },
+  });
+}
+
+/**
+ * Pin a memory for automatic injection into context
+ */
+export async function pinMemory(memoryId: string): Promise<void> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+
+  await db
+    .update(schema.memories)
+    .set({
+      isPinned: true,
+      importanceScore: 100,
+      lastImportanceRecalc: new Date(),
+    })
+    .where(eq(schema.memories.id, memoryId));
+}
+
+/**
+ * Unpin a memory
+ */
+export async function unpinMemory(memoryId: string): Promise<void> {
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+
+  await db
+    .update(schema.memories)
+    .set({
+      isPinned: false,
+      lastImportanceRecalc: new Date(),
+    })
+    .where(eq(schema.memories.id, memoryId));
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+/**
+ * Get all pinned memories for auto-injection into context
+ * Works regardless of governance settings - pinned memories should always be accessible
+ */
+export async function getPinnedMemories(projectId?: string): Promise<any[]> {
+  try {
+    const db = await getDb();
+    const schema = await getSchema();
+
+    let query = (db as any)
+      .select()
+      .from(schema.memories)
+      .where(eq(schema.memories.isPinned, true))
+      .limit(50);
+
+    return await query;
+  } catch (error) {
+    logger.error('Error retrieving pinned memories', error);
+    return [];
+  }
+}
+
+/**
+ * Get pinned memories formatted for context injection
+ */
+export async function getPinnedMemoriesForContext(projectId?: string): Promise<string[]> {
+  const pinned = await getPinnedMemories(projectId);
+  return pinned.map(m => 
+    `[Pinned] ${m.content?.substring(0, 500) || '(no content)'}`
+  );
+}

package/core/security/privacy.ts

@@ -0,0 +1,108 @@
+/**
+ * Privacy System
+ * Handles private tag filtering, secret detection, and path-based exclusions
+ */
+
+import { detectSecrets } from './secret-detector.js';
+
+type PrivacyMode = 'strict' | 'moderate' | 'off';
+
+const DENY_PATTERNS = [
+  '.env', '.env.local', '.env.*.local',
+  '*.key', '*.pem',
+  'secrets.json', 'credentials.json',
+  '.aws', '.ssh', 'node_modules/.env'
+];
+
+const PRIVATE_TAG_REGEX = /<(private|secret)>[\s\S]*?<\/\1>/gi;
+
+export function hasPrivateTags(content: string): boolean {
+  return PRIVATE_TAG_REGEX.test(content);
+}
+
+export function stripPrivateTags(content: string): string {
+  return content.replace(/<private>[\s\S]*?<\/private>/gi, '[PRIVATE]')
+                .replace(/<secret>[\s\S]*?<\/secret>/gi, '[SECRET]');
+}
+
+function isInDenyList(filePath: string): boolean {
+  const normalized = filePath.toLowerCase().replace(/\\/g, '/');
+
+  for (const pattern of DENY_PATTERNS) {
+    if (normalized === pattern) return true;
+    if (pattern.startsWith('*.') && normalized.endsWith(pattern.substring(1))) return true;
+    if (pattern.endsWith('/*')) {
+      const dir = pattern.slice(0, -2);
+      if (normalized.includes(`/${dir}/`) || normalized.startsWith(`${dir}/`)) return true;
+    }
+    if (normalized.includes(pattern)) return true;
+  }
+
+  return false;
+}
+
+export async function shouldStore(
+  content: string,
+  filePath?: string,
+  privacyMode: PrivacyMode = 'moderate'
+): Promise<boolean> {
+  if (privacyMode === 'off') return true;
+  if (hasPrivateTags(content)) return false;
+  if (filePath && isInDenyList(filePath)) return false;
+  if (detectSecrets(content).length > 0) return false;
+
+  return true;
+}
+
+export async function applyPrivacyFilters<T extends { content?: string; details?: Record<string, unknown> }>(
+  items: T[]
+): Promise<T[]> {
+  const results: T[] = [];
+
+  for (const item of items) {
+    const content = item.content || JSON.stringify(item.details || '');
+    if (await shouldStore(content)) {
+      results.push(item);
+    }
+  }
+
+  return results;
+}
+
+export function sanitizeForStorage(content: string, privacyMode: PrivacyMode): string {
+  let sanitized = stripPrivateTags(content);
+
+  if (privacyMode !== 'off') {
+    const secrets = detectSecrets(sanitized);
+    for (let i = secrets.length - 1; i >= 0; i--) {
+      const secret = secrets[i];
+      sanitized = sanitized.substring(0, secret.start) + '[REDACTED]' + sanitized.substring(secret.end);
+    }
+
+    sanitized = sanitized
+      .replace(/([a-zA-Z0-9_-]*[Aa]pi[_-]?[Kk]ey|Bearer\s+[^\s]+)/g, '[API_KEY]')
+      .replace(/(token|jwt|bearer)[\s=:]+[^\s,]+/gi, '[TOKEN]');
+  }
+
+  return sanitized;
+}
+
+export interface PrivacyReport {
+  isPrivate: boolean;
+  hasSecrets: boolean;
+  secretCount: number;
+  secretTypes: string[];
+  shouldStore: boolean;
+}
+
+export async function analyzePrivacy(content: string, filePath?: string): Promise<PrivacyReport> {
+  const secrets = detectSecrets(content);
+
+  return {
+    isPrivate: hasPrivateTags(content),
+    hasSecrets: secrets.length > 0,
+    secretCount: secrets.length,
+    secretTypes: [...new Set(secrets.map(s => s.type))],
+    shouldStore: await shouldStore(content, filePath)
+  };
+}

package/core/security/secret-detector.ts

@@ -0,0 +1,122 @@
+/**
+ * Secret Detector
+ * Pattern-based detection of API keys, tokens, and credentials
+ */
+
+export interface SecretMatch {
+  type: string;
+  pattern: string;
+  value: string;
+  start: number;
+  end: number;
+  confidence: 'high' | 'medium' | 'low';
+}
+
+type Confidence = SecretMatch['confidence'];
+
+interface PatternConfig {
+  pattern: RegExp;
+  type: string;
+  confidence: Confidence;
+}
+
+const HIGH_CONFIDENCE_PATTERNS: PatternConfig[] = [
+  { pattern: /(?:AKIA|ASIA)[0-9A-Z]{16}/g, type: 'aws_access_key', confidence: 'high' },
+  { pattern: /aws_secret_access_key\s*=\s*[^\s]+/gi, type: 'aws_secret_key', confidence: 'high' },
+  { pattern: /-----BEGIN (?:RSA|OPENSSH|PRIVATE) KEY-----/g, type: 'ssh_private_key', confidence: 'high' },
+  { pattern: /-----BEGIN PGP PRIVATE KEY BLOCK-----/g, type: 'gpg_private_key', confidence: 'high' },
+  { pattern: /AIza[0-9A-Za-z\-_]{35}/g, type: 'google_api_key', confidence: 'high' },
+  { pattern: /gh[pousr]_[A-Za-z0-9_]{36,255}/g, type: 'github_token', confidence: 'high' },
+  { pattern: /xox[baprs]-[0-9]{10,13}-[0-9]{10,13}-[A-Za-z0-9_-]{24,34}/g, type: 'slack_token', confidence: 'high' },
+  { pattern: /sk_live_[0-9a-zA-Z]{20,}/g, type: 'stripe_api_key', confidence: 'high' },
+  { pattern: /(mongodb|postgres|mysql|redis|cassandra):\/\/[^\s]+@[^\s]+/gi, type: 'database_url', confidence: 'high' }
+];
+
+const MEDIUM_CONFIDENCE_PATTERNS: PatternConfig[] = [
+  { pattern: /(?:api[_-]?key|apikey|api_secret|secret_key)\s*[:=]\s*['""`]?[A-Za-z0-9_-]{20,}['""`]?/gi, type: 'api_key', confidence: 'medium' },
+  { pattern: /bearer\s+[A-Za-z0-9._-]+/gi, type: 'bearer_token', confidence: 'medium' },
+  { pattern: /eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, type: 'jwt_token', confidence: 'medium' },
+  { pattern: /basic\s+[A-Za-z0-9+/=]+/gi, type: 'basic_auth', confidence: 'medium' },
+  { pattern: /(?:password|passwd|pwd)\s*[:=]\s*['""`]?[^\s'""`]+['""`]?/gi, type: 'password', confidence: 'medium' },
+  { pattern: /(?:connection_string|connstr|conn_string)\s*[:=]\s*['""`][^'""`]+['""`]/gi, type: 'connection_string', confidence: 'medium' }
+];
+
+const LOW_CONFIDENCE_PATTERNS: PatternConfig[] = [
+  { pattern: /\b(?:token|auth_token|access_token|refresh_token)\b/gi, type: 'token_reference', confidence: 'low' },
+  { pattern: /\b(?:secret|secret_key|api_secret|client_secret)\b/gi, type: 'secret_reference', confidence: 'low' },
+  { pattern: /\b(?:credential|cred|credentials)\b/gi, type: 'credential_reference', confidence: 'low' }
+];
+
+const CONFIDENCE_LEVELS: Record<Confidence, number> = { high: 3, medium: 2, low: 1 };
+
+function findMatches(text: string, config: PatternConfig): SecretMatch[] {
+  const regex = new RegExp(config.pattern.source, config.pattern.flags);
+  const matches: SecretMatch[] = [];
+  let match;
+
+  while ((match = regex.exec(text)) !== null) {
+    matches.push({
+      type: config.type,
+      pattern: config.pattern.source,
+      value: match[0],
+      start: match.index,
+      end: match.index + match[0].length,
+      confidence: config.confidence
+    });
+  }
+
+  return matches;
+}
+
+export function detectSecrets(text: string, minConfidence: Confidence = 'medium'): SecretMatch[] {
+  const minLevel = CONFIDENCE_LEVELS[minConfidence];
+  const matches: SecretMatch[] = [];
+
+  for (const config of HIGH_CONFIDENCE_PATTERNS) {
+    matches.push(...findMatches(text, config));
+  }
+
+  if (minLevel <= 2) {
+    for (const config of MEDIUM_CONFIDENCE_PATTERNS) {
+      matches.push(...findMatches(text, config));
+    }
+  }
+
+  if (minLevel <= 1) {
+    for (const config of LOW_CONFIDENCE_PATTERNS) {
+      matches.push(...findMatches(text, config));
+    }
+  }
+
+  return matches;
+}
+
+export function likelyContainsSecrets(text: string): boolean {
+  return detectSecrets(text, 'high').length > 0;
+}
+
+export function getSecretSummary(text: string): { count: number; types: string[] } {
+  const secrets = detectSecrets(text);
+  return {
+    count: secrets.length,
+    types: [...new Set(secrets.map(s => s.type))]
+  };
+}
+
+export function redactSecrets(text: string, replacement = '[REDACTED]'): string {
+  const secrets = detectSecrets(text, 'high');
+  let result = text;
+
+  const sorted = [...secrets].sort((a, b) => b.start - a.start);
+  for (const secret of sorted) {
+    result = result.substring(0, secret.start) + replacement + result.substring(secret.end);
+  }
+
+  return result;
+}
+
+export const SECRET_PATTERNS = {
+  high: HIGH_CONFIDENCE_PATTERNS,
+  medium: MEDIUM_CONFIDENCE_PATTERNS,
+  low: LOW_CONFIDENCE_PATTERNS
+};

package/core/session/auto-load.ts

@@ -0,0 +1,160 @@
+/** Session Auto-Load - Automatically load context when MCP session initializes */
+
+import { logger } from '../logger.js';
+import { config } from '../../config.js';
+import { initializeCoreMemory, getCoreMemory } from '../ingestion/core-memory.js';
+import { search } from '../memory/memories.js';
+import { getProjectContext } from '../context/context.js';
+import { getOrCreateProject } from '../projects.js';
+import { AutoLoadConfig, AutoLoadResult, DEFAULT_AUTO_LOAD_CONFIG } from './types.js';
+import { estimateTokens } from '../context/context-window.js';
+import { getLatestProjectWorkingSetSummary } from './working-set.js';
+import { loadHotCache, getHotCacheSummary } from '../hot-cache.js';
+
+export async function performAutoLoad(
+  projectPath: string,
+  customConfig?: Partial<AutoLoadConfig>
+): Promise<AutoLoadResult> {
+  const startTime = Date.now();
+  const cfg: AutoLoadConfig = { ...DEFAULT_AUTO_LOAD_CONFIG, ...customConfig };
+  const result: AutoLoadResult = {
+    coreMemoryLoaded: false,
+    memoriesLoaded: 0,
+    projectContextLoaded: false,
+    tokensUsed: 0,
+    duration: 0,
+    warnings: [],
+  };
+
+  if (!cfg.enabled) {
+    result.warnings.push('Auto-load disabled by configuration');
+    result.duration = Date.now() - startTime;
+    return result;
+  }
+
+  try {
+    const project = await getOrCreateProject(projectPath);
+    if (!project) {
+      result.warnings.push(`Failed to create/find project: ${projectPath}`);
+      result.duration = Date.now() - startTime;
+      return result;
+    }
+    const projectId = project.id;
+
+try {
+      const workingSetSummary = await getLatestProjectWorkingSetSummary(projectPath);
+      if (workingSetSummary) {
+        result.tokensUsed += estimateTokens(workingSetSummary);
+      }
+    } catch (error) {
+      const msg = error instanceof Error ? error.message : String(error);
+      result.warnings.push(`Failed to load session working set: ${msg}`);
+    }
+
+    // NEW: Load persistent hot cache (Karpathy-style, survives restart)
+    if (cfg.includeCoreMemory) {
+      try {
+        const hotCacheSummary = await getHotCacheSummary({ projectPath, maxEntries: 5 });
+        if (hotCacheSummary) {
+          result.tokensUsed += estimateTokens(hotCacheSummary);
+          logger.info('[AutoLoad] Hot cache loaded from persistent storage');
+        }
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        result.warnings.push(`Failed to load hot cache: ${msg}`);
+        logger.warn(`[AutoLoad] Failed to load hot cache: ${msg}`);
+      }
+    }
+
+    if (cfg.includeCoreMemory) {
+      try {
+        await initializeCoreMemory(projectId);
+        const coreMemory = await getCoreMemory(projectId);
+        if (coreMemory) {
+          result.coreMemoryLoaded = true;
+          const coreMemoryText = Object.values(coreMemory).join('\n');
+          result.tokensUsed += estimateTokens(coreMemoryText);
+          logger.info(`[AutoLoad] Core memory loaded for project ${projectId}`);
+        }
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        result.warnings.push(`Failed to load core memory: ${msg}`);
+        logger.warn(`[AutoLoad] Failed to load core memory: ${msg}`);
+      }
+    }
+
+    if (cfg.includeRecentMemories && cfg.recentMemoryCount > 0) {
+      try {
+        const recentMemories = await search({
+          query: '',
+          project: projectPath,
+          limit: cfg.recentMemoryCount,
+        });
+
+        if (recentMemories && recentMemories.length > 0) {
+          const importantMemories = recentMemories.filter(
+            (m: any) => (m.importanceScore ?? 50) >= cfg.importanceThreshold
+          );
+
+          result.memoriesLoaded = importantMemories.length;
+
+          for (const memory of importantMemories) {
+            result.tokensUsed += estimateTokens(memory.content || '');
+          }
+
+          logger.info(`[AutoLoad] Loaded ${result.memoriesLoaded} high-importance memories`);
+        }
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        result.warnings.push(`Failed to load recent memories: ${msg}`);
+        logger.warn(`[AutoLoad] Failed to load recent memories: ${msg}`);
+      }
+    }
+
+    if (cfg.includeProjectContext) {
+      try {
+        const context = await getProjectContext({
+          project: projectPath,
+          include: ['memories', 'observations'],
+          limit: 5,
+        });
+
+        if (context) {
+          result.projectContextLoaded = true;
+          result.tokensUsed += estimateTokens(JSON.stringify(context));
+          logger.info('[AutoLoad] Project context loaded');
+        }
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        result.warnings.push(`Failed to load project context: ${msg}`);
+        logger.warn(`[AutoLoad] Failed to load project context: ${msg}`);
+      }
+    }
+
+    result.duration = Date.now() - startTime;
+    logger.info(`[AutoLoad] Complete: ${result.memoriesLoaded} memories, ${result.tokensUsed} tokens, ${result.duration}ms`);
+
+    return result;
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    result.warnings.push(`Auto-load failed: ${msg}`);
+    result.duration = Date.now() - startTime;
+    logger.error(`[AutoLoad] Failed: ${msg}`);
+    return result;
+  }
+}
+
+export function getAutoLoadConfig(): AutoLoadConfig {
+  return {
+    enabled: config.sessionAutoLoadEnabled,
+    includeCoreMemory: true,
+    includeRecentMemories: true,
+    recentMemoryCount: config.sessionAutoLoadRecentCount,
+    importanceThreshold: config.sessionAutoLoadImportanceThreshold,
+    includeProjectContext: true,
+  };
+}
+
+export function shouldAutoLoad(): boolean {
+  return config.sessionAutoLoadEnabled;
+}