squish-memory 1.1.5 → 1.2.0

package/core/memory/write-gate.ts

@@ -0,0 +1,288 @@
+/**
+ * Write Gate Enforcement
+ * Validates memories before writing to ensure quality, security, and consistency
+ * Integrates secret detection, trigger detection, and content validation
+ */
+
+import { detectSecrets, redactSecrets, SecretMatch } from '../security/secret-detector.js';
+import { detectMemorySignals, MemorySignals } from './trigger-detector.js';
+import { resolveContradictions } from './contradiction-resolver.js';
+import { supersedeOldTemporalFacts } from './temporal-facts.js';
+import { logger } from '../logger.js';
+
+export interface WriteGateResult {
+  allowed: boolean;
+  sanitized: boolean;
+  warnings: string[];
+  errors: string[];
+  metadata: {
+    secretsDetected: number;
+    signals: MemorySignals | null;
+    contradictions: {
+      found: boolean;
+      count: number;
+    };
+    temporalSupersession: {
+      count: number;
+    };
+  };
+  sanitizedContent?: string;
+}
+
+export interface WriteGateOptions {
+  allowSecrets?: boolean; // If true, redact but allow write
+  minContentLength?: number;
+  maxContentLength?: number;
+  projectId?: string;
+  skipContradictionCheck?: boolean;
+  skipTemporalSupersession?: boolean;
+}
+
+const DEFAULT_OPTIONS: WriteGateOptions = {
+  allowSecrets: false,
+  minContentLength: 5,
+  maxContentLength: 50000,
+  skipContradictionCheck: false,
+  skipTemporalSupersession: false,
+};
+
+/**
+ * Main write gate function - validates and sanitizes content before write
+ */
+export async function enforceWriteGate(
+  content: string,
+  type: string,
+  options: WriteGateOptions = {}
+): Promise<WriteGateResult> {
+  const opts = { ...DEFAULT_OPTIONS, ...options };
+  const result: WriteGateResult = {
+    allowed: true,
+    sanitized: false,
+    warnings: [],
+    errors: [],
+    metadata: {
+      secretsDetected: 0,
+      signals: null,
+      contradictions: { found: false, count: 0 },
+      temporalSupersession: { count: 0 },
+    },
+  };
+
+  let processedContent = content;
+
+  // 1. Content length validation
+  if (content.length < opts.minContentLength!) {
+    result.errors.push(`Content too short (min ${opts.minContentLength} characters)`);
+    result.allowed = false;
+    return result;
+  }
+
+  if (content.length > opts.maxContentLength!) {
+    result.errors.push(`Content too long (max ${opts.maxContentLength} characters)`);
+    result.allowed = false;
+    return result;
+  }
+
+  // 2. Secret detection
+  const secrets = detectSecrets(content, 'high');
+  result.metadata.secretsDetected = secrets.length;
+
+  if (secrets.length > 0) {
+    const secretTypes = [...new Set(secrets.map(s => s.type))];
+    
+    if (!opts.allowSecrets) {
+      result.errors.push(`Potential secrets detected: ${secretTypes.join(', ')}`);
+      result.warnings.push('Content contains sensitive data that must be redacted');
+      result.allowed = false;
+      return result;
+    } else {
+      // Redact secrets but allow write
+      processedContent = redactSecrets(content);
+      result.sanitized = true;
+      result.sanitizedContent = processedContent;
+      result.warnings.push(`Redacted ${secrets.length} potential secrets: ${secretTypes.join(', ')}`);
+    }
+  }
+
+  // 3. Memory signals detection
+  const signals = detectMemorySignals(content);
+  result.metadata.signals = signals;
+
+  // Add warnings for high-priority signals
+  if (signals.priority === 'high') {
+    result.warnings.push('High-priority memory signal detected');
+  }
+
+  if (signals.implicit.correction) {
+    result.warnings.push('Content appears to be a correction - contradiction check recommended');
+  }
+
+  // 4. Contradiction detection (async, non-blocking for write)
+  if (!opts.skipContradictionCheck && signals.implicit.correction) {
+    try {
+      const contradictionResult = await resolveContradictions(
+        processedContent,
+        type,
+        opts.projectId
+      );
+      
+      result.metadata.contradictions = {
+        found: contradictionResult.supersededIds.length > 0,
+        count: contradictionResult.supersededIds.length,
+      };
+      
+      if (contradictionResult.supersededIds.length > 0) {
+        result.warnings.push(
+          `This memory supersedes ${contradictionResult.supersededIds.length} older memories`
+        );
+      }
+    } catch (error) {
+      logger.error('Contradiction check failed', error);
+      result.warnings.push('Contradiction check failed - proceeding with write');
+    }
+  }
+
+  // 5. Temporal fact handling
+  if (!opts.skipTemporalSupersession) {
+    try {
+      // This would be done after the memory is actually written, but we note it here
+      result.metadata.temporalSupersession = { count: 0 };
+    } catch (error) {
+      logger.error('Temporal check failed', error);
+    }
+  }
+
+  // 6. Content quality checks
+  const qualityIssues = checkContentQuality(processedContent);
+  result.warnings.push(...qualityIssues);
+
+  return result;
+}
+
+/**
+ * Quick validation without async operations
+ * Use for fast pre-check before full validation
+ */
+export function quickValidate(content: string): {
+  valid: boolean;
+  errors: string[];
+} {
+  const errors: string[] = [];
+
+  // Check for empty or whitespace-only content
+  if (!content || content.trim().length === 0) {
+    errors.push('Content is empty');
+    return { valid: false, errors };
+  }
+
+  // Check for minimum length
+  if (content.trim().length < 5) {
+    errors.push('Content is too short');
+  }
+
+  // Check for obvious secrets (high confidence only)
+  const secrets = detectSecrets(content, 'high');
+  if (secrets.length > 0) {
+    errors.push(`Content contains ${secrets.length} potential secrets`);
+  }
+
+  return { valid: errors.length === 0, errors };
+}
+
+/**
+ * Check content quality and return warnings
+ */
+function checkContentQuality(content: string): string[] {
+  const warnings: string[] = [];
+
+  // Check for excessive repetition
+  const words = content.toLowerCase().split(/\s+/);
+  const wordCounts = new Map<string, number>();
+  
+  for (const word of words) {
+    if (word.length > 3) { // Only check meaningful words
+      wordCounts.set(word, (wordCounts.get(word) || 0) + 1);
+    }
+  }
+
+  for (const [word, count] of wordCounts.entries()) {
+    if (count > 10) {
+      warnings.push(`Excessive repetition detected: "${word}" appears ${count} times`);
+      break; // Only report one
+    }
+  }
+
+  // Check for very long lines (might be code or minified content)
+  const lines = content.split('\n');
+  for (const line of lines) {
+    if (line.length > 1000) {
+      warnings.push('Content contains very long lines - may not be human-readable');
+      break;
+    }
+  }
+
+  // Check for binary-like content
+  const binaryPattern = /[\x00-\x08\x0E-\x1F]/;
+  if (binaryPattern.test(content)) {
+    warnings.push('Content may contain binary data');
+  }
+
+  return warnings;
+}
+
+/**
+ * Sanitize content for storage
+ * Redacts secrets and normalizes whitespace
+ */
+export function sanitizeForStorage(content: string): string {
+  // Redact secrets
+  let sanitized = redactSecrets(content);
+  
+  // Normalize excessive whitespace
+  sanitized = sanitized.replace(/\n{3,}/g, '\n\n');
+  sanitized = sanitized.replace(/ {2,}/g, ' ');
+  
+  // Trim
+  sanitized = sanitized.trim();
+  
+  return sanitized;
+}
+
+/**
+ * Calculate a quality score for content (0-100)
+ */
+export function calculateContentQualityScore(content: string): number {
+  let score = 100;
+
+  // Deduct for short content
+  if (content.length < 20) {
+    score -= 30;
+  } else if (content.length < 50) {
+    score -= 15;
+  }
+
+  // Deduct for secrets
+  const secrets = detectSecrets(content, 'high');
+  score -= secrets.length * 20;
+
+  // Deduct for repetition
+  const words = content.toLowerCase().split(/\s+/);
+  const uniqueWords = new Set(words.filter(w => w.length > 3));
+  const repetitionRatio = uniqueWords.size / Math.max(1, words.length);
+  if (repetitionRatio < 0.3) {
+    score -= 20;
+  }
+
+  // Bonus for structure (sentences, punctuation)
+  const sentences = content.split(/[.!?]+/).filter(s => s.trim().length > 0);
+  if (sentences.length > 1) {
+    score += 10;
+  }
+
+  // Bonus for reasonable length
+  if (content.length >= 50 && content.length <= 2000) {
+    score += 5;
+  }
+
+  return Math.max(0, Math.min(100, score));
+}

package/core/places/index.ts

@@ -0,0 +1,14 @@
+/**
+ * Places Module - Spatial memory organization
+ * 
+ * Exports:
+ * - places.ts: CRUD operations for places
+ * - rules.ts: Auto-assignment rules engine
+ * - memory-places.ts: Memory-to-place assignments
+ * - walking.ts: Sequential walking through places
+ */
+
+export * from './places.js';
+export * from './rules.js';
+export * from './memory-places.js';
+export * from './walking.js';

package/core/places/memory-places.ts

@@ -0,0 +1,339 @@
+/**
+ * Memory-Place Assignments - Assign memories to places
+ * 
+ * Handles the assignment of memories to places, both:
+ * - Auto-assignment via rules
+ * - Manual assignment by users
+ */
+
+import { randomUUID } from 'crypto';
+import { eq, and } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { logger } from '../logger.js';
+import { getPlaceByType, updatePlaceMemoryCount } from './places.js';
+import { findMatchingPlace } from './rules.js';
+import type { PlaceType } from './places.js';
+
+/**
+ * Assign a memory to a place (auto or manual)
+ */
+export async function assignMemoryToPlace(params: {
+  memoryId: string;
+  placeId: string;
+  isManual?: boolean;
+  ruleId?: string;
+}): Promise<boolean> {
+  const db = await getDb();
+  if (!db) return false;
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+  const id = randomUUID();
+
+  // Check if already assigned
+  const existing = await sqliteDb.select()
+    .from(schema.memoryPlaces)
+    .where(eq(schema.memoryPlaces.memoryId, params.memoryId))
+    .limit(1);
+
+  if (existing.length > 0) {
+    // Update existing
+    await sqliteDb.update(schema.memoryPlaces)
+      .set({
+        placeId: params.placeId,
+        isManual: params.isManual ? 1 : 0,
+        ruleId: params.ruleId || null,
+      })
+      .where(eq(schema.memoryPlaces.memoryId, params.memoryId));
+  } else {
+    // Insert new
+    await sqliteDb.insert(schema.memoryPlaces).values({
+      id,
+      memoryId: params.memoryId,
+      placeId: params.placeId,
+      isManual: params.isManual ? 1 : 0,
+      ruleId: params.ruleId || null,
+    });
+  }
+
+  // Update memory's place reference
+  await sqliteDb.update(schema.memories)
+    .set({ placeId: params.placeId })
+    .where(eq(schema.memories.id, params.memoryId));
+
+  // Update place memory count
+  await updatePlaceMemoryCount(params.placeId);
+
+  logger.info(`[MemoryPlaces] Assigned memory ${params.memoryId} to place ${params.placeId}`);
+  return true;
+}
+
+/**
+ * Auto-assign a memory based on rules
+ */
+export async function autoAssignMemory(params: {
+  memoryId: string;
+  projectId: string;
+  toolName?: string;
+  content?: string;
+  tags?: string[];
+  memoryType?: string;
+}): Promise<{ assigned: boolean; placeId?: string; placeType?: PlaceType }> {
+  const { memoryId, projectId, toolName, content, tags, memoryType } = params;
+
+  // Find matching place via rules
+  const placeType = await findMatchingPlace(projectId, {
+    toolName,
+    content,
+    tags,
+    memoryType,
+  });
+
+  if (!placeType) {
+    logger.info(`[MemoryPlaces] No matching rule for memory ${memoryId}`);
+    return { assigned: false };
+  }
+
+  // Get the place
+  const place = await getPlaceByType(projectId, placeType);
+  if (!place) {
+    logger.warn(`[MemoryPlaces] Place not found: ${placeType}`);
+    return { assigned: false };
+  }
+
+  // Assign
+  const success = await assignMemoryToPlace({
+    memoryId,
+    placeId: place.id,
+    isManual: false,
+  });
+
+  return {
+    assigned: success,
+    placeId: place.id,
+    placeType,
+  };
+}
+
+/**
+ * Manually assign a memory to a place
+ */
+export async function manualAssignMemory(params: {
+  memoryId: string;
+  projectId: string;
+  placeType: PlaceType;
+}): Promise<boolean> {
+  const { memoryId, projectId, placeType } = params;
+
+  // Get the place by type
+  const place = await getPlaceByType(projectId, placeType);
+  if (!place) {
+    logger.warn(`[MemoryPlaces] Place not found: ${placeType}`);
+    return false;
+  }
+
+  return assignMemoryToPlace({
+    memoryId,
+    placeId: place.id,
+    isManual: true,
+  });
+}
+
+/**
+ * Get place for a memory
+ */
+export async function getMemoryPlace(memoryId: string): Promise<string | null> {
+  const db = await getDb();
+  if (!db) return null;
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+
+  const result = await sqliteDb.select()
+    .from(schema.memoryPlaces)
+    .where(eq(schema.memoryPlaces.memoryId, memoryId))
+    .limit(1);
+
+  return result.length > 0 ? result[0].place_id : null;
+}
+
+/**
+ * Get memories for a place
+ */
+export async function getPlaceMemories(placeId: string, limit: number = 50): Promise<string[]> {
+  const db = await getDb();
+  if (!db) return [];
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+
+  const results = await sqliteDb.select({ memoryId: schema.memoryPlaces.memoryId })
+    .from(schema.memoryPlaces)
+    .where(eq(schema.memoryPlaces.placeId, placeId))
+    .limit(limit);
+
+  return results.map((r: any) => r.memoryId);
+}
+
+/**
+ * Remove memory from place
+ */
+export async function removeMemoryFromPlace(memoryId: string): Promise<boolean> {
+  const db = await getDb();
+  if (!db) return false;
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+
+  // Get the place before deleting
+  const existing = await sqliteDb.select()
+    .from(schema.memoryPlaces)
+    .where(eq(schema.memoryPlaces.memoryId, memoryId))
+    .limit(1);
+
+  if (existing.length > 0) {
+    const oldPlaceId = existing[0].place_id;
+    
+    // Delete assignment
+    await sqliteDb.delete(schema.memoryPlaces)
+      .where(eq(schema.memoryPlaces.memoryId, memoryId));
+
+    // Clear memory's place reference
+    await sqliteDb.update(schema.memories)
+      .set({ placeId: null })
+      .where(eq(schema.memories.id, memoryId));
+
+    // Update old place memory count
+    await updatePlaceMemoryCount(oldPlaceId);
+
+    logger.info(`[MemoryPlaces] Removed memory ${memoryId} from place ${oldPlaceId}`);
+  }
+
+  return true;
+}
+
+/**
+ * Initialize memory-place for a project (ensures all memories without places get assigned)
+ */
+export async function initializeProjectPlaces(projectId: string): Promise<{
+  initialized: number;
+  assigned: number;
+}> {
+  const db = await getDb();
+  if (!db) return { initialized: 0, assigned: 0 };
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+
+  // Get all memories without a place_id
+  const memoriesWithoutPlace = await sqliteDb.select({ id: schema.memories.id })
+    .from(schema.memories)
+    .where(and(
+      eq(schema.memories.projectId, projectId),
+    ));
+
+  let assigned = 0;
+  
+  for (const mem of memoriesWithoutPlace) {
+    const result = await autoAssignMemory({
+      memoryId: mem.id,
+      projectId,
+      memoryType: 'observation',
+    });
+    
+    if (result.assigned) assigned++;
+  }
+
+  logger.info(`[MemoryPlaces] Initialized places for project ${projectId}: ${assigned} assigned`);
+  return { initialized: memoriesWithoutPlace.length, assigned };
+}
+
+/**
+ * Auto-archive old memories - move memories > 30 days from active places to Archive
+ * Keeps active places lean and organized
+ */
+export async function autoArchiveOldMemories(projectId: string, daysOld: number = 30): Promise<{
+  archived: number;
+  failed: number;
+}> {
+  const db = await getDb();
+  if (!db) return { archived: 0, failed: 0 };
+
+  const schema = await getSchema();
+  const sqliteDb = db as any;
+  
+  // Get the Archive place
+  const archivePlace = await getPlaceByType(projectId, 'archive');
+  if (!archivePlace) {
+    logger.warn(`[MemoryPlaces] Archive place not found for project ${projectId}`);
+    return { archived: 0, failed: 0 };
+  }
+
+  // Calculate cutoff date (Unix timestamp for SQLite)
+  const cutoffDate = new Date();
+  cutoffDate.setDate(cutoffDate.getDate() - daysOld);
+  const cutoffTimestamp = Math.floor(cutoffDate.getTime() / 1000);
+  
+  // Get all active place IDs (not archive)
+  const allPlaces = await sqliteDb.select()
+    .from(schema.places)
+    .where(eq(schema.places.projectId, projectId));
+  
+  const activePlaceIds = allPlaces
+    .filter((p: any) => p.place_type !== 'archive')
+    .map((p: any) => p.id);
+  
+  if (activePlaceIds.length === 0) {
+    return { archived: 0, failed: 0 };
+  }
+
+  // Find memories in active places that are older than cutoff
+  // Use raw SQL for complex date comparison
+  const oldMemories = await sqliteDb.select({
+    memoryId: schema.memories.id,
+    placeId: schema.memoryPlaces.placeId,
+    createdAt: schema.memories.createdAt,
+  })
+  .from(schema.memories)
+  .innerJoin(schema.memoryPlaces, eq(schema.memories.id, schema.memoryPlaces.memoryId))
+  .where(
+    and(
+      eq(schema.memories.projectId, projectId),
+      // Memories in active places (not archive)
+    )
+  );
+
+  // Actually filter in JS for simplicity
+  const memoriesToArchive = oldMemories.filter((m: any) => 
+    activePlaceIds.includes(m.placeId) && m.createdAt < cutoffTimestamp
+  );
+
+  let archived = 0;
+  let failed = 0;
+  
+  for (const mem of memoriesToArchive) {
+    try {
+      // Move to archive place
+      await sqliteDb.update(schema.memoryPlaces)
+        .set({ placeId: archivePlace.id })
+        .where(eq(schema.memoryPlaces.memoryId, mem.memoryId));
+      
+      // Update the old place's memory count
+      await updatePlaceMemoryCount(mem.placeId);
+      
+      archived++;
+    } catch (e) {
+      logger.warn(`[MemoryPlaces] Failed to archive memory ${mem.memoryId}: ${e}`);
+      failed++;
+    }
+  }
+
+  // Update archive place memory count
+  if (archived > 0) {
+    await updatePlaceMemoryCount(archivePlace.id);
+    logger.info(`[MemoryPlaces] Archived ${archived} old memories to Archive place`);
+  }
+  
+  return { archived, failed };
+}