squish-memory 1.0.2 → 1.2.0

package/core/context/context-paging.ts

@@ -0,0 +1,449 @@
+/**
+ * Context Paging Service - Agent-controlled memory loading
+ * Allows agents to load/evict memories from working set
+ */
+
+import { eq, and, inArray } from 'drizzle-orm';
+import { sql } from 'drizzle-orm';
+import { getMemory } from '../memory/memories.js';
+import { getDbClient } from '../lib/db-client.js';
+
+interface LoadedMemory {
+  id: string;
+  type: string;
+  content: string;
+  contentPreview: string;
+  loadedAt: Date;
+}
+
+/**
+ * Initialize or get a context session
+ * Simplified - just tracks what's loaded, not tokens (Claude manages its own context)
+ */
+export async function initializeContextSession(
+  sessionId: string,
+  projectId: string,
+  userId?: string
+): Promise<void> {
+  const { db, schema } = await getDbClient();
+  const { contextSessions } = schema;
+
+  // Check if session exists
+  const existing = await db
+    .select()
+    .from(contextSessions)
+    .where(eq(contextSessions.sessionId, sessionId))
+    .limit(1);
+
+  if (existing.length === 0) {
+    await db.insert(contextSessions).values({
+      sessionId,
+      projectId: projectId as any,
+      userId: userId || null,
+      loadedMemoryIds: JSON.stringify([]) as any,
+      tokenBudget: 8000,
+      tokensUsed: 0,
+      coreMemoryTokens: 0,
+      loadedMemoriesTokens: 0,
+      metadata: JSON.stringify({}) as any,
+    } as any);
+  }
+}
+
+/**
+ * Load a memory into working set
+ */
+export async function loadMemoryToContext(
+    sessionId: string,
+    memoryId: string
+  ): Promise<{
+    success: boolean;
+    message?: string;
+    memory?: LoadedMemory;
+  }> {
+    const { db, schema } = await getDbClient();
+    const { contextSessions } = schema;
+
+  // Get session
+  const session = await db
+    .select()
+    .from(contextSessions)
+    .where(eq(contextSessions.sessionId, sessionId))
+    .limit(1);
+
+  if (session.length === 0) {
+    return {
+      success: false,
+      message: `Session "${sessionId}" not found. Initialize session first.`,
+    };
+  }
+
+  const currentSession = session[0];
+  const loadedIds = (currentSession.loadedMemoryIds as string[]) || [];
+
+  // Check if already loaded
+  if (loadedIds.includes(memoryId)) {
+    return {
+      success: false,
+      message: `Memory "${memoryId}" is already in working set`,
+    };
+  }
+
+  // Get memory
+  const memory = await getMemory(memoryId);
+  if (!memory) {
+    return {
+      success: false,
+      message: `Memory "${memoryId}" not found`,
+    };
+  }
+
+  // Update session
+  const newLoadedIds = [...loadedIds, memoryId];
+  await db
+    .update(contextSessions)
+    .set({
+      loadedMemoryIds: newLoadedIds as any,
+      updatedAt: new Date() as any,
+    } as any)
+    .where(eq(contextSessions.sessionId, sessionId));
+
+  // Also update memory's contextStatus to 'in-context'
+  await db
+    .update(schema.memories as any)
+    .set({
+      contextStatus: 'in-context' as any,
+    })
+    .where(eq((schema.memories as any).id, memoryId));
+
+  return {
+    success: true,
+    memory: {
+      id: memory.id,
+      type: memory.type,
+      content: memory.content,
+      contentPreview: memory.content.substring(0, 200) + (memory.content.length > 200 ? '...' : ''),
+      loadedAt: new Date(),
+    },
+  };
+}
+
+/**
+ * Evict a memory from working set
+ */
+export async function evictMemoryFromContext(
+    sessionId: string,
+    memoryId: string
+  ): Promise<{
+    success: boolean;
+    message?: string;
+  }> {
+    const { db, schema } = await getDbClient();
+    const { contextSessions } = schema;
+
+  // Get session
+  const session = await db
+    .select()
+    .from(contextSessions)
+    .where(eq(contextSessions.sessionId, sessionId))
+    .limit(1);
+
+  if (session.length === 0) {
+    return {
+      success: false,
+      message: `Session "${sessionId}" not found`,
+    };
+  }
+
+  const currentSession = session[0];
+  const loadedIds = (currentSession.loadedMemoryIds as string[]) || [];
+
+  // Check if loaded
+  if (!loadedIds.includes(memoryId)) {
+    return {
+      success: false,
+      message: `Memory "${memoryId}" is not in working set`,
+    };
+  }
+
+  // Update session
+  const newLoadedIds = loadedIds.filter((id) => id !== memoryId);
+
+  await db
+    .update(contextSessions)
+    .set({
+      loadedMemoryIds: newLoadedIds as any,
+      updatedAt: new Date() as any,
+    } as any)
+    .where(eq(contextSessions.sessionId, sessionId));
+
+  // Also update memory's contextStatus to 'out-of-context'
+  await db
+    .update(schema.memories as any)
+    .set({
+      contextStatus: 'out-of-context' as any,
+    })
+    .where(eq((schema.memories as any).id, memoryId));
+
+  return {
+    success: true,
+    message: `Memory "${memoryId}" removed from working set`,
+  };
+}
+
+/**
+ * View all memories in working set
+ */
+export async function viewLoadedMemories(
+    sessionId: string
+  ): Promise<{
+    success: boolean;
+    memories: LoadedMemory[];
+    count: number;
+  }> {
+    const { db, schema } = await getDbClient();
+    const { contextSessions } = schema;
+
+  // Get session
+  const session = await db
+    .select()
+    .from(contextSessions)
+    .where(eq(contextSessions.sessionId, sessionId))
+    .limit(1);
+
+  if (session.length === 0) {
+    return {
+      success: false,
+      memories: [],
+      count: 0,
+    };
+  }
+
+  const currentSession = session[0];
+  const loadedIds = (currentSession.loadedMemoryIds as string[]) || [];
+
+  if (loadedIds.length === 0) {
+    return {
+      success: true,
+      memories: [],
+      count: 0,
+    };
+  }
+
+  // Get all loaded memories
+  const memories: LoadedMemory[] = [];
+  for (const id of loadedIds) {
+    const memory = await getMemory(id);
+    if (memory) {
+      memories.push({
+        id: memory.id,
+        type: memory.type,
+        content: memory.content,
+        contentPreview: memory.content.substring(0, 200) + (memory.content.length > 200 ? '...' : ''),
+        loadedAt: currentSession.updatedAt,
+      });
+    }
+  }
+
+  return {
+    success: true,
+    memories,
+    count: memories.length,
+  };
+}
+
+/**
+ * Get context status - what's in your working set and what's available
+ */
+export async function getContextStatus(
+    sessionId: string,
+    projectId: string
+  ): Promise<{
+    success: boolean;
+    coreMemory: {
+      sizeBytes: number;
+      maxBytes: number;
+      usagePercent: number;
+    };
+    workingSet: {
+      loadedCount: number;
+      loadedMemories: Array<{
+        id: string;
+        type: string;
+        contentLength: number;
+      }>;
+    };
+    available: {
+      totalMemories: number;
+      totalObservations: number;
+    };
+    note: string;
+  }> {
+    const { db, schema } = await getDbClient();
+    const { contextSessions, memories, learnings } = schema;
+
+  // Get session
+  const session = await db
+    .select()
+    .from(contextSessions)
+    .where(eq(contextSessions.sessionId, sessionId))
+    .limit(1);
+
+  if (session.length === 0) {
+    return {
+      success: false,
+      coreMemory: { sizeBytes: 0, maxBytes: 2048, usagePercent: 0 },
+      workingSet: { loadedCount: 0, loadedMemories: [] },
+      available: { totalMemories: 0, totalObservations: 0 },
+      note: "Session not found. Initialize session first.",
+    };
+  }
+
+  const currentSession = session[0];
+
+  // Get core memory stats
+  const { getCoreMemoryStats } = await import('../ingestion/core-memory.js');
+  const coreStats = await getCoreMemoryStats(projectId);
+
+  // Get loaded memories
+  const loadedIds = (currentSession.loadedMemoryIds as string[]) || [];
+  const loadedMemories = [];
+  for (const id of loadedIds) {
+    const memory = await getMemory(id);
+    if (memory) {
+      loadedMemories.push({
+        id: memory.id,
+        type: memory.type,
+        contentLength: memory.content.length,
+      });
+    }
+  }
+
+  // Get external context stats (counts)
+  const totalMemories = await db
+    .select()
+    .from(memories)
+    .where(eq(memories.projectId, projectId as any));
+
+  const totalObservations = await db
+    .select()
+    .from(learnings)
+    .where(eq(learnings.projectId, projectId as any));
+
+  return {
+    success: true,
+    coreMemory: {
+      sizeBytes: coreStats.totalBytes,
+      maxBytes: coreStats.maxBytes,
+      usagePercent: coreStats.usagePercent,
+    },
+    workingSet: {
+      loadedCount: loadedMemories.length,
+      loadedMemories,
+    },
+    available: {
+      totalMemories: totalMemories.length,
+      totalObservations: totalObservations.length,
+    },
+    note: "Claude manages its own context and token limits. This shows what memories you've loaded into your working set.",
+  };
+}
+
+/**
+ * Clear all loaded memories from working set
+ */
+export async function clearLoadedMemories(sessionId: string): Promise<{
+    success: boolean;
+    message?: string;
+  }> {
+    const { db, schema } = await getDbClient();
+    const { contextSessions } = schema;
+
+  await db
+    .update(contextSessions)
+    .set({
+      loadedMemoryIds: [] as any,
+      updatedAt: new Date() as any,
+    } as any)
+    .where(eq(contextSessions.sessionId, sessionId));
+
+  return {
+    success: true,
+    message: 'All memories cleared from working set',
+  };
+}
+
+/**
+ * Get all memories currently marked as in-context for a session
+ */
+export async function getInContextMemories(
+    sessionId: string
+  ): Promise<LoadedMemory[]> {
+    const { db, schema } = await getDbClient();
+    const { contextSessions } = schema;
+
+  // Get session to find project
+  const session = await db
+    .select()
+    .from(contextSessions)
+    .where(eq(contextSessions.sessionId, sessionId))
+    .limit(1);
+
+  if (session.length === 0) {
+    return [];
+  }
+
+  const { memories } = schema;
+  const projectId = session[0].projectId;
+
+  // Get all in-context memories
+  const inContextMemories = await db
+    .select()
+    .from(memories)
+    .where(
+      and(
+        eq(memories.projectId, projectId as any),
+        eq(memories.contextStatus, 'in-context' as any)
+      )
+    );
+
+  return inContextMemories.map((mem: any) => ({
+    id: mem.id,
+    type: mem.type,
+    content: mem.content,
+    contentPreview: mem.content.substring(0, 200) + (mem.content.length > 200 ? '...' : ''),
+    loadedAt: mem.updatedAt,
+  }));
+}
+
+/**
+ * Get out-of-context (archived) memories for a project
+ */
+export async function getOutOfContextMemories(
+  projectId: string,
+  limit: number = 10
+): Promise<LoadedMemory[]> {
+  const { db, schema } = await getDbClient();
+  const { memories } = schema;
+
+  // Get out-of-context memories, ordered by last accessed
+  const outOfContextMemories = await db
+    .select()
+    .from(memories)
+    .where(
+      and(
+        eq(memories.projectId, projectId as any),
+        eq(memories.contextStatus, 'out-of-context' as any)
+      )
+    )
+    .orderBy(sql`COALESCE(${memories.lastAccessedAt}, ${memories.createdAt}) DESC`)
+    .limit(limit);
+
+  return outOfContextMemories.map((mem: any) => ({
+    id: mem.id,
+    type: mem.type,
+    content: mem.content,
+    contentPreview: mem.content.substring(0, 200) + (mem.content.length > 200 ? '...' : ''),
+    loadedAt: mem.lastAccessedAt || mem.createdAt,
+  }));
+}

package/core/context/context-window.ts

@@ -0,0 +1,234 @@
+import { eq, and, sql } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { createDatabaseClient } from '../storage/database.js';
+import { requireProject } from '../projects.js';
+
+export interface ContextWindowConfig {
+  maxTokens: number;
+  warningThreshold: number;
+  criticalThreshold: number;
+}
+
+export const DEFAULT_CONTEXT_CONFIG: ContextWindowConfig = {
+  maxTokens: 128000,
+  warningThreshold: 0.80,
+  criticalThreshold: 0.95,
+};
+
+export interface TokenUsageStats {
+  coreMemoryTokens: number;
+  memoriesTokens: number;
+  totalTokens: number;
+  maxTokens: number;
+  usagePercent: number;
+  status: 'ok' | 'warning' | 'critical';
+  remainingTokens: number;
+}
+
+export interface OptimizationSuggestion {
+  type: 'drop' | 'summarize' | 'consolidate';
+  memoryId: string;
+  memoryType: string;
+  contentPreview: string;
+  tokens: number;
+  reason: string;
+  priority: number;
+}
+
+export function estimateTokens(content: string): number {
+  if (!content) return 0;
+  return Math.ceil(content.length / 4);
+}
+
+export async function getTokenUsage(projectPath: string): Promise<TokenUsageStats> {
+  const project = await requireProject(projectPath);
+  const projectId = project.id;
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+  const memories = schema.memories; const coreMemory = schema.coreMemory;
+
+  const coreMemoryRows = await db
+    .select()
+    .from(coreMemory)
+    .where(eq(coreMemory.projectId, projectId as any));
+
+  let coreMemoryTokens = 0;
+  for (const row of coreMemoryRows) {
+    coreMemoryTokens += (row as any).tokensEstimate || estimateTokens((row as any).content || '');
+  }
+
+  const memoryRows = await db
+    .select({
+      tokens: (memories as any).tokensEstimate,
+      content: memories.content,
+    })
+    .from(memories)
+    .where(eq(memories.projectId, projectId as any));
+
+  let memoriesTokens = 0;
+  for (const row of memoryRows) {
+    memoriesTokens += (row as any).tokens || estimateTokens((row as any).content || '');
+  }
+
+  const totalTokens = coreMemoryTokens + memoriesTokens;
+  const config = DEFAULT_CONTEXT_CONFIG;
+  const usagePercent = (totalTokens / config.maxTokens) * 100;
+  const remainingTokens = Math.max(0, config.maxTokens - totalTokens);
+
+  let status: 'ok' | 'warning' | 'critical' = 'ok';
+  if (usagePercent >= config.criticalThreshold * 100) {
+    status = 'critical';
+  } else if (usagePercent >= config.warningThreshold * 100) {
+    status = 'warning';
+  }
+
+  return {
+    coreMemoryTokens,
+    memoriesTokens,
+    totalTokens,
+    maxTokens: config.maxTokens,
+    usagePercent,
+    status,
+    remainingTokens,
+  };
+}
+
+export async function checkContextLimit(
+  projectPath: string,
+  additionalTokens: number
+): Promise<{ ok: boolean; warning?: string; stats: TokenUsageStats }> {
+  const stats = await getTokenUsage(projectPath);
+  const projectedTotal = stats.totalTokens + additionalTokens;
+  const projectedPercent = (projectedTotal / stats.maxTokens) * 100;
+
+  if (projectedPercent >= DEFAULT_CONTEXT_CONFIG.criticalThreshold * 100) {
+    return {
+      ok: false,
+      warning: `CRITICAL: Adding ${additionalTokens} tokens would exceed ${(DEFAULT_CONTEXT_CONFIG.criticalThreshold * 100).toFixed(0)}% of context limit (${projectedPercent.toFixed(1)}% projected). Consider optimizing first.`,
+      stats,
+    };
+  }
+
+  if (projectedPercent >= DEFAULT_CONTEXT_CONFIG.warningThreshold * 100) {
+    return {
+      ok: true,
+      warning: `WARNING: Adding ${additionalTokens} tokens would reach ${(projectedPercent.toFixed(1))}% of context limit. Consider optimizing soon.`,
+      stats,
+    };
+  }
+
+  return { ok: true, stats };
+}
+
+export async function getOptimizationSuggestions(
+  projectPath: string
+): Promise<OptimizationSuggestion[]> {
+  const project = await requireProject(projectPath);
+  const projectId = project.id;
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+  const memories = schema.memories;
+
+  const candidates = await db
+    .select({
+      id: memories.id,
+      type: memories.type,
+      content: memories.content,
+      tokens: (memories as any).tokensEstimate,
+      importanceScore: (memories as any).importanceScore,
+      accessCount: (memories as any).accessCount,
+      createdAt: memories.createdAt,
+      isPinned: (memories as any).isPinned,
+      isProtected: (memories as any).isProtected,
+    })
+    .from(memories)
+    .where(eq(memories.projectId, projectId as any))
+    .orderBy(sql`(importance_score * 0.4 + (100 - COALESCE(access_count, 0)) * 0.3 + (strftime('%s', 'now') - created_at / 1000) / 86400 * 0.3) ASC`)
+    .limit(20);
+
+  const suggestions: OptimizationSuggestion[] = [];
+
+  for (const mem of candidates) {
+    const tokens = (mem as any).tokens || estimateTokens((mem as any).content || '');
+    const contentPreview = (mem as any).content?.substring(0, 100) + '...';
+    const isPinned = (mem as any).isPinned === 1 || (mem as any).isPinned === true;
+    const isProtected = (mem as any).isProtected === 1 || (mem as any).isProtected === true;
+    const accessCount = (mem as any).accessCount || 0;
+    const importanceScore = (mem as any).importanceScore || 50;
+
+    if (isPinned || isProtected) {
+      continue;
+    }
+
+    if (importanceScore < 30 && accessCount < 3) {
+      suggestions.push({
+        type: 'drop',
+        memoryId: mem.id,
+        memoryType: mem.type,
+        contentPreview,
+        tokens,
+        reason: `Low importance (${importanceScore}) and rarely accessed (${accessCount} times)`,
+        priority: 1,
+      });
+    } else if (importanceScore < 50 && tokens > 500) {
+      suggestions.push({
+        type: 'summarize',
+        memoryId: mem.id,
+        memoryType: mem.type,
+        contentPreview,
+        tokens,
+        reason: `Large memory (${tokens} tokens) with moderate importance`,
+        priority: 2,
+      });
+    } else if (tokens > 1000) {
+      suggestions.push({
+        type: 'consolidate',
+        memoryId: mem.id,
+        memoryType: mem.type,
+        contentPreview,
+        tokens,
+        reason: `Very large memory (${tokens} tokens) - candidate for consolidation`,
+        priority: 3,
+      });
+    }
+  }
+
+  return suggestions.sort((a, b) => a.priority - b.priority).slice(0, 10);
+}
+
+export async function getContextWindowStatus(projectPath: string): Promise<{
+  config: ContextWindowConfig;
+  usage: TokenUsageStats;
+  suggestions: OptimizationSuggestion[];
+  memoryCount: number;
+  coreMemorySections: number;
+}> {
+   const project = await requireProject(projectPath);
+   const projectId = project.id;
+  const db = createDatabaseClient(await getDb());
+  const schema = await getSchema();
+
+  const [usage, suggestions] = await Promise.all([
+    getTokenUsage(projectPath),
+    getOptimizationSuggestions(projectPath),
+  ]);
+
+  const memoryCount = await db
+    .select({ count: sql`COUNT(*)` })
+    .from(schema.memories)
+    .where(eq(schema.memories.projectId, projectId as any));
+
+  const coreMemorySections = await db
+    .select({ count: sql`COUNT(*)` })
+    .from(schema.coreMemory)
+    .where(eq(schema.coreMemory.projectId, projectId as any));
+
+  return {
+    config: DEFAULT_CONTEXT_CONFIG,
+    usage,
+    suggestions,
+    memoryCount: (memoryCount[0] as any)?.count || 0,
+    coreMemorySections: (coreMemorySections[0] as any)?.count || 0,
+  };
+}

package/core/context/context.ts

@@ -0,0 +1,35 @@
+import { requireProject } from '../projects.js';
+import { getRecent } from '../memory/memories.js';
+import { getObservations } from '../ingestion/learnings.js';
+import { getEntitiesForProject } from '../search/entities.js';
+import { validateLimit } from '../lib/validation.js';
+
+export interface ContextInput {
+  project: string;
+  include?: Array<'memories' | 'observations' | 'entities'>;
+  limit?: number;
+}
+
+export async function getProjectContext(input: ContextInput): Promise<Record<string, unknown>> {
+  const project = await requireProject(input.project);
+  const include = input.include ?? ['memories', 'observations'];
+  const limit = validateLimit(input.limit, 10, 1, 100);
+
+  const result: Record<string, unknown> = { project };
+
+  if (include.includes('memories')) {
+    result.memories = await getRecent(project.path, limit);
+  }
+
+  if (include.includes('observations')) {
+    result.observations = await getObservations(project.path, limit);
+  }
+
+  if (include.includes('entities')) {
+    result.entities = await getEntitiesForProject(project.path, limit);
+  }
+
+  return result;
+}
+
+