squish-memory 1.0.2 → 1.2.0

package/dist/core/{context-paging.js → context/context-paging.js}

@@ -1,28 +1,17 @@
 /**
- * Context Paging Service - Agent-controlled memory loading (Tier 2)
- *
- * Simple memory tracking system that allows agents to:
- * - Load memories into their working set
- * - Evict memories from working set
- * - View what's currently in their working set
- *
- * Note: This does NOT track tokens - Claude is context-aware and manages
- * its own token budget. This just tracks WHAT memories are in the agent's
- * current working set for visibility and management.
+ * Context Paging Service - Agent-controlled memory loading
+ * Allows agents to load/evict memories from working set
  */
 import { eq, and } from 'drizzle-orm';
 import { sql } from 'drizzle-orm';
-import { getDb } from '../db/index.js';
-import { getSchema } from '../db/schema.js';
-import { createDatabaseClient } from './database.js';
-import { getMemoryById } from './memory/memories.js';
+import { getMemory } from '../memory/memories.js';
+import { getDbClient } from '../lib/db-client.js';
 /**
  * Initialize or get a context session
  * Simplified - just tracks what's loaded, not tokens (Claude manages its own context)
  */
 export async function initializeContextSession(sessionId, projectId, userId) {
-    const db = createDatabaseClient(await getDb());
-    const schema = await getSchema();
+    const { db, schema } = await getDbClient();
     const { contextSessions } = schema;
     // Check if session exists
     const existing = await db
@@ -46,11 +35,9 @@ export async function initializeContextSession(sessionId, projectId, userId) {
 }
 /**
  * Load a memory into working set
- * Note: Claude manages its own context - this just tracks what you've loaded
  */
 export async function loadMemoryToContext(sessionId, memoryId) {
-    const db = createDatabaseClient(await getDb());
-    const schema = await getSchema();
+    const { db, schema } = await getDbClient();
     const { contextSessions } = schema;
     // Get session
     const session = await db
@@ -74,7 +61,7 @@ export async function loadMemoryToContext(sessionId, memoryId) {
         };
     }
     // Get memory
-    const memory = await getMemoryById(memoryId);
+    const memory = await getMemory(memoryId);
     if (!memory) {
         return {
             success: false,
@@ -112,8 +99,7 @@ export async function loadMemoryToContext(sessionId, memoryId) {
  * Evict a memory from working set
  */
 export async function evictMemoryFromContext(sessionId, memoryId) {
-    const db = createDatabaseClient(await getDb());
-    const schema = await getSchema();
+    const { db, schema } = await getDbClient();
     const { contextSessions } = schema;
     // Get session
     const session = await db
@@ -161,8 +147,7 @@ export async function evictMemoryFromContext(sessionId, memoryId) {
  * View all memories in working set
  */
 export async function viewLoadedMemories(sessionId) {
-    const db = createDatabaseClient(await getDb());
-    const schema = await getSchema();
+    const { db, schema } = await getDbClient();
     const { contextSessions } = schema;
     // Get session
     const session = await db
@@ -189,7 +174,7 @@ export async function viewLoadedMemories(sessionId) {
     // Get all loaded memories
     const memories = [];
     for (const id of loadedIds) {
-        const memory = await getMemoryById(id);
+        const memory = await getMemory(id);
         if (memory) {
             memories.push({
                 id: memory.id,
@@ -208,12 +193,10 @@ export async function viewLoadedMemories(sessionId) {
 }
 /**
  * Get context status - what's in your working set and what's available
- * Note: Claude manages its own context/tokens - this just shows WHAT you have loaded
  */
 export async function getContextStatus(sessionId, projectId) {
-    const db = createDatabaseClient(await getDb());
-    const schema = await getSchema();
-    const { contextSessions, memories, observations } = schema;
+    const { db, schema } = await getDbClient();
+    const { contextSessions, memories, learnings } = schema;
     // Get session
     const session = await db
         .select()
@@ -231,13 +214,13 @@ export async function getContextStatus(sessionId, projectId) {
     }
     const currentSession = session[0];
     // Get core memory stats
-    const { getCoreMemoryStats } = await import('./core-memory.js');
+    const { getCoreMemoryStats } = await import('../ingestion/core-memory.js');
     const coreStats = await getCoreMemoryStats(projectId);
     // Get loaded memories
     const loadedIds = currentSession.loadedMemoryIds || [];
     const loadedMemories = [];
     for (const id of loadedIds) {
-        const memory = await getMemoryById(id);
+        const memory = await getMemory(id);
         if (memory) {
             loadedMemories.push({
                 id: memory.id,
@@ -253,8 +236,8 @@ export async function getContextStatus(sessionId, projectId) {
         .where(eq(memories.projectId, projectId));
     const totalObservations = await db
         .select()
-        .from(observations)
-        .where(eq(observations.projectId, projectId));
+        .from(learnings)
+        .where(eq(learnings.projectId, projectId));
     return {
         success: true,
         coreMemory: {
@@ -277,8 +260,7 @@ export async function getContextStatus(sessionId, projectId) {
  * Clear all loaded memories from working set
  */
 export async function clearLoadedMemories(sessionId) {
-    const db = createDatabaseClient(await getDb());
-    const schema = await getSchema();
+    const { db, schema } = await getDbClient();
     const { contextSessions } = schema;
     await db
         .update(contextSessions)
@@ -296,8 +278,7 @@ export async function clearLoadedMemories(sessionId) {
  * Get all memories currently marked as in-context for a session
  */
 export async function getInContextMemories(sessionId) {
-    const db = createDatabaseClient(await getDb());
-    const schema = await getSchema();
+    const { db, schema } = await getDbClient();
     const { contextSessions } = schema;
     // Get session to find project
     const session = await db
@@ -327,8 +308,7 @@ export async function getInContextMemories(sessionId) {
  * Get out-of-context (archived) memories for a project
  */
 export async function getOutOfContextMemories(projectId, limit = 10) {
-    const db = createDatabaseClient(await getDb());
-    const schema = await getSchema();
+    const { db, schema } = await getDbClient();
     const { memories } = schema;
     // Get out-of-context memories, ordered by last accessed
     const outOfContextMemories = await db

package/dist/core/context/context-window.d.ts

@@ -0,0 +1,40 @@
+export interface ContextWindowConfig {
+    maxTokens: number;
+    warningThreshold: number;
+    criticalThreshold: number;
+}
+export declare const DEFAULT_CONTEXT_CONFIG: ContextWindowConfig;
+export interface TokenUsageStats {
+    coreMemoryTokens: number;
+    memoriesTokens: number;
+    totalTokens: number;
+    maxTokens: number;
+    usagePercent: number;
+    status: 'ok' | 'warning' | 'critical';
+    remainingTokens: number;
+}
+export interface OptimizationSuggestion {
+    type: 'drop' | 'summarize' | 'consolidate';
+    memoryId: string;
+    memoryType: string;
+    contentPreview: string;
+    tokens: number;
+    reason: string;
+    priority: number;
+}
+export declare function estimateTokens(content: string): number;
+export declare function getTokenUsage(projectPath: string): Promise<TokenUsageStats>;
+export declare function checkContextLimit(projectPath: string, additionalTokens: number): Promise<{
+    ok: boolean;
+    warning?: string;
+    stats: TokenUsageStats;
+}>;
+export declare function getOptimizationSuggestions(projectPath: string): Promise<OptimizationSuggestion[]>;
+export declare function getContextWindowStatus(projectPath: string): Promise<{
+    config: ContextWindowConfig;
+    usage: TokenUsageStats;
+    suggestions: OptimizationSuggestion[];
+    memoryCount: number;
+    coreMemorySections: number;
+}>;
+//# sourceMappingURL=context-window.d.ts.map

package/dist/core/context/context-window.js

@@ -0,0 +1,177 @@
+import { eq, sql } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { createDatabaseClient } from '../storage/database.js';
+import { requireProject } from '../projects.js';
+export const DEFAULT_CONTEXT_CONFIG = {
+    maxTokens: 128000,
+    warningThreshold: 0.80,
+    criticalThreshold: 0.95,
+};
+export function estimateTokens(content) {
+    if (!content)
+        return 0;
+    return Math.ceil(content.length / 4);
+}
+export async function getTokenUsage(projectPath) {
+    const project = await requireProject(projectPath);
+    const projectId = project.id;
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+    const memories = schema.memories;
+    const coreMemory = schema.coreMemory;
+    const coreMemoryRows = await db
+        .select()
+        .from(coreMemory)
+        .where(eq(coreMemory.projectId, projectId));
+    let coreMemoryTokens = 0;
+    for (const row of coreMemoryRows) {
+        coreMemoryTokens += row.tokensEstimate || estimateTokens(row.content || '');
+    }
+    const memoryRows = await db
+        .select({
+        tokens: memories.tokensEstimate,
+        content: memories.content,
+    })
+        .from(memories)
+        .where(eq(memories.projectId, projectId));
+    let memoriesTokens = 0;
+    for (const row of memoryRows) {
+        memoriesTokens += row.tokens || estimateTokens(row.content || '');
+    }
+    const totalTokens = coreMemoryTokens + memoriesTokens;
+    const config = DEFAULT_CONTEXT_CONFIG;
+    const usagePercent = (totalTokens / config.maxTokens) * 100;
+    const remainingTokens = Math.max(0, config.maxTokens - totalTokens);
+    let status = 'ok';
+    if (usagePercent >= config.criticalThreshold * 100) {
+        status = 'critical';
+    }
+    else if (usagePercent >= config.warningThreshold * 100) {
+        status = 'warning';
+    }
+    return {
+        coreMemoryTokens,
+        memoriesTokens,
+        totalTokens,
+        maxTokens: config.maxTokens,
+        usagePercent,
+        status,
+        remainingTokens,
+    };
+}
+export async function checkContextLimit(projectPath, additionalTokens) {
+    const stats = await getTokenUsage(projectPath);
+    const projectedTotal = stats.totalTokens + additionalTokens;
+    const projectedPercent = (projectedTotal / stats.maxTokens) * 100;
+    if (projectedPercent >= DEFAULT_CONTEXT_CONFIG.criticalThreshold * 100) {
+        return {
+            ok: false,
+            warning: `CRITICAL: Adding ${additionalTokens} tokens would exceed ${(DEFAULT_CONTEXT_CONFIG.criticalThreshold * 100).toFixed(0)}% of context limit (${projectedPercent.toFixed(1)}% projected). Consider optimizing first.`,
+            stats,
+        };
+    }
+    if (projectedPercent >= DEFAULT_CONTEXT_CONFIG.warningThreshold * 100) {
+        return {
+            ok: true,
+            warning: `WARNING: Adding ${additionalTokens} tokens would reach ${(projectedPercent.toFixed(1))}% of context limit. Consider optimizing soon.`,
+            stats,
+        };
+    }
+    return { ok: true, stats };
+}
+export async function getOptimizationSuggestions(projectPath) {
+    const project = await requireProject(projectPath);
+    const projectId = project.id;
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+    const memories = schema.memories;
+    const candidates = await db
+        .select({
+        id: memories.id,
+        type: memories.type,
+        content: memories.content,
+        tokens: memories.tokensEstimate,
+        importanceScore: memories.importanceScore,
+        accessCount: memories.accessCount,
+        createdAt: memories.createdAt,
+        isPinned: memories.isPinned,
+        isProtected: memories.isProtected,
+    })
+        .from(memories)
+        .where(eq(memories.projectId, projectId))
+        .orderBy(sql `(importance_score * 0.4 + (100 - COALESCE(access_count, 0)) * 0.3 + (strftime('%s', 'now') - created_at / 1000) / 86400 * 0.3) ASC`)
+        .limit(20);
+    const suggestions = [];
+    for (const mem of candidates) {
+        const tokens = mem.tokens || estimateTokens(mem.content || '');
+        const contentPreview = mem.content?.substring(0, 100) + '...';
+        const isPinned = mem.isPinned === 1 || mem.isPinned === true;
+        const isProtected = mem.isProtected === 1 || mem.isProtected === true;
+        const accessCount = mem.accessCount || 0;
+        const importanceScore = mem.importanceScore || 50;
+        if (isPinned || isProtected) {
+            continue;
+        }
+        if (importanceScore < 30 && accessCount < 3) {
+            suggestions.push({
+                type: 'drop',
+                memoryId: mem.id,
+                memoryType: mem.type,
+                contentPreview,
+                tokens,
+                reason: `Low importance (${importanceScore}) and rarely accessed (${accessCount} times)`,
+                priority: 1,
+            });
+        }
+        else if (importanceScore < 50 && tokens > 500) {
+            suggestions.push({
+                type: 'summarize',
+                memoryId: mem.id,
+                memoryType: mem.type,
+                contentPreview,
+                tokens,
+                reason: `Large memory (${tokens} tokens) with moderate importance`,
+                priority: 2,
+            });
+        }
+        else if (tokens > 1000) {
+            suggestions.push({
+                type: 'consolidate',
+                memoryId: mem.id,
+                memoryType: mem.type,
+                contentPreview,
+                tokens,
+                reason: `Very large memory (${tokens} tokens) - candidate for consolidation`,
+                priority: 3,
+            });
+        }
+    }
+    return suggestions.sort((a, b) => a.priority - b.priority).slice(0, 10);
+}
+export async function getContextWindowStatus(projectPath) {
+    const project = await requireProject(projectPath);
+    const projectId = project.id;
+    const db = createDatabaseClient(await getDb());
+    const schema = await getSchema();
+    const [usage, suggestions] = await Promise.all([
+        getTokenUsage(projectPath),
+        getOptimizationSuggestions(projectPath),
+    ]);
+    const memoryCount = await db
+        .select({ count: sql `COUNT(*)` })
+        .from(schema.memories)
+        .where(eq(schema.memories.projectId, projectId));
+    const coreMemorySections = await db
+        .select({ count: sql `COUNT(*)` })
+        .from(schema.coreMemory)
+        .where(eq(schema.coreMemory.projectId, projectId));
+    return {
+        config: DEFAULT_CONTEXT_CONFIG,
+        usage,
+        suggestions,
+        memoryCount: memoryCount[0]?.count || 0,
+        coreMemorySections: coreMemorySections[0]?.count || 0,
+    };
+}
+//# sourceMappingURL=context-window.js.map

package/dist/core/context/context.js

@@ -0,0 +1,22 @@
+import { requireProject } from '../projects.js';
+import { getRecent } from '../memory/memories.js';
+import { getObservations } from '../ingestion/learnings.js';
+import { getEntitiesForProject } from '../search/entities.js';
+import { validateLimit } from '../lib/validation.js';
+export async function getProjectContext(input) {
+    const project = await requireProject(input.project);
+    const include = input.include ?? ['memories', 'observations'];
+    const limit = validateLimit(input.limit, 10, 1, 100);
+    const result = { project };
+    if (include.includes('memories')) {
+        result.memories = await getRecent(project.path, limit);
+    }
+    if (include.includes('observations')) {
+        result.observations = await getObservations(project.path, limit);
+    }
+    if (include.includes('entities')) {
+        result.entities = await getEntitiesForProject(project.path, limit);
+    }
+    return result;
+}
+//# sourceMappingURL=context.js.map

package/dist/core/embeddings/embeddings.d.ts

@@ -0,0 +1,29 @@
+import { MultimodalInput } from './google-multimodal.js';
+export type EmbeddingProvider = 'local' | 'openai' | 'ollama' | 'lmstudio' | 'transformers' | 'google' | 'none' | 'auto';
+export declare function getEmbedding(input: string | MultimodalInput): Promise<number[] | null>;
+/**
+ * Get embeddings for multiple inputs in parallel batches
+ * Processes inputs in batches to respect rate limits while parallelizing
+ */
+export declare function getBatchEmbeddings(inputs: string[], batchSize?: number): Promise<Array<number[] | null>>;
+/**
+ * Clear the embedding cache
+ */
+export declare function clearEmbeddingCache(): void;
+/**
+ * Get embedding cache statistics
+ */
+export declare function getEmbeddingCacheStats(): {
+    size: number;
+    maxSize: number;
+};
+/**
+ * Check health of all configured embedding providers
+ * Returns availability and latency for each provider
+ */
+export declare function checkEmbeddingProviderHealth(): Promise<Map<string, {
+    available: boolean;
+    latencyMs?: number;
+    error?: string;
+}>>;
+//# sourceMappingURL=embeddings.d.ts.map