squish-memory 0.7.2 → 0.8.0

package/dist/core/memory/hybrid-search.js

@@ -0,0 +1,336 @@
+/**
+ * Hybrid Search - Combines BM25 keyword search with vector semantic search
+ * Uses Reciprocal Rank Fusion (RRF) for intelligent result merging
+ *
+ * Based on research showing 40-60% improvement over pure vector search:
+ * - BM25 excels at exact keyword matches
+ * - Vector search captures semantic similarity
+ * - RRF merges both without score calibration issues
+ */
+import { getDb } from '../../db/index.js';
+import { createDatabaseClient } from '../../core/database.js';
+import { getEmbedding } from '../../core/embeddings.js';
+import { getProjectByPath } from '../../core/projects.js';
+import { fromSqliteTags, normalizeTags } from './serialization.js';
+import { isDatabaseUnavailableError } from '../../core/utils.js';
+/**
+ * Reciprocal Rank Fusion (RRF) constant
+ * Higher values = more influence from lower-ranked items
+ * 60 is the standard value used in research
+ */
+const RRF_K = 60;
+/**
+ * Main hybrid search function - combines BM25 and vector search with RRF
+ */
+export async function hybridSearch(input, options = {}) {
+    const limit = options.limit ?? input.limit ?? 10;
+    const bm25Weight = options.bm25Weight ?? 0.5;
+    const vectorWeight = options.vectorWeight ?? 0.5;
+    // Run both searches in parallel
+    const [bm25Results, vectorResults] = await Promise.all([
+        bm25Search(input, { ...options, limit: limit * 2 }),
+        vectorSearch(input, { ...options, limit: limit * 2 }),
+    ]);
+    // Apply RRF to merge results
+    return reciprocalRankFusion(bm25Results, vectorResults, { limit, bm25Weight, vectorWeight });
+}
+/**
+ * BM25 keyword search using SQLite FTS5
+ */
+async function bm25Search(input, options) {
+    try {
+        const db = createDatabaseClient(await getDb());
+        const sqlite = db.$client;
+        const limit = options.limit ?? 10;
+        const tags = normalizeTags(options.tags ?? input.tags);
+        // Build FTS5 query with proper escaping
+        const ftsQuery = buildFtsQuery(input.query);
+        // Build WHERE conditions
+        const conditions = ['memories_fts MATCH ?'];
+        const params = [ftsQuery];
+        if (input.type) {
+            conditions.push('m.type = ?');
+            params.push(input.type);
+        }
+        if (tags.length) {
+            conditions.push('(' + tags.map(() => 'm.tags LIKE ?').join(' OR ') + ')');
+            params.push(...tags.map((tag) => `%${tag}%`));
+        }
+        let projectId = null;
+        if (input.project) {
+            const project = await getProjectByPath(input.project);
+            if (project) {
+                projectId = project.id;
+                conditions.push('m.project_id = ?');
+                params.push(project.id);
+            }
+        }
+        const whereClause = conditions.join(' AND ');
+        // FTS5 search with BM25 ranking
+        // Note: SQLite doesn't allow rowid access in subquery, so we use direct join
+        const statement = sqlite.prepare(`
+      SELECT
+        m.id as id,
+        m.project_id as projectId,
+        m.type as type,
+        m.content as content,
+        m.summary as summary,
+        m.tags as tags,
+        m.metadata as metadata,
+        bm25(memories_fts) as bm25Score,
+        m.created_at as createdAt
+      FROM memories m
+      INNER JOIN memories_fts ON m.rowid = memories_fts.rowid
+      WHERE ${whereClause}
+      ORDER BY bm25(memories_fts)
+      LIMIT ?
+    `);
+        const rows = statement.all(...params, limit * 3);
+        // Return as ranked results (lower BM25 score = better rank)
+        return rows.map((row, index) => ({
+            id: row.id,
+            rank: index + 1,
+            score: row.bm25Score,
+            result: {
+                id: row.id,
+                projectId: row.projectId,
+                type: row.type,
+                content: row.content,
+                summary: row.summary ?? undefined,
+                tags: fromSqliteTags(row.tags ?? null),
+                metadata: row.metadata ? JSON.parse(row.metadata) : null,
+                createdAt: row.createdAt ? new Date((Number(row.createdAt) || 0) * 1000).toISOString() : undefined,
+            },
+        }));
+    }
+    catch (error) {
+        if (isDatabaseUnavailableError(error)) {
+            return [];
+        }
+        throw error;
+    }
+}
+/**
+ * Vector semantic search using cosine similarity
+ */
+async function vectorSearch(input, options) {
+    try {
+        const db = createDatabaseClient(await getDb());
+        const sqlite = db.$client;
+        const limit = options.limit ?? 10;
+        const tags = normalizeTags(options.tags ?? input.tags);
+        // Get query embedding
+        const queryEmbedding = await getEmbedding(input.query);
+        if (!queryEmbedding) {
+            return [];
+        }
+        // Build WHERE conditions
+        const conditions = [];
+        const params = [];
+        if (input.type) {
+            conditions.push('m.type = ?');
+            params.push(input.type);
+        }
+        if (tags.length) {
+            conditions.push('(' + tags.map(() => 'm.tags LIKE ?').join(' OR ') + ')');
+            params.push(...tags.map((tag) => `%${tag}%`));
+        }
+        let projectId = null;
+        if (input.project) {
+            const project = await getProjectByPath(input.project);
+            if (project) {
+                projectId = project.id;
+                conditions.push('m.project_id = ?');
+                params.push(project.id);
+            }
+        }
+        const whereClause = conditions.length > 0
+            ? 'WHERE ' + conditions.join(' AND ')
+            : '';
+        // Fetch candidates for vector search
+        const statement = sqlite.prepare(`
+      SELECT
+        m.id as id,
+        m.project_id as projectId,
+        m.type as type,
+        m.content as content,
+        m.summary as summary,
+        m.tags as tags,
+        m.metadata as metadata,
+        m.embedding as embedding,
+        m.embedding_json as embeddingJson,
+        m.created_at as createdAt
+      FROM memories m
+      ${whereClause}
+      ORDER BY m.created_at DESC
+      LIMIT ?
+    `);
+        const rows = statement.all(...params, limit * 3);
+        // Calculate cosine similarity for each result
+        const scored = rows
+            .map((row) => {
+            const embedding = parseEmbedding(row.embedding) ?? parseEmbedding(row.embeddingJson);
+            if (!embedding)
+                return null;
+            const similarity = cosineSimilarity(queryEmbedding, embedding);
+            return {
+                id: row.id,
+                projectId: row.projectId,
+                type: row.type,
+                content: row.content,
+                summary: row.summary,
+                tags: row.tags,
+                metadata: row.metadata,
+                createdAt: row.createdAt,
+                similarity,
+            };
+        })
+            .filter((item) => item !== null);
+        // Sort by similarity (descending) and return as ranked results
+        scored.sort((a, b) => b.similarity - a.similarity);
+        return scored.slice(0, limit * 2).map((item, index) => ({
+            id: item.id,
+            rank: index + 1,
+            score: item.similarity,
+            result: {
+                id: item.id,
+                projectId: item.projectId,
+                type: item.type,
+                content: item.content,
+                summary: item.summary ?? undefined,
+                tags: fromSqliteTags(item.tags ?? null),
+                metadata: item.metadata ? JSON.parse(item.metadata) : null,
+                createdAt: item.createdAt ? new Date((Number(item.createdAt) || 0) * 1000).toISOString() : undefined,
+                similarity: item.similarity,
+            },
+        }));
+    }
+    catch (error) {
+        if (isDatabaseUnavailableError(error)) {
+            return [];
+        }
+        throw error;
+    }
+}
+/**
+ * Reciprocal Rank Fusion (RRF) - merges ranked lists without score calibration
+ *
+ * RRF Formula: score(item) = sum(weight_i / (k + rank_i))
+ * Where k is a constant (typically 60) that prevents high ranks from dominating
+ *
+ * Benefits:
+ * - No need to calibrate different scoring systems (BM25 vs cosine similarity)
+ * - Handles items that appear in only one list
+ * - Proven to outperform weighted score fusion in most cases
+ */
+function reciprocalRankFusion(bm25Results, vectorResults, options) {
+    const { limit, bm25Weight, vectorWeight } = options;
+    const scores = new Map();
+    // Process BM25 results
+    for (const item of bm25Results) {
+        const rrfScore = (bm25Weight * 2) / (RRF_K + item.rank);
+        const existing = scores.get(item.id);
+        if (existing) {
+            existing.score += rrfScore;
+        }
+        else {
+            scores.set(item.id, { score: rrfScore, result: item.result });
+        }
+    }
+    // Process vector results
+    for (const item of vectorResults) {
+        const rrfScore = (vectorWeight * 2) / (RRF_K + item.rank);
+        const existing = scores.get(item.id);
+        if (existing) {
+            existing.score += rrfScore;
+        }
+        else {
+            scores.set(item.id, { score: rrfScore, result: item.result });
+        }
+    }
+    // Sort by RRF score (descending) and return top results
+    return Array.from(scores.values())
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit)
+        .map((item) => ({
+        ...item.result,
+        similarity: item.result.similarity ?? 0,
+    }));
+}
+/**
+ * Build FTS5 query string from user input
+ * Handles phrase searches, AND/OR operators, and special characters
+ */
+function buildFtsQuery(query) {
+    // Remove special characters that could break FTS5 syntax
+    let cleaned = query.replace(/[^\w\s"'-]/g, ' ');
+    // If query contains quotes, preserve as phrase search
+    if (cleaned.includes('"')) {
+        return cleaned;
+    }
+    // Split into terms and join with AND for better precision
+    const terms = cleaned.trim().split(/\s+/).filter((t) => t.length > 0);
+    if (terms.length === 0) {
+        return '""'; // Empty query
+    }
+    // For multi-word queries, use NEAR operator for proximity search
+    if (terms.length > 1) {
+        return terms.join(' ');
+    }
+    return terms[0];
+}
+/**
+ * Parse embedding from SQLite storage
+ */
+function parseEmbedding(embeddingData) {
+    if (!embeddingData)
+        return null;
+    if (Array.isArray(embeddingData))
+        return embeddingData;
+    if (embeddingData instanceof Uint8Array || Buffer.isBuffer(embeddingData)) {
+        try {
+            const json = JSON.parse(embeddingData.toString());
+            if (Array.isArray(json))
+                return json;
+        }
+        catch {
+            try {
+                const floatArray = new Float32Array(embeddingData.buffer || embeddingData);
+                return Array.from(floatArray);
+            }
+            catch {
+                return null;
+            }
+        }
+    }
+    if (typeof embeddingData === 'string') {
+        try {
+            const parsed = JSON.parse(embeddingData);
+            if (Array.isArray(parsed))
+                return parsed;
+        }
+        catch {
+            return null;
+        }
+    }
+    return null;
+}
+/**
+ * Calculate cosine similarity between two vectors
+ */
+function cosineSimilarity(a, b) {
+    if (a.length !== b.length)
+        return 0;
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dotProduct += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    if (normA === 0 || normB === 0)
+        return 0;
+    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+//# sourceMappingURL=hybrid-search.js.map

package/dist/core/memory/hybrid-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-search.js","sourceRoot":"","sources":["../../../core/memory/hybrid-search.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAiB,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AAEjE;;;;GAIG;AACH,MAAM,KAAK,GAAG,EAAE,CAAC;AAoBjB;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAkB,EAClB,UAA+B,EAAE;IAEjC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;IACjD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,CAAC;IAC7C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC;IAEjD,gCAAgC;IAChC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACrD,UAAU,CAAC,KAAK,EAAE,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC;QACnD,YAAY,CAAC,KAAK,EAAE,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC;KACtD,CAAC,CAAC;IAEH,6BAA6B;IAC7B,OAAO,oBAAoB,CACzB,WAAW,EACX,aAAa,EACb,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,CACpC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CACvB,KAAkB,EAClB,OAA4B;IAE5B,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,oBAAoB,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,EAAE,CAAC,OAAc,CAAC;QACjC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;QAEvD,wCAAwC;QACxC,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE5C,yBAAyB;QACzB,MAAM,UAAU,GAAa,CAAC,sBAAsB,CAAC,CAAC;QACtD,MAAM,MAAM,GAAU,CAAC,QAAQ,CAAC,CAAC;QAEjC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,UAAU,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtD,IAAI,OAAO,EAAE,CAAC;gBACZ,SAAS,GAAG,OAAO,CAAC,EAAE,CAAC;gBACvB,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAE7C,gCAAgC;QAChC,6EAA6E;QAC7E,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC;;;;;;;;;;;;;cAavB,WAAW;;;KAGpB,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,KAAK,GAAG,CAAC,CAU7C,CAAC;QAEH,4DAA4D;QAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC/B,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,KAAK,GAAG,CAAC;YACf,KAAK,EAAE,GAAG,CAAC,SAAS;YACpB,MAAM,EAAE;gBACN,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,IAAI,EAAE,GAAG,CAAC,IAAW;gBACrB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,SAAS;gBACjC,IAAI,EAAE,cAAc,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC;gBACtC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI;gBACxD,SAAS,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;aACnG;SACF,CAAC,CAAC,CAAC;IACN,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,IAAI,0BAA0B,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CACzB,KAAkB,EAClB,OAA4B;IAE5B,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,oBAAoB,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,EAAE,CAAC,OAAc,CAAC;QACjC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;QAEvD,sBAAsB;QACtB,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,yBAAyB;QACzB,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,MAAM,MAAM,GAAU,EAAE,CAAC;QAEzB,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,UAAU,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,SAAS,GAAkB,IAAI,CAAC;QACpC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtD,IAAI,OAAO,EAAE,CAAC;gBACZ,SAAS,GAAG,OAAO,CAAC,EAAE,CAAC;gBACvB,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC;YACvC,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC;YACrC,CAAC,CAAC,EAAE,CAAC;QAEP,qCAAqC;QACrC,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC;;;;;;;;;;;;;QAa7B,WAAW;;;KAGd,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,KAAK,GAAG,CAAC,CAW7C,CAAC;QAEH,8CAA8C;QAC9C,MAAM,MAAM,GAAG,IAAI;aAChB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACX,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YACrF,IAAI,CAAC,SAAS;gBAAE,OAAO,IAAI,CAAC;YAE5B,MAAM,UAAU,GAAG,gBAAgB,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAC/D,OAAO;gBACL,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,UAAU;aACX,CAAC;QACJ,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAoC,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QAErE,+DAA+D;QAC/D,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAEnD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YACtD,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,KAAK,GAAG,CAAC;YACf,KAAK,EAAE,IAAI,CAAC,UAAU;YACtB,MAAM,EAAE;gBACN,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,IAAI,EAAE,IAAI,CAAC,IAAW;gBACtB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,SAAS;gBAClC,IAAI,EAAE,cAAc,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC;gBACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI;gBAC1D,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;gBACpG,UAAU,EAAE,IAAI,CAAC,UAAU;aAC5B;SACF,CAAC,CAAC,CAAC;IACN,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,IAAI,0BAA0B,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,oBAAoB,CAC3B,WAA2B,EAC3B,aAA6B,EAC7B,OAAoE;IAEpE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;IACpD,MAAM,MAAM,GAAG,IAAI,GAAG,EAA6D,CAAC;IAEpF,uBAAuB;IACvB,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;SAC/B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACd,GAAG,IAAI,CAAC,MAAM;QACd,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC;KACxC,CAAC,CAAC,CAAC;AACR,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,KAAa;IAClC,yDAAyD;IACzD,IAAI,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;IAEhD,sDAAsD;IACtD,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,0DAA0D;IAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,CAAC,cAAc;IAC7B,CAAC;IAED,iEAAiE;IACjE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,aAAkB;IACxC,IAAI,CAAC,aAAa;QAAE,OAAO,IAAI,CAAC;IAEhC,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC;QAAE,OAAO,aAAa,CAAC;IAEvD,IAAI,aAAa,YAAY,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QAC1E,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;YAClD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,MAAM,IAAI,aAAa,CAAC,CAAC;gBAC3E,OAAO,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YACzC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,OAAO,MAAM,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IAEpC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEzC,OAAO,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5D,CAAC"}

package/dist/core/memory/importance.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Importance Scoring System
+ *
+ * Calculates and manages memory importance scores (0-100) with temporal decay.
+ * Based on research from Mnemosyne architecture and ReMe (Memory Replay).
+ *
+ * Scoring factors:
+ * - Base score: 50 (neutral)
+ * - Recency boost: decays over time (exponential decay)
+ * - Access frequency: more frequently accessed = higher importance
+ * - Type weighting: decisions > facts > preferences > context > observations
+ * - User flags: pinned/protected memories get maximum importance
+ */
+import type { Memory } from '../../drizzle/schema.js';
+export interface ImportanceScore {
+    score: number;
+    components: {
+        base: number;
+        recency: number;
+        accessFrequency: number;
+        typeWeight: number;
+        userFlags: number;
+    };
+    explanation: string;
+}
+/**
+ * Calculate importance score for a memory
+ *
+ * Formula: base + recency + accessFrequency + typeWeight + userFlags
+ * All values are clamped to 0-100 range
+ */
+export declare function calculateImportance(memory: Partial<Memory>): ImportanceScore;
+/**
+ * Update importance score for a memory
+ * Used when memory is accessed or modified
+ */
+export declare function updateImportanceScore(memoryId: string, incrementAccess?: boolean): Promise<number>;
+/**
+ * Decay importance scores for old memories
+ * Should be run periodically (e.g., daily) to reduce scores of stale memories
+ *
+ * This function applies exponential decay to all memories that haven't been
+ * recalculated recently, keeping the system's importance scores current.
+ */
+export declare function decayImportanceScores(projectId?: string): Promise<number>;
+/**
+ * Get low-importance memories that are candidates for consolidation
+ * These are old, rarely accessed memories with low importance scores
+ */
+export declare function getLowImportanceMemories(projectId: string, options?: {
+    minAge?: number;
+    maxImportance?: number;
+    limit?: number;
+}): Promise<any[]>;
+/**
+ * Set importance score manually (for user override)
+ */
+export declare function setImportanceScore(memoryId: string, score: number): Promise<void>;
+/**
+ * Pin a memory to prevent decay and consolidation
+ */
+export declare function pinMemory(memoryId: string, pinned?: boolean): Promise<void>;
+/**
+ * Calculate cosine similarity between two vectors
+ * Re-exported for use in consolidation
+ */
+export declare function cosineSimilarity(vecA: number[], vecB: number[]): number;
+//# sourceMappingURL=importance.d.ts.map

package/dist/core/memory/importance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"importance.d.ts","sourceRoot":"","sources":["../../../core/memory/importance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAMtD,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,WAAW,EAAE,MAAM,CAAC;CACrB;AAcD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,eAAe,CA2B5E;AAgHD;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,QAAQ,EAAE,MAAM,EAChB,eAAe,GAAE,OAAe,GAC/B,OAAO,CAAC,MAAM,CAAC,CAsCjB;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAgD/E;AAED;;;GAGG;AACH,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;CACX,GACL,OAAO,CAAC,GAAG,EAAE,CAAC,CA0ChB;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC,CAef;AAED;;GAEG;AACH,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAE,OAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAavF;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAuBvE"}