squish-memory 1.2.0 → 1.2.1

package/dist/core/memory/hybrid-search.js

@@ -1,596 +0,0 @@
-/**
- * Vector Search - Pure semantic search with optional graph boosting + multi-session support
- *
- * Uses cosine similarity on embeddings + optional graph boost
- * BM25 removed - use qmd-client for hot tier (BM25 + vectors + reranking)
- */
-import { getDb } from '../../db/index.js';
-import { createDatabaseClient } from '../storage/database.js';
-import { getEmbedding } from '../../core/embeddings.js';
-import { requireProject } from '../../core/projects.js';
-import { deserializeTags, deserializeMetadata, normalizeTags } from './serialization.js';
-import { computeGraphBoost } from '../search/graph-boost.js';
-import { normalizeTimestamp } from '../lib/utils.js';
-import { parseEmbedding } from '../lib/parse-embedding.js';
-import { cosineSimilarity } from '../utils/vector-operations.js';
-import config from '../../config.js';
-import { getRelatedMemories } from '../associations.js';
-import { getPlaceMemories } from '../places/memory-places.js';
-import { getPlaceByType } from '../places/places.js';
-import { multiHopSearch } from '../graph/multi-hop-retrieval.js';
-import { logger } from '../logger.js';
-/**
- * Detect if query asks about time (temporal queries)
- */
-function isTemporalQuery(query) {
-    const temporalIndicators = [
-        'when', 'how long', 'how many', 'ago', 'since', 'until',
-        'before', 'after', 'earlier', 'later', 'yesterday', 'tomorrow',
-        'last week', 'next week', 'last month', 'next month'
-    ];
-    const lower = query.toLowerCase();
-    return temporalIndicators.some(w => lower.includes(w));
-}
-/**
- * Detect if query spans multiple sessions (multi-hop queries)
- */
-function isMultiSessionQuery(query) {
-    const multiSessionIndicators = [
-        'when', 'then', 'before', 'after', 'earlier', 'later',
-        'previous', 'next', 'first', 'later', 'subsequently',
-        'across', 'between', 'both', 'another', 'different'
-    ];
-    const lower = query.toLowerCase();
-    return multiSessionIndicators.some(w => lower.includes(w));
-}
-/**
- * Check if content contains date/time references
- */
-function hasDateReference(content) {
-    const datePatterns = [
-        /\b\d{4}\b/, // Years: 2023, 2022
-        /\b\d{1,2}\s+(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\b/i, // Month dates
-        /\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2}\b/i,
-        /\b\d{1,2}\/\d{1,2}\/\d{2,4}\b/, // Dates: 5/7/2023
-        /\b(yesterday|today|tomorrow|last week|last month|last year)\b/i,
-        /\b(\d+)\s+(day|week|month|year)s?\s+(ago|before)\b/i,
-    ];
-    const lower = content.toLowerCase();
-    return datePatterns.some(p => p.test(content) || p.test(lower));
-}
-/**
- * Expand query for multi-session retrieval
- */
-function expandQueryForMultiSession(query) {
-    const expansions = [query];
-    const lower = query.toLowerCase();
-    // Add time-based expansions
-    if (lower.includes('when')) {
-        expansions.push(query.replace(/when/i, '').trim());
-    }
-    if (lower.includes('before') || lower.includes('after')) {
-        expansions.push(query.replace(/before|after/i, '').trim());
-    }
-    if (lower.includes('earlier') || lower.includes('later')) {
-        expansions.push(query.replace(/earlier|later/i, '').trim());
-    }
-    // Add "mentioning" expansions for factual queries
-    if (lower.includes('what') || lower.includes('how')) {
-        expansions.push(query + ' mentioned');
-        expansions.push(query + ' said');
-    }
-    return [...new Set(expansions.filter(e => e.length > 2))];
-}
-/**
- * Expand query for temporal retrieval - add date/temporal context
- */
-function expandQueryForTemporal(query) {
-    const expansions = [query];
-    const lower = query.toLowerCase();
-    // For "when" questions, search with entity + date context
-    if (lower.includes('when')) {
-        // Extract entity name (everything after "when did" or "when is")
-        const entityMatch = query.match(/when\s+(?:did|is|was|were)\s+(\w+)/i);
-        if (entityMatch) {
-            const entity = entityMatch[1];
-            // Search with entity name alone (might match date mentions)
-            expansions.push(entity);
-            expansions.push(query.replace(/when\s+(?:did|is|was|were)\s+/i, '').trim());
-        }
-    }
-    // Add "date" and "time" expansions
-    if (lower.includes('when') || lower.includes('how long') || lower.includes('ago')) {
-        expansions.push(query + ' date');
-        expansions.push(query + ' time');
-    }
-    return [...new Set(expansions.filter(e => e.length > 2))];
-}
-/**
- * Score with recency + similarity + entity boost (NO LLM required)
- */
-function scoreWithHeuristics(result, query, now) {
-    let score = result.similarity ?? 0;
-    // 1. Recency boost: Recent = higher (up to +0.1)
-    if (result.createdAt) {
-        const created = new Date(result.createdAt).getTime();
-        const ageHours = (now - created) / (1000 * 60 * 60);
-        const recencyScore = Math.max(0, 0.1 * Math.exp(-ageHours / 720)); // Decay over 30 days
-        score += recencyScore;
-    }
-    // 2. Entity overlap: Query words appearing in content = boost
-    const queryWords = new Set(query.toLowerCase().split(/\s+/).filter(w => w.length > 3));
-    const contentWords = new Set((result.content ?? "").toLowerCase().split(/\s+/));
-    const overlap = [...queryWords].filter(w => contentWords.has(w)).length;
-    score += overlap * 0.02; // Small boost per matching word
-    return score;
-}
-/**
- * Main search function - vectors + graph boost + heuristics + places + sessions
- * Unified search integrating Places, Graph, and Memory
- */
-export async function hybridSearch(input, options = {}) {
-    const limit = options.limit ?? input.limit ?? 10;
-    const enableMultiSession = options.enableMultiSession !== false;
-    const enableHeuristics = options.enableHeuristics !== false;
-    const isMultiHop = enableMultiSession && isMultiSessionQuery(input.query);
-    const isTemporal = isTemporalQuery(input.query);
-    let vectorResults;
-    if (isMultiHop) {
-        // Multi-hop: use expansion to get more coverage
-        const expandedQueries = expandQueryForMultiSession(input.query);
-        const allResults = [];
-        for (const expQuery of expandedQueries) {
-            const expResults = await vectorSearch({ ...input, query: expQuery }, { ...options, limit: Math.ceil(limit * 2) });
-            allResults.push(...expResults);
-        }
-        const byId = new Map();
-        for (const r of allResults) {
-            const existing = byId.get(r.id);
-            if (!existing || (r.similarity ?? 0) > (existing.similarity ?? 0)) {
-                byId.set(r.id, r);
-            }
-        }
-        vectorResults = Array.from(byId.values());
-    }
-    else if (isTemporal) {
-        // Temporal: fetch more results
-        vectorResults = await vectorSearch(input, { ...options, limit: limit * 4 });
-    }
-    else {
-        // Regular query
-        vectorResults = await vectorSearch(input, { ...options, limit: limit * 2 });
-    }
-    // Task 2: Integrate Places into retrieval
-    // If placeId or placeType specified, filter/boost results from that place
-    if (input.placeId || input.placeType) {
-        vectorResults = await applyPlaceFilterAndBoost(vectorResults, input, limit);
-    }
-    // Task 3: Add session temporal scope
-    // If sessionId specified, boost memories from same session
-    if (input.sessionId) {
-        vectorResults = applySessionBoost(vectorResults, input.sessionId);
-    }
-    // TEMPORAL: Boost results with dates for temporal queries
-    if (isTemporal) {
-        vectorResults = applyTemporalBoost(vectorResults);
-    }
-    // Apply heuristics if enabled (recency + entity overlap)
-    if (enableHeuristics) {
-        const now = Date.now();
-        vectorResults = vectorResults.map(r => ({
-            ...r,
-            similarity: scoreWithHeuristics(r, input.query, now)
-        }));
-    }
-    // Graph boost
-    const graphWeight = config.scoringWeights.graphBoost;
-    const candidateIds = vectorResults.map(r => r.id);
-    const graphBoostMap = await computeGraphBoost(candidateIds);
-    let results = applyGraphBoostWithWeight(vectorResults, graphBoostMap, limit, graphWeight);
-    // Expand with associated memories for better coverage
-    if (options.includeAssociations !== false) {
-        results = await expandWithAssociations(results, limit);
-    }
-    // Task 4: Enable multi-hop graph traversal by default
-    // If query is detected as multi-hop, use actual graph traversal
-    if (isMultiHop && options.enableGraphTraversal !== false && input.project) {
-        try {
-            const graphResults = await multiHopSearch({
-                query: input.query,
-                project: input.project,
-                limit: limit,
-                includeVectorResults: false,
-                includeGraphResults: true,
-            });
-            // Merge graph results with vector results
-            const existingIds = new Set(results.map(r => r.id));
-            for (const gr of graphResults) {
-                if (!existingIds.has(gr.id)) {
-                    results.push({
-                        ...gr,
-                        similarity: (gr.similarity ?? 0) * 0.9 // Slightly lower weight
-                    });
-                }
-            }
-            // Re-sort
-            results.sort((a, b) => (b.similarity ?? 0) - (a.similarity ?? 0));
-        }
-        catch (e) {
-            // Multi-hop failed, continue with vector results
-            logger.debug(`[HybridSearch] Multi-hop failed: ${e}`);
-        }
-    }
-    return results;
-}
-/**
- * Task 2: Filter and boost results by place
- */
-async function applyPlaceFilterAndBoost(results, input, limit) {
-    let placeIds = [];
-    // Get place IDs from placeId or placeType
-    if (input.placeType && input.project) {
-        const project = await requireProject(input.project);
-        const place = await getPlaceByType(project.id, input.placeType);
-        if (place) {
-            placeIds = [place.id];
-        }
-    }
-    else if (input.placeId) {
-        placeIds = [input.placeId];
-    }
-    if (placeIds.length === 0) {
-        return results;
-    }
-    // Get memories in this place
-    const placeMemories = new Set();
-    for (const pid of placeIds) {
-        const memIds = await getPlaceMemories(pid, limit * 3);
-        for (const mid of memIds) {
-            placeMemories.add(mid);
-        }
-    }
-    // Boost memories from the specified place
-    const PLACE_BOOST = 0.15;
-    const boosted = results.map(r => {
-        if (placeMemories.has(r.id)) {
-            return {
-                ...r,
-                similarity: (r.similarity ?? 0) + PLACE_BOOST
-            };
-        }
-        return r;
-    });
-    // Re-sort with place boost
-    boosted.sort((a, b) => (b.similarity ?? 0) - (a.similarity ?? 0));
-    return boosted;
-}
-/**
- * Task 3: Boost memories from the same session (temporal)
- */
-function applySessionBoost(results, sessionId) {
-    const SESSION_BOOST = 0.1;
-    const boosted = results.map(r => {
-        // Check if memory's session matches query's session
-        const memSession = r.metadata?.sessionMetadata?.sessionId;
-        if (memSession === sessionId) {
-            return {
-                ...r,
-                similarity: (r.similarity ?? 0) + SESSION_BOOST
-            };
-        }
-        return r;
-    });
-    // Re-sort with session boost
-    boosted.sort((a, b) => (b.similarity ?? 0) - (a.similarity ?? 0));
-    return boosted;
-}
-/**
- * TEMPORAL FIX: Boost memories that contain date references for "when" questions
- * Also boost by date RECENCY - closer to today = higher for temporal queries
- */
-function applyTemporalBoost(results) {
-    const TEMPORAL_BOOST = 0.25; // Moderate boost for date-containing memories
-    const boosted = results.map(r => {
-        let boost = 0;
-        // Boost 1: Has date reference - high priority for temporal
-        if (hasDateReference(r.content ?? "")) {
-            boost += TEMPORAL_BOOST;
-        }
-        return {
-            ...r,
-            similarity: (r.similarity ?? 0) + boost
-        };
-    });
-    // Re-sort with temporal boost
-    boosted.sort((a, b) => (b.similarity ?? 0) - (a.similarity ?? 0));
-    return boosted;
-}
-/**
- * MULTI-HOP FIX: Get memories from adjacent sessions using DB query
- * This provides actual cross-session retrieval for multi-hop queries
- */
-async function applySessionAdjacencyBoost(results, input, limit) {
-    if (!input.project)
-        return results;
-    // Extract session tags from top results
-    const sessionTags = new Set();
-    for (const r of results.slice(0, 5)) {
-        const sessionTag = extractSessionTag(r.tags);
-        if (sessionTag) {
-            sessionTags.add(sessionTag);
-        }
-    }
-    if (sessionTags.size === 0) {
-        return results;
-    }
-    // Get adjacent session tags
-    const adjacentSessionTags = new Set();
-    for (const tag of sessionTags) {
-        const adjacent = getAdjacentSessionTags(tag);
-        for (const adj of adjacent) {
-            adjacentSessionTags.add(adj);
-        }
-    }
-    if (adjacentSessionTags.size === 0) {
-        return results;
-    }
-    // Fetch memories from adjacent sessions
-    const project = await requireProject(input.project);
-    const adjacentMemories = [];
-    for (const sessionTag of adjacentSessionTags) {
-        const memories = await getMemoriesBySession(project.id, sessionTag, limit);
-        adjacentMemories.push(...memories);
-    }
-    if (adjacentMemories.length === 0) {
-        return results;
-    }
-    // Merge with boost
-    const existingIds = new Set(results.map(r => r.id));
-    const ADJACENT_BOOST = 0.3;
-    const merged = [...results];
-    for (const mem of adjacentMemories) {
-        if (!existingIds.has(mem.id)) {
-            merged.push({
-                ...mem,
-                similarity: (mem.similarity ?? 0) + ADJACENT_BOOST
-            });
-        }
-    }
-    // Re-sort
-    merged.sort((a, b) => (b.similarity ?? 0) - (a.similarity ?? 0));
-    return merged.slice(0, limit * 2);
-}
-/**
- * Expand results with directly associated memories
- */
-async function expandWithAssociations(results, limit) {
-    const allIds = new Set(results.map(r => r.id));
-    const expanded = [...results];
-    // Get related for top results only
-    for (const r of results.slice(0, 3)) {
-        try {
-            const related = await getRelatedMemories(r.id, 5);
-            for (const rel of related) {
-                if (!allIds.has(rel.id)) {
-                    allIds.add(rel.id);
-                    expanded.push({
-                        ...rel,
-                        similarity: (rel.similarity ?? 0) * 0.8 // Slightly lower weight
-                    });
-                }
-            }
-        }
-        catch (e) {
-            // Skip errors
-        }
-    }
-    // Re-sort and return top results
-    expanded.sort((a, b) => (b.similarity ?? 0) - (a.similarity ?? 0));
-    return expanded.slice(0, limit);
-}
-/**
- * Get memories by session ID (stored in tags as "dialogue-1", "session-2", etc.)
- */
-async function getMemoriesBySession(projectId, sessionTag, limit) {
-    try {
-        const db = createDatabaseClient(await getDb());
-        const sqlite = db.$client;
-        // Query memories where tags contain the session tag
-        const statement = sqlite.prepare(`
-      SELECT
-        m.id as id,
-        m.project_id as projectId,
-        m.type as type,
-        m.content as content,
-        m.summary as summary,
-        m.tags as tags,
-        m.metadata as metadata,
-        m.embedding as embedding,
-        m.embedding_json as embeddingJson,
-        m.created_at as createdAt
-      FROM memories m
-      WHERE m.project_id = ? AND m.tags LIKE ?
-      ORDER BY m.created_at DESC
-      LIMIT ?
-    `);
-        const rows = statement.all(projectId, `%${sessionTag}%`, limit);
-        return rows.map((row) => ({
-            id: row.id,
-            projectId: row.projectId,
-            type: row.type,
-            content: row.content,
-            summary: row.summary ?? undefined,
-            tags: deserializeTags(row.tags ?? null),
-            metadata: deserializeMetadata(row.metadata ?? null),
-            createdAt: row.createdAt ? (normalizeTimestamp(Number(row.createdAt)) ?? undefined) : undefined,
-            similarity: 0.5, // Default similarity for fetched memories
-        }));
-    }
-    catch (error) {
-        logger.debug(`[HybridSearch] getMemoriesBySession failed: ${error}`);
-        return [];
-    }
-}
-/**
- * Extract session tag from memory tags (e.g., "dialogue-1", "session_1", "D1", "D2")
- */
-function extractSessionTag(tags) {
-    if (!tags)
-        return null;
-    // Look for patterns:
-    // - "dialogue-1", "dialogue_1", "dialogue1"
-    // - "session-2", "session_2", "session2"
-    // - "conv-1", "conv_1", "conv1"
-    // - "D1", "D2" (capital D followed by number)
-    for (const tag of tags) {
-        const lower = tag.toLowerCase();
-        if (/^(dialogue|session|conv|d)[_-]?\d+$/.test(lower)) {
-            return tag;
-        }
-    }
-    return null;
-}
-/**
- * Get adjacent session tags (e.g., dialogue-1 -> dialogue-2, D1 -> D2)
- */
-function getAdjacentSessionTags(currentTag) {
-    const adjacent = [];
-    // Pattern: dialogue-N, session-N, D-N, conv-N (with or without hyphen)
-    const match = currentTag.match(/^(.*?)(\d+)$/i);
-    if (match) {
-        const prefix = match[1];
-        const num = parseInt(match[2], 10);
-        // Adjacent: previous and next
-        if (num > 1) {
-            adjacent.push(`${prefix}${num - 1}`);
-        }
-        adjacent.push(`${prefix}${num + 1}`);
-    }
-    return adjacent;
-}
-/**
- * Vector semantic search using cosine similarity
- */
-async function vectorSearch(input, options) {
-    try {
-        const db = createDatabaseClient(await getDb());
-        const sqlite = db.$client;
-        const limit = options.limit ?? 10;
-        const tags = normalizeTags(options.tags ?? input.tags);
-        // Check for empty query
-        const isEmptyQuery = !input.query || input.query.trim() === '';
-        // Get query embedding (only for non-empty queries)
-        let queryEmbedding = null;
-        if (!isEmptyQuery) {
-            queryEmbedding = await getEmbedding(input.query);
-        }
-        // Build WHERE conditions
-        const conditions = [];
-        const params = [];
-        if (input.type) {
-            conditions.push('m.type = ?');
-            params.push(input.type);
-        }
-        if (tags.length) {
-            conditions.push('(' + tags.map(() => 'm.tags LIKE ?').join(' OR ') + ')');
-            params.push(...tags.map((tag) => `%${tag}%`));
-        }
-        let projectId = null;
-        if (input.project) {
-            const project = await requireProject(input.project);
-            projectId = project.id;
-            conditions.push('m.project_id = ?');
-            params.push(project.id);
-        }
-        const whereClause = conditions.length > 0
-            ? 'WHERE ' + conditions.join(' AND ')
-            : '';
-        // Fetch candidates for vector search
-        const statement = sqlite.prepare(`
-      SELECT
-        m.id as id,
-        m.project_id as projectId,
-        m.type as type,
-        m.content as content,
-        m.summary as summary,
-        m.tags as tags,
-        m.metadata as metadata,
-        m.embedding as embedding,
-        m.embedding_json as embeddingJson,
-        m.created_at as createdAt
-      FROM memories m
-      ${whereClause}
-      ORDER BY m.created_at DESC
-      LIMIT ?
-    `);
-        const rows = statement.all(...params, limit * 3);
-        // If no embedding available, return results ordered by recency
-        if (!queryEmbedding) {
-            return rows.slice(0, limit * 2).map((item) => ({
-                id: item.id,
-                projectId: item.projectId,
-                type: item.type,
-                content: item.content,
-                summary: item.summary ?? undefined,
-                tags: deserializeTags(item.tags ?? null),
-                metadata: deserializeMetadata(item.metadata ?? null),
-                createdAt: item.createdAt ? (normalizeTimestamp(Number(item.createdAt)) ?? undefined) : undefined,
-                similarity: 0,
-            }));
-        }
-        // Calculate cosine similarity for each result
-        const scored = rows
-            .map((row) => {
-            const embedding = parseEmbedding(row.embedding) ?? parseEmbedding(row.embeddingJson);
-            if (!embedding)
-                return null;
-            const similarity = cosineSimilarity(queryEmbedding, embedding);
-            return {
-                id: row.id,
-                projectId: row.projectId,
-                type: row.type,
-                content: row.content,
-                summary: row.summary,
-                tags: row.tags,
-                metadata: row.metadata,
-                createdAt: row.createdAt,
-                similarity,
-            };
-        })
-            .filter((item) => item !== null);
-        // Sort by similarity (descending) and return
-        scored.sort((a, b) => b.similarity - a.similarity);
-        return scored.slice(0, limit * 2).map((item) => ({
-            id: item.id,
-            projectId: item.projectId,
-            type: item.type,
-            content: item.content,
-            summary: item.summary ?? undefined,
-            tags: deserializeTags(item.tags ?? null),
-            metadata: deserializeMetadata(item.metadata ?? null),
-            createdAt: item.createdAt ? (normalizeTimestamp(Number(item.createdAt)) ?? undefined) : undefined,
-            similarity: item.similarity,
-        }));
-    }
-    catch (error) {
-        throw error;
-    }
-}
-/**
- * Apply small graph boost to results
- * Graph boost is ADDITIVE (not dominating)
- */
-function applyGraphBoostWithWeight(results, graphBoostMap, limit, graphWeight) {
-    // Apply SMALL additive boost to each result
-    // graphWeight should be 0.1-0.3, not > 1
-    const boosted = results.map(result => {
-        const boost = graphBoostMap[result.id] ?? 0;
-        // Add small nudge, don't replace similarity
-        const boostedSimilarity = (result.similarity ?? 0) + (boost * graphWeight);
-        return { ...result, similarity: boostedSimilarity };
-    });
-    // Re-sort by boosted similarity
-    boosted.sort((a, b) => (b.similarity ?? 0) - (a.similarity ?? 0));
-    return boosted.slice(0, limit);
-}
-//# sourceMappingURL=hybrid-search.js.map

package/dist/core/memory/importance.d.ts

@@ -1,46 +0,0 @@
-/**
- * Importance Scoring System
- * Calculates and manages memory importance scores (0-100) with temporal decay
- */
-import type { Memory } from '../../db/drizzle/schema.js';
-export interface ImportanceScore {
-    score: number;
-    components: {
-        base: number;
-        recency: number;
-        accessFrequency: number;
-        typeWeight: number;
-        userFlags: number;
-    };
-    explanation: string;
-}
-/**
- * Calculate importance score for a memory
- *
- * Formula: base + recency + accessFrequency + typeWeight + userFlags
- * All values are clamped to 0-100 range
- */
-export declare function calculateImportance(memory: Partial<Memory>): ImportanceScore;
-/**
- * Update importance score for a memory
- * Used when memory is accessed or modified
- */
-export declare function updateImportanceScore(memoryId: string, incrementAccess?: boolean): Promise<number>;
-/**
- * Get low-importance memories that are candidates for consolidation
- * These are old, rarely accessed memories with low importance scores
- */
-export declare function getLowImportanceMemories(projectId: string, options?: {
-    minAge?: number;
-    maxImportance?: number;
-    limit?: number;
-}): Promise<any[]>;
-/**
- * Set importance score manually (for user override)
- */
-export declare function setImportanceScore(memoryId: string, score: number): Promise<void>;
-/**
- * Pin a memory to prevent decay and consolidation
- */
-export declare function pinMemory(memoryId: string, pinned?: boolean): Promise<void>;
-//# sourceMappingURL=importance.d.ts.map