squish-memory 1.2.0 → 1.2.1

package/dist/core/graph/entity-deduplicator.js

@@ -1,183 +0,0 @@
-/**
- * Entity Deduplicator
- *
- * Deduplicates entities using embedding similarity and string matching.
- * Critical for knowledge graph - prevents duplicate nodes for aliases like "AWS" / "Amazon Web Services".
- */
-import { eq } from 'drizzle-orm';
-import { getDb } from '../../db/index.js';
-import { getSchema } from '../../db/schema.js';
-import { cosineSimilarity } from '../utils/vector-operations.js';
-import { parseEmbedding } from '../lib/parse-embedding.js';
-import { logger } from '../logger.js';
-// Common entity aliases for normalization
-const KNOWN_ALIASES = {
-    'amazon web services': ['aws'],
-    'aws': ['amazon web services'],
-    'postgresql': ['postgres', 'pg'],
-    'kubernetes': ['k8s'],
-    'k8s': ['kubernetes'],
-    'javascript': ['js'],
-    'typescript': ['ts'],
-    'node.js': ['nodejs', 'node'],
-    'react.js': ['react'],
-    'vue.js': ['vue'],
-    'machine learning': ['ml'],
-    'artificial intelligence': ['ai'],
-    'large language model': ['llm'],
-    'natural language processing': ['nlp'],
-};
-/**
- * Find and merge duplicate entities in a project.
- */
-export async function deduplicateProjectEntities(projectId, options) {
-    const { similarityThreshold = 0.85, dryRun = false } = options || {};
-    const db = await getDb();
-    const schema = await getSchema();
-    // Get all entities for the project
-    const entities = await db
-        .select()
-        .from(schema.entities)
-        .where(eq(schema.entities.projectId, projectId));
-    if (entities.length < 2) {
-        return {
-            merged: 0,
-            aliases: [],
-            totalEntities: entities.length,
-            uniqueEntities: entities.length,
-        };
-    }
-    const aliases = [];
-    const merged = new Set();
-    let mergeCount = 0;
-    // Compare each pair of entities
-    for (let i = 0; i < entities.length; i++) {
-        if (merged.has(entities[i].id))
-            continue;
-        for (let j = i + 1; j < entities.length; j++) {
-            if (merged.has(entities[j].id))
-                continue;
-            const similarity = await computeEntitySimilarity(entities[i], entities[j]);
-            if (similarity >= similarityThreshold) {
-                // Merge j into i (keep the one with more mentions)
-                const target = (entities[i].mentionCount || 0) >= (entities[j].mentionCount || 0)
-                    ? entities[i]
-                    : entities[j];
-                const source = target.id === entities[i].id ? entities[j] : entities[i];
-                aliases.push({
-                    from: source.name,
-                    to: target.name,
-                    similarity,
-                });
-                if (!dryRun) {
-                    await mergeEntities(source, target, db, schema);
-                }
-                merged.add(source.id);
-                mergeCount++;
-            }
-        }
-    }
-    logger.info('Entity deduplication completed', {
-        projectId,
-        totalEntities: entities.length,
-        merged: mergeCount,
-        dryRun,
-    });
-    return {
-        merged: mergeCount,
-        aliases,
-        totalEntities: entities.length,
-        uniqueEntities: entities.length - mergeCount,
-    };
-}
-/**
- * Compute similarity between two entities.
- * Uses a combination of string matching, known aliases, and embedding similarity.
- */
-async function computeEntitySimilarity(entity1, entity2) {
-    // Must be same type to be considered duplicates
-    if (entity1.type !== entity2.type) {
-        return 0;
-    }
-    const name1 = entity1.name.toLowerCase().trim();
-    const name2 = entity2.name.toLowerCase().trim();
-    // Exact match
-    if (name1 === name2)
-        return 1.0;
-    // Check known aliases
-    const name1Aliases = KNOWN_ALIASES[name1] || [];
-    const name2Aliases = KNOWN_ALIASES[name2] || [];
-    if (name1Aliases.includes(name2) || name2Aliases.includes(name1)) {
-        return 0.99; // Very high confidence for known aliases
-    }
-    // String similarity (Jaccard on words)
-    const words1 = new Set(name1.split(/[\s._-]+/).filter((w) => w.length > 1));
-    const words2 = new Set(name2.split(/[\s._-]+/).filter((w) => w.length > 1));
-    const words1Arr = Array.from(words1);
-    const words2Arr = Array.from(words2);
-    const intersection = words1Arr.filter(w => words2.has(w));
-    const union = new Set([...words1Arr, ...words2Arr]);
-    if (words1.size > 0 && words2.size > 0) {
-        const jaccard = intersection.length / union.size;
-        if (jaccard >= 0.8)
-            return jaccard;
-    }
-    // One contains the other (e.g., "React" in "React.js")
-    if (name1.includes(name2) || name2.includes(name1)) {
-        const shorter = name1.length < name2.length ? name1 : name2;
-        const longer = name1.length < name2.length ? name2 : name1;
-        // Only count as similar if the shorter name is a significant portion
-        if (shorter.length / longer.length >= 0.5) {
-            return 0.85;
-        }
-    }
-    // Embedding similarity (if available)
-    const embedding1 = parseEmbedding(entity1.embedding || entity1.embedding_json);
-    const embedding2 = parseEmbedding(entity2.embedding || entity2.embedding_json);
-    if (embedding1 && embedding2) {
-        const embSimilarity = cosineSimilarity(embedding1, embedding2);
-        if (embSimilarity >= similarityThreshold) {
-            return embSimilarity;
-        }
-    }
-    return 0;
-}
-/**
- * Merge source entity into target entity.
- * Updates all relations pointing to source to point to target instead.
- */
-async function mergeEntities(source, target, db, schema) {
-    // Update all relations pointing FROM source to point FROM target
-    await db
-        .update(schema.entityRelations)
-        .set({ fromEntityId: target.id })
-        .where(eq(schema.entityRelations.fromEntityId, source.id));
-    // Update all relations pointing TO source to point TO target
-    await db
-        .update(schema.entityRelations)
-        .set({ toEntityId: target.id })
-        .where(eq(schema.entityRelations.toEntityId, source.id));
-    // Add source name as alias to target
-    const existingAliases = Array.isArray(target.aliases) ? target.aliases : [];
-    const updatedAliases = [...new Set([...existingAliases, source.name])];
-    await db
-        .update(schema.entities)
-        .set({
-        aliases: updatedAliases,
-        mentionCount: (target.mentionCount || 0) + (source.mentionCount || 0),
-        updatedAt: new Date(),
-    })
-        .where(eq(schema.entities.id, target.id));
-    // Delete source entity
-    await db
-        .delete(schema.entities)
-        .where(eq(schema.entities.id, source.id));
-    logger.debug('Merged entity', {
-        source: source.name,
-        target: target.name,
-        aliases: updatedAliases,
-    });
-}
-// Module-level threshold for embedding similarity
-const similarityThreshold = 0.85;
-//# sourceMappingURL=entity-deduplicator.js.map

package/dist/core/graph/graph-builder.d.ts

@@ -1,46 +0,0 @@
-/**
- * Graph Builder
- *
- * Orchestrates the full pipeline: extract entities and relations from
- * memories and store them in the knowledge graph.
- */
-export interface GraphBuildStats {
-    memoriesProcessed: number;
-    entitiesCreated: number;
-    relationsCreated: number;
-    entitiesDeduplicated: number;
-    errors: number;
-    durationMs: number;
-}
-export interface GraphAddStats {
-    entitiesCreated: number;
-    relationsCreated: number;
-    source: 'llm' | 'regex' | 'fallback' | 'none';
-}
-/**
- * Build or rebuild the entity graph for a project.
- * Processes all memories in the project, extracting entities and relations.
- */
-export declare function buildGraphForProject(projectPath: string, options?: {
-    clearExisting?: boolean;
-    batchSize?: number;
-    preferLLM?: boolean;
-    deduplicate?: boolean;
-}): Promise<GraphBuildStats>;
-/**
- * Add a single memory to the knowledge graph.
- * Used for incremental updates when new memories are stored.
- */
-export declare function addMemoryToGraph(memoryId: string, options?: {
-    preferLLM?: boolean;
-}): Promise<GraphAddStats>;
-/**
- * Get graph statistics for a project.
- */
-export declare function getGraphStats(projectPath: string): Promise<{
-    entityCount: number;
-    relationCount: number;
-    relationTypes: Record<string, number>;
-    avgConnections: number;
-}>;
-//# sourceMappingURL=graph-builder.d.ts.map

package/dist/core/graph/graph-builder.js

@@ -1,174 +0,0 @@
-/**
- * Graph Builder
- *
- * Orchestrates the full pipeline: extract entities and relations from
- * memories and store them in the knowledge graph.
- */
-import { eq } from 'drizzle-orm';
-import { getDb } from '../../db/index.js';
-import { getSchema } from '../../db/schema.js';
-import { logger } from '../logger.js';
-import { config } from '../../config.js';
-import { extractAndStoreRelations } from './relationship-extractor.js';
-import { deduplicateProjectEntities } from './entity-deduplicator.js';
-/**
- * Build or rebuild the entity graph for a project.
- * Processes all memories in the project, extracting entities and relations.
- */
-export async function buildGraphForProject(projectPath, options) {
-    const startTime = Date.now();
-    const { clearExisting = false, batchSize = 10, preferLLM = true, deduplicate = true, } = options || {};
-    const db = await getDb();
-    const schema = await getSchema();
-    // Get project
-    const projectRows = await db
-        .select()
-        .from(schema.projects)
-        .where(eq(schema.projects.path, projectPath))
-        .limit(1);
-    if (projectRows.length === 0) {
-        logger.warn('Project not found for graph build', { projectPath });
-        return {
-            memoriesProcessed: 0,
-            entitiesCreated: 0,
-            relationsCreated: 0,
-            entitiesDeduplicated: 0,
-            errors: 0,
-            durationMs: Date.now() - startTime,
-        };
-    }
-    const projectId = projectRows[0].id;
-    // Clear existing graph if requested
-    if (clearExisting) {
-        const { clearProjectGraph } = await import('./relationship-extractor.js');
-        await clearProjectGraph(projectId);
-        logger.info('Cleared existing graph for rebuild', { projectId });
-    }
-    // Get all active memories for the project
-    const memories = await db
-        .select()
-        .from(schema.memories)
-        .where(eq(schema.memories.projectId, projectId));
-    logger.info('Starting graph build for project', {
-        projectId,
-        memoryCount: memories.length,
-        clearExisting,
-    });
-    let entitiesCreated = 0;
-    let relationsCreated = 0;
-    let errors = 0;
-    // Process memories in batches
-    for (let i = 0; i < memories.length; i += batchSize) {
-        const batch = memories.slice(i, i + batchSize);
-        for (const memory of batch) {
-            try {
-                const result = await extractAndStoreRelations(memory.id, memory.content, projectId, { preferLLM });
-                entitiesCreated += result.entities;
-                relationsCreated += result.relations;
-            }
-            catch (error) {
-                logger.error('Error processing memory for graph', {
-                    memoryId: memory.id,
-                    error: error,
-                });
-                errors++;
-            }
-        }
-        logger.debug('Graph build batch progress', {
-            processed: Math.min(i + batchSize, memories.length),
-            total: memories.length,
-        });
-    }
-    // Deduplicate entities
-    let entitiesDeduplicated = 0;
-    if (deduplicate) {
-        try {
-            const dedupResult = await deduplicateProjectEntities(projectId);
-            entitiesDeduplicated = dedupResult.merged;
-        }
-        catch (error) {
-            logger.error('Error during entity deduplication', { error: error });
-        }
-    }
-    const stats = {
-        memoriesProcessed: memories.length,
-        entitiesCreated,
-        relationsCreated,
-        entitiesDeduplicated,
-        errors,
-        durationMs: Date.now() - startTime,
-    };
-    logger.info('Graph build completed', stats);
-    return stats;
-}
-/**
- * Add a single memory to the knowledge graph.
- * Used for incremental updates when new memories are stored.
- */
-export async function addMemoryToGraph(memoryId, options) {
-    const db = await getDb();
-    const schema = await getSchema();
-    // Get the memory
-    const memories = await db
-        .select()
-        .from(schema.memories)
-        .where(eq(schema.memories.id, memoryId))
-        .limit(1);
-    if (memories.length === 0) {
-        return { entitiesCreated: 0, relationsCreated: 0, source: 'none' };
-    }
-    const memory = memories[0];
-    if (!memory.projectId) {
-        return { entitiesCreated: 0, relationsCreated: 0, source: 'none' };
-    }
-    // Use global config if not explicitly overridden
-    const preferLLM = options?.preferLLM ?? config.llmEnabled;
-    const result = await extractAndStoreRelations(memoryId, memory.content, memory.projectId, { preferLLM });
-    return {
-        entitiesCreated: result.entities,
-        relationsCreated: result.relations,
-        source: result.source,
-    };
-}
-/**
- * Get graph statistics for a project.
- */
-export async function getGraphStats(projectPath) {
-    const db = await getDb();
-    const schema = await getSchema();
-    const projectRows = await db
-        .select({ id: schema.projects.id })
-        .from(schema.projects)
-        .where(eq(schema.projects.path, projectPath))
-        .limit(1);
-    if (projectRows.length === 0) {
-        return { entityCount: 0, relationCount: 0, relationTypes: {}, avgConnections: 0 };
-    }
-    const projectId = projectRows[0].id;
-    const entities = await db
-        .select()
-        .from(schema.entities)
-        .where(eq(schema.entities.projectId, projectId));
-    const relations = await db
-        .select()
-        .from(schema.entityRelations);
-    // Filter relations to only those involving project entities
-    const entityIds = new Set(entities.map((e) => e.id));
-    const projectRelations = relations.filter((r) => entityIds.has(r.fromEntityId) || entityIds.has(r.toEntityId));
-    // Count by type
-    const relationTypes = {};
-    for (const r of projectRelations) {
-        const type = r.type || 'unknown';
-        relationTypes[type] = (relationTypes[type] || 0) + 1;
-    }
-    const avgConnections = entities.length > 0
-        ? (projectRelations.length * 2) / entities.length
-        : 0;
-    return {
-        entityCount: entities.length,
-        relationCount: projectRelations.length,
-        relationTypes,
-        avgConnections: Math.round(avgConnections * 100) / 100,
-    };
-}
-//# sourceMappingURL=graph-builder.js.map

package/dist/core/graph/graph-traversal.d.ts

@@ -1,80 +0,0 @@
-/**
- * Graph Traversal Engine
- *
- * BFS/DFS traversal of the entity relationship graph. This is the
- * core capability that enables multi-hop queries like:
- *
- *   "Was Alice's project affected by Tuesday's outage?"
- *
- * Which requires traversing: Alice → works_on → Project Atlas → uses → PostgreSQL → caused → outage
- *
- * Vector search alone can't answer this because the bridge fact
- * ("Project Atlas uses PostgreSQL") mentions neither Alice nor Tuesday.
- */
-import type { RelationType } from './llm-entity-extractor.js';
-export interface GraphNode {
-    id: string;
-    name: string;
-    type: string;
-    description: string | null;
-    properties: Record<string, unknown> | null;
-}
-export interface GraphEdge {
-    id: string;
-    fromId: string;
-    toId: string;
-    relationType: RelationType | string;
-    weight: number;
-    properties: Record<string, unknown> | null;
-}
-export interface TraversalPath {
-    nodes: GraphNode[];
-    edges: GraphEdge[];
-    totalWeight: number;
-    hopCount: number;
-}
-export interface NeighborhoodResult {
-    center: GraphNode;
-    nodes: GraphNode[];
-    edges: GraphEdge[];
-    radius: number;
-}
-/**
- * BFS traversal from a starting entity, following relationship edges.
- * Returns all reachable entities within maxDepth hops.
- */
-export declare function traverse(startEntityId: string, options?: {
-    maxDepth?: number;
-    relationTypes?: RelationType[];
-    minWeight?: number;
-    direction?: 'outgoing' | 'incoming' | 'both';
-    limit?: number;
-}): Promise<GraphNode[]>;
-/**
- * Find all paths between two entities within a maximum number of hops.
- * Uses BFS with path tracking.
- */
-export declare function findPaths(fromEntityId: string, toEntityId: string, options?: {
-    maxHops?: number;
-    relationTypes?: RelationType[];
-    minWeight?: number;
-    maxPaths?: number;
-}): Promise<TraversalPath[]>;
-/**
- * Get the neighborhood around an entity - all entities within N hops.
- * Returns both the entities and the edges connecting them.
- */
-export declare function getNeighborhood(centerEntityId: string, options?: {
-    radius?: number;
-    relationTypes?: RelationType[];
-    minWeight?: number;
-    limit?: number;
-}): Promise<NeighborhoodResult | null>;
-/**
- * Find entities by name (fuzzy matching).
- */
-export declare function findEntitiesByName(name: string, projectId: string, options?: {
-    limit?: number;
-    fuzzy?: boolean;
-}): Promise<GraphNode[]>;
-//# sourceMappingURL=graph-traversal.d.ts.map