squish-memory 1.1.5 → 1.2.0

package/dist/core/graph/llm-entity-extractor.js

@@ -0,0 +1,313 @@
+/**
+ * LLM Entity Extractor
+ *
+ * Extracts named entities and relationships from memory content using
+ * LLM-powered analysis. Falls back to regex extraction when LLM is unavailable.
+ */
+import { logger } from '../logger.js';
+import { config } from '../../config.js';
+import { extractEntities as regexExtractEntities, } from '../memory/entity-extractor.js';
+// ─── LLM Prompt ─────────────────────────────────────────────────────────────
+const ENTITY_EXTRACTION_PROMPT = `You are an entity and relationship extractor for an AI memory system. Given text, extract:
+
+1. ENTITIES: Named things mentioned (people, projects, systems, tools, concepts, events)
+2. RELATIONSHIPS: How entities connect to each other
+
+Return ONLY valid JSON in this exact format:
+{
+  "entities": [
+    {"name": "Alice", "type": "person", "confidence": 0.95},
+    {"name": "Project Atlas", "type": "concept", "confidence": 0.9}
+  ],
+  "relations": [
+    {"from": "Alice", "to": "Project Atlas", "type": "works_on", "confidence": 0.85, "context": "Alice is the tech lead on Project Atlas"},
+    {"from": "Project Atlas", "to": "PostgreSQL", "type": "uses", "confidence": 0.9, "context": "Project Atlas uses PostgreSQL for its primary datastore"}
+  ]
+}
+
+Entity types: person, file, function, class, concept, tool, date, location, pattern, technique, other
+Relation types: works_on, depends_on, manages, uses, caused, located_in, belongs_to, reports_to, occurred_on, affects, contains, implements, extends, related_to, part_of, owns, created, resolved, blocks
+
+Rules:
+- Extract only explicitly mentioned entities and relationships
+- Use the most specific relation type available
+- Include confidence scores (0-1)
+- Keep entity names as they appear in text (preserve capitalization)
+- For ambiguous relationships, use "related_to"
+- Do NOT invent entities or relationships not present in the text`;
+// ─── LLM Call Abstraction ───────────────────────────────────────────────────
+/**
+ * Call an LLM for entity extraction.
+ * Uses OpenAI-compatible API (works with OpenAI, Ollama, LM Studio, local endpoints).
+ */
+async function callLLM(prompt, content) {
+    // Determine which provider to use for extraction
+    const provider = config.llmProvider;
+    const endpoint = config.llmEndpoint || config.lmStudioUrl || config.ollamaUrl;
+    const model = config.llmExtractionModel;
+    if (!model) {
+        logger.debug('LLM extraction model not configured; using regex fallback');
+        return null;
+    }
+    // Try OpenAI first if configured and provider matches
+    if ((provider === 'openai' || !provider) && config.openAiApiKey) {
+        try {
+            const chatUrl = config.openAiApiUrl.replace('/embeddings', '/chat/completions');
+            const response = await fetch(chatUrl, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${config.openAiApiKey}`,
+                },
+                body: JSON.stringify({
+                    model,
+                    messages: [
+                        { role: 'system', content: prompt },
+                        { role: 'user', content },
+                    ],
+                    temperature: 0.1,
+                    max_tokens: 2000,
+                }),
+            });
+            if (!response.ok) {
+                logger.warn(`LLM entity extraction failed (OpenAI): ${response.status}`);
+                return null;
+            }
+            const payload = await response.json();
+            return payload.choices?.[0]?.message?.content ?? null;
+        }
+        catch (error) {
+            logger.warn('LLM entity extraction error (OpenAI):', { error: error });
+        }
+    }
+    // Try Ollama if configured and provider matches
+    if ((provider === 'ollama' || !provider) && config.ollamaUrl) {
+        try {
+            const response = await fetch(`${config.ollamaUrl}/api/chat`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    model,
+                    messages: [
+                        { role: 'system', content: prompt },
+                        { role: 'user', content },
+                    ],
+                    stream: false,
+                    options: { temperature: 0.1 },
+                }),
+            });
+            if (!response.ok) {
+                logger.warn(`Ollama entity extraction failed: ${response.status}`);
+                return null;
+            }
+            const payload = await response.json();
+            return payload.message?.content ?? null;
+        }
+        catch (error) {
+            logger.warn('LLM entity extraction error (Ollama):', { error: error });
+        }
+    }
+    // Try LM Studio / custom endpoint if configured and provider matches
+    if ((provider === 'lmstudio' || provider === 'local' || !provider) && (config.lmStudioUrl || endpoint)) {
+        try {
+            const baseUrl = endpoint || config.lmStudioUrl;
+            if (!baseUrl)
+                return null;
+            const response = await fetch(`${baseUrl}/v1/chat/completions`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    model,
+                    messages: [
+                        { role: 'system', content: prompt },
+                        { role: 'user', content },
+                    ],
+                    temperature: 0.1,
+                    max_tokens: 2000,
+                }),
+            });
+            if (!response.ok) {
+                logger.warn(`LM Studio entity extraction failed: ${response.status}`);
+                return null;
+            }
+            const payload = await response.json();
+            return payload.choices?.[0]?.message?.content ?? null;
+        }
+        catch (error) {
+            logger.warn('LLM entity extraction error (LM Studio):', { error: error });
+        }
+    }
+    return null;
+}
+// ─── JSON Parsing ────────────────────────────────────────────────────────────
+/**
+ * Parse LLM response into structured extraction result.
+ * Handles various response formats (raw JSON, markdown code blocks, etc.)
+ */
+function parseLLMResponse(response) {
+    // Try to extract JSON from the response
+    let jsonStr = response.trim();
+    // Handle markdown code blocks
+    const codeBlockMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
+    if (codeBlockMatch) {
+        jsonStr = codeBlockMatch[1].trim();
+    }
+    // Try to find JSON object in the response
+    const jsonMatch = jsonStr.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) {
+        logger.debug('No JSON found in LLM response');
+        return null;
+    }
+    try {
+        const parsed = JSON.parse(jsonMatch[0]);
+        // Validate structure
+        if (!parsed.entities || !Array.isArray(parsed.entities)) {
+            logger.debug('LLM response missing entities array');
+            return null;
+        }
+        const entities = parsed.entities
+            .filter((e) => e.name && e.type)
+            .map((e) => ({
+            name: String(e.name),
+            type: validateEntityType(String(e.type)),
+            confidence: typeof e.confidence === 'number' ? Math.min(1, Math.max(0, e.confidence)) : 0.8,
+            startIndex: 0,
+            endIndex: 0,
+            context: String(e.context || ''),
+            normalized: e.normalized ? String(e.normalized) : undefined,
+        }));
+        const relations = (parsed.relations || [])
+            .filter((r) => r.from && r.to && r.type)
+            .map((r) => ({
+            fromEntity: String(r.from),
+            toEntity: String(r.to),
+            relationType: validateRelationType(String(r.type)),
+            confidence: typeof r.confidence === 'number' ? Math.min(1, Math.max(0, r.confidence)) : 0.7,
+            context: String(r.context || ''),
+        }));
+        return { entities, relations };
+    }
+    catch (error) {
+        logger.debug('Failed to parse LLM response as JSON', { error: error });
+        return null;
+    }
+}
+function validateEntityType(type) {
+    const validTypes = ['person', 'file', 'function', 'class', 'concept', 'tool', 'date', 'location', 'pattern', 'technique', 'other'];
+    const lower = type.toLowerCase();
+    if (validTypes.includes(lower))
+        return lower;
+    return 'other';
+}
+function validateRelationType(type) {
+    const validTypes = [
+        'works_on', 'depends_on', 'manages', 'uses', 'caused', 'located_in',
+        'belongs_to', 'reports_to', 'occurred_on', 'affects', 'contains',
+        'implements', 'extends', 'related_to', 'part_of', 'owns', 'created',
+        'resolved', 'blocks',
+    ];
+    const lower = type.toLowerCase();
+    if (validTypes.includes(lower))
+        return lower;
+    return 'related_to';
+}
+// ─── Main Extraction Functions ───────────────────────────────────────────────
+/**
+ * Extract entities and relationships from text using LLM.
+ * Falls back to regex extraction when LLM is unavailable.
+ */
+export async function extractEntitiesAndRelations(content, options) {
+    const { preferLLM = true, maxContentLength = 4000 } = options || {};
+    // Truncate very long content
+    const truncatedContent = content.length > maxContentLength
+        ? content.substring(0, maxContentLength) + '...'
+        : content;
+    // Try LLM extraction first if preferred
+    if (preferLLM) {
+        const llmResult = await extractWithLLM(truncatedContent);
+        if (llmResult) {
+            logger.debug('LLM entity extraction succeeded', {
+                entityCount: llmResult.entities.length,
+                relationCount: llmResult.relations.length,
+            });
+            return { ...llmResult, source: 'llm' };
+        }
+    }
+    // Fall back to regex extraction
+    const regexEntities = await regexExtractEntities(content);
+    if (regexEntities.length > 0) {
+        logger.debug('Regex entity extraction fallback', {
+            entityCount: regexEntities.length,
+        });
+        return {
+            entities: regexEntities,
+            relations: [], // Regex extraction doesn't produce relations
+            source: 'regex',
+        };
+    }
+    return { entities: [], relations: [], source: 'none' };
+}
+/**
+ * Extract entities and relations using LLM.
+ */
+async function extractWithLLM(content) {
+    const response = await callLLM(ENTITY_EXTRACTION_PROMPT, content);
+    if (!response)
+        return null;
+    const parsed = parseLLMResponse(response);
+    if (!parsed)
+        return null;
+    // Validate that extracted entities actually appear in the content
+    const validatedEntities = parsed.entities.filter((entity) => {
+        // Allow entities whose name or a significant substring appears in content
+        const nameLower = entity.name.toLowerCase();
+        const contentLower = content.toLowerCase();
+        // Check if the entity name or a key part of it appears in content
+        const words = nameLower.split(/\s+/);
+        return words.some(word => word.length > 2 && contentLower.includes(word));
+    });
+    // Validate that relation entities exist in the entity list
+    const entityNames = new Set(validatedEntities.map(e => e.name.toLowerCase()));
+    const validatedRelations = parsed.relations.filter((relation) => {
+        const fromExists = entityNames.has(relation.fromEntity.toLowerCase());
+        const toExists = entityNames.has(relation.toEntity.toLowerCase());
+        // Allow relations where at least one entity is known
+        // (the LLM might use slightly different names)
+        return fromExists || toExists ||
+            // Also allow if both entity names appear in the content
+            (content.toLowerCase().includes(relation.fromEntity.toLowerCase()) &&
+                content.toLowerCase().includes(relation.toEntity.toLowerCase()));
+    });
+    return {
+        entities: validatedEntities,
+        relations: validatedRelations,
+    };
+}
+/**
+ * Extract entities and relations from multiple memories in batch.
+ * More efficient than calling extractEntitiesAndRelations for each memory.
+ */
+export async function batchExtractEntitiesAndRelations(contents, options) {
+    const { batchSize = 5 } = options || {};
+    const results = [];
+    // Process in batches to avoid overwhelming the LLM
+    for (let i = 0; i < contents.length; i += batchSize) {
+        const batch = contents.slice(i, i + batchSize);
+        const batchResults = await Promise.all(batch.map(content => extractEntitiesAndRelations(content, options)));
+        results.push(...batchResults);
+    }
+    return results;
+}
+/**
+ * Get the extraction prompt for testing/debugging purposes.
+ */
+export function getExtractionPrompt() {
+    return ENTITY_EXTRACTION_PROMPT;
+}
+/**
+ * Parse an LLM response for testing/debugging purposes.
+ */
+export function testParseLLMResponse(response) {
+    return parseLLMResponse(response);
+}
+//# sourceMappingURL=llm-entity-extractor.js.map

package/dist/core/graph/multi-hop-retrieval.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Multi-Hop Retrieval
+ *
+ * Combines vector search with graph traversal to answer queries that
+ * require following relationships across entities.
+ */
+import { type SearchResult } from '../memory/hybrid-search.js';
+import { type TraversalPath } from './graph-traversal.js';
+export interface MultiHopResult extends SearchResult {
+    /** How this result was found: 'vector' (direct search) or 'graph' (via traversal) */
+    retrievalPath: 'vector' | 'graph' | 'both';
+    /** The graph path that led to this result (if found via graph) */
+    graphPath?: TraversalPath;
+    /** Entities that connected this result to the query */
+    connectingEntities?: string[];
+    /** Hybrid score for backward compatibility */
+    hybridScore?: number;
+}
+export interface MultiHopSearchOptions {
+    /** Search query */
+    query: string;
+    /** Project path for scoping */
+    project?: string;
+    /** Maximum number of results */
+    limit?: number;
+    /** Maximum graph traversal depth */
+    maxHops?: number;
+    /** Whether to include vector-only results */
+    includeVectorResults?: boolean;
+    /** Whether to include graph-expanded results */
+    includeGraphResults?: boolean;
+    /** Minimum graph path weight to include */
+    minPathWeight?: number;
+    /** Session ID for context */
+    sessionId?: string;
+}
+/**
+ * Perform multi-hop search combining vector search with graph traversal.
+ */
+export declare function multiHopSearch(options: MultiHopSearchOptions): Promise<MultiHopResult[]>;
+/** Check if query needs multi-hop graph traversal. */
+export declare function needsMultiHop(query: string): boolean;
+/**
+ * Get a human-readable explanation of how a multi-hop result was found.
+ */
+export declare function explainRetrievalPath(result: MultiHopResult): string;
+export declare const wouldBenefitFromMultiHop: typeof needsMultiHop;
+//# sourceMappingURL=multi-hop-retrieval.d.ts.map

package/dist/core/graph/multi-hop-retrieval.js

@@ -0,0 +1,215 @@
+/**
+ * Multi-Hop Retrieval
+ *
+ * Combines vector search with graph traversal to answer queries that
+ * require following relationships across entities.
+ */
+import { eq } from 'drizzle-orm';
+import { getDb } from '../../db/index.js';
+import { getSchema } from '../../db/schema.js';
+import { hybridSearch } from '../memory/hybrid-search.js';
+import { extractEntitiesAndRelations } from './llm-entity-extractor.js';
+import { extractEntityNames } from '../memory/entity-extractor.js';
+import { findEntitiesByName, findPaths, traverse } from './graph-traversal.js';
+import { logger } from '../logger.js';
+import { config } from '../../config.js';
+// ─── Main Multi-Hop Search ──────────────────────────────────────────────────
+/**
+ * Perform multi-hop search combining vector search with graph traversal.
+ */
+export async function multiHopSearch(options) {
+    const { query, project, limit = 10, maxHops = 3, includeVectorResults = true, includeGraphResults = true, minPathWeight = 1, sessionId, } = options;
+    const results = [];
+    const seenMemoryIds = new Set();
+    // Step 1: Extract query entities (respect global LLM config)
+    let queryEntityNames;
+    try {
+        const extraction = await extractEntitiesAndRelations(query, { preferLLM: config.llmEnabled });
+        queryEntityNames = extraction.entities.map(e => e.name);
+        logger.debug('Multi-hop search: entity extraction succeeded', {
+            query: query.substring(0, 100),
+            entities: queryEntityNames,
+            source: extraction.source,
+        });
+    }
+    catch (error) {
+        // LLM failed or disabled - fall back to simple keyword extraction
+        logger.debug('Multi-hop search: entity extraction failed, falling back to keywords', { error });
+        const simpleEntities = extractEntityNames(query);
+        queryEntityNames = simpleEntities;
+        logger.debug('Multi-hop search: keyword extraction', {
+            entities: queryEntityNames,
+        });
+    }
+    // Step 2: Vector search (baseline results)
+    if (includeVectorResults) {
+        const vectorResults = await hybridSearch({
+            query,
+            project,
+            limit: limit * 2, // Get more candidates for reranking
+        });
+        for (const result of vectorResults) {
+            if (!seenMemoryIds.has(result.id)) {
+                seenMemoryIds.add(result.id);
+                results.push({
+                    ...result,
+                    retrievalPath: 'vector',
+                });
+            }
+        }
+    }
+    // Step 3: Graph expansion (if we have entities)
+    if (includeGraphResults && queryEntityNames.length > 0 && project) {
+        const graphResults = await expandViaGraph(queryEntityNames, project, maxHops, minPathWeight, limit);
+        // Find memories connected to graph-expanded entities
+        for (const graphResult of graphResults) {
+            // Search for memories mentioning the expanded entities
+            const expandedMemories = await hybridSearch({
+                query: graphResult.entityName,
+                project,
+                limit: 5,
+            });
+            for (const memory of expandedMemories) {
+                if (!seenMemoryIds.has(memory.id)) {
+                    seenMemoryIds.add(memory.id);
+                    results.push({
+                        ...memory,
+                        retrievalPath: 'graph',
+                        graphPath: graphResult.path,
+                        connectingEntities: [graphResult.entityName],
+                    });
+                }
+            }
+        }
+    }
+    // Step 4: Rank combined results
+    // Vector results keep their hybrid score, graph results get a boost
+    // for being connected to query entities via the knowledge graph
+    const ranked = rankMultiHopResults(results, queryEntityNames);
+    return ranked.slice(0, limit);
+}
+/**
+ * Expand query entities via the knowledge graph to find related entities.
+ */
+async function expandViaGraph(queryEntityNames, project, maxHops, minPathWeight, limit) {
+    const db = await getDb();
+    const schema = await getSchema();
+    const results = [];
+    const seenEntities = new Set();
+    // Get project ID
+    const projectRows = await db
+        .select({ id: schema.projects.id })
+        .from(schema.projects)
+        .where(eq(schema.projects.path, project))
+        .limit(1);
+    if (projectRows.length === 0)
+        return [];
+    const projectId = projectRows[0].id;
+    // For each query entity, find it in the graph and expand
+    for (const entityName of queryEntityNames) {
+        const entities = await findEntitiesByName(entityName, projectId, { limit: 3 });
+        for (const entity of entities) {
+            if (seenEntities.has(entity.id))
+                continue;
+            seenEntities.add(entity.id);
+            // Traverse the graph from this entity
+            const neighbors = await traverse(entity.id, {
+                maxDepth: maxHops,
+                minWeight: minPathWeight,
+                limit: 20,
+            });
+            for (const neighbor of neighbors) {
+                if (seenEntities.has(neighbor.id))
+                    continue;
+                seenEntities.add(neighbor.id);
+                // Find the path from query entity to this neighbor
+                const paths = await findPaths(entity.id, neighbor.id, {
+                    maxHops,
+                    minWeight: minPathWeight,
+                    maxPaths: 1,
+                });
+                if (paths.length > 0) {
+                    const path = paths[0];
+                    // Score based on path length and weight
+                    const relevanceScore = 1 / (1 + path.hopCount) * (1 / (1 + path.totalWeight * 0.1));
+                    results.push({
+                        entityName: neighbor.name,
+                        entityId: neighbor.id,
+                        path,
+                        relevanceScore,
+                    });
+                }
+            }
+        }
+    }
+    // Sort by relevance score
+    results.sort((a, b) => b.relevanceScore - a.relevanceScore);
+    return results.slice(0, limit);
+}
+// ─── Result Ranking ──────────────────────────────────────────────────────────
+/**
+ * Rank multi-hop results combining vector score with graph connectivity.
+ */
+function rankMultiHopResults(results, queryEntityNames) {
+    return results
+        .map(result => {
+        let score = result.similarity || 0;
+        // Boost for graph-connected results
+        if (result.retrievalPath === 'graph' || result.retrievalPath === 'both') {
+            score *= 1.2; // 20% boost for graph connectivity
+        }
+        // Boost for matching query entities in content
+        const contentLower = (result.content || '').toLowerCase();
+        for (const entity of queryEntityNames) {
+            if (contentLower.includes(entity.toLowerCase())) {
+                score *= 1.1; // 10% boost per entity match
+            }
+        }
+        // Boost for shorter graph paths (more direct connections)
+        if (result.graphPath) {
+            const pathBonus = 1 / (1 + result.graphPath.hopCount * 0.2);
+            score *= (1 + pathBonus * 0.15); // Up to 15% bonus for short paths
+        }
+        return {
+            ...result,
+            similarity: Math.round(score * 1000) / 1000,
+            hybridScore: Math.round(score * 100) / 100, // Backward compat
+        };
+    })
+        .sort((a, b) => (b.similarity || 0) - (a.similarity || 0));
+}
+// ─── Utility ────────────────────────────────────────────────────────────────
+/** Check if query needs multi-hop graph traversal. */
+export function needsMultiHop(query) {
+    // Queries with relationship indicators benefit from multi-hop
+    const relationshipPatterns = [
+        /\b(affected by|caused by|related to|connected to|depends? on|works? on|uses?|manages?)\b/i,
+        /\b(who|what|which|how)\b.*\b(project|team|system|service|database)\b/i,
+        /\b(impact|affect|influence|caus\w+|result|lead to)\b/i,
+    ];
+    return relationshipPatterns.some(pattern => pattern.test(query));
+}
+/**
+ * Get a human-readable explanation of how a multi-hop result was found.
+ */
+export function explainRetrievalPath(result) {
+    if (result.retrievalPath === 'vector') {
+        return `Found via semantic search (score: ${result.similarity?.toFixed(2)})`;
+    }
+    if (result.retrievalPath === 'graph' && result.graphPath) {
+        const pathSteps = result.graphPath.nodes
+            .map(n => n.name)
+            .join(' → ');
+        return `Found via knowledge graph: ${pathSteps} (hops: ${result.graphPath.hopCount})`;
+    }
+    if (result.retrievalPath === 'both' && result.graphPath) {
+        const pathSteps = result.graphPath.nodes
+            .map(n => n.name)
+            .join(' → ');
+        return `Found via both search and graph: ${pathSteps} (score: ${result.similarity?.toFixed(2)})`;
+    }
+    return `Found via ${result.retrievalPath} search`;
+}
+// Backward compatibility alias
+export const wouldBenefitFromMultiHop = needsMultiHop;
+//# sourceMappingURL=multi-hop-retrieval.js.map

package/dist/core/graph/relationship-extractor.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Relationship Extractor
+ *
+ * Populates entity_relations table with typed relationships extracted
+ * from memory content.
+ */
+import { type RelationType } from './llm-entity-extractor.js';
+export interface StoredRelation {
+    id: string;
+    fromEntityId: string;
+    toEntityId: string;
+    fromEntityName: string;
+    toEntityName: string;
+    relationType: RelationType;
+    weight: number;
+    properties: Record<string, unknown> | null;
+}
+/**
+ * Extract entities and relations from memory content and store them in the database.
+ * Returns the stored entities and relations.
+ */
+export declare function extractAndStoreRelations(memoryId: string, content: string, projectId: string, options?: {
+    preferLLM?: boolean;
+    maxContentLength?: number;
+}): Promise<{
+    entities: number;
+    relations: number;
+    source: 'llm' | 'regex' | 'fallback';
+}>;
+/**
+ * Get all relations for a set of entity IDs.
+ * Used by graph traversal to find connected entities.
+ */
+export declare function getEntityRelations(entityIds: string[], relationTypes?: RelationType[]): Promise<StoredRelation[]>;
+/**
+ * Get all entities for a project.
+ */
+export declare function getProjectEntities(projectId: string, limit?: number): Promise<Array<{
+    id: string;
+    name: string;
+    type: string;
+    description: string | null;
+}>>;
+/**
+ * Delete all entities and relations for a project (for graph rebuild).
+ */
+export declare function clearProjectGraph(projectId: string): Promise<void>;
+//# sourceMappingURL=relationship-extractor.d.ts.map