squish-memory 1.2.0 → 1.2.1

package/dist/core/graph/llm-entity-extractor.js

@@ -1,313 +0,0 @@
-/**
- * LLM Entity Extractor
- *
- * Extracts named entities and relationships from memory content using
- * LLM-powered analysis. Falls back to regex extraction when LLM is unavailable.
- */
-import { logger } from '../logger.js';
-import { config } from '../../config.js';
-import { extractEntities as regexExtractEntities, } from '../memory/entity-extractor.js';
-// ─── LLM Prompt ─────────────────────────────────────────────────────────────
-const ENTITY_EXTRACTION_PROMPT = `You are an entity and relationship extractor for an AI memory system. Given text, extract:
-
-1. ENTITIES: Named things mentioned (people, projects, systems, tools, concepts, events)
-2. RELATIONSHIPS: How entities connect to each other
-
-Return ONLY valid JSON in this exact format:
-{
-  "entities": [
-    {"name": "Alice", "type": "person", "confidence": 0.95},
-    {"name": "Project Atlas", "type": "concept", "confidence": 0.9}
-  ],
-  "relations": [
-    {"from": "Alice", "to": "Project Atlas", "type": "works_on", "confidence": 0.85, "context": "Alice is the tech lead on Project Atlas"},
-    {"from": "Project Atlas", "to": "PostgreSQL", "type": "uses", "confidence": 0.9, "context": "Project Atlas uses PostgreSQL for its primary datastore"}
-  ]
-}
-
-Entity types: person, file, function, class, concept, tool, date, location, pattern, technique, other
-Relation types: works_on, depends_on, manages, uses, caused, located_in, belongs_to, reports_to, occurred_on, affects, contains, implements, extends, related_to, part_of, owns, created, resolved, blocks
-
-Rules:
-- Extract only explicitly mentioned entities and relationships
-- Use the most specific relation type available
-- Include confidence scores (0-1)
-- Keep entity names as they appear in text (preserve capitalization)
-- For ambiguous relationships, use "related_to"
-- Do NOT invent entities or relationships not present in the text`;
-// ─── LLM Call Abstraction ───────────────────────────────────────────────────
-/**
- * Call an LLM for entity extraction.
- * Uses OpenAI-compatible API (works with OpenAI, Ollama, LM Studio, local endpoints).
- */
-async function callLLM(prompt, content) {
-    // Determine which provider to use for extraction
-    const provider = config.llmProvider;
-    const endpoint = config.llmEndpoint || config.lmStudioUrl || config.ollamaUrl;
-    const model = config.llmExtractionModel;
-    if (!model) {
-        logger.debug('LLM extraction model not configured; using regex fallback');
-        return null;
-    }
-    // Try OpenAI first if configured and provider matches
-    if ((provider === 'openai' || !provider) && config.openAiApiKey) {
-        try {
-            const chatUrl = config.openAiApiUrl.replace('/embeddings', '/chat/completions');
-            const response = await fetch(chatUrl, {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
-                    Authorization: `Bearer ${config.openAiApiKey}`,
-                },
-                body: JSON.stringify({
-                    model,
-                    messages: [
-                        { role: 'system', content: prompt },
-                        { role: 'user', content },
-                    ],
-                    temperature: 0.1,
-                    max_tokens: 2000,
-                }),
-            });
-            if (!response.ok) {
-                logger.warn(`LLM entity extraction failed (OpenAI): ${response.status}`);
-                return null;
-            }
-            const payload = await response.json();
-            return payload.choices?.[0]?.message?.content ?? null;
-        }
-        catch (error) {
-            logger.warn('LLM entity extraction error (OpenAI):', { error: error });
-        }
-    }
-    // Try Ollama if configured and provider matches
-    if ((provider === 'ollama' || !provider) && config.ollamaUrl) {
-        try {
-            const response = await fetch(`${config.ollamaUrl}/api/chat`, {
-                method: 'POST',
-                headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify({
-                    model,
-                    messages: [
-                        { role: 'system', content: prompt },
-                        { role: 'user', content },
-                    ],
-                    stream: false,
-                    options: { temperature: 0.1 },
-                }),
-            });
-            if (!response.ok) {
-                logger.warn(`Ollama entity extraction failed: ${response.status}`);
-                return null;
-            }
-            const payload = await response.json();
-            return payload.message?.content ?? null;
-        }
-        catch (error) {
-            logger.warn('LLM entity extraction error (Ollama):', { error: error });
-        }
-    }
-    // Try LM Studio / custom endpoint if configured and provider matches
-    if ((provider === 'lmstudio' || provider === 'local' || !provider) && (config.lmStudioUrl || endpoint)) {
-        try {
-            const baseUrl = endpoint || config.lmStudioUrl;
-            if (!baseUrl)
-                return null;
-            const response = await fetch(`${baseUrl}/v1/chat/completions`, {
-                method: 'POST',
-                headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify({
-                    model,
-                    messages: [
-                        { role: 'system', content: prompt },
-                        { role: 'user', content },
-                    ],
-                    temperature: 0.1,
-                    max_tokens: 2000,
-                }),
-            });
-            if (!response.ok) {
-                logger.warn(`LM Studio entity extraction failed: ${response.status}`);
-                return null;
-            }
-            const payload = await response.json();
-            return payload.choices?.[0]?.message?.content ?? null;
-        }
-        catch (error) {
-            logger.warn('LLM entity extraction error (LM Studio):', { error: error });
-        }
-    }
-    return null;
-}
-// ─── JSON Parsing ────────────────────────────────────────────────────────────
-/**
- * Parse LLM response into structured extraction result.
- * Handles various response formats (raw JSON, markdown code blocks, etc.)
- */
-function parseLLMResponse(response) {
-    // Try to extract JSON from the response
-    let jsonStr = response.trim();
-    // Handle markdown code blocks
-    const codeBlockMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
-    if (codeBlockMatch) {
-        jsonStr = codeBlockMatch[1].trim();
-    }
-    // Try to find JSON object in the response
-    const jsonMatch = jsonStr.match(/\{[\s\S]*\}/);
-    if (!jsonMatch) {
-        logger.debug('No JSON found in LLM response');
-        return null;
-    }
-    try {
-        const parsed = JSON.parse(jsonMatch[0]);
-        // Validate structure
-        if (!parsed.entities || !Array.isArray(parsed.entities)) {
-            logger.debug('LLM response missing entities array');
-            return null;
-        }
-        const entities = parsed.entities
-            .filter((e) => e.name && e.type)
-            .map((e) => ({
-            name: String(e.name),
-            type: validateEntityType(String(e.type)),
-            confidence: typeof e.confidence === 'number' ? Math.min(1, Math.max(0, e.confidence)) : 0.8,
-            startIndex: 0,
-            endIndex: 0,
-            context: String(e.context || ''),
-            normalized: e.normalized ? String(e.normalized) : undefined,
-        }));
-        const relations = (parsed.relations || [])
-            .filter((r) => r.from && r.to && r.type)
-            .map((r) => ({
-            fromEntity: String(r.from),
-            toEntity: String(r.to),
-            relationType: validateRelationType(String(r.type)),
-            confidence: typeof r.confidence === 'number' ? Math.min(1, Math.max(0, r.confidence)) : 0.7,
-            context: String(r.context || ''),
-        }));
-        return { entities, relations };
-    }
-    catch (error) {
-        logger.debug('Failed to parse LLM response as JSON', { error: error });
-        return null;
-    }
-}
-function validateEntityType(type) {
-    const validTypes = ['person', 'file', 'function', 'class', 'concept', 'tool', 'date', 'location', 'pattern', 'technique', 'other'];
-    const lower = type.toLowerCase();
-    if (validTypes.includes(lower))
-        return lower;
-    return 'other';
-}
-function validateRelationType(type) {
-    const validTypes = [
-        'works_on', 'depends_on', 'manages', 'uses', 'caused', 'located_in',
-        'belongs_to', 'reports_to', 'occurred_on', 'affects', 'contains',
-        'implements', 'extends', 'related_to', 'part_of', 'owns', 'created',
-        'resolved', 'blocks',
-    ];
-    const lower = type.toLowerCase();
-    if (validTypes.includes(lower))
-        return lower;
-    return 'related_to';
-}
-// ─── Main Extraction Functions ───────────────────────────────────────────────
-/**
- * Extract entities and relationships from text using LLM.
- * Falls back to regex extraction when LLM is unavailable.
- */
-export async function extractEntitiesAndRelations(content, options) {
-    const { preferLLM = true, maxContentLength = 4000 } = options || {};
-    // Truncate very long content
-    const truncatedContent = content.length > maxContentLength
-        ? content.substring(0, maxContentLength) + '...'
-        : content;
-    // Try LLM extraction first if preferred
-    if (preferLLM) {
-        const llmResult = await extractWithLLM(truncatedContent);
-        if (llmResult) {
-            logger.debug('LLM entity extraction succeeded', {
-                entityCount: llmResult.entities.length,
-                relationCount: llmResult.relations.length,
-            });
-            return { ...llmResult, source: 'llm' };
-        }
-    }
-    // Fall back to regex extraction
-    const regexEntities = await regexExtractEntities(content);
-    if (regexEntities.length > 0) {
-        logger.debug('Regex entity extraction fallback', {
-            entityCount: regexEntities.length,
-        });
-        return {
-            entities: regexEntities,
-            relations: [], // Regex extraction doesn't produce relations
-            source: 'regex',
-        };
-    }
-    return { entities: [], relations: [], source: 'none' };
-}
-/**
- * Extract entities and relations using LLM.
- */
-async function extractWithLLM(content) {
-    const response = await callLLM(ENTITY_EXTRACTION_PROMPT, content);
-    if (!response)
-        return null;
-    const parsed = parseLLMResponse(response);
-    if (!parsed)
-        return null;
-    // Validate that extracted entities actually appear in the content
-    const validatedEntities = parsed.entities.filter((entity) => {
-        // Allow entities whose name or a significant substring appears in content
-        const nameLower = entity.name.toLowerCase();
-        const contentLower = content.toLowerCase();
-        // Check if the entity name or a key part of it appears in content
-        const words = nameLower.split(/\s+/);
-        return words.some(word => word.length > 2 && contentLower.includes(word));
-    });
-    // Validate that relation entities exist in the entity list
-    const entityNames = new Set(validatedEntities.map(e => e.name.toLowerCase()));
-    const validatedRelations = parsed.relations.filter((relation) => {
-        const fromExists = entityNames.has(relation.fromEntity.toLowerCase());
-        const toExists = entityNames.has(relation.toEntity.toLowerCase());
-        // Allow relations where at least one entity is known
-        // (the LLM might use slightly different names)
-        return fromExists || toExists ||
-            // Also allow if both entity names appear in the content
-            (content.toLowerCase().includes(relation.fromEntity.toLowerCase()) &&
-                content.toLowerCase().includes(relation.toEntity.toLowerCase()));
-    });
-    return {
-        entities: validatedEntities,
-        relations: validatedRelations,
-    };
-}
-/**
- * Extract entities and relations from multiple memories in batch.
- * More efficient than calling extractEntitiesAndRelations for each memory.
- */
-export async function batchExtractEntitiesAndRelations(contents, options) {
-    const { batchSize = 5 } = options || {};
-    const results = [];
-    // Process in batches to avoid overwhelming the LLM
-    for (let i = 0; i < contents.length; i += batchSize) {
-        const batch = contents.slice(i, i + batchSize);
-        const batchResults = await Promise.all(batch.map(content => extractEntitiesAndRelations(content, options)));
-        results.push(...batchResults);
-    }
-    return results;
-}
-/**
- * Get the extraction prompt for testing/debugging purposes.
- */
-export function getExtractionPrompt() {
-    return ENTITY_EXTRACTION_PROMPT;
-}
-/**
- * Parse an LLM response for testing/debugging purposes.
- */
-export function testParseLLMResponse(response) {
-    return parseLLMResponse(response);
-}
-//# sourceMappingURL=llm-entity-extractor.js.map

package/dist/core/graph/multi-hop-retrieval.d.ts

@@ -1,48 +0,0 @@
-/**
- * Multi-Hop Retrieval
- *
- * Combines vector search with graph traversal to answer queries that
- * require following relationships across entities.
- */
-import { type SearchResult } from '../memory/hybrid-search.js';
-import { type TraversalPath } from './graph-traversal.js';
-export interface MultiHopResult extends SearchResult {
-    /** How this result was found: 'vector' (direct search) or 'graph' (via traversal) */
-    retrievalPath: 'vector' | 'graph' | 'both';
-    /** The graph path that led to this result (if found via graph) */
-    graphPath?: TraversalPath;
-    /** Entities that connected this result to the query */
-    connectingEntities?: string[];
-    /** Hybrid score for backward compatibility */
-    hybridScore?: number;
-}
-export interface MultiHopSearchOptions {
-    /** Search query */
-    query: string;
-    /** Project path for scoping */
-    project?: string;
-    /** Maximum number of results */
-    limit?: number;
-    /** Maximum graph traversal depth */
-    maxHops?: number;
-    /** Whether to include vector-only results */
-    includeVectorResults?: boolean;
-    /** Whether to include graph-expanded results */
-    includeGraphResults?: boolean;
-    /** Minimum graph path weight to include */
-    minPathWeight?: number;
-    /** Session ID for context */
-    sessionId?: string;
-}
-/**
- * Perform multi-hop search combining vector search with graph traversal.
- */
-export declare function multiHopSearch(options: MultiHopSearchOptions): Promise<MultiHopResult[]>;
-/** Check if query needs multi-hop graph traversal. */
-export declare function needsMultiHop(query: string): boolean;
-/**
- * Get a human-readable explanation of how a multi-hop result was found.
- */
-export declare function explainRetrievalPath(result: MultiHopResult): string;
-export declare const wouldBenefitFromMultiHop: typeof needsMultiHop;
-//# sourceMappingURL=multi-hop-retrieval.d.ts.map

package/dist/core/graph/multi-hop-retrieval.js

@@ -1,215 +0,0 @@
-/**
- * Multi-Hop Retrieval
- *
- * Combines vector search with graph traversal to answer queries that
- * require following relationships across entities.
- */
-import { eq } from 'drizzle-orm';
-import { getDb } from '../../db/index.js';
-import { getSchema } from '../../db/schema.js';
-import { hybridSearch } from '../memory/hybrid-search.js';
-import { extractEntitiesAndRelations } from './llm-entity-extractor.js';
-import { extractEntityNames } from '../memory/entity-extractor.js';
-import { findEntitiesByName, findPaths, traverse } from './graph-traversal.js';
-import { logger } from '../logger.js';
-import { config } from '../../config.js';
-// ─── Main Multi-Hop Search ──────────────────────────────────────────────────
-/**
- * Perform multi-hop search combining vector search with graph traversal.
- */
-export async function multiHopSearch(options) {
-    const { query, project, limit = 10, maxHops = 3, includeVectorResults = true, includeGraphResults = true, minPathWeight = 1, sessionId, } = options;
-    const results = [];
-    const seenMemoryIds = new Set();
-    // Step 1: Extract query entities (respect global LLM config)
-    let queryEntityNames;
-    try {
-        const extraction = await extractEntitiesAndRelations(query, { preferLLM: config.llmEnabled });
-        queryEntityNames = extraction.entities.map(e => e.name);
-        logger.debug('Multi-hop search: entity extraction succeeded', {
-            query: query.substring(0, 100),
-            entities: queryEntityNames,
-            source: extraction.source,
-        });
-    }
-    catch (error) {
-        // LLM failed or disabled - fall back to simple keyword extraction
-        logger.debug('Multi-hop search: entity extraction failed, falling back to keywords', { error });
-        const simpleEntities = extractEntityNames(query);
-        queryEntityNames = simpleEntities;
-        logger.debug('Multi-hop search: keyword extraction', {
-            entities: queryEntityNames,
-        });
-    }
-    // Step 2: Vector search (baseline results)
-    if (includeVectorResults) {
-        const vectorResults = await hybridSearch({
-            query,
-            project,
-            limit: limit * 2, // Get more candidates for reranking
-        });
-        for (const result of vectorResults) {
-            if (!seenMemoryIds.has(result.id)) {
-                seenMemoryIds.add(result.id);
-                results.push({
-                    ...result,
-                    retrievalPath: 'vector',
-                });
-            }
-        }
-    }
-    // Step 3: Graph expansion (if we have entities)
-    if (includeGraphResults && queryEntityNames.length > 0 && project) {
-        const graphResults = await expandViaGraph(queryEntityNames, project, maxHops, minPathWeight, limit);
-        // Find memories connected to graph-expanded entities
-        for (const graphResult of graphResults) {
-            // Search for memories mentioning the expanded entities
-            const expandedMemories = await hybridSearch({
-                query: graphResult.entityName,
-                project,
-                limit: 5,
-            });
-            for (const memory of expandedMemories) {
-                if (!seenMemoryIds.has(memory.id)) {
-                    seenMemoryIds.add(memory.id);
-                    results.push({
-                        ...memory,
-                        retrievalPath: 'graph',
-                        graphPath: graphResult.path,
-                        connectingEntities: [graphResult.entityName],
-                    });
-                }
-            }
-        }
-    }
-    // Step 4: Rank combined results
-    // Vector results keep their hybrid score, graph results get a boost
-    // for being connected to query entities via the knowledge graph
-    const ranked = rankMultiHopResults(results, queryEntityNames);
-    return ranked.slice(0, limit);
-}
-/**
- * Expand query entities via the knowledge graph to find related entities.
- */
-async function expandViaGraph(queryEntityNames, project, maxHops, minPathWeight, limit) {
-    const db = await getDb();
-    const schema = await getSchema();
-    const results = [];
-    const seenEntities = new Set();
-    // Get project ID
-    const projectRows = await db
-        .select({ id: schema.projects.id })
-        .from(schema.projects)
-        .where(eq(schema.projects.path, project))
-        .limit(1);
-    if (projectRows.length === 0)
-        return [];
-    const projectId = projectRows[0].id;
-    // For each query entity, find it in the graph and expand
-    for (const entityName of queryEntityNames) {
-        const entities = await findEntitiesByName(entityName, projectId, { limit: 3 });
-        for (const entity of entities) {
-            if (seenEntities.has(entity.id))
-                continue;
-            seenEntities.add(entity.id);
-            // Traverse the graph from this entity
-            const neighbors = await traverse(entity.id, {
-                maxDepth: maxHops,
-                minWeight: minPathWeight,
-                limit: 20,
-            });
-            for (const neighbor of neighbors) {
-                if (seenEntities.has(neighbor.id))
-                    continue;
-                seenEntities.add(neighbor.id);
-                // Find the path from query entity to this neighbor
-                const paths = await findPaths(entity.id, neighbor.id, {
-                    maxHops,
-                    minWeight: minPathWeight,
-                    maxPaths: 1,
-                });
-                if (paths.length > 0) {
-                    const path = paths[0];
-                    // Score based on path length and weight
-                    const relevanceScore = 1 / (1 + path.hopCount) * (1 / (1 + path.totalWeight * 0.1));
-                    results.push({
-                        entityName: neighbor.name,
-                        entityId: neighbor.id,
-                        path,
-                        relevanceScore,
-                    });
-                }
-            }
-        }
-    }
-    // Sort by relevance score
-    results.sort((a, b) => b.relevanceScore - a.relevanceScore);
-    return results.slice(0, limit);
-}
-// ─── Result Ranking ──────────────────────────────────────────────────────────
-/**
- * Rank multi-hop results combining vector score with graph connectivity.
- */
-function rankMultiHopResults(results, queryEntityNames) {
-    return results
-        .map(result => {
-        let score = result.similarity || 0;
-        // Boost for graph-connected results
-        if (result.retrievalPath === 'graph' || result.retrievalPath === 'both') {
-            score *= 1.2; // 20% boost for graph connectivity
-        }
-        // Boost for matching query entities in content
-        const contentLower = (result.content || '').toLowerCase();
-        for (const entity of queryEntityNames) {
-            if (contentLower.includes(entity.toLowerCase())) {
-                score *= 1.1; // 10% boost per entity match
-            }
-        }
-        // Boost for shorter graph paths (more direct connections)
-        if (result.graphPath) {
-            const pathBonus = 1 / (1 + result.graphPath.hopCount * 0.2);
-            score *= (1 + pathBonus * 0.15); // Up to 15% bonus for short paths
-        }
-        return {
-            ...result,
-            similarity: Math.round(score * 1000) / 1000,
-            hybridScore: Math.round(score * 100) / 100, // Backward compat
-        };
-    })
-        .sort((a, b) => (b.similarity || 0) - (a.similarity || 0));
-}
-// ─── Utility ────────────────────────────────────────────────────────────────
-/** Check if query needs multi-hop graph traversal. */
-export function needsMultiHop(query) {
-    // Queries with relationship indicators benefit from multi-hop
-    const relationshipPatterns = [
-        /\b(affected by|caused by|related to|connected to|depends? on|works? on|uses?|manages?)\b/i,
-        /\b(who|what|which|how)\b.*\b(project|team|system|service|database)\b/i,
-        /\b(impact|affect|influence|caus\w+|result|lead to)\b/i,
-    ];
-    return relationshipPatterns.some(pattern => pattern.test(query));
-}
-/**
- * Get a human-readable explanation of how a multi-hop result was found.
- */
-export function explainRetrievalPath(result) {
-    if (result.retrievalPath === 'vector') {
-        return `Found via semantic search (score: ${result.similarity?.toFixed(2)})`;
-    }
-    if (result.retrievalPath === 'graph' && result.graphPath) {
-        const pathSteps = result.graphPath.nodes
-            .map(n => n.name)
-            .join(' → ');
-        return `Found via knowledge graph: ${pathSteps} (hops: ${result.graphPath.hopCount})`;
-    }
-    if (result.retrievalPath === 'both' && result.graphPath) {
-        const pathSteps = result.graphPath.nodes
-            .map(n => n.name)
-            .join(' → ');
-        return `Found via both search and graph: ${pathSteps} (score: ${result.similarity?.toFixed(2)})`;
-    }
-    return `Found via ${result.retrievalPath} search`;
-}
-// Backward compatibility alias
-export const wouldBenefitFromMultiHop = needsMultiHop;
-//# sourceMappingURL=multi-hop-retrieval.js.map

package/dist/core/graph/relationship-extractor.d.ts

@@ -1,48 +0,0 @@
-/**
- * Relationship Extractor
- *
- * Populates entity_relations table with typed relationships extracted
- * from memory content.
- */
-import { type RelationType } from './llm-entity-extractor.js';
-export interface StoredRelation {
-    id: string;
-    fromEntityId: string;
-    toEntityId: string;
-    fromEntityName: string;
-    toEntityName: string;
-    relationType: RelationType;
-    weight: number;
-    properties: Record<string, unknown> | null;
-}
-/**
- * Extract entities and relations from memory content and store them in the database.
- * Returns the stored entities and relations.
- */
-export declare function extractAndStoreRelations(memoryId: string, content: string, projectId: string, options?: {
-    preferLLM?: boolean;
-    maxContentLength?: number;
-}): Promise<{
-    entities: number;
-    relations: number;
-    source: 'llm' | 'regex' | 'fallback';
-}>;
-/**
- * Get all relations for a set of entity IDs.
- * Used by graph traversal to find connected entities.
- */
-export declare function getEntityRelations(entityIds: string[], relationTypes?: RelationType[]): Promise<StoredRelation[]>;
-/**
- * Get all entities for a project.
- */
-export declare function getProjectEntities(projectId: string, limit?: number): Promise<Array<{
-    id: string;
-    name: string;
-    type: string;
-    description: string | null;
-}>>;
-/**
- * Delete all entities and relations for a project (for graph rebuild).
- */
-export declare function clearProjectGraph(projectId: string): Promise<void>;
-//# sourceMappingURL=relationship-extractor.d.ts.map