rag-memory-epf-mcp 1.8.0 → 2.0.0

package/dist/index.js

@@ -7,16 +7,11 @@ import * as sqliteVec from 'sqlite-vec';
 import { get_encoding } from 'tiktoken';
 import path from 'path';
 import { fileURLToPath } from 'url';
-import { pipeline, env } from '@huggingface/transformers';
 // Import our new structured tool system
 import { getAllMCPTools, validateToolArgs, getSystemInfo } from './src/tools/tool-registry.js';
 // Import migration system
 import { MigrationManager } from './src/migrations/migration-manager.js';
 import { migrations } from './src/migrations/migrations.js';
-// Configure Hugging Face transformers for better compatibility
-if (env.backends?.onnx?.wasm) {
-    env.backends.onnx.wasm.wasmPaths = './node_modules/@huggingface/transformers/dist/';
-}
 // Define database file path using environment variable with fallback
 const defaultDbPath = path.join(path.dirname(fileURLToPath(import.meta.url)), 'rag-memory.db');
 const DB_FILE_PATH = process.env.DB_FILE_PATH
@@ -24,6 +19,9 @@ const DB_FILE_PATH = process.env.DB_FILE_PATH
         ? process.env.DB_FILE_PATH
         : path.join(path.dirname(fileURLToPath(import.meta.url)), process.env.DB_FILE_PATH)
     : defaultDbPath;
+const EMBEDDING_MODEL = process.env.EMBEDDING_MODEL || 'qwen3-embedding:8b';
+const OLLAMA_URL = process.env.OLLAMA_URL || 'http://localhost:11434';
+const EMBEDDING_DIM = 4096;
 // Safe rowid for vec0 virtual tables (require literal integer, not parameterized)
 function safeRowid(value) {
     const n = Number(value);
@@ -36,8 +34,9 @@ function safeRowid(value) {
 class RAGKnowledgeGraphManager {
     db = null;
     encoding = null;
-    embeddingModel = null;
     modelInitialized = false;
+    embeddingCache = new Map();
+    EMBEDDING_CACHE_MAX = 500;
     async initialize() {
         console.error('🚀 Initializing RAG Knowledge Graph MCP Server...');
         // Initialize database
@@ -65,20 +64,24 @@ class RAGKnowledgeGraphManager {
     }
     async initializeEmbeddingModel() {
         try {
-            console.error('🤖 Loading embedding model: Qwen3-Embedding-0.6B (1024-dim, 100+ languages)...');
-            // Configure environment to allow remote model downloads
-            env.allowRemoteModels = true;
-            env.allowLocalModels = true;
-            this.embeddingModel = await pipeline('feature-extraction', 'onnx-community/Qwen3-Embedding-0.6B-ONNX', {
-                revision: 'main',
-                dtype: 'fp16',
-            });
+            console.error(`🤖 Connecting to Ollama: ${OLLAMA_URL} (model: ${EMBEDDING_MODEL}, ${EMBEDDING_DIM}-dim)...`);
+            const response = await fetch(`${OLLAMA_URL}/api/tags`);
+            if (!response.ok) {
+                throw new Error(`Ollama not responding: ${response.status}`);
+            }
+            const data = await response.json();
+            const models = data.models || [];
+            const modelFound = models.some((m) => m.name.startsWith(EMBEDDING_MODEL.split(':')[0]));
+            if (!modelFound) {
+                console.error(`⚠️ Model ${EMBEDDING_MODEL} not found in Ollama. Run: ollama pull ${EMBEDDING_MODEL}`);
+                console.error(`📋 Available models: ${models.map((m) => m.name).join(', ') || 'none'}`);
+            }
             this.modelInitialized = true;
-            console.error('✅ Qwen3-Embedding-0.6B model loaded successfully');
+            console.error(`✅ Ollama connected (${models.length} models available)`);
         }
         catch (error) {
-            console.error('❌ Failed to load embedding model:', error);
-            console.error('📋 Falling back to simple embedding generation');
+            console.error('❌ Failed to connect to Ollama:', error instanceof Error ? error.message : error);
+            console.error('📋 Falling back to simple embedding generation. Start Ollama: ollama serve');
             this.modelInitialized = false;
         }
     }
@@ -111,11 +114,8 @@ class RAGKnowledgeGraphManager {
             this.encoding.free();
             this.encoding = null;
         }
-        if (this.embeddingModel) {
-            // Clean up the embedding model if it has cleanup methods
-            this.embeddingModel = null;
-            this.modelInitialized = false;
-        }
+        this.modelInitialized = false;
+        this.embeddingCache.clear();
         if (this.db) {
             this.db.close();
             this.db = null;
@@ -385,12 +385,121 @@ class RAGKnowledgeGraphManager {
         }));
         return { entities, relations };
     }
+    async getNeighbors(entityNames, depth = 1, relationType) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        // Cap depth at 5 to prevent runaway queries
+        const effectiveDepth = Math.min(Math.max(depth, 1), 5);
+        // Convert entity names to IDs
+        const seedIds = entityNames.map(name => `entity_${name.toLowerCase().replace(/[^a-z0-9]/g, '_')}`);
+        // Build dynamic placeholders for the seed IDs
+        const seedPlaceholders = seedIds.map(() => '?').join(',');
+        // Build the recursive CTE query
+        const relationFilter = relationType
+            ? `AND r.relationType = ?`
+            : '';
+        const cteQuery = `
+      WITH RECURSIVE traversal(entity_id, depth, path) AS (
+        -- Base case: seed entities
+        SELECT id, 0, id FROM entities WHERE id IN (${seedPlaceholders})
+        UNION ALL
+        -- Recursive: follow relationships up to max depth
+        SELECT
+          CASE WHEN r.source_entity = t.entity_id THEN r.target_entity ELSE r.source_entity END,
+          t.depth + 1,
+          t.path || ',' || CASE WHEN r.source_entity = t.entity_id THEN r.target_entity ELSE r.source_entity END
+        FROM traversal t
+        JOIN relationships r ON (r.source_entity = t.entity_id OR r.target_entity = t.entity_id)
+        WHERE t.depth < ?
+          ${relationFilter}
+          -- Cycle detection: don't revisit entities already in path
+          AND instr(t.path, CASE WHEN r.source_entity = t.entity_id THEN r.target_entity ELSE r.source_entity END) = 0
+      )
+      SELECT DISTINCT entity_id, MIN(depth) as min_depth, path
+      FROM traversal
+      GROUP BY entity_id
+    `;
+        // Build parameters
+        const params = [...seedIds, effectiveDepth];
+        if (relationType) {
+            params.push(relationType);
+        }
+        const traversalResults = this.db.prepare(cteQuery).all(...params);
+        if (traversalResults.length === 0) {
+            return { entities: [], relations: [], paths: [] };
+        }
+        // Collect all discovered entity IDs
+        const discoveredIds = traversalResults.map(r => r.entity_id);
+        const idPlaceholders = discoveredIds.map(() => '?').join(',');
+        // Fetch entity details
+        const entityRows = this.db.prepare(`
+      SELECT id, name, entityType, observations FROM entities WHERE id IN (${idPlaceholders})
+    `).all(...discoveredIds);
+        // Build id-to-depth and id-to-name maps
+        const idToDepth = new Map();
+        for (const r of traversalResults) {
+            idToDepth.set(r.entity_id, r.min_depth);
+        }
+        const idToName = new Map();
+        for (const row of entityRows) {
+            idToName.set(row.id, row.name);
+        }
+        const entities = entityRows.map(row => ({
+            name: row.name,
+            entityType: row.entityType,
+            observations: JSON.parse(row.observations),
+            depth: idToDepth.get(row.id) ?? 0,
+        }));
+        // Fetch relations between all discovered entities
+        let relQuery = `
+      SELECT
+        r.source_entity,
+        r.target_entity,
+        e1.name as from_name,
+        e2.name as to_name,
+        r.relationType
+      FROM relationships r
+      JOIN entities e1 ON r.source_entity = e1.id
+      JOIN entities e2 ON r.target_entity = e2.id
+      WHERE r.source_entity IN (${idPlaceholders})
+        AND r.target_entity IN (${idPlaceholders})
+    `;
+        const relParams = [...discoveredIds, ...discoveredIds];
+        if (relationType) {
+            relQuery += ` AND r.relationType = ?`;
+            relParams.push(relationType);
+        }
+        const relationRows = this.db.prepare(relQuery).all(...relParams);
+        const relations = relationRows.map(row => ({
+            from: row.from_name,
+            to: row.to_name,
+            relationType: row.relationType,
+            depth: Math.max(idToDepth.get(row.source_entity) ?? 0, idToDepth.get(row.target_entity) ?? 0),
+        }));
+        // Build shortest paths from seed entities to all discovered entities
+        const paths = [];
+        for (const result of traversalResults) {
+            if (result.min_depth === 0)
+                continue; // Skip seed entities themselves
+            const pathIds = result.path.split(',');
+            const pathNames = pathIds.map(id => idToName.get(id) || id).filter(Boolean);
+            if (pathNames.length >= 2) {
+                paths.push({
+                    from: pathNames[0],
+                    to: pathNames[pathNames.length - 1],
+                    path: pathNames,
+                });
+            }
+        }
+        console.error(`✅ getNeighbors: Found ${entities.length} entities, ${relations.length} relations, ${paths.length} paths (depth=${effectiveDepth})`);
+        return { entities, relations, paths };
+    }
     async searchNodes(query, limit = 10, since, until) {
         if (!this.db)
             throw new Error('Database not initialized');
         console.error(`🔍 Semantic entity search: "${query}"`);
-        // Generate query embedding (with instruction prefix for Qwen3)
-        const queryEmbedding = await this.generateEmbedding(query, 1024, true);
+        // Generate query embedding
+        const queryEmbedding = await this.generateEmbedding(query);
         // Perform vector similarity search on entities
         const entityResults = this.db.prepare(`
       SELECT 
@@ -809,55 +918,6 @@ class RAGKnowledgeGraphManager {
             console.error(`  └─ Deleted ${metadata.changes} chunk metadata records`);
         }
     }
-    // Simple configurable term extraction (replacing hardcoded patterns)
-    // Cross-lingual query translation using entity DB + domain dictionary
-    translateQueryCrossLingual(query) {
-        if (!this.db)
-            return null;
-        // Domain-specific Korean→English dictionary
-        const domainDict = {
-            '할랄': 'halal', '인증': 'certification', '상호인정': 'mutual recognition',
-            '인정': 'accreditation', '인증기관': 'certification body',
-            '표준': 'standard', '감사': 'audit', '심사': 'audit review',
-            '도축': 'slaughter', '식품': 'food', '화장품': 'cosmetics',
-            '수출': 'export', '수입': 'import', '무역': 'trade',
-            '방문': 'visit', '협력': 'cooperation', '협약': 'agreement',
-            '제안': 'proposal', '회의': 'meeting', '이메일': 'email',
-            '보고서': 'report', '전략': 'strategy', '사업': 'business',
-            '정부': 'government', '지원': 'support', '바우처': 'voucher',
-            '창업': 'startup', '대학': 'university',
-        };
-        // Build entity Korean→English mapping from observations
-        try {
-            const entities = this.db.prepare(`
-        SELECT name, observations FROM entities
-      `).all();
-            for (const entity of entities) {
-                try {
-                    const obs = JSON.parse(entity.observations);
-                    for (const o of obs) {
-                        const match = o.match(/한국어명:\s*(.+)/);
-                        if (match) {
-                            const koreanName = match[1].trim();
-                            domainDict[koreanName] = entity.name;
-                        }
-                    }
-                }
-                catch { }
-            }
-        }
-        catch { }
-        // Translate Korean terms in query
-        let translated = query;
-        // Sort by length descending to match longer terms first
-        const sortedTerms = Object.entries(domainDict).sort((a, b) => b[0].length - a[0].length);
-        for (const [ko, en] of sortedTerms) {
-            translated = translated.replace(new RegExp(ko, 'g'), en);
-        }
-        // Remove remaining Korean characters and clean up
-        translated = translated.replace(/[\uAC00-\uD7A3]+/g, ' ').replace(/\s+/g, ' ').trim();
-        return translated.length > 2 ? translated : null;
-    }
     extractTermsFromText(text, options = {}) {
         const { minLength = 3, includeCapitalized = true, customPatterns = [] } = options;
         const terms = new Set();
@@ -917,33 +977,58 @@ class RAGKnowledgeGraphManager {
         }
         return chunks;
     }
-    // Generate embeddings using sentence transformers
-    // isQuery: true for search queries (adds instruction prefix), false for documents/entities
-    async generateEmbedding(text, dimensions = 1024, isQuery = false) {
-        if (this.modelInitialized && this.embeddingModel) {
+    // Generate embeddings using Ollama API
+    async generateEmbedding(text) {
+        // Check cache first
+        const cacheKey = text.length > 100 ? text.substring(0, 100) : text;
+        const cached = this.embeddingCache.get(cacheKey);
+        if (cached)
+            return cached;
+        if (this.modelInitialized) {
             try {
-                // Qwen3: add instruction prefix for queries (task-specific instruction improves accuracy)
-                const inputText = isQuery
-                    ? `Instruct: Given a search query, retrieve relevant entities and documents from a knowledge graph about halal certification, accreditation, and standards\nQuery: ${text}`
-                    : text;
-                // Use the real sentence transformer model (Qwen3: last_token pooling)
-                const result = await this.embeddingModel(inputText, { pooling: 'last_token', normalize: true });
-                // Extract the embedding array and convert to Float32Array
-                const embedding = result.data;
-                return new Float32Array(embedding.slice(0, dimensions));
+                const response = await fetch(`${OLLAMA_URL}/api/embed`, {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({
+                        model: EMBEDDING_MODEL,
+                        input: text
+                    })
+                });
+                if (!response.ok) {
+                    throw new Error(`Ollama API error: ${response.status} ${response.statusText}`);
+                }
+                const data = await response.json();
+                const modelResult = new Float32Array(data.embeddings[0]);
+                // Cache the result (LRU: evict oldest if full)
+                if (this.embeddingCache.size >= this.EMBEDDING_CACHE_MAX) {
+                    const firstKey = this.embeddingCache.keys().next().value;
+                    if (firstKey)
+                        this.embeddingCache.delete(firstKey);
+                }
+                this.embeddingCache.set(cacheKey, modelResult);
+                return modelResult;
             }
             catch (error) {
-                console.error(`⚠️ Embedding model failed for text "${text.slice(0, 50)}...":`, error instanceof Error ? error.message : error);
-                // Fall through to enhanced general implementation
+                console.error(`⚠️ Ollama embedding failed for text "${text.slice(0, 50)}...":`, error instanceof Error ? error.message : error);
+                // Fall through to fallback implementation
             }
         }
-        // Enhanced general-purpose semantic embedding
+        // Enhanced general-purpose semantic embedding (fallback)
+        const dimensions = EMBEDDING_DIM;
         const embedding = new Array(dimensions).fill(0);
         // Normalize and tokenize text (preserve Unicode letters including Korean, Arabic, etc.)
         const normalizedText = text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, ' ').replace(/\s+/g, ' ').trim();
         const words = normalizedText.split(' ').filter(word => word.length > 1);
         if (words.length === 0) {
-            return new Float32Array(embedding);
+            const emptyResult = new Float32Array(embedding);
+            // Cache the result (LRU: evict oldest if full)
+            if (this.embeddingCache.size >= this.EMBEDDING_CACHE_MAX) {
+                const firstKey = this.embeddingCache.keys().next().value;
+                if (firstKey)
+                    this.embeddingCache.delete(firstKey);
+            }
+            this.embeddingCache.set(cacheKey, emptyResult);
+            return emptyResult;
         }
         // Enhanced word importance calculation
         const wordFreq = new Map();
@@ -1076,7 +1161,15 @@ class RAGKnowledgeGraphManager {
         // L2 normalization for cosine similarity
         const magnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
         const normalizedEmbedding = magnitude > 0 ? embedding.map(val => val / magnitude) : embedding;
-        return new Float32Array(normalizedEmbedding);
+        const fallbackResult = new Float32Array(normalizedEmbedding);
+        // Cache the result (LRU: evict oldest if full)
+        if (this.embeddingCache.size >= this.EMBEDDING_CACHE_MAX) {
+            const firstKey = this.embeddingCache.keys().next().value;
+            if (firstKey)
+                this.embeddingCache.delete(firstKey);
+        }
+        this.embeddingCache.set(cacheKey, fallbackResult);
+        return fallbackResult;
     }
     // Calculate position-based importance weight
     calculatePositionWeight(position, totalWords) {
@@ -1123,7 +1216,7 @@ class RAGKnowledgeGraphManager {
         if (!document) {
             throw new Error(`Document with ID ${documentId} not found`);
         }
-        const { maxTokens = 200, overlap = 20 } = options;
+        const { maxTokens = 800, overlap = 160 } = options;
         console.error(`🔪 Chunking document: ${documentId} (maxTokens: ${maxTokens}, overlap: ${overlap})`);
         // Clean up existing chunks
         await this.cleanupDocument(documentId);
@@ -1591,18 +1684,8 @@ class RAGKnowledgeGraphManager {
         if (!this.encoding)
             throw new Error('Tokenizer not initialized');
         console.error(`🔍 Enhanced hybrid search: "${query}"`);
-        // Detect Korean in query and build cross-lingual translation
-        const hasKorean = /[\uAC00-\uD7A3]/.test(query);
-        let translatedQuery = null;
-        if (hasKorean) {
-            translatedQuery = this.translateQueryCrossLingual(query);
-            if (translatedQuery) {
-                console.error(`🌐 Cross-lingual: "${query}" → "${translatedQuery}"`);
-            }
-        }
-        // Generate query embedding(s) (with instruction prefix for Qwen3)
-        const queryEmbedding = await this.generateEmbedding(query, 1024, true);
-        const translatedEmbedding = translatedQuery ? await this.generateEmbedding(translatedQuery, 1024, true) : null;
+        // Generate query embedding
+        const queryEmbedding = await this.generateEmbedding(query);
         // Vector search helper
         const searchChunks = (embedding, k) => {
             return this.db.prepare(`
@@ -1628,21 +1711,8 @@ class RAGKnowledgeGraphManager {
         ORDER BY c.distance
       `).all(Buffer.from(embedding.buffer), k);
         };
-        // Dual search: original + translated (if available)
-        const originalResults = searchChunks(queryEmbedding, limit * 3);
-        const translatedResults = translatedEmbedding ? searchChunks(translatedEmbedding, limit * 3) : [];
-        // Merge and deduplicate by chunk_id, keeping best distance
-        const resultMap = new Map();
-        for (const r of originalResults) {
-            resultMap.set(r.chunk_id, r);
-        }
-        for (const r of translatedResults) {
-            const existing = resultMap.get(r.chunk_id);
-            if (!existing || r.distance < existing.distance) {
-                resultMap.set(r.chunk_id, r);
-            }
-        }
-        const vectorResults = Array.from(resultMap.values()).sort((a, b) => a.distance - b.distance);
+        // Vector search
+        const vectorResults = searchChunks(queryEmbedding, limit * 3);
         // FTS5 full-text search as additional signal (Reciprocal Rank Fusion)
         const ftsBoostMap = new Map();
         try {
@@ -1664,21 +1734,14 @@ class RAGKnowledgeGraphManager {
           LIMIT ?
         `).all(ftsExpr, limit * 3);
             };
-            const ftsOriginal = ftsSearchQuery(query);
-            const ftsTranslated = translatedQuery ? ftsSearchQuery(translatedQuery) : [];
-            // Merge FTS5 results, keeping best score per chunk_id
+            const ftsResults = ftsSearchQuery(query);
+            // Build FTS5 boost map
             const ftsResultMap = new Map();
             let rank = 1;
-            for (const r of ftsOriginal) {
+            for (const r of ftsResults) {
                 ftsResultMap.set(r.chunk_id, { chunk_id: r.chunk_id, fts_score: r.fts_score, rank });
                 rank++;
             }
-            for (const r of ftsTranslated) {
-                if (!ftsResultMap.has(r.chunk_id)) {
-                    ftsResultMap.set(r.chunk_id, { chunk_id: r.chunk_id, fts_score: r.fts_score, rank });
-                    rank++;
-                }
-            }
             // Build vector rank map for RRF
             const vectorRankMap = new Map();
             vectorResults.forEach((r, idx) => vectorRankMap.set(r.chunk_id, idx + 1));
@@ -1690,7 +1753,7 @@ class RAGKnowledgeGraphManager {
             }
             // Add FTS5-only results to the vector result pool
             for (const [chunkId] of ftsResultMap) {
-                if (!resultMap.has(chunkId)) {
+                if (!vectorResults.some(r => r.chunk_id === chunkId)) {
                     const chunkRow = this.db.prepare(`
             SELECT
               cm.rowid,
@@ -1732,7 +1795,7 @@ class RAGKnowledgeGraphManager {
         let connectedEntities = new Set();
         let queryMatchedEntities = new Set();
         if (useGraph) {
-            // Vector search: find entities semantically similar to the query (dual search)
+            // Vector search: find entities semantically similar to the query
             try {
                 const searchEntities = (embedding) => {
                     return this.db.prepare(`
@@ -1748,20 +1811,7 @@ class RAGKnowledgeGraphManager {
             ORDER BY ee.distance
           `).all(Buffer.from(embedding.buffer));
                 };
-                // Merge original + translated entity results
-                const entityMap = new Map();
-                for (const e of searchEntities(queryEmbedding)) {
-                    entityMap.set(e.entity_id, e);
-                }
-                if (translatedEmbedding) {
-                    for (const e of searchEntities(translatedEmbedding)) {
-                        const existing = entityMap.get(e.entity_id);
-                        if (!existing || e.distance < existing.distance) {
-                            entityMap.set(e.entity_id, e);
-                        }
-                    }
-                }
-                const similarEntities = Array.from(entityMap.values()).sort((a, b) => a.distance - b.distance);
+                const similarEntities = searchEntities(queryEmbedding);
                 for (const entity of similarEntities) {
                     const similarity = Math.max(0, 1 - entity.distance / 2);
                     if (similarity > 0.5) {
@@ -2127,6 +2177,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                 return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.searchNodes(validatedArgs.query, validatedArgs.limit || 10, validatedArgs.since, validatedArgs.until), null, 2) }] };
             case "openNodes":
                 return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.openNodes(validatedArgs.names), null, 2) }] };
+            case "getNeighbors":
+                return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.getNeighbors(validatedArgs.entityNames, validatedArgs.depth || 1, validatedArgs.relationType), null, 2) }] };
             // New RAG tools
             case "storeDocument":
                 return { content: [{ type: "text", text: JSON.stringify(await ragKgManager.storeDocument(validatedArgs.id, validatedArgs.content, validatedArgs.metadata || {}), null, 2) }] };