rag-memory-epf-mcp 3.3.0 → 3.3.2

package/dist/index.js

@@ -39,6 +39,40 @@ const DB_FILE_PATH = process.env.DB_FILE_PATH
     : defaultDbPath;
 const EMBEDDING_MODEL = process.env.EMBEDDING_MODEL || 'Xenova/bge-m3';
 // Safe rowid for vec0 virtual tables (require literal integer, not parameterized)
+// Trim incomplete UTF-8 multi-byte sequences at chunk boundaries.
+// Continuation bytes match 10xxxxxx (0x80-0xBF); lead bytes indicate how many
+// bytes the sequence needs (0xxxxxxx=1, 110xxxxx=2, 1110xxxx=3, 11110xxx=4).
+// When a chunk is not at the document head/tail, any partial sequence at that
+// edge belongs to an adjacent chunk and must be removed so TextDecoder does
+// not emit U+FFFD. Pass trimHead/trimTail=false to preserve head/tail bytes.
+function trimIncompleteUtf8(bytes, trimHead, trimTail) {
+    let start = 0;
+    let end = bytes.length;
+    if (trimHead) {
+        while (start < end && (bytes[start] & 0xC0) === 0x80)
+            start++;
+    }
+    if (trimTail) {
+        let i = end - 1;
+        while (i >= start && (bytes[i] & 0xC0) === 0x80)
+            i--;
+        if (i >= start) {
+            const lead = bytes[i];
+            let needed = 1;
+            if ((lead & 0x80) === 0)
+                needed = 1;
+            else if ((lead & 0xE0) === 0xC0)
+                needed = 2;
+            else if ((lead & 0xF0) === 0xE0)
+                needed = 3;
+            else if ((lead & 0xF8) === 0xF0)
+                needed = 4;
+            if (end - i < needed)
+                end = i;
+        }
+    }
+    return bytes.subarray(start, end);
+}
 function safeRowid(value) {
     const n = Number(value);
     if (!Number.isInteger(n) || n < 0) {
@@ -154,7 +188,7 @@ class RAGKnowledgeGraphManager {
       VALUES (?, ?, ?, ?, ?)
     `);
         for (const entity of entities) {
-            const entityId = `entity_${entity.name.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
+            const entityId = `entity_${entity.name.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
             const timestamped = (entity.observations || []).map(o => this._timestampObservation(o));
             // Try insert first
             const insertResult = insertStmt.run(entityId, entity.name, entity.entityType, JSON.stringify(timestamped), '{}');
@@ -199,8 +233,8 @@ class RAGKnowledgeGraphManager {
                 { name: relation.from, entityType: 'CONCEPT', observations: [] },
                 { name: relation.to, entityType: 'CONCEPT', observations: [] }
             ]);
-            const sourceId = `entity_${relation.from.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
-            const targetId = `entity_${relation.to.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
+            const sourceId = `entity_${relation.from.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
+            const targetId = `entity_${relation.to.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
             const relationId = `rel_${sourceId}_${relation.relationType}_${targetId}`.toLowerCase();
             const stmt = this.db.prepare(`
         INSERT OR IGNORE INTO relationships 
@@ -219,7 +253,7 @@ class RAGKnowledgeGraphManager {
             throw new Error('Database not initialized');
         const results = [];
         for (const obs of observations) {
-            const entityId = `entity_${obs.entityName.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
+            const entityId = `entity_${obs.entityName.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
             // Get current observations
             const entity = this.db.prepare(`
         SELECT observations FROM entities WHERE id = ?
@@ -250,7 +284,7 @@ class RAGKnowledgeGraphManager {
             throw new Error('Database not initialized');
         console.error(`🗑️ Deleting entities: ${entityNames.join(', ')}`);
         for (const name of entityNames) {
-            const entityId = `entity_${name.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
+            const entityId = `entity_${name.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
             try {
                 // Check if entity exists first
                 const entityExists = this.db.prepare(`
@@ -312,7 +346,7 @@ class RAGKnowledgeGraphManager {
         if (!this.db)
             throw new Error('Database not initialized');
         for (const deletion of deletions) {
-            const entityId = `entity_${deletion.entityName.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
+            const entityId = `entity_${deletion.entityName.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
             const entity = this.db.prepare(`
         SELECT observations FROM entities WHERE id = ?
       `).get(entityId);
@@ -329,8 +363,8 @@ class RAGKnowledgeGraphManager {
         if (!this.db)
             throw new Error('Database not initialized');
         for (const relation of relations) {
-            const sourceId = `entity_${relation.from.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
-            const targetId = `entity_${relation.to.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
+            const sourceId = `entity_${relation.from.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
+            const targetId = `entity_${relation.to.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
             this.db.prepare(`
         DELETE FROM relationships
         WHERE source_entity = ? AND target_entity = ? AND relationType = ?
@@ -343,8 +377,8 @@ class RAGKnowledgeGraphManager {
         let updated = 0;
         let notFound = 0;
         for (const update of updates) {
-            const sourceId = `entity_${update.from.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
-            const targetId = `entity_${update.to.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
+            const sourceId = `entity_${update.from.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
+            const targetId = `entity_${update.to.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
             const relationId = `rel_${sourceId}_${update.relationType}_${targetId}`.toLowerCase();
             // Check if relation exists
             const existing = this.db.prepare(`
@@ -407,7 +441,7 @@ class RAGKnowledgeGraphManager {
         // Cap depth at 5 to prevent runaway queries
         const effectiveDepth = Math.min(Math.max(depth, 1), 5);
         // Convert entity names to IDs
-        const seedIds = entityNames.map(name => `entity_${name.toLowerCase().replace(/[^a-z0-9]/g, '_')}`);
+        const seedIds = entityNames.map(name => `entity_${name.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`);
         // Build dynamic placeholders for the seed IDs
         const seedPlaceholders = seedIds.map(() => '?').join(',');
         // Build the recursive CTE query
@@ -1098,6 +1132,10 @@ class RAGKnowledgeGraphManager {
         return Array.from(terms);
     }
     // Tokenize and chunk text
+    // BPE tokenizers (cl100k_base) split multi-byte UTF-8 sequences across tokens.
+    // Slicing token arrays at arbitrary boundaries can leave incomplete UTF-8
+    // prefix/suffix bytes, which TextDecoder replaces with U+FFFD (�). Trim the
+    // incomplete sequences at chunk boundaries; overlap covers the removed bytes.
     chunkText(text, maxTokens = 800, overlap = 160) {
         if (!this.encoding)
             throw new Error('Tokenizer not initialized');
@@ -1106,7 +1144,10 @@ class RAGKnowledgeGraphManager {
         for (let i = 0; i < tokens.length; i += maxTokens - overlap) {
             const chunkTokens = tokens.slice(i, i + maxTokens);
             const decodedBytes = this.encoding.decode(chunkTokens);
-            const chunkText = new TextDecoder().decode(decodedBytes);
+            const isFirst = i === 0;
+            const isLast = i + chunkTokens.length >= tokens.length;
+            const safeBytes = trimIncompleteUtf8(decodedBytes, !isFirst, !isLast);
+            const chunkText = new TextDecoder('utf-8').decode(safeBytes);
             chunks.push({
                 id: '',
                 document_id: '',
@@ -1364,7 +1405,7 @@ class RAGKnowledgeGraphManager {
       INSERT OR IGNORE INTO chunk_entities (chunk_rowid, entity_id) VALUES (?, ?)
     `);
         for (const entityName of entityNames) {
-            const entityId = `entity_${entityName.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
+            const entityId = `entity_${entityName.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
             // Verify entity exists
             const entity = this.db.prepare(`
         SELECT id FROM entities WHERE id = ?
@@ -2144,7 +2185,7 @@ class RAGKnowledgeGraphManager {
         if (entityNames && entityNames.length > 0) {
             const targetIds = new Set();
             for (const name of entityNames) {
-                const id = `entity_${name.toLowerCase().replace(/[^a-z0-9]/g, '_')}`;
+                const id = `entity_${name.toLowerCase().replace(/[^\p{L}\p{N}]/gu, '_')}`;
                 if (graph.hasNode(id))
                     targetIds.add(id);
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rag-memory-epf-mcp",
-  "version": "3.3.0",
+  "version": "3.3.2",
   "description": "MCP server for project-local RAG memory with knowledge graph and multilingual vector search",
   "license": "MIT",
   "author": "bripin123",