rag-memory-epf-mcp 3.3.1 → 3.3.3

package/dist/index.js

@@ -39,6 +39,40 @@ const DB_FILE_PATH = process.env.DB_FILE_PATH
     : defaultDbPath;
 const EMBEDDING_MODEL = process.env.EMBEDDING_MODEL || 'Xenova/bge-m3';
 // Safe rowid for vec0 virtual tables (require literal integer, not parameterized)
+// Trim incomplete UTF-8 multi-byte sequences at chunk boundaries.
+// Continuation bytes match 10xxxxxx (0x80-0xBF); lead bytes indicate how many
+// bytes the sequence needs (0xxxxxxx=1, 110xxxxx=2, 1110xxxx=3, 11110xxx=4).
+// When a chunk is not at the document head/tail, any partial sequence at that
+// edge belongs to an adjacent chunk and must be removed so TextDecoder does
+// not emit U+FFFD. Pass trimHead/trimTail=false to preserve head/tail bytes.
+function trimIncompleteUtf8(bytes, trimHead, trimTail) {
+    let start = 0;
+    let end = bytes.length;
+    if (trimHead) {
+        while (start < end && (bytes[start] & 0xC0) === 0x80)
+            start++;
+    }
+    if (trimTail) {
+        let i = end - 1;
+        while (i >= start && (bytes[i] & 0xC0) === 0x80)
+            i--;
+        if (i >= start) {
+            const lead = bytes[i];
+            let needed = 1;
+            if ((lead & 0x80) === 0)
+                needed = 1;
+            else if ((lead & 0xE0) === 0xC0)
+                needed = 2;
+            else if ((lead & 0xF0) === 0xE0)
+                needed = 3;
+            else if ((lead & 0xF8) === 0xF0)
+                needed = 4;
+            if (end - i < needed)
+                end = i;
+        }
+    }
+    return bytes.subarray(start, end);
+}
 function safeRowid(value) {
     const n = Number(value);
     if (!Number.isInteger(n) || n < 0) {
@@ -1098,22 +1132,63 @@ class RAGKnowledgeGraphManager {
         return Array.from(terms);
     }
     // Tokenize and chunk text
+    // BPE tokenizers (cl100k_base) split multi-byte UTF-8 sequences across tokens.
+    // Slicing token arrays at arbitrary boundaries can leave incomplete UTF-8
+    // prefix/suffix bytes, which TextDecoder replaces with U+FFFD (�). Trim the
+    // incomplete sequences at chunk boundaries; overlap covers the removed bytes.
+    //
+    // Each chunk records both token-space offsets (start_token/end_token from the
+    // BPE encoder loop) and char-space offsets (start_pos/end_pos into the original
+    // text). Char offsets are recovered with indexOf using a running cursor at the
+    // previous chunk's start, so substr(text, start_pos, end_pos - start_pos)
+    // equals chunk.text. On a coincidental indexOf miss the char offsets are NULL.
     chunkText(text, maxTokens = 800, overlap = 160) {
         if (!this.encoding)
             throw new Error('Tokenizer not initialized');
         const tokens = this.encoding.encode(text);
         const chunks = [];
+        let charCursor = 0;
         for (let i = 0; i < tokens.length; i += maxTokens - overlap) {
             const chunkTokens = tokens.slice(i, i + maxTokens);
             const decodedBytes = this.encoding.decode(chunkTokens);
-            const chunkText = new TextDecoder().decode(decodedBytes);
+            const isFirst = i === 0;
+            const isLast = i + chunkTokens.length >= tokens.length;
+            const safeBytes = trimIncompleteUtf8(decodedBytes, !isFirst, !isLast);
+            const chunkText = new TextDecoder('utf-8').decode(safeBytes);
+            // Recover char-space offsets. First chunk is anchored at 0 because
+            // trimIncompleteUtf8 leaves the leading bytes intact when isFirst is true.
+            let startPos;
+            let endPos;
+            if (isFirst) {
+                startPos = 0;
+                endPos = chunkText.length;
+                charCursor = 0;
+            }
+            else if (chunkText.length === 0) {
+                startPos = null;
+                endPos = null;
+            }
+            else {
+                const idx = text.indexOf(chunkText, charCursor);
+                if (idx >= 0) {
+                    startPos = idx;
+                    endPos = idx + chunkText.length;
+                    charCursor = idx;
+                }
+                else {
+                    startPos = null;
+                    endPos = null;
+                }
+            }
             chunks.push({
                 id: '',
                 document_id: '',
                 chunk_index: chunks.length,
                 text: chunkText,
-                start_pos: i,
-                end_pos: i + chunkTokens.length
+                start_pos: startPos,
+                end_pos: endPos,
+                start_token: i,
+                end_token: i + chunkTokens.length
             });
         }
         return chunks;
@@ -1189,14 +1264,16 @@ class RAGKnowledgeGraphManager {
             // Store chunk metadata (no embedding yet)
             this.db.prepare(`
         INSERT INTO chunk_metadata (
-          chunk_id, document_id, chunk_index, text, start_pos, end_pos
-        ) VALUES (?, ?, ?, ?, ?, ?)
-      `).run(chunkId, documentId, chunk.chunk_index, chunk.text, chunk.start_pos, chunk.end_pos);
+          chunk_id, document_id, chunk_index, text, start_pos, end_pos, start_token, end_token
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+      `).run(chunkId, documentId, chunk.chunk_index, chunk.text, chunk.start_pos, chunk.end_pos, chunk.start_token, chunk.end_token);
             resultChunks.push({
                 id: chunkId,
                 text: chunk.text,
                 startPos: chunk.start_pos,
-                endPos: chunk.end_pos
+                endPos: chunk.end_pos,
+                startToken: chunk.start_token,
+                endToken: chunk.end_token
             });
         }
         console.error(`✅ Document chunked: ${chunks.length} chunks created`);
@@ -1672,6 +1749,8 @@ class RAGKnowledgeGraphManager {
           m.text,
           m.start_pos,
           m.end_pos,
+          m.start_token,
+          m.end_token,
           COALESCE(m.metadata, '{}') as chunk_metadata,
           c.distance,
           COALESCE(d.metadata, '{}') as doc_metadata
@@ -1752,6 +1831,8 @@ class RAGKnowledgeGraphManager {
               cm.text,
               cm.start_pos,
               cm.end_pos,
+              cm.start_token,
+              cm.end_token,
               COALESCE(cm.metadata, '{}') as chunk_metadata,
               COALESCE(d.metadata, '{}') as doc_metadata
             FROM chunk_metadata cm

package/dist/src/migrations/migrations.js

@@ -427,5 +427,83 @@ export const migrations = [
             db.exec(`DROP TABLE IF EXISTS entities_fts`);
             db.exec(`DROP TABLE IF EXISTS chunks_fts`);
         }
+    },
+    // Migration 9: Separate token-space vs char-space chunk offsets.
+    // Before this migration, chunk_metadata.start_pos/end_pos held *token* indices
+    // for document chunks (a leftover from the BPE tokenizer-based chunkText loop)
+    // but already held character lengths (0..text.length) for entity/relationship
+    // chunks. Same column, two meanings — and a column name (`*_pos`) that implies
+    // char offsets in `documents.content`. This migration adds explicit
+    // start_token/end_token columns and reinterprets start_pos/end_pos as character
+    // offsets going forward. Existing document chunks: token data is moved to the
+    // new columns and char offsets are recomputed from documents.content via
+    // indexOf with a running cursor (NULL on miss — caller can re-chunk to fill).
+    // Existing entity/relationship chunks: leave start_pos/end_pos as-is
+    // (already a valid 0..text.length char range against the chunk text itself);
+    // token columns stay NULL since these chunks have no token-space concept.
+    {
+        version: 9,
+        description: 'Add start_token/end_token; reinterpret start_pos/end_pos as char offsets',
+        up: (db) => {
+            // 1) Add columns
+            db.exec(`ALTER TABLE chunk_metadata ADD COLUMN start_token INTEGER`);
+            db.exec(`ALTER TABLE chunk_metadata ADD COLUMN end_token INTEGER`);
+            // 2) Move token data into new columns for document chunks
+            db.exec(`
+        UPDATE chunk_metadata
+          SET start_token = start_pos,
+              end_token = end_pos,
+              start_pos = NULL,
+              end_pos = NULL
+          WHERE chunk_type = 'document'
+      `);
+            // 3) Recompute char offsets via indexOf with a running cursor per document
+            const docRows = db.prepare(`
+        SELECT DISTINCT document_id FROM chunk_metadata
+          WHERE chunk_type = 'document' AND document_id IS NOT NULL
+      `).all();
+            const docContentStmt = db.prepare(`SELECT content FROM documents WHERE id = ?`);
+            const chunksStmt = db.prepare(`
+        SELECT rowid, text FROM chunk_metadata
+          WHERE document_id = ? AND chunk_type = 'document'
+          ORDER BY chunk_index ASC
+      `);
+            const updateStmt = db.prepare(`
+        UPDATE chunk_metadata SET start_pos = ?, end_pos = ? WHERE rowid = ?
+      `);
+            for (const { document_id } of docRows) {
+                const doc = docContentStmt.get(document_id);
+                if (!doc)
+                    continue;
+                const content = doc.content;
+                const chunks = chunksStmt.all(document_id);
+                // Cursor advances by the previous chunk's *start*, not its end, so we can
+                // still locate overlapping chunks. Token-space stride guarantees each
+                // chunk's start is strictly forward of the previous chunk's start.
+                let cursor = 0;
+                for (const c of chunks) {
+                    if (!c.text)
+                        continue;
+                    const idx = content.indexOf(c.text, cursor);
+                    if (idx >= 0) {
+                        updateStmt.run(idx, idx + c.text.length, c.rowid);
+                        cursor = idx;
+                    }
+                    // miss: leave NULL — caller can re-chunk to repair
+                }
+            }
+        },
+        down: (db) => {
+            // SQLite cannot DROP COLUMN cleanly. Best-effort: copy token data back into
+            // start_pos/end_pos for document chunks so a downgrade leaves the legacy
+            // token-space semantics in place.
+            db.exec(`
+        UPDATE chunk_metadata
+          SET start_pos = start_token, end_pos = end_token
+          WHERE chunk_type = 'document' AND start_token IS NOT NULL
+      `);
+            // start_token/end_token columns remain (no DROP COLUMN); they will be
+            // ignored by older code.
+        }
     }
 ];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rag-memory-epf-mcp",
-  "version": "3.3.1",
+  "version": "3.3.3",
   "description": "MCP server for project-local RAG memory with knowledge graph and multilingual vector search",
   "license": "MIT",
   "author": "bripin123",