rag-memory-epf-mcp 3.3.3 → 3.3.5

package/dist/index.js

@@ -1139,15 +1139,19 @@ class RAGKnowledgeGraphManager {
     //
     // Each chunk records both token-space offsets (start_token/end_token from the
     // BPE encoder loop) and char-space offsets (start_pos/end_pos into the original
-    // text). Char offsets are recovered with indexOf using a running cursor at the
-    // previous chunk's start, so substr(text, start_pos, end_pos - start_pos)
-    // equals chunk.text. On a coincidental indexOf miss the char offsets are NULL.
+    // text). Char offsets are Unicode codepoint counts — language-neutral, so SQL
+    // substr, Python str slicing, and JS [...str] iteration all line up. JS's
+    // native UTF-16 indexing differs for supplementary characters (emoji, rare
+    // CJK), so the function maintains parallel UTF-16 and codepoint cursors and
+    // reports codepoint offsets. On a coincidental indexOf miss the char offsets
+    // are NULL.
     chunkText(text, maxTokens = 800, overlap = 160) {
         if (!this.encoding)
             throw new Error('Tokenizer not initialized');
         const tokens = this.encoding.encode(text);
         const chunks = [];
-        let charCursor = 0;
+        let utf16Cursor = 0;
+        let cpCursor = 0;
         for (let i = 0; i < tokens.length; i += maxTokens - overlap) {
             const chunkTokens = tokens.slice(i, i + maxTokens);
             const decodedBytes = this.encoding.decode(chunkTokens);
@@ -1155,25 +1159,31 @@ class RAGKnowledgeGraphManager {
             const isLast = i + chunkTokens.length >= tokens.length;
             const safeBytes = trimIncompleteUtf8(decodedBytes, !isFirst, !isLast);
             const chunkText = new TextDecoder('utf-8').decode(safeBytes);
-            // Recover char-space offsets. First chunk is anchored at 0 because
-            // trimIncompleteUtf8 leaves the leading bytes intact when isFirst is true.
             let startPos;
             let endPos;
             if (isFirst) {
                 startPos = 0;
-                endPos = chunkText.length;
-                charCursor = 0;
+                endPos = [...chunkText].length;
+                utf16Cursor = 0;
+                cpCursor = 0;
             }
             else if (chunkText.length === 0) {
                 startPos = null;
                 endPos = null;
             }
             else {
-                const idx = text.indexOf(chunkText, charCursor);
-                if (idx >= 0) {
-                    startPos = idx;
-                    endPos = idx + chunkText.length;
-                    charCursor = idx;
+                const utfIdx = text.indexOf(chunkText, utf16Cursor);
+                if (utfIdx >= 0) {
+                    // Advance cpCursor by codepoints between the previous cursor and the
+                    // new chunk's start (handles overlap by anchoring at the previous
+                    // chunk's start, not its end).
+                    if (utfIdx > utf16Cursor) {
+                        cpCursor += [...text.slice(utf16Cursor, utfIdx)].length;
+                        utf16Cursor = utfIdx;
+                    }
+                    const cpLen = [...chunkText].length;
+                    startPos = cpCursor;
+                    endPos = cpCursor + cpLen;
                 }
                 else {
                     startPos = null;

package/dist/src/migrations/migrations.js

@@ -428,7 +428,11 @@ export const migrations = [
             db.exec(`DROP TABLE IF EXISTS chunks_fts`);
         }
     },
-    // Migration 9: Separate token-space vs char-space chunk offsets.
+    // Migration 10: Separate token-space vs char-space chunk offsets.
+    // (Slot 9 is intentionally skipped — some user databases from early v3.x
+    // experiments have an unrelated migration recorded at version 9 (Ollama
+    // dimension swap). Reusing that slot would silently no-op against those
+    // databases. Version 10 ensures the migration runs everywhere.)
     // Before this migration, chunk_metadata.start_pos/end_pos held *token* indices
     // for document chunks (a leftover from the BPE tokenizer-based chunkText loop)
     // but already held character lengths (0..text.length) for entity/relationship
@@ -442,13 +446,22 @@ export const migrations = [
     // (already a valid 0..text.length char range against the chunk text itself);
     // token columns stay NULL since these chunks have no token-space concept.
     {
-        version: 9,
+        version: 10,
         description: 'Add start_token/end_token; reinterpret start_pos/end_pos as char offsets',
         up: (db) => {
-            // 1) Add columns
-            db.exec(`ALTER TABLE chunk_metadata ADD COLUMN start_token INTEGER`);
-            db.exec(`ALTER TABLE chunk_metadata ADD COLUMN end_token INTEGER`);
-            // 2) Move token data into new columns for document chunks
+            // 1) Add columns (idempotent — some databases may have been touched by a
+            //    pre-release v9 attempt; tolerate the column already existing).
+            const cols = db.prepare(`PRAGMA table_info(chunk_metadata)`).all()
+                .map(c => c.name);
+            if (!cols.includes('start_token')) {
+                db.exec(`ALTER TABLE chunk_metadata ADD COLUMN start_token INTEGER`);
+            }
+            if (!cols.includes('end_token')) {
+                db.exec(`ALTER TABLE chunk_metadata ADD COLUMN end_token INTEGER`);
+            }
+            // 2) Move token data into new columns for document chunks (only if not
+            //    already moved — guard against re-running in the rare case a prior
+            //    partial run already touched some rows).
             db.exec(`
         UPDATE chunk_metadata
           SET start_token = start_pos,
@@ -456,6 +469,8 @@ export const migrations = [
               start_pos = NULL,
               end_pos = NULL
           WHERE chunk_type = 'document'
+            AND start_token IS NULL
+            AND start_pos IS NOT NULL
       `);
             // 3) Recompute char offsets via indexOf with a running cursor per document
             const docRows = db.prepare(`
@@ -505,5 +520,55 @@ export const migrations = [
             // start_token/end_token columns remain (no DROP COLUMN); they will be
             // ignored by older code.
         }
+    },
+    // Migration 11: Convert chunk_metadata.start_pos/end_pos from JS UTF-16 code
+    // unit indices to Unicode codepoint indices. v3.3.4 stored offsets in JS's
+    // native UTF-16 unit space, which mismatches SQL substr/length and Python
+    // string indexing for any document containing supplementary characters
+    // (emoji, rare CJK). Codepoints are language-neutral. Walks each document's
+    // chunks in chunk_index order, locating each chunk in the source via UTF-16
+    // indexOf and counting codepoints between cursor positions to derive the
+    // codepoint offsets. Idempotent — chunks where indexOf misses keep their
+    // existing values.
+    {
+        version: 11,
+        description: 'Convert chunk_metadata.start_pos/end_pos to Unicode codepoint indices',
+        up: (db) => {
+            const docRows = db.prepare(`SELECT DISTINCT document_id FROM chunk_metadata
+        WHERE chunk_type='document' AND document_id IS NOT NULL
+          AND start_pos IS NOT NULL AND end_pos IS NOT NULL`).all();
+            const getContent = db.prepare(`SELECT content FROM documents WHERE id = ?`);
+            const getChunks = db.prepare(`SELECT rowid, text FROM chunk_metadata
+        WHERE document_id=? AND chunk_type='document'
+          AND start_pos IS NOT NULL AND end_pos IS NOT NULL
+        ORDER BY chunk_index ASC`);
+            const upd = db.prepare(`UPDATE chunk_metadata SET start_pos=?, end_pos=? WHERE rowid=?`);
+            for (const { document_id } of docRows) {
+                const doc = getContent.get(document_id);
+                if (!doc)
+                    continue;
+                const content = doc.content;
+                const chunks = getChunks.all(document_id);
+                let utf16Cursor = 0;
+                let cpCursor = 0;
+                for (const c of chunks) {
+                    if (!c.text)
+                        continue;
+                    const utfIdx = content.indexOf(c.text, utf16Cursor);
+                    if (utfIdx < 0)
+                        continue; // miss: leave existing
+                    if (utfIdx > utf16Cursor) {
+                        cpCursor += [...content.slice(utf16Cursor, utfIdx)].length;
+                        utf16Cursor = utfIdx;
+                    }
+                    const cpLen = [...c.text].length;
+                    upd.run(cpCursor, cpCursor + cpLen, c.rowid);
+                }
+            }
+        },
+        down: (_db) => {
+            // No clean reversal: would need the original document content to recompute
+            // UTF-16 indices, and we never recorded which chunks were touched. No-op.
+        }
     }
 ];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rag-memory-epf-mcp",
-  "version": "3.3.3",
+  "version": "3.3.5",
   "description": "MCP server for project-local RAG memory with knowledge graph and multilingual vector search",
   "license": "MIT",
   "author": "bripin123",