rag-memory-epf-mcp 3.3.2 → 3.3.4

package/dist/index.js

@@ -1136,11 +1136,18 @@ class RAGKnowledgeGraphManager {
     // Slicing token arrays at arbitrary boundaries can leave incomplete UTF-8
     // prefix/suffix bytes, which TextDecoder replaces with U+FFFD (�). Trim the
     // incomplete sequences at chunk boundaries; overlap covers the removed bytes.
+    //
+    // Each chunk records both token-space offsets (start_token/end_token from the
+    // BPE encoder loop) and char-space offsets (start_pos/end_pos into the original
+    // text). Char offsets are recovered with indexOf using a running cursor at the
+    // previous chunk's start, so substr(text, start_pos, end_pos - start_pos)
+    // equals chunk.text. On a coincidental indexOf miss the char offsets are NULL.
     chunkText(text, maxTokens = 800, overlap = 160) {
         if (!this.encoding)
             throw new Error('Tokenizer not initialized');
         const tokens = this.encoding.encode(text);
         const chunks = [];
+        let charCursor = 0;
         for (let i = 0; i < tokens.length; i += maxTokens - overlap) {
             const chunkTokens = tokens.slice(i, i + maxTokens);
             const decodedBytes = this.encoding.decode(chunkTokens);
@@ -1148,13 +1155,40 @@ class RAGKnowledgeGraphManager {
             const isLast = i + chunkTokens.length >= tokens.length;
             const safeBytes = trimIncompleteUtf8(decodedBytes, !isFirst, !isLast);
             const chunkText = new TextDecoder('utf-8').decode(safeBytes);
+            // Recover char-space offsets. First chunk is anchored at 0 because
+            // trimIncompleteUtf8 leaves the leading bytes intact when isFirst is true.
+            let startPos;
+            let endPos;
+            if (isFirst) {
+                startPos = 0;
+                endPos = chunkText.length;
+                charCursor = 0;
+            }
+            else if (chunkText.length === 0) {
+                startPos = null;
+                endPos = null;
+            }
+            else {
+                const idx = text.indexOf(chunkText, charCursor);
+                if (idx >= 0) {
+                    startPos = idx;
+                    endPos = idx + chunkText.length;
+                    charCursor = idx;
+                }
+                else {
+                    startPos = null;
+                    endPos = null;
+                }
+            }
             chunks.push({
                 id: '',
                 document_id: '',
                 chunk_index: chunks.length,
                 text: chunkText,
-                start_pos: i,
-                end_pos: i + chunkTokens.length
+                start_pos: startPos,
+                end_pos: endPos,
+                start_token: i,
+                end_token: i + chunkTokens.length
             });
         }
         return chunks;
@@ -1230,14 +1264,16 @@ class RAGKnowledgeGraphManager {
             // Store chunk metadata (no embedding yet)
             this.db.prepare(`
         INSERT INTO chunk_metadata (
-          chunk_id, document_id, chunk_index, text, start_pos, end_pos
-        ) VALUES (?, ?, ?, ?, ?, ?)
-      `).run(chunkId, documentId, chunk.chunk_index, chunk.text, chunk.start_pos, chunk.end_pos);
+          chunk_id, document_id, chunk_index, text, start_pos, end_pos, start_token, end_token
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+      `).run(chunkId, documentId, chunk.chunk_index, chunk.text, chunk.start_pos, chunk.end_pos, chunk.start_token, chunk.end_token);
             resultChunks.push({
                 id: chunkId,
                 text: chunk.text,
                 startPos: chunk.start_pos,
-                endPos: chunk.end_pos
+                endPos: chunk.end_pos,
+                startToken: chunk.start_token,
+                endToken: chunk.end_token
             });
         }
         console.error(`✅ Document chunked: ${chunks.length} chunks created`);
@@ -1713,6 +1749,8 @@ class RAGKnowledgeGraphManager {
           m.text,
           m.start_pos,
           m.end_pos,
+          m.start_token,
+          m.end_token,
           COALESCE(m.metadata, '{}') as chunk_metadata,
           c.distance,
           COALESCE(d.metadata, '{}') as doc_metadata
@@ -1793,6 +1831,8 @@ class RAGKnowledgeGraphManager {
               cm.text,
               cm.start_pos,
               cm.end_pos,
+              cm.start_token,
+              cm.end_token,
               COALESCE(cm.metadata, '{}') as chunk_metadata,
               COALESCE(d.metadata, '{}') as doc_metadata
             FROM chunk_metadata cm

package/dist/src/migrations/migrations.js

@@ -427,5 +427,98 @@ export const migrations = [
             db.exec(`DROP TABLE IF EXISTS entities_fts`);
             db.exec(`DROP TABLE IF EXISTS chunks_fts`);
         }
+    },
+    // Migration 10: Separate token-space vs char-space chunk offsets.
+    // (Slot 9 is intentionally skipped — some user databases from early v3.x
+    // experiments have an unrelated migration recorded at version 9 (Ollama
+    // dimension swap). Reusing that slot would silently no-op against those
+    // databases. Version 10 ensures the migration runs everywhere.)
+    // Before this migration, chunk_metadata.start_pos/end_pos held *token* indices
+    // for document chunks (a leftover from the BPE tokenizer-based chunkText loop)
+    // but already held character lengths (0..text.length) for entity/relationship
+    // chunks. Same column, two meanings — and a column name (`*_pos`) that implies
+    // char offsets in `documents.content`. This migration adds explicit
+    // start_token/end_token columns and reinterprets start_pos/end_pos as character
+    // offsets going forward. Existing document chunks: token data is moved to the
+    // new columns and char offsets are recomputed from documents.content via
+    // indexOf with a running cursor (NULL on miss — caller can re-chunk to fill).
+    // Existing entity/relationship chunks: leave start_pos/end_pos as-is
+    // (already a valid 0..text.length char range against the chunk text itself);
+    // token columns stay NULL since these chunks have no token-space concept.
+    {
+        version: 10,
+        description: 'Add start_token/end_token; reinterpret start_pos/end_pos as char offsets',
+        up: (db) => {
+            // 1) Add columns (idempotent — some databases may have been touched by a
+            //    pre-release v9 attempt; tolerate the column already existing).
+            const cols = db.prepare(`PRAGMA table_info(chunk_metadata)`).all()
+                .map(c => c.name);
+            if (!cols.includes('start_token')) {
+                db.exec(`ALTER TABLE chunk_metadata ADD COLUMN start_token INTEGER`);
+            }
+            if (!cols.includes('end_token')) {
+                db.exec(`ALTER TABLE chunk_metadata ADD COLUMN end_token INTEGER`);
+            }
+            // 2) Move token data into new columns for document chunks (only if not
+            //    already moved — guard against re-running in the rare case a prior
+            //    partial run already touched some rows).
+            db.exec(`
+        UPDATE chunk_metadata
+          SET start_token = start_pos,
+              end_token = end_pos,
+              start_pos = NULL,
+              end_pos = NULL
+          WHERE chunk_type = 'document'
+            AND start_token IS NULL
+            AND start_pos IS NOT NULL
+      `);
+            // 3) Recompute char offsets via indexOf with a running cursor per document
+            const docRows = db.prepare(`
+        SELECT DISTINCT document_id FROM chunk_metadata
+          WHERE chunk_type = 'document' AND document_id IS NOT NULL
+      `).all();
+            const docContentStmt = db.prepare(`SELECT content FROM documents WHERE id = ?`);
+            const chunksStmt = db.prepare(`
+        SELECT rowid, text FROM chunk_metadata
+          WHERE document_id = ? AND chunk_type = 'document'
+          ORDER BY chunk_index ASC
+      `);
+            const updateStmt = db.prepare(`
+        UPDATE chunk_metadata SET start_pos = ?, end_pos = ? WHERE rowid = ?
+      `);
+            for (const { document_id } of docRows) {
+                const doc = docContentStmt.get(document_id);
+                if (!doc)
+                    continue;
+                const content = doc.content;
+                const chunks = chunksStmt.all(document_id);
+                // Cursor advances by the previous chunk's *start*, not its end, so we can
+                // still locate overlapping chunks. Token-space stride guarantees each
+                // chunk's start is strictly forward of the previous chunk's start.
+                let cursor = 0;
+                for (const c of chunks) {
+                    if (!c.text)
+                        continue;
+                    const idx = content.indexOf(c.text, cursor);
+                    if (idx >= 0) {
+                        updateStmt.run(idx, idx + c.text.length, c.rowid);
+                        cursor = idx;
+                    }
+                    // miss: leave NULL — caller can re-chunk to repair
+                }
+            }
+        },
+        down: (db) => {
+            // SQLite cannot DROP COLUMN cleanly. Best-effort: copy token data back into
+            // start_pos/end_pos for document chunks so a downgrade leaves the legacy
+            // token-space semantics in place.
+            db.exec(`
+        UPDATE chunk_metadata
+          SET start_pos = start_token, end_pos = end_token
+          WHERE chunk_type = 'document' AND start_token IS NOT NULL
+      `);
+            // start_token/end_token columns remain (no DROP COLUMN); they will be
+            // ignored by older code.
+        }
     }
 ];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rag-memory-epf-mcp",
-  "version": "3.3.2",
+  "version": "3.3.4",
   "description": "MCP server for project-local RAG memory with knowledge graph and multilingual vector search",
   "license": "MIT",
   "author": "bripin123",