rag-memory-epf-mcp 3.3.3 → 3.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -428,7 +428,11 @@ export const migrations = [
|
|
|
428
428
|
db.exec(`DROP TABLE IF EXISTS chunks_fts`);
|
|
429
429
|
}
|
|
430
430
|
},
|
|
431
|
-
// Migration
|
|
431
|
+
// Migration 10: Separate token-space vs char-space chunk offsets.
|
|
432
|
+
// (Slot 9 is intentionally skipped — some user databases from early v3.x
|
|
433
|
+
// experiments have an unrelated migration recorded at version 9 (Ollama
|
|
434
|
+
// dimension swap). Reusing that slot would silently no-op against those
|
|
435
|
+
// databases. Version 10 ensures the migration runs everywhere.)
|
|
432
436
|
// Before this migration, chunk_metadata.start_pos/end_pos held *token* indices
|
|
433
437
|
// for document chunks (a leftover from the BPE tokenizer-based chunkText loop)
|
|
434
438
|
// but already held character lengths (0..text.length) for entity/relationship
|
|
@@ -442,13 +446,22 @@ export const migrations = [
|
|
|
442
446
|
// (already a valid 0..text.length char range against the chunk text itself);
|
|
443
447
|
// token columns stay NULL since these chunks have no token-space concept.
|
|
444
448
|
{
|
|
445
|
-
version:
|
|
449
|
+
version: 10,
|
|
446
450
|
description: 'Add start_token/end_token; reinterpret start_pos/end_pos as char offsets',
|
|
447
451
|
up: (db) => {
|
|
448
|
-
// 1) Add columns
|
|
449
|
-
|
|
450
|
-
db.
|
|
451
|
-
|
|
452
|
+
// 1) Add columns (idempotent — some databases may have been touched by a
|
|
453
|
+
// pre-release v9 attempt; tolerate the column already existing).
|
|
454
|
+
const cols = db.prepare(`PRAGMA table_info(chunk_metadata)`).all()
|
|
455
|
+
.map(c => c.name);
|
|
456
|
+
if (!cols.includes('start_token')) {
|
|
457
|
+
db.exec(`ALTER TABLE chunk_metadata ADD COLUMN start_token INTEGER`);
|
|
458
|
+
}
|
|
459
|
+
if (!cols.includes('end_token')) {
|
|
460
|
+
db.exec(`ALTER TABLE chunk_metadata ADD COLUMN end_token INTEGER`);
|
|
461
|
+
}
|
|
462
|
+
// 2) Move token data into new columns for document chunks (only if not
|
|
463
|
+
// already moved — guard against re-running in the rare case a prior
|
|
464
|
+
// partial run already touched some rows).
|
|
452
465
|
db.exec(`
|
|
453
466
|
UPDATE chunk_metadata
|
|
454
467
|
SET start_token = start_pos,
|
|
@@ -456,6 +469,8 @@ export const migrations = [
|
|
|
456
469
|
start_pos = NULL,
|
|
457
470
|
end_pos = NULL
|
|
458
471
|
WHERE chunk_type = 'document'
|
|
472
|
+
AND start_token IS NULL
|
|
473
|
+
AND start_pos IS NOT NULL
|
|
459
474
|
`);
|
|
460
475
|
// 3) Recompute char offsets via indexOf with a running cursor per document
|
|
461
476
|
const docRows = db.prepare(`
|