resplite 1.4.14 → 1.4.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "resplite",
-  "version": "1.4.14",
+  "version": "1.4.18",
   "description": "A RESP2 server with practical Redis compatibility, backed by SQLite",
   "type": "module",
   "main": "src/index.js",

package/src/storage/sqlite/schema.js

@@ -72,6 +72,10 @@ CREATE TABLE IF NOT EXISTS search_indices (
   created_at INTEGER NOT NULL,
   updated_at INTEGER NOT NULL
 );
+
+CREATE TABLE IF NOT EXISTS search_rowid_allocator (
+  id INTEGER PRIMARY KEY AUTOINCREMENT
+);
 `;
 
 /** Type enum: 1=string, 2=hash, 3=set, 4=list, 5=zset */

package/src/storage/sqlite/search.js

@@ -28,6 +28,57 @@ function tableName(idx, suffix) {
   return `search_${suffix}__${idx}`;
 }
 
+/**
+ * Build deterministic sorted field names for FTS column order.
+ * @param {{ fields: { name: string, type: string }[] }} schema
+ * @returns {string[]}
+ */
+function getSortedFieldNames(schema) {
+  return schema.fields.map((f) => f.name).sort();
+}
+
+/**
+ * Encode field values in deterministic FTS column order.
+ * Missing values are normalized to empty string to match insert semantics.
+ * @param {string[]} fieldNames
+ * @param {Record<string, string>} fields
+ * @returns {string[]}
+ */
+function encodeFtsFieldValues(fieldNames, fields) {
+  return fieldNames.map((f) => fields[f] ?? '');
+}
+
+/**
+ * Delete a specific contentless FTS row using the special 'delete' command row.
+ * FTS5 requires passing the exact prior values for all indexed columns.
+ * @param {import('better-sqlite3').Database} db
+ * @param {string} ftsTableName
+ * @param {number} ftsRowid
+ * @param {string[]} fieldNames
+ * @param {Record<string, string>} fields
+ */
+function deleteFtsRow(db, ftsTableName, ftsRowid, fieldNames, fields) {
+  const values = encodeFtsFieldValues(fieldNames, fields);
+  const columns = [ftsTableName, 'rowid', ...fieldNames];
+  const placeholders = columns.map(() => '?').join(', ');
+  db.prepare(`INSERT INTO ${ftsTableName}(${columns.join(', ')}) VALUES (${placeholders})`).run(
+    'delete',
+    ftsRowid,
+    ...values
+  );
+}
+
+/**
+ * Allocate a monotonic FTS rowid that is never reused, even after deletes.
+ * This prevents legacy stale tokens from being remapped to a new document.
+ * @param {import('better-sqlite3').Database} db
+ * @returns {number}
+ */
+function allocateFtsRowid(db) {
+  const info = db.prepare('INSERT INTO search_rowid_allocator DEFAULT VALUES').run();
+  return Number(info.lastInsertRowid);
+}
+
 /**
  * Build canonical schema JSON (fields sorted by name). D.12.2
  * @param {{ name: string, type: string }[]} fields
@@ -138,7 +189,7 @@ export function getIndexMeta(db, name) {
  */
 export function addDocument(db, idx, docId, score, replace, fields) {
   const meta = getIndexMeta(db, idx);
-  const fieldNames = meta.schema.fields.map((f) => f.name);
+  const fieldNames = getSortedFieldNames(meta.schema);
   for (const k of Object.keys(fields)) {
     if (!fieldNames.includes(k)) throw new Error('ERR unknown field');
   }
@@ -154,14 +205,16 @@ export function addDocument(db, idx, docId, score, replace, fields) {
     let ftsRowid;
     if (existing) {
       if (!replace) throw new Error('ERR document exists');
-      // FTS5 contentless bug: INSERT OR REPLACE doesn't remove old tokens.
-      // Solution: assign a new fts_rowid to avoid token pollution.
-      const maxRow = db.prepare(`SELECT COALESCE(MAX(fts_rowid), 0) AS m FROM ${docmapT}`).get();
-      ftsRowid = maxRow.m + 1;
+      const previousRowid = existing.fts_rowid;
+      ftsRowid = allocateFtsRowid(db);
       db.prepare(`UPDATE ${docmapT} SET fts_rowid = ? WHERE doc_id = ?`).run(ftsRowid, docId);
+      const oldDoc = db.prepare(`SELECT fields_json FROM ${docsT} WHERE doc_id = ?`).get(docId);
+      if (oldDoc?.fields_json) {
+        const oldFields = JSON.parse(oldDoc.fields_json);
+        deleteFtsRow(db, ftsT, previousRowid, fieldNames, oldFields);
+      }
     } else {
-      const maxRow = db.prepare(`SELECT COALESCE(MAX(fts_rowid), 0) AS m FROM ${docmapT}`).get();
-      ftsRowid = maxRow.m + 1;
+      ftsRowid = allocateFtsRowid(db);
       db.prepare(`INSERT INTO ${docmapT}(doc_id, fts_rowid) VALUES (?, ?)`).run(docId, ftsRowid);
     }
 
@@ -176,9 +229,8 @@ export function addDocument(db, idx, docId, score, replace, fields) {
       ).run(docId, score, fieldsJson, now, now);
     }
 
-    // FTS5 contentless: insert with new rowid (old rowid becomes orphaned and won't match via docmap join).
-    const ftsColumns = ['rowid', ...fieldNames.sort()];
-    const ftsValues = [ftsRowid, ...fieldNames.sort().map((f) => fields[f] ?? '')];
+    const ftsColumns = ['rowid', ...fieldNames];
+    const ftsValues = [ftsRowid, ...encodeFtsFieldValues(fieldNames, fields)];
     const placeholders = ftsValues.map(() => '?').join(', ');
     const colList = ftsColumns.join(', ');
     db.prepare(`INSERT INTO ${ftsT}(${colList}) VALUES (${placeholders})`).run(...ftsValues);
@@ -218,7 +270,8 @@ export function getDocumentFields(db, idx, docId) {
 }
 
 export function deleteDocument(db, idx, docId) {
-  getIndexMeta(db, idx);
+  const meta = getIndexMeta(db, idx);
+  const fieldNames = getSortedFieldNames(meta.schema);
   const docsT = tableName(idx, 'docs');
   const docmapT = tableName(idx, 'docmap');
   const ftsT = tableName(idx, 'fts');
@@ -226,9 +279,12 @@ export function deleteDocument(db, idx, docId) {
   const row = db.prepare(`SELECT fts_rowid FROM ${docmapT} WHERE doc_id = ?`).get(docId);
   if (!row) return 0;
 
-  // FTS5 contentless does not support DELETE. Remove from docs and docmap; FTS row becomes orphaned
-  // (search results join through docmap so orphaned FTS rows are not returned).
   db.transaction(() => {
+    const docRow = db.prepare(`SELECT fields_json FROM ${docsT} WHERE doc_id = ?`).get(docId);
+    if (docRow?.fields_json) {
+      const fields = JSON.parse(docRow.fields_json);
+      deleteFtsRow(db, ftsT, row.fts_rowid, fieldNames, fields);
+    }
     db.prepare(`DELETE FROM ${docsT} WHERE doc_id = ?`).run(docId);
     db.prepare(`DELETE FROM ${docmapT} WHERE doc_id = ?`).run(docId);
   })();

package/test/integration/search.test.js

@@ -230,6 +230,37 @@ describe('Search integration', () => {
     const g2 = tryParseValue(gorge2, 0).value;
     assert.equal(g2[0], 0, 'gorge* should NOT match martan - this is the bug');
   });
+
+  it('FT.DEL followed by FT.ADD REPLACE does not keep stale tokens', async () => {
+    await sendCommand(port, argv('FT.CREATE', 'del_replace_idx', 'SCHEMA', 'payload', 'TEXT'));
+
+    const addBicho = await sendCommand(
+      port,
+      argv('FT.ADD', 'del_replace_idx', 'DY1O2', '1', 'REPLACE', 'FIELDS', 'payload', 'bicho')
+    );
+    assert.equal(tryParseValue(addBicho, 0).value, 'OK');
+
+    const del = await sendCommand(port, argv('FT.DEL', 'del_replace_idx', 'DY1O2'));
+    assert.equal(tryParseValue(del, 0).value, 1);
+
+    const addGorrion = await sendCommand(
+      port,
+      argv('FT.ADD', 'del_replace_idx', 'DY1O2', '1', 'REPLACE', 'FIELDS', 'payload', 'gorrion')
+    );
+    assert.equal(tryParseValue(addGorrion, 0).value, 'OK');
+
+    const oldPrefix = await sendCommand(port, argv('FT.SEARCH', 'del_replace_idx', 'bicho*', 'NOCONTENT', 'LIMIT', '0', '10'));
+    const oldArr = tryParseValue(oldPrefix, 0).value;
+    assert.equal(oldArr[0], 0, 'bicho* should not match after re-adding DY1O2 with gorrion');
+
+    const newPrefix = await sendCommand(
+      port,
+      argv('FT.SEARCH', 'del_replace_idx', 'gorrion*', 'NOCONTENT', 'LIMIT', '0', '10')
+    );
+    const newArr = tryParseValue(newPrefix, 0).value;
+    assert.equal(newArr[0], 1);
+    assert.equal(newArr[1].toString?.('utf8') ?? newArr[1], 'DY1O2');
+  });
 });
 
 describe('Search persistence', () => {

package/test/unit/search.test.js

@@ -88,6 +88,31 @@ describe('Search layer', () => {
     assert.equal(deleteDocument(db, 'names', 'nonexistent'), 0);
   });
 
+  it('delete + re-add never remaps legacy stale tokens', () => {
+    createIndex(db, 'legacy_stale', [{ name: 'payload', type: 'TEXT' }]);
+    addDocument(db, 'legacy_stale', 'doc1', 1, true, { payload: 'gorrion' });
+
+    const mapped = db
+      .prepare('SELECT fts_rowid FROM search_docmap__legacy_stale WHERE doc_id = ?')
+      .get('doc1');
+    const oldRowid = mapped.fts_rowid;
+
+    // Simulate a legacy-polluted index row where stale term postings exist for the same rowid.
+    db.prepare('INSERT INTO search_fts__legacy_stale(rowid, payload) VALUES (?, ?)').run(oldRowid, 'bicho');
+
+    assert.equal(search(db, 'legacy_stale', 'bicho*', { noContent: true }).total, 1);
+    assert.equal(deleteDocument(db, 'legacy_stale', 'doc1'), 1);
+
+    addDocument(db, 'legacy_stale', 'doc1', 1, true, { payload: 'gorrion' });
+    const remapped = db
+      .prepare('SELECT fts_rowid FROM search_docmap__legacy_stale WHERE doc_id = ?')
+      .get('doc1');
+    assert.notEqual(remapped.fts_rowid, oldRowid);
+
+    assert.equal(search(db, 'legacy_stale', 'gorrion*', { noContent: true }).total, 1);
+    assert.equal(search(db, 'legacy_stale', 'bicho*', { noContent: true }).total, 0);
+  });
+
   it('search with NOCONTENT returns total and doc ids', () => {
     const r = search(db, 'names', 'hello', { noContent: true, offset: 0, count: 10 });
     assert.equal(typeof r.total, 'number');