resplite 1.4.12 → 1.4.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "resplite",
-  "version": "1.4.12",
+  "version": "1.4.16",
   "description": "A RESP2 server with practical Redis compatibility, backed by SQLite",
   "type": "module",
   "main": "src/index.js",

package/src/storage/sqlite/search.js

@@ -28,6 +28,46 @@ function tableName(idx, suffix) {
   return `search_${suffix}__${idx}`;
 }
 
+/**
+ * Build deterministic sorted field names for FTS column order.
+ * @param {{ fields: { name: string, type: string }[] }} schema
+ * @returns {string[]}
+ */
+function getSortedFieldNames(schema) {
+  return schema.fields.map((f) => f.name).sort();
+}
+
+/**
+ * Encode field values in deterministic FTS column order.
+ * Missing values are normalized to empty string to match insert semantics.
+ * @param {string[]} fieldNames
+ * @param {Record<string, string>} fields
+ * @returns {string[]}
+ */
+function encodeFtsFieldValues(fieldNames, fields) {
+  return fieldNames.map((f) => fields[f] ?? '');
+}
+
+/**
+ * Delete a specific contentless FTS row using the special 'delete' command row.
+ * FTS5 requires passing the exact prior values for all indexed columns.
+ * @param {import('better-sqlite3').Database} db
+ * @param {string} ftsTableName
+ * @param {number} ftsRowid
+ * @param {string[]} fieldNames
+ * @param {Record<string, string>} fields
+ */
+function deleteFtsRow(db, ftsTableName, ftsRowid, fieldNames, fields) {
+  const values = encodeFtsFieldValues(fieldNames, fields);
+  const columns = [ftsTableName, 'rowid', ...fieldNames];
+  const placeholders = columns.map(() => '?').join(', ');
+  db.prepare(`INSERT INTO ${ftsTableName}(${columns.join(', ')}) VALUES (${placeholders})`).run(
+    'delete',
+    ftsRowid,
+    ...values
+  );
+}
+
 /**
  * Build canonical schema JSON (fields sorted by name). D.12.2
  * @param {{ name: string, type: string }[]} fields
@@ -138,7 +178,7 @@ export function getIndexMeta(db, name) {
  */
 export function addDocument(db, idx, docId, score, replace, fields) {
   const meta = getIndexMeta(db, idx);
-  const fieldNames = meta.schema.fields.map((f) => f.name);
+  const fieldNames = getSortedFieldNames(meta.schema);
   for (const k of Object.keys(fields)) {
     if (!fieldNames.includes(k)) throw new Error('ERR unknown field');
   }
@@ -155,8 +195,13 @@ export function addDocument(db, idx, docId, score, replace, fields) {
     if (existing) {
       if (!replace) throw new Error('ERR document exists');
       ftsRowid = existing.fts_rowid;
+      const oldDoc = db.prepare(`SELECT fields_json FROM ${docsT} WHERE doc_id = ?`).get(docId);
+      if (oldDoc?.fields_json) {
+        const oldFields = JSON.parse(oldDoc.fields_json);
+        deleteFtsRow(db, ftsT, ftsRowid, fieldNames, oldFields);
+      }
     } else {
-      const maxRow = db.prepare(`SELECT COALESCE(MAX(fts_rowid), 0) AS m FROM ${docmapT}`).get();
+      const maxRow = db.prepare(`SELECT COALESCE(MAX(rowid), 0) AS m FROM ${ftsT}`).get();
       ftsRowid = maxRow.m + 1;
       db.prepare(`INSERT INTO ${docmapT}(doc_id, fts_rowid) VALUES (?, ?)`).run(docId, ftsRowid);
     }
@@ -172,12 +217,11 @@ export function addDocument(db, idx, docId, score, replace, fields) {
       ).run(docId, score, fieldsJson, now, now);
     }
 
-    // FTS5 contentless: cannot DELETE; use INSERT OR REPLACE to overwrite by rowid when replacing.
-    const ftsColumns = ['rowid', ...fieldNames.sort()];
-    const ftsValues = [ftsRowid, ...fieldNames.sort().map((f) => fields[f] ?? '')];
+    const ftsColumns = ['rowid', ...fieldNames];
+    const ftsValues = [ftsRowid, ...encodeFtsFieldValues(fieldNames, fields)];
     const placeholders = ftsValues.map(() => '?').join(', ');
     const colList = ftsColumns.join(', ');
-    db.prepare(`INSERT OR REPLACE INTO ${ftsT}(${colList}) VALUES (${placeholders})`).run(...ftsValues);
+    db.prepare(`INSERT INTO ${ftsT}(${colList}) VALUES (${placeholders})`).run(...ftsValues);
   });
 
   run();
@@ -214,7 +258,8 @@ export function getDocumentFields(db, idx, docId) {
 }
 
 export function deleteDocument(db, idx, docId) {
-  getIndexMeta(db, idx);
+  const meta = getIndexMeta(db, idx);
+  const fieldNames = getSortedFieldNames(meta.schema);
   const docsT = tableName(idx, 'docs');
   const docmapT = tableName(idx, 'docmap');
   const ftsT = tableName(idx, 'fts');
@@ -222,9 +267,12 @@ export function deleteDocument(db, idx, docId) {
   const row = db.prepare(`SELECT fts_rowid FROM ${docmapT} WHERE doc_id = ?`).get(docId);
   if (!row) return 0;
 
-  // FTS5 contentless does not support DELETE. Remove from docs and docmap; FTS row becomes orphaned
-  // (search results join through docmap so orphaned FTS rows are not returned).
   db.transaction(() => {
+    const docRow = db.prepare(`SELECT fields_json FROM ${docsT} WHERE doc_id = ?`).get(docId);
+    if (docRow?.fields_json) {
+      const fields = JSON.parse(docRow.fields_json);
+      deleteFtsRow(db, ftsT, row.fts_rowid, fieldNames, fields);
+    }
     db.prepare(`DELETE FROM ${docsT} WHERE doc_id = ?`).run(docId);
     db.prepare(`DELETE FROM ${docmapT} WHERE doc_id = ?`).run(docId);
   })();

package/test/integration/search.test.js

@@ -196,6 +196,71 @@ describe('Search integration', () => {
     const err = v?.error ?? v;
     assert.ok(String(err).includes('Unknown index name'));
   });
+
+  it('FT.SEARCH prefix query should not match unrelated terms', async () => {
+    // Create a fresh index for this test
+    await sendCommand(port, argv('FT.CREATE', 'bugtest', 'SCHEMA', 'payload', 'TEXT'));
+    
+    // Add document with "gorge" first
+    await sendCommand(port, argv('FT.ADD', 'bugtest', 'DOC1', '1', 'REPLACE', 'FIELDS', 'payload', 'gorge'));
+    
+    // Verify it matches gorge*
+    const gorge1 = await sendCommand(port, argv('FT.SEARCH', 'bugtest', 'gorge*', 'NOCONTENT', 'LIMIT', '0', '10'));
+    const g1 = tryParseValue(gorge1, 0).value;
+    assert.equal(g1[0], 1, 'gorge* should match gorge');
+    assert.equal(g1[1].toString?.('utf8') ?? g1[1], 'DOC1');
+    
+    // Replace with "martan"
+    await sendCommand(port, argv('FT.ADD', 'bugtest', 'DOC1', '1', 'REPLACE', 'FIELDS', 'payload', 'martan'));
+    
+    // Verify FT.GET shows only martan
+    const get = await sendCommand(port, argv('FT.GET', 'bugtest', 'DOC1'));
+    const getArr = tryParseValue(get, 0).value;
+    assert.equal(getArr[0].toString?.('utf8') ?? getArr[0], 'payload');
+    assert.equal(getArr[1].toString?.('utf8') ?? getArr[1], 'martan');
+    
+    // martan* should match
+    const martan = await sendCommand(port, argv('FT.SEARCH', 'bugtest', 'martan*', 'NOCONTENT', 'LIMIT', '0', '10'));
+    const m = tryParseValue(martan, 0).value;
+    assert.equal(m[0], 1, 'martan* should match martan');
+    assert.equal(m[1].toString?.('utf8') ?? m[1], 'DOC1');
+    
+    // gorge* should NOT match martan
+    const gorge2 = await sendCommand(port, argv('FT.SEARCH', 'bugtest', 'gorge*', 'NOCONTENT', 'LIMIT', '0', '10'));
+    const g2 = tryParseValue(gorge2, 0).value;
+    assert.equal(g2[0], 0, 'gorge* should NOT match martan - this is the bug');
+  });
+
+  it('FT.DEL followed by FT.ADD REPLACE does not keep stale tokens', async () => {
+    await sendCommand(port, argv('FT.CREATE', 'del_replace_idx', 'SCHEMA', 'payload', 'TEXT'));
+
+    const addBicho = await sendCommand(
+      port,
+      argv('FT.ADD', 'del_replace_idx', 'DY1O2', '1', 'REPLACE', 'FIELDS', 'payload', 'bicho')
+    );
+    assert.equal(tryParseValue(addBicho, 0).value, 'OK');
+
+    const del = await sendCommand(port, argv('FT.DEL', 'del_replace_idx', 'DY1O2'));
+    assert.equal(tryParseValue(del, 0).value, 1);
+
+    const addGorrion = await sendCommand(
+      port,
+      argv('FT.ADD', 'del_replace_idx', 'DY1O2', '1', 'REPLACE', 'FIELDS', 'payload', 'gorrion')
+    );
+    assert.equal(tryParseValue(addGorrion, 0).value, 'OK');
+
+    const oldPrefix = await sendCommand(port, argv('FT.SEARCH', 'del_replace_idx', 'bicho*', 'NOCONTENT', 'LIMIT', '0', '10'));
+    const oldArr = tryParseValue(oldPrefix, 0).value;
+    assert.equal(oldArr[0], 0, 'bicho* should not match after re-adding DY1O2 with gorrion');
+
+    const newPrefix = await sendCommand(
+      port,
+      argv('FT.SEARCH', 'del_replace_idx', 'gorrion*', 'NOCONTENT', 'LIMIT', '0', '10')
+    );
+    const newArr = tryParseValue(newPrefix, 0).value;
+    assert.equal(newArr[0], 1);
+    assert.equal(newArr[1].toString?.('utf8') ?? newArr[1], 'DY1O2');
+  });
 });
 
 describe('Search persistence', () => {