resplite 1.2.4 → 1.2.8

package/src/migration/index.js

@@ -13,7 +13,7 @@
  *   const info   = await m.preflight();
  *   await m.bulk({ onProgress: console.log });
  *   const status = m.status();
- *   await m.applyDirty();
+ *   await m.applyDirty({ onProgress: console.log });
  *   const result = await m.verify();
  *   await m.close();
  */
@@ -26,6 +26,7 @@ import { runApplyDirty } from './apply-dirty.js';
 import { runVerify } from './verify.js';
 import { runMigrateSearch } from './migrate-search.js';
 import { getRun, getDirtyCounts } from './registry.js';
+import { startDirtyTracker as startDirtyTrackerProcess } from './tracker.js';
 
 /**
  * @typedef {object} MigrationOptions
@@ -47,9 +48,11 @@ import { getRun, getDirtyCounts } from './registry.js';
  * @returns {{
  *   preflight(): Promise<object>,
  *   enableKeyspaceNotifications(opts?: { value?: string, merge?: boolean }): Promise<{ ok: boolean, previous: string|null, applied: string, error?: string }>,
+ *   startDirtyTracker(opts?: { pragmaTemplate?: string, onProgress?: function }): Promise<{ running: true }>,
+ *   stopDirtyTracker(): Promise<{ running: false }>,
  *   bulk(opts?: { resume?: boolean, onProgress?: function }): Promise<object>,
  *   status(): { run: object, dirty: object } | null,
- *   applyDirty(opts?: { batchKeys?: number, maxRps?: number }): Promise<object>,
+ *   applyDirty(opts?: { batchKeys?: number, maxRps?: number, onProgress?: function }): Promise<object>,
  *   verify(opts?: { samplePct?: number, maxSample?: number }): Promise<object>,
  *   migrateSearch(opts?: { onlyIndices?: string[], scanCount?: number, maxRps?: number, batchDocs?: number, maxSuggestions?: number, skipExisting?: boolean, withSuggestions?: boolean, onProgress?: function }): Promise<object>,
  *   close(): Promise<void>,
@@ -69,6 +72,7 @@ export function createMigration({
   if (!to) throw new Error('createMigration: "to" (db path) is required');
 
   let _client = null;
+  let _tracker = null;
 
   async function getClient() {
     if (_client) return _client;
@@ -113,6 +117,41 @@ export function createMigration({
       return setKeyspaceEvents(client, value, { configCommand, merge });
     },
 
+    /**
+     * Start dirty-key tracking in-process for this migration controller.
+     * Use this to run the full minimal-downtime flow in one Node script.
+     *
+     * @param {{
+     *   pragmaTemplate?: string,
+     *   onProgress?: (progress: { runId: string, key: string, event: string, totalEvents: number, at: string }) => void | Promise<void>
+     * }} [opts]
+     */
+    async startDirtyTracker({ pragmaTemplate: pt = pragmaTemplate, onProgress } = {}) {
+      if (_tracker) return { running: true };
+      const id = requireRunId();
+      _tracker = await startDirtyTrackerProcess({
+        from,
+        to,
+        runId: id,
+        pragmaTemplate: pt,
+        configCommand,
+        onProgress,
+      });
+      return { running: true };
+    },
+
+    /**
+     * Stop in-process dirty-key tracking started by `startDirtyTracker`.
+     * Safe to call even if tracking is not running.
+     */
+    async stopDirtyTracker() {
+      if (_tracker) {
+        await _tracker.stop();
+        _tracker = null;
+      }
+      return { running: false };
+    },
+
     /**
      * Step 1 — Bulk import: SCAN all keys from Redis into the destination DB.
      * Resume is on by default: first run starts from 0, later runs continue from checkpoint.
@@ -152,15 +191,16 @@ export function createMigration({
     /**
      * Step 3 — Apply dirty: reconcile keys that changed in Redis during bulk import.
      *
-     * @param {{ batchKeys?: number, maxRps?: number }} [opts]
+     * @param {{ batchKeys?: number, maxRps?: number, onProgress?: (run: object) => void }} [opts]
      */
-    async applyDirty({ batchKeys: bk = batchKeys, maxRps: rps = maxRps } = {}) {
+    async applyDirty({ batchKeys: bk = batchKeys, maxRps: rps = maxRps, onProgress } = {}) {
       const id = requireRunId();
       const client = await getClient();
       return runApplyDirty(client, to, id, {
         pragmaTemplate,
         batch_keys: bk,
         max_rps: rps,
+        onProgress,
       });
     },
 
@@ -208,6 +248,10 @@ export function createMigration({
      * Disconnect from Redis. Call when done with all migration operations.
      */
     async close() {
+      if (_tracker) {
+        await _tracker.stop().catch(() => {});
+        _tracker = null;
+      }
       if (_client) {
         await _client.quit().catch(() => {});
         _client = null;

package/src/migration/migrate-search.js

@@ -5,7 +5,7 @@
  *   1. FT._LIST  → enumerate index names
  *   2. FT.INFO   → read schema (prefix patterns, field attributes)
  *   3. Map RediSearch field types to RespLite TEXT fields
- *   4. FT.CREATE in RespLite (skip if already exists and skipExisting=true)
+ *   4. FT.CREATE in RespLite (or reuse the existing destination index when skipExisting=true)
  *   5. SCAN keys by prefix → HGETALL → addDocument in SQLite batches
  *   6. FT.SUGGET  → import suggestions
  *
@@ -198,6 +198,18 @@ function buildDocFields(hashData, fieldMap, schemaFields) {
   return docFields;
 }
 
+/**
+ * Read document score from Redis hash fields.
+ * Prefers `__score`, then `score`, and falls back to `1.0`.
+ *
+ * @param {Record<string, string>} hashData
+ * @returns {number}
+ */
+function getDocScore(hashData) {
+  const rawScore = hashData['__score'] ?? hashData['score'];
+  return rawScore ? (parseFloat(rawScore) || 1.0) : 1.0;
+}
+
 /**
  * Import suggestions from a RediSearch index via FT.SUGGET "" MAX n WITHSCORES.
  * RediSearch has no cursor for FT.SUGGET; maxSuggestions caps the import.
@@ -249,7 +261,7 @@ async function importSuggestions(redisClient, db, indexName, maxSuggestions) {
  * @param {number}   [options.maxRps=0]             - Max Redis requests/s (0 = unlimited).
  * @param {number}   [options.batchDocs=200]        - Docs per SQLite transaction.
  * @param {number}   [options.maxSuggestions=10000] - Cap for FT.SUGGET import.
- * @param {boolean}  [options.skipExisting=true]    - Skip index if already in RespLite.
+ * @param {boolean}  [options.skipExisting=true]    - Reuse an existing destination index and skip FT.CREATE instead of failing.
  * @param {boolean}  [options.withSuggestions=true] - Also migrate suggestions.
  * @param {(result: IndexResult) => void} [options.onProgress]
  * @returns {Promise<{ indices: IndexResult[], aborted: boolean }>}
@@ -327,6 +339,7 @@ export async function runMigrateSearch(redisClient, dbPath, options = {}) {
         if (e.message.includes('already exists')) {
           if (skipExisting) {
             skipped = true;
+            warnings.push(`Index "${indexName}" already exists in destination; reusing existing schema`);
           } else {
             results.push({ ...errorResult(indexName, 'Index already exists in destination'), warnings });
             continue;
@@ -341,6 +354,7 @@ export async function runMigrateSearch(redisClient, dbPath, options = {}) {
       let docsImported = 0;
       let docsSkipped  = 0;
       let docErrors    = 0;
+      const seenKeys = new Set();
 
       // Batch infrastructure: accumulate HGETALL results, flush in SQLite transactions
       const pendingHashData = new Map();
@@ -351,8 +365,7 @@ export async function runMigrateSearch(redisClient, dbPath, options = {}) {
           const hashData = pendingHashData.get(key);
           if (!hashData) continue;
           const docFields = buildDocFields(hashData, fieldMap, fields);
-          const rawScore  = hashData['__score'] ?? hashData['score'];
-          const score     = rawScore ? (parseFloat(rawScore) || 1.0) : 1.0;
+          const score = getDocScore(hashData);
           addDocument(db, indexName, key, score, true, docFields);
         }
       });
@@ -369,7 +382,7 @@ export async function runMigrateSearch(redisClient, dbPath, options = {}) {
             try {
               const hd = pendingHashData.get(k);
               if (!hd) continue;
-              addDocument(db, indexName, k, 1.0, true, buildDocFields(hd, fieldMap, fields));
+              addDocument(db, indexName, k, getDocScore(hd), true, buildDocFields(hd, fieldMap, fields));
               docsImported++;
             } catch (_e) {
               docErrors++;
@@ -406,11 +419,17 @@ export async function runMigrateSearch(redisClient, dbPath, options = {}) {
               continue;
             }
 
+            if (seenKeys.has(key)) {
+              continue;
+            }
+
             if (!hashData || typeof hashData !== 'object' || Object.keys(hashData).length === 0) {
+              seenKeys.add(key);
               docsSkipped++;
               continue;
             }
 
+            seenKeys.add(key);
             pendingHashData.set(key, hashData);
             pendingKeys.push(key);
 
@@ -454,4 +473,4 @@ function errorResult(name, error) {
 }
 
 // ── Exported helpers (used by tests) ─────────────────────────────────────────
-export { mapFields, buildDocFields };
+export { mapFields, buildDocFields, getDocScore };

package/src/migration/tracker.js

@@ -27,6 +27,13 @@ const KEYEVENT_PATTERN = '__keyevent@0__:*';
  * @param {string} options.runId                           - Migration run identifier.
  * @param {string} [options.pragmaTemplate='default']
  * @param {string} [options.configCommand='CONFIG']        - CONFIG command name (in case it was renamed).
+ * @param {(progress: {
+ *   runId: string,
+ *   key: string,
+ *   event: string,
+ *   totalEvents: number,
+ *   at: string
+ * }) => void | Promise<void>} [options.onProgress]        - Called for each tracked keyspace event.
  * @returns {Promise<{ stop(): Promise<void> }>}
  * @throws {Error} If keyspace notifications are not enabled on Redis.
  */
@@ -36,6 +43,7 @@ export async function startDirtyTracker({
   runId,
   pragmaTemplate = 'default',
   configCommand = 'CONFIG',
+  onProgress,
 } = {}) {
   if (!to) throw new Error('startDirtyTracker: "to" (db path) is required');
   if (!runId) throw new Error('startDirtyTracker: "runId" is required');
@@ -66,12 +74,27 @@ export async function startDirtyTracker({
   );
   await subClient.connect();
 
+  let totalEvents = 0;
   await subClient.pSubscribe(KEYEVENT_PATTERN, (message, channel) => {
     const event = typeof channel === 'string'
       ? channel.split(':').pop()
       : String(channel ?? '').split(':').pop() || 'unknown';
     try {
       upsertDirtyKey(db, runId, message, event);
+      totalEvents += 1;
+      if (onProgress) {
+        Promise.resolve(
+          onProgress({
+            runId,
+            key: message,
+            event,
+            totalEvents,
+            at: new Date().toISOString(),
+          })
+        ).catch((err) => {
+          logError(db, runId, 'dirty_apply', 'Tracker onProgress error: ' + err.message, message);
+        });
+      }
     } catch (err) {
       logError(db, runId, 'dirty_apply', err.message, message);
     }

package/test/integration/migration-dirty-tracker.test.js

@@ -100,13 +100,17 @@ describe('dirty tracker integration', { timeout: 30_000 }, () => {
     const runId  = `tracker-test-${Date.now()}`;
 
     const m = createMigration({ from: REDIS_URL, to: dbPath, runId });
+    const progressEvents = [];
 
     const ks = await m.enableKeyspaceNotifications({ value: 'KEA' });
     assert.ok(ks.ok, `Failed to enable keyspace notifications: ${ks.error}`);
 
-    // ── Start tracker BEFORE bulk ────────────────────────────────────────
-    const tracker = await startDirtyTracker({ from: REDIS_URL, to: dbPath, runId });
-
+    // ── Start tracker BEFORE bulk (same-script API) ─────────────────────
+    await m.startDirtyTracker({
+      onProgress: (p) => {
+        progressEvents.push(p);
+      },
+    });
     try {
       // ── Bulk import (captures the initial snapshot) ──────────────────
       const run = await m.bulk();
@@ -121,7 +125,7 @@ describe('dirty tracker integration', { timeout: 30_000 }, () => {
       // Give the tracker time to process the keyspace events
       await new Promise((r) => setTimeout(r, 600));
     } finally {
-      await tracker.stop();
+      await m.stopDirtyTracker();
     }
 
     // ── Verify dirty keys were captured ─────────────────────────────────
@@ -130,6 +134,7 @@ describe('dirty tracker integration', { timeout: 30_000 }, () => {
       dirty.dirty + dirty.deleted >= 3,
       `Expected ≥3 dirty/deleted keys, got dirty=${dirty.dirty} deleted=${dirty.deleted}`
     );
+    assert.ok(progressEvents.length >= 3, `Expected tracker onProgress events, got ${progressEvents.length}`);
 
     // ── Apply-dirty: reconcile post-bulk changes ─────────────────────────
     const afterApply = await m.applyDirty();

package/test/integration/object-idletime.test.js

@@ -0,0 +1,51 @@
+import { describe, it, before, after } from 'node:test';
+import assert from 'node:assert/strict';
+import { createTestServer } from '../helpers/server.js';
+import { sendCommand, argv } from '../helpers/client.js';
+
+describe('OBJECT IDLETIME integration', () => {
+  let s;
+  let port;
+
+  before(async () => {
+    s = await createTestServer();
+    port = s.port;
+  });
+
+  after(async () => {
+    await s.closeAsync();
+  });
+
+  it('OBJECT IDLETIME missing key returns nil', async () => {
+    const reply = await sendCommand(port, argv('OBJECT', 'IDLETIME', 'nokey'));
+    assert.equal(reply.toString('ascii'), '$-1\r\n');
+  });
+
+  it('OBJECT IDLETIME returns seconds since last write', async () => {
+    await sendCommand(port, argv('SET', 'idlekey', 'v'));
+    const reply = await sendCommand(port, argv('OBJECT', 'IDLETIME', 'idlekey'));
+    const line = reply.toString('ascii');
+    assert.match(line, /^:\d+\r\n$/);
+    const seconds = parseInt(line.replace(/\D/g, ''), 10);
+    assert.ok(seconds >= 0);
+  });
+
+  it('OBJECT IDLETIME increases after write', async () => {
+    await sendCommand(port, argv('SET', 'idlekey2', 'v'));
+    await sendCommand(port, argv('OBJECT', 'IDLETIME', 'idlekey2'));
+    await new Promise((r) => setTimeout(r, 1100));
+    const reply = await sendCommand(port, argv('OBJECT', 'IDLETIME', 'idlekey2'));
+    const seconds = parseInt(reply.toString('ascii').replace(/\D/g, ''), 10);
+    assert.ok(seconds >= 1, 'idle time should be at least 1 second after 1.1s wait');
+  });
+
+  it('OBJECT wrong subcommand returns error', async () => {
+    const reply = await sendCommand(port, argv('OBJECT', 'REFCOUNT', 'k'));
+    assert.ok(reply.toString('ascii').startsWith('-ERR'));
+  });
+
+  it('OBJECT with wrong number of args returns error', async () => {
+    const reply = await sendCommand(port, argv('OBJECT', 'IDLETIME'));
+    assert.ok(reply.toString('ascii').startsWith('-ERR'));
+  });
+});

package/test/unit/migrate-search.test.js

@@ -6,7 +6,7 @@
 
 import { describe, it } from 'node:test';
 import assert from 'node:assert/strict';
-import { runMigrateSearch, mapFields, buildDocFields } from '../../src/migration/migrate-search.js';
+import { runMigrateSearch, mapFields, buildDocFields, getDocScore } from '../../src/migration/migrate-search.js';
 import { openDb } from '../../src/storage/sqlite/db.js';
 import { getIndexMeta, getIndexCounts, search, suggestionGet } from '../../src/storage/sqlite/search.js';
 import { tmpDbPath } from '../helpers/tmp.js';
@@ -188,6 +188,26 @@ describe('buildDocFields', () => {
   });
 });
 
+// ── getDocScore ───────────────────────────────────────────────────────────────
+
+describe('getDocScore', () => {
+  it('prefers __score over score', () => {
+    assert.equal(getDocScore({ __score: '2.5', score: '1.0' }), 2.5);
+  });
+
+  it('uses score when __score is absent', () => {
+    assert.equal(getDocScore({ score: '3.25' }), 3.25);
+  });
+
+  it('falls back to 1.0 for invalid score values', () => {
+    assert.equal(getDocScore({ score: 'not-a-number' }), 1.0);
+  });
+
+  it('falls back to 1.0 when score fields are missing', () => {
+    assert.equal(getDocScore({}), 1.0);
+  });
+});
+
 // ── runMigrateSearch — core behaviour ─────────────────────────────────────────
 
 describe('runMigrateSearch', () => {
@@ -326,7 +346,7 @@ describe('runMigrateSearch', () => {
     assert.equal(idx.docsSkipped, 2);
   });
 
-  it('skipExisting=true skips already-existing index', async () => {
+  it('skipExisting=true reuses an existing index and refreshes documents', async () => {
     const dbPath = tmpDbPath();
     // First run — creates index
     const redis = makeFakeRedis({
@@ -342,6 +362,8 @@ describe('runMigrateSearch', () => {
     assert.equal(result2.indices[0].skipped, true);
     assert.equal(result2.indices[0].created, false);
     assert.equal(result2.indices[0].error, undefined);
+    assert.equal(result2.indices[0].docsImported, 1);
+    assert.ok(result2.indices[0].warnings.some((w) => w.includes('reusing existing schema')));
   });
 
   it('skipExisting=false errors on existing index', async () => {
@@ -374,6 +396,32 @@ describe('runMigrateSearch', () => {
     assert.deepEqual(names, ['a', 'c']);
   });
 
+  it('deduplicates keys that match overlapping index prefixes', async () => {
+    const dbPath = tmpDbPath();
+    const redis = makeFakeRedis({
+      indexNames: ['overlap'],
+      indexInfo: {
+        overlap: {
+          prefixes: ['doc:', 'doc:special:'],
+          attributes: [{ identifier: 'payload', attribute: 'payload', type: 'TEXT' }],
+        },
+      },
+      scanKeys: ['doc:special:1'],
+      hashKeys: {
+        'doc:special:1': { payload: 'hello overlap' },
+      },
+    });
+
+    const result = await runMigrateSearch(redis, dbPath);
+    assert.equal(result.indices.length, 1);
+    assert.equal(result.indices[0].docsImported, 1);
+
+    const db = openDb(dbPath, { pragmaTemplate: 'minimal' });
+    const counts = getIndexCounts(db, 'overlap');
+    assert.equal(counts.num_docs, 1);
+    db.close();
+  });
+
   it('imports suggestions', async () => {
     const dbPath = tmpDbPath();
     const redis = makeFakeRedis({