resplite 1.4.0 → 1.4.2

package/README.md

@@ -166,7 +166,17 @@ await rl.question('Stop app traffic to Redis, then press Enter to apply the fina
 rl.close();
 
 // Step 3 — Apply dirty keys that changed in Redis during bulk
-await m.applyDirty({ onProgress: console.log });
+await m.applyDirty({
+  concurrency: 32,
+  batchKeys: 5000,
+  onProgress: (r) => {
+    console.log(
+      `dirty processed=${r.dirty_keys_processed} pending=${r.dirty_pending} ` +
+      `applied=${r.dirty_keys_applied} deleted=${r.dirty_keys_deleted} ` +
+      `rate=${r.dirty_keys_per_second.toFixed(1)} keys/s eta=${r.dirty_eta_seconds ?? '—'}s`
+    );
+  },
+});
 
 // Step 3b — Stop tracker after cutover
 await m.stopDirtyTracker();

package/docs/generated/migration-apply-dirty-concurrency-progress.md

@@ -0,0 +1,7 @@
+---
+id: 90dhbsvf0f
+type: implementation
+title: Apply dirty migration concurrency and progress
+created: '2026-03-12 14:55:12'
+---
+Improved src/migration/apply-dirty.js to support concurrent dirty-key apply via options.concurrency (chunked Promise.all worker model) while preserving max_rps throttling. Added richer onProgress payload fields: dirty_keys_processed, dirty_pending, dirty_keys_per_second, dirty_eta_seconds, and related counters. Exposed new options through createMigration().applyDirty: concurrency and progressIntervalMs. Updated README migration cutover snippet with high-throughput applyDirty example and progress logging. Added unit test test/unit/migration-apply-dirty.test.js validating concurrency and progress payload. Verified with node --test test/unit/migration-apply-dirty.test.js and npm run test:unit (all passing).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "resplite",
-  "version": "1.4.0",
+  "version": "1.4.2",
   "description": "A RESP2 server with practical Redis compatibility, backed by SQLite",
   "type": "module",
   "main": "src/index.js",

package/src/migration/apply-dirty.js

@@ -16,6 +16,30 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+function buildDirtyProgressPayload(run, startedAtMs, totalProcessed, totalFetched, pendingDirty, pendingDeleted) {
+  if (!run) return null;
+  const elapsedSec = Math.max(0.001, (Date.now() - startedAtMs) / 1000);
+  const keysPerSec = totalProcessed / elapsedSec;
+  const pendingTotal = pendingDirty + pendingDeleted;
+  const etaSeconds = keysPerSec > 0 ? Math.ceil(pendingTotal / keysPerSec) : null;
+  const applied = Number(run.dirty_keys_applied || 0);
+  const deleted = Number(run.dirty_keys_deleted || 0);
+  const reconciled = applied + deleted;
+
+  return {
+    ...run,
+    dirty_elapsed_seconds: elapsedSec,
+    dirty_keys_per_second: keysPerSec,
+    dirty_keys_processed: totalProcessed,
+    dirty_keys_fetched: totalFetched,
+    dirty_reconciled_total: reconciled,
+    dirty_pending: pendingTotal,
+    dirty_pending_dirty: pendingDirty,
+    dirty_pending_deleted: pendingDeleted,
+    dirty_eta_seconds: etaSeconds,
+  };
+}
+
 /**
  * Apply dirty keys: for each key in registry with state=dirty, reimport from Redis or delete in destination.
  * @param {import('redis').RedisClientType} redisClient
@@ -25,10 +49,19 @@ function sleep(ms) {
  * @param {string} [options.pragmaTemplate='default']
  * @param {number} [options.batch_keys=200]
  * @param {number} [options.max_rps=0]
+ * @param {number} [options.concurrency=1]
+ * @param {number} [options.progress_interval_ms=2000]
  * @param {(run: object) => void | Promise<void>} [options.onProgress] - Called after each batch with the current run row.
  */
 export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
-  const { pragmaTemplate = 'default', batch_keys = 200, max_rps = 0, onProgress } = options;
+  const {
+    pragmaTemplate = 'default',
+    batch_keys = 200,
+    max_rps = 0,
+    concurrency = 1,
+    progress_interval_ms = 2000,
+    onProgress,
+  } = options;
 
   const db = openDb(dbPath, { pragmaTemplate });
   const run = getRun(db, runId);
@@ -43,7 +76,41 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
   const storages = { keys, strings, hashes, sets, lists, zsets };
 
   const minIntervalMs = max_rps > 0 ? 1000 / max_rps : 0;
-  let lastKeyTime = 0;
+  const workerCount = Number.isFinite(concurrency) ? Math.max(1, Math.floor(concurrency)) : 1;
+  let nextAllowedAt = 0;
+  const startedAtMs = Date.now();
+  let totalProcessed = 0;
+  let totalFetched = 0;
+  let lastProgressAt = 0;
+
+  async function awaitRateLimit() {
+    if (minIntervalMs <= 0) return;
+    const now = Date.now();
+    const scheduled = Math.max(now, nextAllowedAt);
+    nextAllowedAt = scheduled + minIntervalMs;
+    if (scheduled > now) {
+      await sleep(scheduled - now);
+    }
+  }
+
+  function emitProgress(force = false, pendingDirty = 0, pendingDeleted = 0) {
+    if (!onProgress) return;
+    const now = Date.now();
+    if (!force && now - lastProgressAt < progress_interval_ms) return;
+    const current = getRun(db, runId);
+    lastProgressAt = now;
+    if (current) {
+      const payload = buildDirtyProgressPayload(
+        current,
+        startedAtMs,
+        totalProcessed,
+        totalFetched,
+        pendingDirty,
+        pendingDeleted
+      );
+      Promise.resolve(onProgress(payload)).catch(() => {});
+    }
+  }
 
   for (;;) {
     let r = getRun(db, runId);
@@ -57,79 +124,146 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
     const deletedBatch = getDirtyBatch(db, runId, 'deleted', batch_keys);
     if (dirtyBatch.length === 0 && deletedBatch.length === 0) break;
 
-    const batchSize = dirtyBatch.length + deletedBatch.length;
+    totalFetched += dirtyBatch.length + deletedBatch.length;
+    let aborted = false;
 
     // ── Re-import (or remove) keys that changed while bulk was running ──
-    for (const { key: keyBuf } of dirtyBatch) {
+    for (let i = 0; i < dirtyBatch.length; i += workerCount) {
       r = getRun(db, runId);
-      if (r && r.status === RUN_STATUS.ABORTED) break;
+      if (r && r.status === RUN_STATUS.ABORTED) {
+        aborted = true;
+        break;
+      }
       while (r && r.status === RUN_STATUS.PAUSED) {
         await sleep(2000);
         r = getRun(db, runId);
       }
-
-      if (minIntervalMs > 0) {
-        const elapsed = Date.now() - lastKeyTime;
-        if (elapsed < minIntervalMs) await sleep(minIntervalMs - elapsed);
-        lastKeyTime = Date.now();
+      if (r && r.status === RUN_STATUS.ABORTED) {
+        aborted = true;
+        break;
       }
 
-      const keyName = keyBuf.toString('utf8');
-      try {
-        const type = (await redisClient.type(keyName)).toLowerCase();
-        if (type === 'none' || !type) {
-          keys.delete(keyBuf);
-          markDirtyState(db, runId, keyBuf, 'deleted');
-        } else {
-          const outcome = await importKeyFromRedis(redisClient, keyName, storages, {});
-          if (outcome.ok) {
-            markDirtyState(db, runId, keyBuf, 'applied');
-          } else if (outcome.skipped) {
-            markDirtyState(db, runId, keyBuf, 'skipped');
+      const chunk = dirtyBatch.slice(i, i + workerCount);
+      const results = await Promise.all(
+        chunk.map(async ({ key: keyBuf }) => {
+          const keyName = keyBuf.toString('utf8');
+          try {
+            await awaitRateLimit();
+            const type = (await redisClient.type(keyName)).toLowerCase();
+            if (type === 'none' || !type) {
+              return { keyBuf, keyName, state: 'deleted' };
+            }
+            const outcome = await importKeyFromRedis(redisClient, keyName, storages, {});
+            return { keyBuf, keyName, state: 'imported', outcome };
+          } catch (err) {
+            return { keyBuf, keyName, state: 'exception', error: err };
+          }
+        })
+      );
+
+      for (const result of results) {
+        try {
+          if (result.state === 'deleted') {
+            keys.delete(result.keyBuf);
+            markDirtyState(db, runId, result.keyBuf, 'deleted');
+          } else if (result.state === 'imported') {
+            if (result.outcome.ok) {
+              markDirtyState(db, runId, result.keyBuf, 'applied');
+            } else if (result.outcome.skipped) {
+              markDirtyState(db, runId, result.keyBuf, 'skipped');
+            } else {
+              logError(
+                db,
+                runId,
+                'dirty_apply',
+                result.outcome.error ? 'Import failed' : 'Skipped',
+                result.keyName
+              );
+              markDirtyState(db, runId, result.keyBuf, 'error');
+            }
           } else {
-            logError(db, runId, 'dirty_apply', outcome.error ? 'Import failed' : 'Skipped', keyName);
-            markDirtyState(db, runId, keyBuf, 'error');
+            logError(db, runId, 'dirty_apply', result.error.message, result.keyBuf);
+            markDirtyState(db, runId, result.keyBuf, 'error');
           }
+        } catch (err) {
+          logError(db, runId, 'dirty_apply', err.message, result.keyBuf);
+          markDirtyState(db, runId, result.keyBuf, 'error');
+        } finally {
+          totalProcessed++;
         }
-      } catch (err) {
-        logError(db, runId, 'dirty_apply', err.message, keyBuf);
-        markDirtyState(db, runId, keyBuf, 'error');
       }
+
+      emitProgress(false, Math.max(0, dirtyBatch.length - (i + chunk.length)), deletedBatch.length);
+    }
+
+    if (aborted) {
+      emitProgress(true, dirtyBatch.length, deletedBatch.length);
+      break;
     }
 
     // ── Apply deletions recorded by the tracker (del / expired events) ──
     // The tracker already determined these keys are gone; delete from destination.
     // Marked as 'deleted' in the run counter; state changed away from 'deleted'
     // so the next getDirtyBatch call won't return them again (avoiding infinite loop).
-    for (const { key: keyBuf } of deletedBatch) {
+    for (let i = 0; i < deletedBatch.length; i += workerCount) {
       r = getRun(db, runId);
-      if (r && r.status === RUN_STATUS.ABORTED) break;
+      if (r && r.status === RUN_STATUS.ABORTED) {
+        aborted = true;
+        break;
+      }
       while (r && r.status === RUN_STATUS.PAUSED) {
         await sleep(2000);
         r = getRun(db, runId);
       }
+      if (r && r.status === RUN_STATUS.ABORTED) {
+        aborted = true;
+        break;
+      }
 
-      try {
-        keys.delete(keyBuf);
-        // Increment dirty_keys_deleted counter and transition state out of 'deleted'
-        // so this key is not re-processed in the next batch iteration.
-        const now = Date.now();
-        db.prepare(
-          `UPDATE migration_dirty_keys SET state = 'applied', last_seen_at = ? WHERE run_id = ? AND key = ?`
-        ).run(now, runId, keyBuf);
-        db.prepare(
-          `UPDATE migration_runs SET dirty_keys_deleted = dirty_keys_deleted + 1, updated_at = ? WHERE run_id = ?`
-        ).run(now, runId);
-      } catch (err) {
-        logError(db, runId, 'dirty_apply', err.message, keyBuf);
-        markDirtyState(db, runId, keyBuf, 'error');
+      const chunk = deletedBatch.slice(i, i + workerCount);
+      for (const { key: keyBuf } of chunk) {
+        try {
+          keys.delete(keyBuf);
+          // Increment dirty_keys_deleted counter and transition state out of 'deleted'
+          // so this key is not re-processed in the next batch iteration.
+          const now = Date.now();
+          db.prepare(
+            `UPDATE migration_dirty_keys SET state = 'applied', last_seen_at = ? WHERE run_id = ? AND key = ?`
+          ).run(now, runId, keyBuf);
+          db.prepare(
+            `UPDATE migration_runs SET dirty_keys_deleted = dirty_keys_deleted + 1, updated_at = ? WHERE run_id = ?`
+          ).run(now, runId);
+        } catch (err) {
+          logError(db, runId, 'dirty_apply', err.message, keyBuf);
+          markDirtyState(db, runId, keyBuf, 'error');
+        } finally {
+          totalProcessed++;
+        }
       }
+      emitProgress(false, 0, Math.max(0, deletedBatch.length - (i + chunk.length)));
+      if (aborted) break;
     }
-    if (batchSize > 0 && onProgress) {
-      const run = getRun(db, runId);
-      if (run) Promise.resolve(onProgress(run)).catch(() => {});
+
+    if (aborted) {
+      emitProgress(true, dirtyBatch.length, deletedBatch.length);
+      break;
     }
+
+    const pendingDirty = db.prepare(
+      `SELECT COUNT(*) as n FROM migration_dirty_keys WHERE run_id = ? AND state = 'dirty'`
+    ).get(runId).n;
+    const pendingDeleted = db.prepare(
+      `SELECT COUNT(*) as n FROM migration_dirty_keys WHERE run_id = ? AND state = 'deleted'`
+    ).get(runId).n;
+    emitProgress(true, pendingDirty, pendingDeleted);
   }
 
+  const finalPendingDirty = db.prepare(
+    `SELECT COUNT(*) as n FROM migration_dirty_keys WHERE run_id = ? AND state = 'dirty'`
+  ).get(runId).n;
+  const finalPendingDeleted = db.prepare(
+    `SELECT COUNT(*) as n FROM migration_dirty_keys WHERE run_id = ? AND state = 'deleted'`
+  ).get(runId).n;
+  emitProgress(true, finalPendingDirty, finalPendingDeleted);
   return getRun(db, runId);
 }

package/src/migration/index.js

@@ -36,7 +36,7 @@ import { startDirtyTracker as startDirtyTrackerProcess } from './tracker.js';
  * @property {string} [pragmaTemplate='default']      - PRAGMA preset.
  * @property {number} [scanCount=1000]
  * @property {number} [maxRps=0]                      - Max requests/s (0 = unlimited).
- * @property {number} [concurrency=1]                 - Concurrent imports during bulk migration.
+ * @property {number} [concurrency=1]                 - Concurrent imports during bulk/apply-dirty migration.
  * @property {number} [estimatedTotalKeys=0]          - Optional total-keys estimate for ETA/progress in onProgress.
  * @property {number} [batchKeys=200]
  * @property {number} [batchBytes=67108864]           - 64 MB default.
@@ -54,7 +54,7 @@ import { startDirtyTracker as startDirtyTrackerProcess } from './tracker.js';
  *   stopDirtyTracker(): Promise<{ running: false }>,
  *   bulk(opts?: { resume?: boolean, onProgress?: function }): Promise<object>,
  *   status(): { run: object, dirty: object } | null,
- *   applyDirty(opts?: { batchKeys?: number, maxRps?: number, onProgress?: function }): Promise<object>,
+ *   applyDirty(opts?: { batchKeys?: number, maxRps?: number, concurrency?: number, progressIntervalMs?: number, onProgress?: function }): Promise<object>,
  *   verify(opts?: { samplePct?: number, maxSample?: number }): Promise<object>,
  *   migrateSearch(opts?: { onlyIndices?: string[], scanCount?: number, maxRps?: number, batchDocs?: number, maxSuggestions?: number, skipExisting?: boolean, withSuggestions?: boolean, onProgress?: function }): Promise<object>,
  *   close(): Promise<void>,
@@ -205,15 +205,23 @@ export function createMigration({
     /**
      * Step 3 — Apply dirty: reconcile keys that changed in Redis during bulk import.
      *
-     * @param {{ batchKeys?: number, maxRps?: number, onProgress?: (run: object) => void }} [opts]
+     * @param {{ batchKeys?: number, maxRps?: number, concurrency?: number, progressIntervalMs?: number, onProgress?: (run: object) => void }} [opts]
      */
-    async applyDirty({ batchKeys: bk = batchKeys, maxRps: rps = maxRps, onProgress } = {}) {
+    async applyDirty({
+      batchKeys: bk = batchKeys,
+      maxRps: rps = maxRps,
+      concurrency: c = concurrency,
+      progressIntervalMs: pim = 2000,
+      onProgress,
+    } = {}) {
       const id = requireRunId();
       const client = await getClient();
       return runApplyDirty(client, to, id, {
         pragmaTemplate,
         batch_keys: bk,
         max_rps: rps,
+        concurrency: c,
+        progress_interval_ms: pim,
         onProgress,
       });
     },

package/test/unit/migration-apply-dirty.test.js

@@ -0,0 +1,101 @@
+/**
+ * Unit tests for dirty apply concurrency/progress behavior.
+ */
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { openDb } from '../../src/storage/sqlite/db.js';
+import { runApplyDirty } from '../../src/migration/apply-dirty.js';
+import { createRun, upsertDirtyKey, getDirtyCounts } from '../../src/migration/registry.js';
+import { tmpDbPath } from '../helpers/tmp.js';
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+class FakeRedisStringClient {
+  constructor(initialValues, delayMs = 8) {
+    this.values = new Map(Object.entries(initialValues));
+    this.delayMs = delayMs;
+    this.inFlight = 0;
+    this.maxInFlight = 0;
+  }
+
+  async type(key) {
+    this.inFlight++;
+    this.maxInFlight = Math.max(this.maxInFlight, this.inFlight);
+    try {
+      await sleep(this.delayMs);
+      return this.values.has(key) ? 'string' : 'none';
+    } finally {
+      this.inFlight--;
+    }
+  }
+
+  async pTTL() {
+    this.inFlight++;
+    this.maxInFlight = Math.max(this.maxInFlight, this.inFlight);
+    try {
+      await sleep(this.delayMs);
+      return -1;
+    } finally {
+      this.inFlight--;
+    }
+  }
+
+  async get(key) {
+    this.inFlight++;
+    this.maxInFlight = Math.max(this.maxInFlight, this.inFlight);
+    try {
+      await sleep(this.delayMs);
+      return this.values.get(key) ?? null;
+    } finally {
+      this.inFlight--;
+    }
+  }
+}
+
+describe('migration apply-dirty', () => {
+  it('processes dirty keys with configured concurrency and emits progress payloads', async () => {
+    const dbPath = tmpDbPath();
+    const runId = `apply-dirty-concurrency-${Date.now()}`;
+    const totalKeys = 30;
+
+    const db = openDb(dbPath, { pragmaTemplate: 'minimal' });
+    createRun(db, runId, 'redis://x:6379');
+    const initialValues = {};
+    for (let i = 0; i < totalKeys; i++) {
+      const key = `k:${i}`;
+      initialValues[key] = `v:${i}`;
+      upsertDirtyKey(db, runId, key, 'set');
+    }
+    db.close();
+
+    const fakeRedis = new FakeRedisStringClient(initialValues);
+    const progress = [];
+
+    const run = await runApplyDirty(fakeRedis, dbPath, runId, {
+      pragmaTemplate: 'minimal',
+      batch_keys: totalKeys,
+      concurrency: 8,
+      progress_interval_ms: 0,
+      onProgress: (r) => progress.push(r),
+    });
+
+    assert.equal(run.dirty_keys_applied, totalKeys);
+    assert.equal(run.dirty_keys_deleted, 0);
+    assert.ok(fakeRedis.maxInFlight > 1, `Expected concurrent calls, maxInFlight=${fakeRedis.maxInFlight}`);
+    assert.ok(progress.length >= 1, 'Expected at least one onProgress callback');
+    const last = progress[progress.length - 1];
+    assert.equal(last.dirty_pending, 0);
+    assert.equal(last.dirty_reconciled_total, totalKeys);
+    assert.ok(Number.isFinite(last.dirty_keys_per_second));
+
+    const verifyDb = openDb(dbPath, { pragmaTemplate: 'minimal' });
+    const counts = getDirtyCounts(verifyDb, runId);
+    verifyDb.close();
+    assert.equal(counts.dirty, 0);
+    assert.equal(counts.deleted, 0);
+    assert.equal(counts.applied, totalKeys);
+  });
+});