resplite 1.3.6 → 1.3.8

package/README.md

@@ -107,6 +107,8 @@ const m = createMigration({
   batchKeys:      1000,
   batchBytes:     64 * 1024 * 1024,  // 64 MB
   maxRps:         0,                  // 0 = unlimited
+  concurrency:    8,                  // parallel key imports during bulk
+  // estimatedTotalKeys: info.keyCountEstimate, // optional ETA baseline (can also be set per bulk call)
 
   // If your Redis deployment renamed CONFIG for security:
   // configCommand: 'MYCONFIG',
@@ -137,14 +139,15 @@ await m.startDirtyTracker({
 });
 
 // Step 1 — Bulk import (checkpointed, resumable). Same script to start or continue.
-// Use keyCountEstimate from preflight to show progress % (estimate; actual count may change).
-const total = info.keyCountEstimate || 1;
+// Use keyCountEstimate from preflight to compute ETA/progress during bulk.
 await m.bulk({
   resume: true, 
+  estimatedTotalKeys: info.keyCountEstimate,
   onProgress: (r) => {
-    const pct = total ? ((r.scanned_keys / total) * 100).toFixed(1) : '—';
+    const pct = r.progress_pct != null ? r.progress_pct.toFixed(1) : '—';
+    const eta = r.eta_seconds != null ? `${r.eta_seconds}s` : '—';
     console.log(
-      `scanned=${r.scanned_keys} migrated=${r.migrated_keys} errors=${r.error_keys} progress=${pct}%`
+      `scanned=${r.scanned_keys} migrated=${r.migrated_keys} errors=${r.error_keys} progress=${pct}% eta=${eta} rate=${r.keys_per_second.toFixed(1)} keys/s`
     );
   },
 });

package/docs/generated/migration-bulk-concurrency.md

@@ -0,0 +1,7 @@
+---
+id: tucj9i5nh5
+type: implementation
+title: Bulk migration concurrency added
+created: '2026-03-11 11:09:20'
+---
+Added configurable concurrency to runBulkImport and createMigration.bulk with default 1. Implemented chunked parallel import with shared global max_rps limiter. Added unit tests proving default sequential behavior and concurrent behavior with cap.

package/docs/generated/migration-bulk-eta-onprogress.md

@@ -0,0 +1,7 @@
+---
+id: 105jsp012x
+type: implementation
+title: Bulk onProgress ETA support
+created: '2026-03-11 11:10:48'
+---
+Added ETA/progress metrics to bulk migration onProgress payload. New optional options: estimated_total_keys in runBulkImport, estimatedTotalKeys in createMigration/bulk(). onProgress payload now includes elapsed_seconds, keys_per_second, estimated_total_keys, remaining_keys_estimate, eta_seconds, progress_pct. README migration example updated to print ETA/rate. Added unit test validating ETA fields and final 100%/eta=0 behavior.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "resplite",
-  "version": "1.3.6",
+  "version": "1.3.8",
   "description": "A RESP2 server with practical Redis compatibility, backed by SQLite",
   "type": "module",
   "main": "src/index.js",

package/src/migration/bulk.js

@@ -35,6 +35,27 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+function buildProgressPayload(run, startTimeMs, estimatedTotalKeys) {
+  if (!run) return null;
+  const scanned = Number(run.scanned_keys || 0);
+  const elapsedSec = Math.max(0.001, (Date.now() - startTimeMs) / 1000);
+  const keysPerSec = scanned / elapsedSec;
+  const hasEstimate = Number.isFinite(estimatedTotalKeys) && estimatedTotalKeys > 0;
+  const remainingKeys = hasEstimate ? Math.max(0, estimatedTotalKeys - scanned) : null;
+  const etaSeconds = hasEstimate && keysPerSec > 0 ? Math.ceil(remainingKeys / keysPerSec) : null;
+  const progressPct = hasEstimate ? Math.min(100, (scanned / estimatedTotalKeys) * 100) : null;
+
+  return {
+    ...run,
+    elapsed_seconds: elapsedSec,
+    keys_per_second: keysPerSec,
+    estimated_total_keys: hasEstimate ? estimatedTotalKeys : null,
+    remaining_keys_estimate: remainingKeys,
+    eta_seconds: etaSeconds,
+    progress_pct: progressPct,
+  };
+}
+
 /**
  * Run bulk import: SCAN keys from Redis, import into RespLite DB with checkpointing.
  * On SIGINT/SIGTERM, checkpoint progress, set run status to ABORTED, close DB and rethrow.
@@ -48,6 +69,8 @@ function sleep(ms) {
  * @param {string} [options.pragmaTemplate='default']
  * @param {number} [options.scan_count=1000]
  * @param {number} [options.max_rps=0] - 0 = no limit
+ * @param {number} [options.concurrency=1] - Number of concurrent key imports
+ * @param {number} [options.estimated_total_keys=0] - Optional key count estimate used for ETA/progress
  * @param {number} [options.batch_keys=200]
  * @param {number} [options.batch_bytes=64*1024*1024] - 64MB
  * @param {number} [options.checkpoint_interval_sec=30]
@@ -60,6 +83,8 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
     pragmaTemplate = 'default',
     scan_count = 1000,
     max_rps = 0,
+    concurrency = 1,
+    estimated_total_keys = 0,
     batch_keys = 200,
     batch_bytes = 64 * 1024 * 1024,
     checkpoint_interval_sec = 30,
@@ -100,10 +125,22 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
     }
 
     let lastCheckpointTime = Date.now();
+    const startedAtMs = lastCheckpointTime;
     let batchScanned = 0;
     let batchBytes = 0;
     const minIntervalMs = max_rps > 0 ? 1000 / max_rps : 0;
-    let lastKeyTime = 0;
+    const workerCount = Number.isFinite(concurrency) ? Math.max(1, Math.floor(concurrency)) : 1;
+    let nextAllowedAt = 0;
+
+    async function awaitRateLimit() {
+      if (minIntervalMs <= 0) return;
+      const now = Date.now();
+      const scheduled = Math.max(now, nextAllowedAt);
+      nextAllowedAt = scheduled + minIntervalMs;
+      if (scheduled > now) {
+        await sleep(scheduled - now);
+      }
+    }
 
     outer: do {
       run = getRun(db, runId);
@@ -119,7 +156,7 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
       cursor = parsed.cursor;
       const keyList = parsed.keys || [];
 
-      for (const keyName of keyList) {
+      for (let i = 0; i < keyList.length; i += workerCount) {
         if (abortRequested) break outer;
         run = getRun(db, runId);
         if (run && run.status === RUN_STATUS.ABORTED) break outer;
@@ -128,46 +165,50 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
           run = getRun(db, runId);
         }
 
-        scanned_keys++;
-        if (minIntervalMs > 0) {
-          const elapsed = Date.now() - lastKeyTime;
-          if (elapsed < minIntervalMs) await sleep(minIntervalMs - elapsed);
-          lastKeyTime = Date.now();
-        }
+        const chunk = keyList.slice(i, i + workerCount);
+        const results = await Promise.all(
+          chunk.map(async (keyName) => {
+            await awaitRateLimit();
+            const now = Date.now();
+            const outcome = await importKeyFromRedis(redisClient, keyName, storages, { now });
+            return { keyName, outcome };
+          })
+        );
 
-        const now = Date.now();
-        const outcome = await importKeyFromRedis(redisClient, keyName, storages, { now });
-        if (outcome.ok) {
-          migrated_keys++;
-          migrated_bytes += outcome.bytes || 0;
-          batchScanned++;
-          batchBytes += outcome.bytes || 0;
-        } else if (outcome.skipped) {
-          skipped_keys++;
-        } else {
-          error_keys++;
-          logError(db, runId, 'bulk', outcome.error ? 'Import failed' : 'Skipped', keyName);
-        }
+        for (const { keyName, outcome } of results) {
+          scanned_keys++;
+          if (outcome.ok) {
+            migrated_keys++;
+            migrated_bytes += outcome.bytes || 0;
+            batchScanned++;
+            batchBytes += outcome.bytes || 0;
+          } else if (outcome.skipped) {
+            skipped_keys++;
+          } else {
+            error_keys++;
+            logError(db, runId, 'bulk', outcome.error ? 'Import failed' : 'Skipped', keyName);
+          }
 
-        const now2 = Date.now();
-        const shouldCheckpoint =
-          batchScanned >= batch_keys ||
-          batchBytes >= batch_bytes ||
-          now2 - lastCheckpointTime >= checkpoint_interval_sec * 1000;
-        if (shouldCheckpoint) {
-          updateBulkProgress(db, runId, {
-            scan_cursor: String(cursor),
-            scanned_keys,
-            migrated_keys,
-            skipped_keys,
-            error_keys,
-            migrated_bytes,
-          });
-          lastCheckpointTime = now2;
-          batchScanned = 0;
-          batchBytes = 0;
-          run = getRun(db, runId);
-          if (onProgress && run) onProgress(run);
+          const now2 = Date.now();
+          const shouldCheckpoint =
+            batchScanned >= batch_keys ||
+            batchBytes >= batch_bytes ||
+            now2 - lastCheckpointTime >= checkpoint_interval_sec * 1000;
+          if (shouldCheckpoint) {
+            updateBulkProgress(db, runId, {
+              scan_cursor: String(cursor),
+              scanned_keys,
+              migrated_keys,
+              skipped_keys,
+              error_keys,
+              migrated_bytes,
+            });
+            lastCheckpointTime = now2;
+            batchScanned = 0;
+            batchBytes = 0;
+            run = getRun(db, runId);
+            if (onProgress && run) onProgress(buildProgressPayload(run, startedAtMs, estimated_total_keys));
+          }
         }
       }
     } while (cursor !== 0);
@@ -183,7 +224,7 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
       });
       setRunStatus(db, runId, RUN_STATUS.ABORTED);
       run = getRun(db, runId);
-      if (onProgress && run) onProgress(run);
+      if (onProgress && run) onProgress(buildProgressPayload(run, startedAtMs, estimated_total_keys));
       const err = new Error('Bulk import interrupted by signal (SIGINT/SIGTERM)');
       err.code = 'BULK_ABORTED';
       throw err;
@@ -198,7 +239,9 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
       migrated_bytes,
     });
     setRunStatus(db, runId, RUN_STATUS.COMPLETED);
-    return getRun(db, runId);
+    run = getRun(db, runId);
+    if (onProgress && run) onProgress(buildProgressPayload(run, startedAtMs, estimated_total_keys));
+    return run;
   } catch (err) {
     if (err.code !== 'BULK_ABORTED') {
       setRunStatus(db, runId, RUN_STATUS.FAILED);

package/src/migration/index.js

@@ -36,6 +36,8 @@ import { startDirtyTracker as startDirtyTrackerProcess } from './tracker.js';
  * @property {string} [pragmaTemplate='default']      - PRAGMA preset.
  * @property {number} [scanCount=1000]
  * @property {number} [maxRps=0]                      - Max requests/s (0 = unlimited).
+ * @property {number} [concurrency=1]                 - Concurrent imports during bulk migration.
+ * @property {number} [estimatedTotalKeys=0]          - Optional total-keys estimate for ETA/progress in onProgress.
  * @property {number} [batchKeys=200]
  * @property {number} [batchBytes=67108864]           - 64 MB default.
  * @property {string} [configCommand='CONFIG']        - Redis CONFIG command name. Override if renamed for security.
@@ -65,6 +67,8 @@ export function createMigration({
   pragmaTemplate = 'default',
   scanCount = 1000,
   maxRps = 0,
+  concurrency = 1,
+  estimatedTotalKeys = 0,
   batchKeys = 200,
   batchBytes = 64 * 1024 * 1024,
   configCommand = 'CONFIG',
@@ -156,9 +160,17 @@ export function createMigration({
      * Step 1 — Bulk import: SCAN all keys from Redis into the destination DB.
      * Resume is on by default: first run starts from 0, later runs continue from checkpoint.
      *
-     * @param {{ resume?: boolean, onProgress?: (run: object) => void }} [opts] - resume (default true): start or continue automatically
+     * @param {{ resume?: boolean, concurrency?: number, estimatedTotalKeys?: number, onProgress?: (run: object) => void }} [opts]
+     *   - `resume` (default true): start or continue automatically
+     *   - `concurrency` (default from createMigration options): concurrent key imports
+     *   - `estimatedTotalKeys` (optional): used to compute ETA/progress fields in onProgress
      */
-    async bulk({ resume = true, onProgress } = {}) {
+    async bulk({
+      resume = true,
+      concurrency: c = concurrency,
+      estimatedTotalKeys: et = estimatedTotalKeys,
+      onProgress,
+    } = {}) {
       const id = requireRunId();
       const client = await getClient();
       return runBulkImport(client, to, id, {
@@ -166,6 +178,8 @@ export function createMigration({
         pragmaTemplate,
         scan_count: scanCount,
         max_rps: maxRps,
+        concurrency: c,
+        estimated_total_keys: et,
         batch_keys: batchKeys,
         batch_bytes: batchBytes,
         resume,

package/test/unit/migration-bulk.test.js

@@ -0,0 +1,145 @@
+/**
+ * Unit tests for bulk migration concurrency behavior.
+ */
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { runBulkImport } from '../../src/migration/bulk.js';
+import { tmpDbPath } from '../helpers/tmp.js';
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function makeFakeRedis(valuesByKey, options = {}) {
+  const { getDelayMs = 0 } = options;
+  const keys = Object.keys(valuesByKey);
+
+  let inFlightGets = 0;
+  let maxInFlightGets = 0;
+
+  return {
+    stats: {
+      get maxInFlightGets() {
+        return maxInFlightGets;
+      },
+    },
+
+    async scan(cursor) {
+      if (Number(cursor) !== 0) return { cursor: 0, keys: [] };
+      return { cursor: 0, keys };
+    },
+
+    async type(keyName) {
+      return Object.prototype.hasOwnProperty.call(valuesByKey, keyName) ? 'string' : 'none';
+    },
+
+    async pTTL() {
+      return -1;
+    },
+
+    async get(keyName) {
+      inFlightGets++;
+      maxInFlightGets = Math.max(maxInFlightGets, inFlightGets);
+      try {
+        if (getDelayMs > 0) await sleep(getDelayMs);
+        return valuesByKey[keyName];
+      } finally {
+        inFlightGets--;
+      }
+    },
+  };
+}
+
+describe('runBulkImport concurrency', () => {
+  it('uses sequential processing by default (concurrency=1)', async () => {
+    const redis = makeFakeRedis(
+      {
+        k1: 'v1',
+        k2: 'v2',
+        k3: 'v3',
+        k4: 'v4',
+      },
+      { getDelayMs: 10 }
+    );
+
+    const run = await runBulkImport(redis, tmpDbPath(), `bulk-seq-${Date.now()}`, {
+      sourceUri: 'redis://fake',
+      scan_count: 100,
+      batch_keys: 1,
+    });
+
+    assert.equal(run.status, 'completed');
+    assert.equal(run.scanned_keys, 4);
+    assert.equal(run.migrated_keys, 4);
+    assert.equal(redis.stats.maxInFlightGets, 1);
+  });
+
+  it('processes keys concurrently when concurrency is configured', async () => {
+    const redis = makeFakeRedis(
+      {
+        k1: 'v1',
+        k2: 'v2',
+        k3: 'v3',
+        k4: 'v4',
+        k5: 'v5',
+        k6: 'v6',
+        k7: 'v7',
+        k8: 'v8',
+      },
+      { getDelayMs: 20 }
+    );
+
+    const run = await runBulkImport(redis, tmpDbPath(), `bulk-concurrent-${Date.now()}`, {
+      sourceUri: 'redis://fake',
+      scan_count: 100,
+      concurrency: 4,
+      batch_keys: 2,
+    });
+
+    assert.equal(run.status, 'completed');
+    assert.equal(run.scanned_keys, 8);
+    assert.equal(run.migrated_keys, 8);
+    assert.ok(redis.stats.maxInFlightGets > 1, `expected >1 inflight gets, got ${redis.stats.maxInFlightGets}`);
+    assert.ok(redis.stats.maxInFlightGets <= 4, `expected <=4 inflight gets, got ${redis.stats.maxInFlightGets}`);
+  });
+
+  it('includes ETA/progress fields in onProgress when total estimate is provided', async () => {
+    const redis = makeFakeRedis(
+      {
+        k1: 'v1',
+        k2: 'v2',
+        k3: 'v3',
+        k4: 'v4',
+      },
+      { getDelayMs: 8 }
+    );
+
+    const events = [];
+    const run = await runBulkImport(redis, tmpDbPath(), `bulk-eta-${Date.now()}`, {
+      sourceUri: 'redis://fake',
+      scan_count: 100,
+      batch_keys: 1,
+      estimated_total_keys: 4,
+      onProgress: (r) => events.push(r),
+    });
+
+    assert.equal(run.status, 'completed');
+    assert.ok(events.length >= 2, `expected at least 2 progress events, got ${events.length}`);
+
+    const withEta = events.filter((e) => e.eta_seconds !== null);
+    assert.ok(withEta.length >= 1, 'expected at least one progress event with eta_seconds');
+
+    for (const e of withEta) {
+      assert.equal(e.estimated_total_keys, 4);
+      assert.ok(e.progress_pct >= 0 && e.progress_pct <= 100, `invalid progress_pct=${e.progress_pct}`);
+      assert.ok(e.keys_per_second > 0, `invalid keys_per_second=${e.keys_per_second}`);
+      assert.ok(e.elapsed_seconds > 0, `invalid elapsed_seconds=${e.elapsed_seconds}`);
+    }
+
+    const last = events.at(-1);
+    assert.equal(last.progress_pct, 100);
+    assert.equal(last.eta_seconds, 0);
+    assert.equal(last.remaining_keys_estimate, 0);
+  });
+});