resplite 1.3.8 → 1.4.0

package/README.md

@@ -141,7 +141,6 @@ await m.startDirtyTracker({
 // Step 1 — Bulk import (checkpointed, resumable). Same script to start or continue.
 // Use keyCountEstimate from preflight to compute ETA/progress during bulk.
 await m.bulk({
-  resume: true, 
   estimatedTotalKeys: info.keyCountEstimate,
   onProgress: (r) => {
     const pct = r.progress_pct != null ? r.progress_pct.toFixed(1) : '—';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "resplite",
-  "version": "1.3.8",
+  "version": "1.4.0",
   "description": "A RESP2 server with practical Redis compatibility, backed by SQLite",
   "type": "module",
   "main": "src/index.js",

package/src/migration/import-one.js

@@ -14,16 +14,35 @@ function toBuffer(value) {
   return Buffer.from(String(value), 'utf8');
 }
 
+function parseZscanResult(raw) {
+  if (!Array.isArray(raw) || raw.length < 2) {
+    return { cursor: 0, entries: [] };
+  }
+  const cursor = parseInt(String(raw[0] ?? '0'), 10) || 0;
+  const flat = Array.isArray(raw[1]) ? raw[1] : [];
+  const entries = [];
+  for (let i = 0; i < flat.length; i += 2) {
+    const member = flat[i];
+    const score = flat[i + 1];
+    if (member == null || score == null) continue;
+    entries.push({ value: member, score: Number(score) });
+  }
+  return { cursor, entries };
+}
+
 /**
  * Fetch one key from Redis and write to storages. Idempotent (upsert).
  * @param {import('redis').RedisClientType} redisClient
  * @param {string} keyName
  * @param {{ keys: import('../storage/sqlite/keys.js').ReturnType<import('../storage/sqlite/keys.js').createKeysStorage>; strings: ReturnType<import('../storage/sqlite/strings.js').createStringsStorage>; hashes: ReturnType<import('../storage/sqlite/hashes.js').createHashesStorage>; sets: ReturnType<import('../storage/sqlite/sets.js').createSetsStorage>; lists: ReturnType<import('../storage/sqlite/lists.js').createListsStorage>; zsets: ReturnType<import('../storage/sqlite/zsets.js').createZsetsStorage> }} storages
- * @param {{ now?: number }} options
+ * @param {{ now?: number, zsetScanCount?: number }} options
  * @returns {Promise<{ ok: boolean; skipped?: boolean; error?: boolean; bytes?: number }>}
  */
 export async function importKeyFromRedis(redisClient, keyName, storages, options = {}) {
   const now = options.now ?? Date.now();
+  const zsetScanCount = Number.isFinite(options.zsetScanCount)
+    ? Math.max(10, Math.floor(options.zsetScanCount))
+    : 1000;
   const { keys, strings, hashes, sets, lists, zsets } = storages;
 
   try {
@@ -85,16 +104,46 @@ export async function importKeyFromRedis(redisClient, keyName, storages, options
     }
 
     if (type === 'zset') {
-      const withScores = await redisClient.zRangeWithScores(keyName, 0, -1);
-      if (!withScores || !withScores.length) return { ok: false, skipped: true };
-      const pairs = withScores.map((item) => ({
-        member: toBuffer(item.value),
-        score: Number(item.score),
-      }));
-      for (const p of pairs) bytes += p.member.length + 8;
-      zsets.add(keyBuf, pairs, { updatedAt: now });
-      keys.setExpires(keyBuf, expiresAt, now);
-      return { ok: true, bytes };
+      try {
+        // Use cursor-based reads to avoid loading very large sorted sets in one call.
+        let cursor = 0;
+        let wroteAny = false;
+        do {
+          const raw = await redisClient.sendCommand([
+            'ZSCAN',
+            keyName,
+            String(cursor),
+            'COUNT',
+            String(zsetScanCount),
+          ]);
+          const parsed = parseZscanResult(raw);
+          cursor = parsed.cursor;
+          if (parsed.entries.length === 0) continue;
+          const pairs = parsed.entries.map((item) => ({
+            member: toBuffer(item.value),
+            score: Number(item.score),
+          }));
+          for (const p of pairs) bytes += p.member.length + 8;
+          zsets.add(keyBuf, pairs, { updatedAt: now });
+          wroteAny = true;
+        } while (cursor !== 0);
+
+        if (!wroteAny) return { ok: false, skipped: true };
+        keys.setExpires(keyBuf, expiresAt, now);
+        return { ok: true, bytes };
+      } catch {
+        // Fallback for clients/backends without command passthrough support.
+        const withScores = await redisClient.zRangeWithScores(keyName, 0, -1);
+        if (!withScores || !withScores.length) return { ok: false, skipped: true };
+        const pairs = withScores.map((item) => ({
+          member: toBuffer(item.value),
+          score: Number(item.score),
+        }));
+        for (const p of pairs) bytes += p.member.length + 8;
+        zsets.add(keyBuf, pairs, { updatedAt: now });
+        keys.setExpires(keyBuf, expiresAt, now);
+        return { ok: true, bytes };
+      }
     }
 
     return { ok: false, skipped: true };

package/test/unit/migration-import-one.test.js

@@ -0,0 +1,94 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { importKeyFromRedis } from '../../src/migration/import-one.js';
+
+function makeStorages() {
+  const calls = {
+    zsetAdds: [],
+    setExpires: [],
+  };
+  return {
+    calls,
+    storages: {
+      keys: {
+        setExpires(key, expiresAt, updatedAt) {
+          calls.setExpires.push({ key, expiresAt, updatedAt });
+        },
+      },
+      strings: {},
+      hashes: {},
+      sets: {},
+      lists: {},
+      zsets: {
+        add(key, pairs) {
+          calls.zsetAdds.push({ key, pairs });
+        },
+      },
+    },
+  };
+}
+
+describe('importKeyFromRedis zset handling', () => {
+  it('imports large zsets with ZSCAN chunks', async () => {
+    const { storages, calls } = makeStorages();
+    let scanCalls = 0;
+    const redis = {
+      async type() {
+        return 'zset';
+      },
+      async pTTL() {
+        return -1;
+      },
+      async sendCommand(argv) {
+        assert.equal(argv[0], 'ZSCAN');
+        scanCalls += 1;
+        if (scanCalls === 1) return ['7', ['a', '1', 'b', '2']];
+        if (scanCalls === 2) return ['0', ['c', '3']];
+        return ['0', []];
+      },
+      async zRangeWithScores() {
+        throw new Error('fallback should not be used when ZSCAN works');
+      },
+    };
+
+    const result = await importKeyFromRedis(redis, 'big:zset', storages, { now: 1000, zsetScanCount: 2 });
+
+    assert.equal(result.ok, true);
+    assert.equal(result.skipped, undefined);
+    assert.equal(scanCalls, 2);
+    assert.equal(calls.zsetAdds.length, 2);
+    assert.equal(calls.zsetAdds[0].pairs.length, 2);
+    assert.equal(calls.zsetAdds[1].pairs.length, 1);
+    assert.equal(calls.setExpires.length, 1);
+    // key bytes + member bytes + score metadata estimate (8 bytes/member)
+    assert.equal(result.bytes, Buffer.byteLength('big:zset') + (1 + 1 + 1) + (3 * 8));
+  });
+
+  it('falls back to zRangeWithScores when ZSCAN passthrough is unavailable', async () => {
+    const { storages, calls } = makeStorages();
+    let fallbackUsed = false;
+    const redis = {
+      async type() {
+        return 'zset';
+      },
+      async pTTL() {
+        return -1;
+      },
+      async sendCommand() {
+        throw new Error('sendCommand not supported');
+      },
+      async zRangeWithScores() {
+        fallbackUsed = true;
+        return [{ value: 'member-1', score: 42 }];
+      },
+    };
+
+    const result = await importKeyFromRedis(redis, 'legacy:zset', storages, { now: 1000 });
+
+    assert.equal(result.ok, true);
+    assert.equal(fallbackUsed, true);
+    assert.equal(calls.zsetAdds.length, 1);
+    assert.equal(calls.zsetAdds[0].pairs.length, 1);
+    assert.equal(calls.setExpires.length, 1);
+  });
+});