resplite 1.0.0 → 1.0.4

package/src/engine/engine.js

@@ -9,6 +9,7 @@ import { createHashesStorage } from '../storage/sqlite/hashes.js';
 import { createSetsStorage } from '../storage/sqlite/sets.js';
 import { createListsStorage } from '../storage/sqlite/lists.js';
 import { createZsetsStorage } from '../storage/sqlite/zsets.js';
+import { createBlockingManager } from '../blocking/manager.js';
 import { expectString, expectHash, expectSet, expectList, expectZset, typeName } from './validate.js';
 import { asKey, asValue } from '../util/buffers.js';
 
@@ -45,7 +46,7 @@ export function createEngine(opts = {}) {
     return meta;
   }
 
-  return {
+  const engine = {
     get(key) {
       const k = asKey(key);
       const meta = getKeyMeta(key);
@@ -262,14 +263,18 @@ export function createEngine(opts = {}) {
       const k = asKey(key);
       getKeyMeta(key);
       const buf = values.map((v) => (Buffer.isBuffer(v) ? v : asValue(v)));
-      return lists.lpush(k, buf);
+      const n = lists.lpush(k, buf);
+      if (this._blockingManager) this._blockingManager.wakeup(k);
+      return n;
     },
 
     rpush(key, ...values) {
       const k = asKey(key);
       getKeyMeta(key);
       const buf = values.map((v) => (Buffer.isBuffer(v) ? v : asValue(v)));
-      return lists.rpush(k, buf);
+      const n = lists.rpush(k, buf);
+      if (this._blockingManager) this._blockingManager.wakeup(k);
+      return n;
     },
 
     llen(key) {
@@ -398,7 +403,10 @@ export function createEngine(opts = {}) {
     _lists: lists,
     _zsets: zsets,
     _clock: clock,
+    _blockingManager: null,
   };
+  engine._blockingManager = createBlockingManager(engine, { clock });
+  return engine;
 }
 
 export { KEY_TYPES };

package/src/migration/apply-dirty.js

@@ -0,0 +1,97 @@
+/**
+ * Delta apply: reimport or delete keys from dirty registry (SPEC_F §F.8).
+ */
+
+import { openDb } from '../storage/sqlite/db.js';
+import { createKeysStorage } from '../storage/sqlite/keys.js';
+import { createStringsStorage } from '../storage/sqlite/strings.js';
+import { createHashesStorage } from '../storage/sqlite/hashes.js';
+import { createSetsStorage } from '../storage/sqlite/sets.js';
+import { createListsStorage } from '../storage/sqlite/lists.js';
+import { createZsetsStorage } from '../storage/sqlite/zsets.js';
+import { getRun, getDirtyBatch, markDirtyState, logError, RUN_STATUS } from './registry.js';
+import { importKeyFromRedis } from './import-one.js';
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Apply dirty keys: for each key in registry with state=dirty, reimport from Redis or delete in destination.
+ * @param {import('redis').RedisClientType} redisClient
+ * @param {string} dbPath
+ * @param {string} runId
+ * @param {object} options
+ * @param {string} [options.pragmaTemplate='default']
+ * @param {number} [options.batch_keys=200]
+ * @param {number} [options.max_rps=0]
+ */
+export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
+  const { pragmaTemplate = 'default', batch_keys = 200, max_rps = 0 } = options;
+
+  const db = openDb(dbPath, { pragmaTemplate });
+  const run = getRun(db, runId);
+  if (!run) throw new Error(`Run ${runId} not found`);
+
+  const keys = createKeysStorage(db);
+  const strings = createStringsStorage(db, keys);
+  const hashes = createHashesStorage(db, keys);
+  const sets = createSetsStorage(db, keys);
+  const lists = createListsStorage(db, keys);
+  const zsets = createZsetsStorage(db, keys);
+  const storages = { keys, strings, hashes, sets, lists, zsets };
+
+  const minIntervalMs = max_rps > 0 ? 1000 / max_rps : 0;
+  let lastKeyTime = 0;
+
+  for (;;) {
+    let r = getRun(db, runId);
+    if (r && r.status === RUN_STATUS.ABORTED) break;
+    while (r && r.status === RUN_STATUS.PAUSED) {
+      await sleep(2000);
+      r = getRun(db, runId);
+    }
+
+    const batch = getDirtyBatch(db, runId, 'dirty', batch_keys);
+    if (batch.length === 0) break;
+
+    for (const { key: keyBuf } of batch) {
+      r = getRun(db, runId);
+      if (r && r.status === RUN_STATUS.ABORTED) break;
+      while (r && r.status === RUN_STATUS.PAUSED) {
+        await sleep(2000);
+        r = getRun(db, runId);
+      }
+
+      if (minIntervalMs > 0) {
+        const elapsed = Date.now() - lastKeyTime;
+        if (elapsed < minIntervalMs) await sleep(minIntervalMs - elapsed);
+        lastKeyTime = Date.now();
+      }
+
+      const keyName = keyBuf.toString('utf8');
+      try {
+        const type = (await redisClient.type(keyName)).toLowerCase();
+        if (type === 'none' || !type) {
+          keys.delete(keyBuf);
+          markDirtyState(db, runId, keyBuf, 'deleted');
+        } else {
+          const outcome = await importKeyFromRedis(redisClient, keyName, storages, {});
+          if (outcome.ok) {
+            markDirtyState(db, runId, keyBuf, 'applied');
+          } else if (outcome.skipped) {
+            markDirtyState(db, runId, keyBuf, 'skipped');
+          } else {
+            logError(db, runId, 'dirty_apply', outcome.error ? 'Import failed' : 'Skipped', keyName);
+            markDirtyState(db, runId, keyBuf, 'error');
+          }
+        }
+      } catch (err) {
+        logError(db, runId, 'dirty_apply', err.message, keyBuf);
+        markDirtyState(db, runId, keyBuf, 'error');
+      }
+    }
+  }
+
+  return getRun(db, runId);
+}

package/src/migration/bulk.js

@@ -0,0 +1,181 @@
+/**
+ * Bulk import from Redis with checkpointing, resume, and throttling (SPEC_F §F.7).
+ */
+
+import { openDb } from '../storage/sqlite/db.js';
+import { createKeysStorage } from '../storage/sqlite/keys.js';
+import { createStringsStorage } from '../storage/sqlite/strings.js';
+import { createHashesStorage } from '../storage/sqlite/hashes.js';
+import { createSetsStorage } from '../storage/sqlite/sets.js';
+import { createListsStorage } from '../storage/sqlite/lists.js';
+import { createZsetsStorage } from '../storage/sqlite/zsets.js';
+import {
+  createRun,
+  getRun,
+  updateBulkProgress,
+  setRunStatus,
+  logError,
+  RUN_STATUS,
+} from './registry.js';
+import { importKeyFromRedis } from './import-one.js';
+
+function parseScanResult(result) {
+  if (Array.isArray(result)) {
+    return { cursor: parseInt(result[0], 10), keys: result[1] || [] };
+  }
+  if (result && typeof result === 'object') {
+    const cursor = typeof result.cursor === 'number' ? result.cursor : parseInt(String(result.cursor), 10);
+    const keys = result.keys || [];
+    return { cursor, keys };
+  }
+  return { cursor: 0, keys: [] };
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Run bulk import: SCAN keys from Redis, import into RespLite DB with checkpointing.
+ * @param {import('redis').RedisClientType} redisClient
+ * @param {string} dbPath
+ * @param {string} runId
+ * @param {object} options
+ * @param {string} options.sourceUri
+ * @param {string} [options.pragmaTemplate='default']
+ * @param {number} [options.scan_count=1000]
+ * @param {number} [options.max_rps=0] - 0 = no limit
+ * @param {number} [options.batch_keys=200]
+ * @param {number} [options.batch_bytes=64*1024*1024] - 64MB
+ * @param {number} [options.checkpoint_interval_sec=30]
+ * @param {boolean} [options.resume=false]
+ * @param {function(run): void} [options.onProgress] - called after checkpoint with run row
+ */
+export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
+  const {
+    sourceUri,
+    pragmaTemplate = 'default',
+    scan_count = 1000,
+    max_rps = 0,
+    batch_keys = 200,
+    batch_bytes = 64 * 1024 * 1024,
+    checkpoint_interval_sec = 30,
+    resume = false,
+    onProgress,
+  } = options;
+
+  const db = openDb(dbPath, { pragmaTemplate });
+  const keys = createKeysStorage(db);
+  const strings = createStringsStorage(db, keys);
+  const hashes = createHashesStorage(db, keys);
+  const sets = createSetsStorage(db, keys);
+  const lists = createListsStorage(db, keys);
+  const zsets = createZsetsStorage(db, keys);
+  const storages = { keys, strings, hashes, sets, lists, zsets };
+
+  createRun(db, runId, sourceUri, { scan_count_hint: scan_count });
+  let run = getRun(db, runId);
+  if (!run) throw new Error(`Run ${runId} not found`);
+
+  let cursor = resume && run.scan_cursor !== undefined ? parseInt(String(run.scan_cursor), 10) : 0;
+  let scanned_keys = resume ? (run.scanned_keys || 0) : 0;
+  let migrated_keys = resume ? (run.migrated_keys || 0) : 0;
+  let skipped_keys = resume ? (run.skipped_keys || 0) : 0;
+  let error_keys = resume ? (run.error_keys || 0) : 0;
+  let migrated_bytes = resume ? (run.migrated_bytes || 0) : 0;
+
+  if (!resume) {
+    updateBulkProgress(db, runId, { scan_cursor: String(cursor), scanned_keys, migrated_keys, skipped_keys, error_keys, migrated_bytes });
+  }
+
+  let lastCheckpointTime = Date.now();
+  let batchScanned = 0;
+  let batchBytes = 0;
+  const minIntervalMs = max_rps > 0 ? 1000 / max_rps : 0;
+  let lastKeyTime = 0;
+
+  try {
+    do {
+      run = getRun(db, runId);
+      if (run && run.status === RUN_STATUS.ABORTED) {
+        break;
+      }
+      while (run && run.status === RUN_STATUS.PAUSED) {
+        await sleep(2000);
+        run = getRun(db, runId);
+      }
+
+      const result = await redisClient.scan(cursor, { COUNT: scan_count });
+      const parsed = parseScanResult(result);
+      cursor = parsed.cursor;
+      const keyList = parsed.keys || [];
+
+      for (const keyName of keyList) {
+        run = getRun(db, runId);
+        if (run && run.status === RUN_STATUS.ABORTED) break;
+        while (run && run.status === RUN_STATUS.PAUSED) {
+          await sleep(2000);
+          run = getRun(db, runId);
+        }
+
+        scanned_keys++;
+        if (minIntervalMs > 0) {
+          const elapsed = Date.now() - lastKeyTime;
+          if (elapsed < minIntervalMs) await sleep(minIntervalMs - elapsed);
+          lastKeyTime = Date.now();
+        }
+
+        const now = Date.now();
+        const outcome = await importKeyFromRedis(redisClient, keyName, storages, { now });
+        if (outcome.ok) {
+          migrated_keys++;
+          migrated_bytes += outcome.bytes || 0;
+          batchScanned++;
+          batchBytes += outcome.bytes || 0;
+        } else if (outcome.skipped) {
+          skipped_keys++;
+        } else {
+          error_keys++;
+          logError(db, runId, 'bulk', outcome.error ? 'Import failed' : 'Skipped', keyName);
+        }
+
+        const now2 = Date.now();
+        const shouldCheckpoint =
+          batchScanned >= batch_keys ||
+          batchBytes >= batch_bytes ||
+          now2 - lastCheckpointTime >= checkpoint_interval_sec * 1000;
+        if (shouldCheckpoint) {
+          updateBulkProgress(db, runId, {
+            scan_cursor: String(cursor),
+            scanned_keys,
+            migrated_keys,
+            skipped_keys,
+            error_keys,
+            migrated_bytes,
+          });
+          lastCheckpointTime = now2;
+          batchScanned = 0;
+          batchBytes = 0;
+          run = getRun(db, runId);
+          if (onProgress && run) onProgress(run);
+        }
+      }
+    } while (cursor !== 0);
+
+    updateBulkProgress(db, runId, {
+      scan_cursor: '0',
+      scanned_keys,
+      migrated_keys,
+      skipped_keys,
+      error_keys,
+      migrated_bytes,
+    });
+    setRunStatus(db, runId, RUN_STATUS.COMPLETED);
+    return getRun(db, runId);
+  } catch (err) {
+    setRunStatus(db, runId, RUN_STATUS.FAILED);
+    updateBulkProgress(db, runId, { last_error: err.message });
+    logError(db, runId, 'bulk', err.message, null);
+    throw err;
+  }
+}

package/src/migration/import-one.js

@@ -0,0 +1,106 @@
+/**
+ * Import a single key from Redis into RespLite storages (shared by bulk and delta apply).
+ * SPEC_F §F.7.1, F.8.2.
+ */
+
+import { asKey, asValue } from '../util/buffers.js';
+
+const SUPPORTED_TYPES = new Set(['string', 'hash', 'set', 'list', 'zset']);
+
+function toBuffer(value) {
+  if (value == null) return null;
+  if (Buffer.isBuffer(value)) return value;
+  if (typeof value === 'string') return Buffer.from(value, 'utf8');
+  return Buffer.from(String(value), 'utf8');
+}
+
+/**
+ * Fetch one key from Redis and write to storages. Idempotent (upsert).
+ * @param {import('redis').RedisClientType} redisClient
+ * @param {string} keyName
+ * @param {{ keys: import('../storage/sqlite/keys.js').ReturnType<import('../storage/sqlite/keys.js').createKeysStorage>; strings: ReturnType<import('../storage/sqlite/strings.js').createStringsStorage>; hashes: ReturnType<import('../storage/sqlite/hashes.js').createHashesStorage>; sets: ReturnType<import('../storage/sqlite/sets.js').createSetsStorage>; lists: ReturnType<import('../storage/sqlite/lists.js').createListsStorage>; zsets: ReturnType<import('../storage/sqlite/zsets.js').createZsetsStorage> }} storages
+ * @param {{ now?: number }} options
+ * @returns {Promise<{ ok: boolean; skipped?: boolean; error?: boolean; bytes?: number }>}
+ */
+export async function importKeyFromRedis(redisClient, keyName, storages, options = {}) {
+  const now = options.now ?? Date.now();
+  const { keys, strings, hashes, sets, lists, zsets } = storages;
+
+  try {
+    const type = (await redisClient.type(keyName)).toLowerCase();
+    if (!SUPPORTED_TYPES.has(type)) {
+      return { ok: false, skipped: true };
+    }
+
+    let pttl = await redisClient.pTTL(keyName);
+    if (pttl === -2) pttl = -1;
+    const expiresAt = pttl > 0 ? now + pttl : null;
+    const keyBuf = asKey(keyName);
+
+    let bytes = keyBuf.length;
+
+    if (type === 'string') {
+      const value = await redisClient.get(keyName);
+      if (value === undefined || value === null) return { ok: false, skipped: true };
+      const valBuf = asValue(value);
+      bytes += valBuf.length;
+      strings.set(keyBuf, valBuf, { expiresAt, updatedAt: now });
+      return { ok: true, bytes };
+    }
+
+    if (type === 'hash') {
+      const obj = await redisClient.hGetAll(keyName);
+      if (!obj || typeof obj !== 'object') return { ok: false, skipped: true };
+      const pairs = [];
+      for (const [f, v] of Object.entries(obj)) {
+        const fb = toBuffer(f);
+        const vb = toBuffer(v);
+        pairs.push(fb, vb);
+        bytes += fb.length + vb.length;
+      }
+      if (pairs.length === 0) return { ok: false, skipped: true };
+      hashes.setMultiple(keyBuf, pairs, { updatedAt: now });
+      keys.setExpires(keyBuf, expiresAt, now);
+      return { ok: true, bytes };
+    }
+
+    if (type === 'set') {
+      const members = await redisClient.sMembers(keyName);
+      if (!members || !members.length) return { ok: false, skipped: true };
+      const memberBuffers = members.map((m) => toBuffer(m));
+      for (const b of memberBuffers) bytes += b.length;
+      sets.add(keyBuf, memberBuffers, { updatedAt: now });
+      keys.setExpires(keyBuf, expiresAt, now);
+      return { ok: true, bytes };
+    }
+
+    if (type === 'list') {
+      const elements = await redisClient.lRange(keyName, 0, -1);
+      if (!elements || !elements.length) return { ok: false, skipped: true };
+      const valueBuffers = elements.map((e) => toBuffer(e));
+      for (const b of valueBuffers) bytes += b.length;
+      lists.rpush(keyBuf, valueBuffers, { updatedAt: now });
+      keys.setExpires(keyBuf, expiresAt, now);
+      return { ok: true, bytes };
+    }
+
+    if (type === 'zset') {
+      const withScores = await redisClient.zRangeWithScores(keyName, 0, -1);
+      if (!withScores || !withScores.length) return { ok: false, skipped: true };
+      const pairs = withScores.map((item) => ({
+        member: toBuffer(item.value),
+        score: Number(item.score),
+      }));
+      for (const p of pairs) bytes += p.member.length + 8;
+      zsets.add(keyBuf, pairs, { updatedAt: now });
+      keys.setExpires(keyBuf, expiresAt, now);
+      return { ok: true, bytes };
+    }
+
+    return { ok: false, skipped: true };
+  } catch (err) {
+    return { ok: false, error: true };
+  }
+}
+
+export { SUPPORTED_TYPES };

package/src/migration/preflight.js

@@ -0,0 +1,62 @@
+/**
+ * Preflight check: estimate key count, type distribution, notify-keyspace-events (SPEC_F §F.9 Step 0).
+ */
+
+/**
+ * @param {import('redis').RedisClientType} redisClient
+ * @returns {Promise<{ keyCountEstimate: number; typeDistribution: Record<string, number>; notifyKeyspaceEvents: string | null; recommended: { scan_count: number; max_concurrency: number; max_rps: number } }>}
+ */
+export async function runPreflight(redisClient) {
+  let keyCountEstimate = 0;
+  try {
+    keyCountEstimate = await redisClient.dbSize();
+  } catch (_) {
+    keyCountEstimate = 0;
+  }
+
+  const typeDistribution = { string: 0, hash: 0, set: 0, list: 0, zset: 0, other: 0 };
+  const sampleSize = Math.min(1000, Math.max(100, Math.floor(keyCountEstimate / 10)));
+  let cursor = 0;
+  let sampled = 0;
+
+  if (keyCountEstimate > 0) {
+    do {
+      const result = await redisClient.scan(cursor, { COUNT: 200 });
+      const keys = Array.isArray(result) ? result[1] : (result?.keys || []);
+      cursor = Array.isArray(result) ? parseInt(result[0], 10) : (result?.cursor ?? 0);
+
+      for (const key of keys) {
+        if (sampled >= sampleSize) break;
+        try {
+          const type = (await redisClient.type(key)).toLowerCase();
+          if (typeDistribution[type] !== undefined) typeDistribution[type]++;
+          else typeDistribution.other++;
+          sampled++;
+        } catch (_) {}
+      }
+      if (sampled >= sampleSize) break;
+    } while (cursor !== 0);
+  }
+
+  let notifyKeyspaceEvents = null;
+  try {
+    const config = await redisClient.configGet('notify-keyspace-events');
+    notifyKeyspaceEvents = config && typeof config === 'object' ? config['notify-keyspace-events'] : null;
+    if (typeof notifyKeyspaceEvents !== 'string') notifyKeyspaceEvents = null;
+  } catch (_) {
+    notifyKeyspaceEvents = null;
+  }
+
+  const recommended = {
+    scan_count: 1000,
+    max_concurrency: 32,
+    max_rps: keyCountEstimate > 100000 ? 2000 : 1000,
+  };
+
+  return {
+    keyCountEstimate,
+    typeDistribution,
+    notifyKeyspaceEvents,
+    recommended,
+  };
+}