resplite 1.0.2 → 1.0.6

package/src/migration/bulk.js

@@ -0,0 +1,181 @@
+/**
+ * Bulk import from Redis with checkpointing, resume, and throttling (SPEC_F §F.7).
+ */
+
+import { openDb } from '../storage/sqlite/db.js';
+import { createKeysStorage } from '../storage/sqlite/keys.js';
+import { createStringsStorage } from '../storage/sqlite/strings.js';
+import { createHashesStorage } from '../storage/sqlite/hashes.js';
+import { createSetsStorage } from '../storage/sqlite/sets.js';
+import { createListsStorage } from '../storage/sqlite/lists.js';
+import { createZsetsStorage } from '../storage/sqlite/zsets.js';
+import {
+  createRun,
+  getRun,
+  updateBulkProgress,
+  setRunStatus,
+  logError,
+  RUN_STATUS,
+} from './registry.js';
+import { importKeyFromRedis } from './import-one.js';
+
+function parseScanResult(result) {
+  if (Array.isArray(result)) {
+    return { cursor: parseInt(result[0], 10), keys: result[1] || [] };
+  }
+  if (result && typeof result === 'object') {
+    const cursor = typeof result.cursor === 'number' ? result.cursor : parseInt(String(result.cursor), 10);
+    const keys = result.keys || [];
+    return { cursor, keys };
+  }
+  return { cursor: 0, keys: [] };
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Run bulk import: SCAN keys from Redis, import into RespLite DB with checkpointing.
+ * @param {import('redis').RedisClientType} redisClient
+ * @param {string} dbPath
+ * @param {string} runId
+ * @param {object} options
+ * @param {string} options.sourceUri
+ * @param {string} [options.pragmaTemplate='default']
+ * @param {number} [options.scan_count=1000]
+ * @param {number} [options.max_rps=0] - 0 = no limit
+ * @param {number} [options.batch_keys=200]
+ * @param {number} [options.batch_bytes=64*1024*1024] - 64MB
+ * @param {number} [options.checkpoint_interval_sec=30]
+ * @param {boolean} [options.resume=false]
+ * @param {function(run): void} [options.onProgress] - called after checkpoint with run row
+ */
+export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
+  const {
+    sourceUri,
+    pragmaTemplate = 'default',
+    scan_count = 1000,
+    max_rps = 0,
+    batch_keys = 200,
+    batch_bytes = 64 * 1024 * 1024,
+    checkpoint_interval_sec = 30,
+    resume = false,
+    onProgress,
+  } = options;
+
+  const db = openDb(dbPath, { pragmaTemplate });
+  const keys = createKeysStorage(db);
+  const strings = createStringsStorage(db, keys);
+  const hashes = createHashesStorage(db, keys);
+  const sets = createSetsStorage(db, keys);
+  const lists = createListsStorage(db, keys);
+  const zsets = createZsetsStorage(db, keys);
+  const storages = { keys, strings, hashes, sets, lists, zsets };
+
+  createRun(db, runId, sourceUri, { scan_count_hint: scan_count });
+  let run = getRun(db, runId);
+  if (!run) throw new Error(`Run ${runId} not found`);
+
+  let cursor = resume && run.scan_cursor !== undefined ? parseInt(String(run.scan_cursor), 10) : 0;
+  let scanned_keys = resume ? (run.scanned_keys || 0) : 0;
+  let migrated_keys = resume ? (run.migrated_keys || 0) : 0;
+  let skipped_keys = resume ? (run.skipped_keys || 0) : 0;
+  let error_keys = resume ? (run.error_keys || 0) : 0;
+  let migrated_bytes = resume ? (run.migrated_bytes || 0) : 0;
+
+  if (!resume) {
+    updateBulkProgress(db, runId, { scan_cursor: String(cursor), scanned_keys, migrated_keys, skipped_keys, error_keys, migrated_bytes });
+  }
+
+  let lastCheckpointTime = Date.now();
+  let batchScanned = 0;
+  let batchBytes = 0;
+  const minIntervalMs = max_rps > 0 ? 1000 / max_rps : 0;
+  let lastKeyTime = 0;
+
+  try {
+    do {
+      run = getRun(db, runId);
+      if (run && run.status === RUN_STATUS.ABORTED) {
+        break;
+      }
+      while (run && run.status === RUN_STATUS.PAUSED) {
+        await sleep(2000);
+        run = getRun(db, runId);
+      }
+
+      const result = await redisClient.scan(cursor, { COUNT: scan_count });
+      const parsed = parseScanResult(result);
+      cursor = parsed.cursor;
+      const keyList = parsed.keys || [];
+
+      for (const keyName of keyList) {
+        run = getRun(db, runId);
+        if (run && run.status === RUN_STATUS.ABORTED) break;
+        while (run && run.status === RUN_STATUS.PAUSED) {
+          await sleep(2000);
+          run = getRun(db, runId);
+        }
+
+        scanned_keys++;
+        if (minIntervalMs > 0) {
+          const elapsed = Date.now() - lastKeyTime;
+          if (elapsed < minIntervalMs) await sleep(minIntervalMs - elapsed);
+          lastKeyTime = Date.now();
+        }
+
+        const now = Date.now();
+        const outcome = await importKeyFromRedis(redisClient, keyName, storages, { now });
+        if (outcome.ok) {
+          migrated_keys++;
+          migrated_bytes += outcome.bytes || 0;
+          batchScanned++;
+          batchBytes += outcome.bytes || 0;
+        } else if (outcome.skipped) {
+          skipped_keys++;
+        } else {
+          error_keys++;
+          logError(db, runId, 'bulk', outcome.error ? 'Import failed' : 'Skipped', keyName);
+        }
+
+        const now2 = Date.now();
+        const shouldCheckpoint =
+          batchScanned >= batch_keys ||
+          batchBytes >= batch_bytes ||
+          now2 - lastCheckpointTime >= checkpoint_interval_sec * 1000;
+        if (shouldCheckpoint) {
+          updateBulkProgress(db, runId, {
+            scan_cursor: String(cursor),
+            scanned_keys,
+            migrated_keys,
+            skipped_keys,
+            error_keys,
+            migrated_bytes,
+          });
+          lastCheckpointTime = now2;
+          batchScanned = 0;
+          batchBytes = 0;
+          run = getRun(db, runId);
+          if (onProgress && run) onProgress(run);
+        }
+      }
+    } while (cursor !== 0);
+
+    updateBulkProgress(db, runId, {
+      scan_cursor: '0',
+      scanned_keys,
+      migrated_keys,
+      skipped_keys,
+      error_keys,
+      migrated_bytes,
+    });
+    setRunStatus(db, runId, RUN_STATUS.COMPLETED);
+    return getRun(db, runId);
+  } catch (err) {
+    setRunStatus(db, runId, RUN_STATUS.FAILED);
+    updateBulkProgress(db, runId, { last_error: err.message });
+    logError(db, runId, 'bulk', err.message, null);
+    throw err;
+  }
+}

package/src/migration/import-one.js

@@ -0,0 +1,106 @@
+/**
+ * Import a single key from Redis into RespLite storages (shared by bulk and delta apply).
+ * SPEC_F §F.7.1, F.8.2.
+ */
+
+import { asKey, asValue } from '../util/buffers.js';
+
+const SUPPORTED_TYPES = new Set(['string', 'hash', 'set', 'list', 'zset']);
+
+function toBuffer(value) {
+  if (value == null) return null;
+  if (Buffer.isBuffer(value)) return value;
+  if (typeof value === 'string') return Buffer.from(value, 'utf8');
+  return Buffer.from(String(value), 'utf8');
+}
+
+/**
+ * Fetch one key from Redis and write to storages. Idempotent (upsert).
+ * @param {import('redis').RedisClientType} redisClient
+ * @param {string} keyName
+ * @param {{ keys: import('../storage/sqlite/keys.js').ReturnType<import('../storage/sqlite/keys.js').createKeysStorage>; strings: ReturnType<import('../storage/sqlite/strings.js').createStringsStorage>; hashes: ReturnType<import('../storage/sqlite/hashes.js').createHashesStorage>; sets: ReturnType<import('../storage/sqlite/sets.js').createSetsStorage>; lists: ReturnType<import('../storage/sqlite/lists.js').createListsStorage>; zsets: ReturnType<import('../storage/sqlite/zsets.js').createZsetsStorage> }} storages
+ * @param {{ now?: number }} options
+ * @returns {Promise<{ ok: boolean; skipped?: boolean; error?: boolean; bytes?: number }>}
+ */
+export async function importKeyFromRedis(redisClient, keyName, storages, options = {}) {
+  const now = options.now ?? Date.now();
+  const { keys, strings, hashes, sets, lists, zsets } = storages;
+
+  try {
+    const type = (await redisClient.type(keyName)).toLowerCase();
+    if (!SUPPORTED_TYPES.has(type)) {
+      return { ok: false, skipped: true };
+    }
+
+    let pttl = await redisClient.pTTL(keyName);
+    if (pttl === -2) pttl = -1;
+    const expiresAt = pttl > 0 ? now + pttl : null;
+    const keyBuf = asKey(keyName);
+
+    let bytes = keyBuf.length;
+
+    if (type === 'string') {
+      const value = await redisClient.get(keyName);
+      if (value === undefined || value === null) return { ok: false, skipped: true };
+      const valBuf = asValue(value);
+      bytes += valBuf.length;
+      strings.set(keyBuf, valBuf, { expiresAt, updatedAt: now });
+      return { ok: true, bytes };
+    }
+
+    if (type === 'hash') {
+      const obj = await redisClient.hGetAll(keyName);
+      if (!obj || typeof obj !== 'object') return { ok: false, skipped: true };
+      const pairs = [];
+      for (const [f, v] of Object.entries(obj)) {
+        const fb = toBuffer(f);
+        const vb = toBuffer(v);
+        pairs.push(fb, vb);
+        bytes += fb.length + vb.length;
+      }
+      if (pairs.length === 0) return { ok: false, skipped: true };
+      hashes.setMultiple(keyBuf, pairs, { updatedAt: now });
+      keys.setExpires(keyBuf, expiresAt, now);
+      return { ok: true, bytes };
+    }
+
+    if (type === 'set') {
+      const members = await redisClient.sMembers(keyName);
+      if (!members || !members.length) return { ok: false, skipped: true };
+      const memberBuffers = members.map((m) => toBuffer(m));
+      for (const b of memberBuffers) bytes += b.length;
+      sets.add(keyBuf, memberBuffers, { updatedAt: now });
+      keys.setExpires(keyBuf, expiresAt, now);
+      return { ok: true, bytes };
+    }
+
+    if (type === 'list') {
+      const elements = await redisClient.lRange(keyName, 0, -1);
+      if (!elements || !elements.length) return { ok: false, skipped: true };
+      const valueBuffers = elements.map((e) => toBuffer(e));
+      for (const b of valueBuffers) bytes += b.length;
+      lists.rpush(keyBuf, valueBuffers, { updatedAt: now });
+      keys.setExpires(keyBuf, expiresAt, now);
+      return { ok: true, bytes };
+    }
+
+    if (type === 'zset') {
+      const withScores = await redisClient.zRangeWithScores(keyName, 0, -1);
+      if (!withScores || !withScores.length) return { ok: false, skipped: true };
+      const pairs = withScores.map((item) => ({
+        member: toBuffer(item.value),
+        score: Number(item.score),
+      }));
+      for (const p of pairs) bytes += p.member.length + 8;
+      zsets.add(keyBuf, pairs, { updatedAt: now });
+      keys.setExpires(keyBuf, expiresAt, now);
+      return { ok: true, bytes };
+    }
+
+    return { ok: false, skipped: true };
+  } catch (err) {
+    return { ok: false, error: true };
+  }
+}
+
+export { SUPPORTED_TYPES };

package/src/migration/index.js

@@ -0,0 +1,170 @@
+/**
+ * Programmatic migration API (SPEC_F §F.9).
+ *
+ * Usage:
+ *   import { createMigration } from 'resplite/migration';
+ *
+ *   const m = createMigration({
+ *     from: 'redis://127.0.0.1:6379',
+ *     to:   './data.db',
+ *     runId: 'my-migration-1',
+ *   });
+ *
+ *   const info   = await m.preflight();
+ *   await m.bulk({ onProgress: console.log });
+ *   const status = m.status();
+ *   await m.applyDirty();
+ *   const result = await m.verify();
+ *   await m.close();
+ */
+
+import { createClient } from 'redis';
+import { openDb } from '../storage/sqlite/db.js';
+import { runPreflight } from './preflight.js';
+import { runBulkImport } from './bulk.js';
+import { runApplyDirty } from './apply-dirty.js';
+import { runVerify } from './verify.js';
+import { getRun, getDirtyCounts } from './registry.js';
+
+/**
+ * @typedef {object} MigrationOptions
+ * @property {string} [from='redis://127.0.0.1:6379'] - Source Redis URL.
+ * @property {string} to                              - Destination SQLite DB path.
+ * @property {string} [runId]                         - Unique run identifier (required for bulk/status/applyDirty).
+ * @property {string} [pragmaTemplate='default']      - PRAGMA preset.
+ * @property {number} [scanCount=1000]
+ * @property {number} [maxRps=0]                      - Max requests/s (0 = unlimited).
+ * @property {number} [batchKeys=200]
+ * @property {number} [batchBytes=67108864]           - 64 MB default.
+ */
+
+/**
+ * Create a migration controller bound to a source Redis and destination DB.
+ *
+ * @param {MigrationOptions} options
+ * @returns {{
+ *   preflight(): Promise<import('./preflight.js').PreflightResult>,
+ *   bulk(opts?: { resume?: boolean, onProgress?: function }): Promise<object>,
+ *   status(): { run: object, dirty: object } | null,
+ *   applyDirty(opts?: { batchKeys?: number, maxRps?: number }): Promise<object>,
+ *   verify(opts?: { samplePct?: number, maxSample?: number }): Promise<object>,
+ *   close(): Promise<void>,
+ * }}
+ */
+export function createMigration({
+  from = 'redis://127.0.0.1:6379',
+  to,
+  runId,
+  pragmaTemplate = 'default',
+  scanCount = 1000,
+  maxRps = 0,
+  batchKeys = 200,
+  batchBytes = 64 * 1024 * 1024,
+} = {}) {
+  if (!to) throw new Error('createMigration: "to" (db path) is required');
+
+  let _client = null;
+
+  async function getClient() {
+    if (_client) return _client;
+    _client = createClient({ url: from });
+    _client.on('error', (err) => {
+      /* non-fatal connection errors; callers surface them on next await */
+      void err;
+    });
+    await _client.connect();
+    return _client;
+  }
+
+  function requireRunId() {
+    if (!runId) throw new Error('createMigration: "runId" is required for this operation');
+    return runId;
+  }
+
+  return {
+    /**
+     * Step 0 — Preflight: inspect the source Redis instance.
+     * Returns key count estimate, type distribution, keyspace-events config,
+     * and recommended import parameters.
+     */
+    async preflight() {
+      const client = await getClient();
+      return runPreflight(client);
+    },
+
+    /**
+     * Step 1 — Bulk import: SCAN all keys from Redis into the destination DB.
+     * Supports resume (checkpoint-based) and optional progress callback.
+     *
+     * @param {{ resume?: boolean, onProgress?: (run: object) => void }} [opts]
+     */
+    async bulk({ resume = false, onProgress } = {}) {
+      const id = requireRunId();
+      const client = await getClient();
+      return runBulkImport(client, to, id, {
+        sourceUri: from,
+        pragmaTemplate,
+        scan_count: scanCount,
+        max_rps: maxRps,
+        batch_keys: batchKeys,
+        batch_bytes: batchBytes,
+        resume,
+        onProgress,
+      });
+    },
+
+    /**
+     * Step 2 — Status: read run metadata and dirty-key counts from the DB.
+     * Synchronous — no Redis connection needed.
+     *
+     * @returns {{ run: object, dirty: object } | null}
+     */
+    status() {
+      const id = requireRunId();
+      const db = openDb(to, { pragmaTemplate });
+      const run = getRun(db, id);
+      if (!run) return null;
+      const dirty = getDirtyCounts(db, id);
+      return { run, dirty };
+    },
+
+    /**
+     * Step 3 — Apply dirty: reconcile keys that changed in Redis during bulk import.
+     *
+     * @param {{ batchKeys?: number, maxRps?: number }} [opts]
+     */
+    async applyDirty({ batchKeys: bk = batchKeys, maxRps: rps = maxRps } = {}) {
+      const id = requireRunId();
+      const client = await getClient();
+      return runApplyDirty(client, to, id, {
+        pragmaTemplate,
+        batch_keys: bk,
+        max_rps: rps,
+      });
+    },
+
+    /**
+     * Step 4 — Verify: sample keys from Redis and compare with the destination DB.
+     *
+     * @param {{ samplePct?: number, maxSample?: number }} [opts]
+     * @returns {Promise<{ sampled: number, matched: number, mismatches: Array<{ key: string, reason: string }> }>}
+     */
+    async verify({ samplePct = 0.5, maxSample = 10000 } = {}) {
+      const client = await getClient();
+      return runVerify(client, to, { pragmaTemplate, samplePct, maxSample });
+    },
+
+    /**
+     * Disconnect from Redis. Call when done with all migration operations.
+     */
+    async close() {
+      if (_client) {
+        await _client.quit().catch(() => {});
+        _client = null;
+      }
+    },
+  };
+}
+
+export { runPreflight, runBulkImport, runApplyDirty, runVerify };
+export { getRun, getDirtyCounts, createRun, setRunStatus, logError } from './registry.js';

package/src/migration/preflight.js

@@ -0,0 +1,62 @@
+/**
+ * Preflight check: estimate key count, type distribution, notify-keyspace-events (SPEC_F §F.9 Step 0).
+ */
+
+/**
+ * @param {import('redis').RedisClientType} redisClient
+ * @returns {Promise<{ keyCountEstimate: number; typeDistribution: Record<string, number>; notifyKeyspaceEvents: string | null; recommended: { scan_count: number; max_concurrency: number; max_rps: number } }>}
+ */
+export async function runPreflight(redisClient) {
+  let keyCountEstimate = 0;
+  try {
+    keyCountEstimate = await redisClient.dbSize();
+  } catch (_) {
+    keyCountEstimate = 0;
+  }
+
+  const typeDistribution = { string: 0, hash: 0, set: 0, list: 0, zset: 0, other: 0 };
+  const sampleSize = Math.min(1000, Math.max(100, Math.floor(keyCountEstimate / 10)));
+  let cursor = 0;
+  let sampled = 0;
+
+  if (keyCountEstimate > 0) {
+    do {
+      const result = await redisClient.scan(cursor, { COUNT: 200 });
+      const keys = Array.isArray(result) ? result[1] : (result?.keys || []);
+      cursor = Array.isArray(result) ? parseInt(result[0], 10) : (result?.cursor ?? 0);
+
+      for (const key of keys) {
+        if (sampled >= sampleSize) break;
+        try {
+          const type = (await redisClient.type(key)).toLowerCase();
+          if (typeDistribution[type] !== undefined) typeDistribution[type]++;
+          else typeDistribution.other++;
+          sampled++;
+        } catch (_) {}
+      }
+      if (sampled >= sampleSize) break;
+    } while (cursor !== 0);
+  }
+
+  let notifyKeyspaceEvents = null;
+  try {
+    const config = await redisClient.configGet('notify-keyspace-events');
+    notifyKeyspaceEvents = config && typeof config === 'object' ? config['notify-keyspace-events'] : null;
+    if (typeof notifyKeyspaceEvents !== 'string') notifyKeyspaceEvents = null;
+  } catch (_) {
+    notifyKeyspaceEvents = null;
+  }
+
+  const recommended = {
+    scan_count: 1000,
+    max_concurrency: 32,
+    max_rps: keyCountEstimate > 100000 ? 2000 : 1000,
+  };
+
+  return {
+    keyCountEstimate,
+    typeDistribution,
+    notifyKeyspaceEvents,
+    recommended,
+  };
+}