resplite 1.2.2 → 1.2.6

package/src/migration/bulk.js

@@ -37,6 +37,9 @@ function sleep(ms) {
 
 /**
  * Run bulk import: SCAN keys from Redis, import into RespLite DB with checkpointing.
+ * On SIGINT/SIGTERM, checkpoint progress, set run status to ABORTED, close DB and rethrow.
+ * DB is always closed in a finally block (graceful shutdown when process is interrupted).
+ *
  * @param {import('redis').RedisClientType} redisClient
  * @param {string} dbPath
  * @param {string} runId
@@ -65,41 +68,47 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
   } = options;
 
   const db = openDb(dbPath, { pragmaTemplate });
-  const keys = createKeysStorage(db);
-  const strings = createStringsStorage(db, keys);
-  const hashes = createHashesStorage(db, keys);
-  const sets = createSetsStorage(db, keys);
-  const lists = createListsStorage(db, keys);
-  const zsets = createZsetsStorage(db, keys);
-  const storages = { keys, strings, hashes, sets, lists, zsets };
-
-  createRun(db, runId, sourceUri, { scan_count_hint: scan_count });
-  let run = getRun(db, runId);
-  if (!run) throw new Error(`Run ${runId} not found`);
-
-  let cursor = resume && run.scan_cursor !== undefined ? parseInt(String(run.scan_cursor), 10) : 0;
-  let scanned_keys = resume ? (run.scanned_keys || 0) : 0;
-  let migrated_keys = resume ? (run.migrated_keys || 0) : 0;
-  let skipped_keys = resume ? (run.skipped_keys || 0) : 0;
-  let error_keys = resume ? (run.error_keys || 0) : 0;
-  let migrated_bytes = resume ? (run.migrated_bytes || 0) : 0;
-
-  if (!resume) {
-    updateBulkProgress(db, runId, { scan_cursor: String(cursor), scanned_keys, migrated_keys, skipped_keys, error_keys, migrated_bytes });
-  }
-
-  let lastCheckpointTime = Date.now();
-  let batchScanned = 0;
-  let batchBytes = 0;
-  const minIntervalMs = max_rps > 0 ? 1000 / max_rps : 0;
-  let lastKeyTime = 0;
+  let abortRequested = false;
+  const onSignal = () => {
+    abortRequested = true;
+  };
+  process.on('SIGINT', onSignal);
+  process.on('SIGTERM', onSignal);
 
   try {
-    do {
+    const keys = createKeysStorage(db);
+    const strings = createStringsStorage(db, keys);
+    const hashes = createHashesStorage(db, keys);
+    const sets = createSetsStorage(db, keys);
+    const lists = createListsStorage(db, keys);
+    const zsets = createZsetsStorage(db, keys);
+    const storages = { keys, strings, hashes, sets, lists, zsets };
+
+    createRun(db, runId, sourceUri, { scan_count_hint: scan_count });
+    let run = getRun(db, runId);
+    if (!run) throw new Error(`Run ${runId} not found`);
+
+    let cursor = resume && run.scan_cursor !== undefined ? parseInt(String(run.scan_cursor), 10) : 0;
+    let scanned_keys = resume ? (run.scanned_keys || 0) : 0;
+    let migrated_keys = resume ? (run.migrated_keys || 0) : 0;
+    let skipped_keys = resume ? (run.skipped_keys || 0) : 0;
+    let error_keys = resume ? (run.error_keys || 0) : 0;
+    let migrated_bytes = resume ? (run.migrated_bytes || 0) : 0;
+
+    if (!resume) {
+      updateBulkProgress(db, runId, { scan_cursor: String(cursor), scanned_keys, migrated_keys, skipped_keys, error_keys, migrated_bytes });
+    }
+
+    let lastCheckpointTime = Date.now();
+    let batchScanned = 0;
+    let batchBytes = 0;
+    const minIntervalMs = max_rps > 0 ? 1000 / max_rps : 0;
+    let lastKeyTime = 0;
+
+    outer: do {
       run = getRun(db, runId);
-      if (run && run.status === RUN_STATUS.ABORTED) {
-        break;
-      }
+      if (run && run.status === RUN_STATUS.ABORTED) break;
+      if (abortRequested) break;
       while (run && run.status === RUN_STATUS.PAUSED) {
         await sleep(2000);
         run = getRun(db, runId);
@@ -111,8 +120,9 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
       const keyList = parsed.keys || [];
 
       for (const keyName of keyList) {
+        if (abortRequested) break outer;
         run = getRun(db, runId);
-        if (run && run.status === RUN_STATUS.ABORTED) break;
+        if (run && run.status === RUN_STATUS.ABORTED) break outer;
         while (run && run.status === RUN_STATUS.PAUSED) {
           await sleep(2000);
           run = getRun(db, runId);
@@ -162,6 +172,23 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
       }
     } while (cursor !== 0);
 
+    if (abortRequested) {
+      updateBulkProgress(db, runId, {
+        scan_cursor: String(cursor),
+        scanned_keys,
+        migrated_keys,
+        skipped_keys,
+        error_keys,
+        migrated_bytes,
+      });
+      setRunStatus(db, runId, RUN_STATUS.ABORTED);
+      run = getRun(db, runId);
+      if (onProgress && run) onProgress(run);
+      const err = new Error('Bulk import interrupted by signal (SIGINT/SIGTERM)');
+      err.code = 'BULK_ABORTED';
+      throw err;
+    }
+
     updateBulkProgress(db, runId, {
       scan_cursor: '0',
       scanned_keys,
@@ -173,9 +200,15 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
     setRunStatus(db, runId, RUN_STATUS.COMPLETED);
     return getRun(db, runId);
   } catch (err) {
-    setRunStatus(db, runId, RUN_STATUS.FAILED);
-    updateBulkProgress(db, runId, { last_error: err.message });
-    logError(db, runId, 'bulk', err.message, null);
+    if (err.code !== 'BULK_ABORTED') {
+      setRunStatus(db, runId, RUN_STATUS.FAILED);
+      updateBulkProgress(db, runId, { last_error: err.message });
+      logError(db, runId, 'bulk', err.message, null);
+    }
     throw err;
+  } finally {
+    process.off('SIGINT', onSignal);
+    process.off('SIGTERM', onSignal);
+    db.close();
   }
 }

package/src/migration/index.js

@@ -24,7 +24,9 @@ import { runPreflight, readKeyspaceEvents, setKeyspaceEvents } from './preflight
 import { runBulkImport } from './bulk.js';
 import { runApplyDirty } from './apply-dirty.js';
 import { runVerify } from './verify.js';
+import { runMigrateSearch } from './migrate-search.js';
 import { getRun, getDirtyCounts } from './registry.js';
+import { startDirtyTracker as startDirtyTrackerProcess } from './tracker.js';
 
 /**
  * @typedef {object} MigrationOptions
@@ -46,10 +48,13 @@ import { getRun, getDirtyCounts } from './registry.js';
  * @returns {{
  *   preflight(): Promise<object>,
  *   enableKeyspaceNotifications(opts?: { value?: string, merge?: boolean }): Promise<{ ok: boolean, previous: string|null, applied: string, error?: string }>,
+ *   startDirtyTracker(opts?: { pragmaTemplate?: string, onProgress?: function }): Promise<{ running: true }>,
+ *   stopDirtyTracker(): Promise<{ running: false }>,
  *   bulk(opts?: { resume?: boolean, onProgress?: function }): Promise<object>,
  *   status(): { run: object, dirty: object } | null,
  *   applyDirty(opts?: { batchKeys?: number, maxRps?: number }): Promise<object>,
  *   verify(opts?: { samplePct?: number, maxSample?: number }): Promise<object>,
+ *   migrateSearch(opts?: { onlyIndices?: string[], scanCount?: number, maxRps?: number, batchDocs?: number, maxSuggestions?: number, skipExisting?: boolean, withSuggestions?: boolean, onProgress?: function }): Promise<object>,
  *   close(): Promise<void>,
  * }}
  */
@@ -67,6 +72,7 @@ export function createMigration({
   if (!to) throw new Error('createMigration: "to" (db path) is required');
 
   let _client = null;
+  let _tracker = null;
 
   async function getClient() {
     if (_client) return _client;
@@ -111,6 +117,41 @@ export function createMigration({
       return setKeyspaceEvents(client, value, { configCommand, merge });
     },
 
+    /**
+     * Start dirty-key tracking in-process for this migration controller.
+     * Use this to run the full minimal-downtime flow in one Node script.
+     *
+     * @param {{
+     *   pragmaTemplate?: string,
+     *   onProgress?: (progress: { runId: string, key: string, event: string, totalEvents: number, at: string }) => void | Promise<void>
+     * }} [opts]
+     */
+    async startDirtyTracker({ pragmaTemplate: pt = pragmaTemplate, onProgress } = {}) {
+      if (_tracker) return { running: true };
+      const id = requireRunId();
+      _tracker = await startDirtyTrackerProcess({
+        from,
+        to,
+        runId: id,
+        pragmaTemplate: pt,
+        configCommand,
+        onProgress,
+      });
+      return { running: true };
+    },
+
+    /**
+     * Stop in-process dirty-key tracking started by `startDirtyTracker`.
+     * Safe to call even if tracking is not running.
+     */
+    async stopDirtyTracker() {
+      if (_tracker) {
+        await _tracker.stop();
+        _tracker = null;
+      }
+      return { running: false };
+    },
+
     /**
      * Step 1 — Bulk import: SCAN all keys from Redis into the destination DB.
      * Resume is on by default: first run starts from 0, later runs continue from checkpoint.
@@ -173,10 +214,43 @@ export function createMigration({
       return runVerify(client, to, { pragmaTemplate, samplePct, maxSample });
     },
 
+    /**
+     * Step 5 — Migrate search indices: copy RediSearch index schemas and documents
+     * into RespLite FT.* tables.
+     *
+     * Requires RediSearch (Redis Stack or redis/search module) on the source.
+     * Only HASH-based indices with TEXT/TAG/NUMERIC fields are supported.
+     * TAG and NUMERIC fields are mapped to TEXT.
+     *
+     * @param {{
+     *   onlyIndices?: string[],
+     *   scanCount?: number,
+     *   maxRps?: number,
+     *   batchDocs?: number,
+     *   maxSuggestions?: number,
+     *   skipExisting?: boolean,
+     *   withSuggestions?: boolean,
+     *   onProgress?: (result: object) => void
+     * }} [opts]
+     * @returns {Promise<{ indices: object[], aborted: boolean }>}
+     */
+    async migrateSearch(opts = {}) {
+      const client = await getClient();
+      return runMigrateSearch(client, to, {
+        pragmaTemplate,
+        maxRps,
+        ...opts,
+      });
+    },
+
     /**
      * Disconnect from Redis. Call when done with all migration operations.
      */
     async close() {
+      if (_tracker) {
+        await _tracker.stop().catch(() => {});
+        _tracker = null;
+      }
       if (_client) {
         await _client.quit().catch(() => {});
         _client = null;
@@ -185,6 +259,6 @@ export function createMigration({
   };
 }
 
-export { runPreflight, readKeyspaceEvents, setKeyspaceEvents, runBulkImport, runApplyDirty, runVerify };
+export { runPreflight, readKeyspaceEvents, setKeyspaceEvents, runBulkImport, runApplyDirty, runVerify, runMigrateSearch };
 export { startDirtyTracker } from './tracker.js';
 export { getRun, getDirtyCounts, createRun, setRunStatus, logError } from './registry.js';