resplite 1.3.0 → 1.3.2

package/README.md

@@ -189,7 +189,10 @@ await m.close();
 `resume` defaults to `true`. It doesn't matter whether it's the first run or a resume: the same script works for both starting and continuing. The first run starts from cursor 0; if the process is interrupted (Ctrl+C, crash, etc.), running the script again continues from the last checkpoint. You don't need to pass `resume: false` on the first run or change anything to resume.
 
 **Graceful shutdown**  
-On SIGINT (Ctrl+C) or SIGTERM, the bulk importer checkpoints progress, sets the run status to `aborted`, closes the SQLite database cleanly (so WAL is checkpointed and the file is not left open), then exits. You can safely interrupt a long-running bulk and resume later.
+On SIGINT (Ctrl+C) or SIGTERM, the bulk importer checkpoints progress, sets the run status to `aborted`, closes the SQLite database cleanly (so WAL is checkpointed and the file is not left open), then exits. You can safely interrupt a long-running bulk and resume later. The same applies to **apply-dirty**: Ctrl+C stops the delta apply, sets the run to `aborted`, and exits so you can inspect and retry.
+
+**Errors and stalls**  
+Use `onProgress` to see progress and detect problems. The callback receives the run row (e.g. `scanned_keys`, `migrated_keys`, `dirty_keys_applied`, `last_error`). If progress stops for a long time (e.g. Redis hang or network issue), you may see `_stallWarning: true` and `_stallMessage` in the progress object every 15 seconds. When a key fails to import, the error is logged to `migration_errors` and the run’s `last_error` is set; the real error message is included so you can diagnose. After any failure, check `m.status()` and query `migration_errors` in the DB if needed.
 
 The JS API can run the dirty-key tracker in-process via `m.startDirtyTracker()` / `m.stopDirtyTracker()`, so the full flow stays inside a single script.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "resplite",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "A RESP2 server with practical Redis compatibility, backed by SQLite",
   "type": "module",
   "main": "src/index.js",

package/src/migration/apply-dirty.js

@@ -9,15 +9,20 @@ import { createHashesStorage } from '../storage/sqlite/hashes.js';
 import { createSetsStorage } from '../storage/sqlite/sets.js';
 import { createListsStorage } from '../storage/sqlite/lists.js';
 import { createZsetsStorage } from '../storage/sqlite/zsets.js';
-import { getRun, getDirtyBatch, markDirtyState, logError, RUN_STATUS } from './registry.js';
+import { getRun, getDirtyBatch, markDirtyState, logError, setRunStatus, updateBulkProgress, RUN_STATUS } from './registry.js';
 import { importKeyFromRedis } from './import-one.js';
 
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+const HEARTBEAT_INTERVAL_MS = 15000;
+const STALL_WARNING_MS = 60000;
+
 /**
  * Apply dirty keys: for each key in registry with state=dirty, reimport from Redis or delete in destination.
+ * On SIGINT/SIGTERM, sets run status to ABORTED, closes DB and rethrows so the process can exit and Ctrl+C works.
+ *
  * @param {import('redis').RedisClientType} redisClient
  * @param {string} dbPath
  * @param {string} runId
@@ -25,12 +30,32 @@ function sleep(ms) {
  * @param {string} [options.pragmaTemplate='default']
  * @param {number} [options.batch_keys=200]
  * @param {number} [options.max_rps=0]
- * @param {(run: object) => void | Promise<void>} [options.onProgress] - Called after each batch with the current run row.
+ * @param {(run: object) => void | Promise<void>} [options.onProgress] - Called after each batch and on heartbeat; may receive _stallWarning if no progress for 60s.
  */
 export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
   const { pragmaTemplate = 'default', batch_keys = 200, max_rps = 0, onProgress } = options;
 
   const db = openDb(dbPath, { pragmaTemplate });
+  let abortRequested = false;
+  const onSignal = () => {
+    abortRequested = true;
+  };
+  process.on('SIGINT', onSignal);
+  process.on('SIGTERM', onSignal);
+
+  let heartbeatTimer = null;
+  if (onProgress) {
+    heartbeatTimer = setInterval(() => {
+      const run = getRun(db, runId);
+      if (!run) return;
+      let payload = run;
+      if (run.updated_at && Date.now() - run.updated_at > STALL_WARNING_MS) {
+        payload = { ...run, _stallWarning: true, _stallMessage: 'No progress for 60s — possible hang or Redis timeout' };
+      }
+      Promise.resolve(onProgress(payload)).catch(() => {});
+    }, HEARTBEAT_INTERVAL_MS);
+  }
+
   const run = getRun(db, runId);
   if (!run) throw new Error(`Run ${runId} not found`);
 
@@ -45,7 +70,9 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
   const minIntervalMs = max_rps > 0 ? 1000 / max_rps : 0;
   let lastKeyTime = 0;
 
+  try {
   for (;;) {
+    if (abortRequested) break;
     let r = getRun(db, runId);
     if (r && r.status === RUN_STATUS.ABORTED) break;
     while (r && r.status === RUN_STATUS.PAUSED) {
@@ -61,6 +88,7 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
 
     // ── Re-import (or remove) keys that changed while bulk was running ──
     for (const { key: keyBuf } of dirtyBatch) {
+      if (abortRequested) break;
       r = getRun(db, runId);
       if (r && r.status === RUN_STATUS.ABORTED) break;
       while (r && r.status === RUN_STATUS.PAUSED) {
@@ -87,7 +115,7 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
           } else if (outcome.skipped) {
             markDirtyState(db, runId, keyBuf, 'skipped');
           } else {
-            logError(db, runId, 'dirty_apply', outcome.error ? 'Import failed' : 'Skipped', keyName);
+            logError(db, runId, 'dirty_apply', outcome.errorMessage || (outcome.error ? 'Import failed' : 'Skipped'), keyName);
             markDirtyState(db, runId, keyBuf, 'error');
           }
         }
@@ -102,6 +130,7 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
     // Marked as 'deleted' in the run counter; state changed away from 'deleted'
     // so the next getDirtyBatch call won't return them again (avoiding infinite loop).
     for (const { key: keyBuf } of deletedBatch) {
+      if (abortRequested) break;
       r = getRun(db, runId);
       if (r && r.status === RUN_STATUS.ABORTED) break;
       while (r && r.status === RUN_STATUS.PAUSED) {
@@ -131,5 +160,28 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
     }
   }
 
+  if (abortRequested) {
+    setRunStatus(db, runId, RUN_STATUS.ABORTED);
+    updateBulkProgress(db, runId, { last_error: 'Interrupted by SIGINT/SIGTERM' });
+    const run = getRun(db, runId);
+    if (onProgress && run) Promise.resolve(onProgress(run)).catch(() => {});
+    const err = new Error('Apply dirty interrupted by signal (SIGINT/SIGTERM)');
+    err.code = 'APPLY_DIRTY_ABORTED';
+    throw err;
+  }
+
   return getRun(db, runId);
+  } catch (err) {
+    if (err.code !== 'APPLY_DIRTY_ABORTED') {
+      setRunStatus(db, runId, RUN_STATUS.FAILED);
+      updateBulkProgress(db, runId, { last_error: err.message });
+      logError(db, runId, 'dirty_apply', err.message, null);
+    }
+    throw err;
+  } finally {
+    if (heartbeatTimer) clearInterval(heartbeatTimer);
+    process.off('SIGINT', onSignal);
+    process.off('SIGTERM', onSignal);
+    db.close();
+  }
 }

package/src/migration/bulk.js

@@ -35,6 +35,9 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+const HEARTBEAT_INTERVAL_MS = 15000;
+const STALL_WARNING_MS = 60000;
+
 /**
  * Run bulk import: SCAN keys from Redis, import into RespLite DB with checkpointing.
  * On SIGINT/SIGTERM, checkpoint progress, set run status to ABORTED, close DB and rethrow.
@@ -52,7 +55,7 @@ function sleep(ms) {
  * @param {number} [options.batch_bytes=64*1024*1024] - 64MB
  * @param {number} [options.checkpoint_interval_sec=30]
  * @param {boolean} [options.resume=true] - true: start from 0 or continue from checkpoint; false: always start from 0
- * @param {function(run): void} [options.onProgress] - called after checkpoint with run row
+ * @param {function(run): void} [options.onProgress] - called after checkpoint and on heartbeat; may receive _stallWarning if no progress for 60s
  */
 export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
   const {
@@ -75,6 +78,19 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
   process.on('SIGINT', onSignal);
   process.on('SIGTERM', onSignal);
 
+  let heartbeatTimer = null;
+  if (onProgress) {
+    heartbeatTimer = setInterval(() => {
+      const run = getRun(db, runId);
+      if (!run) return;
+      let payload = run;
+      if (run.updated_at && Date.now() - run.updated_at > STALL_WARNING_MS) {
+        payload = { ...run, _stallWarning: true, _stallMessage: 'No progress for 60s — possible hang or Redis timeout' };
+      }
+      Promise.resolve(onProgress(payload)).catch(() => {});
+    }, HEARTBEAT_INTERVAL_MS);
+  }
+
   try {
     const keys = createKeysStorage(db);
     const strings = createStringsStorage(db, keys);
@@ -146,7 +162,7 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
           skipped_keys++;
         } else {
           error_keys++;
-          logError(db, runId, 'bulk', outcome.error ? 'Import failed' : 'Skipped', keyName);
+          logError(db, runId, 'bulk', outcome.errorMessage || (outcome.error ? 'Import failed' : 'Skipped'), keyName);
         }
 
         const now2 = Date.now();
@@ -207,6 +223,7 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
     }
     throw err;
   } finally {
+    if (heartbeatTimer) clearInterval(heartbeatTimer);
     process.off('SIGINT', onSignal);
     process.off('SIGTERM', onSignal);
     db.close();

package/src/migration/import-one.js

@@ -20,7 +20,7 @@ function toBuffer(value) {
  * @param {string} keyName
  * @param {{ keys: import('../storage/sqlite/keys.js').ReturnType<import('../storage/sqlite/keys.js').createKeysStorage>; strings: ReturnType<import('../storage/sqlite/strings.js').createStringsStorage>; hashes: ReturnType<import('../storage/sqlite/hashes.js').createHashesStorage>; sets: ReturnType<import('../storage/sqlite/sets.js').createSetsStorage>; lists: ReturnType<import('../storage/sqlite/lists.js').createListsStorage>; zsets: ReturnType<import('../storage/sqlite/zsets.js').createZsetsStorage> }} storages
  * @param {{ now?: number }} options
- * @returns {Promise<{ ok: boolean; skipped?: boolean; error?: boolean; bytes?: number }>}
+ * @returns {Promise<{ ok: boolean; skipped?: boolean; error?: boolean; errorMessage?: string; bytes?: number }>}
  */
 export async function importKeyFromRedis(redisClient, keyName, storages, options = {}) {
   const now = options.now ?? Date.now();
@@ -99,7 +99,7 @@ export async function importKeyFromRedis(redisClient, keyName, storages, options
 
     return { ok: false, skipped: true };
   } catch (err) {
-    return { ok: false, error: true };
+    return { ok: false, error: true, errorMessage: err.message };
   }
 }