resplite 1.2.4 → 1.2.6

package/README.md

@@ -217,10 +217,19 @@ const ks = await m.enableKeyspaceNotifications();
 // → { ok: true, previous: '', applied: 'KEA' }
 // If CONFIG is renamed and configCommand was not set, ok=false and error explains how to fix it.
 
+// Step 0c — Start dirty tracking (in-process, same script)
+await m.startDirtyTracker({
+  onProgress: (p) => {
+    // one callback per keyspace event tracked during bulk/cutover
+    console.log(`[dirty ${p.totalEvents}] event=${p.event} key=${p.key}`);
+  },
+});
+
 // Step 1 — Bulk import (checkpointed, resumable). Same script to start or continue.
 // Use keyCountEstimate from preflight to show progress % (estimate; actual count may change).
 const total = info.keyCountEstimate || 1;
 await m.bulk({
+  resume: true, 
   onProgress: (r) => {
     const pct = total ? ((r.scanned_keys / total) * 100).toFixed(1) : '—';
     console.log(
@@ -236,6 +245,9 @@ console.log('bulk status:', run.status, '— dirty counts:', dirty);
 // Step 2 — Apply dirty keys that changed in Redis during bulk
 await m.applyDirty();
 
+// Step 2b — Stop tracker after cutover
+await m.stopDirtyTracker();
+
 // Step 3 — Verify a sample of keys match between Redis and the destination
 const result = await m.verify({ samplePct: 0.5, maxSample: 10000 });
 console.log(`verified ${result.sampled} keys — mismatches: ${result.mismatches.length}`);
@@ -244,13 +256,13 @@ console.log(`verified ${result.sampled} keys — mismatches: ${result.mismatches
 await m.close();
 ```
 
-**Automatic resume (default)**  
+**Bult: Automatic resume (default)**  
 `resume` defaults to `true`. It doesn't matter whether it's the first run or a resume: the same script works for both starting and continuing. The first run starts from cursor 0; if the process is interrupted (Ctrl+C, crash, etc.), running the script again continues from the last checkpoint. You don't need to pass `resume: false` on the first run or change anything to resume.
 
 **Graceful shutdown**  
 On SIGINT (Ctrl+C) or SIGTERM, the bulk importer checkpoints progress, sets the run status to `aborted`, closes the SQLite database cleanly (so WAL is checkpointed and the file is not left open), then exits. You can safely interrupt a long-running bulk and resume later.
 
-The dirty-key tracker (to capture writes during bulk) still runs as a separate process via `npx resplite-dirty-tracker`. The API above handles everything else in a single script.
+The JS API can run the dirty-key tracker in-process via `m.startDirtyTracker()` / `m.stopDirtyTracker()`, so the full flow can run from a single script. You can still use `npx resplite-dirty-tracker start|stop` if you prefer a separate process.
 
 #### Renamed CONFIG command
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "resplite",
-  "version": "1.2.4",
+  "version": "1.2.6",
   "description": "A RESP2 server with practical Redis compatibility, backed by SQLite",
   "type": "module",
   "main": "src/index.js",

package/src/migration/index.js

@@ -26,6 +26,7 @@ import { runApplyDirty } from './apply-dirty.js';
 import { runVerify } from './verify.js';
 import { runMigrateSearch } from './migrate-search.js';
 import { getRun, getDirtyCounts } from './registry.js';
+import { startDirtyTracker as startDirtyTrackerProcess } from './tracker.js';
 
 /**
  * @typedef {object} MigrationOptions
@@ -47,6 +48,8 @@ import { getRun, getDirtyCounts } from './registry.js';
  * @returns {{
  *   preflight(): Promise<object>,
  *   enableKeyspaceNotifications(opts?: { value?: string, merge?: boolean }): Promise<{ ok: boolean, previous: string|null, applied: string, error?: string }>,
+ *   startDirtyTracker(opts?: { pragmaTemplate?: string, onProgress?: function }): Promise<{ running: true }>,
+ *   stopDirtyTracker(): Promise<{ running: false }>,
  *   bulk(opts?: { resume?: boolean, onProgress?: function }): Promise<object>,
  *   status(): { run: object, dirty: object } | null,
  *   applyDirty(opts?: { batchKeys?: number, maxRps?: number }): Promise<object>,
@@ -69,6 +72,7 @@ export function createMigration({
   if (!to) throw new Error('createMigration: "to" (db path) is required');
 
   let _client = null;
+  let _tracker = null;
 
   async function getClient() {
     if (_client) return _client;
@@ -113,6 +117,41 @@ export function createMigration({
       return setKeyspaceEvents(client, value, { configCommand, merge });
     },
 
+    /**
+     * Start dirty-key tracking in-process for this migration controller.
+     * Use this to run the full minimal-downtime flow in one Node script.
+     *
+     * @param {{
+     *   pragmaTemplate?: string,
+     *   onProgress?: (progress: { runId: string, key: string, event: string, totalEvents: number, at: string }) => void | Promise<void>
+     * }} [opts]
+     */
+    async startDirtyTracker({ pragmaTemplate: pt = pragmaTemplate, onProgress } = {}) {
+      if (_tracker) return { running: true };
+      const id = requireRunId();
+      _tracker = await startDirtyTrackerProcess({
+        from,
+        to,
+        runId: id,
+        pragmaTemplate: pt,
+        configCommand,
+        onProgress,
+      });
+      return { running: true };
+    },
+
+    /**
+     * Stop in-process dirty-key tracking started by `startDirtyTracker`.
+     * Safe to call even if tracking is not running.
+     */
+    async stopDirtyTracker() {
+      if (_tracker) {
+        await _tracker.stop();
+        _tracker = null;
+      }
+      return { running: false };
+    },
+
     /**
      * Step 1 — Bulk import: SCAN all keys from Redis into the destination DB.
      * Resume is on by default: first run starts from 0, later runs continue from checkpoint.
@@ -208,6 +247,10 @@ export function createMigration({
      * Disconnect from Redis. Call when done with all migration operations.
      */
     async close() {
+      if (_tracker) {
+        await _tracker.stop().catch(() => {});
+        _tracker = null;
+      }
       if (_client) {
         await _client.quit().catch(() => {});
         _client = null;

package/src/migration/tracker.js

@@ -27,6 +27,13 @@ const KEYEVENT_PATTERN = '__keyevent@0__:*';
  * @param {string} options.runId                           - Migration run identifier.
  * @param {string} [options.pragmaTemplate='default']
  * @param {string} [options.configCommand='CONFIG']        - CONFIG command name (in case it was renamed).
+ * @param {(progress: {
+ *   runId: string,
+ *   key: string,
+ *   event: string,
+ *   totalEvents: number,
+ *   at: string
+ * }) => void | Promise<void>} [options.onProgress]        - Called for each tracked keyspace event.
  * @returns {Promise<{ stop(): Promise<void> }>}
  * @throws {Error} If keyspace notifications are not enabled on Redis.
  */
@@ -36,6 +43,7 @@ export async function startDirtyTracker({
   runId,
   pragmaTemplate = 'default',
   configCommand = 'CONFIG',
+  onProgress,
 } = {}) {
   if (!to) throw new Error('startDirtyTracker: "to" (db path) is required');
   if (!runId) throw new Error('startDirtyTracker: "runId" is required');
@@ -66,12 +74,27 @@ export async function startDirtyTracker({
   );
   await subClient.connect();
 
+  let totalEvents = 0;
   await subClient.pSubscribe(KEYEVENT_PATTERN, (message, channel) => {
     const event = typeof channel === 'string'
       ? channel.split(':').pop()
       : String(channel ?? '').split(':').pop() || 'unknown';
     try {
       upsertDirtyKey(db, runId, message, event);
+      totalEvents += 1;
+      if (onProgress) {
+        Promise.resolve(
+          onProgress({
+            runId,
+            key: message,
+            event,
+            totalEvents,
+            at: new Date().toISOString(),
+          })
+        ).catch((err) => {
+          logError(db, runId, 'dirty_apply', 'Tracker onProgress error: ' + err.message, message);
+        });
+      }
     } catch (err) {
       logError(db, runId, 'dirty_apply', err.message, message);
     }

package/test/integration/migration-dirty-tracker.test.js

@@ -100,13 +100,17 @@ describe('dirty tracker integration', { timeout: 30_000 }, () => {
     const runId  = `tracker-test-${Date.now()}`;
 
     const m = createMigration({ from: REDIS_URL, to: dbPath, runId });
+    const progressEvents = [];
 
     const ks = await m.enableKeyspaceNotifications({ value: 'KEA' });
     assert.ok(ks.ok, `Failed to enable keyspace notifications: ${ks.error}`);
 
-    // ── Start tracker BEFORE bulk ────────────────────────────────────────
-    const tracker = await startDirtyTracker({ from: REDIS_URL, to: dbPath, runId });
-
+    // ── Start tracker BEFORE bulk (same-script API) ─────────────────────
+    await m.startDirtyTracker({
+      onProgress: (p) => {
+        progressEvents.push(p);
+      },
+    });
     try {
       // ── Bulk import (captures the initial snapshot) ──────────────────
       const run = await m.bulk();
@@ -121,7 +125,7 @@ describe('dirty tracker integration', { timeout: 30_000 }, () => {
       // Give the tracker time to process the keyspace events
       await new Promise((r) => setTimeout(r, 600));
     } finally {
-      await tracker.stop();
+      await m.stopDirtyTracker();
     }
 
     // ── Verify dirty keys were captured ─────────────────────────────────
@@ -130,6 +134,7 @@ describe('dirty tracker integration', { timeout: 30_000 }, () => {
       dirty.dirty + dirty.deleted >= 3,
       `Expected ≥3 dirty/deleted keys, got dirty=${dirty.dirty} deleted=${dirty.deleted}`
     );
+    assert.ok(progressEvents.length >= 3, `Expected tracker onProgress events, got ${progressEvents.length}`);
 
     // ── Apply-dirty: reconcile post-bulk changes ─────────────────────────
     const afterApply = await m.applyDirty();