resplite 1.0.6 → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -341,16 +341,18 @@ For large datasets (~30 GB), use the Dirty Key Registry flow so the bulk of the
341
341
  **Enable keyspace notifications in Redis** (required for the dirty-key tracker). Either run at runtime:
342
342
 
343
343
  ```bash
344
- redis-cli CONFIG SET notify-keyspace-events Kgxe
344
+ redis-cli CONFIG SET notify-keyspace-events KEA
345
345
  ```
346
346
 
347
347
  Or add to `redis.conf` and restart Redis:
348
348
 
349
349
  ```
350
- notify-keyspace-events Kgxe
350
+ notify-keyspace-events KEA
351
351
  ```
352
352
 
353
- (`K` = keyspace, `g` = generic, `x` = expired; `e` = keyevent. This lets the tracker see key changes and expirations.)
353
+ (`K` = keyspace prefix, `E` = keyevent prefix, `A` = all event types lets the tracker see every key change and expiration.)
354
+
355
+ > **Renamed CONFIG command?** Some Redis deployments rename `CONFIG` for security. Pass `--config-command <name>` to the CLI tools, or the `configCommand` option to the JS API — see below.
354
356
 
355
357
  1. **Preflight** – Check Redis, key count, type distribution, and that keyspace notifications are enabled:
356
358
  ```bash
@@ -360,6 +362,8 @@ notify-keyspace-events Kgxe
360
362
  2. **Start dirty-key tracker** – Captures keys modified during bulk (requires `notify-keyspace-events` in Redis):
361
363
  ```bash
362
364
  npx resplite-dirty-tracker start --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db
365
+ # If CONFIG was renamed:
366
+ npx resplite-dirty-tracker start --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db --config-command MYCONFIG
363
367
  ```
364
368
 
365
369
  3. **Bulk import** – SCAN and copy all keys; progress is checkpointed and resumable:
@@ -408,14 +412,25 @@ const m = createMigration({
408
412
  batchBytes: 64 * 1024 * 1024, // 64 MB
409
413
  maxRps: 0, // 0 = unlimited
410
414
  pragmaTemplate: 'default',
415
+
416
+ // If your Redis deployment renamed CONFIG for security:
417
+ // configCommand: 'MYCONFIG',
411
418
  });
412
419
 
413
420
  // Step 0 — Preflight: inspect Redis before starting
414
421
  const info = await m.preflight();
415
422
  console.log('keys (estimate):', info.keyCountEstimate);
416
423
  console.log('type distribution:', info.typeDistribution);
424
+ console.log('notify-keyspace-events:', info.notifyKeyspaceEvents);
425
+ console.log('CONFIG available:', info.configCommandAvailable); // false if renamed
417
426
  console.log('recommended params:', info.recommended);
418
427
 
428
+ // Step 0b — Enable keyspace notifications (required for dirty-key tracking)
429
+ // Reads the current value and merges the new flags — existing flags are preserved.
430
+ const ks = await m.enableKeyspaceNotifications();
431
+ // → { ok: true, previous: '', applied: 'KEA' }
432
+ // If CONFIG is renamed and configCommand was not set, ok=false and error explains how to fix it.
433
+
419
434
  // Step 1 — Bulk import (checkpointed, resumable)
420
435
  await m.bulk({
421
436
  resume: false, // true to resume a previous run
@@ -441,6 +456,33 @@ await m.close();
441
456
 
442
457
  The dirty-key tracker (to capture writes during bulk) still runs as a separate process via `npx resplite-dirty-tracker`. The API above handles everything else in a single script.
443
458
 
459
+ #### Renamed CONFIG command
460
+
461
+ If your Redis instance has the `CONFIG` command renamed (a common hardening practice), pass the new name to `createMigration`:
462
+
463
+ ```javascript
464
+ const m = createMigration({
465
+ from: 'redis://10.0.0.10:6379',
466
+ to: './resplite.db',
467
+ runId: 'run_001',
468
+ configCommand: 'MYCONFIG', // the renamed command
469
+ });
470
+
471
+ // preflight will use MYCONFIG GET notify-keyspace-events
472
+ const info = await m.preflight();
473
+ // info.configCommandAvailable → false if the name is wrong
474
+
475
+ // enableKeyspaceNotifications will use MYCONFIG SET notify-keyspace-events KEA
476
+ const result = await m.enableKeyspaceNotifications({ value: 'KEA' });
477
+ ```
478
+
479
+ The same flag is available in the CLI:
480
+
481
+ ```bash
482
+ npx resplite-dirty-tracker start --run-id run_001 --to ./resplite.db \
483
+ --from redis://10.0.0.10:6379 --config-command MYCONFIG
484
+ ```
485
+
444
486
  #### Low-level re-exports
445
487
 
446
488
  If you need more control, the individual functions and registry helpers are also exported:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "resplite",
3
- "version": "1.0.6",
3
+ "version": "1.1.2",
4
4
  "description": "A RESP2 server with practical Redis compatibility, backed by SQLite",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
@@ -4,10 +4,10 @@
4
4
  * Usage: resplite-dirty-tracker start|stop [options]
5
5
  */
6
6
 
7
- import { createClient } from 'redis';
8
7
  import { fileURLToPath } from 'node:url';
9
8
  import { openDb } from '../storage/sqlite/db.js';
10
- import { createRun, getRun, setRunStatus, upsertDirtyKey, logError, RUN_STATUS } from '../migration/registry.js';
9
+ import { getRun, setRunStatus, RUN_STATUS } from '../migration/registry.js';
10
+ import { startDirtyTracker } from '../migration/tracker.js';
11
11
 
12
12
  function parseArgs(argv = process.argv.slice(2)) {
13
13
  const args = { _: [] };
@@ -18,71 +18,39 @@ function parseArgs(argv = process.argv.slice(2)) {
18
18
  else if (arg === '--run-id' && argv[i + 1]) args.runId = argv[++i];
19
19
  else if (arg === '--channels' && argv[i + 1]) args.channels = argv[++i];
20
20
  else if (arg === '--pragma-template' && argv[i + 1]) args.pragmaTemplate = argv[++i];
21
+ else if (arg === '--config-command' && argv[i + 1]) args.configCommand = argv[++i];
21
22
  else if (!arg.startsWith('--')) args._.push(arg);
22
23
  }
23
24
  return args;
24
25
  }
25
26
 
26
- async function checkKeyspaceNotifications(client) {
27
- let val = null;
28
- try {
29
- const config = await client.configGet('notify-keyspace-events');
30
- val = config && typeof config === 'object' ? config['notify-keyspace-events'] : null;
31
- } catch (_) {}
32
- if (!val || val === '') {
33
- throw new Error('Redis notify-keyspace-events is not set. Enable it (e.g. CONFIG SET notify-keyspace-events Kgxe) for the dirty-key tracker.');
34
- }
35
- return val;
36
- }
37
-
38
- async function startTracker(args) {
27
+ async function startTrackerCli(args) {
39
28
  const redisUrl = args.from || process.env.RESPLITE_IMPORT_FROM || 'redis://127.0.0.1:6379';
40
29
  const dbPath = args.to;
41
30
  const runId = args.runId || process.env.RESPLITE_RUN_ID;
42
31
  if (!dbPath || !runId) {
43
- console.error('Usage: resplite-dirty-tracker start --run-id <id> --to <db-path> [--from <redis-url>] [--channels keyevent]');
32
+ console.error('Usage: resplite-dirty-tracker start --run-id <id> --to <db-path> [--from <redis-url>] [--config-command <name>]');
44
33
  process.exit(1);
45
34
  }
46
35
 
47
- const db = openDb(dbPath, { pragmaTemplate: args.pragmaTemplate || 'default' });
48
- createRun(db, runId, redisUrl);
49
-
50
- const mainClient = createClient({ url: redisUrl });
51
- mainClient.on('error', (e) => console.error('Redis (main):', e.message));
52
- await mainClient.connect();
53
- await checkKeyspaceNotifications(mainClient);
54
-
55
- const subClient = mainClient.duplicate();
56
- subClient.on('error', (e) => {
57
- console.error('Redis (sub):', e.message);
58
- logError(db, runId, 'dirty_apply', 'Tracker disconnect: ' + e.message, null);
36
+ const tracker = await startDirtyTracker({
37
+ from: redisUrl,
38
+ to: dbPath,
39
+ runId,
40
+ pragmaTemplate: args.pragmaTemplate || 'default',
41
+ configCommand: args.configCommand || 'CONFIG',
59
42
  });
60
- await subClient.connect();
61
43
 
62
- const pattern = '__keyevent@0__:*';
63
- console.log('Subscribing to', pattern, '...');
64
-
65
- await subClient.pSubscribe(pattern, (message, channel) => {
66
- const event = typeof channel === 'string' ? channel.split(':').pop() : (channel && channel.toString?.())?.split(':').pop() || 'unknown';
67
- const key = message;
68
- try {
69
- upsertDirtyKey(db, runId, key, event);
70
- } catch (err) {
71
- logError(db, runId, 'dirty_apply', err.message, key);
72
- }
73
- });
44
+ console.log('Subscribed to __keyevent@0__:* — dirty tracker running. Ctrl+C to stop.');
74
45
 
75
46
  const shutdown = async () => {
76
47
  console.log('Stopping dirty tracker...');
77
- await subClient.pUnsubscribe(pattern);
78
- await subClient.quit();
79
- await mainClient.quit();
48
+ await tracker.stop();
80
49
  process.exit(0);
81
50
  };
82
51
 
83
52
  process.on('SIGINT', shutdown);
84
53
  process.on('SIGTERM', shutdown);
85
- console.log('Dirty tracker running. Ctrl+C to stop.');
86
54
  }
87
55
 
88
56
  async function stopTracker(args) {
@@ -105,7 +73,7 @@ async function stopTracker(args) {
105
73
  async function main() {
106
74
  const args = parseArgs();
107
75
  const sub = args._[0];
108
- if (sub === 'start') await startTracker(args);
76
+ if (sub === 'start') await startTrackerCli(args);
109
77
  else if (sub === 'stop') await stopTracker(args);
110
78
  else {
111
79
  console.error('Usage: resplite-dirty-tracker <start|stop> --run-id <id> --to <db-path> [--from <redis-url>]');
@@ -121,4 +89,4 @@ if (isMain) {
121
89
  });
122
90
  }
123
91
 
124
- export { parseArgs, startTracker, stopTracker };
92
+ export { parseArgs, stopTracker };
@@ -52,10 +52,12 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
52
52
  r = getRun(db, runId);
53
53
  }
54
54
 
55
- const batch = getDirtyBatch(db, runId, 'dirty', batch_keys);
56
- if (batch.length === 0) break;
55
+ const dirtyBatch = getDirtyBatch(db, runId, 'dirty', batch_keys);
56
+ const deletedBatch = getDirtyBatch(db, runId, 'deleted', batch_keys);
57
+ if (dirtyBatch.length === 0 && deletedBatch.length === 0) break;
57
58
 
58
- for (const { key: keyBuf } of batch) {
59
+ // ── Re-import (or remove) keys that changed while bulk was running ──
60
+ for (const { key: keyBuf } of dirtyBatch) {
59
61
  r = getRun(db, runId);
60
62
  if (r && r.status === RUN_STATUS.ABORTED) break;
61
63
  while (r && r.status === RUN_STATUS.PAUSED) {
@@ -91,6 +93,35 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
91
93
  markDirtyState(db, runId, keyBuf, 'error');
92
94
  }
93
95
  }
96
+
97
+ // ── Apply deletions recorded by the tracker (del / expired events) ──
98
+ // The tracker already determined these keys are gone; delete from destination.
99
+ // Marked as 'deleted' in the run counter; state changed away from 'deleted'
100
+ // so the next getDirtyBatch call won't return them again (avoiding infinite loop).
101
+ for (const { key: keyBuf } of deletedBatch) {
102
+ r = getRun(db, runId);
103
+ if (r && r.status === RUN_STATUS.ABORTED) break;
104
+ while (r && r.status === RUN_STATUS.PAUSED) {
105
+ await sleep(2000);
106
+ r = getRun(db, runId);
107
+ }
108
+
109
+ try {
110
+ keys.delete(keyBuf);
111
+ // Increment dirty_keys_deleted counter and transition state out of 'deleted'
112
+ // so this key is not re-processed in the next batch iteration.
113
+ const now = Date.now();
114
+ db.prepare(
115
+ `UPDATE migration_dirty_keys SET state = 'applied', last_seen_at = ? WHERE run_id = ? AND key = ?`
116
+ ).run(now, runId, keyBuf);
117
+ db.prepare(
118
+ `UPDATE migration_runs SET dirty_keys_deleted = dirty_keys_deleted + 1, updated_at = ? WHERE run_id = ?`
119
+ ).run(now, runId);
120
+ } catch (err) {
121
+ logError(db, runId, 'dirty_apply', err.message, keyBuf);
122
+ markDirtyState(db, runId, keyBuf, 'error');
123
+ }
124
+ }
94
125
  }
95
126
 
96
127
  return getRun(db, runId);
@@ -20,7 +20,7 @@
20
20
 
21
21
  import { createClient } from 'redis';
22
22
  import { openDb } from '../storage/sqlite/db.js';
23
- import { runPreflight } from './preflight.js';
23
+ import { runPreflight, readKeyspaceEvents, setKeyspaceEvents } from './preflight.js';
24
24
  import { runBulkImport } from './bulk.js';
25
25
  import { runApplyDirty } from './apply-dirty.js';
26
26
  import { runVerify } from './verify.js';
@@ -36,6 +36,7 @@ import { getRun, getDirtyCounts } from './registry.js';
36
36
  * @property {number} [maxRps=0] - Max requests/s (0 = unlimited).
37
37
  * @property {number} [batchKeys=200]
38
38
  * @property {number} [batchBytes=67108864] - 64 MB default.
39
+ * @property {string} [configCommand='CONFIG'] - Redis CONFIG command name. Override if renamed for security.
39
40
  */
40
41
 
41
42
  /**
@@ -43,7 +44,8 @@ import { getRun, getDirtyCounts } from './registry.js';
43
44
  *
44
45
  * @param {MigrationOptions} options
45
46
  * @returns {{
46
- * preflight(): Promise<import('./preflight.js').PreflightResult>,
47
+ * preflight(): Promise<object>,
48
+ * enableKeyspaceNotifications(opts?: { value?: string, merge?: boolean }): Promise<{ ok: boolean, previous: string|null, applied: string, error?: string }>,
47
49
  * bulk(opts?: { resume?: boolean, onProgress?: function }): Promise<object>,
48
50
  * status(): { run: object, dirty: object } | null,
49
51
  * applyDirty(opts?: { batchKeys?: number, maxRps?: number }): Promise<object>,
@@ -60,6 +62,7 @@ export function createMigration({
60
62
  maxRps = 0,
61
63
  batchKeys = 200,
62
64
  batchBytes = 64 * 1024 * 1024,
65
+ configCommand = 'CONFIG',
63
66
  } = {}) {
64
67
  if (!to) throw new Error('createMigration: "to" (db path) is required');
65
68
 
@@ -85,11 +88,27 @@ export function createMigration({
85
88
  /**
86
89
  * Step 0 — Preflight: inspect the source Redis instance.
87
90
  * Returns key count estimate, type distribution, keyspace-events config,
91
+ * `configCommandAvailable` (false when CONFIG is renamed/disabled),
88
92
  * and recommended import parameters.
89
93
  */
90
94
  async preflight() {
91
95
  const client = await getClient();
92
- return runPreflight(client);
96
+ return runPreflight(client, { configCommand });
97
+ },
98
+
99
+ /**
100
+ * Enable keyspace notifications on the source Redis (required for the dirty-key tracker).
101
+ * Reads the current value and merges the requested flags so existing flags are preserved.
102
+ * If the CONFIG command has been renamed, pass `configCommand` to `createMigration`.
103
+ *
104
+ * @param {{ value?: string, merge?: boolean }} [opts]
105
+ * - `value` Flags to apply. Defaults to `'KEA'` (keyevent + keyspace + all event types).
106
+ * - `merge` If true (default), merges flags into the existing value instead of overwriting.
107
+ * @returns {Promise<{ ok: boolean, previous: string|null, applied: string, error?: string }>}
108
+ */
109
+ async enableKeyspaceNotifications({ value = 'KEA', merge = true } = {}) {
110
+ const client = await getClient();
111
+ return setKeyspaceEvents(client, value, { configCommand, merge });
93
112
  },
94
113
 
95
114
  /**
@@ -166,5 +185,6 @@ export function createMigration({
166
185
  };
167
186
  }
168
187
 
169
- export { runPreflight, runBulkImport, runApplyDirty, runVerify };
188
+ export { runPreflight, readKeyspaceEvents, setKeyspaceEvents, runBulkImport, runApplyDirty, runVerify };
189
+ export { startDirtyTracker } from './tracker.js';
170
190
  export { getRun, getDirtyCounts, createRun, setRunStatus, logError } from './registry.js';
@@ -2,11 +2,86 @@
2
2
  * Preflight check: estimate key count, type distribution, notify-keyspace-events (SPEC_F §F.9 Step 0).
3
3
  */
4
4
 
5
+ const KEYSPACE_PARAM = 'notify-keyspace-events';
6
+
7
+ /**
8
+ * Read the current `notify-keyspace-events` value from Redis.
9
+ * Uses raw sendCommand so it works even if the CONFIG command has been renamed.
10
+ *
11
+ * @param {import('redis').RedisClientType} redisClient
12
+ * @param {string} [configCommand='CONFIG']
13
+ * @returns {Promise<{ value: string | null; available: boolean }>}
14
+ * `available: false` when the command is disabled or renamed and the default name fails.
15
+ */
16
+ export async function readKeyspaceEvents(redisClient, configCommand = 'CONFIG') {
17
+ try {
18
+ const raw = await redisClient.sendCommand([configCommand, 'GET', KEYSPACE_PARAM]);
19
+ // Redis returns a flat array ['notify-keyspace-events', '<value>']
20
+ if (Array.isArray(raw) && raw.length >= 2) {
21
+ return { value: String(raw[1] ?? ''), available: true };
22
+ }
23
+ // redis@4 may return an object { 'notify-keyspace-events': '<value>' }
24
+ if (raw && typeof raw === 'object' && !Array.isArray(raw)) {
25
+ const v = raw[KEYSPACE_PARAM];
26
+ return { value: typeof v === 'string' ? v : null, available: true };
27
+ }
28
+ return { value: null, available: true };
29
+ } catch (_) {
30
+ return { value: null, available: false };
31
+ }
32
+ }
33
+
5
34
  /**
35
+ * Set `notify-keyspace-events` to `value` on Redis.
36
+ * Optionally merges the new flags into the existing value instead of overwriting.
37
+ *
6
38
  * @param {import('redis').RedisClientType} redisClient
7
- * @returns {Promise<{ keyCountEstimate: number; typeDistribution: Record<string, number>; notifyKeyspaceEvents: string | null; recommended: { scan_count: number; max_concurrency: number; max_rps: number } }>}
39
+ * @param {string} value - Flags to apply (e.g. `'Kgxe'`).
40
+ * @param {{ configCommand?: string; merge?: boolean }} [options]
41
+ * @returns {Promise<{ ok: boolean; previous: string | null; applied: string; error?: string }>}
8
42
  */
9
- export async function runPreflight(redisClient) {
43
+ export async function setKeyspaceEvents(redisClient, value, { configCommand = 'CONFIG', merge = true } = {}) {
44
+ const { value: previous, available } = await readKeyspaceEvents(redisClient, configCommand);
45
+
46
+ if (!available) {
47
+ return {
48
+ ok: false,
49
+ previous: null,
50
+ applied: value,
51
+ error: `CONFIG command not available (it may have been renamed). ` +
52
+ `Use the configCommand option to supply the correct name, ` +
53
+ `or set notify-keyspace-events manually: ${configCommand} SET ${KEYSPACE_PARAM} ${value}`,
54
+ };
55
+ }
56
+
57
+ const applied = merge && previous ? mergeFlags(previous, value) : value;
58
+
59
+ try {
60
+ await redisClient.sendCommand([configCommand, 'SET', KEYSPACE_PARAM, applied]);
61
+ return { ok: true, previous, applied };
62
+ } catch (err) {
63
+ return { ok: false, previous, applied, error: err.message };
64
+ }
65
+ }
66
+
67
+ /**
68
+ * Merge two Redis notify-keyspace-events flag strings.
69
+ * Preserves all flags from both; result is sorted for readability.
70
+ * @param {string} existing
71
+ * @param {string} required
72
+ * @returns {string}
73
+ */
74
+ function mergeFlags(existing, required) {
75
+ const merged = new Set([...existing, ...required]);
76
+ return [...merged].sort().join('');
77
+ }
78
+
79
+ /**
80
+ * @param {import('redis').RedisClientType} redisClient
81
+ * @param {{ configCommand?: string }} [options]
82
+ * @returns {Promise<{ keyCountEstimate: number; typeDistribution: Record<string, number>; notifyKeyspaceEvents: string | null; configCommandAvailable: boolean; recommended: { scan_count: number; max_concurrency: number; max_rps: number } }>}
83
+ */
84
+ export async function runPreflight(redisClient, { configCommand = 'CONFIG' } = {}) {
10
85
  let keyCountEstimate = 0;
11
86
  try {
12
87
  keyCountEstimate = await redisClient.dbSize();
@@ -38,14 +113,8 @@ export async function runPreflight(redisClient) {
38
113
  } while (cursor !== 0);
39
114
  }
40
115
 
41
- let notifyKeyspaceEvents = null;
42
- try {
43
- const config = await redisClient.configGet('notify-keyspace-events');
44
- notifyKeyspaceEvents = config && typeof config === 'object' ? config['notify-keyspace-events'] : null;
45
- if (typeof notifyKeyspaceEvents !== 'string') notifyKeyspaceEvents = null;
46
- } catch (_) {
47
- notifyKeyspaceEvents = null;
48
- }
116
+ const { value: notifyKeyspaceEvents, available: configCommandAvailable } =
117
+ await readKeyspaceEvents(redisClient, configCommand);
49
118
 
50
119
  const recommended = {
51
120
  scan_count: 1000,
@@ -57,6 +126,7 @@ export async function runPreflight(redisClient) {
57
126
  keyCountEstimate,
58
127
  typeDistribution,
59
128
  notifyKeyspaceEvents,
129
+ configCommandAvailable,
60
130
  recommended,
61
131
  };
62
132
  }
@@ -0,0 +1,90 @@
1
+ /**
2
+ * Dirty-key tracker: subscribe to Redis keyspace notifications and record
3
+ * modified/deleted keys in the migration registry (SPEC_F §F.6).
4
+ *
5
+ * Programmatic API:
6
+ * const tracker = await startDirtyTracker({ from, to, runId });
7
+ * // ... run bulk import and other Redis writes ...
8
+ * await tracker.stop();
9
+ */
10
+
11
+ import { createClient } from 'redis';
12
+ import { openDb } from '../storage/sqlite/db.js';
13
+ import { createRun, upsertDirtyKey, logError } from './registry.js';
14
+ import { readKeyspaceEvents } from './preflight.js';
15
+
16
+ const KEYEVENT_PATTERN = '__keyevent@0__:*';
17
+
18
+ /**
19
+ * Start the dirty-key tracker: connect a subscriber client and record every
20
+ * keyspace event into the migration registry.
21
+ *
22
+ * Resolves once the subscription is active. Call `stop()` to disconnect.
23
+ *
24
+ * @param {object} options
25
+ * @param {string} [options.from='redis://127.0.0.1:6379'] - Source Redis URL.
26
+ * @param {string} options.to - SQLite DB path (must already exist or be created by bulk).
27
+ * @param {string} options.runId - Migration run identifier.
28
+ * @param {string} [options.pragmaTemplate='default']
29
+ * @param {string} [options.configCommand='CONFIG'] - CONFIG command name (in case it was renamed).
30
+ * @returns {Promise<{ stop(): Promise<void> }>}
31
+ * @throws {Error} If keyspace notifications are not enabled on Redis.
32
+ */
33
+ export async function startDirtyTracker({
34
+ from = 'redis://127.0.0.1:6379',
35
+ to,
36
+ runId,
37
+ pragmaTemplate = 'default',
38
+ configCommand = 'CONFIG',
39
+ } = {}) {
40
+ if (!to) throw new Error('startDirtyTracker: "to" (db path) is required');
41
+ if (!runId) throw new Error('startDirtyTracker: "runId" is required');
42
+
43
+ const db = openDb(to, { pragmaTemplate });
44
+ createRun(db, runId, from); // idempotent — safe to call even if bulk already created the run
45
+
46
+ const mainClient = createClient({ url: from });
47
+ mainClient.on('error', (err) =>
48
+ logError(db, runId, 'dirty_apply', 'Tracker connection error: ' + err.message, null)
49
+ );
50
+ await mainClient.connect();
51
+
52
+ // Validate keyspace notifications before subscribing
53
+ const { value: eventsValue, available } = await readKeyspaceEvents(mainClient, configCommand);
54
+ if (!available || !eventsValue || eventsValue === '') {
55
+ await mainClient.quit().catch(() => {});
56
+ throw new Error(
57
+ `Redis notify-keyspace-events is not set. ` +
58
+ `Enable it first: ${configCommand} SET notify-keyspace-events KEA\n` +
59
+ `Or call m.enableKeyspaceNotifications() from the programmatic API.`
60
+ );
61
+ }
62
+
63
+ const subClient = mainClient.duplicate();
64
+ subClient.on('error', (err) =>
65
+ logError(db, runId, 'dirty_apply', 'Tracker subscriber error: ' + err.message, null)
66
+ );
67
+ await subClient.connect();
68
+
69
+ await subClient.pSubscribe(KEYEVENT_PATTERN, (message, channel) => {
70
+ const event = typeof channel === 'string'
71
+ ? channel.split(':').pop()
72
+ : String(channel ?? '').split(':').pop() || 'unknown';
73
+ try {
74
+ upsertDirtyKey(db, runId, message, event);
75
+ } catch (err) {
76
+ logError(db, runId, 'dirty_apply', err.message, message);
77
+ }
78
+ });
79
+
80
+ return {
81
+ /**
82
+ * Unsubscribe and disconnect. Safe to call multiple times.
83
+ */
84
+ async stop() {
85
+ await subClient.pUnsubscribe(KEYEVENT_PATTERN).catch(() => {});
86
+ await subClient.quit().catch(() => {});
87
+ await mainClient.quit().catch(() => {});
88
+ },
89
+ };
90
+ }
@@ -0,0 +1,201 @@
1
+ /**
2
+ * Integration test: dirty-key tracker + full migration flow.
3
+ *
4
+ * Requires a local Redis instance on redis://127.0.0.1:6379.
5
+ * The test is skipped automatically if Redis is unavailable or
6
+ * if keyspace notifications cannot be enabled.
7
+ *
8
+ * Flow being tested:
9
+ * 1. Write initial keys to Redis
10
+ * 2. Enable keyspace notifications
11
+ * 3. Start dirty tracker
12
+ * 4. Run bulk import (captures snapshot at T=0)
13
+ * 5. Modify keys in Redis after bulk (tracker records them as dirty)
14
+ * 6. Stop tracker
15
+ * 7. Apply-dirty (reconciles post-bulk changes into destination)
16
+ * 8. Verify (destination matches Redis)
17
+ */
18
+
19
+ import { describe, it, before, after } from 'node:test';
20
+ import assert from 'node:assert/strict';
21
+ import { createClient } from 'redis';
22
+ import { createMigration, startDirtyTracker } from '../../src/migration/index.js';
23
+ import { tmpDbPath } from '../helpers/tmp.js';
24
+
25
+ const REDIS_URL = 'redis://127.0.0.1:6379';
26
+ const PREFIX = `__resplite_tracker_test_${process.pid}__`;
27
+
28
+ /** Connect to local Redis; return null if unavailable. */
29
+ async function tryConnectRedis() {
30
+ const client = createClient({ url: REDIS_URL });
31
+ try {
32
+ await client.connect();
33
+ await client.ping();
34
+ return client;
35
+ } catch {
36
+ await client.quit().catch(() => {});
37
+ return null;
38
+ }
39
+ }
40
+
41
+ /** Delete all keys under PREFIX. */
42
+ async function cleanup(redis) {
43
+ let cursor = 0;
44
+ do {
45
+ const result = await redis.scan(cursor, { MATCH: `${PREFIX}:*`, COUNT: 200 });
46
+ cursor = typeof result.cursor === 'number' ? result.cursor : parseInt(result.cursor, 10);
47
+ const keys = result.keys ?? [];
48
+ if (keys.length) await redis.del(keys);
49
+ } while (cursor !== 0);
50
+ }
51
+
52
+ describe('dirty tracker integration', { timeout: 30_000 }, () => {
53
+ let redis = null;
54
+ let originalEventsValue = '';
55
+
56
+ before(async () => {
57
+ redis = await tryConnectRedis();
58
+ if (!redis) return; // tests will skip inside each `it`
59
+
60
+ // Save existing notify-keyspace-events so we can restore it after
61
+ const raw = await redis.sendCommand(['CONFIG', 'GET', 'notify-keyspace-events']);
62
+ originalEventsValue = Array.isArray(raw) ? (raw[1] ?? '') : '';
63
+
64
+ await cleanup(redis);
65
+ });
66
+
67
+ after(async () => {
68
+ if (!redis) return;
69
+ // Restore original keyspace events value
70
+ await redis.sendCommand(['CONFIG', 'SET', 'notify-keyspace-events', originalEventsValue]);
71
+ await cleanup(redis);
72
+ await redis.quit();
73
+ });
74
+
75
+ it('bulk + tracker + apply-dirty produces a fully reconciled destination', async (t) => {
76
+ if (!redis) {
77
+ t.skip('local Redis not available');
78
+ return;
79
+ }
80
+
81
+ // ── Seed: write initial keys ─────────────────────────────────────────
82
+ const keys = {
83
+ string: `${PREFIX}:str`,
84
+ hash: `${PREFIX}:hash`,
85
+ set: `${PREFIX}:set`,
86
+ list: `${PREFIX}:list`,
87
+ toDelete: `${PREFIX}:will-delete`,
88
+ toModify: `${PREFIX}:will-modify`,
89
+ };
90
+
91
+ await redis.set(keys.string, 'hello');
92
+ await redis.hSet(keys.hash, { field1: 'v1', field2: 'v2' });
93
+ await redis.sAdd(keys.set, ['a', 'b', 'c']);
94
+ await redis.rPush(keys.list, ['x', 'y', 'z']);
95
+ await redis.set(keys.toDelete, 'delete-me');
96
+ await redis.set(keys.toModify, 'original');
97
+
98
+ // ── Setup: enable keyspace notifications ────────────────────────────
99
+ const dbPath = tmpDbPath();
100
+ const runId = `tracker-test-${Date.now()}`;
101
+
102
+ const m = createMigration({ from: REDIS_URL, to: dbPath, runId });
103
+
104
+ const ks = await m.enableKeyspaceNotifications({ value: 'KEA' });
105
+ assert.ok(ks.ok, `Failed to enable keyspace notifications: ${ks.error}`);
106
+
107
+ // ── Start tracker BEFORE bulk ────────────────────────────────────────
108
+ const tracker = await startDirtyTracker({ from: REDIS_URL, to: dbPath, runId });
109
+
110
+ try {
111
+ // ── Bulk import (captures the initial snapshot) ──────────────────
112
+ const run = await m.bulk();
113
+ assert.equal(run.status, 'completed');
114
+ assert.ok(run.migrated_keys >= 6, `Expected ≥6 migrated keys, got ${run.migrated_keys}`);
115
+
116
+ // ── Post-bulk mutations (tracker will record these) ──────────────
117
+ await redis.set(keys.toModify, 'modified-after-bulk');
118
+ await redis.del(keys.toDelete);
119
+ await redis.set(`${PREFIX}:new-key`, 'added-after-bulk');
120
+
121
+ // Give the tracker time to process the keyspace events
122
+ await new Promise((r) => setTimeout(r, 600));
123
+ } finally {
124
+ await tracker.stop();
125
+ }
126
+
127
+ // ── Verify dirty keys were captured ─────────────────────────────────
128
+ const { dirty } = m.status();
129
+ assert.ok(
130
+ dirty.dirty + dirty.deleted >= 3,
131
+ `Expected ≥3 dirty/deleted keys, got dirty=${dirty.dirty} deleted=${dirty.deleted}`
132
+ );
133
+
134
+ // ── Apply-dirty: reconcile post-bulk changes ─────────────────────────
135
+ const afterApply = await m.applyDirty();
136
+ const totalReconciled = afterApply.dirty_keys_applied + afterApply.dirty_keys_deleted;
137
+ assert.ok(totalReconciled >= 3, `Expected ≥3 reconciled keys, got ${totalReconciled}`);
138
+
139
+ // ── Verify: destination should now match Redis for our test prefix ───
140
+ const result = await m.verify({ samplePct: 100, maxSample: 5000 });
141
+
142
+ // Filter mismatches to only those under our test prefix
143
+ const ourMismatches = result.mismatches.filter((mm) => mm.key.startsWith(PREFIX));
144
+ assert.equal(
145
+ ourMismatches.length,
146
+ 0,
147
+ `Unexpected mismatches in test keys:\n${JSON.stringify(ourMismatches, null, 2)}`
148
+ );
149
+
150
+ await m.close();
151
+ });
152
+
153
+ it('startDirtyTracker throws when keyspace notifications are disabled', async (t) => {
154
+ if (!redis) {
155
+ t.skip('local Redis not available');
156
+ return;
157
+ }
158
+
159
+ // Disable keyspace notifications
160
+ await redis.sendCommand(['CONFIG', 'SET', 'notify-keyspace-events', '']);
161
+
162
+ const dbPath = tmpDbPath();
163
+ const runId = `tracker-noevents-${Date.now()}`;
164
+
165
+ await assert.rejects(
166
+ () => startDirtyTracker({ from: REDIS_URL, to: dbPath, runId }),
167
+ /notify-keyspace-events/
168
+ );
169
+
170
+ // Re-enable for subsequent tests
171
+ await redis.sendCommand(['CONFIG', 'SET', 'notify-keyspace-events', 'KEA']);
172
+ });
173
+
174
+ it('enableKeyspaceNotifications merges flags without overwriting existing ones', async (t) => {
175
+ if (!redis) {
176
+ t.skip('local Redis not available');
177
+ return;
178
+ }
179
+
180
+ const dbPath = tmpDbPath();
181
+ const m = createMigration({ from: REDIS_URL, to: dbPath, runId: `ks-merge-${Date.now()}` });
182
+
183
+ // Set a partial value first
184
+ await redis.sendCommand(['CONFIG', 'SET', 'notify-keyspace-events', 'Kg']);
185
+
186
+ const result = await m.enableKeyspaceNotifications({ value: 'KEA', merge: true });
187
+ assert.ok(result.ok, `Expected ok=true: ${result.error}`);
188
+ // Redis may reorder flags (e.g. 'Kg' → 'gK'); only check flag membership
189
+ assert.ok(typeof result.previous === 'string', 'previous should be a string');
190
+ assert.ok(result.previous.includes('K'), `Missing K in previous="${result.previous}"`);
191
+ assert.ok(result.previous.includes('g'), `Missing g in previous="${result.previous}"`);
192
+
193
+ // Applied value must contain K, E, A and the original g
194
+ const applied = result.applied;
195
+ assert.ok(applied.includes('K'), `Missing K in applied="${applied}"`);
196
+ assert.ok(applied.includes('E'), `Missing E in applied="${applied}"`);
197
+ assert.ok(applied.includes('A'), `Missing A in applied="${applied}"`);
198
+
199
+ await m.close();
200
+ });
201
+ });