resplite 1.0.6 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -3
- package/package.json +1 -1
- package/src/cli/resplite-dirty-tracker.js +15 -47
- package/src/migration/apply-dirty.js +34 -3
- package/src/migration/index.js +24 -4
- package/src/migration/preflight.js +80 -10
- package/src/migration/tracker.js +90 -0
- package/test/integration/migration-dirty-tracker.test.js +201 -0
package/README.md
CHANGED
|
@@ -341,16 +341,18 @@ For large datasets (~30 GB), use the Dirty Key Registry flow so the bulk of the
|
|
|
341
341
|
**Enable keyspace notifications in Redis** (required for the dirty-key tracker). Either run at runtime:
|
|
342
342
|
|
|
343
343
|
```bash
|
|
344
|
-
redis-cli CONFIG SET notify-keyspace-events
|
|
344
|
+
redis-cli CONFIG SET notify-keyspace-events KEA
|
|
345
345
|
```
|
|
346
346
|
|
|
347
347
|
Or add to `redis.conf` and restart Redis:
|
|
348
348
|
|
|
349
349
|
```
|
|
350
|
-
notify-keyspace-events
|
|
350
|
+
notify-keyspace-events KEA
|
|
351
351
|
```
|
|
352
352
|
|
|
353
|
-
(`K` = keyspace, `
|
|
353
|
+
(`K` = keyspace prefix, `E` = keyevent prefix, `A` = all event types — lets the tracker see every key change and expiration.)
|
|
354
|
+
|
|
355
|
+
> **Renamed CONFIG command?** Some Redis deployments rename `CONFIG` for security. Pass `--config-command <name>` to the CLI tools, or the `configCommand` option to the JS API — see below.
|
|
354
356
|
|
|
355
357
|
1. **Preflight** – Check Redis, key count, type distribution, and that keyspace notifications are enabled:
|
|
356
358
|
```bash
|
|
@@ -360,6 +362,8 @@ notify-keyspace-events Kgxe
|
|
|
360
362
|
2. **Start dirty-key tracker** – Captures keys modified during bulk (requires `notify-keyspace-events` in Redis):
|
|
361
363
|
```bash
|
|
362
364
|
npx resplite-dirty-tracker start --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db
|
|
365
|
+
# If CONFIG was renamed:
|
|
366
|
+
npx resplite-dirty-tracker start --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db --config-command MYCONFIG
|
|
363
367
|
```
|
|
364
368
|
|
|
365
369
|
3. **Bulk import** – SCAN and copy all keys; progress is checkpointed and resumable:
|
|
@@ -408,14 +412,25 @@ const m = createMigration({
|
|
|
408
412
|
batchBytes: 64 * 1024 * 1024, // 64 MB
|
|
409
413
|
maxRps: 0, // 0 = unlimited
|
|
410
414
|
pragmaTemplate: 'default',
|
|
415
|
+
|
|
416
|
+
// If your Redis deployment renamed CONFIG for security:
|
|
417
|
+
// configCommand: 'MYCONFIG',
|
|
411
418
|
});
|
|
412
419
|
|
|
413
420
|
// Step 0 — Preflight: inspect Redis before starting
|
|
414
421
|
const info = await m.preflight();
|
|
415
422
|
console.log('keys (estimate):', info.keyCountEstimate);
|
|
416
423
|
console.log('type distribution:', info.typeDistribution);
|
|
424
|
+
console.log('notify-keyspace-events:', info.notifyKeyspaceEvents);
|
|
425
|
+
console.log('CONFIG available:', info.configCommandAvailable); // false if renamed
|
|
417
426
|
console.log('recommended params:', info.recommended);
|
|
418
427
|
|
|
428
|
+
// Step 0b — Enable keyspace notifications (required for dirty-key tracking)
|
|
429
|
+
// Reads the current value and merges the new flags — existing flags are preserved.
|
|
430
|
+
const ks = await m.enableKeyspaceNotifications();
|
|
431
|
+
// → { ok: true, previous: '', applied: 'KEA' }
|
|
432
|
+
// If CONFIG is renamed and configCommand was not set, ok=false and error explains how to fix it.
|
|
433
|
+
|
|
419
434
|
// Step 1 — Bulk import (checkpointed, resumable)
|
|
420
435
|
await m.bulk({
|
|
421
436
|
resume: false, // true to resume a previous run
|
|
@@ -441,6 +456,33 @@ await m.close();
|
|
|
441
456
|
|
|
442
457
|
The dirty-key tracker (to capture writes during bulk) still runs as a separate process via `npx resplite-dirty-tracker`. The API above handles everything else in a single script.
|
|
443
458
|
|
|
459
|
+
#### Renamed CONFIG command
|
|
460
|
+
|
|
461
|
+
If your Redis instance has the `CONFIG` command renamed (a common hardening practice), pass the new name to `createMigration`:
|
|
462
|
+
|
|
463
|
+
```javascript
|
|
464
|
+
const m = createMigration({
|
|
465
|
+
from: 'redis://10.0.0.10:6379',
|
|
466
|
+
to: './resplite.db',
|
|
467
|
+
runId: 'run_001',
|
|
468
|
+
configCommand: 'MYCONFIG', // the renamed command
|
|
469
|
+
});
|
|
470
|
+
|
|
471
|
+
// preflight will use MYCONFIG GET notify-keyspace-events
|
|
472
|
+
const info = await m.preflight();
|
|
473
|
+
// info.configCommandAvailable → false if the name is wrong
|
|
474
|
+
|
|
475
|
+
// enableKeyspaceNotifications will use MYCONFIG SET notify-keyspace-events KEA
|
|
476
|
+
const result = await m.enableKeyspaceNotifications({ value: 'KEA' });
|
|
477
|
+
```
|
|
478
|
+
|
|
479
|
+
The same flag is available in the CLI:
|
|
480
|
+
|
|
481
|
+
```bash
|
|
482
|
+
npx resplite-dirty-tracker start --run-id run_001 --to ./resplite.db \
|
|
483
|
+
--from redis://10.0.0.10:6379 --config-command MYCONFIG
|
|
484
|
+
```
|
|
485
|
+
|
|
444
486
|
#### Low-level re-exports
|
|
445
487
|
|
|
446
488
|
If you need more control, the individual functions and registry helpers are also exported:
|
package/package.json
CHANGED
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
* Usage: resplite-dirty-tracker start|stop [options]
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import { createClient } from 'redis';
|
|
8
7
|
import { fileURLToPath } from 'node:url';
|
|
9
8
|
import { openDb } from '../storage/sqlite/db.js';
|
|
10
|
-
import {
|
|
9
|
+
import { getRun, setRunStatus, RUN_STATUS } from '../migration/registry.js';
|
|
10
|
+
import { startDirtyTracker } from '../migration/tracker.js';
|
|
11
11
|
|
|
12
12
|
function parseArgs(argv = process.argv.slice(2)) {
|
|
13
13
|
const args = { _: [] };
|
|
@@ -18,71 +18,39 @@ function parseArgs(argv = process.argv.slice(2)) {
|
|
|
18
18
|
else if (arg === '--run-id' && argv[i + 1]) args.runId = argv[++i];
|
|
19
19
|
else if (arg === '--channels' && argv[i + 1]) args.channels = argv[++i];
|
|
20
20
|
else if (arg === '--pragma-template' && argv[i + 1]) args.pragmaTemplate = argv[++i];
|
|
21
|
+
else if (arg === '--config-command' && argv[i + 1]) args.configCommand = argv[++i];
|
|
21
22
|
else if (!arg.startsWith('--')) args._.push(arg);
|
|
22
23
|
}
|
|
23
24
|
return args;
|
|
24
25
|
}
|
|
25
26
|
|
|
26
|
-
async function
|
|
27
|
-
let val = null;
|
|
28
|
-
try {
|
|
29
|
-
const config = await client.configGet('notify-keyspace-events');
|
|
30
|
-
val = config && typeof config === 'object' ? config['notify-keyspace-events'] : null;
|
|
31
|
-
} catch (_) {}
|
|
32
|
-
if (!val || val === '') {
|
|
33
|
-
throw new Error('Redis notify-keyspace-events is not set. Enable it (e.g. CONFIG SET notify-keyspace-events Kgxe) for the dirty-key tracker.');
|
|
34
|
-
}
|
|
35
|
-
return val;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
async function startTracker(args) {
|
|
27
|
+
async function startTrackerCli(args) {
|
|
39
28
|
const redisUrl = args.from || process.env.RESPLITE_IMPORT_FROM || 'redis://127.0.0.1:6379';
|
|
40
29
|
const dbPath = args.to;
|
|
41
30
|
const runId = args.runId || process.env.RESPLITE_RUN_ID;
|
|
42
31
|
if (!dbPath || !runId) {
|
|
43
|
-
console.error('Usage: resplite-dirty-tracker start --run-id <id> --to <db-path> [--from <redis-url>] [--
|
|
32
|
+
console.error('Usage: resplite-dirty-tracker start --run-id <id> --to <db-path> [--from <redis-url>] [--config-command <name>]');
|
|
44
33
|
process.exit(1);
|
|
45
34
|
}
|
|
46
35
|
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
await checkKeyspaceNotifications(mainClient);
|
|
54
|
-
|
|
55
|
-
const subClient = mainClient.duplicate();
|
|
56
|
-
subClient.on('error', (e) => {
|
|
57
|
-
console.error('Redis (sub):', e.message);
|
|
58
|
-
logError(db, runId, 'dirty_apply', 'Tracker disconnect: ' + e.message, null);
|
|
36
|
+
const tracker = await startDirtyTracker({
|
|
37
|
+
from: redisUrl,
|
|
38
|
+
to: dbPath,
|
|
39
|
+
runId,
|
|
40
|
+
pragmaTemplate: args.pragmaTemplate || 'default',
|
|
41
|
+
configCommand: args.configCommand || 'CONFIG',
|
|
59
42
|
});
|
|
60
|
-
await subClient.connect();
|
|
61
43
|
|
|
62
|
-
|
|
63
|
-
console.log('Subscribing to', pattern, '...');
|
|
64
|
-
|
|
65
|
-
await subClient.pSubscribe(pattern, (message, channel) => {
|
|
66
|
-
const event = typeof channel === 'string' ? channel.split(':').pop() : (channel && channel.toString?.())?.split(':').pop() || 'unknown';
|
|
67
|
-
const key = message;
|
|
68
|
-
try {
|
|
69
|
-
upsertDirtyKey(db, runId, key, event);
|
|
70
|
-
} catch (err) {
|
|
71
|
-
logError(db, runId, 'dirty_apply', err.message, key);
|
|
72
|
-
}
|
|
73
|
-
});
|
|
44
|
+
console.log('Subscribed to __keyevent@0__:* — dirty tracker running. Ctrl+C to stop.');
|
|
74
45
|
|
|
75
46
|
const shutdown = async () => {
|
|
76
47
|
console.log('Stopping dirty tracker...');
|
|
77
|
-
await
|
|
78
|
-
await subClient.quit();
|
|
79
|
-
await mainClient.quit();
|
|
48
|
+
await tracker.stop();
|
|
80
49
|
process.exit(0);
|
|
81
50
|
};
|
|
82
51
|
|
|
83
52
|
process.on('SIGINT', shutdown);
|
|
84
53
|
process.on('SIGTERM', shutdown);
|
|
85
|
-
console.log('Dirty tracker running. Ctrl+C to stop.');
|
|
86
54
|
}
|
|
87
55
|
|
|
88
56
|
async function stopTracker(args) {
|
|
@@ -105,7 +73,7 @@ async function stopTracker(args) {
|
|
|
105
73
|
async function main() {
|
|
106
74
|
const args = parseArgs();
|
|
107
75
|
const sub = args._[0];
|
|
108
|
-
if (sub === 'start') await
|
|
76
|
+
if (sub === 'start') await startTrackerCli(args);
|
|
109
77
|
else if (sub === 'stop') await stopTracker(args);
|
|
110
78
|
else {
|
|
111
79
|
console.error('Usage: resplite-dirty-tracker <start|stop> --run-id <id> --to <db-path> [--from <redis-url>]');
|
|
@@ -121,4 +89,4 @@ if (isMain) {
|
|
|
121
89
|
});
|
|
122
90
|
}
|
|
123
91
|
|
|
124
|
-
export { parseArgs,
|
|
92
|
+
export { parseArgs, stopTracker };
|
|
@@ -52,10 +52,12 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
|
|
|
52
52
|
r = getRun(db, runId);
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
-
const
|
|
56
|
-
|
|
55
|
+
const dirtyBatch = getDirtyBatch(db, runId, 'dirty', batch_keys);
|
|
56
|
+
const deletedBatch = getDirtyBatch(db, runId, 'deleted', batch_keys);
|
|
57
|
+
if (dirtyBatch.length === 0 && deletedBatch.length === 0) break;
|
|
57
58
|
|
|
58
|
-
|
|
59
|
+
// ── Re-import (or remove) keys that changed while bulk was running ──
|
|
60
|
+
for (const { key: keyBuf } of dirtyBatch) {
|
|
59
61
|
r = getRun(db, runId);
|
|
60
62
|
if (r && r.status === RUN_STATUS.ABORTED) break;
|
|
61
63
|
while (r && r.status === RUN_STATUS.PAUSED) {
|
|
@@ -91,6 +93,35 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
|
|
|
91
93
|
markDirtyState(db, runId, keyBuf, 'error');
|
|
92
94
|
}
|
|
93
95
|
}
|
|
96
|
+
|
|
97
|
+
// ── Apply deletions recorded by the tracker (del / expired events) ──
|
|
98
|
+
// The tracker already determined these keys are gone; delete from destination.
|
|
99
|
+
// Marked as 'deleted' in the run counter; state changed away from 'deleted'
|
|
100
|
+
// so the next getDirtyBatch call won't return them again (avoiding infinite loop).
|
|
101
|
+
for (const { key: keyBuf } of deletedBatch) {
|
|
102
|
+
r = getRun(db, runId);
|
|
103
|
+
if (r && r.status === RUN_STATUS.ABORTED) break;
|
|
104
|
+
while (r && r.status === RUN_STATUS.PAUSED) {
|
|
105
|
+
await sleep(2000);
|
|
106
|
+
r = getRun(db, runId);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
try {
|
|
110
|
+
keys.delete(keyBuf);
|
|
111
|
+
// Increment dirty_keys_deleted counter and transition state out of 'deleted'
|
|
112
|
+
// so this key is not re-processed in the next batch iteration.
|
|
113
|
+
const now = Date.now();
|
|
114
|
+
db.prepare(
|
|
115
|
+
`UPDATE migration_dirty_keys SET state = 'applied', last_seen_at = ? WHERE run_id = ? AND key = ?`
|
|
116
|
+
).run(now, runId, keyBuf);
|
|
117
|
+
db.prepare(
|
|
118
|
+
`UPDATE migration_runs SET dirty_keys_deleted = dirty_keys_deleted + 1, updated_at = ? WHERE run_id = ?`
|
|
119
|
+
).run(now, runId);
|
|
120
|
+
} catch (err) {
|
|
121
|
+
logError(db, runId, 'dirty_apply', err.message, keyBuf);
|
|
122
|
+
markDirtyState(db, runId, keyBuf, 'error');
|
|
123
|
+
}
|
|
124
|
+
}
|
|
94
125
|
}
|
|
95
126
|
|
|
96
127
|
return getRun(db, runId);
|
package/src/migration/index.js
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
|
|
21
21
|
import { createClient } from 'redis';
|
|
22
22
|
import { openDb } from '../storage/sqlite/db.js';
|
|
23
|
-
import { runPreflight } from './preflight.js';
|
|
23
|
+
import { runPreflight, readKeyspaceEvents, setKeyspaceEvents } from './preflight.js';
|
|
24
24
|
import { runBulkImport } from './bulk.js';
|
|
25
25
|
import { runApplyDirty } from './apply-dirty.js';
|
|
26
26
|
import { runVerify } from './verify.js';
|
|
@@ -36,6 +36,7 @@ import { getRun, getDirtyCounts } from './registry.js';
|
|
|
36
36
|
* @property {number} [maxRps=0] - Max requests/s (0 = unlimited).
|
|
37
37
|
* @property {number} [batchKeys=200]
|
|
38
38
|
* @property {number} [batchBytes=67108864] - 64 MB default.
|
|
39
|
+
* @property {string} [configCommand='CONFIG'] - Redis CONFIG command name. Override if renamed for security.
|
|
39
40
|
*/
|
|
40
41
|
|
|
41
42
|
/**
|
|
@@ -43,7 +44,8 @@ import { getRun, getDirtyCounts } from './registry.js';
|
|
|
43
44
|
*
|
|
44
45
|
* @param {MigrationOptions} options
|
|
45
46
|
* @returns {{
|
|
46
|
-
* preflight(): Promise<
|
|
47
|
+
* preflight(): Promise<object>,
|
|
48
|
+
* enableKeyspaceNotifications(opts?: { value?: string, merge?: boolean }): Promise<{ ok: boolean, previous: string|null, applied: string, error?: string }>,
|
|
47
49
|
* bulk(opts?: { resume?: boolean, onProgress?: function }): Promise<object>,
|
|
48
50
|
* status(): { run: object, dirty: object } | null,
|
|
49
51
|
* applyDirty(opts?: { batchKeys?: number, maxRps?: number }): Promise<object>,
|
|
@@ -60,6 +62,7 @@ export function createMigration({
|
|
|
60
62
|
maxRps = 0,
|
|
61
63
|
batchKeys = 200,
|
|
62
64
|
batchBytes = 64 * 1024 * 1024,
|
|
65
|
+
configCommand = 'CONFIG',
|
|
63
66
|
} = {}) {
|
|
64
67
|
if (!to) throw new Error('createMigration: "to" (db path) is required');
|
|
65
68
|
|
|
@@ -85,11 +88,27 @@ export function createMigration({
|
|
|
85
88
|
/**
|
|
86
89
|
* Step 0 — Preflight: inspect the source Redis instance.
|
|
87
90
|
* Returns key count estimate, type distribution, keyspace-events config,
|
|
91
|
+
* `configCommandAvailable` (false when CONFIG is renamed/disabled),
|
|
88
92
|
* and recommended import parameters.
|
|
89
93
|
*/
|
|
90
94
|
async preflight() {
|
|
91
95
|
const client = await getClient();
|
|
92
|
-
return runPreflight(client);
|
|
96
|
+
return runPreflight(client, { configCommand });
|
|
97
|
+
},
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Enable keyspace notifications on the source Redis (required for the dirty-key tracker).
|
|
101
|
+
* Reads the current value and merges the requested flags so existing flags are preserved.
|
|
102
|
+
* If the CONFIG command has been renamed, pass `configCommand` to `createMigration`.
|
|
103
|
+
*
|
|
104
|
+
* @param {{ value?: string, merge?: boolean }} [opts]
|
|
105
|
+
* - `value` Flags to apply. Defaults to `'KEA'` (keyevent + keyspace + all event types).
|
|
106
|
+
* - `merge` If true (default), merges flags into the existing value instead of overwriting.
|
|
107
|
+
* @returns {Promise<{ ok: boolean, previous: string|null, applied: string, error?: string }>}
|
|
108
|
+
*/
|
|
109
|
+
async enableKeyspaceNotifications({ value = 'KEA', merge = true } = {}) {
|
|
110
|
+
const client = await getClient();
|
|
111
|
+
return setKeyspaceEvents(client, value, { configCommand, merge });
|
|
93
112
|
},
|
|
94
113
|
|
|
95
114
|
/**
|
|
@@ -166,5 +185,6 @@ export function createMigration({
|
|
|
166
185
|
};
|
|
167
186
|
}
|
|
168
187
|
|
|
169
|
-
export { runPreflight, runBulkImport, runApplyDirty, runVerify };
|
|
188
|
+
export { runPreflight, readKeyspaceEvents, setKeyspaceEvents, runBulkImport, runApplyDirty, runVerify };
|
|
189
|
+
export { startDirtyTracker } from './tracker.js';
|
|
170
190
|
export { getRun, getDirtyCounts, createRun, setRunStatus, logError } from './registry.js';
|
|
@@ -2,11 +2,86 @@
|
|
|
2
2
|
* Preflight check: estimate key count, type distribution, notify-keyspace-events (SPEC_F §F.9 Step 0).
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
+
const KEYSPACE_PARAM = 'notify-keyspace-events';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Read the current `notify-keyspace-events` value from Redis.
|
|
9
|
+
* Uses raw sendCommand so it works even if the CONFIG command has been renamed.
|
|
10
|
+
*
|
|
11
|
+
* @param {import('redis').RedisClientType} redisClient
|
|
12
|
+
* @param {string} [configCommand='CONFIG']
|
|
13
|
+
* @returns {Promise<{ value: string | null; available: boolean }>}
|
|
14
|
+
* `available: false` when the command is disabled or renamed and the default name fails.
|
|
15
|
+
*/
|
|
16
|
+
export async function readKeyspaceEvents(redisClient, configCommand = 'CONFIG') {
|
|
17
|
+
try {
|
|
18
|
+
const raw = await redisClient.sendCommand([configCommand, 'GET', KEYSPACE_PARAM]);
|
|
19
|
+
// Redis returns a flat array ['notify-keyspace-events', '<value>']
|
|
20
|
+
if (Array.isArray(raw) && raw.length >= 2) {
|
|
21
|
+
return { value: String(raw[1] ?? ''), available: true };
|
|
22
|
+
}
|
|
23
|
+
// redis@4 may return an object { 'notify-keyspace-events': '<value>' }
|
|
24
|
+
if (raw && typeof raw === 'object' && !Array.isArray(raw)) {
|
|
25
|
+
const v = raw[KEYSPACE_PARAM];
|
|
26
|
+
return { value: typeof v === 'string' ? v : null, available: true };
|
|
27
|
+
}
|
|
28
|
+
return { value: null, available: true };
|
|
29
|
+
} catch (_) {
|
|
30
|
+
return { value: null, available: false };
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
5
34
|
/**
|
|
35
|
+
* Set `notify-keyspace-events` to `value` on Redis.
|
|
36
|
+
* Optionally merges the new flags into the existing value instead of overwriting.
|
|
37
|
+
*
|
|
6
38
|
* @param {import('redis').RedisClientType} redisClient
|
|
7
|
-
* @
|
|
39
|
+
* @param {string} value - Flags to apply (e.g. `'Kgxe'`).
|
|
40
|
+
* @param {{ configCommand?: string; merge?: boolean }} [options]
|
|
41
|
+
* @returns {Promise<{ ok: boolean; previous: string | null; applied: string; error?: string }>}
|
|
8
42
|
*/
|
|
9
|
-
export async function
|
|
43
|
+
export async function setKeyspaceEvents(redisClient, value, { configCommand = 'CONFIG', merge = true } = {}) {
|
|
44
|
+
const { value: previous, available } = await readKeyspaceEvents(redisClient, configCommand);
|
|
45
|
+
|
|
46
|
+
if (!available) {
|
|
47
|
+
return {
|
|
48
|
+
ok: false,
|
|
49
|
+
previous: null,
|
|
50
|
+
applied: value,
|
|
51
|
+
error: `CONFIG command not available (it may have been renamed). ` +
|
|
52
|
+
`Use the configCommand option to supply the correct name, ` +
|
|
53
|
+
`or set notify-keyspace-events manually: ${configCommand} SET ${KEYSPACE_PARAM} ${value}`,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const applied = merge && previous ? mergeFlags(previous, value) : value;
|
|
58
|
+
|
|
59
|
+
try {
|
|
60
|
+
await redisClient.sendCommand([configCommand, 'SET', KEYSPACE_PARAM, applied]);
|
|
61
|
+
return { ok: true, previous, applied };
|
|
62
|
+
} catch (err) {
|
|
63
|
+
return { ok: false, previous, applied, error: err.message };
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Merge two Redis notify-keyspace-events flag strings.
|
|
69
|
+
* Preserves all flags from both; result is sorted for readability.
|
|
70
|
+
* @param {string} existing
|
|
71
|
+
* @param {string} required
|
|
72
|
+
* @returns {string}
|
|
73
|
+
*/
|
|
74
|
+
function mergeFlags(existing, required) {
|
|
75
|
+
const merged = new Set([...existing, ...required]);
|
|
76
|
+
return [...merged].sort().join('');
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* @param {import('redis').RedisClientType} redisClient
|
|
81
|
+
* @param {{ configCommand?: string }} [options]
|
|
82
|
+
* @returns {Promise<{ keyCountEstimate: number; typeDistribution: Record<string, number>; notifyKeyspaceEvents: string | null; configCommandAvailable: boolean; recommended: { scan_count: number; max_concurrency: number; max_rps: number } }>}
|
|
83
|
+
*/
|
|
84
|
+
export async function runPreflight(redisClient, { configCommand = 'CONFIG' } = {}) {
|
|
10
85
|
let keyCountEstimate = 0;
|
|
11
86
|
try {
|
|
12
87
|
keyCountEstimate = await redisClient.dbSize();
|
|
@@ -38,14 +113,8 @@ export async function runPreflight(redisClient) {
|
|
|
38
113
|
} while (cursor !== 0);
|
|
39
114
|
}
|
|
40
115
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
const config = await redisClient.configGet('notify-keyspace-events');
|
|
44
|
-
notifyKeyspaceEvents = config && typeof config === 'object' ? config['notify-keyspace-events'] : null;
|
|
45
|
-
if (typeof notifyKeyspaceEvents !== 'string') notifyKeyspaceEvents = null;
|
|
46
|
-
} catch (_) {
|
|
47
|
-
notifyKeyspaceEvents = null;
|
|
48
|
-
}
|
|
116
|
+
const { value: notifyKeyspaceEvents, available: configCommandAvailable } =
|
|
117
|
+
await readKeyspaceEvents(redisClient, configCommand);
|
|
49
118
|
|
|
50
119
|
const recommended = {
|
|
51
120
|
scan_count: 1000,
|
|
@@ -57,6 +126,7 @@ export async function runPreflight(redisClient) {
|
|
|
57
126
|
keyCountEstimate,
|
|
58
127
|
typeDistribution,
|
|
59
128
|
notifyKeyspaceEvents,
|
|
129
|
+
configCommandAvailable,
|
|
60
130
|
recommended,
|
|
61
131
|
};
|
|
62
132
|
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dirty-key tracker: subscribe to Redis keyspace notifications and record
|
|
3
|
+
* modified/deleted keys in the migration registry (SPEC_F §F.6).
|
|
4
|
+
*
|
|
5
|
+
* Programmatic API:
|
|
6
|
+
* const tracker = await startDirtyTracker({ from, to, runId });
|
|
7
|
+
* // ... run bulk import and other Redis writes ...
|
|
8
|
+
* await tracker.stop();
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { createClient } from 'redis';
|
|
12
|
+
import { openDb } from '../storage/sqlite/db.js';
|
|
13
|
+
import { createRun, upsertDirtyKey, logError } from './registry.js';
|
|
14
|
+
import { readKeyspaceEvents } from './preflight.js';
|
|
15
|
+
|
|
16
|
+
const KEYEVENT_PATTERN = '__keyevent@0__:*';
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Start the dirty-key tracker: connect a subscriber client and record every
|
|
20
|
+
* keyspace event into the migration registry.
|
|
21
|
+
*
|
|
22
|
+
* Resolves once the subscription is active. Call `stop()` to disconnect.
|
|
23
|
+
*
|
|
24
|
+
* @param {object} options
|
|
25
|
+
* @param {string} [options.from='redis://127.0.0.1:6379'] - Source Redis URL.
|
|
26
|
+
* @param {string} options.to - SQLite DB path (must already exist or be created by bulk).
|
|
27
|
+
* @param {string} options.runId - Migration run identifier.
|
|
28
|
+
* @param {string} [options.pragmaTemplate='default']
|
|
29
|
+
* @param {string} [options.configCommand='CONFIG'] - CONFIG command name (in case it was renamed).
|
|
30
|
+
* @returns {Promise<{ stop(): Promise<void> }>}
|
|
31
|
+
* @throws {Error} If keyspace notifications are not enabled on Redis.
|
|
32
|
+
*/
|
|
33
|
+
export async function startDirtyTracker({
|
|
34
|
+
from = 'redis://127.0.0.1:6379',
|
|
35
|
+
to,
|
|
36
|
+
runId,
|
|
37
|
+
pragmaTemplate = 'default',
|
|
38
|
+
configCommand = 'CONFIG',
|
|
39
|
+
} = {}) {
|
|
40
|
+
if (!to) throw new Error('startDirtyTracker: "to" (db path) is required');
|
|
41
|
+
if (!runId) throw new Error('startDirtyTracker: "runId" is required');
|
|
42
|
+
|
|
43
|
+
const db = openDb(to, { pragmaTemplate });
|
|
44
|
+
createRun(db, runId, from); // idempotent — safe to call even if bulk already created the run
|
|
45
|
+
|
|
46
|
+
const mainClient = createClient({ url: from });
|
|
47
|
+
mainClient.on('error', (err) =>
|
|
48
|
+
logError(db, runId, 'dirty_apply', 'Tracker connection error: ' + err.message, null)
|
|
49
|
+
);
|
|
50
|
+
await mainClient.connect();
|
|
51
|
+
|
|
52
|
+
// Validate keyspace notifications before subscribing
|
|
53
|
+
const { value: eventsValue, available } = await readKeyspaceEvents(mainClient, configCommand);
|
|
54
|
+
if (!available || !eventsValue || eventsValue === '') {
|
|
55
|
+
await mainClient.quit().catch(() => {});
|
|
56
|
+
throw new Error(
|
|
57
|
+
`Redis notify-keyspace-events is not set. ` +
|
|
58
|
+
`Enable it first: ${configCommand} SET notify-keyspace-events KEA\n` +
|
|
59
|
+
`Or call m.enableKeyspaceNotifications() from the programmatic API.`
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const subClient = mainClient.duplicate();
|
|
64
|
+
subClient.on('error', (err) =>
|
|
65
|
+
logError(db, runId, 'dirty_apply', 'Tracker subscriber error: ' + err.message, null)
|
|
66
|
+
);
|
|
67
|
+
await subClient.connect();
|
|
68
|
+
|
|
69
|
+
await subClient.pSubscribe(KEYEVENT_PATTERN, (message, channel) => {
|
|
70
|
+
const event = typeof channel === 'string'
|
|
71
|
+
? channel.split(':').pop()
|
|
72
|
+
: String(channel ?? '').split(':').pop() || 'unknown';
|
|
73
|
+
try {
|
|
74
|
+
upsertDirtyKey(db, runId, message, event);
|
|
75
|
+
} catch (err) {
|
|
76
|
+
logError(db, runId, 'dirty_apply', err.message, message);
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
return {
|
|
81
|
+
/**
|
|
82
|
+
* Unsubscribe and disconnect. Safe to call multiple times.
|
|
83
|
+
*/
|
|
84
|
+
async stop() {
|
|
85
|
+
await subClient.pUnsubscribe(KEYEVENT_PATTERN).catch(() => {});
|
|
86
|
+
await subClient.quit().catch(() => {});
|
|
87
|
+
await mainClient.quit().catch(() => {});
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Integration test: dirty-key tracker + full migration flow.
|
|
3
|
+
*
|
|
4
|
+
* Requires a local Redis instance on redis://127.0.0.1:6379.
|
|
5
|
+
* The test is skipped automatically if Redis is unavailable or
|
|
6
|
+
* if keyspace notifications cannot be enabled.
|
|
7
|
+
*
|
|
8
|
+
* Flow being tested:
|
|
9
|
+
* 1. Write initial keys to Redis
|
|
10
|
+
* 2. Enable keyspace notifications
|
|
11
|
+
* 3. Start dirty tracker
|
|
12
|
+
* 4. Run bulk import (captures snapshot at T=0)
|
|
13
|
+
* 5. Modify keys in Redis after bulk (tracker records them as dirty)
|
|
14
|
+
* 6. Stop tracker
|
|
15
|
+
* 7. Apply-dirty (reconciles post-bulk changes into destination)
|
|
16
|
+
* 8. Verify (destination matches Redis)
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { describe, it, before, after } from 'node:test';
|
|
20
|
+
import assert from 'node:assert/strict';
|
|
21
|
+
import { createClient } from 'redis';
|
|
22
|
+
import { createMigration, startDirtyTracker } from '../../src/migration/index.js';
|
|
23
|
+
import { tmpDbPath } from '../helpers/tmp.js';
|
|
24
|
+
|
|
25
|
+
const REDIS_URL = 'redis://127.0.0.1:6379';
|
|
26
|
+
const PREFIX = `__resplite_tracker_test_${process.pid}__`;
|
|
27
|
+
|
|
28
|
+
/** Connect to local Redis; return null if unavailable. */
|
|
29
|
+
async function tryConnectRedis() {
|
|
30
|
+
const client = createClient({ url: REDIS_URL });
|
|
31
|
+
try {
|
|
32
|
+
await client.connect();
|
|
33
|
+
await client.ping();
|
|
34
|
+
return client;
|
|
35
|
+
} catch {
|
|
36
|
+
await client.quit().catch(() => {});
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Delete all keys under PREFIX. */
|
|
42
|
+
async function cleanup(redis) {
|
|
43
|
+
let cursor = 0;
|
|
44
|
+
do {
|
|
45
|
+
const result = await redis.scan(cursor, { MATCH: `${PREFIX}:*`, COUNT: 200 });
|
|
46
|
+
cursor = typeof result.cursor === 'number' ? result.cursor : parseInt(result.cursor, 10);
|
|
47
|
+
const keys = result.keys ?? [];
|
|
48
|
+
if (keys.length) await redis.del(keys);
|
|
49
|
+
} while (cursor !== 0);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
describe('dirty tracker integration', { timeout: 30_000 }, () => {
|
|
53
|
+
let redis = null;
|
|
54
|
+
let originalEventsValue = '';
|
|
55
|
+
|
|
56
|
+
before(async () => {
|
|
57
|
+
redis = await tryConnectRedis();
|
|
58
|
+
if (!redis) return; // tests will skip inside each `it`
|
|
59
|
+
|
|
60
|
+
// Save existing notify-keyspace-events so we can restore it after
|
|
61
|
+
const raw = await redis.sendCommand(['CONFIG', 'GET', 'notify-keyspace-events']);
|
|
62
|
+
originalEventsValue = Array.isArray(raw) ? (raw[1] ?? '') : '';
|
|
63
|
+
|
|
64
|
+
await cleanup(redis);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
after(async () => {
|
|
68
|
+
if (!redis) return;
|
|
69
|
+
// Restore original keyspace events value
|
|
70
|
+
await redis.sendCommand(['CONFIG', 'SET', 'notify-keyspace-events', originalEventsValue]);
|
|
71
|
+
await cleanup(redis);
|
|
72
|
+
await redis.quit();
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it('bulk + tracker + apply-dirty produces a fully reconciled destination', async (t) => {
|
|
76
|
+
if (!redis) {
|
|
77
|
+
t.skip('local Redis not available');
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// ── Seed: write initial keys ─────────────────────────────────────────
|
|
82
|
+
const keys = {
|
|
83
|
+
string: `${PREFIX}:str`,
|
|
84
|
+
hash: `${PREFIX}:hash`,
|
|
85
|
+
set: `${PREFIX}:set`,
|
|
86
|
+
list: `${PREFIX}:list`,
|
|
87
|
+
toDelete: `${PREFIX}:will-delete`,
|
|
88
|
+
toModify: `${PREFIX}:will-modify`,
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
await redis.set(keys.string, 'hello');
|
|
92
|
+
await redis.hSet(keys.hash, { field1: 'v1', field2: 'v2' });
|
|
93
|
+
await redis.sAdd(keys.set, ['a', 'b', 'c']);
|
|
94
|
+
await redis.rPush(keys.list, ['x', 'y', 'z']);
|
|
95
|
+
await redis.set(keys.toDelete, 'delete-me');
|
|
96
|
+
await redis.set(keys.toModify, 'original');
|
|
97
|
+
|
|
98
|
+
// ── Setup: enable keyspace notifications ────────────────────────────
|
|
99
|
+
const dbPath = tmpDbPath();
|
|
100
|
+
const runId = `tracker-test-${Date.now()}`;
|
|
101
|
+
|
|
102
|
+
const m = createMigration({ from: REDIS_URL, to: dbPath, runId });
|
|
103
|
+
|
|
104
|
+
const ks = await m.enableKeyspaceNotifications({ value: 'KEA' });
|
|
105
|
+
assert.ok(ks.ok, `Failed to enable keyspace notifications: ${ks.error}`);
|
|
106
|
+
|
|
107
|
+
// ── Start tracker BEFORE bulk ────────────────────────────────────────
|
|
108
|
+
const tracker = await startDirtyTracker({ from: REDIS_URL, to: dbPath, runId });
|
|
109
|
+
|
|
110
|
+
try {
|
|
111
|
+
// ── Bulk import (captures the initial snapshot) ──────────────────
|
|
112
|
+
const run = await m.bulk();
|
|
113
|
+
assert.equal(run.status, 'completed');
|
|
114
|
+
assert.ok(run.migrated_keys >= 6, `Expected ≥6 migrated keys, got ${run.migrated_keys}`);
|
|
115
|
+
|
|
116
|
+
// ── Post-bulk mutations (tracker will record these) ──────────────
|
|
117
|
+
await redis.set(keys.toModify, 'modified-after-bulk');
|
|
118
|
+
await redis.del(keys.toDelete);
|
|
119
|
+
await redis.set(`${PREFIX}:new-key`, 'added-after-bulk');
|
|
120
|
+
|
|
121
|
+
// Give the tracker time to process the keyspace events
|
|
122
|
+
await new Promise((r) => setTimeout(r, 600));
|
|
123
|
+
} finally {
|
|
124
|
+
await tracker.stop();
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// ── Verify dirty keys were captured ─────────────────────────────────
|
|
128
|
+
const { dirty } = m.status();
|
|
129
|
+
assert.ok(
|
|
130
|
+
dirty.dirty + dirty.deleted >= 3,
|
|
131
|
+
`Expected ≥3 dirty/deleted keys, got dirty=${dirty.dirty} deleted=${dirty.deleted}`
|
|
132
|
+
);
|
|
133
|
+
|
|
134
|
+
// ── Apply-dirty: reconcile post-bulk changes ─────────────────────────
|
|
135
|
+
const afterApply = await m.applyDirty();
|
|
136
|
+
const totalReconciled = afterApply.dirty_keys_applied + afterApply.dirty_keys_deleted;
|
|
137
|
+
assert.ok(totalReconciled >= 3, `Expected ≥3 reconciled keys, got ${totalReconciled}`);
|
|
138
|
+
|
|
139
|
+
// ── Verify: destination should now match Redis for our test prefix ───
|
|
140
|
+
const result = await m.verify({ samplePct: 100, maxSample: 5000 });
|
|
141
|
+
|
|
142
|
+
// Filter mismatches to only those under our test prefix
|
|
143
|
+
const ourMismatches = result.mismatches.filter((mm) => mm.key.startsWith(PREFIX));
|
|
144
|
+
assert.equal(
|
|
145
|
+
ourMismatches.length,
|
|
146
|
+
0,
|
|
147
|
+
`Unexpected mismatches in test keys:\n${JSON.stringify(ourMismatches, null, 2)}`
|
|
148
|
+
);
|
|
149
|
+
|
|
150
|
+
await m.close();
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it('startDirtyTracker throws when keyspace notifications are disabled', async (t) => {
|
|
154
|
+
if (!redis) {
|
|
155
|
+
t.skip('local Redis not available');
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Disable keyspace notifications
|
|
160
|
+
await redis.sendCommand(['CONFIG', 'SET', 'notify-keyspace-events', '']);
|
|
161
|
+
|
|
162
|
+
const dbPath = tmpDbPath();
|
|
163
|
+
const runId = `tracker-noevents-${Date.now()}`;
|
|
164
|
+
|
|
165
|
+
await assert.rejects(
|
|
166
|
+
() => startDirtyTracker({ from: REDIS_URL, to: dbPath, runId }),
|
|
167
|
+
/notify-keyspace-events/
|
|
168
|
+
);
|
|
169
|
+
|
|
170
|
+
// Re-enable for subsequent tests
|
|
171
|
+
await redis.sendCommand(['CONFIG', 'SET', 'notify-keyspace-events', 'KEA']);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it('enableKeyspaceNotifications merges flags without overwriting existing ones', async (t) => {
|
|
175
|
+
if (!redis) {
|
|
176
|
+
t.skip('local Redis not available');
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const dbPath = tmpDbPath();
|
|
181
|
+
const m = createMigration({ from: REDIS_URL, to: dbPath, runId: `ks-merge-${Date.now()}` });
|
|
182
|
+
|
|
183
|
+
// Set a partial value first
|
|
184
|
+
await redis.sendCommand(['CONFIG', 'SET', 'notify-keyspace-events', 'Kg']);
|
|
185
|
+
|
|
186
|
+
const result = await m.enableKeyspaceNotifications({ value: 'KEA', merge: true });
|
|
187
|
+
assert.ok(result.ok, `Expected ok=true: ${result.error}`);
|
|
188
|
+
// Redis may reorder flags (e.g. 'Kg' → 'gK'); only check flag membership
|
|
189
|
+
assert.ok(typeof result.previous === 'string', 'previous should be a string');
|
|
190
|
+
assert.ok(result.previous.includes('K'), `Missing K in previous="${result.previous}"`);
|
|
191
|
+
assert.ok(result.previous.includes('g'), `Missing g in previous="${result.previous}"`);
|
|
192
|
+
|
|
193
|
+
// Applied value must contain K, E, A and the original g
|
|
194
|
+
const applied = result.applied;
|
|
195
|
+
assert.ok(applied.includes('K'), `Missing K in applied="${applied}"`);
|
|
196
|
+
assert.ok(applied.includes('E'), `Missing E in applied="${applied}"`);
|
|
197
|
+
assert.ok(applied.includes('A'), `Missing A in applied="${applied}"`);
|
|
198
|
+
|
|
199
|
+
await m.close();
|
|
200
|
+
});
|
|
201
|
+
});
|