resplite 1.0.0 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -21
- package/package.json +5 -1
- package/spec/SPEC_F.md +505 -0
- package/src/blocking/manager.js +183 -0
- package/src/cli/resplite-dirty-tracker.js +124 -0
- package/src/cli/resplite-import.js +237 -0
- package/src/commands/blpop.js +50 -0
- package/src/commands/brpop.js +50 -0
- package/src/commands/registry.js +11 -5
- package/src/engine/engine.js +11 -3
- package/src/migration/apply-dirty.js +97 -0
- package/src/migration/bulk.js +181 -0
- package/src/migration/import-one.js +106 -0
- package/src/migration/preflight.js +62 -0
- package/src/migration/registry.js +222 -0
- package/src/migration/verify.js +191 -0
- package/src/server/connection.js +55 -13
- package/src/storage/sqlite/db.js +2 -0
- package/src/storage/sqlite/migration-schema.js +66 -0
- package/tasks/todo.md +19 -0
- package/test/integration/blocking.test.js +107 -0
- package/test/unit/migration-registry.test.js +127 -0
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Blocking list waiters: BLPOP/BRPOP (SPEC_E).
|
|
3
|
+
* In-memory wait queues by key; single delivery; wake on LPUSH/RPUSH.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { asKey } from '../util/buffers.js';
|
|
7
|
+
|
|
8
|
+
function toMapKey(key) {
|
|
9
|
+
const k = Buffer.isBuffer(key) ? key : asKey(key);
|
|
10
|
+
return k.toString('binary');
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @param {object} engine - RESPlite engine (for lpop/rpop on wake)
|
|
15
|
+
* @param {object} [opts]
|
|
16
|
+
* @param {number} [opts.maxWaitersPerKey=10000]
|
|
17
|
+
* @param {number} [opts.maxTotalWaiters=50000]
|
|
18
|
+
* @param {number} [opts.maxKeysPerWait=128]
|
|
19
|
+
* @param {() => number} [opts.clock=Date.now]
|
|
20
|
+
*/
|
|
21
|
+
export function createBlockingManager(engine, opts = {}) {
|
|
22
|
+
const maxWaitersPerKey = opts.maxWaitersPerKey ?? 10000;
|
|
23
|
+
const maxTotalWaiters = opts.maxTotalWaiters ?? 50000;
|
|
24
|
+
const maxKeysPerWait = opts.maxKeysPerWait ?? 128;
|
|
25
|
+
const clock = opts.clock ?? (() => Date.now());
|
|
26
|
+
|
|
27
|
+
/** @type {Map<string, Array<{ waiter: object }>>} key -> deque of waiter refs */
|
|
28
|
+
const waitersByKey = new Map();
|
|
29
|
+
/** @type {Map<string | number, Set<object>>} connectionId -> waiters for cancel */
|
|
30
|
+
const waitersByConnection = new Map();
|
|
31
|
+
let totalWaiters = 0;
|
|
32
|
+
|
|
33
|
+
function removeWaiterFromQueues(waiter) {
|
|
34
|
+
for (const key of waiter.keys) {
|
|
35
|
+
const mapKey = toMapKey(key);
|
|
36
|
+
const q = waitersByKey.get(mapKey);
|
|
37
|
+
if (!q) continue;
|
|
38
|
+
const idx = q.findIndex((ref) => ref.waiter === waiter);
|
|
39
|
+
if (idx !== -1) q.splice(idx, 1);
|
|
40
|
+
if (q.length === 0) waitersByKey.delete(mapKey);
|
|
41
|
+
}
|
|
42
|
+
const connSet = waitersByConnection.get(waiter.connectionId);
|
|
43
|
+
if (connSet) {
|
|
44
|
+
connSet.delete(waiter);
|
|
45
|
+
if (connSet.size === 0) waitersByConnection.delete(waiter.connectionId);
|
|
46
|
+
}
|
|
47
|
+
totalWaiters--;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Try to satisfy one waiter: scan keys in order, pop from first non-empty.
|
|
52
|
+
* @returns {boolean} true if waiter was satisfied and removed
|
|
53
|
+
*/
|
|
54
|
+
function tryWake(waiter) {
|
|
55
|
+
if (waiter.completed || waiter.canceled) return false;
|
|
56
|
+
for (const key of waiter.keys) {
|
|
57
|
+
try {
|
|
58
|
+
const val =
|
|
59
|
+
waiter.kind === 'BLPOP'
|
|
60
|
+
? engine.lpop(key, null)
|
|
61
|
+
: engine.rpop(key, null);
|
|
62
|
+
if (val != null) {
|
|
63
|
+
waiter.completed = true;
|
|
64
|
+
removeWaiterFromQueues(waiter);
|
|
65
|
+
waiter.resolve([key, val]);
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
} catch (_) {
|
|
69
|
+
// wrong type / missing: skip key
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Register a blocked client. Keys in order; timeout in seconds (0 = indefinite).
|
|
77
|
+
* @param {Buffer[]} keys
|
|
78
|
+
* @param {'BLPOP'|'BRPOP'} kind
|
|
79
|
+
* @param {number} timeoutSeconds
|
|
80
|
+
* @param {(value: [Buffer, Buffer]|null) => void} resolve - called with [key, element] or null on timeout
|
|
81
|
+
* @param {string|number} connectionId
|
|
82
|
+
* @returns {{ error?: string }}
|
|
83
|
+
*/
|
|
84
|
+
function registerWaiter(keys, kind, timeoutSeconds, resolve, connectionId) {
|
|
85
|
+
if (keys.length > maxKeysPerWait) {
|
|
86
|
+
return { error: 'ERR too many keys per blocked command' };
|
|
87
|
+
}
|
|
88
|
+
if (totalWaiters >= maxTotalWaiters) {
|
|
89
|
+
return { error: 'ERR too many blocked clients' };
|
|
90
|
+
}
|
|
91
|
+
for (const k of keys) {
|
|
92
|
+
const mapKey = toMapKey(k);
|
|
93
|
+
const q = waitersByKey.get(mapKey) ?? [];
|
|
94
|
+
if (q.length >= maxWaitersPerKey) {
|
|
95
|
+
return { error: 'ERR too many blocked clients' };
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const deadline =
|
|
100
|
+
timeoutSeconds > 0 ? clock() + timeoutSeconds * 1000 : null;
|
|
101
|
+
const waiter = {
|
|
102
|
+
connectionId,
|
|
103
|
+
kind,
|
|
104
|
+
keys: keys.slice(),
|
|
105
|
+
deadline,
|
|
106
|
+
resolve,
|
|
107
|
+
completed: false,
|
|
108
|
+
canceled: false,
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
let timer = null;
|
|
112
|
+
if (deadline != null) {
|
|
113
|
+
timer = setTimeout(() => {
|
|
114
|
+
if (waiter.completed || waiter.canceled) return;
|
|
115
|
+
waiter.completed = true;
|
|
116
|
+
removeWaiterFromQueues(waiter);
|
|
117
|
+
resolve(null);
|
|
118
|
+
}, timeoutSeconds * 1000);
|
|
119
|
+
if (timer.unref) timer.unref();
|
|
120
|
+
}
|
|
121
|
+
waiter.timer = timer;
|
|
122
|
+
|
|
123
|
+
totalWaiters++;
|
|
124
|
+
for (const key of keys) {
|
|
125
|
+
const mapKey = toMapKey(key);
|
|
126
|
+
let q = waitersByKey.get(mapKey);
|
|
127
|
+
if (!q) {
|
|
128
|
+
q = [];
|
|
129
|
+
waitersByKey.set(mapKey, q);
|
|
130
|
+
}
|
|
131
|
+
q.push({ waiter });
|
|
132
|
+
}
|
|
133
|
+
let connSet = waitersByConnection.get(connectionId);
|
|
134
|
+
if (!connSet) {
|
|
135
|
+
connSet = new Set();
|
|
136
|
+
waitersByConnection.set(connectionId, connSet);
|
|
137
|
+
}
|
|
138
|
+
connSet.add(waiter);
|
|
139
|
+
return {};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Called after LPUSH/RPUSH on key. Wake at most one waiter (oldest for this key).
|
|
144
|
+
* @param {Buffer|string} key - key that received the push
|
|
145
|
+
*/
|
|
146
|
+
function wakeup(key) {
|
|
147
|
+
const mapKey = toMapKey(key);
|
|
148
|
+
const q = waitersByKey.get(mapKey);
|
|
149
|
+
if (!q || q.length === 0) return;
|
|
150
|
+
const ref = q[0];
|
|
151
|
+
const waiter = ref.waiter;
|
|
152
|
+
if (waiter.canceled || waiter.completed) {
|
|
153
|
+
removeWaiterFromQueues(waiter);
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
tryWake(waiter);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* On client disconnect: mark waiters canceled and remove from queues.
|
|
161
|
+
* @param {string|number} connectionId
|
|
162
|
+
*/
|
|
163
|
+
function cancel(connectionId) {
|
|
164
|
+
const connSet = waitersByConnection.get(connectionId);
|
|
165
|
+
if (!connSet) return;
|
|
166
|
+
for (const waiter of new Set(connSet)) {
|
|
167
|
+
if (waiter.completed) continue;
|
|
168
|
+
waiter.canceled = true;
|
|
169
|
+
if (waiter.timer) {
|
|
170
|
+
clearTimeout(waiter.timer);
|
|
171
|
+
waiter.timer = null;
|
|
172
|
+
}
|
|
173
|
+
removeWaiterFromQueues(waiter);
|
|
174
|
+
}
|
|
175
|
+
waitersByConnection.delete(connectionId);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
return {
|
|
179
|
+
registerWaiter,
|
|
180
|
+
wakeup,
|
|
181
|
+
cancel,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* resplite-dirty-tracker (SPEC_F §F.6): subscribe to Redis keyspace notifications, record dirty keys in SQLite.
|
|
4
|
+
* Usage: resplite-dirty-tracker start|stop [options]
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { createClient } from 'redis';
|
|
8
|
+
import { fileURLToPath } from 'node:url';
|
|
9
|
+
import { openDb } from '../storage/sqlite/db.js';
|
|
10
|
+
import { createRun, getRun, setRunStatus, upsertDirtyKey, logError, RUN_STATUS } from '../migration/registry.js';
|
|
11
|
+
|
|
12
|
+
function parseArgs(argv = process.argv.slice(2)) {
|
|
13
|
+
const args = { _: [] };
|
|
14
|
+
for (let i = 0; i < argv.length; i++) {
|
|
15
|
+
const arg = argv[i];
|
|
16
|
+
if (arg === '--from' && argv[i + 1]) args.from = argv[++i];
|
|
17
|
+
else if (arg === '--to' && argv[i + 1]) args.to = argv[++i];
|
|
18
|
+
else if (arg === '--run-id' && argv[i + 1]) args.runId = argv[++i];
|
|
19
|
+
else if (arg === '--channels' && argv[i + 1]) args.channels = argv[++i];
|
|
20
|
+
else if (arg === '--pragma-template' && argv[i + 1]) args.pragmaTemplate = argv[++i];
|
|
21
|
+
else if (!arg.startsWith('--')) args._.push(arg);
|
|
22
|
+
}
|
|
23
|
+
return args;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
async function checkKeyspaceNotifications(client) {
|
|
27
|
+
let val = null;
|
|
28
|
+
try {
|
|
29
|
+
const config = await client.configGet('notify-keyspace-events');
|
|
30
|
+
val = config && typeof config === 'object' ? config['notify-keyspace-events'] : null;
|
|
31
|
+
} catch (_) {}
|
|
32
|
+
if (!val || val === '') {
|
|
33
|
+
throw new Error('Redis notify-keyspace-events is not set. Enable it (e.g. CONFIG SET notify-keyspace-events Kgxe) for the dirty-key tracker.');
|
|
34
|
+
}
|
|
35
|
+
return val;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function startTracker(args) {
|
|
39
|
+
const redisUrl = args.from || process.env.RESPLITE_IMPORT_FROM || 'redis://127.0.0.1:6379';
|
|
40
|
+
const dbPath = args.to;
|
|
41
|
+
const runId = args.runId || process.env.RESPLITE_RUN_ID;
|
|
42
|
+
if (!dbPath || !runId) {
|
|
43
|
+
console.error('Usage: resplite-dirty-tracker start --run-id <id> --to <db-path> [--from <redis-url>] [--channels keyevent]');
|
|
44
|
+
process.exit(1);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const db = openDb(dbPath, { pragmaTemplate: args.pragmaTemplate || 'default' });
|
|
48
|
+
createRun(db, runId, redisUrl);
|
|
49
|
+
|
|
50
|
+
const mainClient = createClient({ url: redisUrl });
|
|
51
|
+
mainClient.on('error', (e) => console.error('Redis (main):', e.message));
|
|
52
|
+
await mainClient.connect();
|
|
53
|
+
await checkKeyspaceNotifications(mainClient);
|
|
54
|
+
|
|
55
|
+
const subClient = mainClient.duplicate();
|
|
56
|
+
subClient.on('error', (e) => {
|
|
57
|
+
console.error('Redis (sub):', e.message);
|
|
58
|
+
logError(db, runId, 'dirty_apply', 'Tracker disconnect: ' + e.message, null);
|
|
59
|
+
});
|
|
60
|
+
await subClient.connect();
|
|
61
|
+
|
|
62
|
+
const pattern = '__keyevent@0__:*';
|
|
63
|
+
console.log('Subscribing to', pattern, '...');
|
|
64
|
+
|
|
65
|
+
await subClient.pSubscribe(pattern, (message, channel) => {
|
|
66
|
+
const event = typeof channel === 'string' ? channel.split(':').pop() : (channel && channel.toString?.())?.split(':').pop() || 'unknown';
|
|
67
|
+
const key = message;
|
|
68
|
+
try {
|
|
69
|
+
upsertDirtyKey(db, runId, key, event);
|
|
70
|
+
} catch (err) {
|
|
71
|
+
logError(db, runId, 'dirty_apply', err.message, key);
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
const shutdown = async () => {
|
|
76
|
+
console.log('Stopping dirty tracker...');
|
|
77
|
+
await subClient.pUnsubscribe(pattern);
|
|
78
|
+
await subClient.quit();
|
|
79
|
+
await mainClient.quit();
|
|
80
|
+
process.exit(0);
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
process.on('SIGINT', shutdown);
|
|
84
|
+
process.on('SIGTERM', shutdown);
|
|
85
|
+
console.log('Dirty tracker running. Ctrl+C to stop.');
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async function stopTracker(args) {
|
|
89
|
+
const dbPath = args.to;
|
|
90
|
+
const runId = args.runId || process.env.RESPLITE_RUN_ID;
|
|
91
|
+
if (!dbPath || !runId) {
|
|
92
|
+
console.error('Usage: resplite-dirty-tracker stop --run-id <id> --to <db-path>');
|
|
93
|
+
process.exit(1);
|
|
94
|
+
}
|
|
95
|
+
const db = openDb(dbPath, { pragmaTemplate: args.pragmaTemplate || 'default' });
|
|
96
|
+
const run = getRun(db, runId);
|
|
97
|
+
if (run && run.status === RUN_STATUS.RUNNING) {
|
|
98
|
+
setRunStatus(db, runId, RUN_STATUS.PAUSED);
|
|
99
|
+
console.log('Run', runId, 'status set to paused. Tracker process must be stopped with Ctrl+C if still running.');
|
|
100
|
+
} else {
|
|
101
|
+
console.log('Run', runId, 'status:', run?.status ?? 'not found');
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function main() {
|
|
106
|
+
const args = parseArgs();
|
|
107
|
+
const sub = args._[0];
|
|
108
|
+
if (sub === 'start') await startTracker(args);
|
|
109
|
+
else if (sub === 'stop') await stopTracker(args);
|
|
110
|
+
else {
|
|
111
|
+
console.error('Usage: resplite-dirty-tracker <start|stop> --run-id <id> --to <db-path> [--from <redis-url>]');
|
|
112
|
+
process.exit(1);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const isMain = process.argv[1] === fileURLToPath(import.meta.url) || process.argv[1]?.endsWith('resplite-dirty-tracker.js');
|
|
117
|
+
if (isMain) {
|
|
118
|
+
main().catch((e) => {
|
|
119
|
+
console.error(e);
|
|
120
|
+
process.exit(1);
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export { parseArgs, startTracker, stopTracker };
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* resplite-import CLI (SPEC_F §F.9): preflight, bulk, status, apply-dirty, verify.
|
|
4
|
+
* Usage: resplite-import <preflight|bulk|status|apply-dirty|verify> [options]
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { createClient } from 'redis';
|
|
8
|
+
import { fileURLToPath } from 'node:url';
|
|
9
|
+
import { openDb } from '../storage/sqlite/db.js';
|
|
10
|
+
import { runPreflight } from '../migration/preflight.js';
|
|
11
|
+
import { runBulkImport } from '../migration/bulk.js';
|
|
12
|
+
import { runApplyDirty } from '../migration/apply-dirty.js';
|
|
13
|
+
import { runVerify } from '../migration/verify.js';
|
|
14
|
+
import { getRun, getDirtyCounts } from '../migration/registry.js';
|
|
15
|
+
|
|
16
|
+
const SUBCOMMANDS = ['preflight', 'bulk', 'status', 'apply-dirty', 'verify'];
|
|
17
|
+
|
|
18
|
+
function parseArgs(argv = process.argv.slice(2)) {
|
|
19
|
+
const args = { _: [] };
|
|
20
|
+
for (let i = 0; i < argv.length; i++) {
|
|
21
|
+
const arg = argv[i];
|
|
22
|
+
if (arg === '--from' && argv[i + 1]) {
|
|
23
|
+
args.from = argv[++i];
|
|
24
|
+
} else if (arg === '--to' && argv[i + 1]) {
|
|
25
|
+
args.to = argv[++i];
|
|
26
|
+
} else if (arg === '--run-id' && argv[i + 1]) {
|
|
27
|
+
args.runId = argv[++i];
|
|
28
|
+
} else if (arg === '--scan-count' && argv[i + 1]) {
|
|
29
|
+
args.scanCount = parseInt(argv[++i], 10);
|
|
30
|
+
} else if (arg === '--max-concurrency' && argv[i + 1]) {
|
|
31
|
+
args.maxConcurrency = parseInt(argv[++i], 10);
|
|
32
|
+
} else if (arg === '--max-rps' && argv[i + 1]) {
|
|
33
|
+
args.maxRps = parseInt(argv[++i], 10);
|
|
34
|
+
} else if (arg === '--batch-keys' && argv[i + 1]) {
|
|
35
|
+
args.batchKeys = parseInt(argv[++i], 10);
|
|
36
|
+
} else if (arg === '--batch-bytes' && argv[i + 1]) {
|
|
37
|
+
const v = argv[++i];
|
|
38
|
+
const match = v.match(/^(\d+)(MB|KB|GB)?$/i);
|
|
39
|
+
if (match) {
|
|
40
|
+
let n = parseInt(match[1], 10);
|
|
41
|
+
if (match[2]) {
|
|
42
|
+
if (match[2].toUpperCase() === 'KB') n *= 1024;
|
|
43
|
+
else if (match[2].toUpperCase() === 'MB') n *= 1024 * 1024;
|
|
44
|
+
else if (match[2].toUpperCase() === 'GB') n *= 1024 * 1024 * 1024;
|
|
45
|
+
}
|
|
46
|
+
args.batchBytes = n;
|
|
47
|
+
}
|
|
48
|
+
} else if (arg === '--resume') {
|
|
49
|
+
args.resume = true;
|
|
50
|
+
} else if (arg === '--pragma-template' && argv[i + 1]) {
|
|
51
|
+
args.pragmaTemplate = argv[++i];
|
|
52
|
+
} else if (arg === '--sample' && argv[i + 1]) {
|
|
53
|
+
args.sample = argv[++i];
|
|
54
|
+
} else if (arg.startsWith('--')) {
|
|
55
|
+
args[arg.slice(2).replace(/-/g, '')] = argv[i + 1] ?? true;
|
|
56
|
+
if (argv[i + 1] && !argv[i + 1].startsWith('--')) i++;
|
|
57
|
+
} else {
|
|
58
|
+
args._.push(arg);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return args;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function getRedisUrl(args) {
|
|
65
|
+
return args.from || process.env.RESPLITE_IMPORT_FROM || 'redis://127.0.0.1:6379';
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function getDbPath(args) {
|
|
69
|
+
if (!args.to) {
|
|
70
|
+
console.error('Missing --to <db-path>');
|
|
71
|
+
process.exit(1);
|
|
72
|
+
}
|
|
73
|
+
return args.to;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function getRunId(args, required = false) {
|
|
77
|
+
const id = args.runId || process.env.RESPLITE_RUN_ID;
|
|
78
|
+
if (required && !id) {
|
|
79
|
+
console.error('Missing --run-id <id>');
|
|
80
|
+
process.exit(1);
|
|
81
|
+
}
|
|
82
|
+
return id;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function cmdPreflight(args) {
|
|
86
|
+
const redisUrl = getRedisUrl(args);
|
|
87
|
+
const client = createClient({ url: redisUrl });
|
|
88
|
+
client.on('error', (e) => console.error('Redis:', e.message));
|
|
89
|
+
await client.connect();
|
|
90
|
+
try {
|
|
91
|
+
const result = await runPreflight(client);
|
|
92
|
+
console.log('Preflight:');
|
|
93
|
+
console.log(' Key count (estimate):', result.keyCountEstimate);
|
|
94
|
+
console.log(' Type distribution (sample):', result.typeDistribution);
|
|
95
|
+
console.log(' notify-keyspace-events:', result.notifyKeyspaceEvents ?? '(not set or not readable)');
|
|
96
|
+
if (!result.notifyKeyspaceEvents || result.notifyKeyspaceEvents === '') {
|
|
97
|
+
console.warn(' WARNING: Keyspace notifications disabled. Enable for dirty-key tracker (e.g. "Kgxe").');
|
|
98
|
+
}
|
|
99
|
+
console.log(' Recommended:');
|
|
100
|
+
console.log(' --scan-count', result.recommended.scan_count);
|
|
101
|
+
console.log(' --max-concurrency', result.recommended.max_concurrency);
|
|
102
|
+
console.log(' --max-rps', result.recommended.max_rps);
|
|
103
|
+
} finally {
|
|
104
|
+
await client.quit();
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async function cmdBulk(args) {
|
|
109
|
+
const redisUrl = getRedisUrl(args);
|
|
110
|
+
const dbPath = getDbPath(args);
|
|
111
|
+
const runId = getRunId(args, true);
|
|
112
|
+
const client = createClient({ url: redisUrl });
|
|
113
|
+
client.on('error', (e) => console.error('Redis:', e.message));
|
|
114
|
+
await client.connect();
|
|
115
|
+
try {
|
|
116
|
+
const run = await runBulkImport(client, dbPath, runId, {
|
|
117
|
+
sourceUri: redisUrl,
|
|
118
|
+
pragmaTemplate: args.pragmaTemplate || 'default',
|
|
119
|
+
scan_count: args.scanCount || 1000,
|
|
120
|
+
max_rps: args.maxRps || 0,
|
|
121
|
+
batch_keys: args.batchKeys || 200,
|
|
122
|
+
batch_bytes: args.batchBytes || 64 * 1024 * 1024,
|
|
123
|
+
resume: !!args.resume,
|
|
124
|
+
onProgress: (r) => {
|
|
125
|
+
console.log(` scanned=${r.scanned_keys} migrated=${r.migrated_keys} skipped=${r.skipped_keys} errors=${r.error_keys} cursor=${r.scan_cursor}`);
|
|
126
|
+
},
|
|
127
|
+
});
|
|
128
|
+
console.log('Bulk complete:', run);
|
|
129
|
+
} finally {
|
|
130
|
+
await client.quit();
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async function cmdStatus(args) {
|
|
135
|
+
const dbPath = getDbPath(args);
|
|
136
|
+
const runId = getRunId(args, true);
|
|
137
|
+
const db = openDb(dbPath, { pragmaTemplate: args.pragmaTemplate || 'default' });
|
|
138
|
+
const run = getRun(db, runId);
|
|
139
|
+
if (!run) {
|
|
140
|
+
console.error('Run not found:', runId);
|
|
141
|
+
process.exit(1);
|
|
142
|
+
}
|
|
143
|
+
const dirty = getDirtyCounts(db, runId);
|
|
144
|
+
console.log('Run:', runId);
|
|
145
|
+
console.log(' status:', run.status);
|
|
146
|
+
console.log(' source:', run.source_uri);
|
|
147
|
+
console.log(' scan_cursor:', run.scan_cursor);
|
|
148
|
+
console.log(' scanned_keys:', run.scanned_keys, 'migrated_keys:', run.migrated_keys, 'skipped_keys:', run.skipped_keys, 'error_keys:', run.error_keys);
|
|
149
|
+
console.log(' migrated_bytes:', run.migrated_bytes);
|
|
150
|
+
console.log(' dirty_keys: seen=', run.dirty_keys_seen, 'applied=', run.dirty_keys_applied, 'deleted=', run.dirty_keys_deleted);
|
|
151
|
+
console.log(' dirty by state:', dirty);
|
|
152
|
+
if (run.last_error) console.log(' last_error:', run.last_error);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async function cmdApplyDirty(args) {
|
|
156
|
+
const redisUrl = getRedisUrl(args);
|
|
157
|
+
const dbPath = getDbPath(args);
|
|
158
|
+
const runId = getRunId(args, true);
|
|
159
|
+
const client = createClient({ url: redisUrl });
|
|
160
|
+
client.on('error', (e) => console.error('Redis:', e.message));
|
|
161
|
+
await client.connect();
|
|
162
|
+
try {
|
|
163
|
+
const run = await runApplyDirty(client, dbPath, runId, {
|
|
164
|
+
pragmaTemplate: args.pragmaTemplate || 'default',
|
|
165
|
+
batch_keys: args.batchKeys || 200,
|
|
166
|
+
max_rps: args.maxRps || 0,
|
|
167
|
+
});
|
|
168
|
+
console.log('Apply-dirty complete:', run);
|
|
169
|
+
} finally {
|
|
170
|
+
await client.quit();
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async function cmdVerify(args) {
|
|
175
|
+
const redisUrl = getRedisUrl(args);
|
|
176
|
+
const dbPath = getDbPath(args);
|
|
177
|
+
const client = createClient({ url: redisUrl });
|
|
178
|
+
client.on('error', (e) => console.error('Redis:', e.message));
|
|
179
|
+
await client.connect();
|
|
180
|
+
try {
|
|
181
|
+
let samplePct = 0.5;
|
|
182
|
+
if (args.sample) {
|
|
183
|
+
const m = args.sample.match(/^(\d*\.?\d+)\s*%?$/);
|
|
184
|
+
if (m) samplePct = parseFloat(m[1]);
|
|
185
|
+
}
|
|
186
|
+
const result = await runVerify(client, dbPath, {
|
|
187
|
+
pragmaTemplate: args.pragmaTemplate || 'default',
|
|
188
|
+
samplePct,
|
|
189
|
+
maxSample: 10000,
|
|
190
|
+
});
|
|
191
|
+
console.log('Verify: sampled=', result.sampled, 'matched=', result.matched, 'mismatches=', result.mismatches.length);
|
|
192
|
+
if (result.mismatches.length) {
|
|
193
|
+
result.mismatches.slice(0, 20).forEach((m) => console.log(' ', m.key, m.reason));
|
|
194
|
+
if (result.mismatches.length > 20) console.log(' ... and', result.mismatches.length - 20, 'more');
|
|
195
|
+
}
|
|
196
|
+
} finally {
|
|
197
|
+
await client.quit();
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
async function main() {
|
|
202
|
+
const args = parseArgs();
|
|
203
|
+
const sub = args._[0];
|
|
204
|
+
if (!SUBCOMMANDS.includes(sub)) {
|
|
205
|
+
console.error('Usage: resplite-import <preflight|bulk|status|apply-dirty|verify> [options]');
|
|
206
|
+
console.error(' --from <redis-url> (default: redis://127.0.0.1:6379)');
|
|
207
|
+
console.error(' --to <db-path> (required for bulk, status, apply-dirty, verify)');
|
|
208
|
+
console.error(' --run-id <id> (required for bulk, status, apply-dirty)');
|
|
209
|
+
console.error(' --scan-count N (bulk, default 1000)');
|
|
210
|
+
console.error(' --max-rps N (bulk, apply-dirty)');
|
|
211
|
+
console.error(' --batch-keys N (default 200)');
|
|
212
|
+
console.error(' --batch-bytes N[MB|KB|GB] (default 64MB)');
|
|
213
|
+
console.error(' --resume (bulk: resume from checkpoint)');
|
|
214
|
+
console.error(' --sample 0.5% (verify, default 0.5%)');
|
|
215
|
+
process.exit(1);
|
|
216
|
+
}
|
|
217
|
+
try {
|
|
218
|
+
if (sub === 'preflight') await cmdPreflight(args);
|
|
219
|
+
else if (sub === 'bulk') await cmdBulk(args);
|
|
220
|
+
else if (sub === 'status') await cmdStatus(args);
|
|
221
|
+
else if (sub === 'apply-dirty') await cmdApplyDirty(args);
|
|
222
|
+
else if (sub === 'verify') await cmdVerify(args);
|
|
223
|
+
} catch (err) {
|
|
224
|
+
console.error(err);
|
|
225
|
+
process.exit(1);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const isMain = process.argv[1] === fileURLToPath(import.meta.url) || process.argv[1]?.endsWith('resplite-import.js');
|
|
230
|
+
if (isMain) {
|
|
231
|
+
main().catch((e) => {
|
|
232
|
+
console.error(e);
|
|
233
|
+
process.exit(1);
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export { parseArgs, cmdPreflight, cmdBulk, cmdStatus, cmdApplyDirty, cmdVerify };
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BLPOP key [key ...] timeout
|
|
3
|
+
* Block until one of the keys has an element, or timeout (seconds). timeout 0 = block indefinitely.
|
|
4
|
+
* Returns [key, element] on success, nil on timeout.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { ERRORS } from '../engine/errors.js';
|
|
8
|
+
|
|
9
|
+
function parseTimeout(arg) {
|
|
10
|
+
if (arg == null || arg === '') return null;
|
|
11
|
+
const s = Buffer.isBuffer(arg) ? arg.toString('utf8') : String(arg);
|
|
12
|
+
const n = parseInt(s, 10);
|
|
13
|
+
if (Number.isNaN(n) || s.trim() === '' || String(n) !== s.trim()) return null;
|
|
14
|
+
return n;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function handleBlpop(engine, args, context) {
|
|
18
|
+
if (!args || args.length < 2) {
|
|
19
|
+
return { error: 'ERR wrong number of arguments for \'BLPOP\' command' };
|
|
20
|
+
}
|
|
21
|
+
const timeoutArg = args[args.length - 1];
|
|
22
|
+
const timeoutSeconds = parseTimeout(timeoutArg);
|
|
23
|
+
if (timeoutSeconds === null || timeoutSeconds < 0) {
|
|
24
|
+
return { error: 'ERR timeout is not an integer or out of range' };
|
|
25
|
+
}
|
|
26
|
+
const keys = args.slice(0, -1);
|
|
27
|
+
|
|
28
|
+
try {
|
|
29
|
+
for (const key of keys) {
|
|
30
|
+
const t = engine.type(key);
|
|
31
|
+
if (t !== 'none' && t !== 'list') {
|
|
32
|
+
return { error: ERRORS.WRONGTYPE };
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
for (const key of keys) {
|
|
36
|
+
const val = engine.lpop(key, null);
|
|
37
|
+
if (val != null) {
|
|
38
|
+
return [key, val];
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
} catch (e) {
|
|
42
|
+
const msg = e && e.message ? e.message : String(e);
|
|
43
|
+
return { error: msg.startsWith('ERR ') ? msg : msg.startsWith('WRONGTYPE') ? msg : 'ERR ' + msg };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
if (!context) {
|
|
47
|
+
return { error: 'ERR blocking not supported in this context' };
|
|
48
|
+
}
|
|
49
|
+
return { block: { keys, kind: 'BLPOP', timeoutSeconds } };
|
|
50
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BRPOP key [key ...] timeout
|
|
3
|
+
* Block until one of the keys has an element (pop from right), or timeout (seconds). timeout 0 = block indefinitely.
|
|
4
|
+
* Returns [key, element] on success, nil on timeout.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { ERRORS } from '../engine/errors.js';
|
|
8
|
+
|
|
9
|
+
function parseTimeout(arg) {
|
|
10
|
+
if (arg == null || arg === '') return null;
|
|
11
|
+
const s = Buffer.isBuffer(arg) ? arg.toString('utf8') : String(arg);
|
|
12
|
+
const n = parseInt(s, 10);
|
|
13
|
+
if (Number.isNaN(n) || s.trim() === '' || String(n) !== s.trim()) return null;
|
|
14
|
+
return n;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function handleBrpop(engine, args, context) {
|
|
18
|
+
if (!args || args.length < 2) {
|
|
19
|
+
return { error: 'ERR wrong number of arguments for \'BRPOP\' command' };
|
|
20
|
+
}
|
|
21
|
+
const timeoutArg = args[args.length - 1];
|
|
22
|
+
const timeoutSeconds = parseTimeout(timeoutArg);
|
|
23
|
+
if (timeoutSeconds === null || timeoutSeconds < 0) {
|
|
24
|
+
return { error: 'ERR timeout is not an integer or out of range' };
|
|
25
|
+
}
|
|
26
|
+
const keys = args.slice(0, -1);
|
|
27
|
+
|
|
28
|
+
try {
|
|
29
|
+
for (const key of keys) {
|
|
30
|
+
const t = engine.type(key);
|
|
31
|
+
if (t !== 'none' && t !== 'list') {
|
|
32
|
+
return { error: ERRORS.WRONGTYPE };
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
for (const key of keys) {
|
|
36
|
+
const val = engine.rpop(key, null);
|
|
37
|
+
if (val != null) {
|
|
38
|
+
return [key, val];
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
} catch (e) {
|
|
42
|
+
const msg = e && e.message ? e.message : String(e);
|
|
43
|
+
return { error: msg.startsWith('ERR ') ? msg : msg.startsWith('WRONGTYPE') ? msg : 'ERR ' + msg };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
if (!context) {
|
|
47
|
+
return { error: 'ERR blocking not supported in this context' };
|
|
48
|
+
}
|
|
49
|
+
return { block: { keys, kind: 'BRPOP', timeoutSeconds } };
|
|
50
|
+
}
|
package/src/commands/registry.js
CHANGED
|
@@ -41,6 +41,8 @@ import * as lrange from './lrange.js';
|
|
|
41
41
|
import * as lindex from './lindex.js';
|
|
42
42
|
import * as lpop from './lpop.js';
|
|
43
43
|
import * as rpop from './rpop.js';
|
|
44
|
+
import * as blpop from './blpop.js';
|
|
45
|
+
import * as brpop from './brpop.js';
|
|
44
46
|
import * as scan from './scan.js';
|
|
45
47
|
import * as zadd from './zadd.js';
|
|
46
48
|
import * as zrem from './zrem.js';
|
|
@@ -97,8 +99,10 @@ const HANDLERS = new Map([
|
|
|
97
99
|
['LLEN', (e, a) => llen.handleLlen(e, a)],
|
|
98
100
|
['LRANGE', (e, a) => lrange.handleLrange(e, a)],
|
|
99
101
|
['LINDEX', (e, a) => lindex.handleLindex(e, a)],
|
|
100
|
-
['LPOP', (e, a) => lpop.handleLpop(e, a)],
|
|
101
|
-
['RPOP', (e, a) => rpop.handleRpop(e, a)],
|
|
102
|
+
['LPOP', (e, a, ctx) => lpop.handleLpop(e, a)],
|
|
103
|
+
['RPOP', (e, a, ctx) => rpop.handleRpop(e, a)],
|
|
104
|
+
['BLPOP', (e, a, ctx) => blpop.handleBlpop(e, a, ctx)],
|
|
105
|
+
['BRPOP', (e, a, ctx) => brpop.handleBrpop(e, a, ctx)],
|
|
102
106
|
['SCAN', (e, a) => scan.handleScan(e, a)],
|
|
103
107
|
['ZADD', (e, a) => zadd.handleZadd(e, a)],
|
|
104
108
|
['ZREM', (e, a) => zrem.handleZrem(e, a)],
|
|
@@ -123,9 +127,10 @@ const HANDLERS = new Map([
|
|
|
123
127
|
* Dispatch command. Full argv: [commandNameBuf, ...argBuffers].
|
|
124
128
|
* @param {object} engine
|
|
125
129
|
* @param {Buffer[]} argv - first element is command name, rest are arguments
|
|
126
|
-
* @
|
|
130
|
+
* @param {object} [context] - optional connection context (connectionId, writeResponse) for blocking commands
|
|
131
|
+
* @returns {{ result: unknown } | { error: string } | { quit: true } | { block: object }}
|
|
127
132
|
*/
|
|
128
|
-
export function dispatch(engine, argv) {
|
|
133
|
+
export function dispatch(engine, argv, context) {
|
|
129
134
|
if (!argv || argv.length === 0) {
|
|
130
135
|
return { error: 'ERR wrong number of arguments' };
|
|
131
136
|
}
|
|
@@ -136,9 +141,10 @@ export function dispatch(engine, argv) {
|
|
|
136
141
|
return { error: unsupported() };
|
|
137
142
|
}
|
|
138
143
|
try {
|
|
139
|
-
const result = handler(engine, args);
|
|
144
|
+
const result = handler(engine, args, context);
|
|
140
145
|
if (result && result.quit) return result;
|
|
141
146
|
if (result && result.error) return result;
|
|
147
|
+
if (result && result.block) return result;
|
|
142
148
|
return { result };
|
|
143
149
|
} catch (err) {
|
|
144
150
|
const msg = err && err.message ? err.message : String(err);
|