resplite 1.0.0 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -21
- package/package.json +5 -1
- package/spec/SPEC_F.md +505 -0
- package/src/blocking/manager.js +183 -0
- package/src/cli/resplite-dirty-tracker.js +124 -0
- package/src/cli/resplite-import.js +237 -0
- package/src/commands/blpop.js +50 -0
- package/src/commands/brpop.js +50 -0
- package/src/commands/registry.js +11 -5
- package/src/engine/engine.js +11 -3
- package/src/migration/apply-dirty.js +97 -0
- package/src/migration/bulk.js +181 -0
- package/src/migration/import-one.js +106 -0
- package/src/migration/preflight.js +62 -0
- package/src/migration/registry.js +222 -0
- package/src/migration/verify.js +191 -0
- package/src/server/connection.js +55 -13
- package/src/storage/sqlite/db.js +2 -0
- package/src/storage/sqlite/migration-schema.js +66 -0
- package/tasks/todo.md +19 -0
- package/test/integration/blocking.test.js +107 -0
- package/test/unit/migration-registry.test.js +127 -0
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration registry access (SPEC_F §F.5).
|
|
3
|
+
* Run metadata, dirty key tracking, and error logging.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
const RUN_STATUS = {
|
|
7
|
+
RUNNING: 'running',
|
|
8
|
+
PAUSED: 'paused',
|
|
9
|
+
COMPLETED: 'completed',
|
|
10
|
+
FAILED: 'failed',
|
|
11
|
+
ABORTED: 'aborted',
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* @param {Buffer|string} key
|
|
16
|
+
* @returns {Buffer}
|
|
17
|
+
*/
|
|
18
|
+
function asBlob(key) {
|
|
19
|
+
if (Buffer.isBuffer(key)) return key;
|
|
20
|
+
return Buffer.from(String(key), 'utf8');
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Create a new migration run. Idempotent: if run_id exists, return it.
|
|
25
|
+
* @param {import('better-sqlite3').Database} db
|
|
26
|
+
* @param {string} runId
|
|
27
|
+
* @param {string} sourceUri
|
|
28
|
+
* @param {object} [options] - scan_count_hint
|
|
29
|
+
* @returns {{ run_id: string, created: boolean }}
|
|
30
|
+
*/
|
|
31
|
+
export function createRun(db, runId, sourceUri, options = {}) {
|
|
32
|
+
const { scan_count_hint = 1000 } = options;
|
|
33
|
+
const now = Date.now();
|
|
34
|
+
const stmt = db.prepare(`
|
|
35
|
+
INSERT INTO migration_runs (
|
|
36
|
+
run_id, source_uri, started_at, updated_at, status,
|
|
37
|
+
scan_cursor, scan_count_hint,
|
|
38
|
+
scanned_keys, migrated_keys, skipped_keys, error_keys, migrated_bytes,
|
|
39
|
+
dirty_keys_seen, dirty_keys_applied, dirty_keys_deleted
|
|
40
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0, 0, 0)
|
|
41
|
+
ON CONFLICT(run_id) DO NOTHING
|
|
42
|
+
`);
|
|
43
|
+
stmt.run(runId, sourceUri, now, now, RUN_STATUS.RUNNING, '0', scan_count_hint);
|
|
44
|
+
const created = db.prepare('SELECT changes() as n').get().n > 0;
|
|
45
|
+
if (created) {
|
|
46
|
+
db.prepare('UPDATE migration_runs SET updated_at = ? WHERE run_id = ?').run(now, runId);
|
|
47
|
+
}
|
|
48
|
+
return { run_id: runId, created };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* @param {import('better-sqlite3').Database} db
|
|
53
|
+
* @param {string} runId
|
|
54
|
+
* @returns {Record<string, unknown> | undefined}
|
|
55
|
+
*/
|
|
56
|
+
export function getRun(db, runId) {
|
|
57
|
+
const row = db.prepare('SELECT * FROM migration_runs WHERE run_id = ?').get(runId);
|
|
58
|
+
return row ? (row instanceof Object ? row : undefined) : undefined;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* @param {import('better-sqlite3').Database} db
|
|
63
|
+
* @param {string} runId
|
|
64
|
+
* @param {'running'|'paused'|'completed'|'failed'|'aborted'} status
|
|
65
|
+
*/
|
|
66
|
+
export function setRunStatus(db, runId, status) {
|
|
67
|
+
const now = Date.now();
|
|
68
|
+
db.prepare('UPDATE migration_runs SET status = ?, updated_at = ? WHERE run_id = ?').run(status, now, runId);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Update bulk import progress (cursor and counters).
|
|
73
|
+
* @param {import('better-sqlite3').Database} db
|
|
74
|
+
* @param {string} runId
|
|
75
|
+
* @param {{
|
|
76
|
+
* scan_cursor?: string;
|
|
77
|
+
* scanned_keys?: number;
|
|
78
|
+
* migrated_keys?: number;
|
|
79
|
+
* skipped_keys?: number;
|
|
80
|
+
* error_keys?: number;
|
|
81
|
+
* migrated_bytes?: number;
|
|
82
|
+
* last_error?: string | null;
|
|
83
|
+
* }} updates
|
|
84
|
+
*/
|
|
85
|
+
export function updateBulkProgress(db, runId, updates) {
|
|
86
|
+
const now = Date.now();
|
|
87
|
+
const run = getRun(db, runId);
|
|
88
|
+
if (!run) return;
|
|
89
|
+
|
|
90
|
+
const cursor = updates.scan_cursor !== undefined ? String(updates.scan_cursor) : run.scan_cursor;
|
|
91
|
+
const scanned_keys = updates.scanned_keys !== undefined ? updates.scanned_keys : run.scanned_keys;
|
|
92
|
+
const migrated_keys = updates.migrated_keys !== undefined ? updates.migrated_keys : run.migrated_keys;
|
|
93
|
+
const skipped_keys = updates.skipped_keys !== undefined ? updates.skipped_keys : run.skipped_keys;
|
|
94
|
+
const error_keys = updates.error_keys !== undefined ? updates.error_keys : run.error_keys;
|
|
95
|
+
const migrated_bytes = updates.migrated_bytes !== undefined ? updates.migrated_bytes : run.migrated_bytes;
|
|
96
|
+
const last_error = updates.last_error !== undefined ? updates.last_error : run.last_error;
|
|
97
|
+
|
|
98
|
+
db.prepare(`
|
|
99
|
+
UPDATE migration_runs SET
|
|
100
|
+
scan_cursor = ?,
|
|
101
|
+
scanned_keys = ?,
|
|
102
|
+
migrated_keys = ?,
|
|
103
|
+
skipped_keys = ?,
|
|
104
|
+
error_keys = ?,
|
|
105
|
+
migrated_bytes = ?,
|
|
106
|
+
last_error = ?,
|
|
107
|
+
updated_at = ?
|
|
108
|
+
WHERE run_id = ?
|
|
109
|
+
`).run(cursor, scanned_keys, migrated_keys, skipped_keys, error_keys, migrated_bytes, last_error ?? null, now, runId);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Upsert a dirty key (tracker saw an event). State: dirty or deleted per event type (SPEC_F F.6.2, F.6.3).
|
|
114
|
+
* @param {import('better-sqlite3').Database} db
|
|
115
|
+
* @param {string} runId
|
|
116
|
+
* @param {Buffer|string} key
|
|
117
|
+
* @param {string} event - e.g. "set", "hset", "del", "expired"
|
|
118
|
+
*/
|
|
119
|
+
export function upsertDirtyKey(db, runId, key, event) {
|
|
120
|
+
const keyBlob = asBlob(key);
|
|
121
|
+
const now = Date.now();
|
|
122
|
+
const isDelete = event === 'del' || event === 'unlink' || event === 'expired';
|
|
123
|
+
const state = isDelete ? 'deleted' : 'dirty';
|
|
124
|
+
|
|
125
|
+
const existing = db.prepare('SELECT first_seen_at, state FROM migration_dirty_keys WHERE run_id = ? AND key = ?')
|
|
126
|
+
.get(runId, keyBlob);
|
|
127
|
+
|
|
128
|
+
if (existing) {
|
|
129
|
+
// If was deleted but we see a write again, revert to dirty (SPEC_F F.6.3)
|
|
130
|
+
const newState = existing.state === 'deleted' && !isDelete ? 'dirty' : (isDelete ? 'deleted' : 'dirty');
|
|
131
|
+
db.prepare(`
|
|
132
|
+
UPDATE migration_dirty_keys SET
|
|
133
|
+
last_seen_at = ?,
|
|
134
|
+
events_count = events_count + 1,
|
|
135
|
+
last_event = ?,
|
|
136
|
+
state = ?
|
|
137
|
+
WHERE run_id = ? AND key = ?
|
|
138
|
+
`).run(now, event, newState, runId, keyBlob);
|
|
139
|
+
} else {
|
|
140
|
+
db.prepare(`
|
|
141
|
+
INSERT INTO migration_dirty_keys (run_id, key, first_seen_at, last_seen_at, events_count, last_event, state)
|
|
142
|
+
VALUES (?, ?, ?, ?, 1, ?, ?)
|
|
143
|
+
`).run(runId, keyBlob, now, now, event, state);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
db.prepare('UPDATE migration_runs SET dirty_keys_seen = dirty_keys_seen + 1, updated_at = ? WHERE run_id = ?')
|
|
147
|
+
.run(now, runId);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Get a batch of dirty keys for delta apply. ORDER BY last_seen_at ASC, LIMIT.
|
|
152
|
+
* @param {import('better-sqlite3').Database} db
|
|
153
|
+
* @param {string} runId
|
|
154
|
+
* @param {string} state - 'dirty'
|
|
155
|
+
* @param {number} limit
|
|
156
|
+
* @returns {{ key: Buffer }[]}
|
|
157
|
+
*/
|
|
158
|
+
export function getDirtyBatch(db, runId, state, limit) {
|
|
159
|
+
const rows = db.prepare(`
|
|
160
|
+
SELECT key FROM migration_dirty_keys
|
|
161
|
+
WHERE run_id = ? AND state = ?
|
|
162
|
+
ORDER BY last_seen_at ASC
|
|
163
|
+
LIMIT ?
|
|
164
|
+
`).all(runId, state, limit);
|
|
165
|
+
return rows.map((r) => ({ key: r.key instanceof Buffer ? r.key : Buffer.from(r.key) }));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Mark a dirty key as applied or deleted and update run counters.
|
|
170
|
+
* @param {import('better-sqlite3').Database} db
|
|
171
|
+
* @param {string} runId
|
|
172
|
+
* @param {Buffer|string} key
|
|
173
|
+
* @param {'applied'|'deleted'|'skipped'|'error'} state
|
|
174
|
+
*/
|
|
175
|
+
export function markDirtyState(db, runId, key, state) {
|
|
176
|
+
const keyBlob = asBlob(key);
|
|
177
|
+
const now = Date.now();
|
|
178
|
+
db.prepare('UPDATE migration_dirty_keys SET state = ? WHERE run_id = ? AND key = ?')
|
|
179
|
+
.run(state, runId, keyBlob);
|
|
180
|
+
|
|
181
|
+
if (state === 'applied') {
|
|
182
|
+
db.prepare('UPDATE migration_runs SET dirty_keys_applied = dirty_keys_applied + 1, updated_at = ? WHERE run_id = ?')
|
|
183
|
+
.run(now, runId);
|
|
184
|
+
} else if (state === 'deleted') {
|
|
185
|
+
db.prepare('UPDATE migration_runs SET dirty_keys_deleted = dirty_keys_deleted + 1, updated_at = ? WHERE run_id = ?')
|
|
186
|
+
.run(now, runId);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Log an error for the run (bounded table; consider retention in production).
|
|
192
|
+
* @param {import('better-sqlite3').Database} db
|
|
193
|
+
* @param {string} runId
|
|
194
|
+
* @param {string} stage - 'bulk' | 'dirty_apply' | 'verify'
|
|
195
|
+
* @param {string} message
|
|
196
|
+
* @param {Buffer|string|null} [key]
|
|
197
|
+
*/
|
|
198
|
+
export function logError(db, runId, stage, message, key = null) {
|
|
199
|
+
const now = Date.now();
|
|
200
|
+
const keyBlob = key != null ? asBlob(key) : null;
|
|
201
|
+
db.prepare('INSERT INTO migration_errors (run_id, at, key, stage, message) VALUES (?, ?, ?, ?, ?)')
|
|
202
|
+
.run(runId, now, keyBlob, stage, message);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Get count of dirty keys by state for a run.
|
|
207
|
+
* @param {import('better-sqlite3').Database} db
|
|
208
|
+
* @param {string} runId
|
|
209
|
+
* @returns {Record<string, number>}
|
|
210
|
+
*/
|
|
211
|
+
export function getDirtyCounts(db, runId) {
|
|
212
|
+
const rows = db.prepare(`
|
|
213
|
+
SELECT state, COUNT(*) as n FROM migration_dirty_keys WHERE run_id = ? GROUP BY state
|
|
214
|
+
`).all(runId);
|
|
215
|
+
const out = { dirty: 0, applied: 0, deleted: 0, skipped: 0, error: 0 };
|
|
216
|
+
for (const r of rows) {
|
|
217
|
+
out[r.state] = r.n;
|
|
218
|
+
}
|
|
219
|
+
return out;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export { RUN_STATUS };
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verification: sample keys and compare Redis vs RespLite destination (SPEC_F §F.9 Step 7).
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { openDb } from '../storage/sqlite/db.js';
|
|
6
|
+
import { createKeysStorage } from '../storage/sqlite/keys.js';
|
|
7
|
+
import { createStringsStorage } from '../storage/sqlite/strings.js';
|
|
8
|
+
import { createHashesStorage } from '../storage/sqlite/hashes.js';
|
|
9
|
+
import { createSetsStorage } from '../storage/sqlite/sets.js';
|
|
10
|
+
import { createListsStorage } from '../storage/sqlite/lists.js';
|
|
11
|
+
import { createZsetsStorage } from '../storage/sqlite/zsets.js';
|
|
12
|
+
import { KEY_TYPES } from '../storage/sqlite/schema.js';
|
|
13
|
+
import { asKey } from '../util/buffers.js';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Get key type and value from destination DB for comparison.
|
|
17
|
+
* @param {object} storages
|
|
18
|
+
* @param {Buffer} keyBuf
|
|
19
|
+
* @returns {{ type: string; value?: unknown; ttl?: number } | null}
|
|
20
|
+
*/
|
|
21
|
+
function getKeyFromDestination(storages, keyBuf) {
|
|
22
|
+
const { keys, strings, hashes, sets, lists, zsets } = storages;
|
|
23
|
+
const meta = keys.get(keyBuf);
|
|
24
|
+
if (!meta) return null;
|
|
25
|
+
|
|
26
|
+
const typeNum = meta.type;
|
|
27
|
+
let type = 'unknown';
|
|
28
|
+
if (typeNum === KEY_TYPES.STRING) type = 'string';
|
|
29
|
+
else if (typeNum === KEY_TYPES.HASH) type = 'hash';
|
|
30
|
+
else if (typeNum === KEY_TYPES.SET) type = 'set';
|
|
31
|
+
else if (typeNum === KEY_TYPES.LIST) type = 'list';
|
|
32
|
+
else if (typeNum === KEY_TYPES.ZSET) type = 'zset';
|
|
33
|
+
|
|
34
|
+
let value;
|
|
35
|
+
if (typeNum === KEY_TYPES.STRING) {
|
|
36
|
+
const v = strings.get(keyBuf);
|
|
37
|
+
value = v ? v.toString('utf8') : null;
|
|
38
|
+
} else if (typeNum === KEY_TYPES.HASH) {
|
|
39
|
+
const flat = hashes.getAll(keyBuf);
|
|
40
|
+
const obj = {};
|
|
41
|
+
for (let i = 0; i < flat.length; i += 2) {
|
|
42
|
+
obj[flat[i].toString('utf8')] = flat[i + 1].toString('utf8');
|
|
43
|
+
}
|
|
44
|
+
value = obj;
|
|
45
|
+
} else if (typeNum === KEY_TYPES.SET) {
|
|
46
|
+
value = sets.members(keyBuf).map((b) => b.toString('utf8')).sort();
|
|
47
|
+
} else if (typeNum === KEY_TYPES.LIST) {
|
|
48
|
+
value = lists.lrange(keyBuf, 0, -1).map((b) => b.toString('utf8'));
|
|
49
|
+
} else if (typeNum === KEY_TYPES.ZSET) {
|
|
50
|
+
const flat = zsets.rangeByRank(keyBuf, 0, -1, { withScores: true });
|
|
51
|
+
value = [];
|
|
52
|
+
for (let i = 0; i < flat.length; i += 2) {
|
|
53
|
+
value.push({ member: flat[i].toString('utf8'), score: Number(flat[i + 1]) });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const now = Date.now();
|
|
58
|
+
const expiresAt = meta.expiresAt;
|
|
59
|
+
const ttl = expiresAt != null && expiresAt > now ? Math.floor((expiresAt - now) / 1000) : -1;
|
|
60
|
+
|
|
61
|
+
return { type, value, ttl };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Run verification: sample keys from Redis, compare with destination DB.
|
|
66
|
+
* @param {import('redis').RedisClientType} redisClient
|
|
67
|
+
* @param {string} dbPath
|
|
68
|
+
* @param {object} options
|
|
69
|
+
* @param {string} [options.pragmaTemplate='default']
|
|
70
|
+
* @param {number} [options.samplePct=0.5] - sample percentage (0.5 = 0.5%)
|
|
71
|
+
* @param {number} [options.maxSample=10000]
|
|
72
|
+
* @returns {Promise<{ sampled: number; matched: number; mismatches: Array<{ key: string; reason: string }> }>}
|
|
73
|
+
*/
|
|
74
|
+
export async function runVerify(redisClient, dbPath, options = {}) {
|
|
75
|
+
const { pragmaTemplate = 'default', samplePct = 0.5, maxSample = 10000 } = options;
|
|
76
|
+
|
|
77
|
+
const db = openDb(dbPath, { pragmaTemplate });
|
|
78
|
+
const keys = createKeysStorage(db);
|
|
79
|
+
const strings = createStringsStorage(db, keys);
|
|
80
|
+
const hashes = createHashesStorage(db, keys);
|
|
81
|
+
const sets = createSetsStorage(db, keys);
|
|
82
|
+
const lists = createListsStorage(db, keys);
|
|
83
|
+
const zsets = createZsetsStorage(db, keys);
|
|
84
|
+
const storages = { keys, strings, hashes, sets, lists, zsets };
|
|
85
|
+
|
|
86
|
+
const keyList = [];
|
|
87
|
+
let cursor = 0;
|
|
88
|
+
const takeEvery = Math.max(1, Math.floor(100 / samplePct));
|
|
89
|
+
let index = 0;
|
|
90
|
+
|
|
91
|
+
do {
|
|
92
|
+
const result = await redisClient.scan(cursor, { COUNT: 500 });
|
|
93
|
+
const keysBatch = Array.isArray(result) ? result[1] : (result?.keys || []);
|
|
94
|
+
cursor = Array.isArray(result) ? parseInt(result[0], 10) : (result?.cursor ?? 0);
|
|
95
|
+
for (const k of keysBatch) {
|
|
96
|
+
if (index++ % takeEvery === 0) keyList.push(k);
|
|
97
|
+
if (keyList.length >= maxSample) break;
|
|
98
|
+
}
|
|
99
|
+
if (keyList.length >= maxSample) break;
|
|
100
|
+
} while (cursor !== 0);
|
|
101
|
+
|
|
102
|
+
let matched = 0;
|
|
103
|
+
const mismatches = [];
|
|
104
|
+
|
|
105
|
+
for (const keyName of keyList) {
|
|
106
|
+
const keyBuf = asKey(keyName);
|
|
107
|
+
const dest = getKeyFromDestination(storages, keyBuf);
|
|
108
|
+
try {
|
|
109
|
+
const redisType = (await redisClient.type(keyName)).toLowerCase();
|
|
110
|
+
let redisTtl = await redisClient.pTTL(keyName);
|
|
111
|
+
if (redisTtl === -2) redisTtl = -1;
|
|
112
|
+
else if (redisTtl > 0) redisTtl = Math.floor(redisTtl / 1000);
|
|
113
|
+
|
|
114
|
+
if (!dest) {
|
|
115
|
+
mismatches.push({ key: keyName, reason: 'missing in destination' });
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
if (dest.type !== redisType) {
|
|
119
|
+
mismatches.push({ key: keyName, reason: `type mismatch: Redis=${redisType} dest=${dest.type}` });
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
if (dest.ttl !== undefined && redisTtl >= 0 && dest.ttl !== redisTtl) {
|
|
123
|
+
if (Math.abs(dest.ttl - redisTtl) > 1) {
|
|
124
|
+
mismatches.push({ key: keyName, reason: `TTL mismatch: Redis=${redisTtl}s dest=${dest.ttl}s` });
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if (redisType === 'string') {
|
|
130
|
+
const redisVal = await redisClient.get(keyName);
|
|
131
|
+
const destVal = dest.value;
|
|
132
|
+
if (String(redisVal ?? '') !== String(destVal ?? '')) {
|
|
133
|
+
mismatches.push({ key: keyName, reason: 'value mismatch (string)' });
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
} else if (redisType === 'hash') {
|
|
137
|
+
const redisObj = await redisClient.hGetAll(keyName);
|
|
138
|
+
const destObj = dest.value || {};
|
|
139
|
+
const redisKeys = Object.keys(redisObj || {}).sort();
|
|
140
|
+
const destKeys = Object.keys(destObj).sort();
|
|
141
|
+
if (redisKeys.join(',') !== destKeys.join(',')) {
|
|
142
|
+
mismatches.push({ key: keyName, reason: 'hash field mismatch' });
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
for (const f of redisKeys) {
|
|
146
|
+
if (String((redisObj || {})[f]) !== String(destObj[f])) {
|
|
147
|
+
mismatches.push({ key: keyName, reason: `hash field '${f}' value mismatch` });
|
|
148
|
+
break;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
if (mismatches[mismatches.length - 1]?.key === keyName) continue;
|
|
152
|
+
} else if (redisType === 'set') {
|
|
153
|
+
const redisMembers = (await redisClient.sMembers(keyName)).sort();
|
|
154
|
+
const destMembers = (dest.value || []).slice().sort();
|
|
155
|
+
if (redisMembers.length !== destMembers.length || redisMembers.join(',') !== destMembers.join(',')) {
|
|
156
|
+
mismatches.push({ key: keyName, reason: 'set members mismatch' });
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
} else if (redisType === 'list') {
|
|
160
|
+
const redisList = await redisClient.lRange(keyName, 0, -1);
|
|
161
|
+
const destList = dest.value || [];
|
|
162
|
+
if (redisList.length !== destList.length || redisList.join(',') !== destList.join(',')) {
|
|
163
|
+
mismatches.push({ key: keyName, reason: 'list elements mismatch' });
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
} else if (redisType === 'zset') {
|
|
167
|
+
const redisZ = await redisClient.zRangeWithScores(keyName, 0, -1);
|
|
168
|
+
const destZ = dest.value || [];
|
|
169
|
+
if (redisZ.length !== destZ.length) {
|
|
170
|
+
mismatches.push({ key: keyName, reason: 'zset cardinality mismatch' });
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
for (let i = 0; i < redisZ.length; i++) {
|
|
174
|
+
const r = redisZ[i];
|
|
175
|
+
const d = destZ[i];
|
|
176
|
+
if (r.value !== d.member || Number(r.score) !== Number(d.score)) {
|
|
177
|
+
mismatches.push({ key: keyName, reason: 'zset member/score mismatch' });
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
if (mismatches[mismatches.length - 1]?.key === keyName) continue;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
matched++;
|
|
185
|
+
} catch (err) {
|
|
186
|
+
mismatches.push({ key: keyName, reason: err.message });
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
return { sampled: keyList.length, matched, mismatches };
|
|
191
|
+
}
|
package/src/server/connection.js
CHANGED
|
@@ -6,36 +6,78 @@ import { RESPReader } from '../resp/parser.js';
|
|
|
6
6
|
import { dispatch } from '../commands/registry.js';
|
|
7
7
|
import { encode, encodeSimpleString, encodeError } from '../resp/encoder.js';
|
|
8
8
|
|
|
9
|
+
let nextConnectionId = 0;
|
|
10
|
+
|
|
9
11
|
/**
|
|
10
12
|
* @param {import('net').Socket} socket
|
|
11
13
|
* @param {object} engine
|
|
12
14
|
*/
|
|
13
15
|
export function handleConnection(socket, engine) {
|
|
14
16
|
const reader = new RESPReader();
|
|
17
|
+
const connectionId = ++nextConnectionId;
|
|
18
|
+
const context = {
|
|
19
|
+
connectionId,
|
|
20
|
+
writeResponse(buf) {
|
|
21
|
+
if (socket.writable) socket.write(buf);
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
function writeResult(out) {
|
|
26
|
+
if (out.quit) {
|
|
27
|
+
socket.write(encodeSimpleString('OK'));
|
|
28
|
+
socket.end();
|
|
29
|
+
return true;
|
|
30
|
+
}
|
|
31
|
+
let buf;
|
|
32
|
+
if (out.error) {
|
|
33
|
+
buf = encodeError(out.error);
|
|
34
|
+
} else if (out.result && typeof out.result === 'object' && out.result.simple !== undefined) {
|
|
35
|
+
buf = encodeSimpleString(out.result.simple);
|
|
36
|
+
} else if (out.result && typeof out.result === 'object' && out.result.error !== undefined) {
|
|
37
|
+
buf = encodeError(out.result.error);
|
|
38
|
+
} else {
|
|
39
|
+
buf = encode(out.result);
|
|
40
|
+
}
|
|
41
|
+
socket.write(buf);
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
15
44
|
|
|
16
45
|
socket.on('data', (chunk) => {
|
|
17
46
|
reader.feed(chunk);
|
|
18
47
|
const commands = reader.parseCommands();
|
|
19
48
|
for (const argv of commands) {
|
|
20
|
-
const out = dispatch(engine, argv);
|
|
49
|
+
const out = dispatch(engine, argv, context);
|
|
21
50
|
if (out.quit) {
|
|
22
|
-
|
|
23
|
-
socket.end();
|
|
51
|
+
writeResult(out);
|
|
24
52
|
return;
|
|
25
53
|
}
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
54
|
+
if (out.block) {
|
|
55
|
+
const { keys, kind, timeoutSeconds } = out.block;
|
|
56
|
+
const blockingManager = engine._blockingManager;
|
|
57
|
+
if (!blockingManager) {
|
|
58
|
+
socket.write(encodeError('ERR blocking not available'));
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
const resolve = (value) => {
|
|
62
|
+
const buf = value === null ? encode(null) : encode(value);
|
|
63
|
+
context.writeResponse(buf);
|
|
64
|
+
socket.resume();
|
|
65
|
+
};
|
|
66
|
+
const err = blockingManager.registerWaiter(keys, kind, timeoutSeconds, resolve, connectionId);
|
|
67
|
+
if (err.error) {
|
|
68
|
+
socket.write(encodeError(err.error));
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
socket.pause();
|
|
72
|
+
return;
|
|
35
73
|
}
|
|
36
|
-
|
|
74
|
+
if (writeResult(out)) return;
|
|
37
75
|
}
|
|
38
76
|
});
|
|
39
77
|
|
|
78
|
+
socket.on('close', () => {
|
|
79
|
+
if (engine._blockingManager) engine._blockingManager.cancel(connectionId);
|
|
80
|
+
});
|
|
81
|
+
|
|
40
82
|
socket.on('error', () => {});
|
|
41
83
|
}
|
package/src/storage/sqlite/db.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import Database from 'better-sqlite3';
|
|
6
6
|
import { applyPragmas } from './pragmas.js';
|
|
7
7
|
import { applySchema } from './schema.js';
|
|
8
|
+
import { applyMigrationSchema } from './migration-schema.js';
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* @param {string} path - Database file path
|
|
@@ -16,5 +17,6 @@ export function openDb(path, options = {}) {
|
|
|
16
17
|
const db = new Database(path, dbOptions);
|
|
17
18
|
applyPragmas(db, pragmaTemplate);
|
|
18
19
|
applySchema(db);
|
|
20
|
+
applyMigrationSchema(db);
|
|
19
21
|
return db;
|
|
20
22
|
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration registry schema (SPEC_F §F.5.1).
|
|
3
|
+
* Tables: migration_runs, migration_dirty_keys, migration_errors.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
export const MIGRATION_SCHEMA = `
|
|
7
|
+
CREATE TABLE IF NOT EXISTS migration_runs (
|
|
8
|
+
run_id TEXT PRIMARY KEY,
|
|
9
|
+
source_uri TEXT NOT NULL,
|
|
10
|
+
started_at INTEGER NOT NULL,
|
|
11
|
+
updated_at INTEGER NOT NULL,
|
|
12
|
+
status TEXT NOT NULL,
|
|
13
|
+
|
|
14
|
+
scan_cursor TEXT NOT NULL DEFAULT "0",
|
|
15
|
+
scan_count_hint INTEGER NOT NULL DEFAULT 1000,
|
|
16
|
+
|
|
17
|
+
scanned_keys INTEGER NOT NULL DEFAULT 0,
|
|
18
|
+
migrated_keys INTEGER NOT NULL DEFAULT 0,
|
|
19
|
+
skipped_keys INTEGER NOT NULL DEFAULT 0,
|
|
20
|
+
error_keys INTEGER NOT NULL DEFAULT 0,
|
|
21
|
+
migrated_bytes INTEGER NOT NULL DEFAULT 0,
|
|
22
|
+
|
|
23
|
+
dirty_keys_seen INTEGER NOT NULL DEFAULT 0,
|
|
24
|
+
dirty_keys_applied INTEGER NOT NULL DEFAULT 0,
|
|
25
|
+
dirty_keys_deleted INTEGER NOT NULL DEFAULT 0,
|
|
26
|
+
|
|
27
|
+
last_error TEXT
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
CREATE TABLE IF NOT EXISTS migration_dirty_keys (
|
|
31
|
+
run_id TEXT NOT NULL,
|
|
32
|
+
key BLOB NOT NULL,
|
|
33
|
+
|
|
34
|
+
first_seen_at INTEGER NOT NULL,
|
|
35
|
+
last_seen_at INTEGER NOT NULL,
|
|
36
|
+
events_count INTEGER NOT NULL DEFAULT 1,
|
|
37
|
+
|
|
38
|
+
last_event TEXT,
|
|
39
|
+
state TEXT NOT NULL DEFAULT "dirty",
|
|
40
|
+
|
|
41
|
+
PRIMARY KEY (run_id, key)
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
CREATE INDEX IF NOT EXISTS migration_dirty_keys_state_idx
|
|
45
|
+
ON migration_dirty_keys(run_id, state);
|
|
46
|
+
|
|
47
|
+
CREATE INDEX IF NOT EXISTS migration_dirty_keys_last_seen_idx
|
|
48
|
+
ON migration_dirty_keys(run_id, last_seen_at);
|
|
49
|
+
|
|
50
|
+
CREATE TABLE IF NOT EXISTS migration_errors (
|
|
51
|
+
run_id TEXT NOT NULL,
|
|
52
|
+
at INTEGER NOT NULL,
|
|
53
|
+
key BLOB,
|
|
54
|
+
stage TEXT NOT NULL,
|
|
55
|
+
message TEXT NOT NULL
|
|
56
|
+
);
|
|
57
|
+
CREATE INDEX IF NOT EXISTS migration_errors_at_idx ON migration_errors(run_id, at);
|
|
58
|
+
`;
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Apply migration registry schema to database.
|
|
62
|
+
* @param {import('better-sqlite3').Database} db
|
|
63
|
+
*/
|
|
64
|
+
export function applyMigrationSchema(db) {
|
|
65
|
+
db.exec(MIGRATION_SCHEMA);
|
|
66
|
+
}
|
package/tasks/todo.md
CHANGED
|
@@ -23,3 +23,22 @@
|
|
|
23
23
|
## Not in scope (SPEC §26.3)
|
|
24
24
|
|
|
25
25
|
- RDB parsing, AOF parsing, mirror mode, dual-write
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
# Migration with Dirty Key Registry (SPEC_F)
|
|
30
|
+
|
|
31
|
+
## Done
|
|
32
|
+
|
|
33
|
+
- [x] Migration schema: `migration_runs`, `migration_dirty_keys`, `migration_errors` in `src/storage/sqlite/migration-schema.js`
|
|
34
|
+
- [x] Registry layer: `src/migration/registry.js` (createRun, getRun, updateBulkProgress, upsertDirtyKey, getDirtyBatch, markDirtyState, logError, getDirtyCounts)
|
|
35
|
+
- [x] Bulk importer: `src/migration/bulk.js` with run_id, checkpointing, resume, max_rps, batch_keys/batch_bytes, pause/abort via status
|
|
36
|
+
- [x] Shared import-one: `src/migration/import-one.js` (fetch key from Redis + write to storages; used by bulk and apply-dirty)
|
|
37
|
+
- [x] Delta apply: `src/migration/apply-dirty.js` (apply dirty keys from registry: reimport or delete in destination)
|
|
38
|
+
- [x] Preflight: `src/migration/preflight.js` (key count, type distribution, notify-keyspace-events check, recommendations)
|
|
39
|
+
- [x] Verify: `src/migration/verify.js` (sample keys, compare Redis vs RespLite)
|
|
40
|
+
- [x] CLI `resplite-import`: `src/cli/resplite-import.js` (preflight, bulk, status, apply-dirty, verify)
|
|
41
|
+
- [x] CLI `resplite-dirty-tracker`: `src/cli/resplite-dirty-tracker.js` (start = PSUBSCRIBE keyevent, stop = update run status)
|
|
42
|
+
- [x] package.json `bin`: resplite-import, resplite-dirty-tracker
|
|
43
|
+
- [x] Unit tests: `test/unit/migration-registry.test.js`
|
|
44
|
+
- [x] README: minimal-downtime migration flow and commands
|