resplite 1.0.0 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,222 @@
1
+ /**
2
+ * Migration registry access (SPEC_F §F.5).
3
+ * Run metadata, dirty key tracking, and error logging.
4
+ */
5
+
6
+ const RUN_STATUS = {
7
+ RUNNING: 'running',
8
+ PAUSED: 'paused',
9
+ COMPLETED: 'completed',
10
+ FAILED: 'failed',
11
+ ABORTED: 'aborted',
12
+ };
13
+
14
+ /**
15
+ * @param {Buffer|string} key
16
+ * @returns {Buffer}
17
+ */
18
+ function asBlob(key) {
19
+ if (Buffer.isBuffer(key)) return key;
20
+ return Buffer.from(String(key), 'utf8');
21
+ }
22
+
23
+ /**
24
+ * Create a new migration run. Idempotent: if run_id exists, return it.
25
+ * @param {import('better-sqlite3').Database} db
26
+ * @param {string} runId
27
+ * @param {string} sourceUri
28
+ * @param {object} [options] - scan_count_hint
29
+ * @returns {{ run_id: string, created: boolean }}
30
+ */
31
+ export function createRun(db, runId, sourceUri, options = {}) {
32
+ const { scan_count_hint = 1000 } = options;
33
+ const now = Date.now();
34
+ const stmt = db.prepare(`
35
+ INSERT INTO migration_runs (
36
+ run_id, source_uri, started_at, updated_at, status,
37
+ scan_cursor, scan_count_hint,
38
+ scanned_keys, migrated_keys, skipped_keys, error_keys, migrated_bytes,
39
+ dirty_keys_seen, dirty_keys_applied, dirty_keys_deleted
40
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0, 0, 0)
41
+ ON CONFLICT(run_id) DO NOTHING
42
+ `);
43
+ stmt.run(runId, sourceUri, now, now, RUN_STATUS.RUNNING, '0', scan_count_hint);
44
+ const created = db.prepare('SELECT changes() as n').get().n > 0;
45
+ if (created) {
46
+ db.prepare('UPDATE migration_runs SET updated_at = ? WHERE run_id = ?').run(now, runId);
47
+ }
48
+ return { run_id: runId, created };
49
+ }
50
+
51
+ /**
52
+ * @param {import('better-sqlite3').Database} db
53
+ * @param {string} runId
54
+ * @returns {Record<string, unknown> | undefined}
55
+ */
56
+ export function getRun(db, runId) {
57
+ const row = db.prepare('SELECT * FROM migration_runs WHERE run_id = ?').get(runId);
58
+ return row ? (row instanceof Object ? row : undefined) : undefined;
59
+ }
60
+
61
+ /**
62
+ * @param {import('better-sqlite3').Database} db
63
+ * @param {string} runId
64
+ * @param {'running'|'paused'|'completed'|'failed'|'aborted'} status
65
+ */
66
+ export function setRunStatus(db, runId, status) {
67
+ const now = Date.now();
68
+ db.prepare('UPDATE migration_runs SET status = ?, updated_at = ? WHERE run_id = ?').run(status, now, runId);
69
+ }
70
+
71
+ /**
72
+ * Update bulk import progress (cursor and counters).
73
+ * @param {import('better-sqlite3').Database} db
74
+ * @param {string} runId
75
+ * @param {{
76
+ * scan_cursor?: string;
77
+ * scanned_keys?: number;
78
+ * migrated_keys?: number;
79
+ * skipped_keys?: number;
80
+ * error_keys?: number;
81
+ * migrated_bytes?: number;
82
+ * last_error?: string | null;
83
+ * }} updates
84
+ */
85
+ export function updateBulkProgress(db, runId, updates) {
86
+ const now = Date.now();
87
+ const run = getRun(db, runId);
88
+ if (!run) return;
89
+
90
+ const cursor = updates.scan_cursor !== undefined ? String(updates.scan_cursor) : run.scan_cursor;
91
+ const scanned_keys = updates.scanned_keys !== undefined ? updates.scanned_keys : run.scanned_keys;
92
+ const migrated_keys = updates.migrated_keys !== undefined ? updates.migrated_keys : run.migrated_keys;
93
+ const skipped_keys = updates.skipped_keys !== undefined ? updates.skipped_keys : run.skipped_keys;
94
+ const error_keys = updates.error_keys !== undefined ? updates.error_keys : run.error_keys;
95
+ const migrated_bytes = updates.migrated_bytes !== undefined ? updates.migrated_bytes : run.migrated_bytes;
96
+ const last_error = updates.last_error !== undefined ? updates.last_error : run.last_error;
97
+
98
+ db.prepare(`
99
+ UPDATE migration_runs SET
100
+ scan_cursor = ?,
101
+ scanned_keys = ?,
102
+ migrated_keys = ?,
103
+ skipped_keys = ?,
104
+ error_keys = ?,
105
+ migrated_bytes = ?,
106
+ last_error = ?,
107
+ updated_at = ?
108
+ WHERE run_id = ?
109
+ `).run(cursor, scanned_keys, migrated_keys, skipped_keys, error_keys, migrated_bytes, last_error ?? null, now, runId);
110
+ }
111
+
112
+ /**
113
+ * Upsert a dirty key (tracker saw an event). State: dirty or deleted per event type (SPEC_F F.6.2, F.6.3).
114
+ * @param {import('better-sqlite3').Database} db
115
+ * @param {string} runId
116
+ * @param {Buffer|string} key
117
+ * @param {string} event - e.g. "set", "hset", "del", "expired"
118
+ */
119
+ export function upsertDirtyKey(db, runId, key, event) {
120
+ const keyBlob = asBlob(key);
121
+ const now = Date.now();
122
+ const isDelete = event === 'del' || event === 'unlink' || event === 'expired';
123
+ const state = isDelete ? 'deleted' : 'dirty';
124
+
125
+ const existing = db.prepare('SELECT first_seen_at, state FROM migration_dirty_keys WHERE run_id = ? AND key = ?')
126
+ .get(runId, keyBlob);
127
+
128
+ if (existing) {
129
+ // If was deleted but we see a write again, revert to dirty (SPEC_F F.6.3)
130
+ const newState = existing.state === 'deleted' && !isDelete ? 'dirty' : (isDelete ? 'deleted' : 'dirty');
131
+ db.prepare(`
132
+ UPDATE migration_dirty_keys SET
133
+ last_seen_at = ?,
134
+ events_count = events_count + 1,
135
+ last_event = ?,
136
+ state = ?
137
+ WHERE run_id = ? AND key = ?
138
+ `).run(now, event, newState, runId, keyBlob);
139
+ } else {
140
+ db.prepare(`
141
+ INSERT INTO migration_dirty_keys (run_id, key, first_seen_at, last_seen_at, events_count, last_event, state)
142
+ VALUES (?, ?, ?, ?, 1, ?, ?)
143
+ `).run(runId, keyBlob, now, now, event, state);
144
+ }
145
+
146
+ db.prepare('UPDATE migration_runs SET dirty_keys_seen = dirty_keys_seen + 1, updated_at = ? WHERE run_id = ?')
147
+ .run(now, runId);
148
+ }
149
+
150
+ /**
151
+ * Get a batch of dirty keys for delta apply. ORDER BY last_seen_at ASC, LIMIT.
152
+ * @param {import('better-sqlite3').Database} db
153
+ * @param {string} runId
154
+ * @param {string} state - 'dirty'
155
+ * @param {number} limit
156
+ * @returns {{ key: Buffer }[]}
157
+ */
158
+ export function getDirtyBatch(db, runId, state, limit) {
159
+ const rows = db.prepare(`
160
+ SELECT key FROM migration_dirty_keys
161
+ WHERE run_id = ? AND state = ?
162
+ ORDER BY last_seen_at ASC
163
+ LIMIT ?
164
+ `).all(runId, state, limit);
165
+ return rows.map((r) => ({ key: r.key instanceof Buffer ? r.key : Buffer.from(r.key) }));
166
+ }
167
+
168
+ /**
169
+ * Mark a dirty key as applied or deleted and update run counters.
170
+ * @param {import('better-sqlite3').Database} db
171
+ * @param {string} runId
172
+ * @param {Buffer|string} key
173
+ * @param {'applied'|'deleted'|'skipped'|'error'} state
174
+ */
175
+ export function markDirtyState(db, runId, key, state) {
176
+ const keyBlob = asBlob(key);
177
+ const now = Date.now();
178
+ db.prepare('UPDATE migration_dirty_keys SET state = ? WHERE run_id = ? AND key = ?')
179
+ .run(state, runId, keyBlob);
180
+
181
+ if (state === 'applied') {
182
+ db.prepare('UPDATE migration_runs SET dirty_keys_applied = dirty_keys_applied + 1, updated_at = ? WHERE run_id = ?')
183
+ .run(now, runId);
184
+ } else if (state === 'deleted') {
185
+ db.prepare('UPDATE migration_runs SET dirty_keys_deleted = dirty_keys_deleted + 1, updated_at = ? WHERE run_id = ?')
186
+ .run(now, runId);
187
+ }
188
+ }
189
+
190
+ /**
191
+ * Log an error for the run (bounded table; consider retention in production).
192
+ * @param {import('better-sqlite3').Database} db
193
+ * @param {string} runId
194
+ * @param {string} stage - 'bulk' | 'dirty_apply' | 'verify'
195
+ * @param {string} message
196
+ * @param {Buffer|string|null} [key]
197
+ */
198
+ export function logError(db, runId, stage, message, key = null) {
199
+ const now = Date.now();
200
+ const keyBlob = key != null ? asBlob(key) : null;
201
+ db.prepare('INSERT INTO migration_errors (run_id, at, key, stage, message) VALUES (?, ?, ?, ?, ?)')
202
+ .run(runId, now, keyBlob, stage, message);
203
+ }
204
+
205
+ /**
206
+ * Get count of dirty keys by state for a run.
207
+ * @param {import('better-sqlite3').Database} db
208
+ * @param {string} runId
209
+ * @returns {Record<string, number>}
210
+ */
211
+ export function getDirtyCounts(db, runId) {
212
+ const rows = db.prepare(`
213
+ SELECT state, COUNT(*) as n FROM migration_dirty_keys WHERE run_id = ? GROUP BY state
214
+ `).all(runId);
215
+ const out = { dirty: 0, applied: 0, deleted: 0, skipped: 0, error: 0 };
216
+ for (const r of rows) {
217
+ out[r.state] = r.n;
218
+ }
219
+ return out;
220
+ }
221
+
222
+ export { RUN_STATUS };
@@ -0,0 +1,191 @@
1
+ /**
2
+ * Verification: sample keys and compare Redis vs RespLite destination (SPEC_F §F.9 Step 7).
3
+ */
4
+
5
+ import { openDb } from '../storage/sqlite/db.js';
6
+ import { createKeysStorage } from '../storage/sqlite/keys.js';
7
+ import { createStringsStorage } from '../storage/sqlite/strings.js';
8
+ import { createHashesStorage } from '../storage/sqlite/hashes.js';
9
+ import { createSetsStorage } from '../storage/sqlite/sets.js';
10
+ import { createListsStorage } from '../storage/sqlite/lists.js';
11
+ import { createZsetsStorage } from '../storage/sqlite/zsets.js';
12
+ import { KEY_TYPES } from '../storage/sqlite/schema.js';
13
+ import { asKey } from '../util/buffers.js';
14
+
15
+ /**
16
+ * Get key type and value from destination DB for comparison.
17
+ * @param {object} storages
18
+ * @param {Buffer} keyBuf
19
+ * @returns {{ type: string; value?: unknown; ttl?: number } | null}
20
+ */
21
+ function getKeyFromDestination(storages, keyBuf) {
22
+ const { keys, strings, hashes, sets, lists, zsets } = storages;
23
+ const meta = keys.get(keyBuf);
24
+ if (!meta) return null;
25
+
26
+ const typeNum = meta.type;
27
+ let type = 'unknown';
28
+ if (typeNum === KEY_TYPES.STRING) type = 'string';
29
+ else if (typeNum === KEY_TYPES.HASH) type = 'hash';
30
+ else if (typeNum === KEY_TYPES.SET) type = 'set';
31
+ else if (typeNum === KEY_TYPES.LIST) type = 'list';
32
+ else if (typeNum === KEY_TYPES.ZSET) type = 'zset';
33
+
34
+ let value;
35
+ if (typeNum === KEY_TYPES.STRING) {
36
+ const v = strings.get(keyBuf);
37
+ value = v ? v.toString('utf8') : null;
38
+ } else if (typeNum === KEY_TYPES.HASH) {
39
+ const flat = hashes.getAll(keyBuf);
40
+ const obj = {};
41
+ for (let i = 0; i < flat.length; i += 2) {
42
+ obj[flat[i].toString('utf8')] = flat[i + 1].toString('utf8');
43
+ }
44
+ value = obj;
45
+ } else if (typeNum === KEY_TYPES.SET) {
46
+ value = sets.members(keyBuf).map((b) => b.toString('utf8')).sort();
47
+ } else if (typeNum === KEY_TYPES.LIST) {
48
+ value = lists.lrange(keyBuf, 0, -1).map((b) => b.toString('utf8'));
49
+ } else if (typeNum === KEY_TYPES.ZSET) {
50
+ const flat = zsets.rangeByRank(keyBuf, 0, -1, { withScores: true });
51
+ value = [];
52
+ for (let i = 0; i < flat.length; i += 2) {
53
+ value.push({ member: flat[i].toString('utf8'), score: Number(flat[i + 1]) });
54
+ }
55
+ }
56
+
57
+ const now = Date.now();
58
+ const expiresAt = meta.expiresAt;
59
+ const ttl = expiresAt != null && expiresAt > now ? Math.floor((expiresAt - now) / 1000) : -1;
60
+
61
+ return { type, value, ttl };
62
+ }
63
+
64
+ /**
65
+ * Run verification: sample keys from Redis, compare with destination DB.
66
+ * @param {import('redis').RedisClientType} redisClient
67
+ * @param {string} dbPath
68
+ * @param {object} options
69
+ * @param {string} [options.pragmaTemplate='default']
70
+ * @param {number} [options.samplePct=0.5] - sample percentage (0.5 = 0.5%)
71
+ * @param {number} [options.maxSample=10000]
72
+ * @returns {Promise<{ sampled: number; matched: number; mismatches: Array<{ key: string; reason: string }> }>}
73
+ */
74
+ export async function runVerify(redisClient, dbPath, options = {}) {
75
+ const { pragmaTemplate = 'default', samplePct = 0.5, maxSample = 10000 } = options;
76
+
77
+ const db = openDb(dbPath, { pragmaTemplate });
78
+ const keys = createKeysStorage(db);
79
+ const strings = createStringsStorage(db, keys);
80
+ const hashes = createHashesStorage(db, keys);
81
+ const sets = createSetsStorage(db, keys);
82
+ const lists = createListsStorage(db, keys);
83
+ const zsets = createZsetsStorage(db, keys);
84
+ const storages = { keys, strings, hashes, sets, lists, zsets };
85
+
86
+ const keyList = [];
87
+ let cursor = 0;
88
+ const takeEvery = Math.max(1, Math.floor(100 / samplePct));
89
+ let index = 0;
90
+
91
+ do {
92
+ const result = await redisClient.scan(cursor, { COUNT: 500 });
93
+ const keysBatch = Array.isArray(result) ? result[1] : (result?.keys || []);
94
+ cursor = Array.isArray(result) ? parseInt(result[0], 10) : (result?.cursor ?? 0);
95
+ for (const k of keysBatch) {
96
+ if (index++ % takeEvery === 0) keyList.push(k);
97
+ if (keyList.length >= maxSample) break;
98
+ }
99
+ if (keyList.length >= maxSample) break;
100
+ } while (cursor !== 0);
101
+
102
+ let matched = 0;
103
+ const mismatches = [];
104
+
105
+ for (const keyName of keyList) {
106
+ const keyBuf = asKey(keyName);
107
+ const dest = getKeyFromDestination(storages, keyBuf);
108
+ try {
109
+ const redisType = (await redisClient.type(keyName)).toLowerCase();
110
+ let redisTtl = await redisClient.pTTL(keyName);
111
+ if (redisTtl === -2) redisTtl = -1;
112
+ else if (redisTtl > 0) redisTtl = Math.floor(redisTtl / 1000);
113
+
114
+ if (!dest) {
115
+ mismatches.push({ key: keyName, reason: 'missing in destination' });
116
+ continue;
117
+ }
118
+ if (dest.type !== redisType) {
119
+ mismatches.push({ key: keyName, reason: `type mismatch: Redis=${redisType} dest=${dest.type}` });
120
+ continue;
121
+ }
122
+ if (dest.ttl !== undefined && redisTtl >= 0 && dest.ttl !== redisTtl) {
123
+ if (Math.abs(dest.ttl - redisTtl) > 1) {
124
+ mismatches.push({ key: keyName, reason: `TTL mismatch: Redis=${redisTtl}s dest=${dest.ttl}s` });
125
+ continue;
126
+ }
127
+ }
128
+
129
+ if (redisType === 'string') {
130
+ const redisVal = await redisClient.get(keyName);
131
+ const destVal = dest.value;
132
+ if (String(redisVal ?? '') !== String(destVal ?? '')) {
133
+ mismatches.push({ key: keyName, reason: 'value mismatch (string)' });
134
+ continue;
135
+ }
136
+ } else if (redisType === 'hash') {
137
+ const redisObj = await redisClient.hGetAll(keyName);
138
+ const destObj = dest.value || {};
139
+ const redisKeys = Object.keys(redisObj || {}).sort();
140
+ const destKeys = Object.keys(destObj).sort();
141
+ if (redisKeys.join(',') !== destKeys.join(',')) {
142
+ mismatches.push({ key: keyName, reason: 'hash field mismatch' });
143
+ continue;
144
+ }
145
+ for (const f of redisKeys) {
146
+ if (String((redisObj || {})[f]) !== String(destObj[f])) {
147
+ mismatches.push({ key: keyName, reason: `hash field '${f}' value mismatch` });
148
+ break;
149
+ }
150
+ }
151
+ if (mismatches[mismatches.length - 1]?.key === keyName) continue;
152
+ } else if (redisType === 'set') {
153
+ const redisMembers = (await redisClient.sMembers(keyName)).sort();
154
+ const destMembers = (dest.value || []).slice().sort();
155
+ if (redisMembers.length !== destMembers.length || redisMembers.join(',') !== destMembers.join(',')) {
156
+ mismatches.push({ key: keyName, reason: 'set members mismatch' });
157
+ continue;
158
+ }
159
+ } else if (redisType === 'list') {
160
+ const redisList = await redisClient.lRange(keyName, 0, -1);
161
+ const destList = dest.value || [];
162
+ if (redisList.length !== destList.length || redisList.join(',') !== destList.join(',')) {
163
+ mismatches.push({ key: keyName, reason: 'list elements mismatch' });
164
+ continue;
165
+ }
166
+ } else if (redisType === 'zset') {
167
+ const redisZ = await redisClient.zRangeWithScores(keyName, 0, -1);
168
+ const destZ = dest.value || [];
169
+ if (redisZ.length !== destZ.length) {
170
+ mismatches.push({ key: keyName, reason: 'zset cardinality mismatch' });
171
+ continue;
172
+ }
173
+ for (let i = 0; i < redisZ.length; i++) {
174
+ const r = redisZ[i];
175
+ const d = destZ[i];
176
+ if (r.value !== d.member || Number(r.score) !== Number(d.score)) {
177
+ mismatches.push({ key: keyName, reason: 'zset member/score mismatch' });
178
+ break;
179
+ }
180
+ }
181
+ if (mismatches[mismatches.length - 1]?.key === keyName) continue;
182
+ }
183
+
184
+ matched++;
185
+ } catch (err) {
186
+ mismatches.push({ key: keyName, reason: err.message });
187
+ }
188
+ }
189
+
190
+ return { sampled: keyList.length, matched, mismatches };
191
+ }
@@ -6,36 +6,78 @@ import { RESPReader } from '../resp/parser.js';
6
6
  import { dispatch } from '../commands/registry.js';
7
7
  import { encode, encodeSimpleString, encodeError } from '../resp/encoder.js';
8
8
 
9
+ let nextConnectionId = 0;
10
+
9
11
  /**
10
12
  * @param {import('net').Socket} socket
11
13
  * @param {object} engine
12
14
  */
13
15
  export function handleConnection(socket, engine) {
14
16
  const reader = new RESPReader();
17
+ const connectionId = ++nextConnectionId;
18
+ const context = {
19
+ connectionId,
20
+ writeResponse(buf) {
21
+ if (socket.writable) socket.write(buf);
22
+ },
23
+ };
24
+
25
+ function writeResult(out) {
26
+ if (out.quit) {
27
+ socket.write(encodeSimpleString('OK'));
28
+ socket.end();
29
+ return true;
30
+ }
31
+ let buf;
32
+ if (out.error) {
33
+ buf = encodeError(out.error);
34
+ } else if (out.result && typeof out.result === 'object' && out.result.simple !== undefined) {
35
+ buf = encodeSimpleString(out.result.simple);
36
+ } else if (out.result && typeof out.result === 'object' && out.result.error !== undefined) {
37
+ buf = encodeError(out.result.error);
38
+ } else {
39
+ buf = encode(out.result);
40
+ }
41
+ socket.write(buf);
42
+ return false;
43
+ }
15
44
 
16
45
  socket.on('data', (chunk) => {
17
46
  reader.feed(chunk);
18
47
  const commands = reader.parseCommands();
19
48
  for (const argv of commands) {
20
- const out = dispatch(engine, argv);
49
+ const out = dispatch(engine, argv, context);
21
50
  if (out.quit) {
22
- socket.write(encodeSimpleString('OK'));
23
- socket.end();
51
+ writeResult(out);
24
52
  return;
25
53
  }
26
- let buf;
27
- if (out.error) {
28
- buf = encodeError(out.error);
29
- } else if (out.result && typeof out.result === 'object' && out.result.simple !== undefined) {
30
- buf = encodeSimpleString(out.result.simple);
31
- } else if (out.result && typeof out.result === 'object' && out.result.error !== undefined) {
32
- buf = encodeError(out.result.error);
33
- } else {
34
- buf = encode(out.result);
54
+ if (out.block) {
55
+ const { keys, kind, timeoutSeconds } = out.block;
56
+ const blockingManager = engine._blockingManager;
57
+ if (!blockingManager) {
58
+ socket.write(encodeError('ERR blocking not available'));
59
+ continue;
60
+ }
61
+ const resolve = (value) => {
62
+ const buf = value === null ? encode(null) : encode(value);
63
+ context.writeResponse(buf);
64
+ socket.resume();
65
+ };
66
+ const err = blockingManager.registerWaiter(keys, kind, timeoutSeconds, resolve, connectionId);
67
+ if (err.error) {
68
+ socket.write(encodeError(err.error));
69
+ continue;
70
+ }
71
+ socket.pause();
72
+ return;
35
73
  }
36
- socket.write(buf);
74
+ if (writeResult(out)) return;
37
75
  }
38
76
  });
39
77
 
78
+ socket.on('close', () => {
79
+ if (engine._blockingManager) engine._blockingManager.cancel(connectionId);
80
+ });
81
+
40
82
  socket.on('error', () => {});
41
83
  }
@@ -5,6 +5,7 @@
5
5
  import Database from 'better-sqlite3';
6
6
  import { applyPragmas } from './pragmas.js';
7
7
  import { applySchema } from './schema.js';
8
+ import { applyMigrationSchema } from './migration-schema.js';
8
9
 
9
10
  /**
10
11
  * @param {string} path - Database file path
@@ -16,5 +17,6 @@ export function openDb(path, options = {}) {
16
17
  const db = new Database(path, dbOptions);
17
18
  applyPragmas(db, pragmaTemplate);
18
19
  applySchema(db);
20
+ applyMigrationSchema(db);
19
21
  return db;
20
22
  }
@@ -0,0 +1,66 @@
1
+ /**
2
+ * Migration registry schema (SPEC_F §F.5.1).
3
+ * Tables: migration_runs, migration_dirty_keys, migration_errors.
4
+ */
5
+
6
+ export const MIGRATION_SCHEMA = `
7
+ CREATE TABLE IF NOT EXISTS migration_runs (
8
+ run_id TEXT PRIMARY KEY,
9
+ source_uri TEXT NOT NULL,
10
+ started_at INTEGER NOT NULL,
11
+ updated_at INTEGER NOT NULL,
12
+ status TEXT NOT NULL,
13
+
14
+ scan_cursor TEXT NOT NULL DEFAULT "0",
15
+ scan_count_hint INTEGER NOT NULL DEFAULT 1000,
16
+
17
+ scanned_keys INTEGER NOT NULL DEFAULT 0,
18
+ migrated_keys INTEGER NOT NULL DEFAULT 0,
19
+ skipped_keys INTEGER NOT NULL DEFAULT 0,
20
+ error_keys INTEGER NOT NULL DEFAULT 0,
21
+ migrated_bytes INTEGER NOT NULL DEFAULT 0,
22
+
23
+ dirty_keys_seen INTEGER NOT NULL DEFAULT 0,
24
+ dirty_keys_applied INTEGER NOT NULL DEFAULT 0,
25
+ dirty_keys_deleted INTEGER NOT NULL DEFAULT 0,
26
+
27
+ last_error TEXT
28
+ );
29
+
30
+ CREATE TABLE IF NOT EXISTS migration_dirty_keys (
31
+ run_id TEXT NOT NULL,
32
+ key BLOB NOT NULL,
33
+
34
+ first_seen_at INTEGER NOT NULL,
35
+ last_seen_at INTEGER NOT NULL,
36
+ events_count INTEGER NOT NULL DEFAULT 1,
37
+
38
+ last_event TEXT,
39
+ state TEXT NOT NULL DEFAULT "dirty",
40
+
41
+ PRIMARY KEY (run_id, key)
42
+ );
43
+
44
+ CREATE INDEX IF NOT EXISTS migration_dirty_keys_state_idx
45
+ ON migration_dirty_keys(run_id, state);
46
+
47
+ CREATE INDEX IF NOT EXISTS migration_dirty_keys_last_seen_idx
48
+ ON migration_dirty_keys(run_id, last_seen_at);
49
+
50
+ CREATE TABLE IF NOT EXISTS migration_errors (
51
+ run_id TEXT NOT NULL,
52
+ at INTEGER NOT NULL,
53
+ key BLOB,
54
+ stage TEXT NOT NULL,
55
+ message TEXT NOT NULL
56
+ );
57
+ CREATE INDEX IF NOT EXISTS migration_errors_at_idx ON migration_errors(run_id, at);
58
+ `;
59
+
60
+ /**
61
+ * Apply migration registry schema to database.
62
+ * @param {import('better-sqlite3').Database} db
63
+ */
64
+ export function applyMigrationSchema(db) {
65
+ db.exec(MIGRATION_SCHEMA);
66
+ }
package/tasks/todo.md CHANGED
@@ -23,3 +23,22 @@
23
23
  ## Not in scope (SPEC §26.3)
24
24
 
25
25
  - RDB parsing, AOF parsing, mirror mode, dual-write
26
+
27
+ ---
28
+
29
+ # Migration with Dirty Key Registry (SPEC_F)
30
+
31
+ ## Done
32
+
33
+ - [x] Migration schema: `migration_runs`, `migration_dirty_keys`, `migration_errors` in `src/storage/sqlite/migration-schema.js`
34
+ - [x] Registry layer: `src/migration/registry.js` (createRun, getRun, updateBulkProgress, upsertDirtyKey, getDirtyBatch, markDirtyState, logError, getDirtyCounts)
35
+ - [x] Bulk importer: `src/migration/bulk.js` with run_id, checkpointing, resume, max_rps, batch_keys/batch_bytes, pause/abort via status
36
+ - [x] Shared import-one: `src/migration/import-one.js` (fetch key from Redis + write to storages; used by bulk and apply-dirty)
37
+ - [x] Delta apply: `src/migration/apply-dirty.js` (apply dirty keys from registry: reimport or delete in destination)
38
+ - [x] Preflight: `src/migration/preflight.js` (key count, type distribution, notify-keyspace-events check, recommendations)
39
+ - [x] Verify: `src/migration/verify.js` (sample keys, compare Redis vs RespLite)
40
+ - [x] CLI `resplite-import`: `src/cli/resplite-import.js` (preflight, bulk, status, apply-dirty, verify)
41
+ - [x] CLI `resplite-dirty-tracker`: `src/cli/resplite-dirty-tracker.js` (start = PSUBSCRIBE keyevent, stop = update run status)
42
+ - [x] package.json `bin`: resplite-import, resplite-dirty-tracker
43
+ - [x] Unit tests: `test/unit/migration-registry.test.js`
44
+ - [x] README: minimal-downtime migration flow and commands