resplite 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +207 -212
- package/package.json +1 -1
- package/src/cli/resplite-import.js +4 -2
- package/src/embed.js +10 -0
- package/src/index.js +58 -20
- package/src/migration/bulk.js +2 -2
- package/src/migration/index.js +3 -3
- package/src/server/tcp-server.js +6 -1
package/README.md
CHANGED
|
@@ -87,7 +87,7 @@ OK
|
|
|
87
87
|
|
|
88
88
|
### Standalone server script (fixed port)
|
|
89
89
|
|
|
90
|
-
Run this as a persistent background process (`node server.js`). RESPLite will listen on port 6380 and stay up until the process receives SIGINT (Ctrl+C) or SIGTERM; then it closes the server and exits cleanly.
|
|
90
|
+
Run this as a persistent background process (`node server.js`). RESPLite will listen on port 6380 and stay up until the process receives SIGINT (Ctrl+C) or SIGTERM; then it closes the server and exits cleanly. If you kill the process (e.g. SIGKILL or force quit), all client connections are closed as well — with the default configuration the server runs in the same process, so when the process exits the TCP server and its connections are torn down.
|
|
91
91
|
|
|
92
92
|
```javascript
|
|
93
93
|
// server.js
|
|
@@ -96,13 +96,6 @@ import { createRESPlite } from 'resplite/embed';
|
|
|
96
96
|
const srv = await createRESPlite({ port: 6380, db: './data.db' });
|
|
97
97
|
console.log(`RESPLite listening on ${srv.host}:${srv.port}`);
|
|
98
98
|
|
|
99
|
-
async function shutdown() {
|
|
100
|
-
await srv.close();
|
|
101
|
-
process.exit(0);
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
process.on('SIGINT', shutdown);
|
|
105
|
-
process.on('SIGTERM', shutdown);
|
|
106
99
|
```
|
|
107
100
|
|
|
108
101
|
Then connect from any other script or process:
|
|
@@ -181,6 +174,212 @@ const srv = await createRESPlite({
|
|
|
181
174
|
| `onCommandError` | A command failed (wrong type, invalid args, or handler threw). |
|
|
182
175
|
| `onSocketError` | The connection socket emitted an error (e.g. `ECONNRESET`). |
|
|
183
176
|
|
|
177
|
+
## Migration from Redis
|
|
178
|
+
|
|
179
|
+
RESPLite is a good fit for migrating **non-replicated Redis** instances that have **grown large** (e.g. tens of GB) and where RESPLite's latency is acceptable. The SPEC_F flow (dirty-key tracker, bulk import, cutover) is designed for that scenario with minimal downtime.
|
|
180
|
+
|
|
181
|
+
Migration supports two modes:
|
|
182
|
+
|
|
183
|
+
### Simple one-shot import (legacy)
|
|
184
|
+
|
|
185
|
+
For small datasets or when downtime is acceptable:
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
# Default: redis://127.0.0.1:6379 → ./data.db
|
|
189
|
+
npm run import-from-redis -- --db ./migrated.db
|
|
190
|
+
|
|
191
|
+
# Custom Redis URL
|
|
192
|
+
npm run import-from-redis -- --db ./migrated.db --redis-url redis://127.0.0.1:6379
|
|
193
|
+
|
|
194
|
+
# Or host/port
|
|
195
|
+
npm run import-from-redis -- --db ./migrated.db --host 127.0.0.1 --port 6379
|
|
196
|
+
|
|
197
|
+
# Optional: PRAGMA template for the target DB
|
|
198
|
+
npm run import-from-redis -- --db ./migrated.db --pragma-template performance
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
### Redis with authentication
|
|
202
|
+
|
|
203
|
+
Migration supports Redis instances protected by a password. Use a Redis URL that includes the password (or username and password for Redis 6+ ACL):
|
|
204
|
+
|
|
205
|
+
- **Password only:** `redis://:PASSWORD@host:port`
|
|
206
|
+
- **Username and password:** `redis://username:PASSWORD@host:port`
|
|
207
|
+
|
|
208
|
+
Examples:
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
# One-shot import from authenticated Redis
|
|
212
|
+
npm run import-from-redis -- --db ./migrated.db --redis-url "redis://:mysecret@127.0.0.1:6379"
|
|
213
|
+
|
|
214
|
+
# SPEC_F flow: use --from with the full URL (or set RESPLITE_IMPORT_FROM)
|
|
215
|
+
npx resplite-import preflight --from "redis://:mysecret@10.0.0.10:6379" --to ./resplite.db
|
|
216
|
+
npx resplite-dirty-tracker start --run-id run_001 --from "redis://:mysecret@10.0.0.10:6379" --to ./resplite.db
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
For one-shot import, authentication is only available when using `--redis-url`; the `--host` / `--port` options do not support a password.
|
|
220
|
+
|
|
221
|
+
### Minimal-downtime migration (SPEC_F)
|
|
222
|
+
|
|
223
|
+
For large datasets (~30 GB), use the Dirty Key Registry flow so the bulk of the migration runs online and only a short cutover is needed.
|
|
224
|
+
|
|
225
|
+
**Enable keyspace notifications in Redis** (required for the dirty-key tracker). Either run at runtime:
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
redis-cli CONFIG SET notify-keyspace-events KEA
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Or add to `redis.conf` and restart Redis:
|
|
232
|
+
|
|
233
|
+
```
|
|
234
|
+
notify-keyspace-events KEA
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
(`K` = keyspace prefix, `E` = keyevent prefix, `A` = all event types — lets the tracker see every key change and expiration.)
|
|
238
|
+
|
|
239
|
+
> **Renamed CONFIG command?** Some Redis deployments rename `CONFIG` for security. Pass `--config-command <name>` to the CLI tools, or the `configCommand` option to the JS API — see below.
|
|
240
|
+
|
|
241
|
+
1. **Preflight** – Check Redis, key count, type distribution, and that keyspace notifications are enabled:
|
|
242
|
+
```bash
|
|
243
|
+
npx resplite-import preflight --from redis://10.0.0.10:6379 --to ./resplite.db
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
2. **Start dirty-key tracker** – Captures keys modified during bulk (requires `notify-keyspace-events` in Redis):
|
|
247
|
+
```bash
|
|
248
|
+
npx resplite-dirty-tracker start --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db
|
|
249
|
+
# If CONFIG was renamed:
|
|
250
|
+
npx resplite-dirty-tracker start --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db --config-command MYCONFIG
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
3. **Bulk import** – SCAN and copy all keys; progress is checkpointed and resumable (resume is default; re-run the same command to continue after a stop):
|
|
254
|
+
```bash
|
|
255
|
+
npx resplite-import bulk --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db \
|
|
256
|
+
--scan-count 1000 --max-rps 2000 --batch-keys 200 --batch-bytes 64MB
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
4. **Monitor** – Check run and dirty-key counts:
|
|
260
|
+
```bash
|
|
261
|
+
npx resplite-import status --run-id run_001 --to ./resplite.db
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
5. **Cutover** – Freeze app writes to Redis, then apply remaining dirty keys:
|
|
265
|
+
```bash
|
|
266
|
+
npx resplite-import apply-dirty --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
6. **Stop tracker and switch** – Stop the tracker and point clients to RespLite:
|
|
270
|
+
```bash
|
|
271
|
+
npx resplite-dirty-tracker stop --run-id run_001 --to ./resplite.db
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
7. **Verify** – Optional sampling check between Redis and destination:
|
|
275
|
+
```bash
|
|
276
|
+
npx resplite-import verify --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db --sample 0.5%
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
Then start RespLite with the migrated DB: `RESPLITE_DB=./resplite.db npm start`.
|
|
280
|
+
|
|
281
|
+
### Programmatic migration API (JavaScript)
|
|
282
|
+
|
|
283
|
+
As an alternative to the CLI, the full migration flow is available as a JavaScript API via `resplite/migration`. Useful for embedding the migration inside your own scripts or automation pipelines.
|
|
284
|
+
|
|
285
|
+
```javascript
|
|
286
|
+
import { createMigration } from 'resplite/migration';
|
|
287
|
+
|
|
288
|
+
const m = createMigration({
|
|
289
|
+
from: 'redis://127.0.0.1:6379', // source Redis URL (default)
|
|
290
|
+
to: './resplite.db', // destination SQLite DB path (required)
|
|
291
|
+
runId: 'my-migration-1', // unique run ID (required for bulk/status/applyDirty)
|
|
292
|
+
|
|
293
|
+
// optional — same defaults as the CLI:
|
|
294
|
+
scanCount: 1000,
|
|
295
|
+
batchKeys: 200,
|
|
296
|
+
batchBytes: 64 * 1024 * 1024, // 64 MB
|
|
297
|
+
maxRps: 0, // 0 = unlimited
|
|
298
|
+
pragmaTemplate: 'default',
|
|
299
|
+
|
|
300
|
+
// If your Redis deployment renamed CONFIG for security:
|
|
301
|
+
// configCommand: 'MYCONFIG',
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
// Step 0 — Preflight: inspect Redis before starting
|
|
305
|
+
const info = await m.preflight();
|
|
306
|
+
console.log('keys (estimate):', info.keyCountEstimate);
|
|
307
|
+
console.log('type distribution:', info.typeDistribution);
|
|
308
|
+
console.log('notify-keyspace-events:', info.notifyKeyspaceEvents);
|
|
309
|
+
console.log('CONFIG available:', info.configCommandAvailable); // false if renamed
|
|
310
|
+
console.log('recommended params:', info.recommended);
|
|
311
|
+
|
|
312
|
+
// Step 0b — Enable keyspace notifications (required for dirty-key tracking)
|
|
313
|
+
// Reads the current value and merges the new flags — existing flags are preserved.
|
|
314
|
+
const ks = await m.enableKeyspaceNotifications();
|
|
315
|
+
// → { ok: true, previous: '', applied: 'KEA' }
|
|
316
|
+
// If CONFIG is renamed and configCommand was not set, ok=false and error explains how to fix it.
|
|
317
|
+
|
|
318
|
+
// Step 1 — Bulk import (checkpointed, resumable). Same script to start or continue.
|
|
319
|
+
await m.bulk({
|
|
320
|
+
onProgress: (r) => console.log(
|
|
321
|
+
`scanned=${r.scanned_keys} migrated=${r.migrated_keys} errors=${r.error_keys}`
|
|
322
|
+
),
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
// Check status at any point (synchronous, no Redis needed)
|
|
326
|
+
const { run, dirty } = m.status();
|
|
327
|
+
console.log('bulk status:', run.status, '— dirty counts:', dirty);
|
|
328
|
+
|
|
329
|
+
// Step 2 — Apply dirty keys that changed in Redis during bulk
|
|
330
|
+
await m.applyDirty();
|
|
331
|
+
|
|
332
|
+
// Step 3 — Verify a sample of keys match between Redis and the destination
|
|
333
|
+
const result = await m.verify({ samplePct: 0.5, maxSample: 10000 });
|
|
334
|
+
console.log(`verified ${result.sampled} keys — mismatches: ${result.mismatches.length}`);
|
|
335
|
+
|
|
336
|
+
// Disconnect Redis when done
|
|
337
|
+
await m.close();
|
|
338
|
+
```
|
|
339
|
+
|
|
340
|
+
**Resume automático (por defecto)**
|
|
341
|
+
`resume` viene en `true` por defecto. Da igual si es la primera vez o una reanudación: el mismo script sirve para empezar y para continuar. La primera ejecución arranca desde cursor 0; si el proceso se corta (Ctrl+C, crash, etc.), al volver a ejecutar el script continúa desde el último checkpoint. No hace falta pasar `resume: false` la primera vez ni cambiar nada para reanudar.
|
|
342
|
+
|
|
343
|
+
The dirty-key tracker (to capture writes during bulk) still runs as a separate process via `npx resplite-dirty-tracker`. The API above handles everything else in a single script.
|
|
344
|
+
|
|
345
|
+
#### Renamed CONFIG command
|
|
346
|
+
|
|
347
|
+
If your Redis instance has the `CONFIG` command renamed (a common hardening practice), pass the new name to `createMigration`:
|
|
348
|
+
|
|
349
|
+
```javascript
|
|
350
|
+
const m = createMigration({
|
|
351
|
+
from: 'redis://10.0.0.10:6379',
|
|
352
|
+
to: './resplite.db',
|
|
353
|
+
runId: 'run_001',
|
|
354
|
+
configCommand: 'MYCONFIG', // the renamed command
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
// preflight will use MYCONFIG GET notify-keyspace-events
|
|
358
|
+
const info = await m.preflight();
|
|
359
|
+
// info.configCommandAvailable → false if the name is wrong
|
|
360
|
+
|
|
361
|
+
// enableKeyspaceNotifications will use MYCONFIG SET notify-keyspace-events KEA
|
|
362
|
+
const result = await m.enableKeyspaceNotifications({ value: 'KEA' });
|
|
363
|
+
```
|
|
364
|
+
|
|
365
|
+
The same flag is available in the CLI:
|
|
366
|
+
|
|
367
|
+
```bash
|
|
368
|
+
npx resplite-dirty-tracker start --run-id run_001 --to ./resplite.db \
|
|
369
|
+
--from redis://10.0.0.10:6379 --config-command MYCONFIG
|
|
370
|
+
```
|
|
371
|
+
|
|
372
|
+
#### Low-level re-exports
|
|
373
|
+
|
|
374
|
+
If you need more control, the individual functions and registry helpers are also exported:
|
|
375
|
+
|
|
376
|
+
```javascript
|
|
377
|
+
import {
|
|
378
|
+
runPreflight, runBulkImport, runApplyDirty, runVerify,
|
|
379
|
+
getRun, getDirtyCounts, createRun, setRunStatus, logError,
|
|
380
|
+
} from 'resplite/migration';
|
|
381
|
+
```
|
|
382
|
+
|
|
184
383
|
### Strings, TTL, and key operations
|
|
185
384
|
|
|
186
385
|
```javascript
|
|
@@ -383,210 +582,6 @@ RESPLite implements **47 core Redis commands** (~19% of the ~246 commands in Red
|
|
|
383
582
|
|
|
384
583
|
Unsupported commands return: `ERR command not supported yet`.
|
|
385
584
|
|
|
386
|
-
## Migration from Redis
|
|
387
|
-
|
|
388
|
-
RESPLite is a good fit for migrating **non-replicated Redis** instances that have **grown large** (e.g. tens of GB) and where RESPLite’s latency is acceptable. The SPEC_F flow (dirty-key tracker, bulk import, cutover) is designed for that scenario with minimal downtime.
|
|
389
|
-
|
|
390
|
-
Migration supports two modes:
|
|
391
|
-
|
|
392
|
-
### Simple one-shot import (legacy)
|
|
393
|
-
|
|
394
|
-
For small datasets or when downtime is acceptable:
|
|
395
|
-
|
|
396
|
-
```bash
|
|
397
|
-
# Default: redis://127.0.0.1:6379 → ./data.db
|
|
398
|
-
npm run import-from-redis -- --db ./migrated.db
|
|
399
|
-
|
|
400
|
-
# Custom Redis URL
|
|
401
|
-
npm run import-from-redis -- --db ./migrated.db --redis-url redis://127.0.0.1:6379
|
|
402
|
-
|
|
403
|
-
# Or host/port
|
|
404
|
-
npm run import-from-redis -- --db ./migrated.db --host 127.0.0.1 --port 6379
|
|
405
|
-
|
|
406
|
-
# Optional: PRAGMA template for the target DB
|
|
407
|
-
npm run import-from-redis -- --db ./migrated.db --pragma-template performance
|
|
408
|
-
```
|
|
409
|
-
|
|
410
|
-
### Redis with authentication
|
|
411
|
-
|
|
412
|
-
Migration supports Redis instances protected by a password. Use a Redis URL that includes the password (or username and password for Redis 6+ ACL):
|
|
413
|
-
|
|
414
|
-
- **Password only:** `redis://:PASSWORD@host:port`
|
|
415
|
-
- **Username and password:** `redis://username:PASSWORD@host:port`
|
|
416
|
-
|
|
417
|
-
Examples:
|
|
418
|
-
|
|
419
|
-
```bash
|
|
420
|
-
# One-shot import from authenticated Redis
|
|
421
|
-
npm run import-from-redis -- --db ./migrated.db --redis-url "redis://:mysecret@127.0.0.1:6379"
|
|
422
|
-
|
|
423
|
-
# SPEC_F flow: use --from with the full URL (or set RESPLITE_IMPORT_FROM)
|
|
424
|
-
npx resplite-import preflight --from "redis://:mysecret@10.0.0.10:6379" --to ./resplite.db
|
|
425
|
-
npx resplite-dirty-tracker start --run-id run_001 --from "redis://:mysecret@10.0.0.10:6379" --to ./resplite.db
|
|
426
|
-
```
|
|
427
|
-
|
|
428
|
-
For one-shot import, authentication is only available when using `--redis-url`; the `--host` / `--port` options do not support a password.
|
|
429
|
-
|
|
430
|
-
### Minimal-downtime migration (SPEC_F)
|
|
431
|
-
|
|
432
|
-
For large datasets (~30 GB), use the Dirty Key Registry flow so the bulk of the migration runs online and only a short cutover is needed.
|
|
433
|
-
|
|
434
|
-
**Enable keyspace notifications in Redis** (required for the dirty-key tracker). Either run at runtime:
|
|
435
|
-
|
|
436
|
-
```bash
|
|
437
|
-
redis-cli CONFIG SET notify-keyspace-events KEA
|
|
438
|
-
```
|
|
439
|
-
|
|
440
|
-
Or add to `redis.conf` and restart Redis:
|
|
441
|
-
|
|
442
|
-
```
|
|
443
|
-
notify-keyspace-events KEA
|
|
444
|
-
```
|
|
445
|
-
|
|
446
|
-
(`K` = keyspace prefix, `E` = keyevent prefix, `A` = all event types — lets the tracker see every key change and expiration.)
|
|
447
|
-
|
|
448
|
-
> **Renamed CONFIG command?** Some Redis deployments rename `CONFIG` for security. Pass `--config-command <name>` to the CLI tools, or the `configCommand` option to the JS API — see below.
|
|
449
|
-
|
|
450
|
-
1. **Preflight** – Check Redis, key count, type distribution, and that keyspace notifications are enabled:
|
|
451
|
-
```bash
|
|
452
|
-
npx resplite-import preflight --from redis://10.0.0.10:6379 --to ./resplite.db
|
|
453
|
-
```
|
|
454
|
-
|
|
455
|
-
2. **Start dirty-key tracker** – Captures keys modified during bulk (requires `notify-keyspace-events` in Redis):
|
|
456
|
-
```bash
|
|
457
|
-
npx resplite-dirty-tracker start --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db
|
|
458
|
-
# If CONFIG was renamed:
|
|
459
|
-
npx resplite-dirty-tracker start --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db --config-command MYCONFIG
|
|
460
|
-
```
|
|
461
|
-
|
|
462
|
-
3. **Bulk import** – SCAN and copy all keys; progress is checkpointed and resumable:
|
|
463
|
-
```bash
|
|
464
|
-
npx resplite-import bulk --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db \
|
|
465
|
-
--scan-count 1000 --max-rps 2000 --batch-keys 200 --batch-bytes 64MB --resume
|
|
466
|
-
```
|
|
467
|
-
|
|
468
|
-
4. **Monitor** – Check run and dirty-key counts:
|
|
469
|
-
```bash
|
|
470
|
-
npx resplite-import status --run-id run_001 --to ./resplite.db
|
|
471
|
-
```
|
|
472
|
-
|
|
473
|
-
5. **Cutover** – Freeze app writes to Redis, then apply remaining dirty keys:
|
|
474
|
-
```bash
|
|
475
|
-
npx resplite-import apply-dirty --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db
|
|
476
|
-
```
|
|
477
|
-
|
|
478
|
-
6. **Stop tracker and switch** – Stop the tracker and point clients to RespLite:
|
|
479
|
-
```bash
|
|
480
|
-
npx resplite-dirty-tracker stop --run-id run_001 --to ./resplite.db
|
|
481
|
-
```
|
|
482
|
-
|
|
483
|
-
7. **Verify** – Optional sampling check between Redis and destination:
|
|
484
|
-
```bash
|
|
485
|
-
npx resplite-import verify --run-id run_001 --from redis://10.0.0.10:6379 --to ./resplite.db --sample 0.5%
|
|
486
|
-
```
|
|
487
|
-
|
|
488
|
-
Then start RespLite with the migrated DB: `RESPLITE_DB=./resplite.db npm start`.
|
|
489
|
-
|
|
490
|
-
### Programmatic migration API
|
|
491
|
-
|
|
492
|
-
As an alternative to the CLI, the full migration flow is available as a JavaScript API via `resplite/migration`. Useful for embedding the migration inside your own scripts or automation pipelines.
|
|
493
|
-
|
|
494
|
-
```javascript
|
|
495
|
-
import { createMigration } from 'resplite/migration';
|
|
496
|
-
|
|
497
|
-
const m = createMigration({
|
|
498
|
-
from: 'redis://127.0.0.1:6379', // source Redis URL (default)
|
|
499
|
-
to: './resplite.db', // destination SQLite DB path (required)
|
|
500
|
-
runId: 'my-migration-1', // unique run ID (required for bulk/status/applyDirty)
|
|
501
|
-
|
|
502
|
-
// optional — same defaults as the CLI:
|
|
503
|
-
scanCount: 1000,
|
|
504
|
-
batchKeys: 200,
|
|
505
|
-
batchBytes: 64 * 1024 * 1024, // 64 MB
|
|
506
|
-
maxRps: 0, // 0 = unlimited
|
|
507
|
-
pragmaTemplate: 'default',
|
|
508
|
-
|
|
509
|
-
// If your Redis deployment renamed CONFIG for security:
|
|
510
|
-
// configCommand: 'MYCONFIG',
|
|
511
|
-
});
|
|
512
|
-
|
|
513
|
-
// Step 0 — Preflight: inspect Redis before starting
|
|
514
|
-
const info = await m.preflight();
|
|
515
|
-
console.log('keys (estimate):', info.keyCountEstimate);
|
|
516
|
-
console.log('type distribution:', info.typeDistribution);
|
|
517
|
-
console.log('notify-keyspace-events:', info.notifyKeyspaceEvents);
|
|
518
|
-
console.log('CONFIG available:', info.configCommandAvailable); // false if renamed
|
|
519
|
-
console.log('recommended params:', info.recommended);
|
|
520
|
-
|
|
521
|
-
// Step 0b — Enable keyspace notifications (required for dirty-key tracking)
|
|
522
|
-
// Reads the current value and merges the new flags — existing flags are preserved.
|
|
523
|
-
const ks = await m.enableKeyspaceNotifications();
|
|
524
|
-
// → { ok: true, previous: '', applied: 'KEA' }
|
|
525
|
-
// If CONFIG is renamed and configCommand was not set, ok=false and error explains how to fix it.
|
|
526
|
-
|
|
527
|
-
// Step 1 — Bulk import (checkpointed, resumable)
|
|
528
|
-
await m.bulk({
|
|
529
|
-
resume: false, // true to resume a previous run
|
|
530
|
-
onProgress: (r) => console.log(
|
|
531
|
-
`scanned=${r.scanned_keys} migrated=${r.migrated_keys} errors=${r.error_keys}`
|
|
532
|
-
),
|
|
533
|
-
});
|
|
534
|
-
|
|
535
|
-
// Check status at any point (synchronous, no Redis needed)
|
|
536
|
-
const { run, dirty } = m.status();
|
|
537
|
-
console.log('bulk status:', run.status, '— dirty counts:', dirty);
|
|
538
|
-
|
|
539
|
-
// Step 2 — Apply dirty keys that changed in Redis during bulk
|
|
540
|
-
await m.applyDirty();
|
|
541
|
-
|
|
542
|
-
// Step 3 — Verify a sample of keys match between Redis and the destination
|
|
543
|
-
const result = await m.verify({ samplePct: 0.5, maxSample: 10000 });
|
|
544
|
-
console.log(`verified ${result.sampled} keys — mismatches: ${result.mismatches.length}`);
|
|
545
|
-
|
|
546
|
-
// Disconnect Redis when done
|
|
547
|
-
await m.close();
|
|
548
|
-
```
|
|
549
|
-
|
|
550
|
-
The dirty-key tracker (to capture writes during bulk) still runs as a separate process via `npx resplite-dirty-tracker`. The API above handles everything else in a single script.
|
|
551
|
-
|
|
552
|
-
#### Renamed CONFIG command
|
|
553
|
-
|
|
554
|
-
If your Redis instance has the `CONFIG` command renamed (a common hardening practice), pass the new name to `createMigration`:
|
|
555
|
-
|
|
556
|
-
```javascript
|
|
557
|
-
const m = createMigration({
|
|
558
|
-
from: 'redis://10.0.0.10:6379',
|
|
559
|
-
to: './resplite.db',
|
|
560
|
-
runId: 'run_001',
|
|
561
|
-
configCommand: 'MYCONFIG', // the renamed command
|
|
562
|
-
});
|
|
563
|
-
|
|
564
|
-
// preflight will use MYCONFIG GET notify-keyspace-events
|
|
565
|
-
const info = await m.preflight();
|
|
566
|
-
// info.configCommandAvailable → false if the name is wrong
|
|
567
|
-
|
|
568
|
-
// enableKeyspaceNotifications will use MYCONFIG SET notify-keyspace-events KEA
|
|
569
|
-
const result = await m.enableKeyspaceNotifications({ value: 'KEA' });
|
|
570
|
-
```
|
|
571
|
-
|
|
572
|
-
The same flag is available in the CLI:
|
|
573
|
-
|
|
574
|
-
```bash
|
|
575
|
-
npx resplite-dirty-tracker start --run-id run_001 --to ./resplite.db \
|
|
576
|
-
--from redis://10.0.0.10:6379 --config-command MYCONFIG
|
|
577
|
-
```
|
|
578
|
-
|
|
579
|
-
#### Low-level re-exports
|
|
580
|
-
|
|
581
|
-
If you need more control, the individual functions and registry helpers are also exported:
|
|
582
|
-
|
|
583
|
-
```javascript
|
|
584
|
-
import {
|
|
585
|
-
runPreflight, runBulkImport, runApplyDirty, runVerify,
|
|
586
|
-
getRun, getDirtyCounts, createRun, setRunStatus, logError,
|
|
587
|
-
} from 'resplite/migration';
|
|
588
|
-
```
|
|
589
|
-
|
|
590
585
|
## Scripts
|
|
591
586
|
|
|
592
587
|
| Script | Description |
|
package/package.json
CHANGED
|
@@ -47,6 +47,8 @@ function parseArgs(argv = process.argv.slice(2)) {
|
|
|
47
47
|
}
|
|
48
48
|
} else if (arg === '--resume') {
|
|
49
49
|
args.resume = true;
|
|
50
|
+
} else if (arg === '--no-resume') {
|
|
51
|
+
args.resume = false;
|
|
50
52
|
} else if (arg === '--pragma-template' && argv[i + 1]) {
|
|
51
53
|
args.pragmaTemplate = argv[++i];
|
|
52
54
|
} else if (arg === '--sample' && argv[i + 1]) {
|
|
@@ -120,7 +122,7 @@ async function cmdBulk(args) {
|
|
|
120
122
|
max_rps: args.maxRps || 0,
|
|
121
123
|
batch_keys: args.batchKeys || 200,
|
|
122
124
|
batch_bytes: args.batchBytes || 64 * 1024 * 1024,
|
|
123
|
-
resume:
|
|
125
|
+
resume: args.resume !== false, // default true: start from 0 or continue from checkpoint
|
|
124
126
|
onProgress: (r) => {
|
|
125
127
|
console.log(` scanned=${r.scanned_keys} migrated=${r.migrated_keys} skipped=${r.skipped_keys} errors=${r.error_keys} cursor=${r.scan_cursor}`);
|
|
126
128
|
},
|
|
@@ -210,7 +212,7 @@ async function main() {
|
|
|
210
212
|
console.error(' --max-rps N (bulk, apply-dirty)');
|
|
211
213
|
console.error(' --batch-keys N (default 200)');
|
|
212
214
|
console.error(' --batch-bytes N[MB|KB|GB] (default 64MB)');
|
|
213
|
-
console.error(' --resume
|
|
215
|
+
console.error(' --resume / --no-resume (bulk: default resume=on; use --no-resume to always start from 0)');
|
|
214
216
|
console.error(' --sample 0.5% (verify, default 0.5%)');
|
|
215
217
|
process.exit(1);
|
|
216
218
|
}
|
package/src/embed.js
CHANGED
|
@@ -35,6 +35,7 @@ export { handleConnection, createEngine, openDb };
|
|
|
35
35
|
* @param {number} [options.port=0] Port to listen on (0 = OS-assigned).
|
|
36
36
|
* @param {string} [options.pragmaTemplate='default'] PRAGMA preset (default|performance|safety|minimal|none).
|
|
37
37
|
* @param {RESPliteHooks} [options.hooks] Optional event hooks for observability (onUnknownCommand, onCommandError, onSocketError).
|
|
38
|
+
* @param {boolean} [options.gracefulShutdown=true] If true, register SIGTERM/SIGINT to call close(). Set false if you handle shutdown yourself to avoid double handlers.
|
|
38
39
|
* @returns {Promise<{ port: number, host: string, close: () => Promise<void> }>}
|
|
39
40
|
*/
|
|
40
41
|
export async function createRESPlite({
|
|
@@ -43,6 +44,7 @@ export async function createRESPlite({
|
|
|
43
44
|
port = 0,
|
|
44
45
|
pragmaTemplate = 'default',
|
|
45
46
|
hooks = {},
|
|
47
|
+
gracefulShutdown = true,
|
|
46
48
|
} = {}) {
|
|
47
49
|
const db = openDb(dbPath, { pragmaTemplate });
|
|
48
50
|
const engine = createEngine({ db });
|
|
@@ -71,6 +73,14 @@ export async function createRESPlite({
|
|
|
71
73
|
return closePromise;
|
|
72
74
|
};
|
|
73
75
|
|
|
76
|
+
if (gracefulShutdown) {
|
|
77
|
+
const onSignal = () => {
|
|
78
|
+
close().then(() => process.exit(0));
|
|
79
|
+
};
|
|
80
|
+
process.on('SIGTERM', onSignal);
|
|
81
|
+
process.on('SIGINT', onSignal);
|
|
82
|
+
}
|
|
83
|
+
|
|
74
84
|
return {
|
|
75
85
|
port: server.address().port,
|
|
76
86
|
host,
|
package/src/index.js
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* RESPLite entry point. Start TCP server with SQLite backend.
|
|
3
|
+
*
|
|
4
|
+
* Can be run as CLI (node src/index.js) or used programmatically:
|
|
5
|
+
* import { startServer } from './src/index.js';
|
|
6
|
+
* startServer({ port: 6380, gracefulShutdown: false });
|
|
3
7
|
*/
|
|
4
8
|
|
|
5
9
|
import { createServer } from './server/tcp-server.js';
|
|
@@ -15,23 +19,57 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
|
15
19
|
const DEFAULT_DB_PATH = path.join(process.cwd(), 'data.db');
|
|
16
20
|
const DEFAULT_PORT = 6379;
|
|
17
21
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
22
|
+
/**
|
|
23
|
+
* @param {object} [options]
|
|
24
|
+
* @param {number} [options.port]
|
|
25
|
+
* @param {string} [options.dbPath]
|
|
26
|
+
* @param {string} [options.pragmaTemplate]
|
|
27
|
+
* @param {boolean} [options.gracefulShutdown=true] If true, register SIGTERM/SIGINT to close server and DB. Set false if you handle shutdown yourself.
|
|
28
|
+
*/
|
|
29
|
+
export function startServer(options = {}) {
|
|
30
|
+
const dbPath = options.dbPath ?? process.env.RESPLITE_DB ?? DEFAULT_DB_PATH;
|
|
31
|
+
const port = options.port ?? parseInt(process.env.RESPLITE_PORT || String(DEFAULT_PORT), 10);
|
|
32
|
+
const pragmaTemplate = options.pragmaTemplate ?? process.env.RESPLITE_PRAGMA_TEMPLATE ?? 'default';
|
|
33
|
+
const gracefulShutdown = options.gracefulShutdown !== false;
|
|
34
|
+
|
|
35
|
+
const db = openDb(dbPath, { pragmaTemplate });
|
|
36
|
+
const cache = createCache({ enabled: true });
|
|
37
|
+
const engine = createEngine({ db, cache });
|
|
38
|
+
const sweeper = createExpirationSweeper({
|
|
39
|
+
db,
|
|
40
|
+
clock: () => Date.now(),
|
|
41
|
+
sweepIntervalMs: 1000,
|
|
42
|
+
maxKeysPerSweep: 500,
|
|
43
|
+
});
|
|
44
|
+
sweeper.start();
|
|
45
|
+
|
|
46
|
+
const connections = new Set();
|
|
47
|
+
const server = createServer({ engine, port, connections });
|
|
48
|
+
|
|
49
|
+
if (gracefulShutdown) {
|
|
50
|
+
let shuttingDown = false;
|
|
51
|
+
function shutdown() {
|
|
52
|
+
if (shuttingDown) return;
|
|
53
|
+
shuttingDown = true;
|
|
54
|
+
sweeper.stop();
|
|
55
|
+
for (const socket of connections) socket.destroy();
|
|
56
|
+
connections.clear();
|
|
57
|
+
server.close(() => {
|
|
58
|
+
db.close();
|
|
59
|
+
process.exit(0);
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
process.on('SIGTERM', shutdown);
|
|
63
|
+
process.on('SIGINT', shutdown);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
server.listen(port, () => {
|
|
67
|
+
console.log(`RESPLite listening on port ${port}, db: ${dbPath}`);
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const isCli = process.argv[1] && path.resolve(process.argv[1]) === path.resolve(fileURLToPath(import.meta.url));
|
|
72
|
+
if (isCli) {
|
|
73
|
+
const noGraceful = process.argv.includes('--no-graceful-shutdown');
|
|
74
|
+
startServer({ gracefulShutdown: !noGraceful });
|
|
75
|
+
}
|
package/src/migration/bulk.js
CHANGED
|
@@ -48,7 +48,7 @@ function sleep(ms) {
|
|
|
48
48
|
* @param {number} [options.batch_keys=200]
|
|
49
49
|
* @param {number} [options.batch_bytes=64*1024*1024] - 64MB
|
|
50
50
|
* @param {number} [options.checkpoint_interval_sec=30]
|
|
51
|
-
* @param {boolean} [options.resume=false
|
|
51
|
+
* @param {boolean} [options.resume=true] - true: start from 0 or continue from checkpoint; false: always start from 0
|
|
52
52
|
* @param {function(run): void} [options.onProgress] - called after checkpoint with run row
|
|
53
53
|
*/
|
|
54
54
|
export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
|
|
@@ -60,7 +60,7 @@ export async function runBulkImport(redisClient, dbPath, runId, options = {}) {
|
|
|
60
60
|
batch_keys = 200,
|
|
61
61
|
batch_bytes = 64 * 1024 * 1024,
|
|
62
62
|
checkpoint_interval_sec = 30,
|
|
63
|
-
resume =
|
|
63
|
+
resume = true,
|
|
64
64
|
onProgress,
|
|
65
65
|
} = options;
|
|
66
66
|
|
package/src/migration/index.js
CHANGED
|
@@ -113,11 +113,11 @@ export function createMigration({
|
|
|
113
113
|
|
|
114
114
|
/**
|
|
115
115
|
* Step 1 — Bulk import: SCAN all keys from Redis into the destination DB.
|
|
116
|
-
*
|
|
116
|
+
* Resume is on by default: first run starts from 0, later runs continue from checkpoint.
|
|
117
117
|
*
|
|
118
|
-
* @param {{ resume?: boolean, onProgress?: (run: object) => void }} [opts]
|
|
118
|
+
* @param {{ resume?: boolean, onProgress?: (run: object) => void }} [opts] - resume (default true): start or continue automatically
|
|
119
119
|
*/
|
|
120
|
-
async bulk({ resume =
|
|
120
|
+
async bulk({ resume = true, onProgress } = {}) {
|
|
121
121
|
const id = requireRunId();
|
|
122
122
|
const client = await getClient();
|
|
123
123
|
return runBulkImport(client, to, id, {
|
package/src/server/tcp-server.js
CHANGED
|
@@ -10,10 +10,15 @@ import { handleConnection } from './connection.js';
|
|
|
10
10
|
* @param {object} options.engine
|
|
11
11
|
* @param {number} [options.port=6379]
|
|
12
12
|
* @param {string} [options.host='0.0.0.0']
|
|
13
|
+
* @param {Set<import('node:net').Socket>} [options.connections] If provided, each accepted socket is added here (for graceful shutdown).
|
|
13
14
|
* @returns {import('node:net').Server}
|
|
14
15
|
*/
|
|
15
|
-
export function createServer({ engine, port = 6379, host = '0.0.0.0' }) {
|
|
16
|
+
export function createServer({ engine, port = 6379, host = '0.0.0.0', connections = null }) {
|
|
16
17
|
const server = net.createServer((socket) => {
|
|
18
|
+
if (connections) {
|
|
19
|
+
connections.add(socket);
|
|
20
|
+
socket.once('close', () => connections.delete(socket));
|
|
21
|
+
}
|
|
17
22
|
handleConnection(socket, engine);
|
|
18
23
|
});
|
|
19
24
|
return server;
|