resplite 1.2.2 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -98
- package/package.json +1 -1
- package/spec/SPEC_F.md +94 -0
- package/src/cli/resplite-import.js +74 -17
- package/src/migration/bulk.js +69 -36
- package/src/migration/index.js +32 -1
- package/src/migration/migrate-search.js +457 -0
- package/test/unit/migrate-search.test.js +497 -0
package/README.md
CHANGED
|
@@ -176,11 +176,109 @@ const srv = await createRESPlite({
|
|
|
176
176
|
|
|
177
177
|
## Migration from Redis
|
|
178
178
|
|
|
179
|
-
RESPLite is a good fit for migrating **non-replicated Redis** instances that have **grown large** (e.g. tens of GB) and where RESPLite's latency is acceptable. The
|
|
179
|
+
RESPLite is a good fit for migrating **non-replicated Redis** instances that have **grown large** (e.g. tens of GB) and where RESPLite's latency is acceptable. The flow (dirty-key tracker, bulk import, cutover) is designed for that scenario with minimal downtime.
|
|
180
180
|
|
|
181
181
|
Migration supports two modes:
|
|
182
182
|
|
|
183
|
-
###
|
|
183
|
+
### Programmatic migration API (JavaScript)
|
|
184
|
+
|
|
185
|
+
As an alternative to the CLI, the full migration flow is available as a JavaScript API via `resplite/migration`. Useful for embedding the migration inside your own scripts or automation pipelines.
|
|
186
|
+
|
|
187
|
+
```javascript
|
|
188
|
+
import { createMigration } from 'resplite/migration';
|
|
189
|
+
|
|
190
|
+
const m = createMigration({
|
|
191
|
+
from: 'redis://127.0.0.1:6379', // source Redis URL (default)
|
|
192
|
+
to: './resplite.db', // destination SQLite DB path (required)
|
|
193
|
+
runId: 'my-migration-1', // unique run ID (required for bulk/status/applyDirty)
|
|
194
|
+
|
|
195
|
+
// optional — same defaults as the CLI:
|
|
196
|
+
scanCount: 1000,
|
|
197
|
+
batchKeys: 200,
|
|
198
|
+
batchBytes: 64 * 1024 * 1024, // 64 MB
|
|
199
|
+
maxRps: 0, // 0 = unlimited
|
|
200
|
+
pragmaTemplate: 'default',
|
|
201
|
+
|
|
202
|
+
// If your Redis deployment renamed CONFIG for security:
|
|
203
|
+
// configCommand: 'MYCONFIG',
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
// Step 0 — Preflight: inspect Redis before starting
|
|
207
|
+
const info = await m.preflight();
|
|
208
|
+
console.log('keys (estimate):', info.keyCountEstimate);
|
|
209
|
+
console.log('type distribution:', info.typeDistribution);
|
|
210
|
+
console.log('notify-keyspace-events:', info.notifyKeyspaceEvents);
|
|
211
|
+
console.log('CONFIG available:', info.configCommandAvailable); // false if renamed
|
|
212
|
+
console.log('recommended params:', info.recommended);
|
|
213
|
+
|
|
214
|
+
// Step 0b — Enable keyspace notifications (required for dirty-key tracking)
|
|
215
|
+
// Reads the current value and merges the new flags — existing flags are preserved.
|
|
216
|
+
const ks = await m.enableKeyspaceNotifications();
|
|
217
|
+
// → { ok: true, previous: '', applied: 'KEA' }
|
|
218
|
+
// If CONFIG is renamed and configCommand was not set, ok=false and error explains how to fix it.
|
|
219
|
+
|
|
220
|
+
// Step 1 — Bulk import (checkpointed, resumable). Same script to start or continue.
|
|
221
|
+
// Use keyCountEstimate from preflight to show progress % (estimate; actual count may change).
|
|
222
|
+
const total = info.keyCountEstimate || 1;
|
|
223
|
+
await m.bulk({
|
|
224
|
+
onProgress: (r) => {
|
|
225
|
+
const pct = total ? ((r.scanned_keys / total) * 100).toFixed(1) : '—';
|
|
226
|
+
console.log(
|
|
227
|
+
`scanned=${r.scanned_keys} migrated=${r.migrated_keys} errors=${r.error_keys} progress=${pct}%`
|
|
228
|
+
);
|
|
229
|
+
},
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
// Check status at any point (synchronous, no Redis needed)
|
|
233
|
+
const { run, dirty } = m.status();
|
|
234
|
+
console.log('bulk status:', run.status, '— dirty counts:', dirty);
|
|
235
|
+
|
|
236
|
+
// Step 2 — Apply dirty keys that changed in Redis during bulk
|
|
237
|
+
await m.applyDirty();
|
|
238
|
+
|
|
239
|
+
// Step 3 — Verify a sample of keys match between Redis and the destination
|
|
240
|
+
const result = await m.verify({ samplePct: 0.5, maxSample: 10000 });
|
|
241
|
+
console.log(`verified ${result.sampled} keys — mismatches: ${result.mismatches.length}`);
|
|
242
|
+
|
|
243
|
+
// Disconnect Redis when done
|
|
244
|
+
await m.close();
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
**Automatic resume (default)**
|
|
248
|
+
`resume` defaults to `true`. It doesn't matter whether it's the first run or a resume: the same script works for both starting and continuing. The first run starts from cursor 0; if the process is interrupted (Ctrl+C, crash, etc.), running the script again continues from the last checkpoint. You don't need to pass `resume: false` on the first run or change anything to resume.
|
|
249
|
+
|
|
250
|
+
**Graceful shutdown**
|
|
251
|
+
On SIGINT (Ctrl+C) or SIGTERM, the bulk importer checkpoints progress, sets the run status to `aborted`, closes the SQLite database cleanly (so WAL is checkpointed and the file is not left open), then exits. You can safely interrupt a long-running bulk and resume later.
|
|
252
|
+
|
|
253
|
+
The dirty-key tracker (to capture writes during bulk) still runs as a separate process via `npx resplite-dirty-tracker`. The API above handles everything else in a single script.
|
|
254
|
+
|
|
255
|
+
#### Renamed CONFIG command
|
|
256
|
+
|
|
257
|
+
If your Redis instance has the `CONFIG` command renamed (a common hardening practice), pass the new name to `createMigration`:
|
|
258
|
+
|
|
259
|
+
```javascript
|
|
260
|
+
const m = createMigration({
|
|
261
|
+
from: 'redis://10.0.0.10:6379',
|
|
262
|
+
to: './resplite.db',
|
|
263
|
+
runId: 'run_001',
|
|
264
|
+
configCommand: 'MYCONFIG', // the renamed command
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
// preflight will use MYCONFIG GET notify-keyspace-events
|
|
268
|
+
const info = await m.preflight();
|
|
269
|
+
// info.configCommandAvailable → false if the name is wrong
|
|
270
|
+
|
|
271
|
+
// enableKeyspaceNotifications will use MYCONFIG SET notify-keyspace-events KEA
|
|
272
|
+
const result = await m.enableKeyspaceNotifications({ value: 'KEA' });
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
The same flag is available in the CLI:
|
|
276
|
+
|
|
277
|
+
```bash
|
|
278
|
+
npx resplite-dirty-tracker start --run-id run_001 --to ./resplite.db \
|
|
279
|
+
--from redis://10.0.0.10:6379 --config-command MYCONFIG
|
|
280
|
+
```
|
|
281
|
+
### Simple one-shot import
|
|
184
282
|
|
|
185
283
|
For small datasets or when downtime is acceptable:
|
|
186
284
|
|
|
@@ -211,14 +309,17 @@ Examples:
|
|
|
211
309
|
# One-shot import from authenticated Redis
|
|
212
310
|
npm run import-from-redis -- --db ./migrated.db --redis-url "redis://:mysecret@127.0.0.1:6379"
|
|
213
311
|
|
|
214
|
-
#
|
|
312
|
+
# flow: use --from with the full URL (or set RESPLITE_IMPORT_FROM)
|
|
215
313
|
npx resplite-import preflight --from "redis://:mysecret@10.0.0.10:6379" --to ./resplite.db
|
|
216
314
|
npx resplite-dirty-tracker start --run-id run_001 --from "redis://:mysecret@10.0.0.10:6379" --to ./resplite.db
|
|
217
315
|
```
|
|
218
316
|
|
|
219
317
|
For one-shot import, authentication is only available when using `--redis-url`; the `--host` / `--port` options do not support a password.
|
|
220
318
|
|
|
221
|
-
|
|
319
|
+
**Search indices (FT.\*)**
|
|
320
|
+
The KV bulk migration imports only the Redis keyspace (strings, hashes, sets, lists, zsets). RediSearch index schemas and documents are migrated separately with the `migrate-search` step — see [Migrating RediSearch indices](#migrating-redisearch-indices) below.
|
|
321
|
+
|
|
322
|
+
### Minimal-downtime migration
|
|
222
323
|
|
|
223
324
|
For large datasets (~30 GB), use the Dirty Key Registry flow so the bulk of the migration runs online and only a short cutover is needed.
|
|
224
325
|
|
|
@@ -278,97 +379,6 @@ notify-keyspace-events KEA
|
|
|
278
379
|
|
|
279
380
|
Then start RespLite with the migrated DB: `RESPLITE_DB=./resplite.db npm start`.
|
|
280
381
|
|
|
281
|
-
### Programmatic migration API (JavaScript)
|
|
282
|
-
|
|
283
|
-
As an alternative to the CLI, the full migration flow is available as a JavaScript API via `resplite/migration`. Useful for embedding the migration inside your own scripts or automation pipelines.
|
|
284
|
-
|
|
285
|
-
```javascript
|
|
286
|
-
import { createMigration } from 'resplite/migration';
|
|
287
|
-
|
|
288
|
-
const m = createMigration({
|
|
289
|
-
from: 'redis://127.0.0.1:6379', // source Redis URL (default)
|
|
290
|
-
to: './resplite.db', // destination SQLite DB path (required)
|
|
291
|
-
runId: 'my-migration-1', // unique run ID (required for bulk/status/applyDirty)
|
|
292
|
-
|
|
293
|
-
// optional — same defaults as the CLI:
|
|
294
|
-
scanCount: 1000,
|
|
295
|
-
batchKeys: 200,
|
|
296
|
-
batchBytes: 64 * 1024 * 1024, // 64 MB
|
|
297
|
-
maxRps: 0, // 0 = unlimited
|
|
298
|
-
pragmaTemplate: 'default',
|
|
299
|
-
|
|
300
|
-
// If your Redis deployment renamed CONFIG for security:
|
|
301
|
-
// configCommand: 'MYCONFIG',
|
|
302
|
-
});
|
|
303
|
-
|
|
304
|
-
// Step 0 — Preflight: inspect Redis before starting
|
|
305
|
-
const info = await m.preflight();
|
|
306
|
-
console.log('keys (estimate):', info.keyCountEstimate);
|
|
307
|
-
console.log('type distribution:', info.typeDistribution);
|
|
308
|
-
console.log('notify-keyspace-events:', info.notifyKeyspaceEvents);
|
|
309
|
-
console.log('CONFIG available:', info.configCommandAvailable); // false if renamed
|
|
310
|
-
console.log('recommended params:', info.recommended);
|
|
311
|
-
|
|
312
|
-
// Step 0b — Enable keyspace notifications (required for dirty-key tracking)
|
|
313
|
-
// Reads the current value and merges the new flags — existing flags are preserved.
|
|
314
|
-
const ks = await m.enableKeyspaceNotifications();
|
|
315
|
-
// → { ok: true, previous: '', applied: 'KEA' }
|
|
316
|
-
// If CONFIG is renamed and configCommand was not set, ok=false and error explains how to fix it.
|
|
317
|
-
|
|
318
|
-
// Step 1 — Bulk import (checkpointed, resumable). Same script to start or continue.
|
|
319
|
-
await m.bulk({
|
|
320
|
-
onProgress: (r) => console.log(
|
|
321
|
-
`scanned=${r.scanned_keys} migrated=${r.migrated_keys} errors=${r.error_keys}`
|
|
322
|
-
),
|
|
323
|
-
});
|
|
324
|
-
|
|
325
|
-
// Check status at any point (synchronous, no Redis needed)
|
|
326
|
-
const { run, dirty } = m.status();
|
|
327
|
-
console.log('bulk status:', run.status, '— dirty counts:', dirty);
|
|
328
|
-
|
|
329
|
-
// Step 2 — Apply dirty keys that changed in Redis during bulk
|
|
330
|
-
await m.applyDirty();
|
|
331
|
-
|
|
332
|
-
// Step 3 — Verify a sample of keys match between Redis and the destination
|
|
333
|
-
const result = await m.verify({ samplePct: 0.5, maxSample: 10000 });
|
|
334
|
-
console.log(`verified ${result.sampled} keys — mismatches: ${result.mismatches.length}`);
|
|
335
|
-
|
|
336
|
-
// Disconnect Redis when done
|
|
337
|
-
await m.close();
|
|
338
|
-
```
|
|
339
|
-
|
|
340
|
-
**Resume automático (por defecto)**
|
|
341
|
-
`resume` viene en `true` por defecto. Da igual si es la primera vez o una reanudación: el mismo script sirve para empezar y para continuar. La primera ejecución arranca desde cursor 0; si el proceso se corta (Ctrl+C, crash, etc.), al volver a ejecutar el script continúa desde el último checkpoint. No hace falta pasar `resume: false` la primera vez ni cambiar nada para reanudar.
|
|
342
|
-
|
|
343
|
-
The dirty-key tracker (to capture writes during bulk) still runs as a separate process via `npx resplite-dirty-tracker`. The API above handles everything else in a single script.
|
|
344
|
-
|
|
345
|
-
#### Renamed CONFIG command
|
|
346
|
-
|
|
347
|
-
If your Redis instance has the `CONFIG` command renamed (a common hardening practice), pass the new name to `createMigration`:
|
|
348
|
-
|
|
349
|
-
```javascript
|
|
350
|
-
const m = createMigration({
|
|
351
|
-
from: 'redis://10.0.0.10:6379',
|
|
352
|
-
to: './resplite.db',
|
|
353
|
-
runId: 'run_001',
|
|
354
|
-
configCommand: 'MYCONFIG', // the renamed command
|
|
355
|
-
});
|
|
356
|
-
|
|
357
|
-
// preflight will use MYCONFIG GET notify-keyspace-events
|
|
358
|
-
const info = await m.preflight();
|
|
359
|
-
// info.configCommandAvailable → false if the name is wrong
|
|
360
|
-
|
|
361
|
-
// enableKeyspaceNotifications will use MYCONFIG SET notify-keyspace-events KEA
|
|
362
|
-
const result = await m.enableKeyspaceNotifications({ value: 'KEA' });
|
|
363
|
-
```
|
|
364
|
-
|
|
365
|
-
The same flag is available in the CLI:
|
|
366
|
-
|
|
367
|
-
```bash
|
|
368
|
-
npx resplite-dirty-tracker start --run-id run_001 --to ./resplite.db \
|
|
369
|
-
--from redis://10.0.0.10:6379 --config-command MYCONFIG
|
|
370
|
-
```
|
|
371
|
-
|
|
372
382
|
#### Low-level re-exports
|
|
373
383
|
|
|
374
384
|
If you need more control, the individual functions and registry helpers are also exported:
|
|
@@ -551,9 +561,66 @@ await c2.quit();
|
|
|
551
561
|
await srv2.close();
|
|
552
562
|
```
|
|
553
563
|
|
|
554
|
-
|
|
564
|
+
### Migrating RediSearch indices
|
|
565
|
+
|
|
566
|
+
If your Redis source uses **RediSearch** (Redis Stack or the `redis/search` module), run `migrate-search` after (or during) the KV bulk import. It reads index schemas with `FT.INFO`, creates them in RespLite, and imports documents by scanning the matching hash keys.
|
|
555
567
|
|
|
556
|
-
|
|
568
|
+
**CLI:**
|
|
569
|
+
|
|
570
|
+
```bash
|
|
571
|
+
# Migrate all indices
|
|
572
|
+
npx resplite-import migrate-search \
|
|
573
|
+
--from redis://10.0.0.10:6379 \
|
|
574
|
+
--to ./resplite.db
|
|
575
|
+
|
|
576
|
+
# Migrate specific indices only
|
|
577
|
+
npx resplite-import migrate-search \
|
|
578
|
+
--from redis://10.0.0.10:6379 \
|
|
579
|
+
--to ./resplite.db \
|
|
580
|
+
--index products \
|
|
581
|
+
--index articles
|
|
582
|
+
|
|
583
|
+
# Options
|
|
584
|
+
# --scan-count N SCAN COUNT hint (default 500)
|
|
585
|
+
# --max-rps N throttle Redis reads
|
|
586
|
+
# --batch-docs N docs per SQLite transaction (default 200)
|
|
587
|
+
# --max-suggestions N cap for suggestion import (default 10000)
|
|
588
|
+
# --no-skip overwrite if the index already exists in RespLite
|
|
589
|
+
# --no-suggestions skip suggestion import
|
|
590
|
+
```
|
|
591
|
+
|
|
592
|
+
**Programmatic API:**
|
|
593
|
+
|
|
594
|
+
```javascript
|
|
595
|
+
const m = createMigration({ from, to, runId });
|
|
596
|
+
|
|
597
|
+
const result = await m.migrateSearch({
|
|
598
|
+
onlyIndices: ['products', 'articles'], // omit to migrate all
|
|
599
|
+
batchDocs: 200,
|
|
600
|
+
maxSuggestions: 10000,
|
|
601
|
+
skipExisting: true, // default
|
|
602
|
+
withSuggestions: true, // default
|
|
603
|
+
onProgress: (r) => console.log(r.name, r.docsImported, r.warnings),
|
|
604
|
+
});
|
|
605
|
+
// result.indices: [{ name, created, skipped, docsImported, docsSkipped, docErrors, sugsImported, warnings, error? }]
|
|
606
|
+
// result.aborted: true if interrupted by SIGINT/SIGTERM
|
|
607
|
+
```
|
|
608
|
+
|
|
609
|
+
**What gets migrated:**
|
|
610
|
+
|
|
611
|
+
| RediSearch type | RespLite | Notes |
|
|
612
|
+
|---|---|---|
|
|
613
|
+
| TEXT | TEXT | Direct |
|
|
614
|
+
| TAG | TEXT | Values preserved; TAG filtering lost |
|
|
615
|
+
| NUMERIC | TEXT | Stored as string; numeric range queries not supported |
|
|
616
|
+
| GEO, VECTOR, … | skipped | Warning emitted per field |
|
|
617
|
+
|
|
618
|
+
- Only **HASH**-based indices are supported. JSON (RedisJSON) indices are skipped.
|
|
619
|
+
- A `payload` field is added automatically if none of the source fields maps to it.
|
|
620
|
+
- Suggestions are imported via `FT.SUGGET "" MAX n WITHSCORES` (no cursor; capped at `maxSuggestions`).
|
|
621
|
+
- Graceful shutdown: Ctrl+C finishes the current document, closes SQLite cleanly, and exits with a non-zero code.
|
|
622
|
+
|
|
623
|
+
## Compatibility matrix
|
|
557
624
|
|
|
558
625
|
### Supported (v1)
|
|
559
626
|
|
|
@@ -594,7 +661,7 @@ Unsupported commands return: `ERR command not supported yet`.
|
|
|
594
661
|
| `npm run test:stress` | Stress tests |
|
|
595
662
|
| `npm run benchmark` | Comparative benchmark Redis vs RESPLite |
|
|
596
663
|
| `npm run import-from-redis` | One-shot import from Redis into a SQLite DB |
|
|
597
|
-
| `npx resplite-import` (preflight, bulk, status, apply-dirty, verify) | Migration CLI (
|
|
664
|
+
| `npx resplite-import` (preflight, bulk, status, apply-dirty, verify) | Migration CLI (minimal-downtime flow) |
|
|
598
665
|
| `npx resplite-dirty-tracker <start\|stop>` | Dirty-key tracker for migration cutover |
|
|
599
666
|
|
|
600
667
|
## Specification
|
package/package.json
CHANGED
package/spec/SPEC_F.md
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* Perfect change-data-capture guarantees equivalent to replication logs.
|
|
14
14
|
* Distributed migration across multiple import workers with strict ordering semantics.
|
|
15
15
|
* Full fidelity for unsupported Redis data types (streams, modules, Lua scripts, etc.).
|
|
16
|
+
* **Search indices (FT.\*):** Keyspace migration (`bulk` / `apply-dirty`) copies only the Redis KV data (strings, hashes, sets, lists, zsets). RediSearch index schemas and documents are migrated separately via the `migrate-search` step (§F.10).
|
|
16
17
|
|
|
17
18
|
---
|
|
18
19
|
|
|
@@ -258,6 +259,18 @@ Persist:
|
|
|
258
259
|
|
|
259
260
|
If interrupted, `--resume` restarts from the stored cursor.
|
|
260
261
|
|
|
262
|
+
## F.7.2.1 Graceful shutdown (SIGINT / SIGTERM)
|
|
263
|
+
|
|
264
|
+
When the bulk import process receives SIGINT or SIGTERM it must:
|
|
265
|
+
|
|
266
|
+
* Stop the import loop after the current key (no partial key).
|
|
267
|
+
* Write a final checkpoint (cursor and counters) so progress is persisted.
|
|
268
|
+
* Set run status to `aborted`.
|
|
269
|
+
* Close the SQLite database handle so WAL is checkpointed and the file is not left open.
|
|
270
|
+
* Remove signal handlers and exit (rethrow so the process exits non-zero).
|
|
271
|
+
|
|
272
|
+
This ensures the destination DB is always closed cleanly when the process is killed or interrupted; the next run with `--resume` continues from the last checkpoint.
|
|
273
|
+
|
|
261
274
|
## F.7.3 Throughput controls
|
|
262
275
|
|
|
263
276
|
The importer must support:
|
|
@@ -497,6 +510,87 @@ If dirty tracker disconnects:
|
|
|
497
510
|
|
|
498
511
|
---
|
|
499
512
|
|
|
513
|
+
# F.10 Search Index Migration (FT.* / RediSearch)
|
|
514
|
+
|
|
515
|
+
## F.10.1 Overview
|
|
516
|
+
|
|
517
|
+
When the source is a Redis instance with **RediSearch** (Redis Stack or the `redis/search` module), search indices can be migrated with the `migrate-search` step. This step is independent of the KV bulk import and can be run at any time (before or after `bulk`).
|
|
518
|
+
|
|
519
|
+
## F.10.2 Algorithm
|
|
520
|
+
|
|
521
|
+
For each index in the source:
|
|
522
|
+
|
|
523
|
+
1. **`FT._LIST`** → enumerate all index names.
|
|
524
|
+
2. **`FT.INFO <name>`** → read `index_definition` (key type, prefix patterns) and `attributes` (field names and types).
|
|
525
|
+
3. **Schema mapping** (see §F.10.3).
|
|
526
|
+
4. **`FT.CREATE`** in RespLite with the mapped schema. Skip if already exists (controlled by `skipExisting`).
|
|
527
|
+
5. **SCAN** keys matching each index prefix → **HGETALL** → `addDocument` in SQLite batches.
|
|
528
|
+
6. **`FT.SUGGET "" MAX n WITHSCORES`** → import suggestions into RespLite.
|
|
529
|
+
|
|
530
|
+
## F.10.3 Field type mapping
|
|
531
|
+
|
|
532
|
+
| RediSearch type | RespLite type | Notes |
|
|
533
|
+
|-----------------|---------------|-------|
|
|
534
|
+
| TEXT | TEXT | Direct mapping |
|
|
535
|
+
| TAG | TEXT | Values preserved as-is; TAG filtering semantics lost |
|
|
536
|
+
| NUMERIC | TEXT | Values stored as strings; numeric range queries not supported |
|
|
537
|
+
| GEO, VECTOR, … | — | Skipped with warning |
|
|
538
|
+
|
|
539
|
+
RespLite requires a `payload` TEXT field. If none of the source fields maps to `payload`, a `payload` field is added automatically and synthesised at import time by concatenating all other text values.
|
|
540
|
+
|
|
541
|
+
## F.10.4 Constraints
|
|
542
|
+
|
|
543
|
+
* Only **HASH**-based indices are supported (`key_type = HASH`). JSON indices (RedisJSON) are skipped with an error.
|
|
544
|
+
* Index names must match `[A-Za-z][A-Za-z0-9:_-]{0,63}`. Indices with invalid names are skipped with an error.
|
|
545
|
+
* `FT.SUGGET` has no cursor; suggestions are imported up to `maxSuggestions` (default 10 000).
|
|
546
|
+
* Document score is read from the `__score` or `score` hash field if present; defaults to `1.0`.
|
|
547
|
+
|
|
548
|
+
## F.10.5 Graceful shutdown
|
|
549
|
+
|
|
550
|
+
Same pattern as `bulk` (§F.7.2.1): SIGINT/SIGTERM finishes the current document, closes the SQLite DB cleanly, and exits with a non-zero code.
|
|
551
|
+
|
|
552
|
+
## F.10.6 CLI
|
|
553
|
+
|
|
554
|
+
```bash
|
|
555
|
+
# Migrate all RediSearch indices
|
|
556
|
+
resplite-import migrate-search \
|
|
557
|
+
--from redis://10.0.0.10:6379 \
|
|
558
|
+
--to ./resplite.db
|
|
559
|
+
|
|
560
|
+
# Migrate specific indices only
|
|
561
|
+
resplite-import migrate-search \
|
|
562
|
+
--from redis://10.0.0.10:6379 \
|
|
563
|
+
--to ./resplite.db \
|
|
564
|
+
--index products \
|
|
565
|
+
--index articles
|
|
566
|
+
|
|
567
|
+
# Options
|
|
568
|
+
# --scan-count N SCAN COUNT hint (default 500)
|
|
569
|
+
# --max-rps N throttle (default unlimited)
|
|
570
|
+
# --batch-docs N docs per SQLite transaction (default 200)
|
|
571
|
+
# --max-suggestions N cap for FT.SUGGET (default 10000)
|
|
572
|
+
# --no-skip overwrite if index already exists
|
|
573
|
+
# --no-suggestions skip suggestion import
|
|
574
|
+
```
|
|
575
|
+
|
|
576
|
+
## F.10.7 Programmatic API
|
|
577
|
+
|
|
578
|
+
```javascript
|
|
579
|
+
const m = createMigration({ from, to, runId });
|
|
580
|
+
|
|
581
|
+
const result = await m.migrateSearch({
|
|
582
|
+
onlyIndices: ['products', 'articles'], // omit for all
|
|
583
|
+
batchDocs: 200,
|
|
584
|
+
maxSuggestions: 10000,
|
|
585
|
+
skipExisting: true,
|
|
586
|
+
withSuggestions: true,
|
|
587
|
+
onProgress: (r) => console.log(r.name, r.docsImported, r.warnings),
|
|
588
|
+
});
|
|
589
|
+
// result: { indices: [{ name, created, skipped, docsImported, docsSkipped, docErrors, sugsImported, warnings, error? }], aborted }
|
|
590
|
+
```
|
|
591
|
+
|
|
592
|
+
---
|
|
593
|
+
|
|
500
594
|
# F.12 Operational Guidance (Large datasets)
|
|
501
595
|
|
|
502
596
|
* Use a dedicated Redis replica for reads if possible to reduce load on primary.
|
|
@@ -11,9 +11,10 @@ import { runPreflight } from '../migration/preflight.js';
|
|
|
11
11
|
import { runBulkImport } from '../migration/bulk.js';
|
|
12
12
|
import { runApplyDirty } from '../migration/apply-dirty.js';
|
|
13
13
|
import { runVerify } from '../migration/verify.js';
|
|
14
|
+
import { runMigrateSearch } from '../migration/migrate-search.js';
|
|
14
15
|
import { getRun, getDirtyCounts } from '../migration/registry.js';
|
|
15
16
|
|
|
16
|
-
const SUBCOMMANDS = ['preflight', 'bulk', 'status', 'apply-dirty', 'verify'];
|
|
17
|
+
const SUBCOMMANDS = ['preflight', 'bulk', 'status', 'apply-dirty', 'verify', 'migrate-search'];
|
|
17
18
|
|
|
18
19
|
function parseArgs(argv = process.argv.slice(2)) {
|
|
19
20
|
const args = { _: [] };
|
|
@@ -53,6 +54,17 @@ function parseArgs(argv = process.argv.slice(2)) {
|
|
|
53
54
|
args.pragmaTemplate = argv[++i];
|
|
54
55
|
} else if (arg === '--sample' && argv[i + 1]) {
|
|
55
56
|
args.sample = argv[++i];
|
|
57
|
+
} else if (arg === '--index' && argv[i + 1]) {
|
|
58
|
+
if (!args.index) args.index = [];
|
|
59
|
+
args.index.push(argv[++i]);
|
|
60
|
+
} else if (arg === '--batch-docs' && argv[i + 1]) {
|
|
61
|
+
args.batchDocs = parseInt(argv[++i], 10);
|
|
62
|
+
} else if (arg === '--max-suggestions' && argv[i + 1]) {
|
|
63
|
+
args.maxSuggestions = parseInt(argv[++i], 10);
|
|
64
|
+
} else if (arg === '--no-skip') {
|
|
65
|
+
args.noSkip = true;
|
|
66
|
+
} else if (arg === '--no-suggestions') {
|
|
67
|
+
args.noSuggestions = true;
|
|
56
68
|
} else if (arg.startsWith('--')) {
|
|
57
69
|
args[arg.slice(2).replace(/-/g, '')] = argv[i + 1] ?? true;
|
|
58
70
|
if (argv[i + 1] && !argv[i + 1].startsWith('--')) i++;
|
|
@@ -200,28 +212,73 @@ async function cmdVerify(args) {
|
|
|
200
212
|
}
|
|
201
213
|
}
|
|
202
214
|
|
|
215
|
+
async function cmdMigrateSearch(args) {
|
|
216
|
+
const redisUrl = getRedisUrl(args);
|
|
217
|
+
const dbPath = getDbPath(args);
|
|
218
|
+
const client = createClient({ url: redisUrl });
|
|
219
|
+
client.on('error', (e) => console.error('Redis:', e.message));
|
|
220
|
+
await client.connect();
|
|
221
|
+
try {
|
|
222
|
+
const onlyIndices = args.index
|
|
223
|
+
? (Array.isArray(args.index) ? args.index : [args.index])
|
|
224
|
+
: null;
|
|
225
|
+
|
|
226
|
+
const result = await runMigrateSearch(client, dbPath, {
|
|
227
|
+
pragmaTemplate: args.pragmaTemplate || 'default',
|
|
228
|
+
onlyIndices,
|
|
229
|
+
scanCount: args.scanCount || 500,
|
|
230
|
+
maxRps: args.maxRps || 0,
|
|
231
|
+
batchDocs: args.batchDocs || 200,
|
|
232
|
+
maxSuggestions: args.maxSuggestions || 10000,
|
|
233
|
+
skipExisting: args.noSkip ? false : true,
|
|
234
|
+
withSuggestions: args.noSuggestions ? false : true,
|
|
235
|
+
onProgress: (r) => {
|
|
236
|
+
const status = r.error ? `ERROR: ${r.error}` : (r.skipped ? 'skipped (already exists)' : 'created');
|
|
237
|
+
console.log(` [${r.name}] ${status} — docs=${r.docsImported} skipped=${r.docsSkipped} errors=${r.docErrors} sugs=${r.sugsImported}`);
|
|
238
|
+
if (r.warnings?.length) r.warnings.forEach((w) => console.log(` WARN: ${w}`));
|
|
239
|
+
},
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
if (result.aborted) console.log('Migration aborted by signal.');
|
|
243
|
+
console.log(`Done. Indices processed: ${result.indices.length}`);
|
|
244
|
+
const errors = result.indices.filter((i) => i.error);
|
|
245
|
+
if (errors.length) {
|
|
246
|
+
console.error(` ${errors.length} index(es) failed:`);
|
|
247
|
+
errors.forEach((i) => console.error(` - ${i.name}: ${i.error}`));
|
|
248
|
+
}
|
|
249
|
+
} finally {
|
|
250
|
+
await client.quit();
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
203
254
|
async function main() {
|
|
204
255
|
const args = parseArgs();
|
|
205
256
|
const sub = args._[0];
|
|
206
257
|
if (!SUBCOMMANDS.includes(sub)) {
|
|
207
|
-
console.error('Usage: resplite-import <preflight|bulk|status|apply-dirty|verify> [options]');
|
|
208
|
-
console.error(' --from <redis-url>
|
|
209
|
-
console.error(' --to <db-path>
|
|
210
|
-
console.error(' --run-id <id>
|
|
211
|
-
console.error(' --scan-count N
|
|
212
|
-
console.error(' --max-rps N
|
|
213
|
-
console.error(' --batch-keys N
|
|
214
|
-
console.error(' --batch-bytes N[MB|KB|GB]
|
|
215
|
-
console.error(' --resume / --no-resume
|
|
216
|
-
console.error(' --sample 0.5%
|
|
258
|
+
console.error('Usage: resplite-import <preflight|bulk|status|apply-dirty|verify|migrate-search> [options]');
|
|
259
|
+
console.error(' --from <redis-url> (default: redis://127.0.0.1:6379)');
|
|
260
|
+
console.error(' --to <db-path> (required for bulk, status, apply-dirty, verify, migrate-search)');
|
|
261
|
+
console.error(' --run-id <id> (required for bulk, status, apply-dirty)');
|
|
262
|
+
console.error(' --scan-count N (bulk / migrate-search, default 1000 / 500)');
|
|
263
|
+
console.error(' --max-rps N (bulk, apply-dirty, migrate-search)');
|
|
264
|
+
console.error(' --batch-keys N (default 200)');
|
|
265
|
+
console.error(' --batch-bytes N[MB|KB|GB](default 64MB)');
|
|
266
|
+
console.error(' --resume / --no-resume (bulk: default resume=on)');
|
|
267
|
+
console.error(' --sample 0.5% (verify, default 0.5%)');
|
|
268
|
+
console.error(' --index <name> (migrate-search: repeat for multiple; omit for all indices)');
|
|
269
|
+
console.error(' --batch-docs N (migrate-search: docs per SQLite tx, default 200)');
|
|
270
|
+
console.error(' --max-suggestions N (migrate-search: cap for FT.SUGGET, default 10000)');
|
|
271
|
+
console.error(' --no-skip (migrate-search: overwrite if index exists)');
|
|
272
|
+
console.error(' --no-suggestions (migrate-search: skip suggestion import)');
|
|
217
273
|
process.exit(1);
|
|
218
274
|
}
|
|
219
275
|
try {
|
|
220
|
-
if (sub === 'preflight')
|
|
221
|
-
else if (sub === 'bulk')
|
|
222
|
-
else if (sub === 'status')
|
|
223
|
-
else if (sub === 'apply-dirty')
|
|
224
|
-
else if (sub === 'verify')
|
|
276
|
+
if (sub === 'preflight') await cmdPreflight(args);
|
|
277
|
+
else if (sub === 'bulk') await cmdBulk(args);
|
|
278
|
+
else if (sub === 'status') await cmdStatus(args);
|
|
279
|
+
else if (sub === 'apply-dirty')await cmdApplyDirty(args);
|
|
280
|
+
else if (sub === 'verify') await cmdVerify(args);
|
|
281
|
+
else if (sub === 'migrate-search') await cmdMigrateSearch(args);
|
|
225
282
|
} catch (err) {
|
|
226
283
|
console.error(err);
|
|
227
284
|
process.exit(1);
|
|
@@ -236,4 +293,4 @@ if (isMain) {
|
|
|
236
293
|
});
|
|
237
294
|
}
|
|
238
295
|
|
|
239
|
-
export { parseArgs, cmdPreflight, cmdBulk, cmdStatus, cmdApplyDirty, cmdVerify };
|
|
296
|
+
export { parseArgs, cmdPreflight, cmdBulk, cmdStatus, cmdApplyDirty, cmdVerify, cmdMigrateSearch };
|