resplite 1.2.4 → 1.2.8

package/spec/08-type-sorted-sets.md

@@ -0,0 +1,220 @@
+# RESPLite — Type: Sorted Sets / ZSET (Appendix C)
+
+## C.1 Goals
+
+* Provide a Redis-compatible subset of ZSET commands with efficient range and score queries.
+* Persist in SQLite with appropriate indexing.
+* Keep semantics close to Redis for ordering, score ties, and missing elements.
+
+## C.2 Supported Commands (vNext)
+
+Recommended minimal set:
+
+* `ZADD key [NX|XX] [CH] [INCR] score member [score member ...]` (start with a reduced subset)
+* `ZREM key member [member ...]`
+* `ZCARD key`
+* `ZSCORE key member`
+* `ZRANGE key start stop [WITHSCORES]`
+* `ZREVRANGE key start stop [WITHSCORES]` (optional but useful)
+* `ZRANGEBYSCORE key min max [WITHSCORES] [LIMIT offset count]`
+* `ZREMRANGEBYSCORE key min max` (optional, later)
+* `ZSCAN key cursor [MATCH pattern] [COUNT n]` (later)
+
+**Initial simplification for v1 of ZSET:**
+
+* Support `ZADD key score member [score member ...]` (no flags) returning number of new elements.
+* Add flags later.
+
+## C.3 Redis Semantics
+
+* Sorted set is ordered by:
+
+  1. score ascending
+  2. member lexicographically ascending as tie-breaker (Redis behavior)
+* Wrong type errors same pattern as other types.
+* Non-existent key:
+
+  * `ZCARD` => `0`
+  * `ZRANGE` => empty array
+  * `ZSCORE` => `nil`
+
+## C.4 Data Model (SQLite)
+
+```sql
+CREATE TABLE redis_zsets (
+  key BLOB NOT NULL,
+  member BLOB NOT NULL,
+  score REAL NOT NULL,
+  PRIMARY KEY (key, member),
+  FOREIGN KEY(key) REFERENCES redis_keys(key) ON DELETE CASCADE
+);
+
+CREATE INDEX redis_zsets_key_score_member_idx
+  ON redis_zsets(key, score, member);
+```
+
+Notes:
+
+* `PRIMARY KEY (key, member)` allows upsert of member score.
+* Secondary index supports score range scans and stable ordering by `(score, member)`.
+
+## C.5 Command Behavior
+
+### C.5.1 ZADD
+
+**Minimal v1 behavior:**
+
+* `ZADD key score member [score member ...]`
+* Response: integer count of **new** members added (not updated).
+* If key does not exist: create metadata type zset.
+* For existing member: update score (does not increment return count).
+* Use one transaction:
+
+  * ensure type
+  * upsert all pairs
+  * bump key version
+
+**Later flags (optional):**
+
+* `NX`: only add new
+* `XX`: only update existing
+* `CH`: count changed elements
+* `INCR`: single member increment
+
+### C.5.2 ZREM
+
+* Remove one or more members.
+* Response: integer number removed.
+* If zset becomes empty: delete key metadata.
+
+### C.5.3 ZCARD
+
+* Return cardinality:
+
+  * Prefer `SELECT COUNT(*)` (acceptable).
+  * If performance needs: maintain count in a meta table (not needed initially).
+
+### C.5.4 ZSCORE
+
+* Return bulk string representing score (Redis returns string form), or `nil`.
+* Store as `REAL`, but serialize consistently:
+
+  * Use a stable conversion (avoid scientific notation surprises if possible).
+  * Accept that exact formatting may differ from Redis; document if needed.
+
+### C.5.5 ZRANGE (by rank)
+
+**Request:** `ZRANGE key start stop [WITHSCORES]`
+
+Rank rules like LRANGE:
+
+* start/stop inclusive
+* negative indexes from end
+* clamp
+
+Implementation:
+
+* let `len = ZCARD`
+* normalize range
+* SQL for ordering:
+
+  ```sql
+  SELECT member, score
+  FROM redis_zsets
+  WHERE key=?
+  ORDER BY score ASC, member ASC
+  LIMIT ? OFFSET ?;
+  ```
+* Response:
+
+  * without WITHSCORES: array of members
+  * with WITHSCORES: array `[member1, score1, member2, score2, ...]`
+
+### C.5.6 ZRANGEBYSCORE
+
+**Request:** `ZRANGEBYSCORE key min max [WITHSCORES] [LIMIT offset count]`
+
+Score bounds rules:
+
+* Support numeric `min/max`.
+* Optional later: `(` exclusive bounds, `-inf`, `+inf`.
+
+Implementation:
+
+```sql
+SELECT member, score
+FROM redis_zsets
+WHERE key=? AND score >= ? AND score <= ?
+ORDER BY score ASC, member ASC
+LIMIT ? OFFSET ?;
+```
+
+Return format same as `ZRANGE`.
+
+## C.6 Expiration and Cache
+
+* TTL is in `redis_keys.expires_at`.
+* Lazy expiration removes zset rows via cascade.
+* Cache:
+
+  * Cache `ZSCORE` lookups optionally (key+member) if beneficial.
+  * Avoid caching full zsets early.
+  * Always invalidate on `ZADD/ZREM`.
+
+## C.7 Complexity Targets
+
+* `ZADD`: O(k log n) effectively via index maintenance, practical for SQLite
+* `ZRANGE`: O(m) over returned slice with index support
+* `ZRANGEBYSCORE`: O(m) over match range with index support
+* `ZSCORE`: O(log n) via PK on (key, member)
+
+## C.8 Tests (Required)
+
+* Correct ordering:
+
+  * by score, then by member for ties
+* Rank normalization:
+
+  * negative indices
+  * out of range
+  * start > stop -> empty
+* Score range:
+
+  * boundaries inclusive
+  * LIMIT behavior
+* Wrong type behavior
+* TTL interaction and lazy deletion
+* Persistence across restart
+* Binary member support
+* Concurrency sanity:
+
+  * multiple clients doing `ZADD` on same key does not corrupt ordering or counts
+
+---
+
+## Integration notes (LIST and ZSET)
+
+### Type constants
+
+Extend `redis_keys.type` enum:
+
+* `4 = list`
+* `5 = zset`
+
+### Wrong-type enforcement
+
+Any command on a key must:
+
+1. run lazy-expire check
+2. read `redis_keys.type`
+3. if mismatch, return WRONGTYPE
+
+### Key deletion when empty
+
+For list and zset:
+
+* if becomes empty, delete metadata key row (and meta/items rows if any)
+
+### SCAN behavior
+
+* SCAN should include list/zset keys automatically (it reads from `redis_keys`).

package/spec/{SPEC_D.md → 09-search-ft-commands.md}

@@ -1,4 +1,6 @@
-# Appendix D: Search (FT.*) Specification
+# RESPLite — Search (FT.*) Specification
+
+Originally Appendix D. Goals, data model, FT.CREATE/ADD/DEL/SEARCH/SUG* behavior, SQL templates, parser grammar.
 
 ## D.1 Goals
 

package/spec/{SPEC_E.md → 10-blocking-commands.md}

@@ -1,4 +1,6 @@
-# Appendix E: Blocking Commands Specification (vNext)
+# RESPLite — Blocking Commands Specification (vNext)
+
+Originally Appendix E. BLPOP/BRPOP semantics, wait model, wakeup, tests.
 
 ## E.1 Goals
 

package/spec/{SPEC_F.md → 11-migration-dirty-registry.md}

@@ -1,4 +1,6 @@
-# Appendix F: Migration with Dirty Key Registry (Keyspace Notifications)
+# RESPLite — Migration with Dirty Key Registry (Keyspace Notifications)
+
+Originally Appendix F. Bulk import, dirty key tracker, delta apply, search index migration.
 
 ## F.1 Goals
 
@@ -13,7 +15,7 @@
 * Perfect change-data-capture guarantees equivalent to replication logs.
 * Distributed migration across multiple import workers with strict ordering semantics.
 * Full fidelity for unsupported Redis data types (streams, modules, Lua scripts, etc.).
-* **Search indices (FT.\*):** Keyspace migration (`bulk` / `apply-dirty`) copies only the Redis KV data (strings, hashes, sets, lists, zsets). RediSearch index schemas and documents are migrated separately via the `migrate-search` step (§F.10).
+* **Search indices (FT.\*):** Keyspace migration (`bulk` / `apply-dirty`) copies only the Redis KV data (strings, hashes, sets, lists, zsets). RediSearch index schemas and documents are migrated separately via the `migrate-search` step (§F.12).
 
 ---
 
@@ -71,7 +73,7 @@ A practical baseline is:
   * `z` (zset)
   * `t` (set)
 
-If you need the broadest coverage, use “all” (often `AKE`-style in some docs), but configuration specifics vary by Redis version and operational policy. The migration tool should:
+If you need the broadest coverage, use "all" (often `AKE`-style in some docs), but configuration specifics vary by Redis version and operational policy. The migration tool should:
 
 * detect whether notifications are enabled
 * refuse or warn if they are not enabled
@@ -198,7 +200,7 @@ If only keyspace notifications are available, you will receive:
 * channel includes the key, payload includes the event
   You must support both, but keyevent is simpler.
 
-## F.6.2 Events to treat as “dirty”
+## F.6.2 Events to treat as "dirty"
 
 Mark key as dirty when you see any of:
 
@@ -209,7 +211,7 @@ Mark key as dirty when you see any of:
 * `zadd`, `zrem`, etc.
 * `expire`, `pexpire`, `persist` (TTL changes)
 
-## F.6.3 Events to treat as “deleted”
+## F.6.3 Events to treat as "deleted"
 
 Mark key as deleted when you see:
 
@@ -225,7 +227,7 @@ Keyspace notifications are not a guaranteed durable log:
 * if the tracker disconnects, events can be missed
   Mitigation:
 * treat the final cutover delta as authoritative with the application frozen
-* optionally run one short SCAN after freeze as a “safety sweep” if you want extra assurance
+* optionally run one short SCAN after freeze as a "safety sweep" if you want extra assurance
 
 ---
 
@@ -286,7 +288,7 @@ The importer must support:
 ## F.8.1 When to run delta
 
 * During cutover window, with application writes frozen.
-* Optional: run a “pre-delta” while still live to reduce final delta size.
+* Optional: run a "pre-delta" while still live to reduce final delta size.
 
 ## F.8.2 Delta algorithm
 
@@ -331,7 +333,7 @@ After freeze begins and delta completes:
 
 ---
 
-# F.9 Suggested End-to-End Migration Process (Example)
+# F.9 Suggested End-to-End Migration Process (Programmatic Example)
 
 Assume:
 
@@ -341,119 +343,55 @@ Assume:
 * Supported types: string/hash/set/list/zset
 * Goal: minimal downtime
 
-## Step 0: Preflight
-
-```bash id="4bct0i"
-resplite-import preflight \
-  --from redis://10.0.0.10:6379 \
-  --to ./resplite.db
-```
-
-Outputs:
-
-* estimated key count
-* type distribution sample
-* recommended concurrency and scan count
-* detection of unsupported types
-
-## Step 1: Start Dirty Key Tracker
-
-Start the tracker first, so it captures changes during the entire bulk run.
-
-```bash id="6km4l7"
-resplite-dirty-tracker start \
-  --run-id run_2026_03_03 \
-  --from redis://10.0.0.10:6379 \
-  --to ./resplite.db \
-  --channels keyevent
-```
-
-## Step 2: Run Bulk Import Online
-
-```bash id="a9g2aa"
-resplite-import bulk \
-  --run-id run_2026_03_03 \
-  --from redis://10.0.0.10:6379 \
-  --to ./resplite.db \
-  --scan-count 1000 \
-  --max-concurrency 32 \
-  --max-rps 2000 \
-  --batch-keys 200 \
-  --batch-bytes 64MB \
-  --ttl-mode preserve \
-  --resume
-```
-
-Monitor progress:
-
-```bash id="rkf6uv"
-resplite-import status --run-id run_2026_03_03 --to ./resplite.db
-```
-
-## Step 3 (Optional): Pre-Delta While Still Live
-
-Apply dirty keys while Redis is still live to reduce final delta size:
-
-```bash id="v5xc6f"
-resplite-import apply-dirty \
-  --run-id run_2026_03_03 \
-  --from redis://10.0.0.10:6379 \
-  --to ./resplite.db \
-  --max-concurrency 32 \
-  --max-rps 2000 \
-  --batch-keys 200 \
-  --ttl-mode preserve
-```
-
-You can run this repeatedly (or continuously) while bulk is still running.
-
-## Step 4: Cutover Window (Freeze Writes)
-
-* Put the application into maintenance mode (freeze writes to Redis).
-* Keep dirty tracker running for a moment to capture any last writes.
-
-## Step 5: Final Delta Apply
-
-With writes frozen, apply all remaining dirty keys:
+```javascript id="f9programmatic"
+import { stdin, stdout } from 'node:process';
+import { createInterface } from 'node:readline/promises';
+import { createMigration } from 'resplite/migration';
+
+const m = createMigration({
+  from: 'redis://10.0.0.10:6379',
+  to: './resplite.db',
+  runId: 'run_2026_03_03',
+  scanCount: 1000,
+  batchKeys: 200,
+  batchBytes: 64 * 1024 * 1024,
+  maxRps: 2000,
+});
 
-```bash id="v0x8xo"
-resplite-import apply-dirty \
-  --run-id run_2026_03_03 \
-  --from redis://10.0.0.10:6379 \
-  --to ./resplite.db \
-  --max-concurrency 64 \
-  --max-rps 5000 \
-  --batch-keys 500 \
-  --ttl-mode preserve
-```
+const info = await m.preflight();
+await m.enableKeyspaceNotifications();
+await m.startDirtyTracker();
+
+const total = info.keyCountEstimate || 1;
+await m.bulk({
+  resume: true,
+  onProgress: (r) => {
+    const pct = ((r.scanned_keys / total) * 100).toFixed(1);
+    console.log(`bulk ${pct}% scanned=${r.scanned_keys} migrated=${r.migrated_keys}`);
+  },
+});
 
-Verify no remaining dirty keys:
+console.log(m.status());
 
-```bash id="0ca1y7"
-resplite-import status --run-id run_2026_03_03 --to ./resplite.db
-```
+const rl = createInterface({ input: stdin, output: stdout });
+await rl.question('Freeze writes to Redis, then press Enter to apply the final dirty set...');
+rl.close();
 
-## Step 6: Stop Dirty Tracker and Switch Clients
+await m.applyDirty();
+await m.stopDirtyTracker();
 
-Stop tracker:
+const verify = await m.verify({ samplePct: 0.5, maxSample: 10000 });
+console.log(verify);
 
-```bash id="1v1u1j"
-resplite-dirty-tracker stop --run-id run_2026_03_03 --to ./resplite.db
+await m.close();
 ```
 
-Switch application Redis endpoint to RespLite server (RESP port).
-
-## Step 7: Verification (Post-Cutover)
+Notes:
 
-Run a sampling verification:
-
-```bash id="p6w5q6"
-resplite-import verify \
-  --run-id run_2026_03_03 \
-  --from redis://10.0.0.10:6379 \
-  --to ./resplite.db \
-  --sample 0.5%
-```
+* Start dirty tracking before bulk so it captures writes during the whole import.
+* Keep the tracker running until after the final `applyDirty()`.
+* The cutover window is: freeze writes to Redis, apply the remaining dirty set, stop the tracker, then switch clients to RespLite.
+* `status()` is synchronous and can be polled at any point from the destination DB.
 
 ---
 
@@ -482,7 +420,7 @@ Implementation may use:
 
 * updating `migration_runs.status`
 * a simple control file
-* or a CLI that updates the SQLite run row
+* or another control surface that updates the SQLite run row
 
 ---
 
@@ -498,7 +436,7 @@ If dirty tracker disconnects:
 
 ## F.11.2 Importer crash/restart
 
-* On restart with `--resume`, continue from stored cursor.
+* On restart with resume enabled, continue from stored cursor.
 * Already migrated keys may be overwritten idempotently.
 
 ## F.11.3 Idempotency requirements
@@ -510,24 +448,24 @@ If dirty tracker disconnects:
 
 ---
 
-# F.10 Search Index Migration (FT.* / RediSearch)
+# F.12 Search Index Migration (FT.* / RediSearch)
 
-## F.10.1 Overview
+## F.12.1 Overview
 
 When the source is a Redis instance with **RediSearch** (Redis Stack or the `redis/search` module), search indices can be migrated with the `migrate-search` step. This step is independent of the KV bulk import and can be run at any time (before or after `bulk`).
 
-## F.10.2 Algorithm
+## F.12.2 Algorithm
 
 For each index in the source:
 
 1. **`FT._LIST`** → enumerate all index names.
 2. **`FT.INFO <name>`** → read `index_definition` (key type, prefix patterns) and `attributes` (field names and types).
-3. **Schema mapping** (see §F.10.3).
+3. **Schema mapping** (see §F.12.3).
 4. **`FT.CREATE`** in RespLite with the mapped schema. Skip if already exists (controlled by `skipExisting`).
 5. **SCAN** keys matching each index prefix → **HGETALL** → `addDocument` in SQLite batches.
 6. **`FT.SUGGET "" MAX n WITHSCORES`** → import suggestions into RespLite.
 
-## F.10.3 Field type mapping
+## F.12.3 Field type mapping
 
 | RediSearch type | RespLite type | Notes |
 |-----------------|---------------|-------|
@@ -538,42 +476,18 @@ For each index in the source:
 
 RespLite requires a `payload` TEXT field. If none of the source fields maps to `payload`, a `payload` field is added automatically and synthesised at import time by concatenating all other text values.
 
-## F.10.4 Constraints
+## F.12.4 Constraints
 
 * Only **HASH**-based indices are supported (`key_type = HASH`). JSON indices (RedisJSON) are skipped with an error.
 * Index names must match `[A-Za-z][A-Za-z0-9:_-]{0,63}`. Indices with invalid names are skipped with an error.
 * `FT.SUGGET` has no cursor; suggestions are imported up to `maxSuggestions` (default 10 000).
 * Document score is read from the `__score` or `score` hash field if present; defaults to `1.0`.
 
-## F.10.5 Graceful shutdown
+## F.12.5 Graceful shutdown
 
 Same pattern as `bulk` (§F.7.2.1): SIGINT/SIGTERM finishes the current document, closes the SQLite DB cleanly, and exits with a non-zero code.
 
-## F.10.6 CLI
-
-```bash
-# Migrate all RediSearch indices
-resplite-import migrate-search \
-  --from redis://10.0.0.10:6379 \
-  --to   ./resplite.db
-
-# Migrate specific indices only
-resplite-import migrate-search \
-  --from redis://10.0.0.10:6379 \
-  --to   ./resplite.db \
-  --index products \
-  --index articles
-
-# Options
-#   --scan-count N          SCAN COUNT hint (default 500)
-#   --max-rps N             throttle (default unlimited)
-#   --batch-docs N          docs per SQLite transaction (default 200)
-#   --max-suggestions N     cap for FT.SUGGET (default 10000)
-#   --no-skip               overwrite if index already exists
-#   --no-suggestions        skip suggestion import
-```
-
-## F.10.7 Programmatic API
+## F.12.6 Programmatic API
 
 ```javascript
 const m = createMigration({ from, to, runId });
@@ -591,7 +505,7 @@ const result = await m.migrateSearch({
 
 ---
 
-# F.12 Operational Guidance (Large datasets)
+# F.13 Operational Guidance (Large datasets)
 
 * Use a dedicated Redis replica for reads if possible to reduce load on primary.
 * Keep `max_concurrency` conservative at first; increase only if Redis latency remains stable.

package/src/commands/object.js

@@ -0,0 +1,17 @@
+/**
+ * OBJECT subcommand key - introspection (IDLETIME: seconds since last write).
+ * Only OBJECT IDLETIME key is supported; uses redis_keys.updated_at.
+ */
+
+export function handleObject(engine, args) {
+  if (!args || args.length < 2) {
+    return { error: 'ERR wrong number of arguments for \'OBJECT\' command' };
+  }
+  const sub = (Buffer.isBuffer(args[0]) ? args[0].toString('utf8') : String(args[0])).toUpperCase();
+  if (sub !== 'IDLETIME') {
+    return { error: 'ERR unknown subcommand or wrong number of arguments for \'OBJECT\'. Try OBJECT HELP.' };
+  }
+  const key = args[1];
+  const seconds = engine.objectIdletime(key);
+  return seconds;
+}

package/src/commands/registry.js

@@ -11,6 +11,7 @@ import * as set from './set.js';
 import * as del from './del.js';
 import * as exists from './exists.js';
 import * as type from './type.js';
+import * as object from './object.js';
 import * as mget from './mget.js';
 import * as mset from './mset.js';
 import * as expire from './expire.js';
@@ -75,6 +76,7 @@ const HANDLERS = new Map([
   ['DEL', (e, a) => del.handleDel(e, a)],
   ['EXISTS', (e, a) => exists.handleExists(e, a)],
   ['TYPE', (e, a) => type.handleType(e, a)],
+  ['OBJECT', (e, a) => object.handleObject(e, a)],
   ['MGET', (e, a) => mget.handleMget(e, a)],
   ['MSET', (e, a) => mset.handleMset(e, a)],
   ['EXPIRE', (e, a) => expire.handleExpire(e, a)],

package/src/engine/engine.js

@@ -404,6 +404,17 @@ export function createEngine(opts = {}) {
       return typeName(meta);
     },
 
+    /**
+     * OBJECT IDLETIME: seconds since key was last written (updated_at).
+     * Returns null if key does not exist (Redis: nil).
+     */
+    objectIdletime(key) {
+      const meta = getKeyMeta(key);
+      if (!meta || meta.updatedAt == null) return null;
+      const elapsedMs = clock() - meta.updatedAt;
+      return Math.floor(elapsedMs / 1000);
+    },
+
     scan(cursor, options = {}) {
       const count = options.count ?? 10;
       const offset = parseInt(String(cursor), 10) || 0;

package/src/migration/apply-dirty.js

@@ -25,9 +25,10 @@ function sleep(ms) {
  * @param {string} [options.pragmaTemplate='default']
  * @param {number} [options.batch_keys=200]
  * @param {number} [options.max_rps=0]
+ * @param {(run: object) => void | Promise<void>} [options.onProgress] - Called after each batch with the current run row.
  */
 export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
-  const { pragmaTemplate = 'default', batch_keys = 200, max_rps = 0 } = options;
+  const { pragmaTemplate = 'default', batch_keys = 200, max_rps = 0, onProgress } = options;
 
   const db = openDb(dbPath, { pragmaTemplate });
   const run = getRun(db, runId);
@@ -56,6 +57,8 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
     const deletedBatch = getDirtyBatch(db, runId, 'deleted', batch_keys);
     if (dirtyBatch.length === 0 && deletedBatch.length === 0) break;
 
+    const batchSize = dirtyBatch.length + deletedBatch.length;
+
     // ── Re-import (or remove) keys that changed while bulk was running ──
     for (const { key: keyBuf } of dirtyBatch) {
       r = getRun(db, runId);
@@ -122,6 +125,10 @@ export async function runApplyDirty(redisClient, dbPath, runId, options = {}) {
         markDirtyState(db, runId, keyBuf, 'error');
       }
     }
+    if (batchSize > 0 && onProgress) {
+      const run = getRun(db, runId);
+      if (run) Promise.resolve(onProgress(run)).catch(() => {});
+    }
   }
 
   return getRun(db, runId);