rocksdb-native 1.1.0 → 2.0.0

package/lib/batch.js

@@ -1,20 +1,26 @@
+const c = require('compact-encoding')
 const b4a = require('b4a')
 const binding = require('../binding')
 
 const empty = b4a.alloc(0)
 
-module.exports = class RocksDBBatch {
+class RocksDBBatch {
   constructor (db, opts = {}) {
     const {
-      capacity = 8
+      capacity = 8,
+      encoding = null,
+      keyEncoding = encoding,
+      valueEncoding = encoding
     } = opts
 
     this._db = db
     this._capacity = capacity
-    this._keys = []
-    this._values = []
+    this._operations = []
     this._promises = []
 
+    this._keyEncoding = keyEncoding
+    this._valueEncoding = valueEncoding
+
     this._enqueuePromise = this._enqueuePromise.bind(this)
 
     this._request = null
@@ -31,8 +37,7 @@ module.exports = class RocksDBBatch {
   _onfinished () {
     const resolve = this._resolveRequest
 
-    this._keys = []
-    this._values = []
+    this._operations = []
     this._promises = []
     this._request = null
     this._resolveRequest = null
@@ -40,6 +45,85 @@ module.exports = class RocksDBBatch {
     if (resolve !== null) resolve()
   }
 
+  _resize () {
+    if (this._operations.length <= this._capacity) return false
+
+    while (this._operations.length > this._capacity) {
+      this._capacity *= 2
+    }
+
+    return true
+  }
+
+  async ready () {
+    if (this._handle !== null) return
+
+    if (this._db.opened === false) await this._db.ready()
+
+    this._init()
+  }
+
+  flush () {
+    if (this._request) throw new Error('Request already in progress')
+
+    this._request = new Promise((resolve) => { this._resolveRequest = resolve })
+    this._flush()
+
+    return this._request
+  }
+
+  tryFlush () {
+    if (this._request) throw new Error('Request already in progress')
+
+    this._request = true
+    this._flush()
+  }
+
+  async _flush () {
+    if (this._handle === null) await this.ready()
+  }
+
+  _enqueuePromise (resolve, reject) {
+    this._promises.push({ resolve, reject })
+  }
+
+  _encodeKey (k) {
+    if (this._keyEncoding) return c.encode(this._keyEncoding, k)
+    if (typeof k === 'string') return b4a.from(k)
+    return k
+  }
+
+  _encodeValue (v) {
+    if (this._valueEncoding) return c.encode(this._valueEncoding, v)
+    if (v === null) return empty
+    if (typeof v === 'string') return b4a.from(v)
+    return v
+  }
+
+  _decodeValue (b) {
+    if (this._valueEncoding) return c.decode(this._valueEncoding, b)
+    return b
+  }
+}
+
+exports.ReadBatch = class RocksDBReadBatch extends RocksDBBatch {
+  async _init () {
+    this._handle = binding.readInit(this)
+    this._buffer = binding.readBuffer(this._handle, this._capacity)
+  }
+
+  _resize () {
+    if (super._resize() && this._handle !== null) {
+      this._buffer = binding.readBuffer(this._handle, this._capacity)
+    }
+  }
+
+  async _flush () {
+    await super._flush()
+
+    binding.read(this._db._handle, this._handle, this._operations, this._onread)
+  }
+
   _onread (errs, values) {
     for (let i = 0, n = this._promises.length; i < n; i++) {
       const promise = this._promises[i]
@@ -48,123 +132,158 @@ module.exports = class RocksDBBatch {
       const err = errs[i]
 
       if (err) promise.reject(new Error(err))
-      else promise.resolve(values[i].byteLength === 0 ? null : b4a.from(values[i]))
+      else promise.resolve(values[i].byteLength === 0 ? null : this._decodeValue(b4a.from(values[i])))
     }
 
     this._onfinished()
   }
 
-  _onwrite (err) {
-    for (let i = 0, n = this._promises.length; i < n; i++) {
-      const promise = this._promises[i]
-      if (promise === null) continue
+  get (key) {
+    if (this._request) throw new Error('Request already in progress')
 
-      if (err) promise.reject(new Error(err))
-      else promise.resolve(this._values[i].byteLength === 0 ? null : this._values[i])
-    }
+    const promise = new Promise(this._enqueuePromise)
 
-    this._onfinished()
+    this._operations.push(new RocksDBGet(this._encodeKey(key)))
+    this._resize()
+
+    return promise
   }
 
-  _resize () {
-    if (this._keys.length <= this._capacity) return
+  tryPut (key, value) {
+    if (this._request) throw new Error('Request already in progress')
 
-    while (this._keys.length > this._capacity) {
-      this._capacity *= 2
-    }
+    this._operations.push(new RocksDBPut(this._encodeKey(key), this._encodeValue(value)))
+    this._promises.push(null)
+    this._resize()
+  }
+}
+
+exports.WriteBatch = class RocksDBWriteBatch extends RocksDBBatch {
+  async _init () {
+    this._handle = binding.writeInit(this)
+    this._buffer = binding.writeBuffer(this._handle, this._capacity)
+  }
 
-    if (this._handle !== null) {
-      this._buffer = binding.batchBuffer(this._handle, this._capacity)
+  _resize () {
+    if (super._resize() && this._handle !== null) {
+      this._buffer = binding.writeBuffer(this._handle, this._capacity)
     }
   }
 
-  async ready () {
-    if (this._handle !== null) return
+  async _flush () {
+    await super._flush()
 
-    if (this._db.opened === false) await this._db.ready()
+    binding.write(this._db._handle, this._handle, this._operations, this._onwrite)
+  }
+
+  _onwrite (err) {
+    for (let i = 0, n = this._promises.length; i < n; i++) {
+      const promise = this._promises[i]
+      if (promise === null) continue
 
-    this._handle = binding.batchInit(this._db._handle, this)
+      if (err) promise.reject(new Error(err))
+      else promise.resolve()
+    }
 
-    this._buffer = binding.batchBuffer(this._handle, this._capacity)
+    this._onfinished()
   }
 
-  add (key, value = empty) {
+  put (key, value) {
     if (this._request) throw new Error('Request already in progress')
 
     const promise = new Promise(this._enqueuePromise)
 
-    this._keys.push(this._encodeKey(key))
-    this._values.push(this._encodeValue(value))
+    this._operations.push(new RocksDBPut(this._encodeKey(key), this._encodeValue(value)))
     this._resize()
 
     return promise
   }
 
-  tryAdd (key, value = empty) {
+  tryPut (key, value) {
     if (this._request) throw new Error('Request already in progress')
 
-    this._keys.push(this._encodeKey(key))
-    this._values.push(this._encodeValue(value))
+    this._operations.push(new RocksDBPut(this._encodeKey(key), this._encodeValue(value)))
     this._promises.push(null)
     this._resize()
   }
 
-  read () {
+  delete (key) {
     if (this._request) throw new Error('Request already in progress')
 
-    this._request = new Promise((resolve) => { this._resolveRequest = resolve })
-    this._read()
+    const promise = new Promise(this._enqueuePromise)
 
-    return this._request
+    this._operations.push(new RocksDBDelete(this._encodeKey(key)))
+    this._resize()
+
+    return promise
   }
 
-  tryRead () {
+  tryDelete (key) {
     if (this._request) throw new Error('Request already in progress')
 
-    this._request = true
-    this._read()
+    this._operations.push(new RocksDBDelete(this._encodeKey(key)))
+    this._promises.push(null)
+    this._resize()
   }
 
-  async _read () {
-    if (this._handle === null) await this.ready()
+  deleteRange (start, end) {
+    if (this._request) throw new Error('Request already in progress')
+
+    const promise = new Promise(this._enqueuePromise)
+
+    this._operations.push(new RocksDBDeleteRange(this._encodeKey(start), this._encodeKey(end)))
+    this._resize()
 
-    binding.batchRead(this._handle, this._keys, this._onread)
+    return promise
   }
 
-  write () {
+  tryDeleteRange (start, end) {
     if (this._request) throw new Error('Request already in progress')
 
-    this._request = new Promise((resolve) => { this._resolveRequest = resolve })
-    this._write()
+    this._operations.push(new RocksDBDeleteRange(this._encodeKey(start), this._encodeKey(end)))
+    this._promises.push(null)
+    this._resize()
+  }
+}
 
-    return this._request
+class RocksDBGet {
+  constructor (key) {
+    this.key = key
   }
 
-  tryWrite () {
-    if (this._request) throw new Error('Request already in progress')
+  get type () {
+    return binding.GET
+  }
+}
 
-    this._request = true
-    this._write()
+class RocksDBPut {
+  constructor (key, value) {
+    this.key = key
+    this.value = value
   }
 
-  async _write () {
-    if (this._handle === null) await this.ready()
+  get type () {
+    return binding.PUT
+  }
+}
 
-    binding.batchWrite(this._handle, this._keys, this._values, this._onwrite)
+class RocksDBDelete {
+  constructor (key) {
+    this.key = key
   }
 
-  _enqueuePromise (resolve, reject) {
-    this._promises.push({ resolve, reject })
+  get type () {
+    return binding.DELETE
   }
+}
 
-  _encodeKey (k) {
-    if (typeof k === 'string') return Buffer.from(k)
-    return k
+class RocksDBDeleteRange {
+  constructor (start, end) {
+    this.start = start
+    this.end = end
   }
 
-  _encodeValue (v) {
-    if (v === null) return empty
-    if (typeof v === 'string') return Buffer.from(v)
-    return v
+  get type () {
+    return binding.DELETE_RANGE
   }
 }

package/lib/iterator.js

@@ -1,40 +1,36 @@
 const { Readable } = require('streamx')
+const c = require('compact-encoding')
 const b4a = require('b4a')
 const binding = require('../binding')
 
 const empty = b4a.alloc(0)
 
 module.exports = class RocksDBIterator extends Readable {
-  constructor (db, start, end, opts = {}) {
-    if (typeof start === 'string') start = b4a.from(start)
-    if (typeof end === 'string') end = b4a.from(end)
-
-    if (start && !b4a.isBuffer(start)) {
-      opts = start
-      start = empty
-    } else if (end && !b4a.isBuffer(end)) {
-      opts = end
-      end = empty
-    }
-
+  constructor (db, opts = {}) {
     const {
-      gt = empty,
-      gte = start,
-      lt = end,
-      lte = empty,
+      gt = null,
+      gte = null,
+      lt = null,
+      lte = null,
       reverse = false,
       limit = Infinity,
-      capacity = 8
+      capacity = 8,
+      encoding = null,
+      keyEncoding = encoding,
+      valueEncoding = encoding
     } = opts
 
     super()
 
     this._db = db
 
-    this._gt = toBuffer(gt) || empty
-    this._gte = toBuffer(gte) || empty
-    this._lt = toBuffer(lt) || empty
-    this._lte = toBuffer(lte) || empty
+    this._keyEncoding = keyEncoding
+    this._valueEncoding = valueEncoding
+
+    this._gt = gt ? this._encodeKey(gt) : empty
+    this._gte = gte ? this._encodeKey(gte) : empty
+    this._lt = lt ? this._encodeKey(lt) : empty
+    this._lte = lte ? this._encodeKey(lte) : empty
 
     this._reverse = reverse
     this._limit = limit
@@ -66,7 +62,7 @@ module.exports = class RocksDBIterator extends Readable {
     this._limit -= n
 
     for (let i = 0; i < n; i++) {
-      this.push({ key: b4a.from(keys[i]), value: b4a.from(values[i]) })
+      this.push({ key: this._decodeKey(b4a.from(keys[i])), value: this._decodeValue(b4a.from(values[i])) })
     }
 
     if (n < this._capacity) this.push(null)
@@ -91,12 +87,11 @@ module.exports = class RocksDBIterator extends Readable {
 
     if (this._db.opened === false) await this._db.ready()
 
-    this._handle = binding.iteratorInit(this._db._handle, this,
-      this._onopen,
-      this._onclose,
-      this._onread
-    )
+    this._init()
+  }
 
+  _init () {
+    this._handle = binding.iteratorInit(this, this._onopen, this._onclose, this._onread)
     this._buffer = binding.iteratorBuffer(this._handle, this._capacity)
   }
 
@@ -105,7 +100,7 @@ module.exports = class RocksDBIterator extends Readable {
 
     this._pendingOpen = cb
 
-    binding.iteratorOpen(this._handle, this._gt, this._gte, this._lt, this._lte, this._reverse)
+    binding.iteratorOpen(this._db._handle, this._handle, this._gt, this._gte, this._lt, this._lte, this._reverse)
   }
 
   _read (cb) {
@@ -121,8 +116,20 @@ module.exports = class RocksDBIterator extends Readable {
 
     binding.iteratorClose(this._handle)
   }
-}
 
-function toBuffer (data) {
-  return typeof data === 'string' ? b4a.from(data) : data
+  _encodeKey (k) {
+    if (this._keyEncoding) return c.encode(this._keyEncoding, k)
+    if (typeof k === 'string') return b4a.from(k)
+    return k
+  }
+
+  _decodeKey (b) {
+    if (this._keyEncoding) return c.decode(this._keyEncoding, b)
+    return b
+  }
+
+  _decodeValue (b) {
+    if (this._valueEncoding) return c.decode(this._valueEncoding, b)
+    return b
+  }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rocksdb-native",
-  "version": "1.1.0",
+  "version": "2.0.0",
   "description": "librocksdb bindings for JavaScript",
   "exports": {
     ".": "./index.js",
@@ -34,6 +34,7 @@
   "homepage": "https://github.com/holepunchto/rocksdb-native",
   "dependencies": {
     "b4a": "^1.6.6",
+    "compact-encoding": "^2.15.0",
     "load-addon": "^1.0.0",
     "ready-resource": "^1.0.0",
     "streamx": "^2.16.1"

package/vendor/librocksdb/include/rocksdb.h

@@ -14,18 +14,18 @@ typedef struct rocksdb_open_s rocksdb_open_t;
 typedef struct rocksdb_close_s rocksdb_close_t;
 typedef struct rocksdb_slice_s rocksdb_slice_t;
 typedef struct rocksdb_range_s rocksdb_range_t;
-typedef struct rocksdb_read_range_s rocksdb_read_range_t;
-typedef struct rocksdb_delete_range_s rocksdb_delete_range_t;
 typedef struct rocksdb_iterator_s rocksdb_iterator_t;
-typedef struct rocksdb_batch_s rocksdb_batch_t;
+typedef struct rocksdb_read_s rocksdb_read_t;
+typedef struct rocksdb_read_batch_s rocksdb_read_batch_t;
+typedef struct rocksdb_write_s rocksdb_write_t;
+typedef struct rocksdb_write_batch_s rocksdb_write_batch_t;
 typedef struct rocksdb_s rocksdb_t;
 
 typedef void (*rocksdb_open_cb)(rocksdb_open_t *req, int status);
 typedef void (*rocksdb_close_cb)(rocksdb_close_t *req, int status);
-typedef void (*rocksdb_read_range_cb)(rocksdb_read_range_t *req, int status);
-typedef void (*rocksdb_delete_range_cb)(rocksdb_delete_range_t *req, int status);
 typedef void (*rocksdb_iterator_cb)(rocksdb_iterator_t *iterator, int status);
-typedef void (*rocksdb_batch_cb)(rocksdb_batch_t *batch, int status);
+typedef void (*rocksdb_read_batch_cb)(rocksdb_read_batch_t *batch, int status);
+typedef void (*rocksdb_write_batch_cb)(rocksdb_write_batch_t *batch, int status);
 
 typedef enum {
   rocksdb_compaction_style_level = 0,
@@ -116,11 +116,13 @@ struct rocksdb_range_s {
   rocksdb_slice_t lte;
 };
 
-struct rocksdb_read_range_s {
+struct rocksdb_iterator_s {
   uv_work_t worker;
 
   rocksdb_t *db;
 
+  void *handle; // Opaque iterator pointer
+
   rocksdb_range_t range;
   bool reverse;
 
@@ -132,67 +134,79 @@ struct rocksdb_read_range_s {
 
   char *error;
 
-  rocksdb_read_range_cb cb;
+  rocksdb_iterator_cb cb;
 
   void *data;
 };
 
-struct rocksdb_delete_range_s {
-  uv_work_t worker;
-
-  rocksdb_t *db;
-
-  rocksdb_range_t range;
-
-  char *error;
+typedef enum {
+  rocksdb_get = 1,
+} rocksdb_read_type_t;
 
-  rocksdb_delete_range_cb cb;
+struct rocksdb_read_s {
+  rocksdb_read_type_t type;
 
-  void *data;
+  union {
+    // For get
+    struct {
+      rocksdb_slice_t key;
+      rocksdb_slice_t value;
+    };
+  };
 };
 
-struct rocksdb_iterator_s {
+struct rocksdb_read_batch_s {
   uv_work_t worker;
 
   rocksdb_t *db;
 
-  void *handle; // Opaque iterator pointer
-
-  rocksdb_range_t range;
-  bool reverse;
-
   size_t len;
-  size_t capacity;
 
-  rocksdb_slice_t *keys;
-  rocksdb_slice_t *values;
+  rocksdb_read_t *reads;
 
-  char *error;
+  char **errors;
 
-  rocksdb_iterator_cb cb;
+  rocksdb_read_batch_cb cb;
 
   void *data;
 };
 
-struct rocksdb_batch_s {
+typedef enum {
+  rocksdb_put = 1,
+  rocksdb_delete,
+  rocksdb_delete_range,
+} rocksdb_write_type_t;
+
+struct rocksdb_write_s {
+  rocksdb_write_type_t type;
+
+  union {
+    // For put, delete
+    struct {
+      rocksdb_slice_t key;
+      rocksdb_slice_t value;
+    };
+
+    // For delete_range
+    struct {
+      rocksdb_slice_t start;
+      rocksdb_slice_t end;
+    };
+  };
+};
+
+struct rocksdb_write_batch_s {
   uv_work_t worker;
 
   rocksdb_t *db;
 
   size_t len;
 
-  const rocksdb_slice_t *keys;
-  rocksdb_slice_t *values;
-
-  union {
-    // A single error for the write or delete batch.
-    char *error;
+  rocksdb_write_t *writes;
 
-    // A list of errors for the read batch.
-    char **errors;
-  };
+  char *error;
 
-  rocksdb_batch_cb cb;
+  rocksdb_write_batch_cb cb;
 
   void *data;
 };
@@ -222,16 +236,7 @@ rocksdb_slice_t
 rocksdb_slice_empty (void);
 
 int
-rocksdb_read_range (rocksdb_t *db, rocksdb_read_range_t *req, rocksdb_range_t range, bool reverse, rocksdb_slice_t *keys, rocksdb_slice_t *values, size_t capacity, rocksdb_read_range_cb cb);
-
-int
-rocksdb_delete_range (rocksdb_t *db, rocksdb_delete_range_t *req, rocksdb_range_t range, rocksdb_delete_range_cb cb);
-
-int
-rocksdb_iterator_init (rocksdb_t *db, rocksdb_iterator_t *iterator);
-
-int
-rocksdb_iterator_open (rocksdb_iterator_t *iterator, rocksdb_range_t range, bool reverse, rocksdb_iterator_cb cb);
+rocksdb_iterator_open (rocksdb_t *db, rocksdb_iterator_t *iterator, rocksdb_range_t range, bool reverse, rocksdb_iterator_cb cb);
 
 int
 rocksdb_iterator_close (rocksdb_iterator_t *iterator, rocksdb_iterator_cb cb);
@@ -243,16 +248,10 @@ int
 rocksdb_iterator_read (rocksdb_iterator_t *iterator, rocksdb_slice_t *keys, rocksdb_slice_t *values, size_t capacity, rocksdb_iterator_cb cb);
 
 int
-rocksdb_batch_init (rocksdb_t *db, rocksdb_batch_t *batch);
-
-int
-rocksdb_batch_read (rocksdb_batch_t *batch, const rocksdb_slice_t *keys, rocksdb_slice_t *values, char **errors, size_t len, rocksdb_batch_cb cb);
-
-int
-rocksdb_batch_write (rocksdb_batch_t *batch, const rocksdb_slice_t *keys, const rocksdb_slice_t *values, size_t len, rocksdb_batch_cb cb);
+rocksdb_read (rocksdb_t *db, rocksdb_read_batch_t *batch, rocksdb_read_t *reads, char **errors, size_t len, rocksdb_read_batch_cb cb);
 
 int
-rocksdb_batch_delete (rocksdb_batch_t *batch, const rocksdb_slice_t *keys, const rocksdb_slice_t *values, size_t len, rocksdb_batch_cb cb);
+rocksdb_write (rocksdb_t *db, rocksdb_write_batch_t *batch, rocksdb_write_t *writes, size_t len, rocksdb_write_batch_cb cb);
 
 #ifdef __cplusplus
 }