rocksdb-native 3.3.0 → 3.4.0

package/CMakeLists.txt

@@ -13,7 +13,7 @@ if(target MATCHES "win32")
   add_compile_options(/MT$<$<CONFIG:Debug>:d>)
 endif()
 
-fetch_package("github:holepunchto/librocksdb#2b388ab")
+fetch_package("github:holepunchto/librocksdb#049c4c7")
 
 add_bare_module(rocksdb_native_bare)
 

package/binding.c

@@ -6,32 +6,6 @@
 #include <string.h>
 #include <utf.h>
 
-typedef struct {
-  uint32_t low;
-  uint32_t high;
-} rocksdb_native_uint64_t;
-
-typedef struct {
-  uint32_t read_only;
-  uint32_t create_if_missing;
-  uint32_t create_missing_column_families;
-  uint32_t max_background_jobs;
-  rocksdb_native_uint64_t bytes_per_sync;
-} rocksdb_native_options_t;
-
-typedef struct {
-  uint32_t compation_style;
-  uint32_t enable_blob_files;
-  rocksdb_native_uint64_t min_blob_size;
-  rocksdb_native_uint64_t blob_file_size;
-  uint32_t enable_blob_garbage_collection;
-  rocksdb_native_uint64_t table_block_size;
-  uint32_t table_cache_index_and_filter_blocks;
-  uint32_t table_format_version;
-  uint32_t optimize_filters_for_memory;
-  uint32_t no_block_cache;
-} rocksdb_native_column_family_options_t;
-
 typedef struct {
   rocksdb_column_family_t *handle;
   rocksdb_column_family_descriptor_t descriptor;
@@ -137,11 +111,6 @@ typedef struct {
   rocksdb_snapshot_t handle;
 } rocksdb_native_snapshot_t;
 
-static inline uint64_t
-rocksdb_native__to_uint64(rocksdb_native_uint64_t n) {
-  return n.high * 0x100000000 + n.low;
-}
-
 static void
 rocksdb_native__on_free(js_env_t *env, void *data, void *finalize_hint) {
   free(data);
@@ -325,18 +294,33 @@ static js_value_t *
 rocksdb_native_init(js_env_t *env, js_callback_info_t *info) {
   int err;
 
-  size_t argc = 1;
-  js_value_t *argv[1];
+  size_t argc = 7;
+  js_value_t *argv[7];
 
   err = js_get_callback_info(env, info, &argc, argv, NULL, NULL);
   assert(err == 0);
 
-  assert(argc == 1);
+  assert(argc == 7);
 
-  rocksdb_native_options_t *options;
-  err = js_get_typedarray_info(env, argv[0], NULL, (void **) &options, NULL, NULL, NULL);
+  typedef bool bool_t;
+
+#define V(name, type) \
+  assert(i < argc); \
+  type##_t name; \
+  err = js_get_value_##type(env, argv[i++], &name); \
   assert(err == 0);
 
+  int i = 0;
+
+  V(read_only, bool)
+  V(create_if_missing, bool)
+  V(create_missing_column_families, bool)
+  V(max_background_jobs, int32)
+  V(bytes_per_sync, int64)
+  V(max_open_files, int32)
+  V(use_direct_reads, bool)
+#undef V
+
   uv_loop_t *loop;
   err = js_get_env_loop(env, &loop);
   assert(err == 0);
@@ -352,12 +336,14 @@ rocksdb_native_init(js_env_t *env, js_callback_info_t *info) {
   db->exiting = false;
 
   db->options = (rocksdb_options_t) {
-    0,
-    options->read_only,
-    options->create_if_missing,
-    options->create_missing_column_families,
-    options->max_background_jobs,
-    rocksdb_native__to_uint64(options->bytes_per_sync),
+    1,
+    read_only,
+    create_if_missing,
+    create_missing_column_families,
+    max_background_jobs,
+    bytes_per_sync,
+    max_open_files,
+    use_direct_reads
   };
 
   err = rocksdb_init(loop, &db->handle);
@@ -682,13 +668,13 @@ static js_value_t *
 rocksdb_native_column_family_init(js_env_t *env, js_callback_info_t *info) {
   int err;
 
-  size_t argc = 2;
-  js_value_t *argv[2];
+  size_t argc = 13;
+  js_value_t *argv[13];
 
   err = js_get_callback_info(env, info, &argc, argv, NULL, NULL);
   assert(err == 0);
 
-  assert(argc == 2);
+  assert(argc >= 11 && argc <= 13);
 
   size_t name_len;
   err = js_get_value_string_utf8(env, argv[0], NULL, 0, &name_len);
@@ -700,10 +686,56 @@ rocksdb_native_column_family_init(js_env_t *env, js_callback_info_t *info) {
   err = js_get_value_string_utf8(env, argv[0], name, name_len, NULL);
   assert(err == 0);
 
-  rocksdb_native_column_family_options_t *options;
-  err = js_get_typedarray_info(env, argv[1], NULL, (void **) &options, NULL, NULL, NULL);
+  typedef bool bool_t;
+  typedef double double_t;
+
+#define V(name, type) \
+  assert(i < argc); \
+  type##_t name; \
+  err = js_get_value_##type(env, argv[i++], &name); \
   assert(err == 0);
 
+  int i = 1;
+
+  V(enable_blob_files, bool)
+  V(min_blob_size, int64)
+  V(blob_file_size, int64)
+  V(enable_blob_garbage_collection, bool)
+  V(table_block_size, int64)
+  V(table_cache_index_and_filter_blocks, bool)
+  V(table_format_version, uint32)
+  V(optimize_filters_for_memory, bool)
+  V(no_block_cache, bool)
+  V(filter_policy_type, uint32)
+
+  rocksdb_filter_policy_t filter_policy = {filter_policy_type};
+
+  switch (filter_policy_type) {
+  case rocksdb_bloom_filter_policy: {
+    V(bits_per_key, double)
+
+    filter_policy.bloom = (rocksdb_bloom_filter_options_t) {
+      0,
+      bits_per_key
+    };
+
+    break;
+  }
+  case rocksdb_ribbon_filter_policy: {
+    V(bloom_equivalent_bits_per_key, double)
+    V(bloom_before_level, int32)
+
+    filter_policy.ribbon = (rocksdb_ribbon_filter_options_t) {
+      0,
+      bloom_equivalent_bits_per_key,
+      bloom_before_level,
+    };
+
+    break;
+  }
+  }
+#undef V
+
   uv_loop_t *loop;
   err = js_get_env_loop(env, &loop);
   assert(err == 0);
@@ -721,17 +753,18 @@ rocksdb_native_column_family_init(js_env_t *env, js_callback_info_t *info) {
   column_family->descriptor = (rocksdb_column_family_descriptor_t) {
     (const char *) name,
     {
-      1,
-      options->compation_style,
-      options->enable_blob_files,
-      rocksdb_native__to_uint64(options->min_blob_size),
-      rocksdb_native__to_uint64(options->blob_file_size),
-      options->enable_blob_garbage_collection,
-      rocksdb_native__to_uint64(options->table_block_size),
-      options->table_cache_index_and_filter_blocks,
-      options->table_format_version,
-      options->optimize_filters_for_memory,
-      options->no_block_cache,
+      2,
+      rocksdb_level_compaction,
+      enable_blob_files,
+      min_blob_size,
+      blob_file_size,
+      enable_blob_garbage_collection,
+      table_block_size,
+      table_cache_index_and_filter_blocks,
+      table_format_version,
+      optimize_filters_for_memory,
+      no_block_cache,
+      filter_policy,
     }
   };
 

package/index.js

@@ -1,13 +1,14 @@
 const ColumnFamily = require('./lib/column-family')
 const Iterator = require('./lib/iterator')
 const Snapshot = require('./lib/snapshot')
-const DBState = require('./lib/state')
+const State = require('./lib/state')
+const { BloomFilterPolicy, RibbonFilterPolicy } = require('./lib/filter-policy')
 
 class RocksDB {
   constructor(path, opts = {}) {
     const {
       columnFamily,
-      state = new DBState(this, path, opts),
+      state = new State(this, path, opts),
       snapshot = null,
       keyEncoding = null,
       valueEncoding = null
@@ -174,6 +175,8 @@ class RocksDB {
 module.exports = exports = RocksDB
 
 exports.ColumnFamily = ColumnFamily
+exports.BloomFilterPolicy = BloomFilterPolicy
+exports.RibbonFilterPolicy = RibbonFilterPolicy
 
 function maybeClosed(db) {
   if (db._state.closing || db._index === -1)

package/lib/batch.js

@@ -20,6 +20,7 @@ class RocksDBBatch {
 
     this._request = null
     this._resolve = null
+    this._reject = null
 
     this._handle = null
     this._buffer = null
@@ -38,16 +39,20 @@ class RocksDBBatch {
     this._autoDestroy = autoDestroy
   }
 
-  _onfinished() {
+  _onfinished(err) {
     const resolve = this._resolve
+    const reject = this._reject
 
     this._operations = []
     this._promises = []
     this._request = null
     this._resolve = null
+    this._reject = null
 
     if (this._autoDestroy === true) this.destroy()
-    if (resolve !== null) resolve()
+
+    if (reject !== null && err) reject(err)
+    else if (resolve !== null) resolve()
   }
 
   _resize() {
@@ -92,12 +97,19 @@ class RocksDBBatch {
     }
   }
 
-  flush() {
+  async flush() {
     if (this._request) throw new Error('Request in progress')
     if (this._destroyed) throw new Error('Batch is destroyed')
 
-    this._request = new Promise((resolve) => {
+    const state = this._db._state
+
+    if (state._suspending || state._suspended) {
+      throw new Error('Database is suspended')
+    }
+
+    this._request = new Promise((resolve, reject) => {
       this._resolve = resolve
+      this._reject = reject
     })
 
     this._flush()
@@ -107,6 +119,13 @@ class RocksDBBatch {
 
   tryFlush() {
     if (this._request) throw new Error('Request in progress')
+    if (this._destroyed) throw new Error('Batch is destroyed')
+
+    const state = this._db._state
+
+    if (state._suspending || state._suspended) {
+      throw new Error('Database is suspended')
+    }
 
     this._request = resolved
 
@@ -166,21 +185,26 @@ exports.ReadBatch = class RocksDBReadBatch extends RocksDBBatch {
   }
 
   _onread(errs, values) {
+    let applied = true
+
     for (let i = 0, n = this._promises.length; i < n; i++) {
       const promise = this._promises[i]
       if (promise === null) continue
 
       const err = errs[i]
 
-      if (err) promise.reject(new Error(err))
-      else {
+      if (err) {
+        applied = false
+
+        promise.reject(new Error(err))
+      } else {
         promise.resolve(
           values[i] ? this._decodeValue(Buffer.from(values[i])) : null
         )
       }
     }
 
-    this._onfinished()
+    this._onfinished(applied ? null : new Error('Batch was not applied'))
   }
 
   get(key) {
@@ -227,15 +251,22 @@ exports.WriteBatch = class RocksDBWriteBatch extends RocksDBBatch {
   }
 
   _onwrite(err) {
+    let applied = true
+
     for (let i = 0, n = this._promises.length; i < n; i++) {
       const promise = this._promises[i]
       if (promise === null) continue
 
-      if (err) promise.reject(new Error(err))
-      else promise.resolve()
+      if (err) {
+        applied = false
+
+        promise.reject(new Error(err))
+      } else {
+        promise.resolve()
+      }
     }
 
-    this._onfinished()
+    this._onfinished(applied ? null : new Error('Batch was not applied'))
   }
 
   put(key, value) {

package/lib/column-family.js

@@ -1,4 +1,5 @@
 const binding = require('../binding')
+const { BloomFilterPolicy } = require('./filter-policy')
 
 class RocksDBColumnFamily {
   constructor(name, opts = {}) {
@@ -14,34 +15,59 @@ class RocksDBColumnFamily {
       tableFormatVersion = 6,
       optimizeFiltersForMemory = false,
       blockCache = true,
-      // In case we are cloning
-      settings = null
+      filterPolicy = new BloomFilterPolicy(10)
     } = opts
 
     this._name = name
-    this._settings =
-      settings ||
-      Uint32Array.from([
-        0,
-        enableBlobFiles ? 1 : 0,
-        minBlobSize & 0xffffffff,
-        Math.floor(minBlobSize / 0x100000000),
-        blobFileSize & 0xffffffff,
-        Math.floor(blobFileSize / 0x100000000),
-        enableBlobGarbageCollection ? 1 : 0,
-        tableBlockSize & 0xffffffff,
-        Math.floor(tableBlockSize / 0x100000000),
-        tableCacheIndexAndFilterBlocks ? 1 : 0,
-        tableFormatVersion,
-        optimizeFiltersForMemory ? 1 : 0,
-        blockCache ? 0 : 1
-      ])
-
-    this._handle = binding.columnFamilyInit(name, this._settings)
+    this._options = {
+      enableBlobFiles,
+      minBlobSize,
+      blobFileSize,
+      enableBlobGarbageCollection,
+      tableBlockSize,
+      tableCacheIndexAndFilterBlocks,
+      tableFormatVersion,
+      optimizeFiltersForMemory,
+      blockCache,
+      filterPolicy
+    }
+
+    const filterPolicyArguments = []
+
+    if (filterPolicy === null) filterPolicyArguments.push(0)
+    else {
+      filterPolicyArguments.push(filterPolicy.type)
+
+      switch (filterPolicy.type) {
+        case 1: // Bloom filter policy
+          filterPolicyArguments.push(filterPolicy.bitsPerKey)
+          break
+        case 2: // Ribbon filter policy
+          filterPolicyArguments.push(
+            filterPolicy.bloomEquivalentBitsPerKey,
+            filterPolicy.bloomBeforeLevel
+          )
+          break
+      }
+    }
+
+    this._handle = binding.columnFamilyInit(
+      name,
+      enableBlobFiles,
+      minBlobSize,
+      blobFileSize,
+      enableBlobGarbageCollection,
+      tableBlockSize,
+      tableCacheIndexAndFilterBlocks,
+      tableFormatVersion,
+      optimizeFiltersForMemory,
+      blockCache === false,
+      ...filterPolicyArguments
+    )
   }
 
   cloneSettings(name) {
-    return new RocksDBColumnFamily(name, { settings: this._settings })
+    return new RocksDBColumnFamily(name, this._options)
   }
 
   get name() {

package/lib/filter-policy.js

@@ -0,0 +1,20 @@
+exports.BloomFilterPolicy = class RocksDBBloomFilterPolicy {
+  get type() {
+    return 1
+  }
+
+  constructor(bitsPerKey) {
+    this.bitsPerKey = bitsPerKey
+  }
+}
+
+exports.RibbonFilterPolicy = class RocksDBRibbonFilterPolicy {
+  get type() {
+    return 2
+  }
+
+  constructor(bloomEquivalentBitsPerKey, bloomBeforeLevel = 0) {
+    this.bloomEquivalentBitsPerKey = bloomEquivalentBitsPerKey
+    this.bloomBeforeLevel = bloomBeforeLevel
+  }
+}

package/lib/iterator.js

@@ -97,6 +97,12 @@ module.exports = class RocksDBIterator extends Readable {
   async _open(cb) {
     await this.ready()
 
+    const state = this._db._state
+
+    if (state._suspending || state._suspended) {
+      return cb(new Error('Database is suspended'))
+    }
+
     this._pendingOpen = cb
 
     binding.iteratorOpen(
@@ -117,6 +123,12 @@ module.exports = class RocksDBIterator extends Readable {
   }
 
   _read(cb) {
+    const state = this._db._state
+
+    if (state._suspending || state._suspended) {
+      return cb(new Error('Database is suspended'))
+    }
+
     this._pendingRead = cb
 
     binding.iteratorRead(this._handle, Math.min(this._capacity, this._limit))

package/lib/state.js

@@ -6,7 +6,7 @@ const binding = require('../binding')
 
 const MAX_BATCH_REUSE = 64
 
-module.exports = class DBState extends ReadyResource {
+module.exports = class RocksDBState extends ReadyResource {
   constructor(db, path, opts) {
     super()
 
@@ -17,7 +17,9 @@ module.exports = class DBState extends ReadyResource {
       createIfMissing = true,
       createMissingColumnFamilies = true,
       maxBackgroundJobs = 6,
-      bytesPerSync = 1048576
+      bytesPerSync = 1048576,
+      maxOpenFiles = -1,
+      useDirectReads = false
     } = opts
 
     this.path = path
@@ -27,6 +29,7 @@ module.exports = class DBState extends ReadyResource {
     this.columnFamilies = [columnFamily]
     this.deferSnapshotInit = true
 
+    this._suspended = false
     this._suspending = null
     this._resuming = null
     this._columnsFlushed = false
@@ -40,14 +43,13 @@ module.exports = class DBState extends ReadyResource {
     }
 
     this.handle = binding.init(
-      Uint32Array.from([
-        readOnly ? 1 : 0,
-        createIfMissing ? 1 : 0,
-        createMissingColumnFamilies ? 1 : 0,
-        maxBackgroundJobs,
-        bytesPerSync & 0xffffffff,
-        Math.floor(bytesPerSync / 0x100000000)
-      ])
+      readOnly,
+      createIfMissing,
+      createMissingColumnFamilies,
+      maxBackgroundJobs,
+      bytesPerSync,
+      maxOpenFiles,
+      useDirectReads
     )
   }
 
@@ -126,6 +128,7 @@ module.exports = class DBState extends ReadyResource {
     })
 
     this._columnsFlushed = true
+
     req.handle = binding.open(
       this.handle,
       this,
@@ -191,9 +194,13 @@ module.exports = class DBState extends ReadyResource {
 
     req.handle = binding.suspend(this.handle, req, onsuspend)
 
-    await promise
+    try {
+      await promise
 
-    this._suspending = null
+      this._suspended = true
+    } finally {
+      this._suspending = null
+    }
 
     function onsuspend(err) {
       if (err) req.reject(new Error(err))
@@ -219,9 +226,13 @@ module.exports = class DBState extends ReadyResource {
 
     req.handle = binding.resume(this.handle, req, onresume)
 
-    await promise
+    try {
+      await promise
 
-    this._resuming = null
+      this._suspended = false
+    } finally {
+      this._resuming = null
+    }
 
     function onresume(err) {
       if (err) req.reject(new Error(err))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rocksdb-native",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "description": "librocksdb bindings for JavaScript",
   "exports": {
     ".": "./index.js",