rocksdb-native 3.0.2 → 3.1.1

package/index.js

@@ -1,4 +1,3 @@
-const { ReadBatch, WriteBatch } = require('./lib/batch')
 const ColumnFamily = require('./lib/column-family')
 const Iterator = require('./lib/iterator')
 const Snapshot = require('./lib/snapshot')
@@ -9,13 +8,19 @@ class RocksDB {
     const {
       columnFamily,
       state = new DBState(this, path, opts),
-      snapshot = null
+      snapshot = null,
+      keyEncoding = null,
+      valueEncoding = null
     } = opts
 
     this._state = state
     this._snapshot = snapshot
     this._columnFamily = state.getColumnFamily(columnFamily)
-    this._index = this._state.addSession(this)
+    this._keyEncoding = keyEncoding
+    this._valueEncoding = valueEncoding
+    this._index = -1
+
+    this._state.addSession(this)
   }
 
   get opened() {
@@ -36,25 +41,23 @@ class RocksDB {
 
   session({
     columnFamily = this._columnFamily,
-    snapshot = this._snapshot !== null
+    snapshot = this._snapshot !== null,
+    keyEncoding = this._keyEncoding,
+    valueEncoding = this._valueEncoding
   } = {}) {
-    let snap = null
-
-    if (snapshot) {
-      snap = this._snapshot
-      if (snap === null) snap = new Snapshot(this._state)
-      else snap.ref()
-    }
+    maybeClosed(this)
 
     return new RocksDB(null, {
       state: this._state,
       columnFamily,
-      snapshot: snap
+      snapshot: snapshot ? this._snapshot || new Snapshot(this._state) : null,
+      keyEncoding,
+      valueEncoding
     })
   }
 
-  columnFamily(name) {
-    return this.session({ columnFamily: name })
+  columnFamily(name, opts) {
+    return this.session({ ...opts, columnFamily: name })
   }
 
   snapshot() {
@@ -70,11 +73,7 @@ class RocksDB {
   }
 
   async close({ force } = {}) {
-    if (this._index !== -1) {
-      this._state.removeSession(this)
-      this._index = -1
-      if (this._snapshot) this._snapshot.unref()
-    }
+    if (this._index !== -1) this._state.removeSession(this)
 
     if (force) {
       for (let i = this._state.sessions.length - 1; i >= 0; i--) {
@@ -86,22 +85,28 @@ class RocksDB {
   }
 
   suspend() {
+    maybeClosed(this)
+
     return this._state.suspend()
   }
 
   resume() {
+    maybeClosed(this)
+
     return this._state.resume()
   }
 
   isIdle() {
-    return this._state.refs.isIdle()
+    return this._state.activity.isIdle()
   }
 
   idle() {
-    return this._state.refs.idle()
+    return this._state.activity.idle()
   }
 
   iterator(range, opts) {
+    maybeClosed(this)
+
     return new Iterator(this, { ...range, ...opts })
   }
 
@@ -114,54 +119,59 @@ class RocksDB {
   }
 
   read(opts) {
-    return new ReadBatch(this, opts)
+    maybeClosed(this)
+
+    return this._state.createReadBatch(this, opts)
   }
 
   write(opts) {
-    return new WriteBatch(this, opts)
+    maybeClosed(this)
+
+    return this._state.createWriteBatch(this, opts)
   }
 
   async get(key, opts) {
-    const batch = this.read({ ...opts, capacity: 1 })
-    try {
-      const value = batch.get(key)
-      batch.tryFlush()
-      return await value
-    } finally {
-      batch.destroy()
-    }
+    const batch = this.read({ ...opts, capacity: 1, autoDestroy: true })
+    const value = batch.get(key)
+    batch.tryFlush()
+    return value
   }
 
   async put(key, value, opts) {
-    const batch = this.write({ ...opts, capacity: 1 })
-    try {
-      batch.tryPut(key, value)
-      await batch.flush()
-    } finally {
-      batch.destroy()
-    }
+    const batch = this.write({ ...opts, capacity: 1, autoDestroy: true })
+    batch.tryPut(key, value)
+    await batch.flush()
   }
 
   async delete(key, opts) {
-    const batch = this.write({ ...opts, capacity: 1 })
-    try {
-      batch.tryDelete(key)
-      await batch.flush()
-    } finally {
-      batch.destroy()
-    }
+    const batch = this.write({ ...opts, capacity: 1, autoDestroy: true })
+    batch.tryDelete(key)
+    await batch.flush()
   }
 
   async deleteRange(start, end, opts) {
-    const batch = this.write({ ...opts, capacity: 1 })
-    try {
-      batch.tryDeleteRange(start, end)
-      await batch.flush()
-    } finally {
-      batch.destroy()
-    }
+    const batch = this.write({ ...opts, capacity: 1, autoDestroy: true })
+    batch.tryDeleteRange(start, end)
+    await batch.flush()
+  }
+
+  // used by iterators/batches to ensure no gc/close when active
+
+  _ref() {
+    if (this._snapshot) this._snapshot.ref()
+    this._state.activity.inc()
+  }
+
+  _unref() {
+    if (this._snapshot) this._snapshot.unref()
+    this._state.activity.dec()
   }
 }
 
 module.exports = exports = RocksDB
 exports.ColumnFamily = ColumnFamily
+
+function maybeClosed(db) {
+  if (db._state.closing || db._destroyed)
+    throw new Error('RocksDB session is closed')
+}

package/lib/batch.js

@@ -6,26 +6,16 @@ const resolved = Promise.resolve()
 
 class RocksDBBatch {
   constructor(db, opts = {}) {
-    const {
-      columnFamily = db.defaultColumnFamily,
-      capacity = 8,
-      encoding = null,
-      keyEncoding = encoding,
-      valueEncoding = encoding
-    } = opts
-
-    db._state.ref()
-
-    this._state = db._state
-    this._columnFamily = columnFamily
+    const { capacity = 8, autoDestroy = false } = opts
+
+    db._ref()
+
+    this._db = db
     this._destroyed = false
     this._capacity = capacity
     this._operations = []
     this._promises = []
 
-    this._keyEncoding = keyEncoding
-    this._valueEncoding = valueEncoding
-
     this._enqueuePromise = this._enqueuePromise.bind(this)
 
     this._request = null
@@ -33,8 +23,19 @@ class RocksDBBatch {
 
     this._handle = null
     this._buffer = null
+    this._autoDestroy = autoDestroy
+
+    if (db._state.opened === true) this.ready()
+  }
 
-    if (this._state.opened === true) this.ready()
+  _reuse(db, opts = {}) {
+    const { autoDestroy = false } = opts
+
+    db._ref()
+
+    this._db = db
+    this._destroyed = false
+    this._autoDestroy = autoDestroy
   }
 
   _onfinished() {
@@ -45,6 +46,7 @@ class RocksDBBatch {
     this._request = null
     this._resolve = null
 
+    if (this._autoDestroy === true) this.destroy()
     if (resolve !== null) resolve()
   }
 
@@ -61,7 +63,7 @@ class RocksDBBatch {
   async ready() {
     if (this._handle !== null) return
 
-    if (this._state.opened === false) await this._state.ready()
+    if (this._db._state.opened === false) await this._db._state.ready()
 
     this._init()
   }
@@ -72,7 +74,22 @@ class RocksDBBatch {
 
     this._destroyed = true
 
-    this._state.unref()
+    if (this._promises.length) this._abort()
+
+    this._db._unref()
+    this._onfree()
+  }
+
+  _onfree() {
+    this._db._state.freeBatch(this, false)
+    this._db = null
+  }
+
+  _abort() {
+    for (let i = 0; i < this._promises.length; i++) {
+      const promise = this._promises[i]
+      if (promise !== null) promise.reject(new Error('Batch is destroyed'))
+    }
   }
 
   flush() {
@@ -105,31 +122,25 @@ class RocksDBBatch {
   }
 
   _encodeKey(k) {
-    if (this._keyEncoding) return c.encode(this._keyEncoding, k)
+    if (this._db._keyEncoding) return c.encode(this._db._keyEncoding, k)
     if (typeof k === 'string') return Buffer.from(k)
     return k
   }
 
   _encodeValue(v) {
-    if (this._valueEncoding) return c.encode(this._valueEncoding, v)
+    if (this._db._valueEncoding) return c.encode(this._db._valueEncoding, v)
     if (v === null) return empty
     if (typeof v === 'string') return Buffer.from(v)
     return v
   }
 
   _decodeValue(b) {
-    if (this._valueEncoding) return c.decode(this._valueEncoding, b)
+    if (this._db._valueEncoding) return c.decode(this._db._valueEncoding, b)
     return b
   }
 }
 
 exports.ReadBatch = class RocksDBReadBatch extends RocksDBBatch {
-  constructor(db, opts) {
-    super(db, opts)
-
-    this._snapshot = db._snapshot
-  }
-
   _init() {
     this._handle = binding.readInit()
     this._buffer = binding.readBuffer(this._handle, this._capacity)
@@ -145,10 +156,10 @@ exports.ReadBatch = class RocksDBReadBatch extends RocksDBBatch {
     await super._flush()
 
     binding.read(
-      this._state.handle,
+      this._db._state.handle,
       this._handle,
       this._operations,
-      this._snapshot ? this._snapshot._handle : null,
+      this._db._snapshot ? this._db._snapshot._handle : null,
       this,
       this._onread
     )
@@ -179,7 +190,7 @@ exports.ReadBatch = class RocksDBReadBatch extends RocksDBBatch {
     const promise = new Promise(this._enqueuePromise)
 
     this._operations.push(
-      new RocksDBGet(this._encodeKey(key), this._columnFamily)
+      new RocksDBGet(this._encodeKey(key), this._db._columnFamily)
     )
 
     this._resize()
@@ -200,11 +211,15 @@ exports.WriteBatch = class RocksDBWriteBatch extends RocksDBBatch {
     }
   }
 
+  _onfree() {
+    this._db._state.freeBatch(this, true)
+  }
+
   async _flush() {
     await super._flush()
 
     binding.write(
-      this._state.handle,
+      this._db._state.handle,
       this._handle,
       this._operations,
       this,
@@ -233,7 +248,7 @@ exports.WriteBatch = class RocksDBWriteBatch extends RocksDBBatch {
       new RocksDBPut(
         this._encodeKey(key),
         this._encodeValue(value),
-        this._columnFamily
+        this._db._columnFamily
       )
     )
 
@@ -249,7 +264,7 @@ exports.WriteBatch = class RocksDBWriteBatch extends RocksDBBatch {
       new RocksDBPut(
         this._encodeKey(key),
         this._encodeValue(value),
-        this._columnFamily
+        this._db._columnFamily
       )
     )
 
@@ -264,7 +279,7 @@ exports.WriteBatch = class RocksDBWriteBatch extends RocksDBBatch {
     const promise = new Promise(this._enqueuePromise)
 
     this._operations.push(
-      new RocksDBDelete(this._encodeKey(key), this._columnFamily)
+      new RocksDBDelete(this._encodeKey(key), this._db._columnFamily)
     )
 
     this._resize()
@@ -276,7 +291,7 @@ exports.WriteBatch = class RocksDBWriteBatch extends RocksDBBatch {
     if (this._request) throw new Error('Request already in progress')
 
     this._operations.push(
-      new RocksDBDelete(this._encodeKey(key), this._columnFamily)
+      new RocksDBDelete(this._encodeKey(key), this._db._columnFamily)
     )
 
     this._promises.push(null)
@@ -293,7 +308,7 @@ exports.WriteBatch = class RocksDBWriteBatch extends RocksDBBatch {
       new RocksDBDeleteRange(
         this._encodeKey(start),
         this._encodeKey(end),
-        this._columnFamily
+        this._db._columnFamily
       )
     )
 
@@ -309,7 +324,7 @@ exports.WriteBatch = class RocksDBWriteBatch extends RocksDBBatch {
       new RocksDBDeleteRange(
         this._encodeKey(start),
         this._encodeKey(end),
-        this._columnFamily
+        this._db._columnFamily
       )
     )
 

package/lib/iterator.js

@@ -7,28 +7,20 @@ const empty = Buffer.alloc(0)
 module.exports = class RocksDBIterator extends Readable {
   constructor(db, opts = {}) {
     const {
-      columnFamily = db.defaultColumnFamily,
       gt = null,
       gte = null,
       lt = null,
       lte = null,
       reverse = false,
       limit = Infinity,
-      capacity = 8,
-      encoding = null,
-      keyEncoding = encoding,
-      valueEncoding = encoding
+      capacity = 8
     } = opts
 
     super()
 
-    db._state.ref()
+    db._ref()
 
-    this._state = db._state
-    this._columnFamily = columnFamily
-
-    this._keyEncoding = keyEncoding
-    this._valueEncoding = valueEncoding
+    this._db = db
 
     this._gt = gt ? this._encodeKey(gt) : empty
     this._gte = gte ? this._encodeKey(gte) : empty
@@ -38,7 +30,6 @@ module.exports = class RocksDBIterator extends Readable {
     this._reverse = reverse
     this._limit = limit < 0 ? Infinity : limit
     this._capacity = capacity
-    this._snapshot = db._snapshot
 
     this._pendingOpen = null
     this._pendingRead = null
@@ -47,7 +38,7 @@ module.exports = class RocksDBIterator extends Readable {
     this._buffer = null
     this._handle = null
 
-    if (this._state.opened === true) this.ready()
+    if (this._db._state.opened === true) this.ready()
   }
 
   _onopen(err) {
@@ -80,7 +71,7 @@ module.exports = class RocksDBIterator extends Readable {
   _onclose(err) {
     const cb = this._pendingDestroy
     this._pendingDestroy = null
-    this._state.unref()
+    this._db._unref()
     cb(err)
   }
 
@@ -93,7 +84,7 @@ module.exports = class RocksDBIterator extends Readable {
   async ready() {
     if (this._handle !== null) return
 
-    if (this._state.opened === false) await this._state.ready()
+    if (this._db._state.opened === false) await this._db._state.ready()
 
     this._init()
   }
@@ -109,15 +100,15 @@ module.exports = class RocksDBIterator extends Readable {
     this._pendingOpen = cb
 
     binding.iteratorOpen(
-      this._state.handle,
+      this._db._state.handle,
       this._handle,
-      this._columnFamily._handle,
+      this._db._columnFamily._handle,
       this._gt,
       this._gte,
       this._lt,
       this._lte,
       this._reverse,
-      this._snapshot ? this._snapshot._handle : null,
+      this._db._snapshot ? this._db._snapshot._handle : null,
       this,
       this._onopen,
       this._onclose,
@@ -140,18 +131,21 @@ module.exports = class RocksDBIterator extends Readable {
   }
 
   _encodeKey(k) {
-    if (this._keyEncoding) return c.encode(this._keyEncoding, k)
+    if (this._db._keyEncoding !== null)
+      return c.encode(this._db._keyEncoding, k)
     if (typeof k === 'string') return Buffer.from(k)
     return k
   }
 
   _decodeKey(b) {
-    if (this._keyEncoding) return c.decode(this._keyEncoding, b)
+    if (this._db._keyEncoding !== null)
+      return c.decode(this._db._keyEncoding, b)
     return b
   }
 
   _decodeValue(b) {
-    if (this._valueEncoding) return c.decode(this._valueEncoding, b)
+    if (this._db._valueEncoding !== null)
+      return c.decode(this._db._valueEncoding, b)
     return b
   }
 }

package/lib/snapshot.js

@@ -3,12 +3,11 @@ const binding = require('../binding')
 module.exports = class RocksDBSnapshot {
   constructor(state) {
     this._state = state
-    this._state.snapshots.add(this)
 
     this._handle = null
-    this._refs = 1
+    this._refs = 0 // starts unreffed, easier, must be reffed in first tick
 
-    if (state.opened === true) this._init()
+    if (state.deferSnapshotInit === false) this._init()
   }
 
   _init() {
@@ -22,8 +21,6 @@ module.exports = class RocksDBSnapshot {
   unref() {
     if (--this._refs > 0) return
 
-    this._state.snapshots.delete(this)
-
     if (this._handle === null) return
 
     binding.snapshotDestroy(this._handle)

package/lib/state.js

@@ -1,8 +1,11 @@
 const ReadyResource = require('ready-resource')
 const RefCounter = require('refcounter')
+const { ReadBatch, WriteBatch } = require('./batch')
 const ColumnFamily = require('./column-family')
 const binding = require('../binding')
 
+const MAX_BATCH_REUSE = 64
+
 module.exports = class DBState extends ReadyResource {
   constructor(db, path, opts) {
     super()
@@ -19,15 +22,16 @@ module.exports = class DBState extends ReadyResource {
 
     this.path = path
     this.db = db
-    this.refs = new RefCounter()
-    this.sessionRefs = new RefCounter()
+    this.activity = new RefCounter()
     this.sessions = []
     this.columnFamilies = [columnFamily]
-    this.snapshots = new Set()
+    this.deferSnapshotInit = true
 
     this._suspending = null
     this._resuming = null
     this._columnsFlushed = false
+    this._readBatches = []
+    this._writeBatches = []
 
     for (const col of columnFamilies) {
       this.columnFamilies.push(
@@ -47,24 +51,36 @@ module.exports = class DBState extends ReadyResource {
     )
   }
 
-  addSession(db) {
-    this.sessionRefs.inc()
-    return this.sessions.push(db) - 1
+  createReadBatch(db, opts) {
+    if (this._readBatches.length === 0) return new ReadBatch(db, opts)
+    const batch = this._readBatches.pop()
+    batch._reuse(db, opts)
+    return batch
   }
 
-  removeSession(db) {
-    this.sessionRefs.dec()
-    const head = this.sessions.pop()
-    if (head !== db) this.sessions[(head._index = db._index)] = head
+  createWriteBatch(db, opts) {
+    if (this._writeBatches.length === 0) return new WriteBatch(db, opts)
+    const batch = this._writeBatches.pop()
+    batch._reuse(db, opts)
+    return batch
   }
 
-  ref() {
-    if (this.closing) throw new Error('Database closed')
-    this.refs.inc()
+  freeBatch(batch, writable) {
+    const queue = writable ? this._writeBatches : this._readBatches
+    if (queue.length >= MAX_BATCH_REUSE) return
+    queue.push(batch)
   }
 
-  unref() {
-    this.refs.dec()
+  addSession(db) {
+    db._index = this.sessions.push(db) - 1
+    if (db._snapshot) db._snapshot.ref()
+  }
+
+  removeSession(db) {
+    const head = this.sessions.pop()
+    if (head !== db) this.sessions[(head._index = db._index)] = head
+    db._index = -1
+    if (db._snapshot) db._snapshot.unref()
   }
 
   upsertColumnFamily(c) {
@@ -121,7 +137,10 @@ module.exports = class DBState extends ReadyResource {
 
     await promise
 
-    for (const snapshot of this.snapshots) snapshot._init()
+    this.deferSnapshotInit = false
+    for (const session of this.sessions) {
+      if (session._snapshot) session._snapshot._init()
+    }
 
     function onopen(err) {
       if (err) req.reject(new Error(err))
@@ -130,13 +149,11 @@ module.exports = class DBState extends ReadyResource {
   }
 
   async _close() {
-    while (!this.refs.isIdle() || !this.sessionRefs.isIdle()) {
-      await this.refs.idle()
-      await this.sessionRefs.idle()
-    }
+    while (!this.activity.isIdle()) await this.activity.idle()
+    for (let i = this.sessions.length - 1; i >= 0; i--)
+      await this.sessions[i].close()
 
     for (const columnFamily of this.columnFamilies) columnFamily.destroy()
-    for (const snapshot of this.snapshots) snapshot.destroy()
 
     const req = { resolve: null, reject: null, handle: null }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rocksdb-native",
-  "version": "3.0.2",
+  "version": "3.1.1",
   "description": "librocksdb bindings for JavaScript",
   "exports": {
     ".": "./index.js",