ueberdb2 2.1.0 → 2.2.2

package/index.js

@@ -18,24 +18,12 @@
  */
 
 const cacheAndBufferLayer = require('./lib/CacheAndBufferLayer');
+const logging = require('./lib/logging');
 const util = require('util');
 
-// Returns a logger derived from the given logger (which may be null) that has debug() and
-// isDebugEnabled() methods.
-const normalizeLogger = (logger) => {
-  const logLevelsUsed = ['debug', 'error'];
-  logger = Object.create(logger || {});
-  for (const level of logLevelsUsed) {
-    const enabledFnName = `is${level.charAt(0).toUpperCase() + level.slice(1)}Enabled`;
-    if (typeof logger[level] !== 'function') {
-      logger[level] = () => {};
-      logger[enabledFnName] = () => false;
-    } else if (typeof logger[enabledFnName] !== 'function') {
-      logger[enabledFnName] = () => true;
-    }
-  }
-  return logger;
-};
+const cbDb = {};
+const fns = ['close', 'findKeys', 'flush', 'get', 'getSub', 'init', 'remove', 'set', 'setSub'];
+for (const fn of fns) cbDb[fn] = util.callbackify(cacheAndBufferLayer.Database.prototype[fn]);
 
 const makeDoneCallback = (callback, deprecated) => (err) => {
   if (callback) callback(err);
@@ -62,8 +50,9 @@ exports.Database = class {
     this.dbModule = require(`./databases/${type}_db`);
     this.dbSettings = dbSettings;
     this.wrapperSettings = wrapperSettings;
-    this.logger = normalizeLogger(logger);
+    this.logger = logging.normalizeLogger(logger);
     const db = new this.dbModule.Database(this.dbSettings);
+    db.logger = this.logger;
     this.db = new cacheAndBufferLayer.Database(db, this.wrapperSettings, this.logger);
 
     // Expose the cache wrapper's metrics to the user. See lib/CacheAndBufferLayer.js for details.
@@ -73,12 +62,12 @@ exports.Database = class {
     this.metrics = this.db.metrics;
   }
 
-  init(callback) {
-    if (callback) {
-      util.callbackify(this.db.init.bind(this.db))(callback);
-    } else {
-      return this.db.init();
-    }
+  /**
+   * @param callback - Deprecated. Node-style callback. If null, a Promise is returned.
+   */
+  init(callback = null) {
+    if (callback != null) return cbDb.init.call(this.db, callback);
+    return this.db.init();
   }
 
   /**
@@ -87,104 +76,95 @@ exports.Database = class {
 
   /**
    * Deprecated synonym of flush().
+   *
+   * @param callback - Deprecated. Node-style callback. If null, a Promise is returned.
    */
-  doShutdown(callback) {
-    this.flush(callback);
+  doShutdown(callback = null) {
+    return this.flush(callback);
   }
 
   /**
    * Writes any unsaved changes to the underlying database.
+   *
+   * @param callback - Deprecated. Node-style callback. If null, a Promise is returned.
    */
-  flush(callback) {
-    util.callbackify(this.db.flush.bind(this.db))(callback);
+  flush(callback = null) {
+    if (callback != null) return cbDb.flush.call(this.db, callback);
+    return this.db.flush();
   }
 
-  get(key, callback) {
-    util.callbackify(this.db.get.bind(this.db))(key, (err, val) => callback(err, clone(val)));
+  /**
+   * @param callback - Deprecated. Node-style callback. If null, a Promise is returned.
+   */
+  get(key, callback = null) {
+    if (callback != null) return cbDb.get.call(this.db, key, callback);
+    return this.db.get(key);
   }
 
-  findKeys(key, notKey, callback) {
-    util.callbackify(this.db.findKeys.bind(this.db))(key, notKey, (e, v) => callback(e, clone(v)));
+  /**
+   * @param callback - Deprecated. Node-style callback. If null, a Promise is returned.
+   */
+  findKeys(key, notKey, callback = null) {
+    if (callback != null) return cbDb.findKeys.call(this.db, key, notKey, callback);
+    return this.db.findKeys(key, notKey);
   }
 
   /**
    * Removes an entry from the database if present.
    *
-   * @param cb Called when the write has been committed to the underlying database driver.
-   * @param deprecated Deprecated callback that is called just after cb.
+   * @param cb Deprecated. Node-style callback. Called when the write has been committed to the
+   *     underlying database driver. If null, a Promise is returned.
+   * @param deprecated Deprecated callback that is called just after cb. Ignored if cb is null.
    */
-  remove(key, cb, deprecated = null) {
-    util.callbackify(this.db.remove.bind(this.db))(key, makeDoneCallback(cb, deprecated));
+  remove(key, cb = null, deprecated = null) {
+    if (cb != null) return cbDb.remove.call(this.db, key, makeDoneCallback(cb, deprecated));
+    return this.db.remove(key);
   }
 
   /**
    * Adds or changes the value of an entry.
    *
-   * @param cb Called when the write has been committed to the underlying database driver.
-   * @param deprecated Deprecated callback that is called just after cb.
+   * @param cb Deprecated. Node-style callback. Called when the write has been committed to the
+   *     underlying database driver. If null, a Promise is returned.
+   * @param deprecated Deprecated callback that is called just after cb. Ignored if cb is null.
    */
-  set(key, value, cb, deprecated = null) {
-    util.callbackify(this.db.set.bind(this.db))(
-        key, clone(value), makeDoneCallback(cb, deprecated));
+  set(key, value, cb = null, deprecated = null) {
+    if (cb != null) return cbDb.set.call(this.db, key, value, makeDoneCallback(cb, deprecated));
+    return this.db.set(key, value);
   }
 
-  getSub(key, sub, callback) {
-    util.callbackify(this.db.getSub.bind(this.db))(
-        key, sub, (err, val) => callback(err, clone(val)));
+  /**
+   * @param callback - Deprecated. Node-style callback. If null, a Promise is returned.
+   */
+  getSub(key, sub, callback = null) {
+    if (callback != null) return cbDb.getSub.call(this.db, key, sub, callback);
+    return this.db.getSub(key, sub);
   }
 
   /**
    * Adds or changes a subvalue of an entry.
    *
-   * @param cb Called when the write has been committed to the underlying database driver.
-   * @param deprecated Deprecated callback that is called just after cb.
+   * @param cb Deprecated. Node-style callback. Called when the write has been committed to the
+   *     underlying database driver. If null, a Promise is returned.
+   * @param deprecated Deprecated callback that is called just after cb. Ignored if cb is null.
    */
-  setSub(key, sub, value, cb, deprecated = null) {
-    util.callbackify(this.db.setSub.bind(this.db))(
-        key, sub, clone(value), makeDoneCallback(cb, deprecated));
+  setSub(key, sub, value, cb = null, deprecated = null) {
+    if (cb != null) {
+      return cbDb.setSub.call(this.db, key, sub, value, makeDoneCallback(cb, deprecated));
+    }
+    return this.db.setSub(key, sub, value);
   }
 
   /**
    * Flushes unwritten changes then closes the connection to the underlying database. After this
    * returns, any future call to a method on this object may result in an error.
+   *
+   * @param callback - Deprecated. Node-style callback. If null, a Promise is returned.
    */
-  close(callback) {
-    util.callbackify(this.db.close.bind(this.db))(callback);
-  }
-};
-
-const clone = (obj, key = '') => {
-  // Handle the 3 simple types, and null or undefined
-  if (null == obj || 'object' !== typeof obj) return obj;
-
-  if (typeof obj.toJSON === 'function') return clone(obj.toJSON(key));
-
-  // Handle Date
-  if (obj instanceof Date) {
-    const copy = new Date();
-    copy.setTime(obj.getTime());
-    return copy;
+  close(callback = null) {
+    if (callback != null) return cbDb.close.call(this.db, callback);
+    return this.db.close();
   }
-
-  // Handle Array
-  if (obj instanceof Array) {
-    const copy = [];
-    for (let i = 0, len = obj.length; i < len; ++i) {
-      copy[i] = clone(obj[i], String(i));
-    }
-    return copy;
-  }
-
-  // Handle Object
-  if (obj instanceof Object) {
-    const copy = {};
-    for (const attr in obj) {
-      if (Object.prototype.hasOwnProperty.call(obj, attr)) copy[attr] = clone(obj[attr], attr);
-    }
-    return copy;
-  }
-
-  throw new Error("Unable to copy obj! Its type isn't supported.");
 };
 
 /**

package/lib/AbstractDatabase.js

@@ -1,5 +1,9 @@
 'use strict';
 
+const logging = require('./logging');
+
+const nullLogger = logging.normalizeLogger(null);
+
 module.exports = class AbstractDatabase {
   constructor() {
     if (new.target === module.exports) {
@@ -8,6 +12,7 @@ module.exports = class AbstractDatabase {
     for (const fn of ['init', 'close', 'get', 'findKeys', 'remove', 'set']) {
       if (typeof this[fn] !== 'function') throw new TypeError(`method ${fn} not defined`);
     }
+    this.logger = nullLogger;
   }
 
   /**

package/lib/CacheAndBufferLayer.js

@@ -165,6 +165,11 @@ exports.Database = class {
     //     underlying database and we are awaiting commit.
     this.buffer = new LRU(this.settings.cache, (k, v) => !v.dirty && !v.writingInProgress);
 
+    // Either null if flushing is currently allowed, or a Promise that will resolve when it is OK to
+    // start flushing. The Promise has a `count` property that tracks the number of operations that
+    // are currently preventing flush() from running.
+    this._flushPaused = null;
+
     // Maps database key to a Promise that is resolved when the record is unlocked.
     this._locks = new Map();
 
@@ -245,6 +250,33 @@ exports.Database = class {
     this._locks.delete(key);
   }
 
+  // Block flush() until _resumeFlush() is called. This is needed so that a call to flush() after a
+  // write (set(), setSub(), or remove() call) in the same ECMAScript macro- or microtask will see
+  // the enqueued write and flush it.
+  //
+  // An alternative would be to change flush() to schedule its actions in a future microtask after
+  // the write has been queued in the buffer, but:
+  //
+  //   * That would be fragile: Every use of await moves the subsequent processing to a new
+  //     microtask, so flush() would need to do a number of `await Promise.resolve();` calls equal
+  //     to the number of awaits before a write is actually buffered.
+  //
+  //   * It won't work for setSub() because it must wait for a read to complete before it buffers
+  //     the write.
+  _pauseFlush() {
+    if (this._flushPaused == null) {
+      this._flushPaused = new SelfContainedPromise();
+      this._flushPaused.count = 0;
+    }
+    ++this._flushPaused.count;
+  }
+
+  _resumeFlush() {
+    if (--this._flushPaused.count > 0) return;
+    this._flushPaused.done();
+    this._flushPaused = null;
+  }
+
   /**
    * wraps the init function of the original DB
    */
@@ -266,12 +298,14 @@ exports.Database = class {
    * Gets the value trough the wrapper.
    */
   async get(key) {
+    let v;
     await this._lock(key);
     try {
-      return await this._getLocked(key);
+      v = await this._getLocked(key);
     } finally {
       this._unlock(key);
     }
+    return clone(v);
   }
 
   async _getLocked(key) {
@@ -334,12 +368,13 @@ exports.Database = class {
    * returns the key entries via callback.
    */
   async findKeys(key, notKey) {
-    const bufferKey = `${key}-${notKey}`;
+    await this.flush();
     const keyValues = await this.wrappedDB.findKeys(key, notKey);
     if (this.logger.isDebugEnabled()) {
-      this.logger.debug(`GET    - ${bufferKey} - ${JSON.stringify(keyValues)} - from database `);
+      this.logger.debug(
+          `GET    - ${key}-${notKey} - ${JSON.stringify(keyValues)} - from database `);
     }
-    return keyValues;
+    return clone(keyValues);
   }
 
   /**
@@ -354,9 +389,19 @@ exports.Database = class {
    * Sets the value trough the wrapper
    */
   async set(key, value) {
-    await this._lock(key);
-    const p = this._setLocked(key, value);
-    this._unlock(key);
+    value = clone(value);
+    let p;
+    this._pauseFlush();
+    try {
+      await this._lock(key);
+      try {
+        p = this._setLocked(key, value);
+      } finally {
+        this._unlock(key);
+      }
+    } finally {
+      this._resumeFlush();
+    }
     await p;
   }
 
@@ -406,44 +451,53 @@ exports.Database = class {
    * Sets a subvalue
    */
   async setSub(key, sub, value) {
+    value = clone(value);
     if (this.logger.isDebugEnabled()) {
       this.logger.debug(`SETSUB - ${key}${JSON.stringify(sub)} - ${JSON.stringify(value)}`);
     }
     let p;
-    await this._lock(key);
+    this._pauseFlush();
     try {
-      let base;
+      await this._lock(key);
       try {
-        const fullValue = await this._getLocked(key);
-        base = {fullValue};
-        // Emulate a pointer to the property that should be set to `value`.
-        const ptr = {obj: base, prop: 'fullValue'};
-        for (let i = 0; i < sub.length; i++) {
-          let o = ptr.obj[ptr.prop];
-          if (o == null) ptr.obj[ptr.prop] = o = {};
-          // If o is a primitive (string, number, etc.), then setting `o.foo` has no effect because
-          // ECMAScript automatically wraps primitives in a temporary wrapper object.
-          if (typeof o !== 'object') {
-            throw new TypeError(
-                `Cannot set property ${JSON.stringify(sub[i])} on non-object ` +
+        let base;
+        try {
+          const fullValue = await this._getLocked(key);
+          base = {fullValue};
+          // Emulate a pointer to the property that should be set to `value`.
+          const ptr = {obj: base, prop: 'fullValue'};
+          for (let i = 0; i < sub.length; i++) {
+            if (sub[i] === '__proto__') {
+              throw new Error('Modifying object prototype is not supported for security reasons');
+            }
+            let o = ptr.obj[ptr.prop];
+            if (o == null) ptr.obj[ptr.prop] = o = {};
+            // If o is a primitive (string, number, etc.), then setting `o.foo` has no effect
+            // because ECMAScript automatically wraps primitives in a temporary wrapper object.
+            if (typeof o !== 'object') {
+              throw new TypeError(
+                  `Cannot set property ${JSON.stringify(sub[i])} on non-object ` +
                   `${JSON.stringify(o)} (key: ${JSON.stringify(key)} ` +
                   `value in db: ${JSON.stringify(fullValue)} ` +
                   `sub: ${JSON.stringify(sub.slice(0, i + 1))})`);
+            }
+            ptr.obj = ptr.obj[ptr.prop];
+            ptr.prop = sub[i];
           }
-          ptr.obj = ptr.obj[ptr.prop];
-          ptr.prop = sub[i];
+          ptr.obj[ptr.prop] = value;
+        } catch (err) {
+          // this._setLocked() will not be called but it should still count as a write failure.
+          ++this.metrics.writes;
+          ++this.metrics.writesFailed;
+          ++this.metrics.writesFinished;
+          throw err;
         }
-        ptr.obj[ptr.prop] = value;
-      } catch (err) {
-        // this._setLocked() will not be called but it should still count as a write failure.
-        ++this.metrics.writes;
-        ++this.metrics.writesFailed;
-        ++this.metrics.writesFinished;
-        throw err;
+        p = this._setLocked(key, base.fullValue);
+      } finally {
+        this._unlock(key);
       }
-      p = this._setLocked(key, base.fullValue);
     } finally {
-      this._unlock(key);
+      this._resumeFlush();
     }
     await p;
   }
@@ -454,13 +508,14 @@ exports.Database = class {
    *     "bla"]
    */
   async getSub(key, sub) {
+    let subvalue;
     await this._lock(key);
     try {
       // get the full value
       const value = await this._getLocked(key);
 
       // everything is correct, navigate to the subvalue and return it
-      let subvalue = value;
+      subvalue = value;
 
       for (let i = 0; i < sub.length; i++) {
         // test if the subvalue exist
@@ -476,10 +531,10 @@ exports.Database = class {
       if (this.logger.isDebugEnabled()) {
         this.logger.debug(`GETSUB - ${key}${JSON.stringify(sub)} - ${JSON.stringify(subvalue)}`);
       }
-      return subvalue;
     } finally {
       this._unlock(key);
     }
+    return clone(subvalue);
   }
 
   /**
@@ -489,6 +544,7 @@ exports.Database = class {
     if (this._flushDone == null) {
       this._flushDone = (async () => {
         while (true) {
+          while (this._flushPaused != null) await this._flushPaused;
           const dirtyEntries = [];
           for (const entry of this.buffer) {
             if (entry[1].dirty && !entry[1].writingInProgress) {
@@ -573,10 +629,12 @@ exports.Database = class {
   }
 };
 
-const clone = (obj) => {
+const clone = (obj, key = '') => {
   // Handle the 3 simple types, and null or undefined
   if (null == obj || 'object' !== typeof obj) return obj;
 
+  if (typeof obj.toJSON === 'function') return clone(obj.toJSON(key));
+
   // Handle Date
   if (obj instanceof Date) {
     const copy = new Date();
@@ -588,7 +646,7 @@ const clone = (obj) => {
   if (obj instanceof Array) {
     const copy = [];
     for (let i = 0, len = obj.length; i < len; ++i) {
-      copy[i] = clone(obj[i]);
+      copy[i] = clone(obj[i], String(i));
     }
     return copy;
   }
@@ -597,7 +655,7 @@ const clone = (obj) => {
   if (obj instanceof Object) {
     const copy = {};
     for (const attr in obj) {
-      if (Object.prototype.hasOwnProperty.call(obj, attr)) copy[attr] = clone(obj[attr]);
+      if (Object.prototype.hasOwnProperty.call(obj, attr)) copy[attr] = clone(obj[attr], attr);
     }
     return copy;
   }

package/lib/logging.js

@@ -0,0 +1,29 @@
+'use strict';
+
+const {Console} = require('console');
+const {stdout, stderr} = require('process');
+
+class ConsoleLogger extends Console {
+  constructor(opts = {}) { super({stdout, stderr, inspectOptions: {depth: Infinity}, ...opts}); }
+  isDebugEnabled() { return false; }
+  isInfoEnabled() { return true; }
+  isWarnEnabled() { return true; }
+  isErrorEnabled() { return true; }
+}
+
+exports.ConsoleLogger = ConsoleLogger;
+
+exports.normalizeLogger = (logger) => {
+  const logLevelsUsed = ['debug', 'info', 'warn', 'error'];
+  logger = Object.create(logger || {});
+  for (const level of logLevelsUsed) {
+    const enabledFnName = `is${level.charAt(0).toUpperCase() + level.slice(1)}Enabled`;
+    if (typeof logger[level] !== 'function') {
+      logger[level] = () => {};
+      logger[enabledFnName] = () => false;
+    } else if (typeof logger[enabledFnName] !== 'function') {
+      logger[enabledFnName] = () => true;
+    }
+  }
+  return logger;
+};

package/package.json

@@ -21,24 +21,24 @@
     }
   ],
   "dependencies": {
-    "async": "^3.2.2",
+    "async": "^3.2.3",
     "cassandra-driver": "^4.6.3",
     "dirty": "^1.1.3",
-    "elasticsearch": "^16.7.2",
+    "elasticsearch": "^16.7.3",
     "mongodb": "^3.7.3",
     "mssql": "^8.1.0",
     "mysql": "2.18.1",
-    "nano": "^9.0.5",
-    "pg": "^8.7.1",
+    "nano": "^10.0.0",
+    "pg": "^8.7.3",
     "redis": "^3.1.2",
     "rethinkdb": "^2.4.2",
-    "simple-git": "^3.6.0"
+    "simple-git": "^3.7.1"
   },
   "optionalDependencies": {
-    "sqlite3": "github:mapbox/node-sqlite3#593c9d498be2510d286349134537e3bf89401c4a"
+    "sqlite3": "^5.0.6"
   },
   "devDependencies": {
-    "cli-table": "^0.3.8",
+    "cli-table": "^0.3.11",
     "eslint": "^7.32.0",
     "eslint-config-etherpad": "^2.0.3",
     "eslint-plugin-cypress": "^2.12.1",
@@ -57,7 +57,7 @@
     "url": "https://github.com/ether/ueberDB.git"
   },
   "main": "./index",
-  "version": "2.1.0",
+  "version": "2.2.2",
   "bugs": {
     "url": "https://github.com/ether/ueberDB/issues"
   },
@@ -117,6 +117,6 @@
     "root": true
   },
   "engines": {
-    "node": "^10.17.0 || >=11.14.0"
+    "node": ">=14.15.0"
   }
 }

package/test/lib/databases.js

@@ -51,13 +51,14 @@ exports.databases = {
       removeMax: 0.3,
     },
   },
-  /* Disabled due to "Document update conflict" error in findKeys()
   couch: {
     host: '127.0.0.1',
     port: 5984,
     database: 'ueberdb',
     user: 'ueberdb',
     password: 'ueberdb',
+    speeds: {
+      findKeysMax: 30,
+    },
   },
-  */
 };