ueberdb2 1.4.16

package/README.md

@@ -0,0 +1,356 @@
+# UeberDB2: Abstract your databases
+
+## About
+
+✓ UeberDB turns every database into a simple key value store by providing a
+layer of abstraction between your software and your database.
+
+✓ UeberDB uses a cache and buffer to make databases faster. Reads are cached and
+writes are done in a bulk. This can be turned off.
+
+✓ UeberDB does bulk writing ergo reduces the overhead of database transactions.
+
+✓ UeberDB uses a simple and clean syntax ergo getting started is easy.
+
+## Database Support
+
+* Couch
+* Dirty
+* Elasticsearch
+* Maria
+* Mongo
+* MsSQL
+* MySQL
+* Postgres (single connection and with connection pool)
+* Redis
+* Rethink
+* SQLite
+
+## Install
+
+```
+npm install ueberdb2
+```
+
+## Examples
+
+### Basic
+
+```javascript
+const ueberdb = require('ueberdb2');
+
+// mysql
+const db = new ueberdb.Database('mysql', {
+  user: 'root',
+  host: 'localhost',
+  password: '',
+  database: 'store',
+  engine: 'InnoDB',
+});
+
+// dirty to file system
+//const db = new ueberdb.Database('dirty', {filename: 'var/dirty.db'});
+
+async function example(db) {
+  await db.init();
+
+  // no need for await because it's already in cache.
+  db.set('valueA', {a: 1, b: 2});
+
+  db.get('valueA', function (err, value) {
+    // close the database connection.
+    db.close(function () {
+      process.exit(0);
+    });
+  });
+}
+
+example(db);
+```
+
+### findKeys
+
+```javascript
+const ueberdb = require('ueberdb2');
+const db = new ueberdb.Database('dirty', {filename: 'var/dirty.db'});
+
+async function example(db){
+  await db.init();
+
+  // no need for await because it's already in cache.
+  db.set('valueA', {a: 1, b: 2});
+  db.set('valueA:h1', {a: 1, b: 2});
+  db.set('valueA:h2', {a: 3, b: 4});
+
+  db.findKeys('valueA:*', null, function (err, value) { // TODO: Check this
+    // value will be ['valueA:h1', 'valueA:h2']
+    db.close(function () {
+      process.exit(0);
+    });
+  });
+}
+
+example(db);
+```
+
+### Getting and setting subkeys
+
+ueberDB can store complex JSON objects. Sometimes you only want to get or set a
+specific (sub-)property of the stored object. The `.getSub()` and `.setSub()`
+methods make this easier.
+
+#### `getSub`
+
+```javascript
+db.getSub(key, propertyPath, callback);
+```
+
+Fetches the object stored at `key`, walks the property path given in
+`propertyPath`, and returns the value at that location. `propertyPath` must be
+an array. If `propertyPath` is an empty array then `getSub()` is equivalent to
+`get()`. Returns a nullish value (`null` or `undefined`) if the record does not
+exist or if the given property path does not exist.
+
+Examples:
+
+```javascript
+db.set(key, {prop1: {prop2: ['value']}}, (err) => {
+  if (err != null) throw err;
+
+  db.getSub(key, ['prop1', 'prop2', '0'], (err, val) => {
+    if (err != null) throw err;
+    console.log('1.', val); // prints "1. value"
+  });
+
+  db.getSub(key, ['prop1', 'prop2'], (err, val) => {
+    if (err != null) throw err;
+    console.log('2.', val); // prints "2. [ 'value' ]"
+  });
+
+  db.getSub(key, ['prop1'], (err, val) => {
+    if (err != null) throw err;
+    console.log('3.', val); // prints "3. { prop2: [ 'value' ] }"
+  });
+
+  db.getSub(key, [], (err, val) => {
+    if (err != null) throw err;
+    console.log('4.', val); // prints "4. { prop1: { prop2: [ 'value' ] } }"
+  });
+
+  db.getSub(key, ['does', 'not', 'exist'], (err, val) => {
+    if (err != null) throw err;
+    console.log('5.', val); // prints "5. null" or "5. undefined"
+  });
+});
+```
+
+#### `setSub`
+
+```javascript
+db.setSub(key, propertyPath, value, cb);
+```
+
+Fetches the object stored at `key`, walks the property path given in
+`propertyPath`, and sets the value at that location to `value`. `propertyPath`
+must be an array. If `propertyPath` is an empty array then `setSub()` is
+equivalent to `set()`. Empty objects are created as needed if the property path
+does not exist (including if `key` does not exist in the database). It is an
+error to attempt to set a property on a non-object. `cb` is optional and is
+called when the database driver has reported that the change has been written.
+
+Examples:
+
+```javascript
+// Assumption: The database does not yet have any records.
+
+// Equivalent to db.set('key1', 'value', cb):
+db.setSub('key1', [], 'value', cb);
+
+// Equivalent to db.set('key2', {prop1: {prop2: {0: 'value'}}}, cb):
+db.setSub('key2', ['prop1', 'prop2', '0'], 'value', cb):
+
+db.set('key3', {prop1: 'value'}, (err) => {
+  if (err != null) return cb(err);
+  // Equivalent to db.set('key3', {prop1: 'value', prop2: 'other value'}, cb):
+  db.setSub('key3', ['prop2'], 'other value', cb);
+});
+
+db.set('key3', {prop1: 'value'}, (err) => {
+  if (err != null) return cb(err);
+  // TypeError: Cannot set property "badProp" on non-object "value":
+  db.setSub('key3', ['prop1', 'badProp'], 'foo', cb);
+});
+```
+
+### Disable the read cache
+
+Set the `cache` wrapper option to 0 to force every read operation to go directly
+to the database driver (except for reads of written values that have not yet
+been committed to the database):
+
+```javascript
+const ueberdb = require('ueberdb2');
+
+(async () => {
+  const db = new ueberdb.Database(
+      'dirty', {filename: 'var/dirty.db'}, {cache: 0});
+  await db.init();
+  db.set('valueA', {a: 1, b: 2});
+  db.get('valueA', (err, value) => {
+    console.log(JSON.stringify(value));
+    db.close(() => {
+      process.exit(0);
+    });
+  });
+})();
+```
+
+### Disable write buffering
+
+Set the `writeInterval` wrapper option to 0 to force writes to go directly to
+the database driver:
+
+```javascript
+const ueberdb = require('ueberdb2');
+
+(async () => {
+  const db = new ueberdb.Database(
+      'dirty', {filename: 'var/dirty.db'}, {writeInterval: 0});
+  await db.init();
+  db.set('valueA', {a: 1, b: 2});
+  db.get('valueA', (err, value) => {
+    console.log(JSON.stringify(value));
+    db.close(() => {
+      process.exit(0);
+    });
+  });
+})();
+```
+
+## Feature support
+
+|        | Get | Set | findKeys | Remove | getSub | setSub | doBulk |CI Coverage|
+|--------|-----|-----|----------|--------|--------|--------|--------|--------|
+|  cassandra |  ✓  |  ✓  |    *     |   ✓    |   ✓    |   ✓    |   ✓    |
+|  couchdb |  ✓  |  ✓  |    ✓     |   ✓    |   ✓    |   ✓    |   ✓    |
+|  dirty |  ✓  |  ✓  |    ✓     |   ✓    |   ✓    |   ✓    |        |   ✓   |
+|  dirty_git |  ✓  |  ✓  |    ✓     |   ✓    |   ✓    |   ✓    |        |
+|  elasticsearch |  ✓  |  ✓  |    *     |   ✓    |   ✓    |   ✓    |   ✓    |
+|  maria |  ✓  |  ✓  |    ✓     |   ✓    |   ✓    |   ✓    |   ✓    |
+|  mysql |  ✓  |  ✓  |    ✓     |   ✓    |   ✓    |   ✓    |   ✓    |   ✓   |
+|  postgres  |  ✓  |  ✓  |    ✓     |   ✓    |   ✓    |   ✓    |   ✓    |   ✓   |
+|  redis |  ✓  |  ✓  |    *     |   ✓    |   ✓    |   ✓    |   ✓    |   ✓   |
+|  rethinkdb |  ✓  |  ✓  |    *     |   ✓    |   ✓    |   ✓    |   ✓    |
+|  sqlite | ✓  |  ✓  |    ✓     |   ✓    |   ✓    |   ✓    |   ✓    |   ✓    |
+
+## Limitations
+
+### findKeys query support
+
+The following characters should be avoided in keys `\^$.|?*+()[{` as they will
+cause findKeys to fail.
+
+### findKeys database support*
+
+The following have limitations on findKeys
+
+* redis (Only keys of the format \*:\*:\*)
+* cassandra (Only keys of the format \*:\*:\*)
+* elasticsearch (Only keys of the format \*:\*:\*)
+* rethink (Currently doesn't work)
+
+For details on how it works please refer to the wiki:
+https://github.com/ether/UeberDB/wiki/findKeys-functionality
+
+### Scaling, High availability and disaster recovery.
+
+To scale UeberDB you should use sharding especially for real time applications.
+An example of this is sharding given Pads within Etherpad based on their initial
+pad authors geographical location. High availability and disaster recovery can
+be provided through replication of your database however YMMV on passing
+Settings to your database library. Do not be under the illusion that UeberDB
+provides any Stateless capabilities, it does not. An option is to use something
+like rethinkdb and set cache to 0 but YMMV.
+
+### Key Length Restrictions
+
+Your Key Length will be limited by the database you chose to use but keep into
+account portability within your application.
+
+### doBulk operations on .set out of memory
+
+doBulk operations that chain IE a large number of .set without a pause to handle
+the channel clearance can cause a `Javascript out of heap memory`. It's very
+rare this happens and is usually due to a bug in software causing a constant
+write to the database.
+
+## MySQL /MariaDB Advice
+
+You should create your database as utf8mb4_bin.
+
+## Redis TLS communication
+
+If you enabled TLS on your Redis database (available since Redis 6.0) you will
+need to change your connections parameters, here is an example:
+
+```
+settings:
+    {
+      host:
+      port: rediss://<redis_database_address>:<redis_database_port>
+      socket:
+      database:
+      password:
+      client_options
+    }
+```
+
+Do not provide a `host` value.
+
+If you don't provide a certificate on the client side, you need to add the
+environment variable `NODE_TLS_REJECT_UNAUTHORIZED = 0` and add the flag
+`--tls-auth-clients no` when launching the redis-server to accept connections.
+
+## How to add support for another database
+
+1. Add the database driver to `packages.json`, this will happen automatically if
+   you run `npm install %yourdatabase%`
+1. Create `databases/DATABASENAME_db.js` and have it export a `Database` class
+   that derives from `lib/AbstractDatabase.js`. Implement the required
+   functions.
+1. Add a service for the database to the test job in
+   `.github/workflows/npmpublish.yml`.
+1. Add an entry to `test/lib/databases.js` for your database and configure it to
+   work with the service added to the GitHub workflow.
+1. Install and start the database server and configure it to work with the
+   settings in your `test/lib/databases.js` entry.
+1. Run `npm test` to ensure that it works.
+
+## License
+
+[Apache License v2](http://www.apache.org/licenses/LICENSE-2.0.html)
+
+## What's changed from UeberDB?
+
+* Dropped broken databases: CrateDB, LevelDB, LMDB (probably a
+  breaking change for some people)
+* Introduced CI.
+* Introduced better testing.
+* Fixed broken database clients IE Redis.
+* Updated Depdendencies where possible.
+* Tidied file structure.
+* Improved documentation.
+* Sensible name for software makes it clear that it's maintained by The Etherpad
+  Foundation.
+* Make db.init await / async
+
+### Dirty_Git Easter Egg.
+
+* I suck at hiding Easter eggs..
+
+Dirty_git will `commit` and `push` to Git on every `set`. To use `git init` or
+`git clone` within your dirty database location and then set your upstream IE
+`git remote add origin git://whztevz`.
+
+The logic behind dirty git is that you can still use dirty but you can also have
+offsite backups. It's noisy and spammy but it can be useful.

package/SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Please email contact@etherpad.org to report security related issues.

package/databases/cassandra_db.js

@@ -0,0 +1,250 @@
+'use strict';
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+const cassandra = require('cassandra-driver');
+
+exports.Database = class extends AbstractDatabase {
+  /**
+   * @param {Object} settings The required settings object to initiate the Cassandra database
+   * @param {String[]} settings.clientOptions See
+   *     http://www.datastax.com/drivers/nodejs/2.0/global.html#ClientOptions for a full set of
+   *     options that can be used
+   * @param {String} settings.columnFamily The column family that should be used to store data. The
+   *     column family will be created if it doesn't exist
+   * @param {Function} [settings.logger] Function that will be used to pass on log events emitted by
+   *     the Cassandra driver. See https://github.com/datastax/nodejs-driver#logging for more
+   *     information
+   */
+  constructor(settings) {
+    super();
+    if (!settings.clientOptions) {
+      throw new Error('The Cassandra client options should be defined');
+    }
+    if (!settings.columnFamily) {
+      throw new Error('The Cassandra column family should be defined');
+    }
+
+    this.settings = {};
+    this.settings.clientOptions = settings.clientOptions;
+    this.settings.columnFamily = settings.columnFamily;
+    this.settings.logger = settings.logger;
+  }
+
+  /**
+   * Initializes the Cassandra client, connects to Cassandra and creates the CF if it didn't exist
+   * already
+   *
+   * @param  {Function}   callback        Standard callback method.
+   * @param  {Error}      callback.err    An error object (if any.)
+   */
+  init(callback) {
+    // Create a client
+    this.client = new cassandra.Client(this.settings.clientOptions);
+
+    // Pass on log messages if a logger has been configured
+    if (this.settings.logger) {
+      this.client.on('log', this.settings.logger);
+    }
+
+    // Check whether our column family already exists and create it if necessary
+    this.client.execute(
+        'SELECT columnfamily_name FROM system.schema_columnfamilies WHERE keyspace_name = ?',
+        [this.settings.clientOptions.keyspace],
+        (err, result) => {
+          if (err) {
+            return callback(err);
+          }
+
+          let isDefined = false;
+          const length = result.rows.length;
+          for (let i = 0; i < length; i++) {
+            if (result.rows[i].columnfamily_name === this.settings.columnFamily) {
+              isDefined = true;
+              break;
+            }
+          }
+
+          if (isDefined) {
+            return callback(null);
+          } else {
+            const cql =
+                `CREATE COLUMNFAMILY "${this.settings.columnFamily}" ` +
+                '(key text PRIMARY KEY, data text)';
+            this.client.execute(cql, callback);
+          }
+        });
+  }
+
+  /**
+   * Gets a value from Cassandra
+   *
+   * @param  {String}     key               The key for which the value should be retrieved
+   * @param  {Function}   callback          Standard callback method
+   * @param  {Error}      callback.err      An error object, if any
+   * @param  {String}     callback.value    The value for the given key (if any)
+   */
+  get(key, callback) {
+    const cql = `SELECT data FROM "${this.settings.columnFamily}" WHERE key = ?`;
+    this.client.execute(cql, [key], (err, result) => {
+      if (err) {
+        return callback(err);
+      }
+
+      if (!result.rows || result.rows.length === 0) {
+        return callback(null, null);
+      }
+
+      return callback(null, result.rows[0].data);
+    });
+  }
+
+  /**
+   * Cassandra has no native `findKeys` method. This function implements a naive filter by
+   * retrieving *all* the keys and filtering those. This should obviously be used with the utmost
+   * care and is probably not something you want to run in production.
+   *
+   * @param  {String}     key               The filter for keys that should match
+   * @param  {String}     [notKey]          The filter for keys that shouldn't match
+   * @param  {Function}   callback          Standard callback method
+   * @param  {Error}      callback.err      An error object, if any
+   * @param  {String[]}   callback.keys     An array of keys that match the specified filters
+   */
+  findKeys(key, notKey, callback) {
+    let cql = null;
+    if (!notKey) {
+      // Get all the keys
+      cql = `SELECT key FROM "${this.settings.columnFamily}"`;
+      this.client.execute(cql, (err, result) => {
+        if (err) {
+          return callback(err);
+        }
+
+        // Construct a regular expression based on the given key
+        const regex = new RegExp(`^${key.replace(/\*/g, '.*')}$`);
+
+        const keys = [];
+        result.rows.forEach((row) => {
+          if (regex.test(row.key)) {
+            keys.push(row.key);
+          }
+        });
+
+        return callback(null, keys);
+      });
+    } else if (notKey === '*:*:*') {
+      // restrict key to format 'text:*'
+      const matches = /^([^:]+):\*$/.exec(key);
+      if (matches) {
+        // Get the 'text' bit out of the key and get all those keys from a special column.
+        // We can retrieve them from this column as we're duplicating them on .set/.remove
+        cql = `SELECT * from "${this.settings.columnFamily}" WHERE key = ?`;
+        this.client.execute(cql, [`ueberdb:keys:${matches[1]}`], (err, result) => {
+          if (err) {
+            return callback(err);
+          }
+
+          if (!result.rows || result.rows.length === 0) {
+            return callback(null, []);
+          }
+
+          const keys = result.rows.map((row) => row.data);
+          return callback(null, keys);
+        });
+      } else {
+        const msg =
+            'Cassandra db only supports key patterns like pad:* when notKey is set to *:*:*';
+        return callback(new Error(msg), null);
+      }
+    } else {
+      return callback(new Error('Cassandra db currently only supports *:*:* as notKey'), null);
+    }
+  }
+
+  /**
+   * Sets a value for a key
+   *
+   * @param  {String}     key             The key to set
+   * @param  {String}     value           The value associated to this key
+   * @param  {Function}   callback        Standard callback method
+   * @param  {Error}      callback.err    An error object, if any
+   */
+  set(key, value, callback) {
+    this.doBulk([{type: 'set', key, value}], callback);
+  }
+
+  /**
+   * Removes a key and it's value from the column family
+   *
+   * @param  {String}     key             The key to remove
+   * @param  {Function}   callback        Standard callback method
+   * @param  {Error}      callback.err    An error object, if any
+   */
+  remove(key, callback) {
+    this.doBulk([{type: 'remove', key}], callback);
+  }
+
+  /**
+   * Performs multiple operations in one action
+   *
+   * @param  {Object[]}   bulk            The set of operations that should be performed
+   * @param  {Function}   callback        Standard callback method
+   * @param  {Error}      callback.err    An error object, if any
+   */
+  doBulk(bulk, callback) {
+    const queries = [];
+    bulk.forEach((operation) => {
+      // We support finding keys of the form `test:*`. If anything matches, we will try and save
+      // this
+      const matches = /^([^:]+):([^:]+)$/.exec(operation.key);
+      if (operation.type === 'set') {
+        queries.push({
+          query: `UPDATE "${this.settings.columnFamily}" SET data = ? WHERE key = ?`,
+          params: [operation.value, operation.key],
+        });
+
+        if (matches) {
+          queries.push({
+            query: `UPDATE "${this.settings.columnFamily}" SET data = ? WHERE key = ?`,
+            params: ['1', `ueberdb:keys:${matches[1]}`],
+          });
+        }
+      } else if (operation.type === 'remove') {
+        queries.push({
+          query: `DELETE FROM "${this.settings.columnFamily}" WHERE key=?`,
+          params: [operation.key],
+        });
+
+        if (matches) {
+          queries.push({
+            query: `DELETE FROM "${this.settings.columnFamily}" WHERE key = ?`,
+            params: [`ueberdb:keys:${matches[1]}`],
+          });
+        }
+      }
+    });
+    this.client.batch(queries, {prepare: true}, callback);
+  }
+
+  /**
+   * Closes the Cassandra connection
+   *
+   * @param  {Function}   callback        Standard callback method
+   * @param  {Error}      callback.err    Error object in case something goes wrong
+   */
+  close(callback) {
+    this.pool.shutdown(callback);
+  }
+};