ueberdb2 1.4.16

package/databases/postgres_db.js

@@ -0,0 +1,198 @@
+'use strict';
+/**
+ * 2011 Peter 'Pita' Martischka
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+const async = require('async');
+const pg = require('pg');
+
+exports.Database = class extends AbstractDatabase {
+  constructor(settings) {
+    super();
+    if (typeof settings === 'string') settings = {connectionString: settings};
+    this.settings = settings;
+
+    this.settings.cache = settings.cache || 1000;
+    this.settings.writeInterval = 100;
+    this.settings.json = true;
+
+    // Pool specific defaults
+    this.settings.max = this.settings.max || 20;
+    this.settings.min = this.settings.min || 4;
+    this.settings.idleTimeoutMillis = this.settings.idleTimeoutMillis || 1000;
+
+    this.db = new pg.Pool(this.settings);
+  }
+
+  init(callback) {
+    const testTableExists = "SELECT 1 as exists FROM pg_tables WHERE tablename = 'store'";
+
+    const createTable = 'CREATE TABLE IF NOT EXISTS store (' +
+                        '"key" character varying(100) NOT NULL, ' +
+                        '"value" text NOT NULL, ' +
+                        'CONSTRAINT store_pkey PRIMARY KEY (key))';
+
+    // this variable will be given a value depending on the result of the
+    // feature detection
+    this.upsertStatement = null;
+
+    /*
+     * - Detects if this Postgres version supports INSERT .. ON CONFLICT
+     *   UPDATE (PostgreSQL >= 9.5 and CockroachDB)
+     * - If upsert is not supported natively, creates in the DB a pl/pgsql
+     *   function that emulates it
+     * - Performs a side effect, setting this.upsertStatement to the sql
+     *   statement that needs to be used, based on the detection result
+     * - calls the callback
+     */
+    const detectUpsertMethod = (callback) => {
+      const upsertViaFunction = 'SELECT ueberdb_insert_or_update($1,$2)';
+      const upsertNatively =
+          'INSERT INTO store(key, value) VALUES ($1, $2) ' +
+          'ON CONFLICT (key) DO UPDATE SET value = excluded.value';
+      const createFunc =
+          'CREATE OR REPLACE FUNCTION ueberdb_insert_or_update(character varying, text) ' +
+          'RETURNS void AS $$ ' +
+          'BEGIN ' +
+          '  IF EXISTS( SELECT * FROM store WHERE key = $1 ) THEN ' +
+          '    UPDATE store SET value = $2 WHERE key = $1; ' +
+          '  ELSE ' +
+          '    INSERT INTO store(key,value) VALUES( $1, $2 ); ' +
+          '  END IF; ' +
+          '  RETURN; ' +
+          'END; ' +
+          '$$ LANGUAGE plpgsql;';
+
+      const testNativeUpsert = `EXPLAIN ${upsertNatively}`;
+
+      this.db.query(testNativeUpsert, ['test-key', 'test-value'], (err) => {
+        if (err) {
+          // the UPSERT statement failed: we will have to emulate it via
+          // an sql function
+          this.upsertStatement = upsertViaFunction;
+
+          // actually create the emulation function
+          this.db.query(createFunc, [], callback);
+
+          return;
+        }
+
+        // if we get here, the EXPLAIN UPSERT succeeded, and we can use a
+        // native UPSERT
+        this.upsertStatement = upsertNatively;
+        callback();
+      });
+    };
+
+    this.db.query(testTableExists, (err, result) => {
+      if (err != null) return callback(err);
+      if (result.rows.length === 0) {
+        this.db.query(createTable, (err) => {
+          if (err != null) return callback(err);
+          detectUpsertMethod(callback);
+        });
+      } else {
+        detectUpsertMethod(callback);
+      }
+    });
+  }
+
+  get(key, callback) {
+    this.db.query('SELECT value FROM store WHERE key=$1', [key], (err, results) => {
+      let value = null;
+
+      if (!err && results.rows.length === 1) {
+        value = results.rows[0].value;
+      }
+
+      callback(err, value);
+    });
+  }
+
+  findKeys(key, notKey, callback) {
+    let query = 'SELECT key FROM store WHERE key LIKE $1';
+    const params = [];
+    // desired keys are %key:%, e.g. pad:%
+    key = key.replace(/\*/g, '%');
+    params.push(key);
+
+    if (notKey != null) {
+      // not desired keys are notKey:%, e.g. %:%:%
+      notKey = notKey.replace(/\*/g, '%');
+      query += ' AND key NOT LIKE $2';
+      params.push(notKey);
+    }
+    this.db.query(query, params, (err, results) => {
+      const value = [];
+
+      if (!err && results.rows.length > 0) {
+        results.rows.forEach((val) => {
+          value.push(val.key);
+        });
+      }
+
+      callback(err, value);
+    });
+  }
+
+  set(key, value, callback) {
+    if (key.length > 100) {
+      callback('Your Key can only be 100 chars');
+    } else {
+      this.db.query(this.upsertStatement, [key, value], callback);
+    }
+  }
+
+  remove(key, callback) {
+    this.db.query('DELETE FROM store WHERE key=$1', [key], callback);
+  }
+
+  doBulk(bulk, callback) {
+    const replaceVALs = [];
+    let removeSQL = 'DELETE FROM store WHERE key IN (';
+    const removeVALs = [];
+
+    let removeCount = 0;
+
+    for (const i in bulk) {
+      if (bulk[i].type === 'set') {
+        replaceVALs.push([bulk[i].key, bulk[i].value]);
+      } else if (bulk[i].type === 'remove') {
+        if (removeCount !== 0) removeSQL += ',';
+        removeCount += 1;
+
+        removeSQL += `$${removeCount}`;
+        removeVALs.push(bulk[i].key);
+      }
+    }
+
+    removeSQL += ');';
+
+    const functions = replaceVALs.map((v) => (cb) => this.db.query(this.upsertStatement, v, cb));
+
+    const removeFunction = (callback) => {
+      if (!removeVALs.length < 1) this.db.query(removeSQL, removeVALs, callback);
+      else callback();
+    };
+    functions.push(removeFunction);
+
+    async.parallel(functions, callback);
+  }
+
+  close(callback) {
+    this.db.end(callback);
+  }
+};

package/databases/postgrespool_db.js

@@ -0,0 +1,11 @@
+'use strict';
+
+const postgres = require('./postgres_db');
+
+exports.Database = class extends postgres.Database {
+  constructor(settings) {
+    console.warn('ueberdb: The postgrespool database driver is deprecated ' +
+                 'and will be removed in a future version. Use postgres instead.');
+    super(settings);
+  }
+};

package/databases/redis_db.js

@@ -0,0 +1,128 @@
+'use strict';
+/* eslint new-cap: ["error", {"capIsNewExceptions": ["KEYS", "SMEMBERS"]}] */
+
+/**
+ * 2011 Peter 'Pita' Martischka
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+const async = require('async');
+const redis = require('redis');
+
+exports.Database = class extends AbstractDatabase {
+  constructor(settings) {
+    super();
+    this.client = null;
+    this.settings = settings || {};
+  }
+
+  auth(callback) {
+    if (!this.settings.password) return callback();
+    this.client.auth(this.settings.password, callback);
+  }
+
+  select(callback) {
+    if (!this.settings.database) return callback();
+    this.client.select(this.settings.database, callback);
+  }
+
+  _deprecatedInit(callback) {
+    if (this.settings.socket) {
+      // Deprecated, but kept for backwards compatibility.
+      this.client = redis.createClient(this.settings.socket,
+          this.settings.client_options);
+    } else {
+      // Deprecated, but kept for backwards compatibility.
+      this.client = redis.createClient(this.settings.port,
+          this.settings.host, this.settings.client_options);
+    }
+
+    this.client.database = this.settings.database;
+    async.waterfall([this.auth.bind(this), this.select.bind(this)], callback);
+  }
+
+  init(callback) {
+    if (this.settings.socket || this.settings.client_options) return this._deprecatedInit(callback);
+    this.client = redis.createClient(this.settings);
+    callback();
+  }
+
+  get(key, callback) {
+    this.client.get(key, callback);
+  }
+
+  findKeys(key, notKey, callback) {
+    // As redis provides only limited support for getting a list of all
+    // available keys we have to limit key and notKey here.
+    // See http://redis.io/commands/keys
+    if (notKey == null) {
+      this.client.KEYS(key, callback);
+    } else if (notKey === '*:*:*') {
+      // restrict key to format "text:*"
+      const matches = /^([^:]+):\*$/.exec(key);
+      if (matches) {
+        this.client.SMEMBERS(`ueberDB:keys:${matches[1]}`, callback);
+      } else {
+        const msg = 'redis db only supports key patterns like pad:* when notKey is set to *:*:*';
+        callback(new Error(msg), null);
+      }
+    } else {
+      callback(new Error('redis db currently only supports *:*:* as notKey'), null);
+    }
+  }
+
+  set(key, value, callback) {
+    const matches = /^([^:]+):([^:]+)$/.exec(key);
+    if (matches) {
+      this.client.sadd([`ueberDB:keys:${matches[1]}`, matches[0]]);
+    }
+    this.client.set(key, value, callback);
+  }
+
+  remove(key, callback) {
+    const matches = /^([^:]+):([^:]+)$/.exec(key);
+    if (matches) {
+      this.client.srem([`ueberDB:keys:${matches[1]}`, matches[0]]);
+    }
+    this.client.del(key, callback);
+  }
+
+  doBulk(bulk, callback) {
+    const multi = this.client.multi();
+
+    for (const {key, type, value} of bulk) {
+      const matches = /^([^:]+):([^:]+)$/.exec(key);
+      if (type === 'set') {
+        if (matches) {
+          multi.sadd([`ueberDB:keys:${matches[1]}`, matches[0]]);
+        }
+        multi.set(key, value);
+      } else if (type === 'remove') {
+        if (matches) {
+          multi.srem([`ueberDB:keys:${matches[1]}`, matches[0]]);
+        }
+        multi.del(key);
+      }
+    }
+
+    multi.exec(callback);
+  }
+
+  close(callback) {
+    this.client.quit(() => {
+      callback();
+    });
+  }
+};

package/databases/rethink_db.js

@@ -0,0 +1,98 @@
+'use strict';
+/**
+ * 2016 Remi Arnaud
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+const r = require('rethinkdb');
+const async = require('async');
+
+exports.Database = class extends AbstractDatabase {
+  constructor(settings) {
+    super();
+    if (!settings) settings = {};
+    if (!settings.host) { settings.host = 'localhost'; }
+    if (!settings.port) { settings.port = 28015; }
+    if (!settings.db) { settings.db = 'test'; }
+    if (!settings.table) { settings.table = 'test'; }
+
+    this.host = settings.host;
+    this.db = settings.db;
+    this.port = settings.port;
+    this.table = settings.table;
+    this.connection = null;
+  }
+
+  init(callback) {
+    r.connect(this, (err, conn) => {
+      if (err) throw err;
+      this.connection = conn;
+
+      r.table(this.table).run(this.connection, (err, cursor) => {
+        if (err) {
+          // assuming table does not exists
+          r.tableCreate(this.table).run(this.connection, callback);
+        } else if (callback) { callback(null, cursor); }
+      });
+    });
+  }
+
+  get(key, callback) {
+    r.table(this.table).get(key).run(this.connection, (err, item) => {
+      callback(err, (item ? item.content : item));
+    });
+  }
+
+  findKeys(key, notKey, callback) {
+    const keys = [];
+    const regex = this.createFindRegex(key, notKey);
+    r.filter((item) => {
+      if (item.id.search(regex) !== -1) {
+        keys.push(item.id);
+      }
+    }).run(this.connection, callback);
+  }
+
+  set(key, value, callback) {
+    r.table(this.table)
+        .insert({id: key, content: value}, {conflict: 'replace'})
+        .run(this.connection, callback);
+  }
+
+  doBulk(bulk, callback) {
+    const _in = [];
+    const _out = [];
+
+    for (const i in bulk) {
+      if (bulk[i].type === 'set') {
+        _in.push({id: bulk[i].key, content: bulk[i].value});
+      } else if (bulk[i].type === 'remove') {
+        _out.push(bulk[i].key);
+      }
+    }
+    async.parallel([
+      (cb) => { r.table(this.table).insert(_in, {conflict: 'replace'}).run(this.connection, cb); },
+      (cb) => { r.table(this.table).getAll(_out).delete().run(this.connection, cb); },
+    ], callback);
+  }
+
+  remove(key, callback) {
+    r.table(this.table).get(key).delete().run(this.connection, callback);
+  }
+
+  close(callback) {
+    this.connection.close(callback);
+  }
+};

package/databases/sqlite_db.js

@@ -0,0 +1,158 @@
+'use strict';
+/**
+ * 2011 Peter 'Pita' Martischka
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+let sqlite3;
+try {
+  sqlite3 = require('sqlite3');
+} catch (err) {
+  throw new Error(
+      'sqlite3 not found. It was removed from ueberdb\'s dependencies because it requires ' +
+      'compilation which fails on several systems. If you still want to use sqlite, run ' +
+      '"npm install sqlite3" in your etherpad-lite ./src directory.');
+}
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+const util = require('util');
+
+const escape = (val) => `'${val.replace(/'/g, "''")}'`;
+
+exports.Database = class extends AbstractDatabase {
+  constructor(settings) {
+    super();
+    this.db = null;
+
+    if (!settings || !settings.filename) {
+      settings = {filename: ':memory:'};
+    }
+
+    this.settings = settings;
+
+    // set settings for the dbWrapper
+    if (settings.filename === ':memory:') {
+      this.settings.cache = 0;
+      this.settings.writeInterval = 0;
+      this.settings.json = true;
+    } else {
+      this.settings.cache = 1000;
+      this.settings.writeInterval = 100;
+      this.settings.json = true;
+    }
+  }
+
+  async _query(sql, params = []) {
+    // It is unclear how util.promisify() deals with variadic functions, so it is not used here.
+    return await new Promise((resolve, reject) => {
+      // According to sqlite3's documentation, .run() method (and maybe .all() and .get(); the
+      // documentation is unclear) might call the callback multiple times. That's OK -- ECMAScript
+      // guarantees that it is safe to call a Promise executor's resolve and reject functions
+      // multiple times. The subsequent calls are ignored, except Node.js's 'process' object emits a
+      // 'multipleResolves' event to aid in debugging.
+      this.db.all(sql, params, (err, rows) => {
+        if (err != null) return reject(err);
+        resolve(rows);
+      });
+    });
+  }
+
+  // Temporary callbackified version of _query. This will be removed once all database objects are
+  // asyncified.
+  _queryCb(sql, params, callback) {
+    // It is unclear how util.callbackify() handles optional parameters, so it is not used here.
+    const p = this._query(sql, params);
+    if (callback) p.then((rows) => callback(null, rows), (err) => callback(err || new Error(err)));
+  }
+
+  init(callback) {
+    util.callbackify(async () => {
+      this.db = await new Promise((resolve, reject) => {
+        new sqlite3.Database(this.settings.filename, function (err) {
+          if (err != null) return reject(err);
+          // The use of `this` relies on an undocumented feature of sqlite3:
+          // https://github.com/mapbox/node-sqlite3/issues/1408
+          resolve(this);
+        });
+      });
+      await this._query('CREATE TABLE IF NOT EXISTS store (key TEXT PRIMARY KEY, value TEXT)');
+    })(callback);
+  }
+
+  get(key, callback) {
+    this._queryCb(
+        'SELECT value FROM store WHERE key = ?', [key],
+        (err, rows) => callback(err, err == null && rows && rows.length ? rows[0].value : null));
+  }
+
+  findKeys(key, notKey, callback) {
+    let query = 'SELECT key FROM store WHERE key LIKE ?';
+    const params = [];
+    // desired keys are %key:%, e.g. pad:%
+    key = key.replace(/\*/g, '%');
+    params.push(key);
+
+    if (notKey != null) {
+      // not desired keys are notKey:%, e.g. %:%:%
+      notKey = notKey.replace(/\*/g, '%');
+      query += ' AND key NOT LIKE ?';
+      params.push(notKey);
+    }
+
+    this._queryCb(query, params, (err, results) => {
+      const value = [];
+
+      if (!err && Object.keys(results).length > 0) {
+        results.forEach((val) => {
+          value.push(val.key);
+        });
+      }
+
+      callback(err, value);
+    });
+  }
+
+  set(key, value, callback) {
+    this._queryCb('REPLACE INTO store VALUES (?,?)', [key, value], callback);
+  }
+
+  remove(key, callback) {
+    this._queryCb('DELETE FROM store WHERE key = ?', [key], callback);
+  }
+
+  doBulk(bulk, callback) {
+    let sql = 'BEGIN TRANSACTION;\n';
+    for (const i in bulk) {
+      if (bulk[i].type === 'set') {
+        sql += `REPLACE INTO store VALUES (${escape(bulk[i].key)}, ${escape(bulk[i].value)});\n`;
+      } else if (bulk[i].type === 'remove') {
+        sql += `DELETE FROM store WHERE key = ${escape(bulk[i].key)};\n`;
+      }
+    }
+    sql += 'END TRANSACTION;';
+
+    this.db.exec(sql, (err) => {
+      if (err) {
+        console.error('ERROR WITH SQL: ');
+        console.error(sql);
+      }
+
+      callback(err);
+    });
+  }
+
+  close(callback) {
+    this.db.close(callback);
+  }
+};