ueberdb2 1.4.16

package/databases/couch_db.js

@@ -0,0 +1,201 @@
+'use strict';
+/**
+ * 2012 Max 'Azul' Wiehle
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+const nano = require('nano');
+const async = require('async');
+
+const DESIGN_NAME = 'ueberDb';
+const DESIGN_PATH = `_design/${DESIGN_NAME}`;
+
+const handleError = (er) => {
+  if (er) throw new Error(er);
+};
+
+exports.Database = class extends AbstractDatabase {
+  constructor(settings) {
+    super();
+    this.db = null;
+    this.client = null;
+    this.settings = settings;
+
+    // force some settings
+    // used by CacheAndBufferLayer.js
+    this.settings.cache = 1000;
+    this.settings.writeInterval = 100;
+    this.settings.json = false;
+  }
+
+  init(callback) {
+    const settings = this.settings;
+    let client = null;
+    let db = null;
+
+    const config = {
+      url: `http://${settings.host}:${settings.port}`,
+      requestDefaults: {
+        pool: {
+          maxSockets: settings.maxListeners || 1,
+        },
+        auth: {
+          user: settings.user,
+          pass: settings.password,
+        },
+      },
+    };
+
+    const createDb = () => {
+      client.db.create(settings.database, (er, body) => {
+        if (er) return callback(er);
+        return setDb();
+      });
+    };
+
+    const setDb = () => {
+      db = client.use(settings.database);
+      checkUeberDbDesignDocument(db);
+      this.client = client;
+      this.db = db;
+      callback();
+    };
+
+    // Always ensure that couchDb has at least an empty design doc for UeberDb use
+    // this will be necessary for the `findKeys` method
+    const checkUeberDbDesignDocument = () => {
+      db.head(DESIGN_PATH, (er, _, header) => {
+        if (er && er.statusCode === 404) return db.insert({views: {}}, DESIGN_PATH, handleError);
+        if (er) throw new Error(er);
+      });
+    };
+
+    client = nano(config);
+    client.db.get(settings.database, (er, body) => {
+      if (er && er.statusCode === 404) return createDb();
+      if (er) return callback(er);
+      return setDb();
+    });
+  }
+
+  get(key, callback) {
+    const db = this.db;
+    db.get(key, (er, doc) => {
+      if (er && er.statusCode !== 404) {
+        console.log('GET');
+        console.log(er);
+      }
+      if (doc == null) return callback(null, null);
+      callback(null, doc.value);
+    });
+  }
+
+  findKeys(key, notKey, callback) {
+    const regex = this.createFindRegex(key, notKey);
+    const queryKey = `${key}__${notKey}`;
+    const db = this.db;
+
+    // always look up if the query haven't be done before
+    const checkQuery = () => {
+      db.get(DESIGN_PATH, (er, doc) => {
+        handleError(er);
+        const queryExists = queryKey in doc.views;
+        if (!queryExists) return createQuery(doc);
+        makeQuery();
+      });
+    };
+
+    // Cache the query for faster reuse in the future
+    const createQuery = (doc) => {
+      const mapFunction = {
+        map: `function (doc) { if (${regex}.test(doc._id)) { emit(doc._id, null); } }`,
+      };
+      doc.views[queryKey] = mapFunction;
+      db.insert(doc, DESIGN_PATH, (er) => {
+        handleError(er);
+        makeQuery();
+      });
+    };
+
+    // If this is the first time the request is used, this can take a while…
+    const makeQuery = (er) => {
+      db.view(DESIGN_NAME, queryKey, (er, docs) => {
+        handleError(er);
+        docs = docs.rows.map((doc) => doc.key);
+        callback(null, docs);
+      });
+    };
+
+    checkQuery();
+  }
+
+  set(key, value, callback) {
+    const db = this.db;
+    db.get(key, (er, doc) => {
+      if (doc == null) return db.insert({_id: key, value}, callback);
+      db.insert({_id: key, _rev: doc._rev, value}, callback);
+    });
+  }
+
+  remove(key, callback) {
+    const db = this.db;
+    db.head(key, (er, _, header) => {
+      if (er && er.statusCode === 404) return callback(null);
+      if (er) return callback(er);
+      // etag has additional quotation marks, remove them
+      const etag = JSON.parse(header).etag;
+      db.destroy(key, etag, (er, body) => {
+        if (er) return callback(er);
+        callback(null);
+      });
+    });
+  }
+
+  doBulk(bulk, callback) {
+    const db = this.db;
+    const keys = bulk.map((op) => op.key);
+    const revs = {};
+    const setters = [];
+    async.series([
+      (callback) => {
+        db.fetchRevs({keys}, (er, r) => {
+          if (er) throw new Error(JSON.stringify(er));
+          const rows = r.rows;
+          for (const j in r.rows) {
+            // couchDB will return error instead of value if key does not exist
+            if (rows[j].value != null) revs[rows[j].key] = rows[j].value.rev;
+          }
+          callback();
+        });
+      },
+      (callback) => {
+        for (const item of bulk) {
+          const set = {_id: item.key};
+          if (revs[item.key] != null) set._rev = revs[item.key];
+          if (item.type === 'set') set.value = item.value;
+          if (item.type === 'remove') set._deleted = true;
+          setters.push(set);
+        }
+        callback();
+      },
+    ], (err) => {
+      db.bulk({docs: setters}, callback);
+    });
+  }
+
+  close(callback) {
+    if (callback) callback();
+  }
+};

package/databases/dirty_db.js

@@ -0,0 +1,80 @@
+'use strict';
+/**
+ * 2011 Peter 'Pita' Martischka
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+*
+* Fair warning that length may not provide the correct value upon load.
+* See https://github.com/ether/etherpad-lite/pull/3984
+*
+*/
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+const Dirty = require('dirty');
+
+exports.Database = class extends AbstractDatabase {
+  constructor(settings) {
+    super();
+    this.db = null;
+
+    if (!settings || !settings.filename) {
+      settings = {filename: null};
+    }
+
+    this.settings = settings;
+
+    // set default settings
+    this.settings.cache = 0;
+    this.settings.writeInterval = 0;
+    this.settings.json = false;
+  }
+
+  init(callback) {
+    this.db = new Dirty(this.settings.filename);
+    this.db.on('load', (err) => {
+      callback();
+    });
+  }
+
+  get(key, callback) {
+    callback(null, this.db.get(key));
+  }
+
+  findKeys(key, notKey, callback) {
+    const keys = [];
+    const regex = this.createFindRegex(key, notKey);
+    this.db.forEach((key, val) => {
+      if (key.search(regex) !== -1) {
+        keys.push(key);
+      }
+    });
+    callback(null, keys);
+  }
+
+  set(key, value, callback) {
+    this.db.set(key, value, callback);
+  }
+
+  remove(key, callback) {
+    this.db.rm(key, callback);
+  }
+
+  close(callback) {
+    this.db.close();
+    this.db = null;
+    if (callback) callback();
+  }
+};

package/databases/dirty_git_db.js

@@ -0,0 +1,78 @@
+'use strict';
+/**
+ * 2011 Peter 'Pita' Martischka
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+const Dirty = require('dirty');
+
+exports.Database = class extends AbstractDatabase {
+  constructor(settings) {
+    super();
+    this.db = null;
+
+    if (!settings || !settings.filename) {
+      settings = {filename: null};
+    }
+
+    this.settings = settings;
+
+    // set default settings
+    this.settings.cache = 0;
+    this.settings.writeInterval = 0;
+    this.settings.json = false;
+  }
+
+  init(callback) {
+    this.db = new Dirty(this.settings.filename);
+    this.db.on('load', (err) => {
+      callback();
+    });
+  }
+
+  get(key, callback) {
+    callback(null, this.db.get(key));
+  }
+
+  findKeys(key, notKey, callback) {
+    const keys = [];
+    const regex = this.createFindRegex(key, notKey);
+    this.db.forEach((key, val) => {
+      if (key.search(regex) !== -1) {
+        keys.push(key);
+      }
+    });
+    callback(null, keys);
+  }
+
+  set(key, value, callback) {
+    this.db.set(key, value, callback);
+    const databasePath = require('path').dirname(this.settings.filename);
+    require('simple-git')(databasePath)
+        .silent(true)
+        .add('./*.db')
+        .commit('Automated commit...')
+        .push(['-u', 'origin', 'master'], () => console.debug('Stored git commit'));
+  }
+
+  remove(key, callback) {
+    this.db.rm(key, callback);
+  }
+
+  close(callback) {
+    this.db.close();
+    if (callback) callback();
+  }
+};

package/databases/elasticsearch_db.js

@@ -0,0 +1,288 @@
+'use strict';
+/**
+ * 2015 Visionist, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+const es = require('elasticsearch');
+
+// initialize w/ default settings
+const elasticsearchSettings = {
+  hostname: '127.0.0.1',
+  port: '9200',
+  base_index: 'ueberes',
+
+  // for a list of valid API values see:
+  // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html#config-options
+  api: '7.6',
+};
+
+let client;
+
+exports.Database = class extends AbstractDatabase {
+  constructor(settings) {
+    super();
+    this.db = null;
+
+    this.settings = settings || {};
+
+    // update settings if they were provided
+    if (this.settings.host) {
+      elasticsearchSettings.hostname = this.settings.host;
+    }
+
+    if (this.settings.port) {
+      elasticsearchSettings.port = this.settings.port;
+    }
+
+    if (this.settings.base_index) {
+      elasticsearchSettings.base_index = this.settings.base_index;
+    }
+
+    if (this.settings.api) {
+      elasticsearchSettings.api = this.settings.api;
+    }
+  }
+
+  /**
+   * Initialize the elasticsearch client, then ping the server to ensure that a
+   * connection was made.
+   */
+  init(callback) {
+    // create elasticsearch client
+    client = new es.Client({
+      host: `${elasticsearchSettings.hostname}:${elasticsearchSettings.port}`,
+      apiVersion: elasticsearchSettings.api,
+      // log: "trace" // useful for debugging
+    });
+
+    // test the connection
+    client.ping({
+      requestTimeout: 3000,
+    }, (error) => {
+      if (error) {
+        console.error('unable to communicate with elasticsearch');
+      }
+
+      callback(error);
+    });
+  }
+
+  /**
+   *  This function provides read functionality to the database.
+   *
+   *  @param {String} key Key, of the format "test:test1" or, optionally, of the
+   *    format "test:test1:check:check1"
+   *  @param {function} callback Function will be called in the event of an error or
+   *    upon completion of a successful database retrieval.
+   */
+  get(key, callback) {
+    client.get(getIndexTypeId(key), (error, response) => {
+      parseResponse(error, response, callback);
+    });
+  }
+
+  /**
+   *  The three key scenarios for this are:
+   *      (test:test1, null) ; (test:*, *:*:*) ; (test:*, null)
+   *
+   *  TODO This currently works only for the second implementation above.
+   *
+   *  For more information:
+   *    - See the #Limitations section of the ueberDB README.
+   *    - See https://github.com/Pita/ueberDB/wiki/findKeys-functionality, as well
+   *      as the sqlite and mysql implementations.
+   *
+   *  @param key Search key, which uses an asterisk (*) as the wild card.
+   *  @param notKey Used to filter the result set
+   *  @param callback First param is error, second is result
+   */
+  findKeys(key, notKey, callback) {
+    const splitKey = key.split(':');
+
+    client.search({
+      index: elasticsearchSettings.base_index,
+      type: splitKey[0],
+      size: 100, // this is a pretty random threshold...
+    }, (error, response) => {
+      if (error) {
+        console.error('findkeys', error);
+        callback(error);
+        return;
+      }
+
+      if (!error && response.hits) {
+        const keys = [];
+        for (let counter = 0; counter < response.hits.total; counter++) {
+          keys.push(`${splitKey[0]}:${response.hits.hits[counter]._id}`);
+        }
+        callback(null, keys);
+      }
+    });
+  }
+
+  /**
+   *  This function provides write functionality to the database.
+   *
+   *  @param {String} key Key, of the format "test:test1" or, optionally, of the
+   *    format "test:test1:check:check1"
+   *  @param {JSON|String} value The value to be stored to the database.  The value is
+   *    always converted to {val:value} before being written to the database, to account
+   *    for situations where the value is just a string.
+   *  @param {function} callback Function will be called in the event of an error or on
+   *    completion of a successful database write.
+   */
+  set(key, value, callback) {
+    const options = getIndexTypeId(key);
+
+    options.body = {
+      val: value,
+    };
+
+    client.index(options, (error, response) => {
+      parseResponse(error, response, callback);
+    });
+  }
+
+  /**
+   *  This function provides delete functionality to the database.
+   *
+   *  The index, type, and ID will be parsed from the key, and this document will
+   *  be deleted from the database.
+   *
+   *  @param {String} key Key, of the format "test:test1" or, optionally, of the
+   *    format "test:test1:check:check1"
+   *  @param {function} callback Function will be called in the event of an error or on
+   *    completion of a successful database write.
+   */
+  remove(key, callback) {
+    client.delete(key, (error, response) => {
+      parseResponse(error, response, callback);
+    });
+  }
+
+  /**
+   *  This uses the bulk upload functionality of elasticsearch (url:port/_bulk).
+   *
+   *  The CacheAndBufferLayer will periodically (every this.settings.writeInterval)
+   *  flush writes that have already been done in the local cache out to the database.
+   *
+   *  @param {Array} bulk An array of JSON data in the format:
+   *      {"type":type, "key":key, "value":value}
+   *  @param {function} callback This function will be called on an error or upon the
+   *      successful completion of the database write.
+   */
+  doBulk(bulk, callback) {
+    // bulk is an array of JSON:
+    // example: [{"type":"set", "key":"sessionstorage:{id}", "value":{"cookie":{...}}]
+
+    const operations = [];
+
+    for (let counter = 0; counter < bulk.length; counter++) {
+      const indexTypeId = getIndexTypeId(bulk[counter].key);
+      const operationPayload = {
+        _index: indexTypeId.index,
+        _type: indexTypeId.type,
+        _id: indexTypeId.id,
+      };
+
+      switch (bulk[counter].type) {
+        case 'set':
+          operations.push({index: operationPayload});
+          operations.push({val: JSON.parse(bulk[counter].value)});
+          break;
+        case 'remove':
+          operations.push({delete: operationPayload});
+          break;
+        default:
+          continue;
+      }
+    }
+
+    // send bulk request
+    client.bulk({
+      body: operations,
+    }, (error, response) => {
+      parseResponse(error, response, callback);
+    });
+  }
+
+  close(callback) {
+    callback(null);
+  }
+};
+
+/** ************************
+ **** Helper functions ****
+ **************************/
+
+/**
+ *  This function parses a given key into an object with three
+ *  fields, .index, .type, and .id.  This object can then be
+ *  used to build an elasticsearch path or to access an object
+ *  for bulk updates.
+ *
+ *  @param {String} key Key, of the format "test:test1" or, optionally, of the
+ *    format "test:test1:check:check1"
+ */
+const getIndexTypeId = (key) => {
+  const returnObject = {};
+
+  const splitKey = key.split(':');
+
+  if (splitKey.length === 4) {
+    /*
+     * This is for keys like test:test1:check:check1.
+     * These keys are stored at /base_index-test-check/test1/check1
+     */
+    returnObject.index = `${elasticsearchSettings.base_index}-${splitKey[0]}-${splitKey[2]}`;
+    returnObject.type = encodeURIComponent(splitKey[1]);
+    returnObject.id = splitKey[3];
+  } else {
+    // everything else ('test:test1') is stored /base_index/test/test1
+    returnObject.index = elasticsearchSettings.base_index;
+    returnObject.type = splitKey[0];
+    returnObject.id = encodeURIComponent(splitKey[1]);
+  }
+
+  return returnObject;
+};
+
+/**
+ * Extract data from elasticsearch responses, handle errors, handle callbacks.
+ */
+const parseResponse = (error, response, callback) => {
+  if (error) {
+    // don't treat not found as an error (is this specific to etherpad?)
+    if (error.message === 'Not Found' && !response.found) {
+      callback(null, null);
+      return;
+    } else {
+      console.error('elasticsearch_db: ', error);
+    }
+  }
+
+  if (!error && response) {
+    response = response._source;
+
+    if (response) {
+      response = response.val;
+    }
+
+    response = JSON.stringify(response);
+  }
+
+  callback(error, response);
+};

package/databases/mock_db.js

@@ -0,0 +1,42 @@
+'use strict';
+
+const events = require('events');
+
+exports.Database = class extends events.EventEmitter {
+  constructor(settings) {
+    super();
+    this.settings = {
+      writeInterval: 1,
+      ...settings,
+    };
+    settings.mock = this;
+  }
+
+  close(cb) {
+    this.emit('close', cb);
+  }
+
+  doBulk(ops, cb) {
+    this.emit('doBulk', ops, cb);
+  }
+
+  findKeys(key, notKey, cb) {
+    this.emit('findKeys', key, notKey, cb);
+  }
+
+  get(key, cb) {
+    this.emit('get', key, cb);
+  }
+
+  init(cb) {
+    this.emit('init', cb);
+  }
+
+  remove(key, cb) {
+    this.emit('remove', key, cb);
+  }
+
+  set(key, value, cb) {
+    this.emit('set', key, value, cb);
+  }
+};