ueberdb2 1.4.16

package/databases/mongodb_db.js

@@ -0,0 +1,136 @@
+'use strict';
+/**
+ * 2020 Sylchauf
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+
+exports.Database = class extends AbstractDatabase {
+  constructor(settings) {
+    super();
+    this.settings = settings;
+
+    if (!this.settings.url) throw new Error('You must specify a mongodb url');
+    // For backwards compatibility:
+    if (this.settings.database == null) this.settings.database = this.settings.dbName;
+
+    if (!this.settings.collection) this.settings.collection = 'ueberdb';
+  }
+
+  clearPing() {
+    if (this.interval) {
+      clearInterval(this.interval);
+    }
+  }
+
+  schedulePing() {
+    this.clearPing();
+    this.interval = setInterval(() => {
+      this.database.command({
+        ping: 1,
+      });
+    }, 10000);
+  }
+
+  init(callback) {
+    const MongoClient = require('mongodb').MongoClient;
+
+    MongoClient.connect(this.settings.url, (err, client) => {
+      if (!err) {
+        this.client = client;
+        this.database = client.db(this.settings.database);
+        this.collection = this.database.collection(this.settings.collection);
+      }
+
+      callback(err);
+    });
+
+    this.schedulePing();
+  }
+
+  get(key, callback) {
+    this.collection.findOne({_id: key}, (err, document) => {
+      if (err) callback(err);
+      else callback(null, document ? document.value : null);
+    });
+
+    this.schedulePing();
+  }
+
+  findKeys(key, notKey, callback) {
+    const selector = {
+      $and: [
+        {_id: {$regex: `${key.replace(/\*/g, '')}`}},
+      ],
+    };
+
+    if (notKey) {
+      selector.$and.push({_id: {$not: {$regex: `${notKey.replace(/\*/g, '')}`}}});
+    }
+
+    this.collection.find(selector, async (err, res) => {
+      if (err) {
+        callback(err);
+      } else {
+        const data = await res.toArray();
+
+        callback(null, data.map((i) => i._id));
+      }
+    });
+
+    this.schedulePing();
+  }
+
+  set(key, value, callback) {
+    if (key.length > 100) {
+      callback('Your Key can only be 100 chars');
+    } else {
+      this.collection.update({_id: key}, {$set: {value}}, {upsert: true}, callback);
+    }
+
+    this.schedulePing();
+  }
+
+  remove(key, callback) {
+    this.collection.remove({_id: key}, callback);
+
+    this.schedulePing();
+  }
+
+  doBulk(bulk, callback) {
+    const bulkMongo = this.collection.initializeOrderedBulkOp();
+
+    for (const i in bulk) {
+      if (bulk[i].type === 'set') {
+        bulkMongo.find({_id: bulk[i].key}).upsert().updateOne({$set: {value: bulk[i].value}});
+      } else if (bulk[i].type === 'remove') {
+        bulkMongo.find({_id: bulk[i].key}).deleteOne();
+      }
+    }
+
+    bulkMongo.execute().then((res) => {
+      callback(null, res);
+    }).catch((error) => {
+      callback(error);
+    });
+
+    this.schedulePing();
+  }
+
+  close(callback) {
+    this.clearPing();
+    this.client.close(callback);
+  }
+};

package/databases/mssql_db.js

@@ -0,0 +1,218 @@
+'use strict';
+/* eslint new-cap: ["error", {"capIsNewExceptions": ["mssql.NVarChar"]}] */
+
+/**
+ * 2019 - exspecto@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * Note: This requires MS SQL Server >= 2008 due to the usage of the MERGE statement
+ *
+ */
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+const async = require('async');
+const mssql = require('mssql');
+
+exports.Database = class extends AbstractDatabase {
+  constructor(settings) {
+    super();
+    settings = settings || {};
+
+    if (settings.json != null) {
+      settings.parseJSON = settings.json;
+    }
+
+    // set the request timeout to 5 minutes
+    settings.requestTimeout = 300000;
+
+    settings.server = settings.host;
+    this.settings = settings;
+
+    /*
+      Turning off the cache and write buffer here. You
+      can reenable it, but also take a look at maxInserts in
+      the doBulk function to decide how you want to split it up.
+    */
+    this.settings.cache = 0;
+    this.settings.writeInterval = 0;
+  }
+
+  init(callback) {
+    const sqlCreate =
+        "IF OBJECT_ID(N'dbo.store', N'U') IS NULL" +
+        '  BEGIN' +
+        '    CREATE TABLE [store] (' +
+        '      [key] NVARCHAR(100) PRIMARY KEY,' +
+        '      [value] NTEXT NOT NULL' +
+        '    );' +
+        '  END';
+
+    new mssql.ConnectionPool(this.settings).connect().then((pool) => {
+      this.db = pool;
+
+      const request = new mssql.Request(this.db);
+
+      request.query(sqlCreate, (err) => {
+        callback(err);
+      });
+
+      this.db.on('error', (err) => {
+        console.log(err);
+      });
+    });
+  }
+
+  get(key, callback) {
+    const request = new mssql.Request(this.db);
+
+    request.input('key', mssql.NVarChar(100), key);
+
+    request.query('SELECT [value] FROM [store] WHERE [key] = @key', (err, results) => {
+      let value = null;
+
+      if (!err && results.rowsAffected[0] === 1) {
+        value = results.recordset[0].value;
+      }
+
+      callback(err, value);
+    });
+  }
+
+  findKeys(key, notKey, callback) {
+    const request = new mssql.Request(this.db);
+    let query = 'SELECT [key] FROM [store] WHERE [key] LIKE @key';
+
+    // desired keys are key, e.g. pad:%
+    key = key.replace(/\*/g, '%');
+
+    request.input('key', mssql.NVarChar(100), key);
+
+    if (notKey != null) {
+      // not desired keys are notKey, e.g. %:%:%
+      notKey = notKey.replace(/\*/g, '%');
+      request.input('notkey', mssql.NVarChar(100), notKey);
+      query += ' AND [key] NOT LIKE @notkey';
+    }
+
+    request.query(query, (err, results) => {
+      const value = [];
+
+      if (!err && results.rowsAffected[0] > 0) {
+        for (let i = 0; i < results.recordset.length; i++) {
+          value.push(results.recordset[i].key);
+        }
+      }
+
+      callback(err, value);
+    });
+  }
+
+  set(key, value, callback) {
+    const request = new mssql.Request(this.db);
+
+    if (key.length > 100) {
+      callback('Your Key can only be 100 chars');
+    } else {
+      const query =
+          'MERGE [store] t USING (SELECT @key [key], @value [value]) s' +
+          ' ON t.[key] = s.[key]' +
+          ' WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value]' +
+          ' WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);';
+
+      request.input('key', mssql.NVarChar(100), key);
+      request.input('value', mssql.NText, value);
+
+      request.query(query, (err, info) => {
+        callback(err);
+      });
+    }
+  }
+
+  remove(key, callback) {
+    const request = new mssql.Request(this.db);
+    request.input('key', mssql.NVarChar(100), key);
+    request.query('DELETE FROM [store] WHERE [key] = @key', callback);
+  }
+
+  doBulk(bulk, callback) {
+    const maxInserts = 100;
+    const request = new mssql.Request(this.db);
+    let firstReplace = true;
+    let firstRemove = true;
+    const replacements = [];
+    let removeSQL = 'DELETE FROM [store] WHERE [key] IN (';
+
+    for (const i in bulk) {
+      if (bulk[i].type === 'set') {
+        if (firstReplace) {
+          replacements.push('BEGIN TRANSACTION;');
+          firstReplace = false;
+        } else if (i % maxInserts === 0) {
+          replacements.push('\nCOMMIT TRANSACTION;\nBEGIN TRANSACTION;\n');
+        }
+
+        replacements.push(
+            `MERGE [store] t USING (SELECT '${bulk[i].key}' [key], '${bulk[i].value}' [value]) s`,
+            'ON t.[key] = s.[key]',
+            'WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value]',
+            'WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);');
+      } else if (bulk[i].type === 'remove') {
+        if (!firstRemove) {
+          removeSQL += ',';
+        }
+
+        firstRemove = false;
+        removeSQL += `'${bulk[i].key}'`;
+      }
+    }
+
+    removeSQL += ');';
+    replacements.push('COMMIT TRANSACTION;');
+
+    async.parallel(
+        [
+          (callback) => {
+            if (!firstReplace) {
+              request.batch(replacements.join('\n'), (err, results) => {
+                if (err) {
+                  callback(err);
+                }
+                callback(err, results);
+              });
+            } else {
+              callback();
+            }
+          },
+          (callback) => {
+            if (!firstRemove) {
+              request.query(removeSQL, callback);
+            } else {
+              callback();
+            }
+          },
+        ],
+        (err, results) => {
+          if (err) {
+            callback(err);
+          }
+          callback(err, results);
+        }
+    );
+  }
+
+  close(callback) {
+    this.db.close(callback);
+  }
+};

package/databases/mysql_db.js

@@ -0,0 +1,178 @@
+'use strict';
+/**
+ * 2011 Peter 'Pita' Martischka
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AbstractDatabase = require('../lib/AbstractDatabase');
+const mysql = require('mysql');
+const util = require('util');
+
+exports.Database = class extends AbstractDatabase {
+  constructor(settings) {
+    super();
+    this.logger = console;
+    this._mysqlSettings = {
+      charset: 'utf8mb4', // temp hack needs a proper fix..
+      ...settings,
+    };
+    this.settings = {
+      engine: 'InnoDB',
+      // Limit the query size to avoid timeouts or other failures.
+      bulkLimit: 100,
+      json: true,
+      queryTimeout: 60000,
+    };
+    this._pool = null; // Initialized in init();
+  }
+
+  get isAsync() { return true; }
+
+  async _query(options) {
+    try {
+      return await new Promise((resolve, reject) => {
+        options = {timeout: this.settings.queryTimeout, ...options};
+        this._pool.query(options, (err, ...args) => err != null ? reject(err) : resolve(args));
+      });
+    } catch (err) {
+      this.logger.error(`${err.fatal ? 'Fatal ' : ''}MySQL error: ${err.stack || err}`);
+      throw err;
+    }
+  }
+
+  async init() {
+    this._pool = mysql.createPool(this._mysqlSettings);
+    const {database, charset} = this._mysqlSettings;
+
+    const sqlCreate = `${'CREATE TABLE IF NOT EXISTS `store` ( ' +
+                  '`key` VARCHAR( 100 ) NOT NULL COLLATE utf8mb4_bin, ' +
+                  '`value` LONGTEXT COLLATE utf8mb4_bin NOT NULL , ' +
+                  'PRIMARY KEY ( `key` ) ' +
+                  ') ENGINE='}${this.settings.engine} CHARSET=utf8mb4 COLLATE=utf8mb4_bin;`;
+
+    const sqlAlter = 'ALTER TABLE store MODIFY `key` VARCHAR(100) COLLATE utf8mb4_bin;';
+
+    await this._query({sql: sqlCreate});
+
+    // Checks for Database charset et al
+    const dbCharSet =
+        'SELECT DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME ' +
+        `FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = '${database}'`;
+    let [result] = await this._query({sql: dbCharSet});
+
+    result = JSON.parse(JSON.stringify(result));
+    if (result[0].DEFAULT_CHARACTER_SET_NAME !== charset) {
+      this.logger.error(`Database is not configured with charset ${charset} -- ` +
+                        'This may lead to crashes when certain characters are pasted in pads');
+      this.logger.log(result[0], charset);
+    }
+
+    if (result[0].DEFAULT_COLLATION_NAME.indexOf(charset) === -1) {
+      this.logger.error(
+          `Database is not configured with collation name that includes ${charset} -- ` +
+            'This may lead to crashes when certain characters are pasted in pads');
+      this.logger.log(result[0], charset, result[0].DEFAULT_COLLATION_NAME);
+    }
+
+    const tableCharSet =
+        'SELECT CCSA.character_set_name AS character_set_name ' +
+        'FROM information_schema.`TABLES` ' +
+        'T,information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY` CCSA ' +
+        'WHERE CCSA.collation_name = T.table_collation ' +
+        `AND T.table_schema = '${database}' ` +
+        "AND T.table_name = 'store'";
+    [result] = await this._query({sql: tableCharSet});
+    if (!result[0]) {
+      this.logger.warn('Data has no character_set_name value -- ' +
+                       'This may lead to crashes when certain characters are pasted in pads');
+    }
+    if (result[0] && (result[0].character_set_name !== charset)) {
+      this.logger.error(`table is not configured with charset ${charset} -- ` +
+                        'This may lead to crashes when certain characters are pasted in pads');
+      this.logger.log(result[0], charset);
+    }
+
+    // check migration level, alter if not migrated
+    const level = await this.get('MYSQL_MIGRATION_LEVEL');
+
+    if (level !== '1') {
+      await this._query({sql: sqlAlter});
+      await this.set('MYSQL_MIGRATION_LEVEL', '1');
+    }
+  }
+
+  async get(key) {
+    const [results] = await this._query({
+      sql: 'SELECT `value` FROM `store` WHERE `key` = ? AND BINARY `key` = ?',
+      values: [key, key],
+    });
+    return results.length === 1 ? results[0].value : null;
+  }
+
+  async findKeys(key, notKey) {
+    let query = 'SELECT `key` FROM `store` WHERE `key` LIKE ?';
+    const params = [];
+
+    // desired keys are key, e.g. pad:%
+    key = key.replace(/\*/g, '%');
+    params.push(key);
+
+    if (notKey != null) {
+      // not desired keys are notKey, e.g. %:%:%
+      notKey = notKey.replace(/\*/g, '%');
+      query += ' AND `key` NOT LIKE ?';
+      params.push(notKey);
+    }
+    const [results] = await this._query({sql: query, values: params});
+    return results.map((val) => val.key);
+  }
+
+  async set(key, value) {
+    if (key.length > 100) throw new Error('Your Key can only be 100 chars');
+    await this._query({sql: 'REPLACE INTO `store` VALUES (?,?)', values: [key, value]});
+  }
+
+  async remove(key) {
+    await this._query({
+      sql: 'DELETE FROM `store` WHERE `key` = ? AND BINARY `key` = ?',
+      values: [key, key],
+    });
+  }
+
+  async doBulk(bulk) {
+    const replaces = [];
+    const deletes = [];
+    for (const op of bulk) {
+      switch (op.type) {
+        case 'set': replaces.push([op.key, op.value]); break;
+        case 'remove': deletes.push(op.key); break;
+        default: throw new Error(`unknown op type: ${op.type}`);
+      }
+    }
+    await Promise.all([
+      replaces.length ? this._query({
+        sql: 'REPLACE INTO `store` VALUES ?;',
+        values: [replaces],
+      }) : null,
+      deletes.length ? this._query({
+        sql: 'DELETE FROM `store` WHERE `key` IN (?) AND BINARY `key` IN (?);',
+        values: [deletes, deletes],
+      }) : null,
+    ]);
+  }
+
+  async close() {
+    await util.promisify(this._pool.end.bind(this._pool))();
+  }
+};