ueberdb2 4.0.11 → 4.0.17

package/dist/databases/mysql_db.js

@@ -0,0 +1,170 @@
+"use strict";
+/**
+ * 2011 Peter 'Pita' Martischka
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Database = void 0;
+const AbstractDatabase_1 = __importDefault(require("../lib/AbstractDatabase"));
+const mysql_1 = __importDefault(require("mysql"));
+const util_1 = __importDefault(require("util"));
+const Database = class extends AbstractDatabase_1.default {
+    _mysqlSettings;
+    _pool;
+    constructor(settings) {
+        super();
+        this.logger = console;
+        this._mysqlSettings = {
+            charset: 'utf8mb4',
+            ...settings,
+        };
+        this.settings = {
+            engine: 'InnoDB',
+            // Limit the query size to avoid timeouts or other failures.
+            bulkLimit: 100,
+            json: true,
+            queryTimeout: 60000,
+        };
+        this._pool = null; // Initialized in init();
+    }
+    get isAsync() { return true; }
+    async _query(options) {
+        try {
+            return await new Promise((resolve, reject) => {
+                options = { timeout: this.settings.queryTimeout, ...options };
+                // @ts-ignore
+                this._pool && this._pool.query(options, (err, ...args) => err != null ? reject(err) : resolve(args));
+            });
+        }
+        catch (err) {
+            this.logger.error(`${err.fatal ? 'Fatal ' : ''}MySQL error: ${err.stack || err}`);
+            throw err;
+        }
+    }
+    async init() {
+        // @ts-ignore
+        this._pool = mysql_1.default.createPool(this._mysqlSettings);
+        const { database, charset } = this._mysqlSettings;
+        const sqlCreate = `${'CREATE TABLE IF NOT EXISTS `store` ( ' +
+            '`key` VARCHAR( 100 ) NOT NULL COLLATE utf8mb4_bin, ' +
+            '`value` LONGTEXT COLLATE utf8mb4_bin NOT NULL , ' +
+            'PRIMARY KEY ( `key` ) ' +
+            ') ENGINE='}${this.settings.engine} CHARSET=utf8mb4 COLLATE=utf8mb4_bin;`;
+        const sqlAlter = 'ALTER TABLE store MODIFY `key` VARCHAR(100) COLLATE utf8mb4_bin;';
+        await this._query({ sql: sqlCreate });
+        // Checks for Database charset et al
+        const dbCharSet = 'SELECT DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME ' +
+            `FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = '${database}'`;
+        // @ts-ignore
+        let [result] = await this._query({ sql: dbCharSet });
+        result = JSON.parse(JSON.stringify(result));
+        if (result[0].DEFAULT_CHARACTER_SET_NAME !== charset) {
+            this.logger.error(`Database is not configured with charset ${charset} -- ` +
+                'This may lead to crashes when certain characters are pasted in pads');
+            this.logger.log(result[0], charset);
+        }
+        if (result[0].DEFAULT_COLLATION_NAME.indexOf(charset) === -1) {
+            this.logger.error(`Database is not configured with collation name that includes ${charset} -- ` +
+                'This may lead to crashes when certain characters are pasted in pads');
+            this.logger.log(result[0], charset, result[0].DEFAULT_COLLATION_NAME);
+        }
+        const tableCharSet = 'SELECT CCSA.character_set_name AS character_set_name ' +
+            'FROM information_schema.`TABLES` ' +
+            'T,information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY` CCSA ' +
+            'WHERE CCSA.collation_name = T.table_collation ' +
+            `AND T.table_schema = '${database}' ` +
+            "AND T.table_name = 'store'";
+        [result] = await this._query({ sql: tableCharSet });
+        if (!result[0]) {
+            this.logger.warn('Data has no character_set_name value -- ' +
+                'This may lead to crashes when certain characters are pasted in pads');
+        }
+        if (result[0] && (result[0].character_set_name !== charset)) {
+            this.logger.error(`table is not configured with charset ${charset} -- ` +
+                'This may lead to crashes when certain characters are pasted in pads');
+            this.logger.log(result[0], charset);
+        }
+        // check migration level, alter if not migrated
+        const level = await this.get('MYSQL_MIGRATION_LEVEL');
+        if (level !== '1') {
+            await this._query({ sql: sqlAlter });
+            await this.set('MYSQL_MIGRATION_LEVEL', '1');
+        }
+    }
+    async get(key) {
+        const [results] = await this._query({
+            sql: 'SELECT `value` FROM `store` WHERE `key` = ? AND BINARY `key` = ?',
+            values: [key, key],
+        });
+        return results.length === 1 ? results[0].value : null;
+    }
+    async findKeys(key, notKey) {
+        let query = 'SELECT `key` FROM `store` WHERE `key` LIKE ?';
+        const params = [];
+        // desired keys are key, e.g. pad:%
+        key = key.replace(/\*/g, '%');
+        params.push(key);
+        if (notKey != null) {
+            // not desired keys are notKey, e.g. %:%:%
+            notKey = notKey.replace(/\*/g, '%');
+            query += ' AND `key` NOT LIKE ?';
+            params.push(notKey);
+        }
+        const [results] = await this._query({ sql: query, values: params });
+        return results.map((val) => val.key);
+    }
+    async set(key, value) {
+        if (key.length > 100)
+            throw new Error('Your Key can only be 100 chars');
+        await this._query({ sql: 'REPLACE INTO `store` VALUES (?,?)', values: [key, value] });
+    }
+    async remove(key) {
+        await this._query({
+            sql: 'DELETE FROM `store` WHERE `key` = ? AND BINARY `key` = ?',
+            values: [key, key],
+        });
+    }
+    async doBulk(bulk) {
+        const replaces = [];
+        const deletes = [];
+        for (const op of bulk) {
+            switch (op.type) {
+                case 'set':
+                    replaces.push([op.key, op.value]);
+                    break;
+                case 'remove':
+                    deletes.push(op.key);
+                    break;
+                default: throw new Error(`unknown op type: ${op.type}`);
+            }
+        }
+        await Promise.all([
+            replaces.length ? this._query({
+                sql: 'REPLACE INTO `store` VALUES ?;',
+                values: [replaces],
+            }) : null,
+            deletes.length ? this._query({
+                sql: 'DELETE FROM `store` WHERE `key` IN (?) AND BINARY `key` IN (?);',
+                values: [deletes, deletes],
+            }) : null,
+        ]);
+    }
+    async close() {
+        await util_1.default.promisify(this._pool.end.bind(this._pool))();
+    }
+};
+exports.Database = Database;

package/dist/databases/postgres_db.js

@@ -0,0 +1,192 @@
+"use strict";
+/**
+ * 2011 Peter 'Pita' Martischka
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Database = void 0;
+const AbstractDatabase_1 = __importDefault(require("../lib/AbstractDatabase"));
+const async_1 = __importDefault(require("async"));
+const pg_1 = __importDefault(require("pg"));
+const Database = class extends AbstractDatabase_1.default {
+    db;
+    upsertStatement;
+    constructor(settings) {
+        super();
+        if (typeof settings === 'string')
+            settings = { connectionString: settings };
+        this.settings = settings;
+        this.settings.cache = settings.cache || 1000;
+        this.settings.writeInterval = 100;
+        this.settings.json = true;
+        // Pool specific defaults
+        this.settings.max = this.settings.max || 20;
+        this.settings.min = this.settings.min || 4;
+        this.settings.idleTimeoutMillis = this.settings.idleTimeoutMillis || 1000;
+        // @ts-ignore
+        this.db = new pg_1.default.Pool(this.settings);
+    }
+    init(callback) {
+        const testTableExists = "SELECT 1 as exists FROM pg_tables WHERE tablename = 'store'";
+        const createTable = 'CREATE TABLE IF NOT EXISTS store (' +
+            '"key" character varying(100) NOT NULL, ' +
+            '"value" text NOT NULL, ' +
+            'CONSTRAINT store_pkey PRIMARY KEY (key))';
+        // this variable will be given a value depending on the result of the
+        // feature detection
+        this.upsertStatement = null;
+        /*
+         * - Detects if this Postgres version supports INSERT .. ON CONFLICT
+         *   UPDATE (PostgreSQL >= 9.5 and CockroachDB)
+         * - If upsert is not supported natively, creates in the DB a pl/pgsql
+         *   function that emulates it
+         * - Performs a side effect, setting this.upsertStatement to the sql
+         *   statement that needs to be used, based on the detection result
+         * - calls the callback
+         */
+        const detectUpsertMethod = (callback) => {
+            const upsertViaFunction = 'SELECT ueberdb_insert_or_update($1,$2)';
+            const upsertNatively = 'INSERT INTO store(key, value) VALUES ($1, $2) ' +
+                'ON CONFLICT (key) DO UPDATE SET value = excluded.value';
+            const createFunc = 'CREATE OR REPLACE FUNCTION ueberdb_insert_or_update(character varying, text) ' +
+                'RETURNS void AS $$ ' +
+                'BEGIN ' +
+                '  IF EXISTS( SELECT * FROM store WHERE key = $1 ) THEN ' +
+                '    UPDATE store SET value = $2 WHERE key = $1; ' +
+                '  ELSE ' +
+                '    INSERT INTO store(key,value) VALUES( $1, $2 ); ' +
+                '  END IF; ' +
+                '  RETURN; ' +
+                'END; ' +
+                '$$ LANGUAGE plpgsql;';
+            const testNativeUpsert = `EXPLAIN ${upsertNatively}`;
+            this.db.query(testNativeUpsert, ['test-key', 'test-value'], (err) => {
+                if (err) {
+                    // the UPSERT statement failed: we will have to emulate it via
+                    // an sql function
+                    this.upsertStatement = upsertViaFunction;
+                    // actually create the emulation function
+                    this.db.query(createFunc, [], callback);
+                    return;
+                }
+                // if we get here, the EXPLAIN UPSERT succeeded, and we can use a
+                // native UPSERT
+                this.upsertStatement = upsertNatively;
+                callback();
+            });
+        };
+        this.db.query(testTableExists, (err, result) => {
+            if (err != null)
+                return callback(err);
+            if (result.rows.length === 0) {
+                this.db.query(createTable, (err) => {
+                    if (err != null)
+                        return callback(err);
+                    // @ts-ignore
+                    detectUpsertMethod(callback);
+                });
+            }
+            else {
+                // @ts-ignore
+                detectUpsertMethod(callback);
+            }
+        });
+    }
+    get(key, callback) {
+        this.db.query('SELECT value FROM store WHERE key=$1', [key], (err, results) => {
+            let value = null;
+            if (!err && results.rows.length === 1) {
+                value = results.rows[0].value;
+            }
+            callback(err, value);
+        });
+    }
+    findKeys(key, notKey, callback) {
+        let query = 'SELECT key FROM store WHERE key LIKE $1';
+        const params = [];
+        // desired keys are %key:%, e.g. pad:%
+        key = key.replace(/\*/g, '%');
+        params.push(key);
+        if (notKey != null) {
+            // not desired keys are notKey:%, e.g. %:%:%
+            notKey = notKey.replace(/\*/g, '%');
+            query += ' AND key NOT LIKE $2';
+            params.push(notKey);
+        }
+        this.db.query(query, params, (err, results) => {
+            const value = [];
+            if (!err && results.rows.length > 0) {
+                results.rows.forEach((val) => {
+                    value.push(val.key);
+                });
+            }
+            callback(err, value);
+        });
+    }
+    set(key, value, callback) {
+        if (key.length > 100) {
+            const val = '';
+            callback(Error('Your Key can only be 100 chars'), val);
+        }
+        else if (this.upsertStatement != null) {
+            this.db.query(this.upsertStatement, [key, value], callback);
+        }
+    }
+    remove(key, callback) {
+        this.db.query('DELETE FROM store WHERE key=$1', [key], callback);
+    }
+    doBulk(bulk, callback) {
+        const replaceVALs = [];
+        let removeSQL = 'DELETE FROM store WHERE key IN (';
+        const removeVALs = [];
+        let removeCount = 0;
+        for (const i in bulk) {
+            if (bulk[i].type === 'set') {
+                replaceVALs.push([bulk[i].key, bulk[i].value]);
+            }
+            else if (bulk[i].type === 'remove') {
+                if (removeCount !== 0)
+                    removeSQL += ',';
+                removeCount += 1;
+                removeSQL += `$${removeCount}`;
+                removeVALs.push(bulk[i].key);
+            }
+        }
+        removeSQL += ');';
+        if (!this.upsertStatement) {
+            return;
+        }
+        // @ts-ignore
+        const functions = replaceVALs.map((v) => (cb) => this.db.query(this.upsertStatement, v, cb));
+        const removeFunction = (callback) => {
+            // @ts-ignore
+            if (!(removeVALs.length) > 1) {
+                this.db.query(removeSQL, removeVALs, callback);
+            }
+            else {
+                callback();
+            }
+        };
+        functions.push(removeFunction);
+        // @ts-ignore
+        async_1.default.parallel(functions, callback);
+    }
+    close(callback) {
+        this.db.end(callback);
+    }
+};
+exports.Database = Database;

package/dist/databases/postgrespool_db.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Database = void 0;
+const postgres = require('./postgres_db');
+const Database = class PostgresDB extends postgres.Database {
+    constructor(settings) {
+        console.warn('ueberdb: The postgrespool database driver is deprecated ' +
+            'and will be removed in a future version. Use postgres instead.');
+        super(settings);
+    }
+};
+exports.Database = Database;

package/dist/databases/redis_db.js

@@ -0,0 +1,105 @@
+"use strict";
+/**
+ * 2011 Peter 'Pita' Martischka
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Database = void 0;
+const AbstractDatabase_1 = __importDefault(require("../lib/AbstractDatabase"));
+const redis_1 = require("redis");
+const Database = class RedisDB extends AbstractDatabase_1.default {
+    _client;
+    constructor(settings) {
+        super();
+        this._client = null;
+        this.settings = settings || {};
+    }
+    get isAsync() { return true; }
+    async init() {
+        this._client = (0, redis_1.createClient)({ url: this.settings.url });
+        if (this._client) {
+            await this._client.connect();
+            await this._client.ping();
+        }
+    }
+    async get(key) {
+        if (this._client == null)
+            return null;
+        return await this._client.get(key);
+    }
+    async findKeys(key, notKey) {
+        if (this._client == null)
+            return null;
+        const [type] = /^([^:*]+):\*$/.exec(key) || [];
+        if (type != null && ['*:*:*', `${key}:*`].includes(notKey)) {
+            // Performance optimization for a common Etherpad case.
+            return await this._client.sMembers(`ueberDB:keys:${type}`);
+        }
+        let keys = await this._client.keys(key.replace(/[?[\]\\]/g, '\\$&'));
+        if (notKey != null) {
+            const regex = this.createFindRegex(key, notKey);
+            keys = keys.filter((k) => regex.test(k));
+        }
+        return keys;
+    }
+    async set(key, value) {
+        if (this._client == null)
+            return null;
+        const matches = /^([^:]+):([^:]+)$/.exec(key);
+        await Promise.all([
+            matches && this._client.sAdd(`ueberDB:keys:${matches[1]}`, matches[0]),
+            this._client.set(key, value),
+        ]);
+    }
+    async remove(key) {
+        if (this._client == null)
+            return null;
+        const matches = /^([^:]+):([^:]+)$/.exec(key);
+        await Promise.all([
+            matches && this._client.sRem(`ueberDB:keys:${matches[1]}`, matches[0]),
+            this._client.del(key),
+        ]);
+    }
+    async doBulk(bulk) {
+        if (this._client == null)
+            return null;
+        const multi = this._client.multi();
+        for (const { key, type, value } of bulk) {
+            const matches = /^([^:]+):([^:]+)$/.exec(key);
+            if (type === 'set') {
+                if (matches) {
+                    multi.sAdd(`ueberDB:keys:${matches[1]}`, matches[0]);
+                }
+                multi.set(key, value);
+            }
+            else if (type === 'remove') {
+                if (matches) {
+                    multi.sRem(`ueberDB:keys:${matches[1]}`, matches[0]);
+                }
+                multi.del(key);
+            }
+        }
+        await multi.exec();
+    }
+    async close() {
+        if (this._client == null)
+            return null;
+        await this._client.quit();
+        this._client = null;
+    }
+};
+exports.Database = Database;

package/dist/databases/rethink_db.js

@@ -0,0 +1,123 @@
+"use strict";
+/**
+ * 2016 Remi Arnaud
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Database = void 0;
+const AbstractDatabase_1 = __importDefault(require("../lib/AbstractDatabase"));
+const rethinkdb_1 = __importDefault(require("rethinkdb"));
+const async_1 = __importDefault(require("async"));
+const Database = class extends AbstractDatabase_1.default {
+    host;
+    db;
+    port;
+    table;
+    connection;
+    constructor(settings) {
+        super();
+        if (!settings)
+            settings = {};
+        if (!settings.host) {
+            settings.host = 'localhost';
+        }
+        if (!settings.port) {
+            settings.port = 28015;
+        }
+        if (!settings.db) {
+            settings.db = 'test';
+        }
+        if (!settings.table) {
+            settings.table = 'test';
+        }
+        this.host = settings.host;
+        this.db = settings.db;
+        this.port = settings.port;
+        this.table = settings.table;
+        this.connection = null;
+    }
+    init(callback) {
+        // @ts-ignore
+        rethinkdb_1.default.connect(this, (err, conn) => {
+            if (err)
+                throw err;
+            this.connection = conn;
+            rethinkdb_1.default.table(this.table).run(this.connection, (err, cursor) => {
+                if (err) {
+                    // assuming table does not exists
+                    // @ts-ignore
+                    rethinkdb_1.default.tableCreate(this.table).run(this.connection, callback);
+                }
+                else if (callback) {
+                    callback(null, cursor);
+                }
+            });
+        });
+    }
+    get(key, callback) {
+        // @ts-ignore
+        rethinkdb_1.default.table(this.table).get(key).run(this.connection, (err, item) => {
+            // @ts-ignore
+            callback(err, (item ? item.content : item));
+        });
+    }
+    findKeys(key, notKey, callback) {
+        const keys = [];
+        const regex = this.createFindRegex(key, notKey);
+        // @ts-ignore
+        rethinkdb_1.default.filter((item) => {
+            if (item.id.search(regex) !== -1) {
+                keys.push(item.id);
+            }
+        }).run(this.connection, callback);
+    }
+    set(key, value, callback) {
+        rethinkdb_1.default.table(this.table)
+            .insert({ id: key, content: value }, { conflict: 'replace' })
+            .run(this.connection, callback);
+    }
+    doBulk(bulk, callback) {
+        const _in = [];
+        const _out = [];
+        for (const i in bulk) {
+            if (bulk[i].type === 'set') {
+                _in.push({ id: bulk[i].key, content: bulk[i].value });
+            }
+            else if (bulk[i].type === 'remove') {
+                _out.push(bulk[i].key);
+            }
+        }
+        async_1.default.parallel([
+            (cb) => {
+                rethinkdb_1.default.table(this.table).insert(_in, { conflict: 'replace' }).run(this.connection, cb);
+            },
+            (cb) => {
+                rethinkdb_1.default.table(this.table).getAll(_out).delete().run(this.connection, cb);
+            },
+        ], callback);
+    }
+    remove(key, callback) {
+        // @ts-ignore
+        rethinkdb_1.default.table(this.table).get(key).delete().run(this.connection, callback);
+    }
+    close(callback) {
+        if (this.connection) {
+            this.connection.close(callback);
+        }
+    }
+};
+exports.Database = Database;