ueberdb2 4.0.11 → 4.0.17

package/dist/databases/elasticsearch_db.js

@@ -0,0 +1,251 @@
+"use strict";
+/**
+ * 2015 Visionist, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Database = void 0;
+const AbstractDatabase_1 = __importDefault(require("../lib/AbstractDatabase"));
+const assert_1 = require("assert");
+const buffer_1 = require("buffer");
+const crypto_1 = require("crypto");
+const elasticsearch7_1 = require("elasticsearch7");
+const schema = '2';
+const keyToId = (key) => {
+    const keyBuf = buffer_1.Buffer.from(key);
+    return keyBuf.length > 512 ? (0, crypto_1.createHash)('sha512').update(keyBuf).digest('hex') : key;
+};
+const mappings = {
+    // _id is expected to equal key, unless the UTF-8 encoded key is > 512 bytes, in which case it is
+    // the hex-encoded sha512 hash of the UTF-8 encoded key.
+    properties: {
+        key: { type: 'wildcard' },
+        value: { type: 'object', enabled: false }, // Values should be opaque to Elasticsearch.
+    },
+};
+const migrateToSchema2 = async (client, v1BaseIndex, v2Index, logger) => {
+    let recordsMigratedLastLogged = 0;
+    let recordsMigrated = 0;
+    const totals = new Map();
+    logger.info('Attempting elasticsearch record migration from schema v1 at base index ' +
+        `${v1BaseIndex} to schema v2 at index ${v2Index}...`);
+    const { body: indices } = await client.indices.get({ index: [v1BaseIndex, `${v1BaseIndex}-*-*`] });
+    const scrollIds = new Map();
+    const q = [];
+    try {
+        for (const index of Object.keys(indices)) {
+            const { body: res } = await client.search({ index, scroll: '10m' });
+            scrollIds.set(index, res._scroll_id);
+            q.push({ index, res });
+        }
+        while (q.length) {
+            const { index, res: { hits: { hits, total: { value: total } } } } = q.shift();
+            if (hits.length === 0)
+                continue;
+            totals.set(index, total);
+            const body = [];
+            for (const { _id, _type, _source: { val } } of hits) {
+                let key = `${_type}:${_id}`;
+                if (v1BaseIndex && index !== v1BaseIndex) {
+                    const parts = index.slice(v1BaseIndex.length + 1).split('-');
+                    if (parts.length !== 2) {
+                        throw new Error(`unable to migrate records from index ${index} due to data ambiguity`);
+                    }
+                    key = `${parts[0]}:${decodeURIComponent(_type)}:${parts[1]}:${_id}`;
+                }
+                body.push({ index: { _id: keyToId(key) } }, { key, value: JSON.parse(val) });
+            }
+            await client.bulk({ index: v2Index, body });
+            recordsMigrated += hits.length;
+            if (Math.floor(recordsMigrated / 100) > Math.floor(recordsMigratedLastLogged / 100)) {
+                // @ts-ignore
+                const total = [...totals.values()].reduce((a, b) => a + b, 0);
+                logger.info(`Migrated ${recordsMigrated} records out of ${total}`);
+                recordsMigratedLastLogged = recordsMigrated;
+            }
+            q.push({ index, res: (await client.scroll({ scroll: '5m', scrollId: scrollIds.get(index) })).body });
+        }
+        logger.info(`Finished migrating ${recordsMigrated} records`);
+    }
+    finally {
+        // @ts-ignore
+        await Promise.all([...scrollIds.values()].map((scrollId) => client.clearScroll({ scrollId })));
+    }
+};
+const Database = class extends AbstractDatabase_1.default {
+    _client;
+    _index;
+    _indexClean;
+    _q;
+    constructor(settings) {
+        super();
+        this._client = null;
+        this.settings = {
+            host: '127.0.0.1',
+            port: '9200',
+            base_index: 'ueberes',
+            migrate_to_newer_schema: false,
+            // for a list of valid API values see:
+            // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html#config-options
+            api: '7.6',
+            ...settings || {},
+            json: false, // Elasticsearch will do the JSON conversion as necessary.
+        };
+        this._index = `${this.settings.base_index}_s${schema}`;
+        this._q = { index: this._index };
+        this._indexClean = true;
+    }
+    get isAsync() { return true; }
+    async _refreshIndex() {
+        if (this._indexClean)
+            return;
+        this._indexClean = true;
+        await this._client.indices.refresh(this._q);
+    }
+    /**
+     * Initialize the elasticsearch client, then ping the server to ensure that a
+     * connection was made.
+     */
+    async init() {
+        // create elasticsearch client
+        const client = new elasticsearch7_1.Client({
+            node: `http://${this.settings.host}:${this.settings.port}`,
+        });
+        await client.ping();
+        if (!(await client.indices.exists({ index: this._index })).body) {
+            let tmpIndex;
+            // @ts-ignore
+            const { body: migrate } = await client.indices.exists({ index: this.settings.base_index });
+            if (migrate && !this.settings.migrate_to_newer_schema) {
+                throw new Error(`Data exists under the legacy index (schema) named ${this.settings.base_index}. ` +
+                    'Set migrate_to_newer_schema to true to copy the existing data to a new index ' +
+                    `named ${this._index}.`);
+            }
+            let attempt = 0;
+            while (true) {
+                tmpIndex = `${this._index}_${migrate ? 'migrate_attempt_' : 'i'}${attempt++}`;
+                if (!(await client.indices.exists({ index: tmpIndex })).body)
+                    break;
+            }
+            await client.indices.create({ index: tmpIndex, body: { mappings } });
+            if (migrate)
+                await migrateToSchema2(client, this.settings.base_index, tmpIndex, this.logger);
+            await client.indices.putAlias({ index: tmpIndex, name: this._index });
+        }
+        const indices = Object.values((await client.indices.get({ index: this._index })).body);
+        (0, assert_1.equal)(indices.length, 1);
+        try {
+            // @ts-ignore
+            assert.deepEqual(indices[0].mappings, mappings);
+        }
+        catch (err) {
+            this.logger.warn(`Index ${this._index} mappings does not match expected; ` +
+                `attempting to use index anyway. Details: ${err}`);
+        }
+        this._client = client;
+    }
+    /**
+     *  This function provides read functionality to the database.
+     *
+     *  @param {String} key Key
+     */
+    async get(key) {
+        const { body } = await this._client.get({ ...this._q, id: keyToId(key) }, { ignore: [404] });
+        if (!body.found)
+            return null;
+        return body._source.value;
+    }
+    /**
+     *  @param key Search key, which uses an asterisk (*) as the wild card.
+     *  @param notKey Used to filter the result set
+     */
+    async findKeys(key, notKey) {
+        await this._refreshIndex();
+        const q = {
+            ...this._q,
+            body: {
+                query: {
+                    bool: {
+                        filter: { wildcard: { key: { value: key } } },
+                        ...notKey == null ? {} : {
+                            must_not: { wildcard: { key: { value: notKey } } },
+                        },
+                    },
+                },
+            },
+        };
+        const { body: { hits: { hits } } } = await this._client.search(q);
+        return hits.map((h) => h._source.key);
+    }
+    /**
+     *  This function provides write functionality to the database.
+     *
+     *  @param {String} key Record identifier.
+     *  @param {JSON|String} value The value to store in the database.
+     */
+    async set(key, value) {
+        this._indexClean = false;
+        await this._client.index({ ...this._q, id: keyToId(key), body: { key, value } });
+    }
+    /**
+     *  This function provides delete functionality to the database.
+     *
+     *  The index, type, and ID will be parsed from the key, and this document will
+     *  be deleted from the database.
+     *
+     *  @param {String} key Record identifier.
+     */
+    async remove(key) {
+        this._indexClean = false;
+        await this._client.delete({ ...this._q, id: keyToId(key) }, { ignore: [404] });
+    }
+    /**
+     *  This uses the bulk upload functionality of elasticsearch (url:port/_bulk).
+     *
+     *  The CacheAndBufferLayer will periodically (every this.settings.writeInterval)
+     *  flush writes that have already been done in the local cache out to the database.
+     *
+     *  @param {Array} bulk An array of JSON data in the format:
+     *      {"type":type, "key":key, "value":value}
+     */
+    async doBulk(bulk) {
+        // bulk is an array of JSON:
+        // example: [{"type":"set", "key":"sessionstorage:{id}", "value":{"cookie":{...}}]
+        const operations = [];
+        for (const { type, key, value } of bulk) {
+            this._indexClean = false;
+            switch (type) {
+                case 'set':
+                    operations.push({ index: { _id: keyToId(key) } });
+                    operations.push({ key, value });
+                    break;
+                case 'remove':
+                    operations.push({ delete: { _id: keyToId(key) } });
+                    break;
+                default:
+                    continue;
+            }
+        }
+        await this._client.bulk({ ...this._q, body: operations });
+    }
+    async close() {
+        if (this._client != null)
+            this._client.close();
+        this._client = null;
+    }
+};
+exports.Database = Database;

package/dist/databases/memory_db.js

@@ -0,0 +1,39 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Database = void 0;
+const AbstractDatabase_1 = __importDefault(require("../lib/AbstractDatabase"));
+const Database = class MemoryDB extends AbstractDatabase_1.default {
+    _data;
+    constructor(settings) {
+        super();
+        this.settings = settings;
+        settings.json = false;
+        settings.cache = 0;
+        settings.writeInterval = 0;
+        this._data = null;
+    }
+    get isAsync() { return true; }
+    close() {
+        this._data = null;
+    }
+    findKeys(key, notKey) {
+        const regex = this.createFindRegex(key, notKey);
+        return [...this._data.keys()].filter((k) => regex.test(k));
+    }
+    get(key) {
+        return this._data.get(key);
+    }
+    init() {
+        this._data = this.settings.data || new Map();
+    }
+    remove(key) {
+        this._data.delete(key);
+    }
+    set(key, value) {
+        this._data.set(key, value);
+    }
+};
+exports.Database = Database;

package/dist/databases/mock_db.js

@@ -0,0 +1,40 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Database = void 0;
+const events_1 = __importDefault(require("events"));
+const Database = class extends events_1.default.EventEmitter {
+    settings;
+    constructor(settings) {
+        super();
+        this.settings = {
+            writeInterval: 1,
+            ...settings,
+        };
+        settings.mock = this;
+    }
+    close(cb) {
+        this.emit('close', cb);
+    }
+    doBulk(ops, cb) {
+        this.emit('doBulk', ops, cb);
+    }
+    findKeys(key, notKey, cb) {
+        this.emit('findKeys', key, notKey, cb);
+    }
+    get(key, cb) {
+        this.emit('get', key, cb);
+    }
+    init(cb) {
+        this.emit('init', cb);
+    }
+    remove(key, cb) {
+        this.emit('remove', key, cb);
+    }
+    set(key, value, cb) {
+        this.emit('set', key, value, cb);
+    }
+};
+exports.Database = Database;

package/dist/databases/mongodb_db.js

@@ -0,0 +1,127 @@
+"use strict";
+/**
+ * 2020 Sylchauf
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const AbstractDatabase_1 = __importDefault(require("../lib/AbstractDatabase"));
+exports.Database = class extends AbstractDatabase_1.default {
+    interval;
+    database;
+    client;
+    collection;
+    constructor(settings) {
+        super();
+        this.settings = settings;
+        if (!this.settings.url)
+            throw new Error('You must specify a mongodb url');
+        // For backwards compatibility:
+        if (this.settings.database == null)
+            this.settings.database = this.settings.dbName;
+        if (!this.settings.collection)
+            this.settings.collection = 'ueberdb';
+    }
+    clearPing() {
+        if (this.interval) {
+            clearInterval(this.interval);
+        }
+    }
+    schedulePing() {
+        this.clearPing();
+        this.interval = setInterval(() => {
+            this.database.command({
+                ping: 1,
+            });
+        }, 10000);
+    }
+    init(callback) {
+        const MongoClient = require('mongodb').MongoClient;
+        MongoClient.connect(this.settings.url, (err, client) => {
+            if (!err) {
+                this.client = client;
+                this.database = client.db(this.settings.database);
+                this.collection = this.database.collection(this.settings.collection);
+            }
+            callback(err);
+        });
+        this.schedulePing();
+    }
+    get(key, callback) {
+        this.collection.findOne({ _id: key }, (err, document) => {
+            if (err)
+                callback(err);
+            else
+                callback(null, document ? document.value : null);
+        });
+        this.schedulePing();
+    }
+    findKeys(key, notKey, callback) {
+        const selector = {
+            $and: [
+                { _id: { $regex: `${key.replace(/\*/g, '')}` } },
+            ],
+        };
+        if (notKey) {
+            // @ts-ignore
+            selector.$and.push({ _id: { $not: { $regex: `${notKey.replace(/\*/g, '')}` } } });
+        }
+        this.collection.find(selector, async (err, res) => {
+            if (err) {
+                callback(err);
+            }
+            else {
+                const data = await res.toArray();
+                callback(null, data.map((i) => i._id));
+            }
+        });
+        this.schedulePing();
+    }
+    set(key, value, callback) {
+        if (key.length > 100) {
+            callback('Your Key can only be 100 chars');
+        }
+        else {
+            this.collection.update({ _id: key }, { $set: { value } }, { upsert: true }, callback);
+        }
+        this.schedulePing();
+    }
+    remove(key, callback) {
+        this.collection.remove({ _id: key }, callback);
+        this.schedulePing();
+    }
+    doBulk(bulk, callback) {
+        const bulkMongo = this.collection.initializeOrderedBulkOp();
+        for (const i in bulk) {
+            if (bulk[i].type === 'set') {
+                bulkMongo.find({ _id: bulk[i].key }).upsert().updateOne({ $set: { value: bulk[i].value } });
+            }
+            else if (bulk[i].type === 'remove') {
+                bulkMongo.find({ _id: bulk[i].key }).deleteOne();
+            }
+        }
+        bulkMongo.execute().then((res) => {
+            callback(null, res);
+        }).catch((error) => {
+            callback(error);
+        });
+        this.schedulePing();
+    }
+    close(callback) {
+        this.clearPing();
+        this.client.close(callback);
+    }
+};

package/dist/databases/mssql_db.js

@@ -0,0 +1,187 @@
+"use strict";
+/* eslint new-cap: ["error", {"capIsNewExceptions": ["mssql.NVarChar"]}] */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Database = void 0;
+/**
+ * 2019 - exspecto@gmail.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * Note: This requires MS SQL Server >= 2008 due to the usage of the MERGE statement
+ *
+ */
+const AbstractDatabase_1 = __importDefault(require("../lib/AbstractDatabase"));
+const async_1 = __importDefault(require("async"));
+const mssql_1 = __importDefault(require("mssql"));
+const Database = class MSSQL extends AbstractDatabase_1.default {
+    db;
+    constructor(settings) {
+        super();
+        settings = settings || {};
+        if (settings.json != null) {
+            settings.parseJSON = settings.json;
+        }
+        // set the request timeout to 5 minutes
+        settings.requestTimeout = 300000;
+        settings.server = settings.host;
+        this.settings = settings;
+        /*
+          Turning off the cache and write buffer here. You
+          can reenable it, but also take a look at maxInserts in
+          the doBulk function to decide how you want to split it up.
+        */
+        this.settings.cache = 0;
+        this.settings.writeInterval = 0;
+    }
+    init(callback) {
+        const sqlCreate = "IF OBJECT_ID(N'dbo.store', N'U') IS NULL" +
+            '  BEGIN' +
+            '    CREATE TABLE [store] (' +
+            '      [key] NVARCHAR(100) PRIMARY KEY,' +
+            '      [value] NTEXT NOT NULL' +
+            '    );' +
+            '  END';
+        // @ts-ignore
+        new mssql_1.default.ConnectionPool(this.settings).connect().then((pool) => {
+            this.db = pool;
+            const request = new mssql_1.default.Request(this.db);
+            request.query(sqlCreate, (err) => {
+                callback(err);
+            });
+            this.db.on('error', (err) => {
+                console.log(err);
+            });
+        });
+    }
+    get(key, callback) {
+        const request = new mssql_1.default.Request(this.db);
+        request.input('key', mssql_1.default.NVarChar(100), key);
+        request.query('SELECT [value] FROM [store] WHERE [key] = @key', (err, results) => {
+            let value = null;
+            if (!err && results && results.rowsAffected[0] === 1) {
+                // @ts-ignore
+                value = results.recordset[0].value;
+            }
+            callback(err, value);
+        });
+    }
+    findKeys(key, notKey, callback) {
+        const request = new mssql_1.default.Request(this.db);
+        let query = 'SELECT [key] FROM [store] WHERE [key] LIKE @key';
+        // desired keys are key, e.g. pad:%
+        key = key.replace(/\*/g, '%');
+        request.input('key', mssql_1.default.NVarChar(100), key);
+        if (notKey != null) {
+            // not desired keys are notKey, e.g. %:%:%
+            notKey = notKey.replace(/\*/g, '%');
+            request.input('notkey', mssql_1.default.NVarChar(100), notKey);
+            query += ' AND [key] NOT LIKE @notkey';
+        }
+        request.query(query, (err, results) => {
+            const value = [];
+            if (!err && results && results.rowsAffected[0] > 0) {
+                for (let i = 0; i < results.recordset.length; i++) {
+                    value.push(results.recordset[i].key);
+                }
+            }
+            callback(err, value);
+        });
+    }
+    set(key, value, callback) {
+        const request = new mssql_1.default.Request(this.db);
+        if (key.length > 100) {
+            callback('Your Key can only be 100 chars');
+        }
+        else {
+            const query = 'MERGE [store] t USING (SELECT @key [key], @value [value]) s' +
+                ' ON t.[key] = s.[key]' +
+                ' WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value]' +
+                ' WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);';
+            request.input('key', mssql_1.default.NVarChar(100), key);
+            request.input('value', mssql_1.default.NText, value);
+            request.query(query, (err, info) => {
+                callback(err ? err.toString() : '');
+            });
+        }
+    }
+    remove(key, callback) {
+        const request = new mssql_1.default.Request(this.db);
+        request.input('key', mssql_1.default.NVarChar(100), key);
+        request.query('DELETE FROM [store] WHERE [key] = @key', callback);
+    }
+    doBulk(bulk, callback) {
+        const maxInserts = 100;
+        const request = new mssql_1.default.Request(this.db);
+        let firstReplace = true;
+        let firstRemove = true;
+        const replacements = [];
+        let removeSQL = 'DELETE FROM [store] WHERE [key] IN (';
+        for (const i in bulk) {
+            if (bulk[i].type === 'set') {
+                if (firstReplace) {
+                    replacements.push('BEGIN TRANSACTION;');
+                    firstReplace = false;
+                }
+                else if (Number(i) % maxInserts === 0) {
+                    replacements.push('\nCOMMIT TRANSACTION;\nBEGIN TRANSACTION;\n');
+                }
+                replacements.push(`MERGE [store] t USING (SELECT '${bulk[i].key}' [key], '${bulk[i].value}' [value]) s`, 'ON t.[key] = s.[key]', 'WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value]', 'WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);');
+            }
+            else if (bulk[i].type === 'remove') {
+                if (!firstRemove) {
+                    removeSQL += ',';
+                }
+                firstRemove = false;
+                removeSQL += `'${bulk[i].key}'`;
+            }
+        }
+        removeSQL += ');';
+        replacements.push('COMMIT TRANSACTION;');
+        async_1.default.parallel([
+            (callback) => {
+                if (!firstReplace) {
+                    request.batch(replacements.join('\n'), (err, results) => {
+                        if (err) {
+                            callback(err);
+                        }
+                        callback(err, results);
+                    });
+                }
+                else {
+                    callback();
+                }
+            },
+            (callback) => {
+                if (!firstRemove) {
+                    request.query(removeSQL, callback);
+                }
+                else {
+                    callback();
+                }
+            },
+        ], (err, results) => {
+            if (err) {
+                callback(err);
+            }
+            callback(err, results);
+        });
+    }
+    close(callback) {
+        this.db && this.db.close(callback);
+    }
+};
+exports.Database = Database;