ueberdb2 4.1.8 → 4.1.10

package/dist/databases/elasticsearch_db.js

@@ -1,269 +1,243 @@
-"use strict";
-/**
- * 2015 Visionist, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS-IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Database = void 0;
-const AbstractDatabase_1 = __importDefault(require("../lib/AbstractDatabase"));
-const assert_1 = __importStar(require("assert"));
-const buffer_1 = require("buffer");
-const crypto_1 = require("crypto");
-const elasticsearch8_1 = require("elasticsearch8");
-const schema = '2';
-const keyToId = (key) => {
-    const keyBuf = buffer_1.Buffer.from(key);
-    return keyBuf.length > 512 ? (0, crypto_1.createHash)('sha512').update(keyBuf).digest('hex') : key;
-};
-const mappings = {
-    // _id is expected to equal key, unless the UTF-8 encoded key is > 512 bytes, in which case it is
-    // the hex-encoded sha512 hash of the UTF-8 encoded key.
-    properties: {
-        key: { type: 'wildcard' },
-        value: { type: 'object', enabled: false }, // Values should be opaque to Elasticsearch.
-    },
-};
-const migrateToSchema2 = async (client, v1BaseIndex, v2Index, logger) => {
-    let recordsMigratedLastLogged = 0;
-    let recordsMigrated = 0;
-    const totals = new Map();
-    logger.info('Attempting elasticsearch record migration from schema v1 at base index ' +
-        `${v1BaseIndex} to schema v2 at index ${v2Index}...`);
-    const indices = await client.indices.get({ index: [v1BaseIndex, `${v1BaseIndex}-*-*`] });
-    const scrollIds = new Map();
-    const q = [];
-    try {
-        for (const index of Object.keys(indices)) {
-            const res = await client.search({ index, scroll: '10m' });
-            scrollIds.set(index, res._scroll_id);
-            q.push({ index, res });
-        }
-        while (q.length) {
-            const { index, res: { hits: { hits, total: { value: total } } } } = q.shift();
-            if (hits.length === 0)
-                continue;
-            totals.set(index, total);
-            const body = [];
-            for (const { _id, _type, _source: { val } } of hits) {
-                let key = `${_type}:${_id}`;
-                if (v1BaseIndex && index !== v1BaseIndex) {
-                    const parts = index.slice(v1BaseIndex.length + 1).split('-');
-                    if (parts.length !== 2) {
-                        throw new Error(`unable to migrate records from index ${index} due to data ambiguity`);
-                    }
-                    key = `${parts[0]}:${decodeURIComponent(_type)}:${parts[1]}:${_id}`;
-                }
-                body.push({ index: { _id: keyToId(key) } }, { key, value: JSON.parse(val) });
-            }
-            await client.bulk({ index: v2Index, body });
-            recordsMigrated += hits.length;
-            if (Math.floor(recordsMigrated / 100) > Math.floor(recordsMigratedLastLogged / 100)) {
-                const total = [...totals.values()].reduce((a, b) => a + b, 0);
-                logger.info(`Migrated ${recordsMigrated} records out of ${total}`);
-                recordsMigratedLastLogged = recordsMigrated;
-            }
-            q.push({ index, res: (await client.scroll({ scroll: '5m', scroll_id: scrollIds.get(index) })) });
-        }
-        logger.info(`Finished migrating ${recordsMigrated} records`);
-    }
-    finally {
-        await Promise.all([...scrollIds.values()].map((scrollId) => client.clearScroll({ scroll_id: scrollId })));
-    }
-};
-const Database = class extends AbstractDatabase_1.default {
-    _client;
-    _index;
-    _indexClean;
-    _q;
-    constructor(settings) {
-        super();
-        this._client = null;
-        this.settings = {
-            host: '127.0.0.1',
-            port: '9200',
-            base_index: 'ueberes',
-            migrate_to_newer_schema: false,
-            // for a list of valid API values see:
-            // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html#config-options
-            api: '7.6',
-            ...settings || {},
-            json: false, // Elasticsearch will do the JSON conversion as necessary.
-        };
-        this._index = `${this.settings.base_index}_s${schema}`;
-        this._q = { index: this._index };
-        this._indexClean = true;
-    }
-    get isAsync() { return true; }
-    async _refreshIndex() {
-        if (this._indexClean)
-            return;
-        this._indexClean = true;
-        await this._client.indices.refresh(this._q);
-    }
-    /**
-     * Initialize the elasticsearch client, then ping the server to ensure that a
-     * connection was made.
-     */
-    async init() {
-        // create elasticsearch client
-        const client = new elasticsearch8_1.Client({
-            node: `http://${this.settings.host}:${this.settings.port}`,
-        });
-        await client.ping();
-        if (!(await client.indices.exists({ index: this._index }))) {
-            let tmpIndex;
-            const exists = await client.indices.exists({ index: this.settings.base_index });
-            if (exists && !this.settings.migrate_to_newer_schema) {
-                throw new Error(`Data exists under the legacy index (schema) named ${this.settings.base_index}. ` +
-                    'Set migrate_to_newer_schema to true to copy the existing data to a new index ' +
-                    `named ${this._index}.`);
-            }
-            let attempt = 0;
-            while (true) {
-                tmpIndex = `${this._index}_${exists ? 'migrate_attempt_' : 'i'}${attempt++}`;
-                if (!(await client.indices.exists({ index: tmpIndex })))
-                    break;
-            }
-            await client.indices.create({ index: tmpIndex, mappings: mappings });
-            if (exists)
-                await migrateToSchema2(client, this.settings.base_index, tmpIndex, this.logger);
-            await client.indices.putAlias({ index: tmpIndex, name: this._index });
-        }
-        const indices = Object.values((await client.indices.get({ index: this._index })));
-        (0, assert_1.equal)(indices.length, 1);
-        try {
-            assert_1.default.deepEqual(indices[0].mappings, mappings);
-        }
-        catch (err) {
-            this.logger.warn(`Index ${this._index} mappings does not match expected; ` +
-                `attempting to use index anyway. Details: ${err}`);
-        }
-        this._client = client;
-    }
-    /**
-     *  This function provides read functionality to the database.
-     *
-     *  @param {String} key Key
-     */
-    async get(key) {
-        const res = await this._client.get({ ...this._q, id: keyToId(key) }, { ignore: [404] });
-        if (!res.found)
-            return null;
-        return res._source.value;
-    }
-    /**
-     *  @param key Search key, which uses an asterisk (*) as the wild card.
-     *  @param notKey Used to filter the result set
-     */
-    async findKeys(key, notKey) {
-        await this._refreshIndex();
-        const q = {
-            ...this._q,
-            body: {
-                query: {
-                    bool: {
-                        filter: { wildcard: { key: { value: key } } },
-                        ...notKey == null ? {} : {
-                            must_not: { wildcard: { key: { value: notKey } } },
-                        },
-                    },
-                },
-            },
-        };
-        const { hits: hits } = await this._client.search(q);
-        return hits.hits.map((h) => h._source.key);
-    }
-    /**
-     *  This function provides write functionality to the database.
-     *
-     *  @param {String} key Record identifier.
-     *  @param {JSON|String} value The value to store in the database.
-     */
-    async set(key, value) {
-        this._indexClean = false;
-        await this._client.index({ ...this._q, id: keyToId(key), body: { key, value } });
-    }
-    /**
-     *  This function provides delete functionality to the database.
-     *
-     *  The index, type, and ID will be parsed from the key, and this document will
-     *  be deleted from the database.
-     *
-     *  @param {String} key Record identifier.
-     */
-    async remove(key) {
-        this._indexClean = false;
-        await this._client.delete({ ...this._q, id: keyToId(key) }, { ignore: [404] });
-    }
-    /**
-     *  This uses the bulk upload functionality of elasticsearch (url:port/_bulk).
-     *
-     *  The CacheAndBufferLayer will periodically (every this.settings.writeInterval)
-     *  flush writes that have already been done in the local cache out to the database.
-     *
-     *  @param {Array} bulk An array of JSON data in the format:
-     *      {"type":type, "key":key, "value":value}
-     */
-    async doBulk(bulk) {
-        // bulk is an array of JSON:
-        // example: [{"type":"set", "key":"sessionstorage:{id}", "value":{"cookie":{...}}]
-        const operations = [];
-        for (const { type, key, value } of bulk) {
-            this._indexClean = false;
-            switch (type) {
-                case 'set':
-                    operations.push({ index: { _id: keyToId(key) } });
-                    operations.push({ key, value });
-                    break;
-                case 'remove':
-                    operations.push({ delete: { _id: keyToId(key) } });
-                    break;
-                default:
-            }
-        }
-        await this._client.bulk({ ...this._q, body: operations });
-    }
-    async close() {
-        if (this._client != null)
-            this._client.close();
-        this._client = null;
-    }
+'use strict';
+
+var AbstractDatabase = require('../lib/AbstractDatabase.js');
+var assert$0 = require('assert');
+var buffer = require('buffer');
+var crypto = require('crypto');
+var es = require('elasticsearch8');
+
+/**
+ * 2015 Visionist, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const schema = '2';
+const keyToId = (key) => {
+    const keyBuf = buffer.Buffer.from(key);
+    return keyBuf.length > 512 ? crypto.createHash('sha512').update(keyBuf).digest('hex') : key;
+};
+const mappings = {
+    // _id is expected to equal key, unless the UTF-8 encoded key is > 512 bytes, in which case it is
+    // the hex-encoded sha512 hash of the UTF-8 encoded key.
+    properties: {
+        key: { type: 'wildcard' },
+        value: { type: 'object', enabled: false }, // Values should be opaque to Elasticsearch.
+    },
+};
+const migrateToSchema2 = async (client, v1BaseIndex, v2Index, logger) => {
+    let recordsMigratedLastLogged = 0;
+    let recordsMigrated = 0;
+    const totals = new Map();
+    logger.info('Attempting elasticsearch record migration from schema v1 at base index ' +
+        `${v1BaseIndex} to schema v2 at index ${v2Index}...`);
+    const indices = await client.indices.get({ index: [v1BaseIndex, `${v1BaseIndex}-*-*`] });
+    const scrollIds = new Map();
+    const q = [];
+    try {
+        for (const index of Object.keys(indices)) {
+            const res = await client.search({ index, scroll: '10m' });
+            scrollIds.set(index, res._scroll_id);
+            q.push({ index, res });
+        }
+        while (q.length) {
+            const { index, res: { hits: { hits, total: { value: total } } } } = q.shift();
+            if (hits.length === 0)
+                continue;
+            totals.set(index, total);
+            const body = [];
+            for (const { _id, _type, _source: { val } } of hits) {
+                let key = `${_type}:${_id}`;
+                if (v1BaseIndex && index !== v1BaseIndex) {
+                    const parts = index.slice(v1BaseIndex.length + 1).split('-');
+                    if (parts.length !== 2) {
+                        throw new Error(`unable to migrate records from index ${index} due to data ambiguity`);
+                    }
+                    key = `${parts[0]}:${decodeURIComponent(_type)}:${parts[1]}:${_id}`;
+                }
+                body.push({ index: { _id: keyToId(key) } }, { key, value: JSON.parse(val) });
+            }
+            await client.bulk({ index: v2Index, body });
+            recordsMigrated += hits.length;
+            if (Math.floor(recordsMigrated / 100) > Math.floor(recordsMigratedLastLogged / 100)) {
+                const total = [...totals.values()].reduce((a, b) => a + b, 0);
+                logger.info(`Migrated ${recordsMigrated} records out of ${total}`);
+                recordsMigratedLastLogged = recordsMigrated;
+            }
+            q.push({ index, res: (await client.scroll({ scroll: '5m', scroll_id: scrollIds.get(index) })) });
+        }
+        logger.info(`Finished migrating ${recordsMigrated} records`);
+    }
+    finally {
+        await Promise.all([...scrollIds.values()].map((scrollId) => client.clearScroll({ scroll_id: scrollId })));
+    }
+};
+const Database = class extends AbstractDatabase {
+    _client;
+    _index;
+    _indexClean;
+    _q;
+    constructor(settings) {
+        super();
+        this._client = null;
+        this.settings = {
+            host: '127.0.0.1',
+            port: '9200',
+            base_index: 'ueberes',
+            migrate_to_newer_schema: false,
+            // for a list of valid API values see:
+            // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html#config-options
+            api: '7.6',
+            ...settings || {},
+            json: false, // Elasticsearch will do the JSON conversion as necessary.
+        };
+        this._index = `${this.settings.base_index}_s${schema}`;
+        this._q = { index: this._index };
+        this._indexClean = true;
+    }
+    get isAsync() { return true; }
+    async _refreshIndex() {
+        if (this._indexClean)
+            return;
+        this._indexClean = true;
+        await this._client.indices.refresh(this._q);
+    }
+    /**
+     * Initialize the elasticsearch client, then ping the server to ensure that a
+     * connection was made.
+     */
+    async init() {
+        // create elasticsearch client
+        const client = new es.Client({
+            node: `http://${this.settings.host}:${this.settings.port}`,
+        });
+        await client.ping();
+        if (!(await client.indices.exists({ index: this._index }))) {
+            let tmpIndex;
+            const exists = await client.indices.exists({ index: this.settings.base_index });
+            if (exists && !this.settings.migrate_to_newer_schema) {
+                throw new Error(`Data exists under the legacy index (schema) named ${this.settings.base_index}. ` +
+                    'Set migrate_to_newer_schema to true to copy the existing data to a new index ' +
+                    `named ${this._index}.`);
+            }
+            let attempt = 0;
+            while (true) {
+                tmpIndex = `${this._index}_${exists ? 'migrate_attempt_' : 'i'}${attempt++}`;
+                if (!(await client.indices.exists({ index: tmpIndex })))
+                    break;
+            }
+            await client.indices.create({ index: tmpIndex, mappings: mappings });
+            if (exists)
+                await migrateToSchema2(client, this.settings.base_index, tmpIndex, this.logger);
+            await client.indices.putAlias({ index: tmpIndex, name: this._index });
+        }
+        const indices = Object.values((await client.indices.get({ index: this._index })));
+        assert$0.equal(indices.length, 1);
+        try {
+            assert$0.deepEqual(indices[0].mappings, mappings);
+        }
+        catch (err) {
+            this.logger.warn(`Index ${this._index} mappings does not match expected; ` +
+                `attempting to use index anyway. Details: ${err}`);
+        }
+        this._client = client;
+    }
+    /**
+     *  This function provides read functionality to the database.
+     *
+     *  @param {String} key Key
+     */
+    async get(key) {
+        const res = await this._client.get({ ...this._q, id: keyToId(key) }, { ignore: [404] });
+        if (!res.found)
+            return null;
+        return res._source.value;
+    }
+    /**
+     *  @param key Search key, which uses an asterisk (*) as the wild card.
+     *  @param notKey Used to filter the result set
+     */
+    async findKeys(key, notKey) {
+        await this._refreshIndex();
+        const q = {
+            ...this._q,
+            body: {
+                query: {
+                    bool: {
+                        filter: { wildcard: { key: { value: key } } },
+                        ...notKey == null ? {} : {
+                            must_not: { wildcard: { key: { value: notKey } } },
+                        },
+                    },
+                },
+            },
+        };
+        const { hits: hits } = await this._client.search(q);
+        return hits.hits.map((h) => h._source.key);
+    }
+    /**
+     *  This function provides write functionality to the database.
+     *
+     *  @param {String} key Record identifier.
+     *  @param {JSON|String} value The value to store in the database.
+     */
+    async set(key, value) {
+        this._indexClean = false;
+        await this._client.index({ ...this._q, id: keyToId(key), body: { key, value } });
+    }
+    /**
+     *  This function provides delete functionality to the database.
+     *
+     *  The index, type, and ID will be parsed from the key, and this document will
+     *  be deleted from the database.
+     *
+     *  @param {String} key Record identifier.
+     */
+    async remove(key) {
+        this._indexClean = false;
+        await this._client.delete({ ...this._q, id: keyToId(key) }, { ignore: [404] });
+    }
+    /**
+     *  This uses the bulk upload functionality of elasticsearch (url:port/_bulk).
+     *
+     *  The CacheAndBufferLayer will periodically (every this.settings.writeInterval)
+     *  flush writes that have already been done in the local cache out to the database.
+     *
+     *  @param {Array} bulk An array of JSON data in the format:
+     *      {"type":type, "key":key, "value":value}
+     */
+    async doBulk(bulk) {
+        // bulk is an array of JSON:
+        // example: [{"type":"set", "key":"sessionstorage:{id}", "value":{"cookie":{...}}]
+        const operations = [];
+        for (const { type, key, value } of bulk) {
+            this._indexClean = false;
+            switch (type) {
+                case 'set':
+                    operations.push({ index: { _id: keyToId(key) } });
+                    operations.push({ key, value });
+                    break;
+                case 'remove':
+                    operations.push({ delete: { _id: keyToId(key) } });
+                    break;
+            }
+        }
+        await this._client.bulk({ ...this._q, body: operations });
+    }
+    async close() {
+        if (this._client != null)
+            this._client.close();
+        this._client = null;
+    }
 };
+
 exports.Database = Database;

package/dist/databases/memory_db.js

@@ -1,39 +1,37 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Database = void 0;
-const AbstractDatabase_1 = __importDefault(require("../lib/AbstractDatabase"));
-const Database = class MemoryDB extends AbstractDatabase_1.default {
-    _data;
-    constructor(settings) {
-        super();
-        this.settings = settings;
-        settings.json = false;
-        settings.cache = 0;
-        settings.writeInterval = 0;
-        this._data = null;
-    }
-    get isAsync() { return true; }
-    close() {
-        this._data = null;
-    }
-    findKeys(key, notKey) {
-        const regex = this.createFindRegex(key, notKey);
-        return [...this._data.keys()].filter((k) => regex.test(k));
-    }
-    get(key) {
-        return this._data.get(key);
-    }
-    init() {
-        this._data = this.settings.data || new Map();
-    }
-    remove(key) {
-        this._data.delete(key);
-    }
-    set(key, value) {
-        this._data.set(key, value);
-    }
+'use strict';
+
+var AbstractDatabase = require('../lib/AbstractDatabase.js');
+
+const Database = class MemoryDB extends AbstractDatabase {
+    _data;
+    constructor(settings) {
+        super();
+        this.settings = settings;
+        settings.json = false;
+        settings.cache = 0;
+        settings.writeInterval = 0;
+        this._data = null;
+    }
+    get isAsync() { return true; }
+    close() {
+        this._data = null;
+    }
+    findKeys(key, notKey) {
+        const regex = this.createFindRegex(key, notKey);
+        return [...this._data.keys()].filter((k) => regex.test(k));
+    }
+    get(key) {
+        return this._data.get(key);
+    }
+    init() {
+        this._data = this.settings.data || new Map();
+    }
+    remove(key) {
+        this._data.delete(key);
+    }
+    set(key, value) {
+        this._data.set(key, value);
+    }
 };
+
 exports.Database = Database;