ueberdb2 4.1.1 → 4.1.3

package/databases/elasticsearch_db.ts

@@ -1,257 +0,0 @@
-/**
- * 2015 Visionist, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS-IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AbstractDatabase, {Settings} from '../lib/AbstractDatabase';
-import {equal, strict} from 'assert';
-
-import {Buffer} from 'buffer';
-import {createHash} from 'crypto';
-import {Client} from 'elasticsearch7';
-import {BulkObject} from './cassandra_db';
-
-const schema = '2';
-
-const keyToId = (key:string) => {
-  const keyBuf = Buffer.from(key);
-  return keyBuf.length > 512 ? createHash('sha512').update(keyBuf).digest('hex') : key;
-};
-
-const mappings = {
-  // _id is expected to equal key, unless the UTF-8 encoded key is > 512 bytes, in which case it is
-  // the hex-encoded sha512 hash of the UTF-8 encoded key.
-  properties: {
-    key: {type: 'wildcard'}, // For findKeys, and because _id is limited to 512 bytes.
-    value: {type: 'object', enabled: false}, // Values should be opaque to Elasticsearch.
-  },
-};
-
-const migrateToSchema2 = async (client: any, v1BaseIndex: string | undefined, v2Index: string, logger: any) => {
-  let recordsMigratedLastLogged = 0;
-  let recordsMigrated = 0;
-  const totals = new Map();
-  logger.info('Attempting elasticsearch record migration from schema v1 at base index ' +
-              `${v1BaseIndex} to schema v2 at index ${v2Index}...`);
-  const {body: indices} = await client.indices.get({index: [v1BaseIndex, `${v1BaseIndex}-*-*`]});
-  const scrollIds = new Map();
-  const q = [];
-  try {
-    for (const index of Object.keys(indices)) {
-      const {body: res} = await client.search({index, scroll: '10m'});
-      scrollIds.set(index, res._scroll_id);
-      q.push({index, res});
-    }
-    while (q.length) {
-      const {index, res: {hits: {hits, total: {value: total}}}}:any = q.shift();
-      if (hits.length === 0) continue;
-      totals.set(index, total);
-      const body = [];
-      for (const {_id, _type, _source: {val}} of hits) {
-        let key = `${_type}:${_id}`;
-        if (v1BaseIndex && index !== v1BaseIndex) {
-          const parts = index.slice(v1BaseIndex.length + 1).split('-');
-          if (parts.length !== 2) {
-            throw new Error(`unable to migrate records from index ${index} due to data ambiguity`);
-          }
-          key = `${parts[0]}:${decodeURIComponent(_type)}:${parts[1]}:${_id}`;
-        }
-        body.push({index: {_id: keyToId(key)}}, {key, value: JSON.parse(val)});
-      }
-      await client.bulk({index: v2Index, body});
-      recordsMigrated += hits.length;
-      if (Math.floor(recordsMigrated / 100) > Math.floor(recordsMigratedLastLogged / 100)) {
-        // @ts-ignore
-        const total = [...totals.values()].reduce((a, b) => a + b, 0);
-        logger.info(`Migrated ${recordsMigrated} records out of ${total}`);
-        recordsMigratedLastLogged = recordsMigrated;
-      }
-      q.push(
-          {index, res: (await client.scroll({scroll: '5m', scrollId: scrollIds.get(index)})).body});
-    }
-    logger.info(`Finished migrating ${recordsMigrated} records`);
-  } finally {
-    // @ts-ignore
-    await Promise.all([...scrollIds.values()].map((scrollId) => client.clearScroll({scrollId})));
-  }
-};
-
-export const Database = class extends AbstractDatabase {
-  private _client: any;
-  private readonly _index: any;
-  private _indexClean: boolean;
-  private readonly _q: {index: any};
-  constructor(settings:Settings) {
-    super();
-    this._client = null;
-    this.settings = {
-      host: '127.0.0.1',
-      port: '9200',
-      base_index: 'ueberes',
-      migrate_to_newer_schema: false,
-      // for a list of valid API values see:
-      // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html#config-options
-      api: '7.6',
-      ...settings || {},
-      json: false, // Elasticsearch will do the JSON conversion as necessary.
-    };
-    this._index = `${this.settings.base_index}_s${schema}`;
-    this._q = {index: this._index};
-    this._indexClean = true;
-  }
-
-  get isAsync() { return true; }
-
-  async _refreshIndex() {
-    if (this._indexClean) return;
-    this._indexClean = true;
-    await this._client.indices.refresh(this._q);
-  }
-
-  /**
-   * Initialize the elasticsearch client, then ping the server to ensure that a
-   * connection was made.
-   */
-  async init() {
-    // create elasticsearch client
-    const client = new Client({
-      node: `http://${this.settings.host}:${this.settings.port}`,
-    });
-    await client.ping();
-    if (!(await client.indices.exists({index: this._index})).body) {
-      let tmpIndex;
-      // @ts-ignore
-      const {body: migrate} = await client.indices.exists({index: this.settings.base_index});
-      if (migrate && !this.settings.migrate_to_newer_schema) {
-        throw new Error(
-            `Data exists under the legacy index (schema) named ${this.settings.base_index}. ` +
-            'Set migrate_to_newer_schema to true to copy the existing data to a new index ' +
-            `named ${this._index}.`);
-      }
-      let attempt = 0;
-      while (true) {
-        tmpIndex = `${this._index}_${migrate ? 'migrate_attempt_' : 'i'}${attempt++}`;
-        if (!(await client.indices.exists({index: tmpIndex})).body) break;
-      }
-      await client.indices.create({index: tmpIndex, body: {mappings}});
-      if (migrate) await migrateToSchema2(client, this.settings.base_index, tmpIndex, this.logger);
-      await client.indices.putAlias({index: tmpIndex, name: this._index});
-    }
-    const indices = Object.values((await client.indices.get({index: this._index})).body);
-    equal(indices.length, 1);
-    try {
-      // @ts-ignore
-      assert.deepEqual(indices[0].mappings, mappings);
-    } catch (err) {
-      this.logger.warn(`Index ${this._index} mappings does not match expected; ` +
-                       `attempting to use index anyway. Details: ${err}`);
-    }
-    this._client = client;
-  }
-
-  /**
-   *  This function provides read functionality to the database.
-   *
-   *  @param {String} key Key
-   */
-  async get(key:string) {
-    const {body} = await this._client.get({...this._q, id: keyToId(key)}, {ignore: [404]});
-    if (!body.found) return null;
-    return body._source.value;
-  }
-
-  /**
-   *  @param key Search key, which uses an asterisk (*) as the wild card.
-   *  @param notKey Used to filter the result set
-   */
-  async findKeys(key:string, notKey:string) {
-    await this._refreshIndex();
-    const q = {
-      ...this._q,
-      body: {
-        query: {
-          bool: {
-            filter: {wildcard: {key: {value: key}}},
-            ...notKey == null ? {} : {
-              must_not: {wildcard: {key: {value: notKey}}},
-            },
-          },
-        },
-      },
-    };
-    const {body: {hits: {hits}}} = await this._client.search(q);
-    return hits.map((h:{_source:{key:string}}) => h._source.key);
-  }
-
-  /**
-   *  This function provides write functionality to the database.
-   *
-   *  @param {String} key Record identifier.
-   *  @param {JSON|String} value The value to store in the database.
-   */
-  async set(key: string, value:string) {
-    this._indexClean = false;
-    await this._client.index({...this._q, id: keyToId(key), body: {key, value}});
-  }
-
-  /**
-   *  This function provides delete functionality to the database.
-   *
-   *  The index, type, and ID will be parsed from the key, and this document will
-   *  be deleted from the database.
-   *
-   *  @param {String} key Record identifier.
-   */
-  async remove(key:string) {
-    this._indexClean = false;
-    await this._client.delete({...this._q, id: keyToId(key)}, {ignore: [404]});
-  }
-
-  /**
-   *  This uses the bulk upload functionality of elasticsearch (url:port/_bulk).
-   *
-   *  The CacheAndBufferLayer will periodically (every this.settings.writeInterval)
-   *  flush writes that have already been done in the local cache out to the database.
-   *
-   *  @param {Array} bulk An array of JSON data in the format:
-   *      {"type":type, "key":key, "value":value}
-   */
-  async doBulk(bulk: BulkObject[]) {
-    // bulk is an array of JSON:
-    // example: [{"type":"set", "key":"sessionstorage:{id}", "value":{"cookie":{...}}]
-
-    const operations = [];
-
-    for (const {type, key, value} of bulk) {
-      this._indexClean = false;
-      switch (type) {
-        case 'set':
-          operations.push({index: {_id: keyToId(key)}});
-          operations.push({key, value});
-          break;
-        case 'remove':
-          operations.push({delete: {_id: keyToId(key)}});
-          break;
-        default:
-          continue;
-      }
-    }
-    await this._client.bulk({...this._q, body: operations});
-  }
-
-  async close() {
-    if (this._client != null) this._client.close();
-    this._client = null;
-  }
-};

package/databases/memory_db.ts

@@ -1,41 +0,0 @@
-import AbstractDatabase, {Settings} from '../lib/AbstractDatabase';
-
-
-export const Database = class MemoryDB extends AbstractDatabase {
-  private _data: any;
-  constructor(settings:Settings) {
-    super();
-    this.settings = settings;
-    settings.json = false;
-    settings.cache = 0;
-    settings.writeInterval = 0;
-    this._data = null;
-  }
-
-  get isAsync() { return true; }
-
-  close() {
-    this._data = null;
-  }
-
-  findKeys(key:string, notKey:string) {
-    const regex = this.createFindRegex(key, notKey);
-    return [...this._data.keys()].filter((k) => regex.test(k));
-  }
-
-  get(key:string) {
-    return this._data.get(key);
-  }
-
-  init() {
-    this._data = this.settings.data || new Map();
-  }
-
-  remove(key:string) {
-    this._data.delete(key);
-  }
-
-  set(key:string, value:string) {
-    this._data.set(key, value);
-  }
-};

package/databases/mock_db.ts

@@ -1,43 +0,0 @@
-import {Settings} from '../lib/AbstractDatabase';
-
-import events from 'events';
-
-export const Database = class extends events.EventEmitter {
-  private settings: Settings;
-  constructor(settings:Settings) {
-    super();
-    this.settings = {
-      writeInterval: 1,
-      ...settings,
-    };
-    settings.mock = this;
-  }
-
-  close(cb: ()=>{}) {
-    this.emit('close', cb);
-  }
-
-  doBulk(ops:string, cb: ()=>{}) {
-    this.emit('doBulk', ops, cb);
-  }
-
-  findKeys(key:string, notKey:string, cb:()=>{}) {
-    this.emit('findKeys', key, notKey, cb);
-  }
-
-  get(key:string, cb:()=>{}) {
-    this.emit('get', key, cb);
-  }
-
-  init(cb:()=>{}) {
-    this.emit('init', cb);
-  }
-
-  remove(key:string, cb:()=>{}) {
-    this.emit('remove', key, cb);
-  }
-
-  set(key:string, value:string, cb:()=>{}) {
-    this.emit('set', key, value, cb);
-  }
-};

package/databases/mongodb_db.ts

@@ -1,142 +0,0 @@
-/**
- * 2020 Sylchauf
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS-IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AbstractDatabase, {Settings} from '../lib/AbstractDatabase';
-import {Func} from 'mocha';
-import {BulkObject} from './cassandra_db';
-
-exports.Database = class extends AbstractDatabase {
-  private interval: any;
-  private database: any;
-  private client: any;
-  private collection: any;
-  constructor(settings:Settings) {
-    super();
-    this.settings = settings;
-
-    if (!this.settings.url) throw new Error('You must specify a mongodb url');
-    // For backwards compatibility:
-    if (this.settings.database == null) this.settings.database = this.settings.dbName;
-
-    if (!this.settings.collection) this.settings.collection = 'ueberdb';
-  }
-
-  clearPing() {
-    if (this.interval) {
-      clearInterval(this.interval);
-    }
-  }
-
-  schedulePing() {
-    this.clearPing();
-    this.interval = setInterval(() => {
-      this.database.command({
-        ping: 1,
-      });
-    }, 10000);
-  }
-
-  init(callback:Function) {
-    const MongoClient = require('mongodb').MongoClient;
-
-    MongoClient.connect(this.settings.url, (err:any, client:any) => {
-      if (!err) {
-        this.client = client;
-        this.database = client.db(this.settings.database);
-        this.collection = this.database.collection(this.settings.collection);
-      }
-
-      callback(err);
-    });
-
-    this.schedulePing();
-  }
-
-  get(key:string, callback:Function) {
-    this.collection.findOne({_id: key}, (err:any, document:any) => {
-      if (err) callback(err);
-      else callback(null, document ? document.value : null);
-    });
-
-    this.schedulePing();
-  }
-
-  findKeys(key:string, notKey:string, callback:Function) {
-    const selector = {
-      $and: [
-        {_id: {$regex: `${key.replace(/\*/g, '')}`}},
-      ],
-    };
-
-    if (notKey) {
-      // @ts-ignore
-      selector.$and.push({_id: {$not: {$regex: `${notKey.replace(/\*/g, '')}`}}});
-    }
-
-    this.collection.find(selector, async (err:any, res:any) => {
-      if (err) {
-        callback(err);
-      } else {
-        const data = await res.toArray();
-
-        callback(null, data.map((i:any) => i._id));
-      }
-    });
-
-    this.schedulePing();
-  }
-
-  set(key:string, value:string, callback:Function) {
-    if (key.length > 100) {
-      callback('Your Key can only be 100 chars');
-    } else {
-      this.collection.update({_id: key}, {$set: {value}}, {upsert: true}, callback);
-    }
-
-    this.schedulePing();
-  }
-
-  remove(key:string, callback:Function) {
-    this.collection.remove({_id: key}, callback);
-
-    this.schedulePing();
-  }
-
-  doBulk(bulk:BulkObject[], callback:Function) {
-    const bulkMongo = this.collection.initializeOrderedBulkOp();
-
-    for (const i in bulk) {
-      if (bulk[i].type === 'set') {
-        bulkMongo.find({_id: bulk[i].key}).upsert().updateOne({$set: {value: bulk[i].value}});
-      } else if (bulk[i].type === 'remove') {
-        bulkMongo.find({_id: bulk[i].key}).deleteOne();
-      }
-    }
-
-    bulkMongo.execute().then((res:any) => {
-      callback(null, res);
-    }).catch((error:any) => {
-      callback(error);
-    });
-
-    this.schedulePing();
-  }
-
-  close(callback:any) {
-    this.clearPing();
-    this.client.close(callback);
-  }
-};

package/databases/mssql_db.ts

@@ -1,226 +0,0 @@
-/* eslint new-cap: ["error", {"capIsNewExceptions": ["mssql.NVarChar"]}] */
-
-/**
- * 2019 - exspecto@gmail.com
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS-IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * Note: This requires MS SQL Server >= 2008 due to the usage of the MERGE statement
- *
- */
-
-import AbstractDatabase, {Settings} from '../lib/AbstractDatabase';
-import async from 'async';
-import mssql, {ConnectionPool} from 'mssql';
-import {BulkObject} from './cassandra_db';
-
-type RowResult = {
-    key: string;
-};
-
-
-export const Database = class MSSQL extends AbstractDatabase {
-  private db: ConnectionPool | undefined;
-  constructor(settings:Settings) {
-    super();
-    settings = settings || {};
-
-    if (settings.json != null) {
-      settings.parseJSON = settings.json;
-    }
-
-    // set the request timeout to 5 minutes
-    settings.requestTimeout = 300000;
-
-    settings.server = settings.host;
-    this.settings = settings;
-
-    /*
-      Turning off the cache and write buffer here. You
-      can reenable it, but also take a look at maxInserts in
-      the doBulk function to decide how you want to split it up.
-    */
-    this.settings.cache = 0;
-    this.settings.writeInterval = 0;
-  }
-
-  init(callback:(err: any)=>{}) {
-    const sqlCreate =
-        "IF OBJECT_ID(N'dbo.store', N'U') IS NULL" +
-        '  BEGIN' +
-        '    CREATE TABLE [store] (' +
-        '      [key] NVARCHAR(100) PRIMARY KEY,' +
-        '      [value] NTEXT NOT NULL' +
-        '    );' +
-        '  END';
-
-    // @ts-ignore
-    new mssql.ConnectionPool(this.settings).connect().then((pool) => {
-      this.db = pool;
-
-      const request = new mssql.Request(this.db);
-
-      request.query(sqlCreate, (err) => {
-        callback(err);
-      });
-
-      this.db.on('error', (err) => {
-        console.log(err);
-      });
-    });
-  }
-
-  get(key:string, callback:(err?:Error, value?:string)=>{}) {
-    const request = new mssql.Request(this.db);
-
-    request.input('key', mssql.NVarChar(100), key);
-
-    request.query('SELECT [value] FROM [store] WHERE [key] = @key', (err, results) => {
-      let value = null;
-
-      if (!err && results && results.rowsAffected[0] === 1) {
-        // @ts-ignore
-        value = results.recordset[0].value;
-      }
-
-      callback(err, value);
-    });
-  }
-
-  findKeys(key:string, notKey:string, callback:(err: Error | undefined, value:string[])=>{}) {
-    const request = new mssql.Request(this.db);
-    let query = 'SELECT [key] FROM [store] WHERE [key] LIKE @key';
-
-    // desired keys are key, e.g. pad:%
-    key = key.replace(/\*/g, '%');
-
-    request.input('key', mssql.NVarChar(100), key);
-
-    if (notKey != null) {
-      // not desired keys are notKey, e.g. %:%:%
-      notKey = notKey.replace(/\*/g, '%');
-      request.input('notkey', mssql.NVarChar(100), notKey);
-      query += ' AND [key] NOT LIKE @notkey';
-    }
-
-    request.query(query, (err, results) => {
-      const value:string[] = [];
-
-      if (!err && results && results.rowsAffected[0] > 0) {
-        for (let i = 0; i < results.recordset.length; i++) {
-          value.push((results.recordset[i] as RowResult).key);
-        }
-      }
-
-      callback(err, value);
-    });
-  }
-
-  set(key:string, value:string, callback: (val:string)=>{}) {
-    const request = new mssql.Request(this.db);
-
-    if (key.length > 100) {
-      callback('Your Key can only be 100 chars');
-    } else {
-      const query =
-          'MERGE [store] t USING (SELECT @key [key], @value [value]) s' +
-          ' ON t.[key] = s.[key]' +
-          ' WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value]' +
-          ' WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);';
-
-      request.input('key', mssql.NVarChar(100), key);
-      request.input('value', mssql.NText, value);
-
-      request.query(query, (err, info) => {
-        callback(err ? err.toString() : '');
-      });
-    }
-  }
-
-  remove(key:string, callback:()=>{}) {
-    const request = new mssql.Request(this.db);
-    request.input('key', mssql.NVarChar(100), key);
-    request.query('DELETE FROM [store] WHERE [key] = @key', callback);
-  }
-
-  doBulk(bulk: BulkObject[], callback:(err:any, results?: any)=>{}) {
-    const maxInserts = 100;
-    const request = new mssql.Request(this.db);
-    let firstReplace = true;
-    let firstRemove = true;
-    const replacements: string[] = [];
-    let removeSQL = 'DELETE FROM [store] WHERE [key] IN (';
-
-    for (const i in bulk) {
-      if (bulk[i].type === 'set') {
-        if (firstReplace) {
-          replacements.push('BEGIN TRANSACTION;');
-          firstReplace = false;
-        } else if (Number(i) % maxInserts === 0) {
-          replacements.push('\nCOMMIT TRANSACTION;\nBEGIN TRANSACTION;\n');
-        }
-
-        replacements.push(
-            `MERGE [store] t USING (SELECT '${bulk[i].key}' [key], '${bulk[i].value}' [value]) s`,
-            'ON t.[key] = s.[key]',
-            'WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value]',
-            'WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);');
-      } else if (bulk[i].type === 'remove') {
-        if (!firstRemove) {
-          removeSQL += ',';
-        }
-
-        firstRemove = false;
-        removeSQL += `'${bulk[i].key}'`;
-      }
-    }
-
-    removeSQL += ');';
-    replacements.push('COMMIT TRANSACTION;');
-
-    async.parallel(
-        [
-          (callback) => {
-            if (!firstReplace) {
-              request.batch(replacements.join('\n'), (err, results) => {
-                if (err) {
-                  callback(err);
-                }
-                callback(err, results);
-              });
-            } else {
-              callback();
-            }
-          },
-          (callback) => {
-            if (!firstRemove) {
-              request.query(removeSQL, callback);
-            } else {
-              callback();
-            }
-          },
-        ],
-        (err, results) => {
-          if (err) {
-            callback(err);
-          }
-          callback(err, results);
-        },
-    );
-  }
-
-  close(callback: (err?:Error)=>{}) {
-    this.db && this.db.close(callback);
-  }
-};