ueberdb2 2.2.3 → 4.0.0

package/.eslintrc.cjs

@@ -0,0 +1,27 @@
+'use strict';
+
+// This is a workaround for https://github.com/eslint/eslint/issues/3458
+require('eslint-config-etherpad/patch/modern-module-resolution');
+
+module.exports = {
+  root: true,
+  extends: 'etherpad/node',
+  overrides: [
+    {
+      files: [
+        'test/**/*',
+      ],
+      extends: 'etherpad/tests/backend',
+      overrides: [
+        {
+          files: [
+            'test/lib/**/*',
+          ],
+          rules: {
+            'mocha/no-exports': 'off',
+          },
+        },
+      ],
+    },
+  ],
+};

package/.github/workflows/npmpublish.yml

@@ -19,6 +19,12 @@ jobs:
         env:
           COUCHDB_USER: ueberdb
           COUCHDB_PASSWORD: ueberdb
+      elasticsearch:
+        image: elasticsearch:7.17.3
+        ports:
+          - 9200:9200
+        env:
+          discovery.type: single-node
       mongo:
         image: mongo
         ports:
@@ -56,7 +62,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 12
+          node-version: 14
       - run: npm ci
       # Optional dependencies must be installed manually.
       - run: npm i sqlite3
@@ -86,7 +92,7 @@ jobs:
       -
         uses: actions/setup-node@v3
         with:
-          node-version: 12
+          node-version: 14
           registry-url: https://registry.npmjs.org/
           cache: 'npm'
       # This is required if the package has a prepare script that uses something

package/CHANGELOG.md

@@ -1,5 +1,62 @@
 # Notable Changes
 
+## v4.0.0
+
+Compatibility changes:
+
+  * `redis`: The `socket` and `client_options` settings, deprecated since
+    v1.3.1, have been removed.
+  * `redis`: The client configuration object has changed with the new version of
+    the `redis` client library. See the [`redis` client library
+    documentation](https://github.com/redis/node-redis/blob/redis%404.1.0/docs/client-configuration.md)
+    for details.
+
+Bug fixes:
+
+  * `redis`: Several `findKeys()` fixes.
+
+Updated database dependencies:
+
+  * `redis`: Updated `redis` from 3.1.2 to 4.1.0.
+
+## v3.0.1
+
+Bug fixes:
+
+  * Fixed `findKeys()` calls containing special regular expression characters
+    (applicable to the database drivers that use the glob-to-regex helper
+    function).
+
+## v3.0.0
+
+Compatibility changes:
+
+  * Minimum supported Node.js version is now 14.15.0.
+  * `elasticsearch`: New index name and mapping (schema). To automatically copy
+    existing data to the new index when the ueberdb client is initialized, set
+    the `migrate_to_newer_schema` option to `true`.
+  * As mentioned in the v2.2.0 changes, passing callbacks to the database
+    methods is deprecated. Use the returned Promises instead.
+  * `postgrespool`: As mentioned in the v1.4.15 changes, `postgrespool` is
+    deprecated. Use `postgres` instead.
+  * `redis`: As mentioned in the v1.3.1 changes, the `socket` and
+    `client_options` settings are deprecated. Pass the [client options
+    object](https://www.npmjs.com/package/redis/v/3.1.2#options-object-properties)
+    directly.
+
+Bug fixes:
+
+  * `elasticsearch`: Rewrote driver to fix numerous bugs and modernize the code.
+
+Updated database dependencies:
+
+  * `couch`: Updated `nano` to 10.0.0.
+  * `dirty_git`: Updated `simple-git` to 3.7.1.
+  * `elasticsearch`: Switched the client library from the deprecated
+    `elasticsearch` to `@elastic/elasticsearch` version 7.17.0.
+  * `postgres`: Updated `pg` to 8.7.3.
+  * `sqlite`: Updated `sqlite3` to 5.0.6.
+
 ## v2.2.0
 
 Compatibility changes:

package/README.md

@@ -272,16 +272,8 @@ You should create your database as utf8mb4_bin.
 If you enabled TLS on your Redis database (available since Redis 6.0) you will
 need to change your connections parameters, here is an example:
 
-```
-settings:
-    {
-      host:
-      port: rediss://<redis_database_address>:<redis_database_port>
-      socket:
-      database:
-      password:
-      client_options
-    }
+```javascript
+const db = new ueberdb.Database('redis', {url: 'rediss://localhost'});
 ```
 
 Do not provide a `host` value.

package/databases/elasticsearch_db.js

@@ -16,144 +16,184 @@
  */
 
 const AbstractDatabase = require('../lib/AbstractDatabase');
-const es = require('elasticsearch');
+const assert = require('assert').strict;
+const {Buffer} = require('buffer');
+const crypto = require('crypto');
+const es = require('elasticsearch7');
 
-// initialize w/ default settings
-const elasticsearchSettings = {
-  hostname: '127.0.0.1',
-  port: '9200',
-  base_index: 'ueberes',
+const schema = '2';
 
-  // for a list of valid API values see:
-  // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html#config-options
-  api: '7.6',
+const keyToId = (key) => {
+  const keyBuf = Buffer.from(key);
+  return keyBuf.length > 512 ? crypto.createHash('sha512').update(keyBuf).digest('hex') : key;
 };
 
-let client;
+const mappings = {
+  // _id is expected to equal key, unless the UTF-8 encoded key is > 512 bytes, in which case it is
+  // the hex-encoded sha512 hash of the UTF-8 encoded key.
+  properties: {
+    key: {type: 'wildcard'}, // For findKeys, and because _id is limited to 512 bytes.
+    value: {type: 'object', enabled: false}, // Values should be opaque to Elasticsearch.
+  },
+};
+
+const migrateToSchema2 = async (client, v1BaseIndex, v2Index, logger) => {
+  let recordsMigratedLastLogged = 0;
+  let recordsMigrated = 0;
+  const totals = new Map();
+  logger.info('Attempting elasticsearch record migration from schema v1 at base index ' +
+              `${v1BaseIndex} to schema v2 at index ${v2Index}...`);
+  const {body: indices} = await client.indices.get({index: [v1BaseIndex, `${v1BaseIndex}-*-*`]});
+  const scrollIds = new Map();
+  const q = [];
+  try {
+    for (const index of Object.keys(indices)) {
+      const {body: res} = await client.search({index, scroll: '10m'});
+      scrollIds.set(index, res._scroll_id);
+      q.push({index, res});
+    }
+    while (q.length) {
+      const {index, res: {hits: {hits, total: {value: total}}}} = q.shift();
+      if (hits.length === 0) continue;
+      totals.set(index, total);
+      const body = [];
+      for (const {_id, _type, _source: {val}} of hits) {
+        let key = `${_type}:${_id}`;
+        if (index !== v1BaseIndex) {
+          const parts = index.slice(v1BaseIndex.length + 1).split('-');
+          if (parts.length !== 2) {
+            throw new Error(`unable to migrate records from index ${index} due to data ambiguity`);
+          }
+          key = `${parts[0]}:${decodeURIComponent(_type)}:${parts[1]}:${_id}`;
+        }
+        body.push({index: {_id: keyToId(key)}}, {key, value: JSON.parse(val)});
+      }
+      await client.bulk({index: v2Index, body});
+      recordsMigrated += hits.length;
+      if (Math.floor(recordsMigrated / 100) > Math.floor(recordsMigratedLastLogged / 100)) {
+        const total = [...totals.values()].reduce((a, b) => a + b, 0);
+        logger.info(`Migrated ${recordsMigrated} records out of ${total}`);
+        recordsMigratedLastLogged = recordsMigrated;
+      }
+      q.push(
+          {index, res: (await client.scroll({scroll: '5m', scrollId: scrollIds.get(index)})).body});
+    }
+    logger.info(`Finished migrating ${recordsMigrated} records`);
+  } finally {
+    await Promise.all([...scrollIds.values()].map((scrollId) => client.clearScroll({scrollId})));
+  }
+};
 
 exports.Database = class extends AbstractDatabase {
   constructor(settings) {
     super();
-    this.db = null;
-
-    this.settings = settings || {};
-
-    // update settings if they were provided
-    if (this.settings.host) {
-      elasticsearchSettings.hostname = this.settings.host;
-    }
-
-    if (this.settings.port) {
-      elasticsearchSettings.port = this.settings.port;
-    }
+    this._client = null;
+    this.settings = {
+      host: '127.0.0.1',
+      port: '9200',
+      base_index: 'ueberes',
+      migrate_to_newer_schema: false,
+      // for a list of valid API values see:
+      // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html#config-options
+      api: '7.6',
+      ...settings || {},
+      json: false, // Elasticsearch will do the JSON conversion as necessary.
+    };
+    this._index = `${this.settings.base_index}_s${schema}`;
+    this._q = {index: this._index};
+    this._indexClean = true;
+  }
 
-    if (this.settings.base_index) {
-      elasticsearchSettings.base_index = this.settings.base_index;
-    }
+  get isAsync() { return true; }
 
-    if (this.settings.api) {
-      elasticsearchSettings.api = this.settings.api;
-    }
+  async _refreshIndex() {
+    if (this._indexClean) return;
+    this._indexClean = true;
+    await this._client.indices.refresh(this._q);
   }
 
   /**
    * Initialize the elasticsearch client, then ping the server to ensure that a
    * connection was made.
    */
-  init(callback) {
+  async init() {
     // create elasticsearch client
-    client = new es.Client({
-      host: `${elasticsearchSettings.hostname}:${elasticsearchSettings.port}`,
-      apiVersion: elasticsearchSettings.api,
-      // log: "trace" // useful for debugging
+    const client = new es.Client({
+      node: `http://${this.settings.host}:${this.settings.port}`,
     });
-
-    // test the connection
-    client.ping({
-      requestTimeout: 3000,
-    }, (error) => {
-      if (error) {
-        console.error('unable to communicate with elasticsearch');
+    await client.ping();
+    if (!(await client.indices.exists({index: this._index})).body) {
+      let tmpIndex;
+      const {body: migrate} = await client.indices.exists({index: this.settings.base_index});
+      if (migrate && !this.settings.migrate_to_newer_schema) {
+        throw new Error(
+            `Data exists under the legacy index (schema) named ${this.settings.base_index}. ` +
+            'Set migrate_to_newer_schema to true to copy the existing data to a new index ' +
+            `named ${this._index}.`);
       }
-
-      callback(error);
-    });
+      let attempt = 0;
+      while (true) {
+        tmpIndex = `${this._index}_${migrate ? 'migrate_attempt_' : 'i'}${attempt++}`;
+        if (!(await client.indices.exists({index: tmpIndex})).body) break;
+      }
+      await client.indices.create({index: tmpIndex, body: {mappings}});
+      if (migrate) await migrateToSchema2(client, this.settings.base_index, tmpIndex, this.logger);
+      await client.indices.putAlias({index: tmpIndex, name: this._index});
+    }
+    const indices = Object.values((await client.indices.get({index: this._index})).body);
+    assert.equal(indices.length, 1);
+    try {
+      assert.deepEqual(indices[0].mappings, mappings);
+    } catch (err) {
+      this.logger.warn(`Index ${this._index} mappings does not match expected; ` +
+                       `attempting to use index anyway. Details: ${err}`);
+    }
+    this._client = client;
   }
 
   /**
    *  This function provides read functionality to the database.
    *
-   *  @param {String} key Key, of the format "test:test1" or, optionally, of the
-   *    format "test:test1:check:check1"
-   *  @param {function} callback Function will be called in the event of an error or
-   *    upon completion of a successful database retrieval.
+   *  @param {String} key Key
    */
-  get(key, callback) {
-    client.get(getIndexTypeId(key), (error, response) => {
-      parseResponse(error, response, callback);
-    });
+  async get(key) {
+    const {body} = await this._client.get({...this._q, id: keyToId(key)}, {ignore: [404]});
+    if (!body.found) return null;
+    return body._source.value;
   }
 
   /**
-   *  The three key scenarios for this are:
-   *      (test:test1, null) ; (test:*, *:*:*) ; (test:*, null)
-   *
-   *  TODO This currently works only for the second implementation above.
-   *
-   *  For more information:
-   *    - See the #Limitations section of the ueberDB README.
-   *    - See https://github.com/Pita/ueberDB/wiki/findKeys-functionality, as well
-   *      as the sqlite and mysql implementations.
-   *
    *  @param key Search key, which uses an asterisk (*) as the wild card.
    *  @param notKey Used to filter the result set
-   *  @param callback First param is error, second is result
    */
-  findKeys(key, notKey, callback) {
-    const splitKey = key.split(':');
-
-    client.search({
-      index: elasticsearchSettings.base_index,
-      type: splitKey[0],
-      size: 100, // this is a pretty random threshold...
-    }, (error, response) => {
-      if (error) {
-        console.error('findkeys', error);
-        callback(error);
-        return;
-      }
-
-      if (!error && response.hits) {
-        const keys = [];
-        for (let counter = 0; counter < response.hits.total; counter++) {
-          keys.push(`${splitKey[0]}:${response.hits.hits[counter]._id}`);
-        }
-        callback(null, keys);
-      }
-    });
+  async findKeys(key, notKey) {
+    await this._refreshIndex();
+    const q = {
+      ...this._q,
+      body: {
+        query: {
+          bool: {
+            filter: {wildcard: {key: {value: key}}},
+            ...notKey == null ? {} : {
+              must_not: {wildcard: {key: {value: notKey}}},
+            },
+          },
+        },
+      },
+    };
+    const {body: {hits: {hits}}} = await this._client.search(q);
+    return hits.map((h) => h._source.key);
   }
 
   /**
    *  This function provides write functionality to the database.
    *
-   *  @param {String} key Key, of the format "test:test1" or, optionally, of the
-   *    format "test:test1:check:check1"
-   *  @param {JSON|String} value The value to be stored to the database.  The value is
-   *    always converted to {val:value} before being written to the database, to account
-   *    for situations where the value is just a string.
-   *  @param {function} callback Function will be called in the event of an error or on
-   *    completion of a successful database write.
+   *  @param {String} key Record identifier.
+   *  @param {JSON|String} value The value to store in the database.
    */
-  set(key, value, callback) {
-    const options = getIndexTypeId(key);
-
-    options.body = {
-      val: value,
-    };
-
-    client.index(options, (error, response) => {
-      parseResponse(error, response, callback);
-    });
+  async set(key, value) {
+    this._indexClean = false;
+    await this._client.index({...this._q, id: keyToId(key), body: {key, value}});
   }
 
   /**
@@ -162,15 +202,11 @@ exports.Database = class extends AbstractDatabase {
    *  The index, type, and ID will be parsed from the key, and this document will
    *  be deleted from the database.
    *
-   *  @param {String} key Key, of the format "test:test1" or, optionally, of the
-   *    format "test:test1:check:check1"
-   *  @param {function} callback Function will be called in the event of an error or on
-   *    completion of a successful database write.
+   *  @param {String} key Record identifier.
    */
-  remove(key, callback) {
-    client.delete(key, (error, response) => {
-      parseResponse(error, response, callback);
-    });
+  async remove(key) {
+    this._indexClean = false;
+    await this._client.delete({...this._q, id: keyToId(key)}, {ignore: [404]});
   }
 
   /**
@@ -181,108 +217,32 @@ exports.Database = class extends AbstractDatabase {
    *
    *  @param {Array} bulk An array of JSON data in the format:
    *      {"type":type, "key":key, "value":value}
-   *  @param {function} callback This function will be called on an error or upon the
-   *      successful completion of the database write.
    */
-  doBulk(bulk, callback) {
+  async doBulk(bulk) {
     // bulk is an array of JSON:
     // example: [{"type":"set", "key":"sessionstorage:{id}", "value":{"cookie":{...}}]
 
     const operations = [];
 
-    for (let counter = 0; counter < bulk.length; counter++) {
-      const indexTypeId = getIndexTypeId(bulk[counter].key);
-      const operationPayload = {
-        _index: indexTypeId.index,
-        _type: indexTypeId.type,
-        _id: indexTypeId.id,
-      };
-
-      switch (bulk[counter].type) {
+    for (const {type, key, value} of bulk) {
+      this._indexClean = false;
+      switch (type) {
         case 'set':
-          operations.push({index: operationPayload});
-          operations.push({val: JSON.parse(bulk[counter].value)});
+          operations.push({index: {_id: keyToId(key)}});
+          operations.push({key, value});
           break;
         case 'remove':
-          operations.push({delete: operationPayload});
+          operations.push({delete: {_id: keyToId(key)}});
           break;
         default:
           continue;
       }
     }
-
-    // send bulk request
-    client.bulk({
-      body: operations,
-    }, (error, response) => {
-      parseResponse(error, response, callback);
-    });
-  }
-
-  close(callback) {
-    callback(null);
-  }
-};
-
-/** ************************
- **** Helper functions ****
- **************************/
-
-/**
- *  This function parses a given key into an object with three
- *  fields, .index, .type, and .id.  This object can then be
- *  used to build an elasticsearch path or to access an object
- *  for bulk updates.
- *
- *  @param {String} key Key, of the format "test:test1" or, optionally, of the
- *    format "test:test1:check:check1"
- */
-const getIndexTypeId = (key) => {
-  const returnObject = {};
-
-  const splitKey = key.split(':');
-
-  if (splitKey.length === 4) {
-    /*
-     * This is for keys like test:test1:check:check1.
-     * These keys are stored at /base_index-test-check/test1/check1
-     */
-    returnObject.index = `${elasticsearchSettings.base_index}-${splitKey[0]}-${splitKey[2]}`;
-    returnObject.type = encodeURIComponent(splitKey[1]);
-    returnObject.id = splitKey[3];
-  } else {
-    // everything else ('test:test1') is stored /base_index/test/test1
-    returnObject.index = elasticsearchSettings.base_index;
-    returnObject.type = splitKey[0];
-    returnObject.id = encodeURIComponent(splitKey[1]);
-  }
-
-  return returnObject;
-};
-
-/**
- * Extract data from elasticsearch responses, handle errors, handle callbacks.
- */
-const parseResponse = (error, response, callback) => {
-  if (error) {
-    // don't treat not found as an error (is this specific to etherpad?)
-    if (error.message === 'Not Found' && !response.found) {
-      callback(null, null);
-      return;
-    } else {
-      console.error('elasticsearch_db: ', error);
-    }
+    await this._client.bulk({...this._q, body: operations});
   }
 
-  if (!error && response) {
-    response = response._source;
-
-    if (response) {
-      response = response.val;
-    }
-
-    response = JSON.stringify(response);
+  async close() {
+    if (this._client != null) this._client.close();
+    this._client = null;
   }
-
-  callback(error, response);
 };

package/databases/mssql_db.js

@@ -208,7 +208,7 @@ exports.Database = class extends AbstractDatabase {
             callback(err);
           }
           callback(err, results);
-        }
+        },
     );
   }
 

package/databases/redis_db.js

@@ -1,5 +1,4 @@
 'use strict';
-/* eslint new-cap: ["error", {"capIsNewExceptions": ["KEYS", "SMEMBERS"]}] */
 
 /**
  * 2011 Peter 'Pita' Martischka
@@ -18,111 +17,80 @@
  */
 
 const AbstractDatabase = require('../lib/AbstractDatabase');
-const async = require('async');
 const redis = require('redis');
 
 exports.Database = class extends AbstractDatabase {
   constructor(settings) {
     super();
-    this.client = null;
+    this._client = null;
     this.settings = settings || {};
   }
 
-  auth(callback) {
-    if (!this.settings.password) return callback();
-    this.client.auth(this.settings.password, callback);
-  }
-
-  select(callback) {
-    if (!this.settings.database) return callback();
-    this.client.select(this.settings.database, callback);
-  }
+  get isAsync() { return true; }
 
-  _deprecatedInit(callback) {
-    if (this.settings.socket) {
-      // Deprecated, but kept for backwards compatibility.
-      this.client = redis.createClient(this.settings.socket,
-          this.settings.client_options);
-    } else {
-      // Deprecated, but kept for backwards compatibility.
-      this.client = redis.createClient(this.settings.port,
-          this.settings.host, this.settings.client_options);
-    }
-
-    this.client.database = this.settings.database;
-    async.waterfall([this.auth.bind(this), this.select.bind(this)], callback);
+  async init() {
+    this._client = redis.createClient(this.settings);
+    await this._client.connect();
+    await this._client.ping();
   }
 
-  init(callback) {
-    if (this.settings.socket || this.settings.client_options) return this._deprecatedInit(callback);
-    this.client = redis.createClient(this.settings);
-    callback();
+  async get(key) {
+    return await this._client.get(key);
   }
 
-  get(key, callback) {
-    this.client.get(key, callback);
-  }
-
-  findKeys(key, notKey, callback) {
-    // As redis provides only limited support for getting a list of all
-    // available keys we have to limit key and notKey here.
-    // See http://redis.io/commands/keys
-    if (notKey == null) {
-      this.client.KEYS(key, callback);
-    } else if (notKey === '*:*:*') {
-      // restrict key to format "text:*"
-      const matches = /^([^:]+):\*$/.exec(key);
-      if (matches) {
-        this.client.SMEMBERS(`ueberDB:keys:${matches[1]}`, callback);
-      } else {
-        const msg = 'redis db only supports key patterns like pad:* when notKey is set to *:*:*';
-        callback(new Error(msg), null);
-      }
-    } else {
-      callback(new Error('redis db currently only supports *:*:* as notKey'), null);
+  async findKeys(key, notKey) {
+    const [type] = /^([^:*]+):\*$/.exec(key) || [];
+    if (type != null && ['*:*:*', `${key}:*`].includes(notKey)) {
+      // Performance optimization for a common Etherpad case.
+      return await this._client.sMembers(`ueberDB:keys:${type}`);
+    }
+    let keys = await this._client.keys(key.replace(/[?[\]\\]/g, '\\$&'));
+    if (notKey != null) {
+      const regex = this.createFindRegex(key, notKey);
+      keys = keys.filter((k) => regex.test(k));
     }
+    return keys;
   }
 
-  set(key, value, callback) {
+  async set(key, value) {
     const matches = /^([^:]+):([^:]+)$/.exec(key);
-    if (matches) {
-      this.client.sadd([`ueberDB:keys:${matches[1]}`, matches[0]]);
-    }
-    this.client.set(key, value, callback);
+    await Promise.all([
+      matches && this._client.sAdd(`ueberDB:keys:${matches[1]}`, matches[0]),
+      this._client.set(key, value),
+    ]);
   }
 
-  remove(key, callback) {
+  async remove(key) {
     const matches = /^([^:]+):([^:]+)$/.exec(key);
-    if (matches) {
-      this.client.srem([`ueberDB:keys:${matches[1]}`, matches[0]]);
-    }
-    this.client.del(key, callback);
+    await Promise.all([
+      matches && this._client.sRem(`ueberDB:keys:${matches[1]}`, matches[0]),
+      this._client.del(key),
+    ]);
   }
 
-  doBulk(bulk, callback) {
-    const multi = this.client.multi();
+  async doBulk(bulk) {
+    const multi = this._client.multi();
 
     for (const {key, type, value} of bulk) {
       const matches = /^([^:]+):([^:]+)$/.exec(key);
       if (type === 'set') {
         if (matches) {
-          multi.sadd([`ueberDB:keys:${matches[1]}`, matches[0]]);
+          multi.sAdd(`ueberDB:keys:${matches[1]}`, matches[0]);
         }
         multi.set(key, value);
       } else if (type === 'remove') {
         if (matches) {
-          multi.srem([`ueberDB:keys:${matches[1]}`, matches[0]]);
+          multi.sRem(`ueberDB:keys:${matches[1]}`, matches[0]);
         }
         multi.del(key);
       }
     }
 
-    multi.exec(callback);
+    await multi.exec();
   }
 
-  close(callback) {
-    this.client.quit(() => {
-      callback();
-    });
+  async close() {
+    await this._client.quit();
+    this._client = null;
   }
 };

package/lib/AbstractDatabase.js

@@ -4,6 +4,11 @@ const logging = require('./logging');
 
 const nullLogger = logging.normalizeLogger(null);
 
+// Format: All characters match themselves except * matches any zero or more characters. No
+// backslash escaping is supported, so it is impossible to create a pattern that matches only the
+// '*' character.
+const simpleGlobToRegExp = (s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*');
+
 module.exports = class AbstractDatabase {
   constructor() {
     if (new.target === module.exports) {
@@ -19,13 +24,8 @@ module.exports = class AbstractDatabase {
    * For findKey regex. Used by document dbs like mongodb or dirty.
    */
   createFindRegex(key, notKey) {
-    let regex = '';
-    key = key.replace(/\*/g, '.*');
-    regex = `(?=^${key}$)`;
-    if (notKey != null) {
-      notKey = notKey.replace(/\*/g, '.*');
-      regex += `(?!${notKey}$)`;
-    }
+    let regex = `^(?=${simpleGlobToRegExp(key)}$)`;
+    if (notKey != null) regex += `(?!${simpleGlobToRegExp(notKey)}$)`;
     return new RegExp(regex);
   }
 

package/lib/CacheAndBufferLayer.js

@@ -622,10 +622,12 @@ exports.Database = class {
       }
       if (success) entries.forEach((entry) => markDone(entry, null));
     }
-    // At this point we could call db.buffer.evictOld() to ensure that the number of entries in
-    // this.buffer is at or below capacity, but if we haven't run out of memory by this point then
-    // it should be safe to continue using the memory until the next call to this.buffer.set()
-    // evicts the old entries. This saves some CPU cycles at the expense of memory.
+    // This call to this.buffer.evictOld() can be safely removed (if we haven't run out of memory by
+    // this point then it is probably safe to continue using the memory until the next call to
+    // this.buffer.set() evicts the old entries), except removing it would cause some reads to be
+    // satisfied from the cache even when this.settings.cache = 0. That would contradict the
+    // documented behavior for cache = 0.
+    this.buffer.evictOld();
   }
 };
 

package/package.json

@@ -21,35 +21,29 @@
     }
   ],
   "dependencies": {
-    "async": "^3.2.2",
+    "async": "^3.2.3",
     "cassandra-driver": "^4.6.3",
     "dirty": "^1.1.3",
-    "elasticsearch": "^16.7.2",
+    "elasticsearch7": "npm:@elastic/elasticsearch@^7.17.0",
     "mongodb": "^3.7.3",
     "mssql": "^8.1.0",
     "mysql": "2.18.1",
-    "nano": "^9.0.5",
-    "pg": "^8.7.1",
-    "redis": "^3.1.2",
+    "nano": "^10.0.0",
+    "pg": "^8.7.3",
+    "redis": "^4.1.0",
     "rethinkdb": "^2.4.2",
-    "simple-git": "^3.6.0"
+    "simple-git": "^3.7.1"
   },
   "optionalDependencies": {
-    "sqlite3": "github:mapbox/node-sqlite3#593c9d498be2510d286349134537e3bf89401c4a"
+    "sqlite3": "^5.0.6"
   },
   "devDependencies": {
-    "cli-table": "^0.3.8",
-    "eslint": "^7.32.0",
-    "eslint-config-etherpad": "^2.0.3",
-    "eslint-plugin-cypress": "^2.12.1",
-    "eslint-plugin-eslint-comments": "^3.2.0",
-    "eslint-plugin-mocha": "^9.0.0",
-    "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-prefer-arrow": "^1.2.3",
-    "eslint-plugin-promise": "^5.1.1",
-    "eslint-plugin-you-dont-need-lodash-underscore": "^6.12.0",
-    "mocha": "^9.2.2",
+    "cli-table": "^0.3.11",
+    "eslint": "^8.14.0",
+    "eslint-config-etherpad": "^3.0.13",
+    "mocha": "^10.0.0",
     "randexp": "^0.5.3",
+    "typescript": "^4.6.4",
     "wtfnode": "^0.9.1"
   },
   "repository": {
@@ -57,7 +51,7 @@
     "url": "https://github.com/ether/ueberDB.git"
   },
   "main": "./index",
-  "version": "2.2.3",
+  "version": "4.0.0",
   "bugs": {
     "url": "https://github.com/ether/ueberDB/issues"
   },
@@ -77,46 +71,7 @@
       "email": "john@mclear.co.uk"
     }
   ],
-  "eslintConfig": {
-    "overrides": [
-      {
-        "files": [
-          "**/.eslintrc.js"
-        ],
-        "extends": "etherpad/node"
-      },
-      {
-        "files": [
-          "**/*"
-        ],
-        "excludedFiles": [
-          "**/.eslintrc.js"
-        ],
-        "extends": "etherpad/node"
-      },
-      {
-        "files": [
-          "test/**/*"
-        ],
-        "excludedFiles": [
-          "**/.eslintrc.js"
-        ],
-        "extends": "etherpad/tests/backend",
-        "overrides": [
-          {
-            "files": [
-              "test/lib/**/*"
-            ],
-            "rules": {
-              "mocha/no-exports": "off"
-            }
-          }
-        ]
-      }
-    ],
-    "root": true
-  },
   "engines": {
-    "node": "^10.17.0 || >=11.14.0"
+    "node": ">=14.15.0"
   }
 }

package/test/lib/databases.js

@@ -61,4 +61,10 @@ exports.databases = {
       findKeysMax: 30,
     },
   },
+  elasticsearch: {
+    base_index: 'ueberdb_test',
+    speeds: {
+      findKeysMax: 30,
+    },
+  },
 };

package/test/test.js

@@ -21,7 +21,7 @@ after(async function () {
     console.error('node should have exited by now but something is keeping it open ' +
                   'such as an open connection or active timer');
     wtfnode.dump();
-    process.exit(1); // eslint-disable-line no-process-exit
+    process.exit(1); // eslint-disable-line n/no-process-exit
   }, 5000).unref();
 });
 
@@ -142,7 +142,6 @@ describe(__filename, function () {
 
               it('findKeys with exclusion works', async function () {
                 if (database === 'mongodb') this.skip(); // TODO: Fix mongodb.
-                if (database === 'redis') this.skip(); // TODO: Fix redis.
                 const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
                 await Promise.all([
                   db.set(key, true),
@@ -237,7 +236,7 @@ describe(__filename, function () {
               });
 
               it('speed is acceptable', async function () {
-                this.timeout(60000);
+                this.timeout(180000);
 
                 const {speeds: {
                   count = 1000,

package/test/test_elasticsearch.js

@@ -0,0 +1,133 @@
+'use strict';
+
+const assert = require('assert').strict;
+const es = require('elasticsearch7');
+const {databases: {elasticsearch: cfg}} = require('./lib/databases');
+const logging = require('../lib/logging');
+const ueberdb = require('../index');
+
+const logger = new class extends logging.ConsoleLogger {
+  info() {}
+  isInfoEnabled() { return false; }
+}();
+
+describe(__filename, function () {
+  this.timeout(60000);
+
+  const {base_index = 'ueberdb_test'} = cfg;
+  let client;
+  let db;
+
+  beforeEach(async function () {
+    client = new es.Client({
+      node: `http://${cfg.host || '127.0.0.1'}:${cfg.port || '9200'}`,
+    });
+    await client.indices.delete({index: `${base_index}*`}, {ignore: [404]});
+  });
+
+  afterEach(async function () {
+    if (db != null) await db.close();
+    db = null;
+    await client.indices.delete({index: `${base_index}*`}, {ignore: [404]});
+    client.close();
+    client = null;
+  });
+
+  describe('migration to schema v2', function () {
+    describe('no old data', function () {
+      for (const migrate of [false, true]) {
+        it(`migration ${migrate ? 'en' : 'dis'}abled`, async function () {
+          const settings = {base_index, ...cfg};
+          delete settings.migrate_to_newer_schema;
+          db = new ueberdb.Database('elasticsearch', settings, {}, logger);
+          await db.init();
+          const indices = [];
+          const {body: res} = await client.indices.get({index: `${base_index}*`});
+          for (const [k, v] of Object.entries(res)) {
+            indices.push(k);
+            indices.push(...Object.keys(v.aliases));
+          }
+          assert.deepEqual(indices.sort(), [`${base_index}_s2`, `${base_index}_s2_i0`].sort());
+        });
+      }
+    });
+
+    describe('existing data', function () {
+      const data = new Map([
+        ['foo:number', 42],
+        ['foo:string', 'value'],
+        ['foo:object', {k: 'v'}],
+        ['foo:p:s:number', 42],
+        ['foo:p:s:string', 'value'],
+        ['foo:p:s:object', {k: 'v'}],
+      ]);
+
+      const setOld = async (k, v) => {
+        const kp = k.split(':');
+        const index = kp.length === 4 ? `${base_index}-${kp[0]}-${kp[2]}` : base_index;
+        await client.index({
+          index,
+          type: kp.length === 4 ? encodeURIComponent(kp[1]) : kp[0],
+          id: kp.length === 4 ? kp[3] : encodeURIComponent(kp[1]),
+          body: {
+            // The old elasticsearch driver was inconsistent: doBulk() called JSON.parse() on the
+            // value from ueberdb before writing, but set() did not. We'll assume that any existing
+            // data came from set() writes, not doBulk() writes.
+            val: JSON.stringify(v),
+          },
+        });
+        await client.indices.refresh({index});
+      };
+
+      beforeEach(async function () {
+        await Promise.all([...data].map(async ([k, v]) => await setOld(k, v)));
+      });
+
+      it('migration disabled => init error', async function () {
+        const settings = {base_index, ...cfg};
+        delete settings.migrate_to_newer_schema;
+        db = new ueberdb.Database('elasticsearch', settings, {}, logger);
+        await assert.rejects(db.init(), /migrate_to_newer_schema/);
+      });
+
+      it('migration enabled', async function () {
+        const settings = {base_index, ...cfg, migrate_to_newer_schema: true};
+        db = new ueberdb.Database('elasticsearch', settings, {}, logger);
+        await db.init();
+        await Promise.all([...data].map(async ([k, v]) => {
+          assert.deepEqual(await db.get(k), v);
+        }));
+      });
+
+      it('each attempt uses a new index', async function () {
+        await setOld('a-x:b:c-x:d', 'v'); // Force a conversion failure.
+        const settings = {base_index, ...cfg, migrate_to_newer_schema: true};
+        db = new ueberdb.Database('elasticsearch', settings, {}, logger);
+        const getIndices =
+            async () => Object.keys((await client.indices.get({index: `${base_index}_s2*`})).body);
+        assert.deepEqual(await getIndices(), []);
+        await assert.rejects(db.init(), /ambig/);
+        assert.deepEqual(await getIndices(), [`${base_index}_s2_migrate_attempt_0`]);
+        await assert.rejects(db.init(), /ambig/);
+        assert.deepEqual((await getIndices()).sort(), [
+          `${base_index}_s2_migrate_attempt_0`,
+          `${base_index}_s2_migrate_attempt_1`,
+        ]);
+      });
+
+      it('final name not created until success', async function () {
+      });
+
+      describe('ambiguous key', function () {
+        for (const k of ['a:b:c-x:d', 'a-x:b:c:d', 'a-x:b:c-x:d']) {
+          it(k, async function () {
+            await setOld(k, 'v');
+            const settings = {base_index, ...cfg, migrate_to_newer_schema: true};
+            db = new ueberdb.Database('elasticsearch', settings, {}, logger);
+            await assert.rejects(db.init(), /ambig/);
+          });
+        }
+      });
+    });
+  });
+});