ueberdb2 4.0.11 → 4.0.17

package/lib/AbstractDatabase.js

@@ -1,37 +0,0 @@
-'use strict';
-
-const logging = require('./logging');
-
-const nullLogger = logging.normalizeLogger(null);
-
-// Format: All characters match themselves except * matches any zero or more characters. No
-// backslash escaping is supported, so it is impossible to create a pattern that matches only the
-// '*' character.
-const simpleGlobToRegExp = (s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*');
-
-module.exports = class AbstractDatabase {
-  constructor() {
-    if (new.target === module.exports) {
-      throw new TypeError('cannot instantiate Abstract Database directly');
-    }
-    for (const fn of ['init', 'close', 'get', 'findKeys', 'remove', 'set']) {
-      if (typeof this[fn] !== 'function') throw new TypeError(`method ${fn} not defined`);
-    }
-    this.logger = nullLogger;
-  }
-
-  /**
-   * For findKey regex. Used by document dbs like mongodb or dirty.
-   */
-  createFindRegex(key, notKey) {
-    let regex = `^(?=${simpleGlobToRegExp(key)}$)`;
-    if (notKey != null) regex += `(?!${simpleGlobToRegExp(notKey)}$)`;
-    return new RegExp(regex);
-  }
-
-  doBulk(operations, cb) {
-    throw new Error('the doBulk method must be implemented if write caching is enabled');
-  }
-
-  get isAsync() { return false; }
-};

package/test/test.js

@@ -1,328 +0,0 @@
-'use strict';
-
-const wtfnode = require('wtfnode'); // This should be first so that it can instrument everything.
-
-const Clitable = require('cli-table');
-const Randexp = require('randexp');
-const assert = require('assert').strict;
-const databases = require('./lib/databases').databases;
-const fs = require('fs').promises;
-const logging = require('../lib/logging');
-const ueberdb = require('../index');
-
-const maxKeyLength = 100;
-const randomString = (length = maxKeyLength) => new Randexp(new RegExp(`.{${length}}`)).gen();
-
-// eslint-disable-next-line mocha/no-top-level-hooks
-after(async function () {
-  // Add a timeout to forcibly exit if something is keeping node from exiting cleanly.
-  // The timeout is unref()ed so that it doesn't prevent node from exiting when done.
-  setTimeout(() => {
-    console.error('node should have exited by now but something is keeping it open ' +
-                  'such as an open connection or active timer');
-    wtfnode.dump();
-    process.exit(1); // eslint-disable-line n/no-process-exit
-  }, 5000).unref();
-});
-
-describe(__filename, function () {
-  let speedTable;
-  let db;
-
-  before(async function () {
-    speedTable = new Clitable({
-      head: [
-        'Database',
-        'read cache',
-        'write buffer',
-        '#',
-        'ms/set',
-        'ms/get',
-        'ms/findKeys',
-        'ms/remove',
-        'total ms',
-        'total ms/#',
-      ],
-      colWidths: [15, 15, 15, 8, 13, 13, 13, 13, 13, 13],
-    });
-  });
-
-  after(async function () {
-    console.log(speedTable.toString());
-  });
-
-  for (const database of Object.keys(databases)) {
-    const dbSettings = databases[database];
-    describe(database, function () {
-      for (const readCache of [false, true]) {
-        describe(`${readCache ? '' : 'no '}read cache`, function () {
-          for (const writeBuffer of [false, true]) {
-            describe(`${writeBuffer ? '' : 'no '}write buffer`, function () {
-              this.timeout(5000);
-
-              before(async function () {
-                if (dbSettings.filename) await fs.unlink(dbSettings.filename).catch(() => {});
-                db = new ueberdb.Database(database, dbSettings, {
-                  ...(readCache ? {} : {cache: 0}),
-                  ...(writeBuffer ? {} : {writeInterval: 0}),
-                }, new logging.ConsoleLogger());
-                await db.init();
-              });
-
-              after(async function () {
-                await db.close();
-                if (dbSettings.filename) await fs.unlink(dbSettings.filename).catch(() => {});
-              });
-
-              describe('white space in key is not ignored', function () {
-                for (const space of [false, true]) {
-                  describe(`key ${space ? 'has' : 'does not have'} a trailing space`, function () {
-                    let input;
-                    let key;
-
-                    before(async function () {
-                      input = {a: 1, b: new Randexp(/.+/).gen()};
-                      key = randomString(maxKeyLength - 1) + (space ? ' ' : '');
-                      await db.set(key, input);
-                    });
-
-                    it('get(key) -> record', async function () {
-                      const output = await db.get(key);
-                      assert.equal(JSON.stringify(output), JSON.stringify(input));
-                    });
-
-                    it('get(`${key} `) -> nullish', async function () {
-                      const output = await db.get(`${key} `);
-                      assert(output == null);
-                    });
-
-                    if (space) {
-                      it('get(key.slice(0, -1)) -> nullish', async function () {
-                        const output = await db.get(key.slice(0, -1));
-                        assert(output == null);
-                      });
-                    }
-                  });
-                }
-              });
-
-              it('get of unknown key -> nullish', async function () {
-                const key = randomString();
-                assert(await db.get(key) == null);
-              });
-
-              it('set+get works', async function () {
-                const input = {a: 1, b: new Randexp(/.+/).gen()};
-                const key = randomString();
-                await db.set(key, input);
-                const output = await db.get(key);
-                assert.equal(JSON.stringify(output), JSON.stringify(input));
-              });
-
-              it('set+get with random key/value works', async function () {
-                const input = {testLongString: new Randexp(/[a-f0-9]{50000}/).gen()};
-                const key = randomString();
-                await db.set(key, input);
-                const output = await db.get(key);
-                assert.equal(JSON.stringify(output), JSON.stringify(input));
-              });
-
-              it('findKeys works', async function () {
-                if (database === 'mongodb') this.skip(); // TODO: Fix mongodb.
-                // TODO setting a key with non ascii chars
-                const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
-                await Promise.all([
-                  db.set(key, true),
-                  db.set(`${key}a`, true),
-                  db.set(`nonmatching_${key}`, false),
-                ]);
-                const keys = await db.findKeys(`${key}*`, null);
-                assert.deepEqual(keys.sort(), [key, `${key}a`]);
-              });
-
-              it('findKeys with exclusion works', async function () {
-                if (database === 'mongodb') this.skip(); // TODO: Fix mongodb.
-                const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
-                await Promise.all([
-                  db.set(key, true),
-                  db.set(`${key}a`, true),
-                  db.set(`${key}b`, false),
-                  db.set(`${key}b2`, false),
-                  db.set(`nonmatching_${key}`, false),
-                ]);
-                const keys = await db.findKeys(`${key}*`, `${key}b*`);
-                assert.deepEqual(keys.sort(), [key, `${key}a`].sort());
-              });
-
-              it('findKeys with no matches works', async function () {
-                const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
-                await db.set(key, true);
-                const keys = await db.findKeys(`${key}_nomatch_*`, null);
-                assert.deepEqual(keys, []);
-              });
-
-              it('findKeys with no wildcard works', async function () {
-                const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
-                await db.set(key, true);
-                const keys = await db.findKeys(key, null);
-                assert.deepEqual(keys, [key]);
-              });
-
-              it('remove works', async function () {
-                const input = {a: 1, b: new Randexp(/.+/).gen()};
-                const key = randomString();
-                await db.set(key, input);
-                assert.equal(JSON.stringify(await db.get(key)), JSON.stringify(input));
-                await db.remove(key);
-                assert(await db.get(key) == null);
-              });
-
-              it('getSub of existing property works', async function () {
-                await db.set('k', {sub1: {sub2: 'v'}});
-                assert.equal(await db.getSub('k', ['sub1', 'sub2']), 'v');
-                assert.deepEqual(await db.getSub('k', ['sub1']), {sub2: 'v'});
-                assert.deepEqual(await db.getSub('k', []), {sub1: {sub2: 'v'}});
-              });
-
-              it('getSub of missing property returns nullish', async function () {
-                await db.set('k', {sub1: {}});
-                assert(await db.getSub('k', ['sub1', 'sub2']) == null);
-
-                await db.set('k', {});
-                assert(await db.getSub('k', ['sub1', 'sub2']) == null);
-                assert(await db.getSub('k', ['sub1']) == null);
-
-                await db.remove('k');
-                assert(await db.getSub('k', ['sub1', 'sub2']) == null);
-                assert(await db.getSub('k', ['sub1']) == null);
-                assert(await db.getSub('k', []) == null);
-              });
-
-              it('setSub can modify an existing property', async function () {
-                await db.set('k', {sub1: {sub2: 'v'}});
-                await db.setSub('k', ['sub1', 'sub2'], 'v2');
-                assert.deepEqual(await db.get('k'), {sub1: {sub2: 'v2'}});
-
-                await db.setSub('k', ['sub1'], 'v2');
-                assert.deepEqual(await db.get('k'), {sub1: 'v2'});
-
-                await db.setSub('k', [], 'v3');
-                assert.equal(await db.get('k'), 'v3');
-              });
-
-              it('setSub can add a new property', async function () {
-                await db.remove('k');
-                await db.setSub('k', [], {});
-                assert.deepEqual(await db.get('k'), {});
-                await db.setSub('k', ['sub1'], {});
-                assert.deepEqual(await db.get('k'), {sub1: {}});
-                await db.setSub('k', ['sub1', 'sub2'], 'v');
-                assert.deepEqual(await db.get('k'), {sub1: {sub2: 'v'}});
-
-                await db.remove('k');
-                await db.setSub('k', ['sub1', 'sub2'], 'v');
-                assert.deepEqual(await db.get('k'), {sub1: {sub2: 'v'}});
-              });
-
-              it('setSub rejects attempts to set properties on primitives', async function () {
-                for (const v of ['hello world', 42, true]) {
-                  await db.set('k', v);
-                  assert.rejects(db.setSub('k', ['sub'], 'x'), {
-                    name: 'TypeError',
-                    message: /property "sub" on non-object/,
-                  });
-                  assert.deepEqual(await db.get('k'), v);
-                }
-              });
-
-              it('speed is acceptable', async function () {
-                this.timeout(180000);
-
-                const {speeds: {
-                  count = 1000,
-                  setMax = 3,
-                  getMax = 0.1,
-                  findKeysMax = 3,
-                  removeMax = 1,
-                } = {}} = dbSettings || {};
-
-                const input = {a: 1, b: new Randexp(/.+/).gen()};
-                // TODO setting a key with non ascii chars
-                const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
-                // Pre-allocate an array before starting the timer so that time spent growing the
-                // array doesn't throw off the benchmarks.
-                const promises = [...Array(count + 1)].map(() => null);
-
-                const timers = {start: Date.now()};
-
-                for (let i = 0; i < count; ++i) promises[i] = db.set(key + i, input);
-                promises[count] = db.flush();
-                await Promise.all(promises);
-                timers.set = Date.now();
-
-                for (let i = 0; i < count; ++i) promises[i] = db.get(key + i);
-                await Promise.all(promises);
-                timers.get = Date.now();
-
-                for (let i = 0; i < count; ++i) promises[i] = db.findKeys(key + i, null);
-                await Promise.all(promises);
-                timers.findKeys = Date.now();
-
-                for (let i = 0; i < count; ++i) promises[i] = db.remove(key + i);
-                promises[count] = db.flush();
-                await Promise.all(promises);
-                timers.remove = Date.now();
-
-                const timePerOp = {
-                  set: (timers.set - timers.start) / count,
-                  get: (timers.get - timers.set) / count,
-                  findKeys: (timers.findKeys - timers.get) / count,
-                  remove: (timers.remove - timers.findKeys) / count,
-                };
-                speedTable.push([
-                  database,
-                  readCache ? 'yes' : 'no',
-                  writeBuffer ? 'yes' : 'no',
-                  count,
-                  timePerOp.set,
-                  timePerOp.get,
-                  timePerOp.findKeys,
-                  timePerOp.remove,
-                  timers.remove - timers.start,
-                  (timers.remove - timers.start) / count,
-                ]);
-
-                // Removes the "Acceptable ms/op" column if there is no enforced limit.
-                const filterColumn = (row) => {
-                  if (readCache && writeBuffer) return row;
-                  row.splice(1, 1);
-                  return row;
-                };
-                const acceptableTable = new Clitable({
-                  head: filterColumn(['op', 'Acceptable ms/op', 'Actual ms/op']),
-                  colWidths: filterColumn([10, 18, 18]),
-                });
-                acceptableTable.push(...[
-                  ['set', setMax, timePerOp.set],
-                  ['get', getMax, timePerOp.get],
-                  ['findKeys', findKeysMax, timePerOp.findKeys],
-                  ['remove', removeMax, timePerOp.remove],
-                ].map(filterColumn));
-                console.log(acceptableTable.toString());
-
-                if (readCache && writeBuffer) {
-                  assert(setMax >= timePerOp.set);
-                  assert(getMax >= timePerOp.get);
-                  assert(findKeysMax >= timePerOp.findKeys);
-                  assert(removeMax >= timePerOp.remove);
-                }
-              });
-            });
-          }
-        });
-      }
-    });
-  }
-});
-
-// TODO: Need test which prefills with 1e7 of data then does a get.

package/test/test_bulk.js

@@ -1,69 +0,0 @@
-'use strict';
-
-const assert = require('assert').strict;
-const ueberdb = require('../index');
-const util = require('util');
-
-const range = (N) => [...Array(N).keys()];
-
-describe(__filename, function () {
-  let db = null;
-  let mock = null;
-  const createDb = async (wrapperSettings) => {
-    const settings = {};
-    db = new ueberdb.Database('mock', settings, wrapperSettings);
-    mock = settings.mock;
-    mock.once('init', (cb) => cb());
-    await db.init();
-  };
-
-  afterEach(async function () {
-    if (mock != null) {
-      mock.removeAllListeners();
-      mock.once('close', (cb) => cb());
-      mock = null;
-    }
-    if (db != null) {
-      await db.close();
-      db = null;
-    }
-  });
-
-  describe('bulkLimit', function () {
-    const bulkLimits = [0, false, null, undefined, '', 1, 2];
-    for (const bulkLimit of bulkLimits) {
-      it(bulkLimit === undefined ? 'undefined' : JSON.stringify(bulkLimit), async function () {
-        await createDb({bulkLimit});
-        const gotWrites = [];
-        mock.on('set', util.callbackify(async (k, v) => gotWrites.push(1)));
-        mock.on('doBulk', util.callbackify(async (ops) => gotWrites.push(ops.length)));
-        const N = 10;
-        await Promise.all(range(N).map((i) => db.set(`key${i}`, `val${i}`)));
-        const wantLimit = bulkLimit || N;
-        const wantWrites = range(N / wantLimit).map((i) => wantLimit);
-        assert.deepEqual(gotWrites, wantWrites);
-      });
-    }
-  });
-
-  it('bulk failures are retried individually', async function () {
-    await createDb({});
-    const gotDoBulkCalls = [];
-    mock.on('doBulk', util.callbackify(async (ops) => {
-      gotDoBulkCalls.push(ops.length);
-      throw new Error('test');
-    }));
-    const gotWrites = new Map();
-    const wantWrites = new Map();
-    mock.on('set', util.callbackify(async (k, v) => gotWrites.set(k, v)));
-    const N = 10;
-    await Promise.all(range(N).map(async (i) => {
-      const k = `key${i}`;
-      const v = `val${i}`;
-      wantWrites.set(k, JSON.stringify(v));
-      await db.set(k, v);
-    }));
-    assert.deepEqual(gotDoBulkCalls, [N]);
-    assert.deepEqual(gotWrites, wantWrites);
-  });
-});

package/test/test_getSub.js

@@ -1,31 +0,0 @@
-'use strict';
-
-const assert = require('assert').strict;
-const ueberdb = require('../index');
-
-describe(__filename, function () {
-  let db;
-
-  beforeEach(async function () {
-    db = new ueberdb.Database('memory', {}, {});
-    await db.init();
-    await db.set('k', {s: 'v'});
-  });
-
-  afterEach(async function () {
-    if (db != null) await db.close();
-    db = null;
-  });
-
-  it('getSub stops at non-objects', async function () {
-    assert(await db.getSub('k', ['s', 'length']) == null);
-  });
-
-  it('getSub ignores non-own properties', async function () {
-    assert(await db.getSub('k', ['toString']) == null);
-  });
-
-  it('getSub ignores __proto__', async function () {
-    assert(await db.getSub('k', ['__proto__']) == null);
-  });
-});

package/test/test_lru.js

@@ -1,145 +0,0 @@
-'use strict';
-
-const LRU = require('../lib/CacheAndBufferLayer').exportedForTesting.LRU;
-const assert = require('assert').strict;
-
-describe(__filename, function () {
-  describe('capacity = 0', function () {
-    it('constructor does not throw', async function () {
-      new LRU(0);
-    });
-
-    describe('behavior when empty', function () {
-      it('get() returns nullish', async function () {
-        assert((new LRU(0)).get('k') == null);
-      });
-
-      it('empty iteration', async function () {
-        assert.equal([...(new LRU(0))].length, 0);
-      });
-
-      it('evictOld() does not throw', async function () {
-        (new LRU(0)).evictOld();
-      });
-    });
-
-    describe('single entry with evictable = false', function () {
-      let evictable, lru, key, val;
-
-      beforeEach(async function () {
-        evictable = false;
-        lru = new LRU(0, () => evictable);
-        key = 'k';
-        val = 'v';
-        lru.set(key, val);
-      });
-
-      it('get() works', async function () {
-        assert.equal(lru.get(key), val);
-      });
-
-      it('iterate works', async function () {
-        assert.deepEqual([...lru], [[key, val]]);
-      });
-
-      it('re-set() works', async function () {
-        const val2 = 'v2';
-        lru.set(key, val2);
-        assert.equal(lru.get(key), val2);
-        assert.deepEqual([...lru], [[key, val2]]);
-      });
-
-      it('evictOld() does not evict', async function () {
-        lru.evictOld();
-        assert.deepEqual([...lru], [[key, val]]);
-      });
-
-      it('evictOld() evicts after setting evictable = true', async function () {
-        evictable = true;
-        lru.evictOld();
-        assert.deepEqual([...lru], []);
-      });
-    });
-
-    describe('set immediately evicts if evictable', function () {
-      it('explicitly evictable', async function () {
-        const lru = new LRU(0, () => true);
-        lru.set('k', 'v');
-        assert(lru.get('k') == null);
-        assert.deepEqual([...lru], []);
-      });
-
-      it('is evictable by default', async function () {
-        const lru = new LRU(0);
-        lru.set('k', 'v');
-        assert(lru.get('k') == null);
-        assert.deepEqual([...lru], []);
-      });
-    });
-  });
-
-  describe('capacity = 2', function () {
-    let evictable, lru;
-
-    beforeEach(async function () {
-      evictable = () => false;
-      lru = new LRU(2, (k, v) => evictable(k, v));
-    });
-
-    it('iterates oldest first', async function () {
-      lru.set(0, '0');
-      lru.set(1, '1');
-      let i = 0;
-      for (const [k, v] of lru) {
-        assert.equal(k, i);
-        assert.equal(v, `${i}`);
-        i++;
-      }
-      assert.equal(i, 2);
-    });
-
-    it('get(k) updates recently used', async function () {
-      lru.set(0, '0');
-      lru.set(1, '1');
-      assert.equal(lru.get(0), '0');
-      assert.deepEqual([...lru], [[1, '1'], [0, '0']]);
-    });
-
-    it('get(k, false) does not update recently used', async function () {
-      lru.set(0, '0');
-      lru.set(1, '1');
-      assert.equal(lru.get(0, false), '0');
-      assert.deepEqual([...lru], [[0, '0'], [1, '1']]);
-    });
-
-    it('re-set() updates recently used', async function () {
-      lru.set(0, '0');
-      lru.set(1, '1');
-      lru.set(0, '00');
-      assert.deepEqual([...lru], [[1, '1'], [0, '00']]);
-    });
-
-    it('evictOld() only evicts evictable entries', async function () {
-      evictable = () => false;
-      lru.set(0, '0');
-      lru.set(1, '1');
-      lru.set(2, '2');
-      lru.set(3, '3');
-      assert.deepEqual([...lru], [[0, '0'], [1, '1'], [2, '2'], [3, '3']]);
-      evictable = (k) => k >= 2;
-      lru.evictOld();
-      // The newer entries should be evicted because the older are dirty/writingInProgress.
-      assert.deepEqual([...lru], [[0, '0'], [1, '1']]);
-    });
-
-    it('evictOld() does nothing if at or below capacity', async function () {
-      evictable = () => true;
-      lru.set(0, '0');
-      lru.evictOld();
-      assert.deepEqual([...lru], [[0, '0']]);
-      lru.set(1, '1');
-      lru.evictOld();
-      assert.deepEqual([...lru], [[0, '0'], [1, '1']]);
-    });
-  });
-});

package/test/test_memory.js

@@ -1,31 +0,0 @@
-'use strict';
-
-const assert = require('assert').strict;
-const memory = require('../databases/memory_db');
-
-describe(__filename, function () {
-  describe('data option', function () {
-    it('uses existing records from data option', async function () {
-      const db = new memory.Database({data: new Map([['foo', 'bar']])});
-      await db.init();
-      assert.equal(await db.get('foo'), 'bar');
-    });
-
-    it('updates existing map', async function () {
-      const data = new Map();
-      const db = new memory.Database({data});
-      await db.init();
-      await db.set('foo', 'bar');
-      assert.equal(data.get('foo'), 'bar');
-    });
-
-    it('does not clear map on close', async function () {
-      const data = new Map();
-      const db = new memory.Database({data});
-      await db.init();
-      await db.set('foo', 'bar');
-      await db.close();
-      assert.equal(data.get('foo'), 'bar');
-    });
-  });
-});

package/test/test_postgres.js

@@ -1,16 +0,0 @@
-'use strict';
-
-const assert = require('assert').strict;
-const {databases} = require('./lib/databases');
-const ueberdb = require('../index');
-
-describe(__filename, function () {
-  it('connection string instead of settings object', async function () {
-    const {user, password, host, database} = databases.postgres;
-    const db =
-        new ueberdb.Database('postgres', `postgres://${user}:${password}@${host}/${database}`);
-    await db.init();
-    await db.set('key', 'val');
-    assert.equal(await db.get('key'), 'val');
-  });
-});