ueberdb2 1.4.16

package/test/test.js

@@ -0,0 +1,312 @@
+'use strict';
+
+const wtfnode = require('wtfnode'); // This should be first so that it can instrument everything.
+
+const Clitable = require('cli-table');
+const Randexp = require('randexp');
+const assert = require('assert').strict;
+const databases = require('./lib/databases').databases;
+const fs = require('fs').promises;
+const ueberdb = require('../index');
+const util = require('util');
+
+const maxKeyLength = 100;
+const randomString = (length = maxKeyLength) => new Randexp(new RegExp(`.{${length}}`)).gen();
+
+// eslint-disable-next-line mocha/no-top-level-hooks
+after(async function () {
+  // Add a timeout to forcibly exit if something is keeping node from exiting cleanly.
+  // The timeout is unref()ed so that it doesn't prevent node from exiting when done.
+  setTimeout(() => {
+    console.error('node should have exited by now but something is keeping it open ' +
+                  'such as an open connection or active timer');
+    wtfnode.dump();
+    process.exit(1); // eslint-disable-line no-process-exit
+  }, 5000).unref();
+});
+
+// Returns an object with promisified equivalents of ueberdb.Database methods.
+const promisifyDb = (db) => {
+  const ret = {};
+  const fns = ['init', 'close', 'get', 'getSub', 'findKeys', 'set', 'setSub', 'remove', 'flush'];
+  for (const fn of fns) ret[fn] = util.promisify(db[fn].bind(db));
+  return ret;
+};
+
+describe(__filename, function () {
+  let speedTable;
+  let db;
+  let pdb;
+
+  before(async function () {
+    speedTable = new Clitable({
+      head: [
+        'Database',
+        'read cache',
+        'write buffer',
+        '#',
+        'ms/set',
+        'ms/get',
+        'ms/findKeys',
+        'ms/remove',
+        'total ms',
+        'total ms/#',
+      ],
+      colWidths: [15, 15, 15, 8, 13, 13, 13, 13, 13, 13],
+    });
+  });
+
+  after(async function () {
+    console.log(speedTable.toString());
+  });
+
+  for (const database of Object.keys(databases)) {
+    const dbSettings = databases[database];
+    describe(database, function () {
+      for (const readCache of [false, true]) {
+        describe(`${readCache ? '' : 'no '}read cache`, function () {
+          for (const writeBuffer of [false, true]) {
+            describe(`${writeBuffer ? '' : 'no '}write buffer`, function () {
+              this.timeout(5000);
+
+              before(async function () {
+                if (dbSettings.filename) await fs.unlink(dbSettings.filename).catch(() => {});
+                db = new ueberdb.Database(database, dbSettings, {
+                  ...(readCache ? {} : {cache: 0}),
+                  ...(writeBuffer ? {} : {writeInterval: 0}),
+                });
+                pdb = promisifyDb(db);
+                await pdb.init();
+              });
+
+              after(async function () {
+                await pdb.close();
+                if (dbSettings.filename) await fs.unlink(dbSettings.filename).catch(() => {});
+              });
+
+              describe('white space in key is not ignored', function () {
+                for (const space of [false, true]) {
+                  describe(`key ${space ? 'has' : 'does not have'} a trailing space`, function () {
+                    let input;
+                    let key;
+
+                    before(async function () {
+                      input = {a: 1, b: new Randexp(/.+/).gen()};
+                      key = randomString(maxKeyLength - 1) + (space ? ' ' : '');
+                      await pdb.set(key, input);
+                    });
+
+                    it('get(key) -> record', async function () {
+                      const output = await pdb.get(key);
+                      assert.equal(JSON.stringify(output), JSON.stringify(input));
+                    });
+
+                    it('get(`${key} `) -> nullish', async function () {
+                      const output = await pdb.get(`${key} `);
+                      assert(output == null);
+                    });
+
+                    if (space) {
+                      it('get(key.slice(0, -1)) -> nullish', async function () {
+                        const output = await pdb.get(key.slice(0, -1));
+                        assert(output == null);
+                      });
+                    }
+                  });
+                }
+              });
+
+              it('get of unknown key -> nullish', async function () {
+                const key = randomString();
+                assert(await pdb.get(key) == null);
+              });
+
+              it('set+get works', async function () {
+                const input = {a: 1, b: new Randexp(/.+/).gen()};
+                const key = randomString();
+                await pdb.set(key, input);
+                const output = await pdb.get(key);
+                assert.equal(JSON.stringify(output), JSON.stringify(input));
+              });
+
+              it('set+get with random key/value works', async function () {
+                const input = {testLongString: new Randexp(/[a-f0-9]{50000}/).gen()};
+                const key = randomString();
+                await pdb.set(key, input);
+                const output = await pdb.get(key);
+                assert.equal(JSON.stringify(output), JSON.stringify(input));
+              });
+
+              it('findKeys works', async function () {
+                const input = {a: 1, b: new Randexp(/.+/).gen()};
+                // TODO setting a key with non ascii chars
+                const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
+                await Promise.all([
+                  pdb.set(`${key}:test2`, input),
+                  pdb.set(`${key}:test`, input),
+                ]);
+                const output = await pdb.findKeys(`${key}:*`, null);
+                for (const keyVal of output) {
+                  const output = await pdb.get(keyVal);
+                  assert.equal(JSON.stringify(output), JSON.stringify(input));
+                }
+              });
+
+              it('remove works', async function () {
+                const input = {a: 1, b: new Randexp(/.+/).gen()};
+                const key = randomString();
+                await pdb.set(key, input);
+                assert.equal(JSON.stringify(await pdb.get(key)), JSON.stringify(input));
+                await pdb.remove(key);
+                assert(await pdb.get(key) == null);
+              });
+
+              it('getSub of existing property works', async function () {
+                await pdb.set('k', {sub1: {sub2: 'v'}});
+                assert.equal(await pdb.getSub('k', ['sub1', 'sub2']), 'v');
+                assert.deepEqual(await pdb.getSub('k', ['sub1']), {sub2: 'v'});
+                assert.deepEqual(await pdb.getSub('k', []), {sub1: {sub2: 'v'}});
+              });
+
+              it('getSub of missing property returns nullish', async function () {
+                await pdb.set('k', {sub1: {}});
+                assert(await pdb.getSub('k', ['sub1', 'sub2']) == null);
+
+                await pdb.set('k', {});
+                assert(await pdb.getSub('k', ['sub1', 'sub2']) == null);
+                assert(await pdb.getSub('k', ['sub1']) == null);
+
+                await pdb.remove('k');
+                assert(await pdb.getSub('k', ['sub1', 'sub2']) == null);
+                assert(await pdb.getSub('k', ['sub1']) == null);
+                assert(await pdb.getSub('k', []) == null);
+              });
+
+              it('setSub can modify an existing property', async function () {
+                await pdb.set('k', {sub1: {sub2: 'v'}});
+                await pdb.setSub('k', ['sub1', 'sub2'], 'v2');
+                assert.deepEqual(await pdb.get('k'), {sub1: {sub2: 'v2'}});
+
+                await pdb.setSub('k', ['sub1'], 'v2');
+                assert.deepEqual(await pdb.get('k'), {sub1: 'v2'});
+
+                await pdb.setSub('k', [], 'v3');
+                assert.equal(await pdb.get('k'), 'v3');
+              });
+
+              it('setSub can add a new property', async function () {
+                await pdb.remove('k');
+                await pdb.setSub('k', [], {});
+                assert.deepEqual(await pdb.get('k'), {});
+                await pdb.setSub('k', ['sub1'], {});
+                assert.deepEqual(await pdb.get('k'), {sub1: {}});
+                await pdb.setSub('k', ['sub1', 'sub2'], 'v');
+                assert.deepEqual(await pdb.get('k'), {sub1: {sub2: 'v'}});
+
+                await pdb.remove('k');
+                await pdb.setSub('k', ['sub1', 'sub2'], 'v');
+                assert.deepEqual(await pdb.get('k'), {sub1: {sub2: 'v'}});
+              });
+
+              it('setSub rejects attempts to set properties on primitives', async function () {
+                for (const v of ['hello world', 42, true]) {
+                  await pdb.set('k', v);
+                  assert.rejects(pdb.setSub('k', ['sub'], 'x'), {
+                    name: 'TypeError',
+                    message: /property "sub" on non-object/,
+                  });
+                  assert.deepEqual(await pdb.get('k'), v);
+                }
+              });
+
+              it('speed is acceptable', async function () {
+                this.timeout(60000);
+
+                const {speeds: {
+                  count = 1000,
+                  setMax = 3,
+                  getMax = 0.1,
+                  findKeysMax = 3,
+                  removeMax = 1,
+                } = {}} = dbSettings || {};
+
+                const input = {a: 1, b: new Randexp(/.+/).gen()};
+                // TODO setting a key with non ascii chars
+                const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
+                // Pre-allocate an array before starting the timer so that time spent growing the
+                // array doesn't throw off the benchmarks.
+                const promises = [...Array(count + 1)].map(() => null);
+
+                const timers = {start: Date.now()};
+
+                for (let i = 0; i < count; ++i) promises[i] = pdb.set(key + i, input);
+                promises[count] = pdb.flush();
+                await Promise.all(promises);
+                timers.set = Date.now();
+
+                for (let i = 0; i < count; ++i) promises[i] = pdb.get(key + i);
+                await Promise.all(promises);
+                timers.get = Date.now();
+
+                for (let i = 0; i < count; ++i) promises[i] = pdb.findKeys(key + i, null);
+                await Promise.all(promises);
+                timers.findKeys = Date.now();
+
+                for (let i = 0; i < count; ++i) promises[i] = pdb.remove(key + i);
+                promises[count] = pdb.flush();
+                await Promise.all(promises);
+                timers.remove = Date.now();
+
+                const timePerOp = {
+                  set: (timers.set - timers.start) / count,
+                  get: (timers.get - timers.set) / count,
+                  findKeys: (timers.findKeys - timers.get) / count,
+                  remove: (timers.remove - timers.findKeys) / count,
+                };
+                speedTable.push([
+                  database,
+                  readCache ? 'yes' : 'no',
+                  writeBuffer ? 'yes' : 'no',
+                  count,
+                  timePerOp.set,
+                  timePerOp.get,
+                  timePerOp.findKeys,
+                  timePerOp.remove,
+                  timers.remove - timers.start,
+                  (timers.remove - timers.start) / count,
+                ]);
+
+                // Removes the "Acceptable ms/op" column if there is no enforced limit.
+                const filterColumn = (row) => {
+                  if (readCache && writeBuffer) return row;
+                  row.splice(1, 1);
+                  return row;
+                };
+                const acceptableTable = new Clitable({
+                  head: filterColumn(['op', 'Acceptable ms/op', 'Actual ms/op']),
+                  colWidths: filterColumn([10, 18, 18]),
+                });
+                acceptableTable.push(...[
+                  ['set', setMax, timePerOp.set],
+                  ['get', getMax, timePerOp.get],
+                  ['findKeys', findKeysMax, timePerOp.findKeys],
+                  ['remove', removeMax, timePerOp.remove],
+                ].map(filterColumn));
+                console.log(acceptableTable.toString());
+
+                if (readCache && writeBuffer) {
+                  assert(setMax >= timePerOp.set);
+                  assert(getMax >= timePerOp.get);
+                  assert(findKeysMax >= timePerOp.findKeys);
+                  assert(removeMax >= timePerOp.remove);
+                }
+              });
+            });
+          }
+        });
+      }
+    });
+  }
+});
+
+// TODO: Need test which prefills with 1e7 of data then does a get.

package/test/test_bulk.js

@@ -0,0 +1,71 @@
+'use strict';
+
+const assert = require('assert').strict;
+const ueberdb = require('../index');
+const util = require('util');
+
+const range = (N) => [...Array(N).keys()];
+
+describe(__filename, function () {
+  let db = null;
+  let mock = null;
+  const createDb = async (wrapperSettings) => {
+    const settings = {};
+    const udb = new ueberdb.Database('mock', settings, wrapperSettings);
+    mock = settings.mock;
+    db = {};
+    for (const fn of ['init', 'close', 'set']) db[fn] = util.promisify(udb[fn].bind(udb));
+    mock.once('init', (cb) => cb());
+    await db.init();
+  };
+
+  afterEach(async function () {
+    if (mock != null) {
+      mock.removeAllListeners();
+      mock.once('close', (cb) => cb());
+      mock = null;
+    }
+    if (db != null) {
+      await db.close();
+      db = null;
+    }
+  });
+
+  describe('bulkLimit', function () {
+    const bulkLimits = [0, false, null, undefined, '', 1, 2];
+    for (const bulkLimit of bulkLimits) {
+      it(bulkLimit === undefined ? 'undefined' : JSON.stringify(bulkLimit), async function () {
+        await createDb({bulkLimit});
+        const gotWrites = [];
+        mock.on('set', util.callbackify(async (k, v) => gotWrites.push(1)));
+        mock.on('doBulk', util.callbackify(async (ops) => gotWrites.push(ops.length)));
+        const N = 10;
+        await Promise.all(range(N).map((i) => db.set(`key${i}`, `val${i}`)));
+        const wantLimit = bulkLimit || N;
+        const wantWrites = range(N / wantLimit).map((i) => wantLimit);
+        assert.deepEqual(gotWrites, wantWrites);
+      });
+    }
+  });
+
+  it('bulk failures are retried individually', async function () {
+    await createDb({});
+    const gotDoBulkCalls = [];
+    mock.on('doBulk', util.callbackify(async (ops) => {
+      gotDoBulkCalls.push(ops.length);
+      throw new Error('test');
+    }));
+    const gotWrites = new Map();
+    const wantWrites = new Map();
+    mock.on('set', util.callbackify(async (k, v) => gotWrites.set(k, v)));
+    const N = 10;
+    await Promise.all(range(N).map(async (i) => {
+      const k = `key${i}`;
+      const v = `val${i}`;
+      wantWrites.set(k, JSON.stringify(v));
+      await db.set(k, v);
+    }));
+    assert.deepEqual(gotDoBulkCalls, [N]);
+    assert.deepEqual(gotWrites, wantWrites);
+  });
+});

package/test/test_lru.js

@@ -0,0 +1,145 @@
+'use strict';
+
+const LRU = require('../lib/CacheAndBufferLayer').exportedForTesting.LRU;
+const assert = require('assert').strict;
+
+describe(__filename, function () {
+  describe('capacity = 0', function () {
+    it('constructor does not throw', async function () {
+      new LRU(0);
+    });
+
+    describe('behavior when empty', function () {
+      it('get() returns nullish', async function () {
+        assert((new LRU(0)).get('k') == null);
+      });
+
+      it('empty iteration', async function () {
+        assert.equal([...(new LRU(0))].length, 0);
+      });
+
+      it('evictOld() does not throw', async function () {
+        (new LRU(0)).evictOld();
+      });
+    });
+
+    describe('single entry with evictable = false', function () {
+      let evictable, lru, key, val;
+
+      beforeEach(async function () {
+        evictable = false;
+        lru = new LRU(0, () => evictable);
+        key = 'k';
+        val = 'v';
+        lru.set(key, val);
+      });
+
+      it('get() works', async function () {
+        assert.equal(lru.get(key), val);
+      });
+
+      it('iterate works', async function () {
+        assert.deepEqual([...lru], [[key, val]]);
+      });
+
+      it('re-set() works', async function () {
+        const val2 = 'v2';
+        lru.set(key, val2);
+        assert.equal(lru.get(key), val2);
+        assert.deepEqual([...lru], [[key, val2]]);
+      });
+
+      it('evictOld() does not evict', async function () {
+        lru.evictOld();
+        assert.deepEqual([...lru], [[key, val]]);
+      });
+
+      it('evictOld() evicts after setting evictable = true', async function () {
+        evictable = true;
+        lru.evictOld();
+        assert.deepEqual([...lru], []);
+      });
+    });
+
+    describe('set immediately evicts if evictable', function () {
+      it('explicitly evictable', async function () {
+        const lru = new LRU(0, () => true);
+        lru.set('k', 'v');
+        assert(lru.get('k') == null);
+        assert.deepEqual([...lru], []);
+      });
+
+      it('is evictable by default', async function () {
+        const lru = new LRU(0);
+        lru.set('k', 'v');
+        assert(lru.get('k') == null);
+        assert.deepEqual([...lru], []);
+      });
+    });
+  });
+
+  describe('capacity = 2', function () {
+    let evictable, lru;
+
+    beforeEach(async function () {
+      evictable = () => false;
+      lru = new LRU(2, (k, v) => evictable(k, v));
+    });
+
+    it('iterates oldest first', async function () {
+      lru.set(0, '0');
+      lru.set(1, '1');
+      let i = 0;
+      for (const [k, v] of lru) {
+        assert.equal(k, i);
+        assert.equal(v, `${i}`);
+        i++;
+      }
+      assert.equal(i, 2);
+    });
+
+    it('get(k) updates recently used', async function () {
+      lru.set(0, '0');
+      lru.set(1, '1');
+      assert.equal(lru.get(0), '0');
+      assert.deepEqual([...lru], [[1, '1'], [0, '0']]);
+    });
+
+    it('get(k, false) does not update recently used', async function () {
+      lru.set(0, '0');
+      lru.set(1, '1');
+      assert.equal(lru.get(0, false), '0');
+      assert.deepEqual([...lru], [[0, '0'], [1, '1']]);
+    });
+
+    it('re-set() updates recently used', async function () {
+      lru.set(0, '0');
+      lru.set(1, '1');
+      lru.set(0, '00');
+      assert.deepEqual([...lru], [[1, '1'], [0, '00']]);
+    });
+
+    it('evictOld() only evicts evictable entries', async function () {
+      evictable = () => false;
+      lru.set(0, '0');
+      lru.set(1, '1');
+      lru.set(2, '2');
+      lru.set(3, '3');
+      assert.deepEqual([...lru], [[0, '0'], [1, '1'], [2, '2'], [3, '3']]);
+      evictable = (k) => k >= 2;
+      lru.evictOld();
+      // The newer entries should be evicted because the older are dirty/writingInProgress.
+      assert.deepEqual([...lru], [[0, '0'], [1, '1']]);
+    });
+
+    it('evictOld() does nothing if at or below capacity', async function () {
+      evictable = () => true;
+      lru.set(0, '0');
+      lru.evictOld();
+      assert.deepEqual([...lru], [[0, '0']]);
+      lru.set(1, '1');
+      lru.evictOld();
+      assert.deepEqual([...lru], [[0, '0'], [1, '1']]);
+    });
+  });
+});