ueberdb2 4.0.11 → 4.0.17

package/dist/test/lib/databases.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.databases = void 0;
+const os = require('os');
+exports.databases = {
+    memory: {},
+    dirty: {
+        filename: `${os.tmpdir()}/ueberdb-test.db`,
+        speeds: {
+            setMax: 1,
+            getMax: 0.1,
+            findKeysMax: 0.5,
+        },
+    },
+    sqlite: {
+        filename: `${os.tmpdir()}/ueberdb-test.sqlite`,
+        speeds: {
+            setMax: 0.6,
+            getMax: 0.5,
+            findKeysMax: 2.5,
+            removeMax: 0.5,
+        },
+    },
+    mysql: {
+        user: 'ueberdb',
+        host: '127.0.0.1',
+        password: 'ueberdb',
+        database: 'ueberdb',
+        charset: 'utf8mb4',
+        speeds: {
+            findKeysMax: 6,
+            getMax: 1,
+        },
+    },
+    postgres: {
+        user: 'ueberdb',
+        host: 'localhost',
+        password: 'ueberdb',
+        database: 'ueberdb',
+        charset: 'utf8mb4',
+        speeds: {
+            setMax: 6,
+        },
+    },
+    redis: {},
+    mongodb: {
+        url: 'mongodb://127.0.0.1:27017',
+        database: 'mydb_test',
+        speeds: {
+            setMax: 0.2,
+            getMax: 0.05,
+            removeMax: 0.3,
+        },
+    },
+    couch: {
+        host: '127.0.0.1',
+        port: 5984,
+        database: 'ueberdb',
+        user: 'ueberdb',
+        password: 'ueberdb',
+        speeds: {
+            findKeysMax: 30,
+        },
+    },
+    elasticsearch: {
+        base_index: 'ueberdb_test',
+        speeds: {
+            findKeysMax: 30,
+        }, host: '127.0.0.1',
+        port: '9200',
+    },
+};

package/dist/test/test.js

@@ -0,0 +1,373 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+// @ts-expect-error TS(7016): Could not find a declaration file for module 'wtfn... Remove this comment to see the full error message
+const wtfnode_1 = __importDefault(require("wtfnode"));
+// @ts-expect-error TS(7016): Could not find a declaration file for module 'cli-... Remove this comment to see the full error message
+const cli_table_1 = __importDefault(require("cli-table"));
+// @ts-ignore
+const randexp_1 = __importDefault(require("randexp"));
+const assert_1 = __importDefault(require("assert"));
+const databases_1 = require("./lib/databases");
+const fs_1 = require("fs");
+const logging_1 = __importDefault(require("../lib/logging"));
+const ueberdb = __importStar(require("../index"));
+'use strict';
+const assert = assert_1.default.strict;
+const databases = { databases: databases_1.databases }.databases;
+const fs = { promises: fs_1.promises }.promises;
+const maxKeyLength = 100;
+const randomString = (length = maxKeyLength) => new randexp_1.default(new RegExp(`.{${length}}`)).gen();
+// eslint-disable-next-line mocha/no-top-level-hooks
+after(async () => {
+    // Add a timeout to forcibly exit if something is keeping node from exiting cleanly.
+    // The timeout is unref()ed so that it doesn't prevent node from exiting when done.
+    setTimeout(() => {
+        console.error('node should have exited by now but something is keeping it open ' +
+            'such as an open connection or active timer');
+        wtfnode_1.default.dump();
+        process.exit(1); // eslint-disable-line n/no-process-exit
+    }, 5000).unref();
+});
+describe(__filename, () => {
+    let speedTable;
+    let db;
+    before(async () => {
+        speedTable = new cli_table_1.default({
+            head: [
+                'Database',
+                'read cache',
+                'write buffer',
+                '#',
+                'ms/set',
+                'ms/get',
+                'ms/findKeys',
+                'ms/remove',
+                'total ms',
+                'total ms/#',
+            ],
+            colWidths: [15, 15, 15, 8, 13, 13, 13, 13, 13, 13],
+        });
+    });
+    after(async () => {
+        console.log(speedTable.toString());
+    });
+    Object.keys(databases).filter((v) => v === "sqlite")
+        .forEach((database) => {
+        // @ts-ignore
+        const dbSettings = databases[database];
+        describe(database, () => {
+            for (const readCache of [false, true]) {
+                describe(`${readCache ? '' : 'no '}read cache`, () => {
+                    for (const writeBuffer of [false, true]) {
+                        describe(`${writeBuffer ? '' : 'no '}write buffer`, function () {
+                            this.timeout(5000);
+                            before(async () => {
+                                if (dbSettings.filename) {
+                                    await fs.unlink(dbSettings.filename).catch(() => { });
+                                }
+                                db = new ueberdb.Database(database, dbSettings, {
+                                    ...(readCache ? {} : { cache: 0 }),
+                                    ...(writeBuffer ? {} : { writeInterval: 0 }),
+                                }, new logging_1.default.ConsoleLogger());
+                                await db.init();
+                            });
+                            after(async () => {
+                                await db.close();
+                                if (dbSettings.filename) {
+                                    await fs.unlink(dbSettings.filename).catch(() => { });
+                                }
+                            });
+                            describe('white space in key is not ignored', () => {
+                                for (const space of [false, true]) {
+                                    describe(`key ${space ? 'has' : 'does not have'} a trailing space`, () => {
+                                        let input;
+                                        let key;
+                                        before(async () => {
+                                            input = { a: 1, b: new randexp_1.default(/.+/).gen() };
+                                            key = randomString(maxKeyLength - 1) + (space ? ' ' : '');
+                                            await db.set(key, input);
+                                        });
+                                        it('get(key) -> record', async () => {
+                                            const output = await db.get(key);
+                                            // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                            assert.equal(JSON.stringify(output), JSON.stringify(input));
+                                        });
+                                        it('get(`${key} `) -> nullish', async () => {
+                                            const output = await db.get(`${key} `);
+                                            // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                            assert(output == null);
+                                        });
+                                        if (space) {
+                                            it('get(key.slice(0, -1)) -> nullish', async () => {
+                                                const output = await db.get(key.slice(0, -1));
+                                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                                assert(output == null);
+                                            });
+                                        }
+                                    });
+                                }
+                            });
+                            it('get of unknown key -> nullish', async () => {
+                                const key = randomString();
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert((await db.get(key)) == null);
+                            });
+                            it('set+get works', async () => {
+                                const input = { a: 1, b: new randexp_1.default(/.+/).gen() };
+                                const key = randomString();
+                                await db.set(key, input);
+                                const output = await db.get(key);
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.equal(JSON.stringify(output), JSON.stringify(input));
+                            });
+                            it('set+get with random key/value works', async () => {
+                                const input = { testLongString: new randexp_1.default(/[a-f0-9]{50000}/).gen() };
+                                const key = randomString();
+                                await db.set(key, input);
+                                const output = await db.get(key);
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.equal(JSON.stringify(output), JSON.stringify(input));
+                            });
+                            it('findKeys works', async function () {
+                                if (database === 'mongodb') {
+                                    this.skip();
+                                } // TODO: Fix mongodb.
+                                // TODO setting a key with non ascii chars
+                                const key = new randexp_1.default(/([a-z]\w{0,20})foo\1/).gen();
+                                await Promise.all([
+                                    db.set(key, true),
+                                    db.set(`${key}a`, true),
+                                    db.set(`nonmatching_${key}`, false),
+                                ]);
+                                const keys = await db.findKeys(`${key}*`, null);
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(keys.sort(), [key, `${key}a`]);
+                            });
+                            it('findKeys with exclusion works', async function () {
+                                if (database === 'mongodb') {
+                                    this.skip();
+                                } // TODO: Fix mongodb.
+                                const key = new randexp_1.default(/([a-z]\w{0,20})foo\1/).gen();
+                                await Promise.all([
+                                    db.set(key, true),
+                                    db.set(`${key}a`, true),
+                                    db.set(`${key}b`, false),
+                                    db.set(`${key}b2`, false),
+                                    db.set(`nonmatching_${key}`, false),
+                                ]);
+                                const keys = await db.findKeys(`${key}*`, `${key}b*`);
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(keys.sort(), [key, `${key}a`].sort());
+                            });
+                            it('findKeys with no matches works', async () => {
+                                const key = new randexp_1.default(/([a-z]\w{0,20})foo\1/).gen();
+                                await db.set(key, true);
+                                const keys = await db.findKeys(`${key}_nomatch_*`, null);
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(keys, []);
+                            });
+                            it('findKeys with no wildcard works', async () => {
+                                const key = new randexp_1.default(/([a-z]\w{0,20})foo\1/).gen();
+                                await db.set(key, true);
+                                const keys = await db.findKeys(key, null);
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(keys, [key]);
+                            });
+                            it('remove works', async () => {
+                                const input = { a: 1, b: new randexp_1.default(/.+/).gen() };
+                                const key = randomString();
+                                await db.set(key, input);
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.equal(JSON.stringify(await db.get(key)), JSON.stringify(input));
+                                await db.remove(key);
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert((await db.get(key)) == null);
+                            });
+                            it('getSub of existing property works', async () => {
+                                await db.set('k', { sub1: { sub2: 'v' } });
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.equal(await db.getSub('k', ['sub1', 'sub2']), 'v');
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(await db.getSub('k', ['sub1']), { sub2: 'v' });
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(await db.getSub('k', []), { sub1: { sub2: 'v' } });
+                            });
+                            it('getSub of missing property returns nullish', async () => {
+                                await db.set('k', { sub1: {} });
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert((await db.getSub('k', ['sub1', 'sub2'])) == null);
+                                await db.set('k', {});
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert((await db.getSub('k', ['sub1', 'sub2'])) == null);
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert((await db.getSub('k', ['sub1'])) == null);
+                                await db.remove('k');
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert((await db.getSub('k', ['sub1', 'sub2'])) == null);
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert((await db.getSub('k', ['sub1'])) == null);
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert((await db.getSub('k', [])) == null);
+                            });
+                            it('setSub can modify an existing property', async () => {
+                                await db.set('k', { sub1: { sub2: 'v' } });
+                                await db.setSub('k', ['sub1', 'sub2'], 'v2');
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(await db.get('k'), { sub1: { sub2: 'v2' } });
+                                await db.setSub('k', ['sub1'], 'v2');
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(await db.get('k'), { sub1: 'v2' });
+                                await db.setSub('k', [], 'v3');
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.equal(await db.get('k'), 'v3');
+                            });
+                            it('setSub can add a new property', async () => {
+                                await db.remove('k');
+                                await db.setSub('k', [], {});
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(await db.get('k'), {});
+                                await db.setSub('k', ['sub1'], {});
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(await db.get('k'), { sub1: {} });
+                                await db.setSub('k', ['sub1', 'sub2'], 'v');
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(await db.get('k'), { sub1: { sub2: 'v' } });
+                                await db.remove('k');
+                                await db.setSub('k', ['sub1', 'sub2'], 'v');
+                                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                assert.deepEqual(await db.get('k'), { sub1: { sub2: 'v' } });
+                            });
+                            it('setSub rejects attempts to set properties on primitives', async () => {
+                                for (const v of ['hello world', 42, true]) {
+                                    await db.set('k', v);
+                                    assert.rejects(db.setSub('k', ['sub'], 'x'), {
+                                        name: 'TypeError',
+                                        message: /property "sub" on non-object/,
+                                    });
+                                    // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                    assert.deepEqual(await db.get('k'), v);
+                                }
+                            });
+                            it('speed is acceptable', async function () {
+                                this.timeout(180000);
+                                const { speeds: { count = 1000, setMax = 3, getMax = 0.1, findKeysMax = 3, removeMax = 1 } = {} } = dbSettings || {};
+                                const input = { a: 1, b: new randexp_1.default(/.+/).gen() };
+                                // TODO setting a key with non ascii chars
+                                const key = new randexp_1.default(/([a-z]\w{0,20})foo\1/).gen();
+                                // Pre-allocate an array before starting the timer so that time spent growing the
+                                // array doesn't throw off the benchmarks.
+                                const promises = [...Array(count + 1)].map(() => null);
+                                const timers = { start: Date.now() };
+                                for (let i = 0; i < count; ++i) {
+                                    promises[i] = db.set(key + i, input);
+                                }
+                                promises[count] = db.flush();
+                                await Promise.all(promises);
+                                // @ts-expect-error TS(2339): Property 'set' does not exist on type '{ start: nu... Remove this comment to see the full error message
+                                timers.set = Date.now();
+                                for (let i = 0; i < count; ++i) {
+                                    promises[i] = db.get(key + i);
+                                }
+                                await Promise.all(promises);
+                                // @ts-expect-error TS(2339): Property 'get' does not exist on type '{ start: nu... Remove this comment to see the full error message
+                                timers.get = Date.now();
+                                for (let i = 0; i < count; ++i) {
+                                    promises[i] = db.findKeys(key + i, null);
+                                }
+                                await Promise.all(promises);
+                                // @ts-expect-error TS(2339): Property 'findKeys' does not exist on type '{ star... Remove this comment to see the full error message
+                                timers.findKeys = Date.now();
+                                for (let i = 0; i < count; ++i) {
+                                    promises[i] = db.remove(key + i);
+                                }
+                                promises[count] = db.flush();
+                                await Promise.all(promises);
+                                // @ts-expect-error TS(2339): Property 'remove' does not exist on type '{ start:... Remove this comment to see the full error message
+                                timers.remove = Date.now();
+                                const timePerOp = {
+                                    // @ts-expect-error TS(2339): Property 'set' does not exist on type '{ start: nu... Remove this comment to see the full error message
+                                    set: (timers.set - timers.start) / count,
+                                    // @ts-expect-error TS(2339): Property 'get' does not exist on type '{ start: nu... Remove this comment to see the full error message
+                                    get: (timers.get - timers.set) / count,
+                                    // @ts-expect-error TS(2339): Property 'findKeys' does not exist on type '{ star... Remove this comment to see the full error message
+                                    findKeys: (timers.findKeys - timers.get) / count,
+                                    // @ts-expect-error TS(2339): Property 'remove' does not exist on type '{ start:... Remove this comment to see the full error message
+                                    remove: (timers.remove - timers.findKeys) / count,
+                                };
+                                speedTable.push([
+                                    database,
+                                    readCache ? 'yes' : 'no',
+                                    writeBuffer ? 'yes' : 'no',
+                                    count,
+                                    timePerOp.set,
+                                    timePerOp.get,
+                                    timePerOp.findKeys,
+                                    timePerOp.remove,
+                                    // @ts-expect-error TS(2339): Property 'remove' does not exist on type '{ start:... Remove this comment to see the full error message
+                                    timers.remove - timers.start,
+                                    // @ts-expect-error TS(2339): Property 'remove' does not exist on type '{ start:... Remove this comment to see the full error message
+                                    (timers.remove - timers.start) / count,
+                                ]);
+                                // Removes the "Acceptable ms/op" column if there is no enforced limit.
+                                const filterColumn = (row) => {
+                                    if (readCache && writeBuffer) {
+                                        return row;
+                                    }
+                                    row.splice(1, 1);
+                                    return row;
+                                };
+                                const acceptableTable = new cli_table_1.default({
+                                    head: filterColumn(['op', 'Acceptable ms/op', 'Actual ms/op']),
+                                    colWidths: filterColumn([10, 18, 18]),
+                                });
+                                acceptableTable.push(...[
+                                    ['set', setMax, timePerOp.set],
+                                    ['get', getMax, timePerOp.get],
+                                    ['findKeys', findKeysMax, timePerOp.findKeys],
+                                    ['remove', removeMax, timePerOp.remove],
+                                ].map(filterColumn));
+                                console.log(acceptableTable.toString());
+                                if (readCache && writeBuffer) {
+                                    // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                    assert(setMax >= timePerOp.set);
+                                    // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                    assert(getMax >= timePerOp.get);
+                                    // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                    assert(findKeysMax >= timePerOp.findKeys);
+                                    // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                                    assert(removeMax >= timePerOp.remove);
+                                }
+                            });
+                        });
+                    }
+                });
+            }
+        });
+    });
+});

package/dist/test/test_bulk.js

@@ -0,0 +1,74 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const assert_1 = __importDefault(require("assert"));
+const index_1 = require("../index");
+const util_1 = __importDefault(require("util"));
+'use strict';
+const assert = assert_1.default.strict;
+const range = (N) => [...Array(N).keys()];
+describe(__filename, () => {
+    let db = null;
+    let mock = null;
+    const createDb = async (wrapperSettings) => {
+        const settings = {};
+        db = new index_1.Database('mock', settings, wrapperSettings);
+        // @ts-expect-error TS(2339): Property 'mock' does not exist on type '{}'.
+        mock = settings.mock;
+        mock.once('init', (cb) => cb());
+        await db.init();
+    };
+    afterEach(async () => {
+        if (mock != null) {
+            mock.removeAllListeners();
+            mock.once('close', (cb) => cb());
+            mock = null;
+        }
+        if (db != null) {
+            await db.close();
+            db = null;
+        }
+    });
+    describe('bulkLimit', () => {
+        const bulkLimits = [0, false, null, undefined, '', 1, 2];
+        for (const bulkLimit of bulkLimits) {
+            it(bulkLimit === undefined ? 'undefined' : JSON.stringify(bulkLimit), async () => {
+                await createDb({ bulkLimit });
+                const gotWrites = [];
+                mock.on('set', util_1.default.callbackify(async (k, v) => gotWrites.push(1)));
+                mock.on('doBulk', util_1.default.callbackify(async (ops) => gotWrites.push(ops.length)));
+                const N = 10;
+                await Promise.all(range(N).map((i) => db.set(`key${i}`, `val${i}`)));
+                const wantLimit = bulkLimit || N;
+                // @ts-expect-error TS(2363): The right-hand side of an arithmetic operation mus... Remove this comment to see the full error message
+                const wantWrites = range(N / wantLimit).map((i) => wantLimit);
+                // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+                assert.deepEqual(gotWrites, wantWrites);
+            });
+        }
+    });
+    it('bulk failures are retried individually', async () => {
+        await createDb({});
+        const gotDoBulkCalls = [];
+        mock.on('doBulk', util_1.default.callbackify(async (ops) => {
+            gotDoBulkCalls.push(ops.length);
+            throw new Error('test');
+        }));
+        const gotWrites = new Map();
+        const wantWrites = new Map();
+        mock.on('set', util_1.default.callbackify(async (k, v) => gotWrites.set(k, v)));
+        const N = 10;
+        await Promise.all(range(N).map(async (i) => {
+            const k = `key${i}`;
+            const v = `val${i}`;
+            wantWrites.set(k, JSON.stringify(v));
+            await db.set(k, v);
+        }));
+        // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+        assert.deepEqual(gotDoBulkCalls, [N]);
+        // @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
+        assert.deepEqual(gotWrites, wantWrites);
+    });
+});