ueberdb2 4.0.11 → 4.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintignore +2 -0
- package/.eslintrc.cjs +44 -5
- package/.github/workflows/npmpublish.yml +3 -3
- package/databases/{cassandra_db.js → cassandra_db.ts} +45 -30
- package/databases/{couch_db.js → couch_db.ts} +78 -31
- package/databases/{dirty_db.js → dirty_db.ts} +19 -14
- package/databases/{dirty_git_db.js → dirty_git_db.ts} +19 -15
- package/databases/{elasticsearch_db.js → elasticsearch_db.ts} +30 -21
- package/databases/{memory_db.js → memory_db.ts} +8 -8
- package/databases/mock_db.ts +43 -0
- package/databases/{mongodb_db.js → mongodb_db.ts} +22 -16
- package/databases/{mssql_db.js → mssql_db.ts} +29 -21
- package/databases/{mysql_db.js → mysql_db.ts} +20 -15
- package/databases/{postgres_db.js → postgres_db.ts} +37 -22
- package/databases/{postgrespool_db.js → postgrespool_db.ts} +3 -3
- package/databases/redis_db.ts +129 -0
- package/databases/{rethink_db.js → rethink_db.ts} +35 -19
- package/databases/{sqlite_db.js → sqlite_db.ts} +37 -36
- package/dist/databases/cassandra_db.js +237 -0
- package/dist/databases/couch_db.js +181 -0
- package/dist/databases/dirty_db.js +78 -0
- package/dist/databases/dirty_git_db.js +77 -0
- package/dist/databases/elasticsearch_db.js +251 -0
- package/dist/databases/memory_db.js +39 -0
- package/dist/databases/mock_db.js +40 -0
- package/dist/databases/mongodb_db.js +127 -0
- package/dist/databases/mssql_db.js +187 -0
- package/dist/databases/mysql_db.js +170 -0
- package/dist/databases/postgres_db.js +192 -0
- package/dist/databases/postgrespool_db.js +12 -0
- package/dist/databases/redis_db.js +105 -0
- package/dist/databases/rethink_db.js +123 -0
- package/dist/databases/sqlite_db.js +140 -0
- package/dist/index.js +215 -0
- package/dist/lib/AbstractDatabase.js +38 -0
- package/dist/lib/CacheAndBufferLayer.js +657 -0
- package/dist/lib/logging.js +34 -0
- package/dist/test/lib/databases.js +72 -0
- package/dist/test/test.js +373 -0
- package/dist/test/test_bulk.js +74 -0
- package/dist/test/test_elasticsearch.js +157 -0
- package/dist/test/test_findKeys.js +69 -0
- package/dist/test/test_flush.js +83 -0
- package/dist/test/test_getSub.js +57 -0
- package/dist/test/test_lru.js +155 -0
- package/dist/test/test_memory.js +59 -0
- package/dist/test/test_metrics.js +772 -0
- package/dist/test/test_mysql.js +91 -0
- package/dist/test/test_postgres.js +40 -0
- package/dist/test/test_setSub.js +48 -0
- package/dist/test/test_tojson.js +62 -0
- package/docker-compose.yml +44 -0
- package/{index.js → index.ts} +76 -25
- package/lib/AbstractDatabase.ts +79 -0
- package/lib/{CacheAndBufferLayer.js → CacheAndBufferLayer.ts} +17 -16
- package/lib/{logging.js → logging.ts} +10 -6
- package/package.json +18 -3
- package/test/lib/{databases.js → databases.ts} +8 -5
- package/test/test.ts +328 -0
- package/test/test_bulk.ts +69 -0
- package/test/{test_elasticsearch.js → test_elasticsearch.ts} +48 -53
- package/test/{test_findKeys.js → test_findKeys.ts} +15 -17
- package/test/{test_flush.js → test_flush.ts} +16 -22
- package/test/test_getSub.ts +28 -0
- package/test/test_lru.ts +151 -0
- package/test/test_memory.ts +32 -0
- package/test/{test_metrics.js → test_metrics.ts} +73 -68
- package/test/{test_mysql.js → test_mysql.ts} +16 -22
- package/test/test_postgres.ts +16 -0
- package/test/{test_setSub.js → test_setSub.ts} +8 -12
- package/test/test_tojson.ts +34 -0
- package/databases/mock_db.js +0 -42
- package/databases/redis_db.js +0 -96
- package/lib/AbstractDatabase.js +0 -37
- package/test/test.js +0 -328
- package/test/test_bulk.js +0 -69
- package/test/test_getSub.js +0 -31
- package/test/test_lru.js +0 -145
- package/test/test_memory.js +0 -31
- package/test/test_postgres.js +0 -16
- package/test/test_tojson.js +0 -37
package/package.json
CHANGED
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
"cassandra-driver": "^4.6.4",
|
|
26
26
|
"dirty": "^1.1.3",
|
|
27
27
|
"elasticsearch7": "npm:@elastic/elasticsearch@^7.17.0",
|
|
28
|
+
"eslint-plugin-import": "^2.26.0",
|
|
28
29
|
"mongodb": "^3.7.3",
|
|
29
30
|
"mssql": "^9.1.1",
|
|
30
31
|
"mysql": "2.18.1",
|
|
@@ -39,11 +40,21 @@
|
|
|
39
40
|
"sqlite3": "^5.1.6"
|
|
40
41
|
},
|
|
41
42
|
"devDependencies": {
|
|
43
|
+
"@types/async": "^3.2.20",
|
|
44
|
+
"@types/mocha": "^10.0.1",
|
|
45
|
+
"@types/mongodb": "^4.0.7",
|
|
46
|
+
"@types/mssql": "^8.1.2",
|
|
47
|
+
"@types/mysql": "^2.15.21",
|
|
48
|
+
"@types/node": "^20.3.2",
|
|
49
|
+
"@types/pg": "^8.10.2",
|
|
50
|
+
"@types/rethinkdb": "^2.3.17",
|
|
42
51
|
"cli-table": "^0.3.11",
|
|
43
52
|
"eslint": "^8.43.0",
|
|
44
53
|
"eslint-config-etherpad": "^3.0.13",
|
|
45
54
|
"mocha": "^10.2.0",
|
|
46
55
|
"randexp": "^0.5.3",
|
|
56
|
+
"ts-migrate": "^0.1.35",
|
|
57
|
+
"ts-node": "^10.9.1",
|
|
47
58
|
"typescript": "^4.9.5",
|
|
48
59
|
"wtfnode": "^0.9.1"
|
|
49
60
|
},
|
|
@@ -51,8 +62,8 @@
|
|
|
51
62
|
"type": "git",
|
|
52
63
|
"url": "https://github.com/ether/ueberDB.git"
|
|
53
64
|
},
|
|
54
|
-
"main": "./index",
|
|
55
|
-
"version": "4.0.
|
|
65
|
+
"main": "./dist/index",
|
|
66
|
+
"version": "4.0.17",
|
|
56
67
|
"bugs": {
|
|
57
68
|
"url": "https://github.com/ether/ueberDB/issues"
|
|
58
69
|
},
|
|
@@ -60,7 +71,11 @@
|
|
|
60
71
|
"scripts": {
|
|
61
72
|
"lint": "eslint .",
|
|
62
73
|
"lint:fix": "eslint --fix .",
|
|
63
|
-
"test": "mocha test/test*.js"
|
|
74
|
+
"test": "tsc && mocha ./dist/test/test*.js",
|
|
75
|
+
"test:watch": "mocha ./dist/test/test*.js --watch",
|
|
76
|
+
"test-debug": "tsc && mocha --inspect-brk ./dist/test/test*.js",
|
|
77
|
+
"build": "tsc",
|
|
78
|
+
"publish": "npm run build && npm publish"
|
|
64
79
|
},
|
|
65
80
|
"_npmUser": {
|
|
66
81
|
"name": "johnyma22",
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
|
|
1
|
+
const os = require('os');
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
|
|
4
|
+
export const databases = {
|
|
4
5
|
memory: {},
|
|
5
6
|
dirty: {
|
|
6
|
-
filename:
|
|
7
|
+
filename: `${os.tmpdir()}/ueberdb-test.db`,
|
|
7
8
|
speeds: {
|
|
8
9
|
setMax: 1,
|
|
9
10
|
getMax: 0.1,
|
|
@@ -11,7 +12,7 @@ exports.databases = {
|
|
|
11
12
|
},
|
|
12
13
|
},
|
|
13
14
|
sqlite: {
|
|
14
|
-
filename:
|
|
15
|
+
filename: `${os.tmpdir()}/ueberdb-test.sqlite`,
|
|
15
16
|
speeds: {
|
|
16
17
|
setMax: 0.6,
|
|
17
18
|
getMax: 0.5,
|
|
@@ -65,6 +66,8 @@ exports.databases = {
|
|
|
65
66
|
base_index: 'ueberdb_test',
|
|
66
67
|
speeds: {
|
|
67
68
|
findKeysMax: 30,
|
|
68
|
-
},
|
|
69
|
+
}, host: '127.0.0.1',
|
|
70
|
+
port: '9200',
|
|
71
|
+
|
|
69
72
|
},
|
|
70
73
|
};
|
package/test/test.ts
ADDED
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
// @ts-expect-error TS(7016): Could not find a declaration file for module 'wtfn... Remove this comment to see the full error message
|
|
2
|
+
import wtfnode from 'wtfnode';
|
|
3
|
+
// @ts-expect-error TS(7016): Could not find a declaration file for module 'cli-... Remove this comment to see the full error message
|
|
4
|
+
import Clitable from 'cli-table';
|
|
5
|
+
// @ts-ignore
|
|
6
|
+
import Randexp from 'randexp';
|
|
7
|
+
import assert$0 from 'assert';
|
|
8
|
+
import {databases as databases$0} from './lib/databases';
|
|
9
|
+
import {promises} from 'fs';
|
|
10
|
+
import logging from '../lib/logging';
|
|
11
|
+
import * as ueberdb from '../index';
|
|
12
|
+
'use strict';
|
|
13
|
+
const assert = assert$0.strict;
|
|
14
|
+
const databases = {databases: databases$0}.databases;
|
|
15
|
+
const fs = {promises}.promises;
|
|
16
|
+
const maxKeyLength = 100;
|
|
17
|
+
const randomString = (length = maxKeyLength) => new Randexp(new RegExp(`.{${length}}`)).gen();
|
|
18
|
+
// eslint-disable-next-line mocha/no-top-level-hooks
|
|
19
|
+
after(async () => {
|
|
20
|
+
// Add a timeout to forcibly exit if something is keeping node from exiting cleanly.
|
|
21
|
+
// The timeout is unref()ed so that it doesn't prevent node from exiting when done.
|
|
22
|
+
setTimeout(() => {
|
|
23
|
+
console.error('node should have exited by now but something is keeping it open ' +
|
|
24
|
+
'such as an open connection or active timer');
|
|
25
|
+
wtfnode.dump();
|
|
26
|
+
process.exit(1); // eslint-disable-line n/no-process-exit
|
|
27
|
+
}, 5000).unref();
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
describe(__filename, () => {
|
|
31
|
+
let speedTable: any;
|
|
32
|
+
let db: any;
|
|
33
|
+
before(async () => {
|
|
34
|
+
speedTable = new Clitable({
|
|
35
|
+
head: [
|
|
36
|
+
'Database',
|
|
37
|
+
'read cache',
|
|
38
|
+
'write buffer',
|
|
39
|
+
'#',
|
|
40
|
+
'ms/set',
|
|
41
|
+
'ms/get',
|
|
42
|
+
'ms/findKeys',
|
|
43
|
+
'ms/remove',
|
|
44
|
+
'total ms',
|
|
45
|
+
'total ms/#',
|
|
46
|
+
],
|
|
47
|
+
colWidths: [15, 15, 15, 8, 13, 13, 13, 13, 13, 13],
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
after(async () => {
|
|
51
|
+
console.log(speedTable.toString());
|
|
52
|
+
});
|
|
53
|
+
Object.keys(databases).filter((v)=>v === "sqlite")
|
|
54
|
+
.forEach((database) => {
|
|
55
|
+
// @ts-ignore
|
|
56
|
+
const dbSettings = databases[database];
|
|
57
|
+
describe(database, () => {
|
|
58
|
+
for (const readCache of [false, true]) {
|
|
59
|
+
describe(`${readCache ? '' : 'no '}read cache`, () => {
|
|
60
|
+
for (const writeBuffer of [false, true]) {
|
|
61
|
+
describe(`${writeBuffer ? '' : 'no '}write buffer`, function (this: any) {
|
|
62
|
+
this.timeout(5000);
|
|
63
|
+
before(async () => {
|
|
64
|
+
if (dbSettings.filename) { await fs.unlink(dbSettings.filename).catch(() => { }); }
|
|
65
|
+
db = new ueberdb.Database(database, dbSettings, {
|
|
66
|
+
...(readCache ? {} : {cache: 0}),
|
|
67
|
+
...(writeBuffer ? {} : {writeInterval: 0}),
|
|
68
|
+
}, new logging.ConsoleLogger());
|
|
69
|
+
await db.init();
|
|
70
|
+
});
|
|
71
|
+
after(async () => {
|
|
72
|
+
await db.close();
|
|
73
|
+
if (dbSettings.filename) { await fs.unlink(dbSettings.filename).catch(() => { }); }
|
|
74
|
+
});
|
|
75
|
+
describe('white space in key is not ignored', () => {
|
|
76
|
+
for (const space of [false, true]) {
|
|
77
|
+
describe(`key ${space ? 'has' : 'does not have'} a trailing space`, () => {
|
|
78
|
+
let input: any;
|
|
79
|
+
let key: any;
|
|
80
|
+
before(async () => {
|
|
81
|
+
input = {a: 1, b: new Randexp(/.+/).gen()};
|
|
82
|
+
key = randomString(maxKeyLength - 1) + (space ? ' ' : '');
|
|
83
|
+
await db.set(key, input);
|
|
84
|
+
});
|
|
85
|
+
it('get(key) -> record', async () => {
|
|
86
|
+
const output = await db.get(key);
|
|
87
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
88
|
+
assert.equal(JSON.stringify(output), JSON.stringify(input));
|
|
89
|
+
});
|
|
90
|
+
it('get(`${key} `) -> nullish', async () => {
|
|
91
|
+
const output = await db.get(`${key} `);
|
|
92
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
93
|
+
assert(output == null);
|
|
94
|
+
});
|
|
95
|
+
if (space) {
|
|
96
|
+
it('get(key.slice(0, -1)) -> nullish', async () => {
|
|
97
|
+
const output = await db.get(key.slice(0, -1));
|
|
98
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
99
|
+
assert(output == null);
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
it('get of unknown key -> nullish', async () => {
|
|
106
|
+
const key = randomString();
|
|
107
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
108
|
+
assert((await db.get(key)) == null);
|
|
109
|
+
});
|
|
110
|
+
it('set+get works', async () => {
|
|
111
|
+
const input = {a: 1, b: new Randexp(/.+/).gen()};
|
|
112
|
+
const key = randomString();
|
|
113
|
+
await db.set(key, input);
|
|
114
|
+
const output = await db.get(key);
|
|
115
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
116
|
+
assert.equal(JSON.stringify(output), JSON.stringify(input));
|
|
117
|
+
});
|
|
118
|
+
it('set+get with random key/value works', async () => {
|
|
119
|
+
const input = {testLongString: new Randexp(/[a-f0-9]{50000}/).gen()};
|
|
120
|
+
const key = randomString();
|
|
121
|
+
await db.set(key, input);
|
|
122
|
+
const output = await db.get(key);
|
|
123
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
124
|
+
assert.equal(JSON.stringify(output), JSON.stringify(input));
|
|
125
|
+
});
|
|
126
|
+
it('findKeys works', async function (this: any) {
|
|
127
|
+
if (database === 'mongodb') { this.skip(); } // TODO: Fix mongodb.
|
|
128
|
+
// TODO setting a key with non ascii chars
|
|
129
|
+
const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
|
|
130
|
+
await Promise.all([
|
|
131
|
+
db.set(key, true),
|
|
132
|
+
db.set(`${key}a`, true),
|
|
133
|
+
db.set(`nonmatching_${key}`, false),
|
|
134
|
+
]);
|
|
135
|
+
const keys = await db.findKeys(`${key}*`, null);
|
|
136
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
137
|
+
assert.deepEqual(keys.sort(), [key, `${key}a`]);
|
|
138
|
+
});
|
|
139
|
+
it('findKeys with exclusion works', async function (this: any) {
|
|
140
|
+
if (database === 'mongodb') { this.skip(); } // TODO: Fix mongodb.
|
|
141
|
+
const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
|
|
142
|
+
await Promise.all([
|
|
143
|
+
db.set(key, true),
|
|
144
|
+
db.set(`${key}a`, true),
|
|
145
|
+
db.set(`${key}b`, false),
|
|
146
|
+
db.set(`${key}b2`, false),
|
|
147
|
+
db.set(`nonmatching_${key}`, false),
|
|
148
|
+
]);
|
|
149
|
+
const keys = await db.findKeys(`${key}*`, `${key}b*`);
|
|
150
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
151
|
+
assert.deepEqual(keys.sort(), [key, `${key}a`].sort());
|
|
152
|
+
});
|
|
153
|
+
it('findKeys with no matches works', async () => {
|
|
154
|
+
const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
|
|
155
|
+
await db.set(key, true);
|
|
156
|
+
const keys = await db.findKeys(`${key}_nomatch_*`, null);
|
|
157
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
158
|
+
assert.deepEqual(keys, []);
|
|
159
|
+
});
|
|
160
|
+
it('findKeys with no wildcard works', async () => {
|
|
161
|
+
const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
|
|
162
|
+
await db.set(key, true);
|
|
163
|
+
const keys = await db.findKeys(key, null);
|
|
164
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
165
|
+
assert.deepEqual(keys, [key]);
|
|
166
|
+
});
|
|
167
|
+
it('remove works', async () => {
|
|
168
|
+
const input = {a: 1, b: new Randexp(/.+/).gen()};
|
|
169
|
+
const key = randomString();
|
|
170
|
+
await db.set(key, input);
|
|
171
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
172
|
+
assert.equal(JSON.stringify(await db.get(key)), JSON.stringify(input));
|
|
173
|
+
await db.remove(key);
|
|
174
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
175
|
+
assert((await db.get(key)) == null);
|
|
176
|
+
});
|
|
177
|
+
it('getSub of existing property works', async () => {
|
|
178
|
+
await db.set('k', {sub1: {sub2: 'v'}});
|
|
179
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
180
|
+
assert.equal(await db.getSub('k', ['sub1', 'sub2']), 'v');
|
|
181
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
182
|
+
assert.deepEqual(await db.getSub('k', ['sub1']), {sub2: 'v'});
|
|
183
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
184
|
+
assert.deepEqual(await db.getSub('k', []), {sub1: {sub2: 'v'}});
|
|
185
|
+
});
|
|
186
|
+
it('getSub of missing property returns nullish', async () => {
|
|
187
|
+
await db.set('k', {sub1: {}});
|
|
188
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
189
|
+
assert((await db.getSub('k', ['sub1', 'sub2'])) == null);
|
|
190
|
+
await db.set('k', {});
|
|
191
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
192
|
+
assert((await db.getSub('k', ['sub1', 'sub2'])) == null);
|
|
193
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
194
|
+
assert((await db.getSub('k', ['sub1'])) == null);
|
|
195
|
+
await db.remove('k');
|
|
196
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
197
|
+
assert((await db.getSub('k', ['sub1', 'sub2'])) == null);
|
|
198
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
199
|
+
assert((await db.getSub('k', ['sub1'])) == null);
|
|
200
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
201
|
+
assert((await db.getSub('k', [])) == null);
|
|
202
|
+
});
|
|
203
|
+
it('setSub can modify an existing property', async () => {
|
|
204
|
+
await db.set('k', {sub1: {sub2: 'v'}});
|
|
205
|
+
await db.setSub('k', ['sub1', 'sub2'], 'v2');
|
|
206
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
207
|
+
assert.deepEqual(await db.get('k'), {sub1: {sub2: 'v2'}});
|
|
208
|
+
await db.setSub('k', ['sub1'], 'v2');
|
|
209
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
210
|
+
assert.deepEqual(await db.get('k'), {sub1: 'v2'});
|
|
211
|
+
await db.setSub('k', [], 'v3');
|
|
212
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
213
|
+
assert.equal(await db.get('k'), 'v3');
|
|
214
|
+
});
|
|
215
|
+
it('setSub can add a new property', async () => {
|
|
216
|
+
await db.remove('k');
|
|
217
|
+
await db.setSub('k', [], {});
|
|
218
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
219
|
+
assert.deepEqual(await db.get('k'), {});
|
|
220
|
+
await db.setSub('k', ['sub1'], {});
|
|
221
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
222
|
+
assert.deepEqual(await db.get('k'), {sub1: {}});
|
|
223
|
+
await db.setSub('k', ['sub1', 'sub2'], 'v');
|
|
224
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
225
|
+
assert.deepEqual(await db.get('k'), {sub1: {sub2: 'v'}});
|
|
226
|
+
await db.remove('k');
|
|
227
|
+
await db.setSub('k', ['sub1', 'sub2'], 'v');
|
|
228
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
229
|
+
assert.deepEqual(await db.get('k'), {sub1: {sub2: 'v'}});
|
|
230
|
+
});
|
|
231
|
+
it('setSub rejects attempts to set properties on primitives', async () => {
|
|
232
|
+
for (const v of ['hello world', 42, true]) {
|
|
233
|
+
await db.set('k', v);
|
|
234
|
+
assert.rejects(db.setSub('k', ['sub'], 'x'), {
|
|
235
|
+
name: 'TypeError',
|
|
236
|
+
message: /property "sub" on non-object/,
|
|
237
|
+
});
|
|
238
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
239
|
+
assert.deepEqual(await db.get('k'), v);
|
|
240
|
+
}
|
|
241
|
+
});
|
|
242
|
+
it('speed is acceptable', async function (this: any) {
|
|
243
|
+
this.timeout(180000);
|
|
244
|
+
const {speeds: {count = 1000, setMax = 3, getMax = 0.1, findKeysMax = 3, removeMax = 1} = {}} = dbSettings || {};
|
|
245
|
+
const input = {a: 1, b: new Randexp(/.+/).gen()};
|
|
246
|
+
// TODO setting a key with non ascii chars
|
|
247
|
+
const key = new Randexp(/([a-z]\w{0,20})foo\1/).gen();
|
|
248
|
+
// Pre-allocate an array before starting the timer so that time spent growing the
|
|
249
|
+
// array doesn't throw off the benchmarks.
|
|
250
|
+
const promises = [...Array(count + 1)].map(() => null);
|
|
251
|
+
const timers = {start: Date.now()};
|
|
252
|
+
for (let i = 0; i < count; ++i) { promises[i] = db.set(key + i, input); }
|
|
253
|
+
promises[count] = db.flush();
|
|
254
|
+
await Promise.all(promises);
|
|
255
|
+
// @ts-expect-error TS(2339): Property 'set' does not exist on type '{ start: nu... Remove this comment to see the full error message
|
|
256
|
+
timers.set = Date.now();
|
|
257
|
+
for (let i = 0; i < count; ++i) { promises[i] = db.get(key + i); }
|
|
258
|
+
await Promise.all(promises);
|
|
259
|
+
// @ts-expect-error TS(2339): Property 'get' does not exist on type '{ start: nu... Remove this comment to see the full error message
|
|
260
|
+
timers.get = Date.now();
|
|
261
|
+
for (let i = 0; i < count; ++i) { promises[i] = db.findKeys(key + i, null); }
|
|
262
|
+
await Promise.all(promises);
|
|
263
|
+
// @ts-expect-error TS(2339): Property 'findKeys' does not exist on type '{ star... Remove this comment to see the full error message
|
|
264
|
+
timers.findKeys = Date.now();
|
|
265
|
+
for (let i = 0; i < count; ++i) { promises[i] = db.remove(key + i); }
|
|
266
|
+
promises[count] = db.flush();
|
|
267
|
+
await Promise.all(promises);
|
|
268
|
+
// @ts-expect-error TS(2339): Property 'remove' does not exist on type '{ start:... Remove this comment to see the full error message
|
|
269
|
+
timers.remove = Date.now();
|
|
270
|
+
const timePerOp = {
|
|
271
|
+
// @ts-expect-error TS(2339): Property 'set' does not exist on type '{ start: nu... Remove this comment to see the full error message
|
|
272
|
+
set: (timers.set - timers.start) / count,
|
|
273
|
+
// @ts-expect-error TS(2339): Property 'get' does not exist on type '{ start: nu... Remove this comment to see the full error message
|
|
274
|
+
get: (timers.get - timers.set) / count,
|
|
275
|
+
// @ts-expect-error TS(2339): Property 'findKeys' does not exist on type '{ star... Remove this comment to see the full error message
|
|
276
|
+
findKeys: (timers.findKeys - timers.get) / count,
|
|
277
|
+
// @ts-expect-error TS(2339): Property 'remove' does not exist on type '{ start:... Remove this comment to see the full error message
|
|
278
|
+
remove: (timers.remove - timers.findKeys) / count,
|
|
279
|
+
};
|
|
280
|
+
speedTable.push([
|
|
281
|
+
database,
|
|
282
|
+
readCache ? 'yes' : 'no',
|
|
283
|
+
writeBuffer ? 'yes' : 'no',
|
|
284
|
+
count,
|
|
285
|
+
timePerOp.set,
|
|
286
|
+
timePerOp.get,
|
|
287
|
+
timePerOp.findKeys,
|
|
288
|
+
timePerOp.remove,
|
|
289
|
+
// @ts-expect-error TS(2339): Property 'remove' does not exist on type '{ start:... Remove this comment to see the full error message
|
|
290
|
+
timers.remove - timers.start,
|
|
291
|
+
// @ts-expect-error TS(2339): Property 'remove' does not exist on type '{ start:... Remove this comment to see the full error message
|
|
292
|
+
(timers.remove - timers.start) / count,
|
|
293
|
+
]);
|
|
294
|
+
// Removes the "Acceptable ms/op" column if there is no enforced limit.
|
|
295
|
+
const filterColumn = (row: any) => {
|
|
296
|
+
if (readCache && writeBuffer) { return row; }
|
|
297
|
+
row.splice(1, 1);
|
|
298
|
+
return row;
|
|
299
|
+
};
|
|
300
|
+
const acceptableTable = new Clitable({
|
|
301
|
+
head: filterColumn(['op', 'Acceptable ms/op', 'Actual ms/op']),
|
|
302
|
+
colWidths: filterColumn([10, 18, 18]),
|
|
303
|
+
});
|
|
304
|
+
acceptableTable.push(...[
|
|
305
|
+
['set', setMax, timePerOp.set],
|
|
306
|
+
['get', getMax, timePerOp.get],
|
|
307
|
+
['findKeys', findKeysMax, timePerOp.findKeys],
|
|
308
|
+
['remove', removeMax, timePerOp.remove],
|
|
309
|
+
].map(filterColumn));
|
|
310
|
+
console.log(acceptableTable.toString());
|
|
311
|
+
if (readCache && writeBuffer) {
|
|
312
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
313
|
+
assert(setMax >= timePerOp.set);
|
|
314
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
315
|
+
assert(getMax >= timePerOp.get);
|
|
316
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
317
|
+
assert(findKeysMax >= timePerOp.findKeys);
|
|
318
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
319
|
+
assert(removeMax >= timePerOp.remove);
|
|
320
|
+
}
|
|
321
|
+
});
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
});
|
|
327
|
+
});
|
|
328
|
+
});
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import assert$0 from 'assert';
|
|
2
|
+
import {Database} from '../index';
|
|
3
|
+
import util from 'util';
|
|
4
|
+
'use strict';
|
|
5
|
+
const assert = assert$0.strict;
|
|
6
|
+
const range = (N: any) => [...Array(N).keys()];
|
|
7
|
+
describe(__filename, () => {
|
|
8
|
+
let db: any = null;
|
|
9
|
+
let mock: any = null;
|
|
10
|
+
const createDb = async (wrapperSettings: any) => {
|
|
11
|
+
const settings = {};
|
|
12
|
+
db = new Database('mock', settings, wrapperSettings);
|
|
13
|
+
// @ts-expect-error TS(2339): Property 'mock' does not exist on type '{}'.
|
|
14
|
+
mock = settings.mock;
|
|
15
|
+
mock.once('init', (cb: any) => cb());
|
|
16
|
+
await db.init();
|
|
17
|
+
};
|
|
18
|
+
afterEach(async () => {
|
|
19
|
+
if (mock != null) {
|
|
20
|
+
mock.removeAllListeners();
|
|
21
|
+
mock.once('close', (cb: any) => cb());
|
|
22
|
+
mock = null;
|
|
23
|
+
}
|
|
24
|
+
if (db != null) {
|
|
25
|
+
await db.close();
|
|
26
|
+
db = null;
|
|
27
|
+
}
|
|
28
|
+
});
|
|
29
|
+
describe('bulkLimit', () => {
|
|
30
|
+
const bulkLimits = [0, false, null, undefined, '', 1, 2];
|
|
31
|
+
for (const bulkLimit of bulkLimits) {
|
|
32
|
+
it(bulkLimit === undefined ? 'undefined' : JSON.stringify(bulkLimit), async () => {
|
|
33
|
+
await createDb({bulkLimit});
|
|
34
|
+
const gotWrites: any = [];
|
|
35
|
+
mock.on('set', util.callbackify(async (k: any, v: any) => gotWrites.push(1)));
|
|
36
|
+
mock.on('doBulk', util.callbackify(async (ops: any) => gotWrites.push(ops.length)));
|
|
37
|
+
const N = 10;
|
|
38
|
+
await Promise.all(range(N).map((i) => db.set(`key${i}`, `val${i}`)));
|
|
39
|
+
const wantLimit = bulkLimit || N;
|
|
40
|
+
// @ts-expect-error TS(2363): The right-hand side of an arithmetic operation mus... Remove this comment to see the full error message
|
|
41
|
+
const wantWrites = range(N / wantLimit).map((i) => wantLimit);
|
|
42
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
43
|
+
assert.deepEqual(gotWrites, wantWrites);
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
it('bulk failures are retried individually', async () => {
|
|
48
|
+
await createDb({});
|
|
49
|
+
const gotDoBulkCalls: any = [];
|
|
50
|
+
mock.on('doBulk', util.callbackify(async (ops: any) => {
|
|
51
|
+
gotDoBulkCalls.push(ops.length);
|
|
52
|
+
throw new Error('test');
|
|
53
|
+
}));
|
|
54
|
+
const gotWrites = new Map();
|
|
55
|
+
const wantWrites = new Map();
|
|
56
|
+
mock.on('set', util.callbackify(async (k: any, v: any) => gotWrites.set(k, v)));
|
|
57
|
+
const N = 10;
|
|
58
|
+
await Promise.all(range(N).map(async (i) => {
|
|
59
|
+
const k = `key${i}`;
|
|
60
|
+
const v = `val${i}`;
|
|
61
|
+
wantWrites.set(k, JSON.stringify(v));
|
|
62
|
+
await db.set(k, v);
|
|
63
|
+
}));
|
|
64
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
65
|
+
assert.deepEqual(gotDoBulkCalls, [N]);
|
|
66
|
+
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
67
|
+
assert.deepEqual(gotWrites, wantWrites);
|
|
68
|
+
});
|
|
69
|
+
});
|
|
@@ -1,43 +1,40 @@
|
|
|
1
|
+
import {deepEqual, rejects} from 'assert';
|
|
2
|
+
import es from 'elasticsearch7';
|
|
3
|
+
import {databases} from './lib/databases';
|
|
4
|
+
import logging from '../lib/logging';
|
|
5
|
+
import * as ueberdb from '../index';
|
|
1
6
|
'use strict';
|
|
2
|
-
|
|
3
|
-
const assert = require('assert').strict;
|
|
4
|
-
const es = require('elasticsearch7');
|
|
5
|
-
const {databases: {elasticsearch: cfg}} = require('./lib/databases');
|
|
6
|
-
const logging = require('../lib/logging');
|
|
7
|
-
const ueberdb = require('../index');
|
|
8
|
-
|
|
7
|
+
const {databases: {elasticsearch: cfg}} = {databases};
|
|
9
8
|
const logger = new class extends logging.ConsoleLogger {
|
|
10
|
-
info() {}
|
|
9
|
+
info() { }
|
|
11
10
|
isInfoEnabled() { return false; }
|
|
12
11
|
}();
|
|
13
12
|
|
|
14
|
-
describe(__filename, function () {
|
|
13
|
+
describe(__filename, function (this: any) {
|
|
15
14
|
this.timeout(60000);
|
|
16
|
-
|
|
17
15
|
const {base_index = 'ueberdb_test'} = cfg;
|
|
18
|
-
let client;
|
|
19
|
-
let db;
|
|
20
|
-
|
|
21
|
-
beforeEach(async function () {
|
|
16
|
+
let client: any;
|
|
17
|
+
let db: any;
|
|
18
|
+
beforeEach(async () => {
|
|
22
19
|
client = new es.Client({
|
|
23
20
|
node: `http://${cfg.host || '127.0.0.1'}:${cfg.port || '9200'}`,
|
|
24
21
|
});
|
|
25
22
|
await client.indices.delete({index: `${base_index}*`}, {ignore: [404]});
|
|
26
23
|
});
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
if (db != null) await db.close();
|
|
24
|
+
afterEach(async () => {
|
|
25
|
+
if (db != null) { await db.close(); }
|
|
30
26
|
db = null;
|
|
31
27
|
await client.indices.delete({index: `${base_index}*`}, {ignore: [404]});
|
|
32
28
|
client.close();
|
|
33
29
|
client = null;
|
|
34
30
|
});
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
describe('no old data', function () {
|
|
31
|
+
describe('migration to schema v2', () => {
|
|
32
|
+
describe('no old data', () => {
|
|
38
33
|
for (const migrate of [false, true]) {
|
|
39
|
-
it(`migration ${migrate ? 'en' : 'dis'}abled`, async
|
|
40
|
-
|
|
34
|
+
it(`migration ${migrate ? 'en' : 'dis'}abled`, async () => {
|
|
35
|
+
// @ts-ignore
|
|
36
|
+
const settings = {base_index, migrate_to_newer_schema: undefined,
|
|
37
|
+
...cfg};
|
|
41
38
|
delete settings.migrate_to_newer_schema;
|
|
42
39
|
db = new ueberdb.Database('elasticsearch', settings, {}, logger);
|
|
43
40
|
await db.init();
|
|
@@ -45,14 +42,15 @@ describe(__filename, function () {
|
|
|
45
42
|
const {body: res} = await client.indices.get({index: `${base_index}*`});
|
|
46
43
|
for (const [k, v] of Object.entries(res)) {
|
|
47
44
|
indices.push(k);
|
|
45
|
+
// @ts-expect-error TS(2571): Object is of type 'unknown'.
|
|
48
46
|
indices.push(...Object.keys(v.aliases));
|
|
49
47
|
}
|
|
50
|
-
|
|
48
|
+
deepEqual(indices.sort(), [`${base_index}_s2`, `${base_index}_s2_i0`].sort());
|
|
51
49
|
});
|
|
52
50
|
}
|
|
53
51
|
});
|
|
54
|
-
|
|
55
|
-
|
|
52
|
+
describe('existing data', () => {
|
|
53
|
+
// @ts-expect-error TS(2769): No overload matches this call.
|
|
56
54
|
const data = new Map([
|
|
57
55
|
['foo:number', 42],
|
|
58
56
|
['foo:string', 'value'],
|
|
@@ -61,8 +59,7 @@ describe(__filename, function () {
|
|
|
61
59
|
['foo:p:s:string', 'value'],
|
|
62
60
|
['foo:p:s:object', {k: 'v'}],
|
|
63
61
|
]);
|
|
64
|
-
|
|
65
|
-
const setOld = async (k, v) => {
|
|
62
|
+
const setOld = async (k: any, v: any) => {
|
|
66
63
|
const kp = k.split(':');
|
|
67
64
|
const index = kp.length === 4 ? `${base_index}-${kp[0]}-${kp[2]}` : base_index;
|
|
68
65
|
await client.index({
|
|
@@ -78,53 +75,51 @@ describe(__filename, function () {
|
|
|
78
75
|
});
|
|
79
76
|
await client.indices.refresh({index});
|
|
80
77
|
};
|
|
81
|
-
|
|
82
|
-
beforeEach(async function () {
|
|
78
|
+
beforeEach(async () => {
|
|
83
79
|
await Promise.all([...data].map(async ([k, v]) => await setOld(k, v)));
|
|
84
80
|
});
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
const settings = {base_index,
|
|
81
|
+
it('migration disabled => init error', async () => {
|
|
82
|
+
// @ts-ignore
|
|
83
|
+
const settings = {base_index, migrate_to_newer_schema: undefined,
|
|
84
|
+
...cfg};
|
|
88
85
|
delete settings.migrate_to_newer_schema;
|
|
89
86
|
db = new ueberdb.Database('elasticsearch', settings, {}, logger);
|
|
90
|
-
await
|
|
87
|
+
await rejects(db.init(), /migrate_to_newer_schema/);
|
|
91
88
|
});
|
|
92
|
-
|
|
93
|
-
|
|
89
|
+
it('migration enabled', async () => {
|
|
90
|
+
// @ts-ignore
|
|
94
91
|
const settings = {base_index, ...cfg, migrate_to_newer_schema: true};
|
|
95
92
|
db = new ueberdb.Database('elasticsearch', settings, {}, logger);
|
|
96
93
|
await db.init();
|
|
97
94
|
await Promise.all([...data].map(async ([k, v]) => {
|
|
98
|
-
|
|
95
|
+
deepEqual(await db.get(k), v);
|
|
99
96
|
}));
|
|
100
97
|
});
|
|
101
|
-
|
|
102
|
-
it('each attempt uses a new index', async function () {
|
|
98
|
+
it('each attempt uses a new index', async () => {
|
|
103
99
|
await setOld('a-x:b:c-x:d', 'v'); // Force a conversion failure.
|
|
104
|
-
|
|
100
|
+
cfg.base_index = base_index;
|
|
101
|
+
const settings = {...cfg, migrate_to_newer_schema: true};
|
|
105
102
|
db = new ueberdb.Database('elasticsearch', settings, {}, logger);
|
|
106
|
-
const getIndices =
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
await
|
|
110
|
-
|
|
111
|
-
await
|
|
112
|
-
assert.deepEqual((await getIndices()).sort(), [
|
|
103
|
+
const getIndices = async () => Object.keys((await client.indices.get({index: `${base_index}_s2*`})).body);
|
|
104
|
+
deepEqual(await getIndices(), []);
|
|
105
|
+
await rejects(db.init(), /ambig/);
|
|
106
|
+
deepEqual(await getIndices(), [`${base_index}_s2_migrate_attempt_0`]);
|
|
107
|
+
await rejects(db.init(), /ambig/);
|
|
108
|
+
deepEqual((await getIndices()).sort(), [
|
|
113
109
|
`${base_index}_s2_migrate_attempt_0`,
|
|
114
110
|
`${base_index}_s2_migrate_attempt_1`,
|
|
115
111
|
]);
|
|
116
112
|
});
|
|
117
|
-
|
|
118
|
-
it('final name not created until success', async function () {
|
|
113
|
+
it('final name not created until success', async () => {
|
|
119
114
|
});
|
|
120
|
-
|
|
121
|
-
describe('ambiguous key', function () {
|
|
115
|
+
describe('ambiguous key', () => {
|
|
122
116
|
for (const k of ['a:b:c-x:d', 'a-x:b:c:d', 'a-x:b:c-x:d']) {
|
|
123
|
-
it(k, async
|
|
117
|
+
it(k, async () => {
|
|
124
118
|
await setOld(k, 'v');
|
|
125
|
-
|
|
119
|
+
cfg.base_index = base_index;
|
|
120
|
+
const settings = {...cfg, migrate_to_newer_schema: true};
|
|
126
121
|
db = new ueberdb.Database('elasticsearch', settings, {}, logger);
|
|
127
|
-
await
|
|
122
|
+
await rejects(db.init(), /ambig/);
|
|
128
123
|
});
|
|
129
124
|
}
|
|
130
125
|
});
|