ueberdb2 4.1.8 → 4.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/databases/cassandra_db.js +233 -235
- package/dist/databases/couch_db.js +171 -173
- package/dist/databases/dirty_db.js +73 -76
- package/dist/databases/dirty_git_db.js +57 -75
- package/dist/databases/elasticsearch_db.js +241 -267
- package/dist/databases/memory_db.js +35 -37
- package/dist/databases/mock_db.js +36 -38
- package/dist/databases/mongodb_db.js +131 -133
- package/dist/databases/mssql_db.js +183 -185
- package/dist/databases/mysql_db.js +166 -168
- package/dist/databases/postgres_db.js +188 -190
- package/dist/databases/postgrespool_db.js +10 -10
- package/dist/databases/redis_db.js +118 -120
- package/dist/databases/rethink_db.js +119 -121
- package/dist/databases/sqlite_db.js +135 -137
- package/dist/index.js +195 -213
- package/lib/AbstractDatabase.ts +79 -0
- package/lib/CacheAndBufferLayer.ts +665 -0
- package/lib/logging.ts +32 -0
- package/package.json +16 -12
- package/dist/lib/AbstractDatabase.js +0 -38
- package/dist/lib/CacheAndBufferLayer.js +0 -657
- package/dist/lib/logging.js +0 -34
- package/dist/test/lib/databases.js +0 -74
- package/dist/test/test.js +0 -327
- package/dist/test/test_bulk.js +0 -74
- package/dist/test/test_elasticsearch.js +0 -157
- package/dist/test/test_findKeys.js +0 -69
- package/dist/test/test_flush.js +0 -83
- package/dist/test/test_getSub.js +0 -57
- package/dist/test/test_lru.js +0 -155
- package/dist/test/test_memory.js +0 -59
- package/dist/test/test_metrics.js +0 -772
- package/dist/test/test_mysql.js +0 -90
- package/dist/test/test_postgres.js +0 -40
- package/dist/test/test_setSub.js +0 -48
- package/dist/test/test_tojson.js +0 -62
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.databases = void 0;
|
|
4
|
-
const os = require('os');
|
|
5
|
-
exports.databases = {
|
|
6
|
-
memory: {},
|
|
7
|
-
dirty: {
|
|
8
|
-
filename: `${os.tmpdir()}/ueberdb-test.db`,
|
|
9
|
-
speeds: {
|
|
10
|
-
setMax: 1,
|
|
11
|
-
getMax: 0.1,
|
|
12
|
-
findKeysMax: 0.5,
|
|
13
|
-
},
|
|
14
|
-
},
|
|
15
|
-
sqlite: {
|
|
16
|
-
filename: `${os.tmpdir()}/ueberdb-test.sqlite`,
|
|
17
|
-
speeds: {
|
|
18
|
-
setMax: 0.6,
|
|
19
|
-
getMax: 0.5,
|
|
20
|
-
findKeysMax: 2.5,
|
|
21
|
-
removeMax: 0.5,
|
|
22
|
-
},
|
|
23
|
-
},
|
|
24
|
-
mysql: {
|
|
25
|
-
user: 'ueberdb',
|
|
26
|
-
host: '127.0.0.1',
|
|
27
|
-
password: 'ueberdb',
|
|
28
|
-
database: 'ueberdb',
|
|
29
|
-
charset: 'utf8mb4',
|
|
30
|
-
speeds: {
|
|
31
|
-
findKeysMax: 6,
|
|
32
|
-
getMax: 1,
|
|
33
|
-
},
|
|
34
|
-
},
|
|
35
|
-
postgres: {
|
|
36
|
-
user: 'ueberdb',
|
|
37
|
-
host: 'localhost',
|
|
38
|
-
password: 'ueberdb',
|
|
39
|
-
database: 'ueberdb',
|
|
40
|
-
charset: 'utf8mb4',
|
|
41
|
-
speeds: {
|
|
42
|
-
setMax: 6,
|
|
43
|
-
},
|
|
44
|
-
},
|
|
45
|
-
redis: {
|
|
46
|
-
url: 'redis://localhost/'
|
|
47
|
-
},
|
|
48
|
-
mongodb: {
|
|
49
|
-
url: 'mongodb://127.0.0.1:27017',
|
|
50
|
-
database: 'mydb_test',
|
|
51
|
-
speeds: {
|
|
52
|
-
setMax: 0.2,
|
|
53
|
-
getMax: 0.05,
|
|
54
|
-
removeMax: 0.3,
|
|
55
|
-
},
|
|
56
|
-
},
|
|
57
|
-
couch: {
|
|
58
|
-
host: '127.0.0.1',
|
|
59
|
-
port: 5984,
|
|
60
|
-
database: 'ueberdb',
|
|
61
|
-
user: 'ueberdb',
|
|
62
|
-
password: 'ueberdb',
|
|
63
|
-
speeds: {
|
|
64
|
-
findKeysMax: 30,
|
|
65
|
-
},
|
|
66
|
-
},
|
|
67
|
-
elasticsearch: {
|
|
68
|
-
base_index: 'ueberdb_test',
|
|
69
|
-
speeds: {
|
|
70
|
-
findKeysMax: 30,
|
|
71
|
-
}, host: '127.0.0.1',
|
|
72
|
-
port: '9200',
|
|
73
|
-
},
|
|
74
|
-
};
|
package/dist/test/test.js
DELETED
|
@@ -1,327 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
-
};
|
|
28
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
// @ts-expect-error TS(7016): Could not find a declaration file for module 'wtfn... Remove this comment to see the full error message
|
|
30
|
-
const wtfnode_1 = __importDefault(require("wtfnode"));
|
|
31
|
-
// @ts-expect-error TS(7016): Could not find a declaration file for module 'cli-... Remove this comment to see the full error message
|
|
32
|
-
const cli_table_1 = __importDefault(require("cli-table"));
|
|
33
|
-
// @ts-ignore
|
|
34
|
-
const randexp_1 = __importDefault(require("randexp"));
|
|
35
|
-
const databases_1 = require("./lib/databases");
|
|
36
|
-
const fs_1 = require("fs");
|
|
37
|
-
const logging_1 = __importDefault(require("../lib/logging"));
|
|
38
|
-
const ueberdb = __importStar(require("../index"));
|
|
39
|
-
'use strict';
|
|
40
|
-
const assert_1 = require("assert");
|
|
41
|
-
const fs = { promises: fs_1.promises }.promises;
|
|
42
|
-
const maxKeyLength = 100;
|
|
43
|
-
const randomString = (length = maxKeyLength) => new randexp_1.default(new RegExp(`.{${length}}`)).gen();
|
|
44
|
-
// eslint-disable-next-line mocha/no-top-level-hooks
|
|
45
|
-
after(async () => {
|
|
46
|
-
// Add a timeout to forcibly exit if something is keeping node from exiting cleanly.
|
|
47
|
-
// The timeout is unref()ed so that it doesn't prevent node from exiting when done.
|
|
48
|
-
setTimeout(() => {
|
|
49
|
-
console.error('node should have exited by now but something is keeping it open ' +
|
|
50
|
-
'such as an open connection or active timer');
|
|
51
|
-
wtfnode_1.default.dump();
|
|
52
|
-
process.exit(1); // eslint-disable-line n/no-process-exit
|
|
53
|
-
}, 5000).unref();
|
|
54
|
-
});
|
|
55
|
-
describe(__filename, () => {
|
|
56
|
-
let speedTable;
|
|
57
|
-
let db;
|
|
58
|
-
before(async () => {
|
|
59
|
-
speedTable = new cli_table_1.default({
|
|
60
|
-
head: [
|
|
61
|
-
'Database',
|
|
62
|
-
'read cache',
|
|
63
|
-
'write buffer',
|
|
64
|
-
'#',
|
|
65
|
-
'ms/set',
|
|
66
|
-
'ms/get',
|
|
67
|
-
'ms/findKeys',
|
|
68
|
-
'ms/remove',
|
|
69
|
-
'total ms',
|
|
70
|
-
'total ms/#',
|
|
71
|
-
],
|
|
72
|
-
colWidths: [15, 15, 15, 8, 13, 13, 13, 13, 13, 13],
|
|
73
|
-
});
|
|
74
|
-
});
|
|
75
|
-
after(async () => {
|
|
76
|
-
console.log(speedTable.toString());
|
|
77
|
-
});
|
|
78
|
-
Object.keys(databases_1.databases)
|
|
79
|
-
.forEach((database) => {
|
|
80
|
-
const dbSettings = databases_1.databases[database];
|
|
81
|
-
describe(database, () => {
|
|
82
|
-
for (const readCache of [false, true]) {
|
|
83
|
-
describe(`${readCache ? '' : 'no '}read cache`, () => {
|
|
84
|
-
for (const writeBuffer of [false, true]) {
|
|
85
|
-
describe(`${writeBuffer ? '' : 'no '}write buffer`, function () {
|
|
86
|
-
this.timeout(5000);
|
|
87
|
-
before(async () => {
|
|
88
|
-
if (dbSettings.filename) {
|
|
89
|
-
await fs.unlink(dbSettings.filename).catch(() => { });
|
|
90
|
-
}
|
|
91
|
-
db = new ueberdb.Database(database, dbSettings, {
|
|
92
|
-
...(readCache ? {} : { cache: 0 }),
|
|
93
|
-
...(writeBuffer ? {} : { writeInterval: 0 }),
|
|
94
|
-
}, new logging_1.default.ConsoleLogger());
|
|
95
|
-
await db.init();
|
|
96
|
-
});
|
|
97
|
-
after(async () => {
|
|
98
|
-
await db.close();
|
|
99
|
-
if (dbSettings.filename) {
|
|
100
|
-
await fs.unlink(dbSettings.filename).catch(() => { });
|
|
101
|
-
}
|
|
102
|
-
});
|
|
103
|
-
describe('white space in key is not ignored', () => {
|
|
104
|
-
for (const space of [false, true]) {
|
|
105
|
-
describe(`key ${space ? 'has' : 'does not have'} a trailing space`, () => {
|
|
106
|
-
let input;
|
|
107
|
-
let key;
|
|
108
|
-
before(async () => {
|
|
109
|
-
input = { a: 1, b: new randexp_1.default(/.+/).gen() };
|
|
110
|
-
key = randomString(maxKeyLength - 1) + (space ? ' ' : '');
|
|
111
|
-
await db.set(key, input);
|
|
112
|
-
});
|
|
113
|
-
it('get(key) -> record', async () => {
|
|
114
|
-
const output = await db.get(key);
|
|
115
|
-
(0, assert_1.equal)(JSON.stringify(output), JSON.stringify(input));
|
|
116
|
-
});
|
|
117
|
-
it('get(`${key} `) -> nullish', async () => {
|
|
118
|
-
const output = await db.get(`${key} `);
|
|
119
|
-
(0, assert_1.equal)(output == null, true);
|
|
120
|
-
});
|
|
121
|
-
if (space) {
|
|
122
|
-
it('get(key.slice(0, -1)) -> nullish', async () => {
|
|
123
|
-
const output = await db.get(key.slice(0, -1));
|
|
124
|
-
(0, assert_1.equal)(output == null, true);
|
|
125
|
-
});
|
|
126
|
-
}
|
|
127
|
-
});
|
|
128
|
-
}
|
|
129
|
-
});
|
|
130
|
-
it('get of unknown key -> nullish', async () => {
|
|
131
|
-
const key = randomString();
|
|
132
|
-
(0, assert_1.equal)((await db.get(key)) == null, true);
|
|
133
|
-
});
|
|
134
|
-
it('set+get works', async () => {
|
|
135
|
-
const input = { a: 1, b: new randexp_1.default(/.+/).gen() };
|
|
136
|
-
const key = randomString();
|
|
137
|
-
await db.set(key, input);
|
|
138
|
-
const output = await db.get(key);
|
|
139
|
-
(0, assert_1.equal)(JSON.stringify(output), JSON.stringify(input));
|
|
140
|
-
});
|
|
141
|
-
it('set+get with random key/value works', async () => {
|
|
142
|
-
const input = { testLongString: new randexp_1.default(/[a-f0-9]{50000}/).gen() };
|
|
143
|
-
const key = randomString();
|
|
144
|
-
await db.set(key, input);
|
|
145
|
-
const output = await db.get(key);
|
|
146
|
-
(0, assert_1.equal)(JSON.stringify(output), JSON.stringify(input));
|
|
147
|
-
});
|
|
148
|
-
it('findKeys works', async function () {
|
|
149
|
-
if (database === 'mongodb') {
|
|
150
|
-
this.skip();
|
|
151
|
-
} // TODO: Fix mongodb.
|
|
152
|
-
// TODO setting a key with non ascii chars
|
|
153
|
-
const key = new randexp_1.default(/([a-z]\w{0,20})foo\1/).gen();
|
|
154
|
-
await Promise.all([
|
|
155
|
-
db.set(key, true),
|
|
156
|
-
db.set(`${key}a`, true),
|
|
157
|
-
db.set(`nonmatching_${key}`, false),
|
|
158
|
-
]);
|
|
159
|
-
const keys = await db.findKeys(`${key}*`, null);
|
|
160
|
-
(0, assert_1.deepEqual)(keys.sort(), [key, `${key}a`]);
|
|
161
|
-
});
|
|
162
|
-
it('findKeys with exclusion works', async function () {
|
|
163
|
-
if (database === 'mongodb') {
|
|
164
|
-
this.skip();
|
|
165
|
-
} // TODO: Fix mongodb.
|
|
166
|
-
const key = new randexp_1.default(/([a-z]\w{0,20})foo\1/).gen();
|
|
167
|
-
await Promise.all([
|
|
168
|
-
db.set(key, true),
|
|
169
|
-
db.set(`${key}a`, true),
|
|
170
|
-
db.set(`${key}b`, false),
|
|
171
|
-
db.set(`${key}b2`, false),
|
|
172
|
-
db.set(`nonmatching_${key}`, false),
|
|
173
|
-
]);
|
|
174
|
-
const keys = await db.findKeys(`${key}*`, `${key}b*`);
|
|
175
|
-
(0, assert_1.deepEqual)(keys.sort(), [key, `${key}a`].sort());
|
|
176
|
-
});
|
|
177
|
-
it('findKeys with no matches works', async () => {
|
|
178
|
-
const key = new randexp_1.default(/([a-z]\w{0,20})foo\1/).gen();
|
|
179
|
-
await db.set(key, true);
|
|
180
|
-
const keys = await db.findKeys(`${key}_nomatch_*`, null);
|
|
181
|
-
(0, assert_1.deepEqual)(keys, []);
|
|
182
|
-
});
|
|
183
|
-
it('findKeys with no wildcard works', async () => {
|
|
184
|
-
const key = new randexp_1.default(/([a-z]\w{0,20})foo\1/).gen();
|
|
185
|
-
await db.set(key, true);
|
|
186
|
-
const keys = await db.findKeys(key, null);
|
|
187
|
-
(0, assert_1.deepEqual)(keys, [key]);
|
|
188
|
-
});
|
|
189
|
-
it('remove works', async () => {
|
|
190
|
-
const input = { a: 1, b: new randexp_1.default(/.+/).gen() };
|
|
191
|
-
const key = randomString();
|
|
192
|
-
await db.set(key, input);
|
|
193
|
-
(0, assert_1.equal)(JSON.stringify(await db.get(key)), JSON.stringify(input));
|
|
194
|
-
await db.remove(key);
|
|
195
|
-
(0, assert_1.equal)((await db.get(key)) == null, true);
|
|
196
|
-
});
|
|
197
|
-
it('getSub of existing property works', async () => {
|
|
198
|
-
await db.set('k', { sub1: { sub2: 'v' } });
|
|
199
|
-
(0, assert_1.equal)(await db.getSub('k', ['sub1', 'sub2']), 'v');
|
|
200
|
-
(0, assert_1.deepEqual)(await db.getSub('k', ['sub1']), { sub2: 'v' });
|
|
201
|
-
(0, assert_1.deepEqual)(await db.getSub('k', []), { sub1: { sub2: 'v' } });
|
|
202
|
-
});
|
|
203
|
-
it('getSub of missing property returns nullish', async () => {
|
|
204
|
-
await db.set('k', { sub1: {} });
|
|
205
|
-
(0, assert_1.equal)((await db.getSub('k', ['sub1', 'sub2'])) == null, true);
|
|
206
|
-
await db.set('k', {});
|
|
207
|
-
(0, assert_1.equal)((await db.getSub('k', ['sub1', 'sub2'])) == null, true);
|
|
208
|
-
(0, assert_1.equal)((await db.getSub('k', ['sub1'])) == null, true);
|
|
209
|
-
await db.remove('k');
|
|
210
|
-
(0, assert_1.equal)((await db.getSub('k', ['sub1', 'sub2'])) == null, true);
|
|
211
|
-
(0, assert_1.equal)((await db.getSub('k', ['sub1'])) == null, true);
|
|
212
|
-
(0, assert_1.equal)((await db.getSub('k', [])) == null, true);
|
|
213
|
-
});
|
|
214
|
-
it('setSub can modify an existing property', async () => {
|
|
215
|
-
await db.set('k', { sub1: { sub2: 'v' } });
|
|
216
|
-
await db.setSub('k', ['sub1', 'sub2'], 'v2');
|
|
217
|
-
(0, assert_1.deepEqual)(await db.get('k'), { sub1: { sub2: 'v2' } });
|
|
218
|
-
await db.setSub('k', ['sub1'], 'v2');
|
|
219
|
-
(0, assert_1.deepEqual)(await db.get('k'), { sub1: 'v2' });
|
|
220
|
-
await db.setSub('k', [], 'v3');
|
|
221
|
-
(0, assert_1.equal)(await db.get('k'), 'v3');
|
|
222
|
-
});
|
|
223
|
-
it('setSub can add a new property', async () => {
|
|
224
|
-
await db.remove('k');
|
|
225
|
-
await db.setSub('k', [], {});
|
|
226
|
-
(0, assert_1.deepEqual)(await db.get('k'), {});
|
|
227
|
-
await db.setSub('k', ['sub1'], {});
|
|
228
|
-
(0, assert_1.deepEqual)(await db.get('k'), { sub1: {} });
|
|
229
|
-
await db.setSub('k', ['sub1', 'sub2'], 'v');
|
|
230
|
-
(0, assert_1.deepEqual)(await db.get('k'), { sub1: { sub2: 'v' } });
|
|
231
|
-
await db.remove('k');
|
|
232
|
-
await db.setSub('k', ['sub1', 'sub2'], 'v');
|
|
233
|
-
(0, assert_1.deepEqual)(await db.get('k'), { sub1: { sub2: 'v' } });
|
|
234
|
-
});
|
|
235
|
-
it('setSub rejects attempts to set properties on primitives', async () => {
|
|
236
|
-
for (const v of ['hello world', 42, true]) {
|
|
237
|
-
await db.set('k', v);
|
|
238
|
-
await (0, assert_1.rejects)(db.setSub('k', ['sub'], 'x'), {
|
|
239
|
-
name: 'TypeError',
|
|
240
|
-
message: /property "sub" on non-object/,
|
|
241
|
-
});
|
|
242
|
-
(0, assert_1.deepEqual)(await db.get('k'), v);
|
|
243
|
-
}
|
|
244
|
-
});
|
|
245
|
-
it('speed is acceptable', async function () {
|
|
246
|
-
this.timeout(180000);
|
|
247
|
-
const { speeds: { count = 1000, setMax = 3, getMax = 0.1, findKeysMax = 3, removeMax = 1 } = {} } = dbSettings || {};
|
|
248
|
-
const input = { a: 1, b: new randexp_1.default(/.+/).gen() };
|
|
249
|
-
// TODO setting a key with non ascii chars
|
|
250
|
-
const key = new randexp_1.default(/([a-z]\w{0,20})foo\1/).gen();
|
|
251
|
-
// Pre-allocate an array before starting the timer so that time spent growing the
|
|
252
|
-
// array doesn't throw off the benchmarks.
|
|
253
|
-
const promises = [...Array(count + 1)].map(() => null);
|
|
254
|
-
const timers = { start: Date.now() };
|
|
255
|
-
for (let i = 0; i < count; ++i) {
|
|
256
|
-
promises[i] = db.set(key + i, input);
|
|
257
|
-
}
|
|
258
|
-
promises[count] = db.flush();
|
|
259
|
-
await Promise.all(promises);
|
|
260
|
-
timers.set = Date.now();
|
|
261
|
-
for (let i = 0; i < count; ++i) {
|
|
262
|
-
promises[i] = db.get(key + i);
|
|
263
|
-
}
|
|
264
|
-
await Promise.all(promises);
|
|
265
|
-
timers.get = Date.now();
|
|
266
|
-
for (let i = 0; i < count; ++i) {
|
|
267
|
-
promises[i] = db.findKeys(key + i, null);
|
|
268
|
-
}
|
|
269
|
-
await Promise.all(promises);
|
|
270
|
-
timers.findKeys = Date.now();
|
|
271
|
-
for (let i = 0; i < count; ++i) {
|
|
272
|
-
promises[i] = db.remove(key + i);
|
|
273
|
-
}
|
|
274
|
-
promises[count] = db.flush();
|
|
275
|
-
await Promise.all(promises);
|
|
276
|
-
timers.remove = Date.now();
|
|
277
|
-
const timePerOp = {
|
|
278
|
-
set: (timers.set - timers.start) / count,
|
|
279
|
-
get: (timers.get - timers.set) / count,
|
|
280
|
-
findKeys: (timers.findKeys - timers.get) / count,
|
|
281
|
-
remove: (timers.remove - timers.findKeys) / count,
|
|
282
|
-
};
|
|
283
|
-
speedTable.push([
|
|
284
|
-
database,
|
|
285
|
-
readCache ? 'yes' : 'no',
|
|
286
|
-
writeBuffer ? 'yes' : 'no',
|
|
287
|
-
count,
|
|
288
|
-
timePerOp.set,
|
|
289
|
-
timePerOp.get,
|
|
290
|
-
timePerOp.findKeys,
|
|
291
|
-
timePerOp.remove,
|
|
292
|
-
timers.remove - timers.start,
|
|
293
|
-
(timers.remove - timers.start) / count,
|
|
294
|
-
]);
|
|
295
|
-
// Removes the "Acceptable ms/op" column if there is no enforced limit.
|
|
296
|
-
const filterColumn = (row) => {
|
|
297
|
-
if (readCache && writeBuffer) {
|
|
298
|
-
return row;
|
|
299
|
-
}
|
|
300
|
-
row.splice(1, 1);
|
|
301
|
-
return row;
|
|
302
|
-
};
|
|
303
|
-
const acceptableTable = new cli_table_1.default({
|
|
304
|
-
head: filterColumn(['op', 'Acceptable ms/op', 'Actual ms/op']),
|
|
305
|
-
colWidths: filterColumn([10, 18, 18]),
|
|
306
|
-
});
|
|
307
|
-
acceptableTable.push(...[
|
|
308
|
-
['set', setMax, timePerOp.set],
|
|
309
|
-
['get', getMax, timePerOp.get],
|
|
310
|
-
['findKeys', findKeysMax, timePerOp.findKeys],
|
|
311
|
-
['remove', removeMax, timePerOp.remove],
|
|
312
|
-
].map(filterColumn));
|
|
313
|
-
console.log(acceptableTable.toString());
|
|
314
|
-
if (readCache && writeBuffer) {
|
|
315
|
-
(0, assert_1.equal)(setMax >= timePerOp.set, true);
|
|
316
|
-
(0, assert_1.equal)(getMax >= timePerOp.get, true);
|
|
317
|
-
(0, assert_1.equal)(findKeysMax >= timePerOp.findKeys, true);
|
|
318
|
-
(0, assert_1.equal)(removeMax >= timePerOp.remove, true);
|
|
319
|
-
}
|
|
320
|
-
});
|
|
321
|
-
});
|
|
322
|
-
}
|
|
323
|
-
});
|
|
324
|
-
}
|
|
325
|
-
});
|
|
326
|
-
});
|
|
327
|
-
});
|
package/dist/test/test_bulk.js
DELETED
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const assert_1 = __importDefault(require("assert"));
|
|
7
|
-
const index_1 = require("../index");
|
|
8
|
-
const util_1 = __importDefault(require("util"));
|
|
9
|
-
'use strict';
|
|
10
|
-
const assert = assert_1.default.strict;
|
|
11
|
-
const range = (N) => [...Array(N).keys()];
|
|
12
|
-
describe(__filename, () => {
|
|
13
|
-
let db = null;
|
|
14
|
-
let mock = null;
|
|
15
|
-
const createDb = async (wrapperSettings) => {
|
|
16
|
-
const settings = {};
|
|
17
|
-
db = new index_1.Database('mock', settings, wrapperSettings);
|
|
18
|
-
// @ts-expect-error TS(2339): Property 'mock' does not exist on type '{}'.
|
|
19
|
-
mock = settings.mock;
|
|
20
|
-
mock.once('init', (cb) => cb());
|
|
21
|
-
await db.init();
|
|
22
|
-
};
|
|
23
|
-
afterEach(async () => {
|
|
24
|
-
if (mock != null) {
|
|
25
|
-
mock.removeAllListeners();
|
|
26
|
-
mock.once('close', (cb) => cb());
|
|
27
|
-
mock = null;
|
|
28
|
-
}
|
|
29
|
-
if (db != null) {
|
|
30
|
-
await db.close();
|
|
31
|
-
db = null;
|
|
32
|
-
}
|
|
33
|
-
});
|
|
34
|
-
describe('bulkLimit', () => {
|
|
35
|
-
const bulkLimits = [0, false, null, undefined, '', 1, 2];
|
|
36
|
-
for (const bulkLimit of bulkLimits) {
|
|
37
|
-
it(bulkLimit === undefined ? 'undefined' : JSON.stringify(bulkLimit), async () => {
|
|
38
|
-
await createDb({ bulkLimit });
|
|
39
|
-
const gotWrites = [];
|
|
40
|
-
mock.on('set', util_1.default.callbackify(async (k, v) => gotWrites.push(1)));
|
|
41
|
-
mock.on('doBulk', util_1.default.callbackify(async (ops) => gotWrites.push(ops.length)));
|
|
42
|
-
const N = 10;
|
|
43
|
-
await Promise.all(range(N).map((i) => db.set(`key${i}`, `val${i}`)));
|
|
44
|
-
const wantLimit = bulkLimit || N;
|
|
45
|
-
// @ts-expect-error TS(2363): The right-hand side of an arithmetic operation mus... Remove this comment to see the full error message
|
|
46
|
-
const wantWrites = range(N / wantLimit).map((i) => wantLimit);
|
|
47
|
-
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
48
|
-
assert.deepEqual(gotWrites, wantWrites);
|
|
49
|
-
});
|
|
50
|
-
}
|
|
51
|
-
});
|
|
52
|
-
it('bulk failures are retried individually', async () => {
|
|
53
|
-
await createDb({});
|
|
54
|
-
const gotDoBulkCalls = [];
|
|
55
|
-
mock.on('doBulk', util_1.default.callbackify(async (ops) => {
|
|
56
|
-
gotDoBulkCalls.push(ops.length);
|
|
57
|
-
throw new Error('test');
|
|
58
|
-
}));
|
|
59
|
-
const gotWrites = new Map();
|
|
60
|
-
const wantWrites = new Map();
|
|
61
|
-
mock.on('set', util_1.default.callbackify(async (k, v) => gotWrites.set(k, v)));
|
|
62
|
-
const N = 10;
|
|
63
|
-
await Promise.all(range(N).map(async (i) => {
|
|
64
|
-
const k = `key${i}`;
|
|
65
|
-
const v = `val${i}`;
|
|
66
|
-
wantWrites.set(k, JSON.stringify(v));
|
|
67
|
-
await db.set(k, v);
|
|
68
|
-
}));
|
|
69
|
-
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
70
|
-
assert.deepEqual(gotDoBulkCalls, [N]);
|
|
71
|
-
// @ts-expect-error TS(2775): Assertions require every name in the call target t... Remove this comment to see the full error message
|
|
72
|
-
assert.deepEqual(gotWrites, wantWrites);
|
|
73
|
-
});
|
|
74
|
-
});
|
|
@@ -1,157 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
-
};
|
|
28
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
const assert_1 = require("assert");
|
|
30
|
-
const elasticsearch8_1 = __importDefault(require("elasticsearch8"));
|
|
31
|
-
const databases_1 = require("./lib/databases");
|
|
32
|
-
const logging_1 = __importDefault(require("../lib/logging"));
|
|
33
|
-
const ueberdb = __importStar(require("../index"));
|
|
34
|
-
'use strict';
|
|
35
|
-
const { databases: { elasticsearch: cfg } } = { databases: databases_1.databases };
|
|
36
|
-
const logger = new class extends logging_1.default.ConsoleLogger {
|
|
37
|
-
info() { }
|
|
38
|
-
isInfoEnabled() { return false; }
|
|
39
|
-
}();
|
|
40
|
-
describe(__filename, function () {
|
|
41
|
-
this.timeout(60000);
|
|
42
|
-
const { base_index = 'ueberdb_test' } = cfg;
|
|
43
|
-
let client;
|
|
44
|
-
let db;
|
|
45
|
-
beforeEach(async () => {
|
|
46
|
-
client = new elasticsearch8_1.default.Client({
|
|
47
|
-
node: `http://${cfg.host || '127.0.0.1'}:${cfg.port || '9200'}`,
|
|
48
|
-
});
|
|
49
|
-
await client.indices.delete({ index: `${base_index}*` }, { ignore: [404] });
|
|
50
|
-
});
|
|
51
|
-
afterEach(async () => {
|
|
52
|
-
if (db != null) {
|
|
53
|
-
await db.close();
|
|
54
|
-
}
|
|
55
|
-
db = null;
|
|
56
|
-
await client.indices.delete({ index: `${base_index}*` }, { ignore: [404] });
|
|
57
|
-
client.close();
|
|
58
|
-
client = null;
|
|
59
|
-
});
|
|
60
|
-
describe('migration to schema v2', () => {
|
|
61
|
-
describe('no old data', () => {
|
|
62
|
-
for (const migrate of [false, true]) {
|
|
63
|
-
it(`migration ${migrate ? 'en' : 'dis'}abled`, async () => {
|
|
64
|
-
// @ts-ignore
|
|
65
|
-
const settings = { base_index, migrate_to_newer_schema: undefined,
|
|
66
|
-
...cfg };
|
|
67
|
-
delete settings.migrate_to_newer_schema;
|
|
68
|
-
db = new ueberdb.Database('elasticsearch', settings, {}, logger);
|
|
69
|
-
await db.init();
|
|
70
|
-
const indices = [];
|
|
71
|
-
const res = await client.indices.get({ index: `${base_index}*` });
|
|
72
|
-
for (const [k, v] of Object.entries(res)) {
|
|
73
|
-
indices.push(k);
|
|
74
|
-
// @ts-expect-error TS(2571): Object is of type 'unknown'.
|
|
75
|
-
indices.push(...Object.keys(v.aliases));
|
|
76
|
-
}
|
|
77
|
-
(0, assert_1.deepEqual)(indices.sort(), [`${base_index}_s2`, `${base_index}_s2_i0`].sort());
|
|
78
|
-
});
|
|
79
|
-
}
|
|
80
|
-
});
|
|
81
|
-
describe('existing data', () => {
|
|
82
|
-
// @ts-expect-error TS(2769): No overload matches this call.
|
|
83
|
-
const data = new Map([
|
|
84
|
-
['foo:number', 42],
|
|
85
|
-
['foo:string', 'value'],
|
|
86
|
-
['foo:object', { k: 'v' }],
|
|
87
|
-
['foo:p:s:number', 42],
|
|
88
|
-
['foo:p:s:string', 'value'],
|
|
89
|
-
['foo:p:s:object', { k: 'v' }],
|
|
90
|
-
]);
|
|
91
|
-
const setOld = async (k, v) => {
|
|
92
|
-
const kp = k.split(':');
|
|
93
|
-
const index = kp.length === 4 ? `${base_index}-${kp[0]}-${kp[2]}` : base_index;
|
|
94
|
-
await client.index({
|
|
95
|
-
index,
|
|
96
|
-
type: kp.length === 4 ? encodeURIComponent(kp[1]) : kp[0],
|
|
97
|
-
id: kp.length === 4 ? kp[3] : encodeURIComponent(kp[1]),
|
|
98
|
-
body: {
|
|
99
|
-
// The old elasticsearch driver was inconsistent: doBulk() called JSON.parse() on the
|
|
100
|
-
// value from ueberdb before writing, but set() did not. We'll assume that any existing
|
|
101
|
-
// data came from set() writes, not doBulk() writes.
|
|
102
|
-
val: JSON.stringify(v),
|
|
103
|
-
},
|
|
104
|
-
});
|
|
105
|
-
await client.indices.refresh({ index });
|
|
106
|
-
};
|
|
107
|
-
beforeEach(async () => {
|
|
108
|
-
await Promise.all([...data].map(async ([k, v]) => await setOld(k, v)));
|
|
109
|
-
});
|
|
110
|
-
it('migration disabled => init error', async () => {
|
|
111
|
-
// @ts-ignore
|
|
112
|
-
const settings = { base_index, migrate_to_newer_schema: undefined,
|
|
113
|
-
...cfg };
|
|
114
|
-
delete settings.migrate_to_newer_schema;
|
|
115
|
-
db = new ueberdb.Database('elasticsearch', settings, {}, logger);
|
|
116
|
-
await (0, assert_1.rejects)(db.init(), /migrate_to_newer_schema/);
|
|
117
|
-
});
|
|
118
|
-
it('migration enabled', async () => {
|
|
119
|
-
// @ts-ignore
|
|
120
|
-
const settings = { base_index, ...cfg, migrate_to_newer_schema: true };
|
|
121
|
-
db = new ueberdb.Database('elasticsearch', settings, {}, logger);
|
|
122
|
-
await db.init();
|
|
123
|
-
await Promise.all([...data].map(async ([k, v]) => {
|
|
124
|
-
(0, assert_1.deepEqual)(await db.get(k), v);
|
|
125
|
-
}));
|
|
126
|
-
});
|
|
127
|
-
it('each attempt uses a new index', async () => {
|
|
128
|
-
await setOld('a-x:b:c-x:d', 'v'); // Force a conversion failure.
|
|
129
|
-
cfg.base_index = base_index;
|
|
130
|
-
const settings = { ...cfg, migrate_to_newer_schema: true };
|
|
131
|
-
db = new ueberdb.Database('elasticsearch', settings, {}, logger);
|
|
132
|
-
const getIndices = async () => Object.keys((await client.indices.get({ index: `${base_index}_s2*` })));
|
|
133
|
-
(0, assert_1.deepEqual)(await getIndices(), []);
|
|
134
|
-
await (0, assert_1.rejects)(db.init(), /ambig/);
|
|
135
|
-
(0, assert_1.deepEqual)(await getIndices(), [`${base_index}_s2_migrate_attempt_0`]);
|
|
136
|
-
await (0, assert_1.rejects)(db.init(), /ambig/);
|
|
137
|
-
(0, assert_1.deepEqual)((await getIndices()).sort(), [
|
|
138
|
-
`${base_index}_s2_migrate_attempt_0`,
|
|
139
|
-
`${base_index}_s2_migrate_attempt_1`,
|
|
140
|
-
]);
|
|
141
|
-
});
|
|
142
|
-
it('final name not created until success', async () => {
|
|
143
|
-
});
|
|
144
|
-
describe('ambiguous key', () => {
|
|
145
|
-
for (const k of ['a:b:c-x:d', 'a-x:b:c:d', 'a-x:b:c-x:d']) {
|
|
146
|
-
it(k, async () => {
|
|
147
|
-
await setOld(k, 'v');
|
|
148
|
-
cfg.base_index = base_index;
|
|
149
|
-
const settings = { ...cfg, migrate_to_newer_schema: true };
|
|
150
|
-
db = new ueberdb.Database('elasticsearch', settings, {}, logger);
|
|
151
|
-
await (0, assert_1.rejects)(db.init(), /ambig/);
|
|
152
|
-
});
|
|
153
|
-
}
|
|
154
|
-
});
|
|
155
|
-
});
|
|
156
|
-
});
|
|
157
|
-
});
|