spice-couchbase 2.0.63 → 2.0.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/connectionHealth.js +114 -149
- package/build/connectionHealth.test.js +301 -316
- package/package.json +1 -1
- package/src/connectionHealth.js +48 -7
- package/src/connectionHealth.test.js +53 -4
|
@@ -1,29 +1,20 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
-
function ownKeys(
|
|
4
|
-
|
|
5
|
-
function
|
|
6
|
-
|
|
7
|
-
function
|
|
8
|
-
|
|
9
|
-
function
|
|
10
|
-
|
|
11
|
-
function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
|
|
12
|
-
|
|
3
|
+
function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
|
|
4
|
+
function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
|
|
5
|
+
function _defineProperty(e, r, t) { return (r = _toPropertyKey(r)) in e ? Object.defineProperty(e, r, { value: t, enumerable: !0, configurable: !0, writable: !0 }) : e[r] = t, e; }
|
|
6
|
+
function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == typeof i ? i : i + ""; }
|
|
7
|
+
function _toPrimitive(t, e) { if ("object" != typeof t || !t) return t; var r; if ("undefined" != typeof Symbol && void 0 !== (r = t[Symbol.toPrimitive])) { var i = r.call(t, e || "default"); if ("object" != typeof i) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === e ? String : Number)(t); }
|
|
8
|
+
function asyncGeneratorStep(n, t, e, r, o, a, c) { try { var i = n[a](c), u = i.value; } catch (n) { return void e(n); } i.done ? t(u) : Promise.resolve(u).then(r, o); }
|
|
9
|
+
function _asyncToGenerator(n) { return function () { var t = this, e = arguments; return new Promise(function (r, o) { var a = n.apply(t, e); function _next(n) { asyncGeneratorStep(a, r, o, _next, _throw, "next", n); } function _throw(n) { asyncGeneratorStep(a, r, o, _next, _throw, "throw", n); } _next(void 0); }); }; }
|
|
13
10
|
var assert = require("node:assert/strict");
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
test
|
|
17
|
-
} = require("node:test");
|
|
18
|
-
|
|
11
|
+
var _require = require("node:test"),
|
|
12
|
+
test = _require.test;
|
|
19
13
|
var couchbase = require("couchbase");
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
isConnectivityError
|
|
25
|
-
} = require("./connectionHealth");
|
|
26
|
-
|
|
14
|
+
var _require2 = require("./connectionHealth"),
|
|
15
|
+
createHealthMonitor = _require2.createHealthMonitor,
|
|
16
|
+
buildConnectOptions = _require2.buildConnectOptions,
|
|
17
|
+
isConnectivityError = _require2.isConnectivityError;
|
|
27
18
|
var config = {
|
|
28
19
|
nodes: "couchbase://db",
|
|
29
20
|
bucket: "saturn",
|
|
@@ -31,29 +22,28 @@ var config = {
|
|
|
31
22
|
username: "u",
|
|
32
23
|
password: "p"
|
|
33
24
|
};
|
|
25
|
+
var timeoutError = () => new couchbase.UnambiguousTimeoutError();
|
|
34
26
|
|
|
35
|
-
|
|
27
|
+
// A fake cluster whose probe behaviour can be flipped between healthy and
|
|
36
28
|
// wedged. `hang` makes operations never settle, like a stuck SDK.
|
|
37
|
-
|
|
38
|
-
|
|
39
29
|
function fakeCluster(id) {
|
|
40
|
-
var {
|
|
41
|
-
healthy =
|
|
42
|
-
|
|
43
|
-
|
|
30
|
+
var _ref = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {},
|
|
31
|
+
_ref$healthy = _ref.healthy,
|
|
32
|
+
healthy = _ref$healthy === void 0 ? true : _ref$healthy,
|
|
33
|
+
_ref$hang = _ref.hang,
|
|
34
|
+
hang = _ref$hang === void 0 ? false : _ref$hang;
|
|
44
35
|
var cluster = {
|
|
45
36
|
id,
|
|
46
37
|
healthy,
|
|
47
38
|
hang,
|
|
48
39
|
closed: 0,
|
|
49
40
|
queries: 0,
|
|
50
|
-
|
|
41
|
+
kvStates: [couchbase.PingState.Ok],
|
|
51
42
|
close() {
|
|
52
43
|
return _asyncToGenerator(function* () {
|
|
53
44
|
cluster.closed += 1;
|
|
54
45
|
})();
|
|
55
46
|
},
|
|
56
|
-
|
|
57
47
|
query() {
|
|
58
48
|
cluster.queries += 1;
|
|
59
49
|
if (cluster.hang) return new Promise(() => {});
|
|
@@ -63,49 +53,49 @@ function fakeCluster(id) {
|
|
|
63
53
|
}]
|
|
64
54
|
}) : Promise.reject(timeoutError());
|
|
65
55
|
},
|
|
66
|
-
|
|
67
56
|
bucket(name) {
|
|
68
57
|
var bucket = {
|
|
69
58
|
name,
|
|
70
59
|
cluster,
|
|
71
|
-
|
|
72
60
|
waitUntilReady() {
|
|
73
61
|
return _asyncToGenerator(function* () {
|
|
74
62
|
if (!cluster.healthy) throw timeoutError();
|
|
75
63
|
})();
|
|
76
64
|
},
|
|
77
|
-
|
|
65
|
+
ping() {
|
|
66
|
+
if (cluster.hang) return new Promise(() => {});
|
|
67
|
+
if (!cluster.healthy) return Promise.reject(timeoutError());
|
|
68
|
+
return Promise.resolve({
|
|
69
|
+
services: {
|
|
70
|
+
kv: cluster.kvStates.map(state => ({
|
|
71
|
+
state
|
|
72
|
+
}))
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
},
|
|
78
76
|
scope(scopeName) {
|
|
79
77
|
return {
|
|
80
78
|
name: scopeName,
|
|
81
79
|
bucket,
|
|
82
|
-
|
|
83
80
|
collection(collectionName) {
|
|
84
81
|
return {
|
|
85
82
|
name: collectionName,
|
|
86
83
|
cluster,
|
|
87
|
-
|
|
84
|
+
// Like `_default` inside a named scope: the collection does
|
|
85
|
+
// not exist, so KV operations on it always time out.
|
|
88
86
|
exists() {
|
|
89
|
-
|
|
90
|
-
return cluster.healthy ? Promise.resolve({
|
|
91
|
-
exists: false
|
|
92
|
-
}) : Promise.reject(timeoutError());
|
|
87
|
+
return Promise.reject(timeoutError());
|
|
93
88
|
}
|
|
94
|
-
|
|
95
89
|
};
|
|
96
90
|
}
|
|
97
|
-
|
|
98
91
|
};
|
|
99
92
|
}
|
|
100
|
-
|
|
101
93
|
};
|
|
102
94
|
return bucket;
|
|
103
95
|
}
|
|
104
|
-
|
|
105
96
|
};
|
|
106
97
|
return cluster;
|
|
107
98
|
}
|
|
108
|
-
|
|
109
99
|
function handlesFor(cluster) {
|
|
110
100
|
var bucket = cluster.bucket(config.bucket);
|
|
111
101
|
var scope = bucket.scope(config.scope);
|
|
@@ -116,13 +106,12 @@ function handlesFor(cluster) {
|
|
|
116
106
|
collection: scope.collection("_default")
|
|
117
107
|
};
|
|
118
108
|
}
|
|
119
|
-
|
|
120
109
|
function setup() {
|
|
121
|
-
var {
|
|
122
|
-
settings =
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
110
|
+
var _ref2 = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {},
|
|
111
|
+
_ref2$settings = _ref2.settings,
|
|
112
|
+
settings = _ref2$settings === void 0 ? {} : _ref2$settings,
|
|
113
|
+
connect = _ref2.connect,
|
|
114
|
+
clock = _ref2.clock;
|
|
126
115
|
var initial = fakeCluster("initial");
|
|
127
116
|
var handles = handlesFor(initial);
|
|
128
117
|
var globals = {
|
|
@@ -171,9 +160,7 @@ function setup() {
|
|
|
171
160
|
retireGraceMs: 5,
|
|
172
161
|
exitAfterMs: 10000
|
|
173
162
|
}, settings),
|
|
174
|
-
connect: connect ||
|
|
175
|
-
/*#__PURE__*/
|
|
176
|
-
_asyncToGenerator(function* () {
|
|
163
|
+
connect: connect || /*#__PURE__*/_asyncToGenerator(function* () {
|
|
177
164
|
var cluster = fakeCluster("fresh-".concat(opened.length + 1));
|
|
178
165
|
opened.push(cluster);
|
|
179
166
|
return cluster;
|
|
@@ -195,9 +182,7 @@ function setup() {
|
|
|
195
182
|
logs
|
|
196
183
|
};
|
|
197
184
|
}
|
|
198
|
-
|
|
199
185
|
var sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
200
|
-
|
|
201
186
|
test("only connectivity errors count", () => {
|
|
202
187
|
assert.equal(isConnectivityError(timeoutError()), true);
|
|
203
188
|
assert.equal(isConnectivityError(new couchbase.AmbiguousTimeoutError()), true);
|
|
@@ -212,7 +197,7 @@ test("connect options carry only the timeouts that are set", () => {
|
|
|
212
197
|
username: "u",
|
|
213
198
|
password: "p"
|
|
214
199
|
});
|
|
215
|
-
assert.deepEqual(buildConnectOptions(_objectSpread({}, config, {
|
|
200
|
+
assert.deepEqual(buildConnectOptions(_objectSpread(_objectSpread({}, config), {}, {
|
|
216
201
|
configProfile: "wanDevelopment",
|
|
217
202
|
timeouts: {
|
|
218
203
|
queryTimeout: "30000",
|
|
@@ -228,43 +213,104 @@ test("connect options carry only the timeouts that are set", () => {
|
|
|
228
213
|
}
|
|
229
214
|
});
|
|
230
215
|
});
|
|
231
|
-
test("a healthy probe leaves the connection alone",
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
initial
|
|
239
|
-
} = setup();
|
|
240
|
-
assert.equal((yield monitor.check("default")), true);
|
|
216
|
+
test("a healthy probe leaves the connection alone", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
217
|
+
var _setup = setup(),
|
|
218
|
+
monitor = _setup.monitor,
|
|
219
|
+
opened = _setup.opened,
|
|
220
|
+
globals = _setup.globals,
|
|
221
|
+
initial = _setup.initial;
|
|
222
|
+
assert.equal(yield monitor.check("default"), true);
|
|
241
223
|
assert.equal(opened.length, 0);
|
|
242
224
|
assert.equal(globals.database.clusters.default, initial);
|
|
243
225
|
assert.equal(monitor.status().default.healthy, true);
|
|
244
226
|
}));
|
|
245
|
-
test("
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
227
|
+
test("the probe does not depend on the boot-time collection existing", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
228
|
+
// The fake collection's KV operations always time out; ping still succeeds.
|
|
229
|
+
var _setup2 = setup({
|
|
230
|
+
settings: {
|
|
231
|
+
probeFailureThreshold: 1
|
|
232
|
+
}
|
|
233
|
+
}),
|
|
234
|
+
monitor = _setup2.monitor,
|
|
235
|
+
opened = _setup2.opened;
|
|
236
|
+
assert.equal(yield monitor.check("default"), true);
|
|
237
|
+
assert.equal(opened.length, 0);
|
|
238
|
+
assert.equal(monitor.status().default.healthy, true);
|
|
239
|
+
}));
|
|
240
|
+
test("one answering KV node is enough, none is unhealthy", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
241
|
+
var _setup3 = setup({
|
|
242
|
+
settings: {
|
|
243
|
+
probeFailureThreshold: 1
|
|
244
|
+
}
|
|
245
|
+
}),
|
|
246
|
+
monitor = _setup3.monitor,
|
|
247
|
+
initial = _setup3.initial,
|
|
248
|
+
opened = _setup3.opened;
|
|
249
|
+
var _couchbase$PingState = couchbase.PingState,
|
|
250
|
+
Ok = _couchbase$PingState.Ok,
|
|
251
|
+
Timeout = _couchbase$PingState.Timeout,
|
|
252
|
+
Failed = _couchbase$PingState.Error;
|
|
253
|
+
initial.kvStates = [Timeout, Ok, Failed];
|
|
254
|
+
assert.equal(yield monitor.check("default"), true);
|
|
255
|
+
assert.equal(opened.length, 0);
|
|
256
|
+
initial.kvStates = [Timeout, Failed];
|
|
257
|
+
yield monitor.check("default");
|
|
258
|
+
assert.equal(opened.length, 1);
|
|
259
|
+
}));
|
|
260
|
+
test("probe failures say which check failed", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
261
|
+
var _setup4 = setup({
|
|
262
|
+
settings: {
|
|
263
|
+
probeFailureThreshold: 1
|
|
264
|
+
},
|
|
265
|
+
connect: function () {
|
|
266
|
+
var _connect = _asyncToGenerator(function* () {
|
|
267
|
+
return fakeCluster("bad", {
|
|
268
|
+
healthy: false
|
|
269
|
+
});
|
|
270
|
+
});
|
|
271
|
+
function connect() {
|
|
272
|
+
return _connect.apply(this, arguments);
|
|
273
|
+
}
|
|
274
|
+
return connect;
|
|
275
|
+
}()
|
|
276
|
+
}),
|
|
277
|
+
monitor = _setup4.monitor,
|
|
278
|
+
initial = _setup4.initial;
|
|
279
|
+
initial.kvStates = [];
|
|
280
|
+
yield monitor.check("default");
|
|
281
|
+
assert.match(monitor.status().default.lastError, /waitUntilReady failed/);
|
|
282
|
+
var query = setup({
|
|
283
|
+
settings: {
|
|
284
|
+
probeFailureThreshold: 1
|
|
285
|
+
}
|
|
286
|
+
});
|
|
287
|
+
query.initial.query = () => Promise.reject(timeoutError());
|
|
288
|
+
yield query.monitor.check("default");
|
|
289
|
+
assert.match(query.logs.join("\n"), /unhealthy: Query probe failed/);
|
|
290
|
+
var kv = setup({
|
|
291
|
+
settings: {
|
|
292
|
+
probeFailureThreshold: 1
|
|
293
|
+
}
|
|
294
|
+
});
|
|
295
|
+
kv.initial.kvStates = [];
|
|
296
|
+
yield kv.monitor.check("default");
|
|
297
|
+
assert.match(kv.logs.join("\n"), /unhealthy: KV ping failed: no endpoint responded/);
|
|
298
|
+
}));
|
|
299
|
+
test("non-connectivity errors never trigger a probe", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
300
|
+
var _setup5 = setup(),
|
|
301
|
+
monitor = _setup5.monitor,
|
|
302
|
+
initial = _setup5.initial;
|
|
253
303
|
for (var i = 0; i < 10; i++) {
|
|
254
304
|
monitor.recordOutcome("default", new couchbase.DocumentNotFoundError());
|
|
255
305
|
}
|
|
256
|
-
|
|
257
306
|
yield sleep(5);
|
|
258
307
|
assert.equal(initial.queries, 0);
|
|
259
308
|
}));
|
|
260
|
-
test("one failed probe is not enough, two trigger a reconnect",
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
initial,
|
|
266
|
-
opened
|
|
267
|
-
} = setup();
|
|
309
|
+
test("one failed probe is not enough, two trigger a reconnect", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
310
|
+
var _setup6 = setup(),
|
|
311
|
+
monitor = _setup6.monitor,
|
|
312
|
+
initial = _setup6.initial,
|
|
313
|
+
opened = _setup6.opened;
|
|
268
314
|
initial.healthy = false;
|
|
269
315
|
yield monitor.check("default");
|
|
270
316
|
assert.equal(opened.length, 0);
|
|
@@ -274,34 +320,24 @@ _asyncToGenerator(function* () {
|
|
|
274
320
|
assert.equal(monitor.status().default.healthy, true);
|
|
275
321
|
assert.equal(monitor.status().default.generation, 2);
|
|
276
322
|
}));
|
|
277
|
-
test("a burst of operation timeouts probes and reconnects at once",
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
opened,
|
|
284
|
-
globals
|
|
285
|
-
} = setup();
|
|
323
|
+
test("a burst of operation timeouts probes and reconnects at once", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
324
|
+
var _setup7 = setup(),
|
|
325
|
+
monitor = _setup7.monitor,
|
|
326
|
+
initial = _setup7.initial,
|
|
327
|
+
opened = _setup7.opened,
|
|
328
|
+
globals = _setup7.globals;
|
|
286
329
|
initial.healthy = false;
|
|
287
|
-
|
|
288
|
-
for (var i = 0; i < 3; i++) {
|
|
289
|
-
monitor.recordOutcome("default", timeoutError());
|
|
290
|
-
}
|
|
291
|
-
|
|
330
|
+
for (var i = 0; i < 3; i++) monitor.recordOutcome("default", timeoutError());
|
|
292
331
|
yield sleep(10);
|
|
293
332
|
assert.equal(opened.length, 1);
|
|
294
333
|
assert.equal(globals.database.clusters.default, opened[0]);
|
|
295
334
|
}));
|
|
296
|
-
test("timeouts outside the window do not add up",
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
opened,
|
|
303
|
-
time
|
|
304
|
-
} = setup();
|
|
335
|
+
test("timeouts outside the window do not add up", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
336
|
+
var _setup8 = setup(),
|
|
337
|
+
monitor = _setup8.monitor,
|
|
338
|
+
initial = _setup8.initial,
|
|
339
|
+
opened = _setup8.opened,
|
|
340
|
+
time = _setup8.time;
|
|
305
341
|
initial.healthy = false;
|
|
306
342
|
monitor.recordOutcome("default", timeoutError());
|
|
307
343
|
monitor.recordOutcome("default", timeoutError());
|
|
@@ -311,15 +347,12 @@ _asyncToGenerator(function* () {
|
|
|
311
347
|
assert.equal(initial.queries, 0);
|
|
312
348
|
assert.equal(opened.length, 0);
|
|
313
349
|
}));
|
|
314
|
-
test("reconnect swaps every global handle and resets collection caches",
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
opened,
|
|
321
|
-
globals
|
|
322
|
-
} = setup();
|
|
350
|
+
test("reconnect swaps every global handle and resets collection caches", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
351
|
+
var _setup9 = setup(),
|
|
352
|
+
monitor = _setup9.monitor,
|
|
353
|
+
initial = _setup9.initial,
|
|
354
|
+
opened = _setup9.opened,
|
|
355
|
+
globals = _setup9.globals;
|
|
323
356
|
initial.healthy = false;
|
|
324
357
|
yield monitor.check("default");
|
|
325
358
|
yield monitor.check("default");
|
|
@@ -331,13 +364,10 @@ _asyncToGenerator(function* () {
|
|
|
331
364
|
assert.deepEqual(globals.checked_collections, {});
|
|
332
365
|
assert.equal(globals.db.collections.size, 0);
|
|
333
366
|
}));
|
|
334
|
-
test("the replaced cluster is closed after the grace period",
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
monitor,
|
|
339
|
-
initial
|
|
340
|
-
} = setup();
|
|
367
|
+
test("the replaced cluster is closed after the grace period", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
368
|
+
var _setup0 = setup(),
|
|
369
|
+
monitor = _setup0.monitor,
|
|
370
|
+
initial = _setup0.initial;
|
|
341
371
|
initial.healthy = false;
|
|
342
372
|
yield monitor.check("default");
|
|
343
373
|
yield monitor.check("default");
|
|
@@ -345,57 +375,46 @@ _asyncToGenerator(function* () {
|
|
|
345
375
|
yield sleep(20);
|
|
346
376
|
assert.equal(initial.closed, 1);
|
|
347
377
|
}));
|
|
348
|
-
test("concurrent triggers share a single reconnect",
|
|
349
|
-
/*#__PURE__*/
|
|
350
|
-
_asyncToGenerator(function* () {
|
|
378
|
+
test("concurrent triggers share a single reconnect", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
351
379
|
var release;
|
|
352
380
|
var opened = [];
|
|
353
|
-
var {
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
});
|
|
381
|
+
var _setup1 = setup({
|
|
382
|
+
settings: {
|
|
383
|
+
probeFailureThreshold: 1,
|
|
384
|
+
reconnectTimeoutMs: 1000
|
|
385
|
+
},
|
|
386
|
+
connect: () => new Promise(resolve => {
|
|
387
|
+
release = () => {
|
|
388
|
+
var cluster = fakeCluster("fresh");
|
|
389
|
+
opened.push(cluster);
|
|
390
|
+
resolve(cluster);
|
|
391
|
+
};
|
|
392
|
+
})
|
|
393
|
+
}),
|
|
394
|
+
monitor = _setup1.monitor,
|
|
395
|
+
initial = _setup1.initial;
|
|
369
396
|
initial.healthy = false;
|
|
370
397
|
var first = monitor.check("default");
|
|
371
398
|
yield sleep(5);
|
|
372
399
|
var others = [monitor.check("default"), monitor.reconnect("default")];
|
|
373
|
-
|
|
374
|
-
for (var i = 0; i < 5; i++) {
|
|
375
|
-
monitor.recordOutcome("default", timeoutError());
|
|
376
|
-
}
|
|
377
|
-
|
|
400
|
+
for (var i = 0; i < 5; i++) monitor.recordOutcome("default", timeoutError());
|
|
378
401
|
release();
|
|
379
402
|
yield Promise.all([first, ...others]);
|
|
380
403
|
assert.equal(opened.length, 1);
|
|
381
404
|
assert.equal(monitor.status().default.generation, 2);
|
|
382
405
|
}));
|
|
383
|
-
test("a probe that hangs is cut off by the hard deadline",
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
});
|
|
395
|
-
initial.hang = true; // The deadline timer is unref'd; keep the loop alive while the probe hangs.
|
|
396
|
-
|
|
406
|
+
test("a probe that hangs is cut off by the hard deadline", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
407
|
+
var _setup10 = setup({
|
|
408
|
+
settings: {
|
|
409
|
+
probeFailureThreshold: 1
|
|
410
|
+
}
|
|
411
|
+
}),
|
|
412
|
+
monitor = _setup10.monitor,
|
|
413
|
+
initial = _setup10.initial,
|
|
414
|
+
opened = _setup10.opened;
|
|
415
|
+
initial.hang = true;
|
|
416
|
+
// The deadline timer is unref'd; keep the loop alive while the probe hangs.
|
|
397
417
|
var keepAlive = setInterval(() => {}, 1000);
|
|
398
|
-
|
|
399
418
|
try {
|
|
400
419
|
var started = Date.now();
|
|
401
420
|
yield monitor.check("default");
|
|
@@ -405,42 +424,38 @@ _asyncToGenerator(function* () {
|
|
|
405
424
|
clearInterval(keepAlive);
|
|
406
425
|
}
|
|
407
426
|
}));
|
|
408
|
-
test("a failed reconnect backs off and keeps the old cluster",
|
|
409
|
-
/*#__PURE__*/
|
|
410
|
-
_asyncToGenerator(function* () {
|
|
427
|
+
test("a failed reconnect backs off and keeps the old cluster", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
411
428
|
var opened = [];
|
|
412
|
-
var {
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
var cluster = fakeCluster("bad-".concat(opened.length + 1), {
|
|
424
|
-
healthy: false
|
|
429
|
+
var _setup11 = setup({
|
|
430
|
+
settings: {
|
|
431
|
+
probeFailureThreshold: 1
|
|
432
|
+
},
|
|
433
|
+
connect: function () {
|
|
434
|
+
var _connect2 = _asyncToGenerator(function* () {
|
|
435
|
+
var cluster = fakeCluster("bad-".concat(opened.length + 1), {
|
|
436
|
+
healthy: false
|
|
437
|
+
});
|
|
438
|
+
opened.push(cluster);
|
|
439
|
+
return cluster;
|
|
425
440
|
});
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
});
|
|
441
|
+
function connect() {
|
|
442
|
+
return _connect2.apply(this, arguments);
|
|
443
|
+
}
|
|
444
|
+
return connect;
|
|
445
|
+
}()
|
|
446
|
+
}),
|
|
447
|
+
monitor = _setup11.monitor,
|
|
448
|
+
initial = _setup11.initial,
|
|
449
|
+
globals = _setup11.globals,
|
|
450
|
+
time = _setup11.time;
|
|
437
451
|
initial.healthy = false;
|
|
438
452
|
yield monitor.check("default");
|
|
439
453
|
assert.equal(opened.length, 1);
|
|
440
454
|
assert.equal(opened[0].closed, 1, "unverified cluster is closed");
|
|
441
455
|
assert.equal(globals.database.clusters.default, initial);
|
|
442
|
-
assert.equal(monitor.status().default.reconnectAttempts, 1);
|
|
456
|
+
assert.equal(monitor.status().default.reconnectAttempts, 1);
|
|
443
457
|
|
|
458
|
+
// Inside the backoff window: probe only, no new connect.
|
|
444
459
|
yield monitor.check("default");
|
|
445
460
|
assert.equal(opened.length, 1);
|
|
446
461
|
time.now += 150;
|
|
@@ -448,31 +463,26 @@ _asyncToGenerator(function* () {
|
|
|
448
463
|
assert.equal(opened.length, 2);
|
|
449
464
|
assert.equal(monitor.status().default.reconnectAttempts, 2);
|
|
450
465
|
}));
|
|
451
|
-
test("a connection that recovers on its own is marked healthy again",
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
},
|
|
462
|
-
connect: function () {
|
|
463
|
-
var _connect2 = _asyncToGenerator(function* () {
|
|
464
|
-
return fakeCluster("bad", {
|
|
465
|
-
healthy: false
|
|
466
|
+
test("a connection that recovers on its own is marked healthy again", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
467
|
+
var _setup12 = setup({
|
|
468
|
+
settings: {
|
|
469
|
+
probeFailureThreshold: 1
|
|
470
|
+
},
|
|
471
|
+
connect: function () {
|
|
472
|
+
var _connect3 = _asyncToGenerator(function* () {
|
|
473
|
+
return fakeCluster("bad", {
|
|
474
|
+
healthy: false
|
|
475
|
+
});
|
|
466
476
|
});
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
return
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
477
|
+
function connect() {
|
|
478
|
+
return _connect3.apply(this, arguments);
|
|
479
|
+
}
|
|
480
|
+
return connect;
|
|
481
|
+
}()
|
|
482
|
+
}),
|
|
483
|
+
monitor = _setup12.monitor,
|
|
484
|
+
initial = _setup12.initial,
|
|
485
|
+
time = _setup12.time;
|
|
476
486
|
initial.healthy = false;
|
|
477
487
|
yield monitor.check("default");
|
|
478
488
|
assert.equal(monitor.status().default.healthy, false);
|
|
@@ -484,33 +494,28 @@ _asyncToGenerator(function* () {
|
|
|
484
494
|
assert.equal(status.reconnectAttempts, 0);
|
|
485
495
|
assert.equal(status.unhealthySince, null);
|
|
486
496
|
}));
|
|
487
|
-
test("staying unhealthy past the limit terminates the process once",
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
exitAfterMs: 1000
|
|
499
|
-
},
|
|
500
|
-
connect: function () {
|
|
501
|
-
var _connect3 = _asyncToGenerator(function* () {
|
|
502
|
-
return fakeCluster("bad", {
|
|
503
|
-
healthy: false
|
|
497
|
+
test("staying unhealthy past the limit terminates the process once", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
498
|
+
var _setup13 = setup({
|
|
499
|
+
settings: {
|
|
500
|
+
probeFailureThreshold: 1,
|
|
501
|
+
exitAfterMs: 1000
|
|
502
|
+
},
|
|
503
|
+
connect: function () {
|
|
504
|
+
var _connect4 = _asyncToGenerator(function* () {
|
|
505
|
+
return fakeCluster("bad", {
|
|
506
|
+
healthy: false
|
|
507
|
+
});
|
|
504
508
|
});
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
return
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
509
|
+
function connect() {
|
|
510
|
+
return _connect4.apply(this, arguments);
|
|
511
|
+
}
|
|
512
|
+
return connect;
|
|
513
|
+
}()
|
|
514
|
+
}),
|
|
515
|
+
monitor = _setup13.monitor,
|
|
516
|
+
initial = _setup13.initial,
|
|
517
|
+
time = _setup13.time,
|
|
518
|
+
terminated = _setup13.terminated;
|
|
514
519
|
initial.healthy = false;
|
|
515
520
|
yield monitor.check("default");
|
|
516
521
|
assert.equal(terminated.length, 0);
|
|
@@ -520,55 +525,45 @@ _asyncToGenerator(function* () {
|
|
|
520
525
|
yield monitor.check("default");
|
|
521
526
|
assert.equal(terminated.length, 1);
|
|
522
527
|
}));
|
|
523
|
-
test("exitAfterMs of 0 never terminates",
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
exitAfterMs: 0
|
|
535
|
-
},
|
|
536
|
-
connect: function () {
|
|
537
|
-
var _connect4 = _asyncToGenerator(function* () {
|
|
538
|
-
return fakeCluster("bad", {
|
|
539
|
-
healthy: false
|
|
528
|
+
test("exitAfterMs of 0 never terminates", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
529
|
+
var _setup14 = setup({
|
|
530
|
+
settings: {
|
|
531
|
+
probeFailureThreshold: 1,
|
|
532
|
+
exitAfterMs: 0
|
|
533
|
+
},
|
|
534
|
+
connect: function () {
|
|
535
|
+
var _connect5 = _asyncToGenerator(function* () {
|
|
536
|
+
return fakeCluster("bad", {
|
|
537
|
+
healthy: false
|
|
538
|
+
});
|
|
540
539
|
});
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
return
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
540
|
+
function connect() {
|
|
541
|
+
return _connect5.apply(this, arguments);
|
|
542
|
+
}
|
|
543
|
+
return connect;
|
|
544
|
+
}()
|
|
545
|
+
}),
|
|
546
|
+
monitor = _setup14.monitor,
|
|
547
|
+
initial = _setup14.initial,
|
|
548
|
+
time = _setup14.time,
|
|
549
|
+
terminated = _setup14.terminated;
|
|
550
550
|
initial.healthy = false;
|
|
551
|
-
|
|
552
551
|
for (var i = 0; i < 5; i++) {
|
|
553
552
|
time.now += 100000;
|
|
554
553
|
yield monitor.check("default");
|
|
555
554
|
}
|
|
556
|
-
|
|
557
555
|
assert.equal(terminated.length, 0);
|
|
558
556
|
}));
|
|
559
|
-
test("stop() halts reconnects and closes retired clusters",
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
retireGraceMs: 60000
|
|
570
|
-
}
|
|
571
|
-
});
|
|
557
|
+
test("stop() halts reconnects and closes retired clusters", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
558
|
+
var _setup15 = setup({
|
|
559
|
+
settings: {
|
|
560
|
+
probeFailureThreshold: 1,
|
|
561
|
+
retireGraceMs: 60000
|
|
562
|
+
}
|
|
563
|
+
}),
|
|
564
|
+
monitor = _setup15.monitor,
|
|
565
|
+
initial = _setup15.initial,
|
|
566
|
+
opened = _setup15.opened;
|
|
572
567
|
initial.healthy = false;
|
|
573
568
|
yield monitor.check("default");
|
|
574
569
|
assert.equal(opened.length, 1);
|
|
@@ -579,42 +574,35 @@ _asyncToGenerator(function* () {
|
|
|
579
574
|
yield monitor.reconnect("default");
|
|
580
575
|
assert.equal(opened.length, 1);
|
|
581
576
|
}));
|
|
582
|
-
test("the interval probe runs when enabled",
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
probeFailureThreshold: 1
|
|
593
|
-
}
|
|
594
|
-
});
|
|
577
|
+
test("the interval probe runs when enabled", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
578
|
+
var _setup16 = setup({
|
|
579
|
+
settings: {
|
|
580
|
+
healthIntervalMs: 5,
|
|
581
|
+
probeFailureThreshold: 1
|
|
582
|
+
}
|
|
583
|
+
}),
|
|
584
|
+
monitor = _setup16.monitor,
|
|
585
|
+
initial = _setup16.initial,
|
|
586
|
+
opened = _setup16.opened;
|
|
595
587
|
initial.healthy = false;
|
|
596
588
|
yield sleep(40);
|
|
597
589
|
assert.ok(opened.length >= 1);
|
|
598
590
|
yield monitor.stop();
|
|
599
591
|
}));
|
|
600
|
-
test("a Database instance follows the swapped globals",
|
|
601
|
-
/*#__PURE__*/
|
|
602
|
-
_asyncToGenerator(function* () {
|
|
592
|
+
test("a Database instance follows the swapped globals", /*#__PURE__*/_asyncToGenerator(function* () {
|
|
603
593
|
var path = require.resolve("./index");
|
|
604
|
-
|
|
605
594
|
delete require.cache[path];
|
|
606
595
|
var previousSpice = global.spice;
|
|
607
|
-
var {
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
})
|
|
617
|
-
global.spice = _objectSpread({}, globals, {
|
|
596
|
+
var _setup17 = setup({
|
|
597
|
+
settings: {
|
|
598
|
+
probeFailureThreshold: 1
|
|
599
|
+
}
|
|
600
|
+
}),
|
|
601
|
+
monitor = _setup17.monitor,
|
|
602
|
+
initial = _setup17.initial,
|
|
603
|
+
opened = _setup17.opened,
|
|
604
|
+
globals = _setup17.globals;
|
|
605
|
+
global.spice = _objectSpread(_objectSpread({}, globals), {}, {
|
|
618
606
|
config: {
|
|
619
607
|
database: {
|
|
620
608
|
connections: {
|
|
@@ -622,24 +610,21 @@ _asyncToGenerator(function* () {
|
|
|
622
610
|
}
|
|
623
611
|
}
|
|
624
612
|
}
|
|
625
|
-
});
|
|
626
|
-
|
|
613
|
+
});
|
|
614
|
+
// Point the shared module-level monitor at this test's monitor.
|
|
627
615
|
var key = Symbol.for("spice-couchbase.health-monitor");
|
|
628
616
|
var previousMonitor = globalThis[key];
|
|
629
617
|
globalThis[key] = monitor;
|
|
630
|
-
|
|
631
618
|
try {
|
|
632
619
|
// Keep a reference to the same globals object the monitor mutates.
|
|
633
|
-
global.spice.database = globals.database;
|
|
634
|
-
|
|
620
|
+
global.spice.database = globals.database;
|
|
621
|
+
// The constructor reads the bucket name off the boot-time collection.
|
|
635
622
|
globals.database.connections.default._scope = {
|
|
636
623
|
_bucket: {
|
|
637
624
|
_name: "saturn"
|
|
638
625
|
}
|
|
639
626
|
};
|
|
640
|
-
|
|
641
627
|
var Database = require("./index");
|
|
642
|
-
|
|
643
628
|
var db = new Database("default");
|
|
644
629
|
assert.equal(db.cluster, initial);
|
|
645
630
|
var oldCollection = db.collection;
|