spice-couchbase 2.0.62 → 2.0.63

package/build/connectionHealth.test.js

@@ -0,0 +1,659 @@
+"use strict";
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
+
+function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
+
+function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
+
+var assert = require("node:assert/strict");
+
+var {
+  test
+} = require("node:test");
+
+var couchbase = require("couchbase");
+
+var {
+  createHealthMonitor,
+  buildConnectOptions,
+  isConnectivityError
+} = require("./connectionHealth");
+
+var config = {
+  nodes: "couchbase://db",
+  bucket: "saturn",
+  scope: "app",
+  username: "u",
+  password: "p"
+};
+
+var timeoutError = () => new couchbase.UnambiguousTimeoutError(); // A fake cluster whose probe behaviour can be flipped between healthy and
+// wedged. `hang` makes operations never settle, like a stuck SDK.
+
+
+function fakeCluster(id) {
+  var {
+    healthy = true,
+    hang = false
+  } = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
+  var cluster = {
+    id,
+    healthy,
+    hang,
+    closed: 0,
+    queries: 0,
+
+    close() {
+      return _asyncToGenerator(function* () {
+        cluster.closed += 1;
+      })();
+    },
+
+    query() {
+      cluster.queries += 1;
+      if (cluster.hang) return new Promise(() => {});
+      return cluster.healthy ? Promise.resolve({
+        rows: [{
+          $1: 1
+        }]
+      }) : Promise.reject(timeoutError());
+    },
+
+    bucket(name) {
+      var bucket = {
+        name,
+        cluster,
+
+        waitUntilReady() {
+          return _asyncToGenerator(function* () {
+            if (!cluster.healthy) throw timeoutError();
+          })();
+        },
+
+        scope(scopeName) {
+          return {
+            name: scopeName,
+            bucket,
+
+            collection(collectionName) {
+              return {
+                name: collectionName,
+                cluster,
+
+                exists() {
+                  if (cluster.hang) return new Promise(() => {});
+                  return cluster.healthy ? Promise.resolve({
+                    exists: false
+                  }) : Promise.reject(timeoutError());
+                }
+
+              };
+            }
+
+          };
+        }
+
+      };
+      return bucket;
+    }
+
+  };
+  return cluster;
+}
+
+function handlesFor(cluster) {
+  var bucket = cluster.bucket(config.bucket);
+  var scope = bucket.scope(config.scope);
+  return {
+    cluster,
+    bucket,
+    scope,
+    collection: scope.collection("_default")
+  };
+}
+
+function setup() {
+  var {
+    settings = {},
+    connect,
+    clock
+  } = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
+  var initial = fakeCluster("initial");
+  var handles = handlesFor(initial);
+  var globals = {
+    database: {
+      clusters: {
+        default: handles.cluster
+      },
+      buckets: {
+        default: handles.bucket
+      },
+      scopes: {
+        default: handles.scope
+      },
+      connections: {
+        default: handles.collection
+      }
+    },
+    checked_collections: {
+      users: true
+    },
+    db: {
+      collections: new Set(["users"])
+    }
+  };
+  var opened = [];
+  var time = clock || {
+    now: 1000
+  };
+  var terminated = [];
+  var logs = [];
+  var log = {
+    log: m => logs.push(m),
+    warn: m => logs.push(m),
+    error: m => logs.push(m)
+  };
+  var monitor = createHealthMonitor({
+    settings: _objectSpread({
+      healthIntervalMs: 0,
+      probeTimeoutMs: 20,
+      probeFailureThreshold: 2,
+      failureThreshold: 3,
+      failureWindowMs: 1000,
+      reconnectTimeoutMs: 50,
+      reconnectBackoffMinMs: 100,
+      reconnectBackoffMaxMs: 1000,
+      retireGraceMs: 5,
+      exitAfterMs: 10000
+    }, settings),
+    connect: connect ||
+    /*#__PURE__*/
+    _asyncToGenerator(function* () {
+      var cluster = fakeCluster("fresh-".concat(opened.length + 1));
+      opened.push(cluster);
+      return cluster;
+    }),
+    now: () => time.now,
+    random: () => 0.5,
+    log,
+    getGlobals: () => globals,
+    terminate: () => terminated.push(time.now)
+  });
+  monitor.register("default", config);
+  return {
+    monitor,
+    initial,
+    globals,
+    opened,
+    time,
+    terminated,
+    logs
+  };
+}
+
+var sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+test("only connectivity errors count", () => {
+  assert.equal(isConnectivityError(timeoutError()), true);
+  assert.equal(isConnectivityError(new couchbase.AmbiguousTimeoutError()), true);
+  assert.equal(isConnectivityError(new couchbase.RequestCanceledError()), true);
+  assert.equal(isConnectivityError(new couchbase.ServiceNotAvailableError()), true);
+  assert.equal(isConnectivityError(new couchbase.DocumentNotFoundError()), false);
+  assert.equal(isConnectivityError(new Error("syntax error")), false);
+  assert.equal(isConnectivityError(undefined), false);
+});
+test("connect options carry only the timeouts that are set", () => {
+  assert.deepEqual(buildConnectOptions(config), {
+    username: "u",
+    password: "p"
+  });
+  assert.deepEqual(buildConnectOptions(_objectSpread({}, config, {
+    configProfile: "wanDevelopment",
+    timeouts: {
+      queryTimeout: "30000",
+      kvTimeout: undefined,
+      connectTimeout: ""
+    }
+  })), {
+    username: "u",
+    password: "p",
+    configProfile: "wanDevelopment",
+    timeouts: {
+      queryTimeout: 30000
+    }
+  });
+});
+test("a healthy probe leaves the connection alone",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    opened,
+    globals,
+    initial
+  } = setup();
+  assert.equal((yield monitor.check("default")), true);
+  assert.equal(opened.length, 0);
+  assert.equal(globals.database.clusters.default, initial);
+  assert.equal(monitor.status().default.healthy, true);
+}));
+test("non-connectivity errors never trigger a probe",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial
+  } = setup();
+
+  for (var i = 0; i < 10; i++) {
+    monitor.recordOutcome("default", new couchbase.DocumentNotFoundError());
+  }
+
+  yield sleep(5);
+  assert.equal(initial.queries, 0);
+}));
+test("one failed probe is not enough, two trigger a reconnect",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial,
+    opened
+  } = setup();
+  initial.healthy = false;
+  yield monitor.check("default");
+  assert.equal(opened.length, 0);
+  assert.equal(monitor.status().default.healthy, true);
+  yield monitor.check("default");
+  assert.equal(opened.length, 1);
+  assert.equal(monitor.status().default.healthy, true);
+  assert.equal(monitor.status().default.generation, 2);
+}));
+test("a burst of operation timeouts probes and reconnects at once",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial,
+    opened,
+    globals
+  } = setup();
+  initial.healthy = false;
+
+  for (var i = 0; i < 3; i++) {
+    monitor.recordOutcome("default", timeoutError());
+  }
+
+  yield sleep(10);
+  assert.equal(opened.length, 1);
+  assert.equal(globals.database.clusters.default, opened[0]);
+}));
+test("timeouts outside the window do not add up",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial,
+    opened,
+    time
+  } = setup();
+  initial.healthy = false;
+  monitor.recordOutcome("default", timeoutError());
+  monitor.recordOutcome("default", timeoutError());
+  time.now += 2000;
+  monitor.recordOutcome("default", timeoutError());
+  yield sleep(10);
+  assert.equal(initial.queries, 0);
+  assert.equal(opened.length, 0);
+}));
+test("reconnect swaps every global handle and resets collection caches",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial,
+    opened,
+    globals
+  } = setup();
+  initial.healthy = false;
+  yield monitor.check("default");
+  yield monitor.check("default");
+  var fresh = opened[0];
+  assert.equal(globals.database.clusters.default, fresh);
+  assert.equal(globals.database.buckets.default.cluster, fresh);
+  assert.equal(globals.database.scopes.default.bucket.cluster, fresh);
+  assert.equal(globals.database.connections.default.cluster, fresh);
+  assert.deepEqual(globals.checked_collections, {});
+  assert.equal(globals.db.collections.size, 0);
+}));
+test("the replaced cluster is closed after the grace period",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial
+  } = setup();
+  initial.healthy = false;
+  yield monitor.check("default");
+  yield monitor.check("default");
+  assert.equal(initial.closed, 0);
+  yield sleep(20);
+  assert.equal(initial.closed, 1);
+}));
+test("concurrent triggers share a single reconnect",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var release;
+  var opened = [];
+  var {
+    monitor,
+    initial
+  } = setup({
+    settings: {
+      probeFailureThreshold: 1,
+      reconnectTimeoutMs: 1000
+    },
+    connect: () => new Promise(resolve => {
+      release = () => {
+        var cluster = fakeCluster("fresh");
+        opened.push(cluster);
+        resolve(cluster);
+      };
+    })
+  });
+  initial.healthy = false;
+  var first = monitor.check("default");
+  yield sleep(5);
+  var others = [monitor.check("default"), monitor.reconnect("default")];
+
+  for (var i = 0; i < 5; i++) {
+    monitor.recordOutcome("default", timeoutError());
+  }
+
+  release();
+  yield Promise.all([first, ...others]);
+  assert.equal(opened.length, 1);
+  assert.equal(monitor.status().default.generation, 2);
+}));
+test("a probe that hangs is cut off by the hard deadline",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial,
+    opened
+  } = setup({
+    settings: {
+      probeFailureThreshold: 1
+    }
+  });
+  initial.hang = true; // The deadline timer is unref'd; keep the loop alive while the probe hangs.
+
+  var keepAlive = setInterval(() => {}, 1000);
+
+  try {
+    var started = Date.now();
+    yield monitor.check("default");
+    assert.ok(Date.now() - started < 1500);
+    assert.equal(opened.length, 1);
+  } finally {
+    clearInterval(keepAlive);
+  }
+}));
+test("a failed reconnect backs off and keeps the old cluster",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var opened = [];
+  var {
+    monitor,
+    initial,
+    globals,
+    time
+  } = setup({
+    settings: {
+      probeFailureThreshold: 1
+    },
+    connect: function () {
+      var _connect = _asyncToGenerator(function* () {
+        var cluster = fakeCluster("bad-".concat(opened.length + 1), {
+          healthy: false
+        });
+        opened.push(cluster);
+        return cluster;
+      });
+
+      function connect() {
+        return _connect.apply(this, arguments);
+      }
+
+      return connect;
+    }()
+  });
+  initial.healthy = false;
+  yield monitor.check("default");
+  assert.equal(opened.length, 1);
+  assert.equal(opened[0].closed, 1, "unverified cluster is closed");
+  assert.equal(globals.database.clusters.default, initial);
+  assert.equal(monitor.status().default.reconnectAttempts, 1); // Inside the backoff window: probe only, no new connect.
+
+  yield monitor.check("default");
+  assert.equal(opened.length, 1);
+  time.now += 150;
+  yield monitor.check("default");
+  assert.equal(opened.length, 2);
+  assert.equal(monitor.status().default.reconnectAttempts, 2);
+}));
+test("a connection that recovers on its own is marked healthy again",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial,
+    time
+  } = setup({
+    settings: {
+      probeFailureThreshold: 1
+    },
+    connect: function () {
+      var _connect2 = _asyncToGenerator(function* () {
+        return fakeCluster("bad", {
+          healthy: false
+        });
+      });
+
+      function connect() {
+        return _connect2.apply(this, arguments);
+      }
+
+      return connect;
+    }()
+  });
+  initial.healthy = false;
+  yield monitor.check("default");
+  assert.equal(monitor.status().default.healthy, false);
+  initial.healthy = true;
+  time.now += 5000;
+  yield monitor.check("default");
+  var status = monitor.status().default;
+  assert.equal(status.healthy, true);
+  assert.equal(status.reconnectAttempts, 0);
+  assert.equal(status.unhealthySince, null);
+}));
+test("staying unhealthy past the limit terminates the process once",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial,
+    time,
+    terminated
+  } = setup({
+    settings: {
+      probeFailureThreshold: 1,
+      exitAfterMs: 1000
+    },
+    connect: function () {
+      var _connect3 = _asyncToGenerator(function* () {
+        return fakeCluster("bad", {
+          healthy: false
+        });
+      });
+
+      function connect() {
+        return _connect3.apply(this, arguments);
+      }
+
+      return connect;
+    }()
+  });
+  initial.healthy = false;
+  yield monitor.check("default");
+  assert.equal(terminated.length, 0);
+  time.now += 1500;
+  yield monitor.check("default");
+  time.now += 1500;
+  yield monitor.check("default");
+  assert.equal(terminated.length, 1);
+}));
+test("exitAfterMs of 0 never terminates",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial,
+    time,
+    terminated
+  } = setup({
+    settings: {
+      probeFailureThreshold: 1,
+      exitAfterMs: 0
+    },
+    connect: function () {
+      var _connect4 = _asyncToGenerator(function* () {
+        return fakeCluster("bad", {
+          healthy: false
+        });
+      });
+
+      function connect() {
+        return _connect4.apply(this, arguments);
+      }
+
+      return connect;
+    }()
+  });
+  initial.healthy = false;
+
+  for (var i = 0; i < 5; i++) {
+    time.now += 100000;
+    yield monitor.check("default");
+  }
+
+  assert.equal(terminated.length, 0);
+}));
+test("stop() halts reconnects and closes retired clusters",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial,
+    opened
+  } = setup({
+    settings: {
+      probeFailureThreshold: 1,
+      retireGraceMs: 60000
+    }
+  });
+  initial.healthy = false;
+  yield monitor.check("default");
+  assert.equal(opened.length, 1);
+  yield monitor.stop();
+  assert.equal(initial.closed, 1);
+  opened[0].healthy = false;
+  yield monitor.check("default");
+  yield monitor.reconnect("default");
+  assert.equal(opened.length, 1);
+}));
+test("the interval probe runs when enabled",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var {
+    monitor,
+    initial,
+    opened
+  } = setup({
+    settings: {
+      healthIntervalMs: 5,
+      probeFailureThreshold: 1
+    }
+  });
+  initial.healthy = false;
+  yield sleep(40);
+  assert.ok(opened.length >= 1);
+  yield monitor.stop();
+}));
+test("a Database instance follows the swapped globals",
+/*#__PURE__*/
+_asyncToGenerator(function* () {
+  var path = require.resolve("./index");
+
+  delete require.cache[path];
+  var previousSpice = global.spice;
+  var {
+    monitor,
+    initial,
+    opened,
+    globals
+  } = setup({
+    settings: {
+      probeFailureThreshold: 1
+    }
+  });
+  global.spice = _objectSpread({}, globals, {
+    config: {
+      database: {
+        connections: {
+          default: _objectSpread({}, config)
+        }
+      }
+    }
+  }); // Point the shared module-level monitor at this test's monitor.
+
+  var key = Symbol.for("spice-couchbase.health-monitor");
+  var previousMonitor = globalThis[key];
+  globalThis[key] = monitor;
+
+  try {
+    // Keep a reference to the same globals object the monitor mutates.
+    global.spice.database = globals.database; // The constructor reads the bucket name off the boot-time collection.
+
+    globals.database.connections.default._scope = {
+      _bucket: {
+        _name: "saturn"
+      }
+    };
+
+    var Database = require("./index");
+
+    var db = new Database("default");
+    assert.equal(db.cluster, initial);
+    var oldCollection = db.collection;
+    assert.equal(oldCollection.cluster, initial);
+    initial.healthy = false;
+    yield monitor.check("default");
+    assert.equal(opened.length, 1);
+    assert.equal(db.cluster, opened[0]);
+    assert.equal(db.collection.cluster, opened[0]);
+    assert.notEqual(db.collection, oldCollection);
+    assert.deepEqual(Database.healthStatus().default.generation, 2);
+  } finally {
+    globalThis[key] = previousMonitor;
+    global.spice = previousSpice;
+    delete require.cache[path];
+  }
+}));