spice-couchbase 2.0.62 → 2.0.64

package/build/connectionHealth.test.js

@@ -0,0 +1,644 @@
+"use strict";
+
+function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
+function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
+function _defineProperty(e, r, t) { return (r = _toPropertyKey(r)) in e ? Object.defineProperty(e, r, { value: t, enumerable: !0, configurable: !0, writable: !0 }) : e[r] = t, e; }
+function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == typeof i ? i : i + ""; }
+function _toPrimitive(t, e) { if ("object" != typeof t || !t) return t; var r; if ("undefined" != typeof Symbol && void 0 !== (r = t[Symbol.toPrimitive])) { var i = r.call(t, e || "default"); if ("object" != typeof i) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === e ? String : Number)(t); }
+function asyncGeneratorStep(n, t, e, r, o, a, c) { try { var i = n[a](c), u = i.value; } catch (n) { return void e(n); } i.done ? t(u) : Promise.resolve(u).then(r, o); }
+function _asyncToGenerator(n) { return function () { var t = this, e = arguments; return new Promise(function (r, o) { var a = n.apply(t, e); function _next(n) { asyncGeneratorStep(a, r, o, _next, _throw, "next", n); } function _throw(n) { asyncGeneratorStep(a, r, o, _next, _throw, "throw", n); } _next(void 0); }); }; }
+var assert = require("node:assert/strict");
+var _require = require("node:test"),
+  test = _require.test;
+var couchbase = require("couchbase");
+var _require2 = require("./connectionHealth"),
+  createHealthMonitor = _require2.createHealthMonitor,
+  buildConnectOptions = _require2.buildConnectOptions,
+  isConnectivityError = _require2.isConnectivityError;
+var config = {
+  nodes: "couchbase://db",
+  bucket: "saturn",
+  scope: "app",
+  username: "u",
+  password: "p"
+};
+var timeoutError = () => new couchbase.UnambiguousTimeoutError();
+
+// A fake cluster whose probe behaviour can be flipped between healthy and
+// wedged. `hang` makes operations never settle, like a stuck SDK.
+function fakeCluster(id) {
+  var _ref = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {},
+    _ref$healthy = _ref.healthy,
+    healthy = _ref$healthy === void 0 ? true : _ref$healthy,
+    _ref$hang = _ref.hang,
+    hang = _ref$hang === void 0 ? false : _ref$hang;
+  var cluster = {
+    id,
+    healthy,
+    hang,
+    closed: 0,
+    queries: 0,
+    kvStates: [couchbase.PingState.Ok],
+    close() {
+      return _asyncToGenerator(function* () {
+        cluster.closed += 1;
+      })();
+    },
+    query() {
+      cluster.queries += 1;
+      if (cluster.hang) return new Promise(() => {});
+      return cluster.healthy ? Promise.resolve({
+        rows: [{
+          $1: 1
+        }]
+      }) : Promise.reject(timeoutError());
+    },
+    bucket(name) {
+      var bucket = {
+        name,
+        cluster,
+        waitUntilReady() {
+          return _asyncToGenerator(function* () {
+            if (!cluster.healthy) throw timeoutError();
+          })();
+        },
+        ping() {
+          if (cluster.hang) return new Promise(() => {});
+          if (!cluster.healthy) return Promise.reject(timeoutError());
+          return Promise.resolve({
+            services: {
+              kv: cluster.kvStates.map(state => ({
+                state
+              }))
+            }
+          });
+        },
+        scope(scopeName) {
+          return {
+            name: scopeName,
+            bucket,
+            collection(collectionName) {
+              return {
+                name: collectionName,
+                cluster,
+                // Like `_default` inside a named scope: the collection does
+                // not exist, so KV operations on it always time out.
+                exists() {
+                  return Promise.reject(timeoutError());
+                }
+              };
+            }
+          };
+        }
+      };
+      return bucket;
+    }
+  };
+  return cluster;
+}
+function handlesFor(cluster) {
+  var bucket = cluster.bucket(config.bucket);
+  var scope = bucket.scope(config.scope);
+  return {
+    cluster,
+    bucket,
+    scope,
+    collection: scope.collection("_default")
+  };
+}
+function setup() {
+  var _ref2 = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {},
+    _ref2$settings = _ref2.settings,
+    settings = _ref2$settings === void 0 ? {} : _ref2$settings,
+    connect = _ref2.connect,
+    clock = _ref2.clock;
+  var initial = fakeCluster("initial");
+  var handles = handlesFor(initial);
+  var globals = {
+    database: {
+      clusters: {
+        default: handles.cluster
+      },
+      buckets: {
+        default: handles.bucket
+      },
+      scopes: {
+        default: handles.scope
+      },
+      connections: {
+        default: handles.collection
+      }
+    },
+    checked_collections: {
+      users: true
+    },
+    db: {
+      collections: new Set(["users"])
+    }
+  };
+  var opened = [];
+  var time = clock || {
+    now: 1000
+  };
+  var terminated = [];
+  var logs = [];
+  var log = {
+    log: m => logs.push(m),
+    warn: m => logs.push(m),
+    error: m => logs.push(m)
+  };
+  var monitor = createHealthMonitor({
+    settings: _objectSpread({
+      healthIntervalMs: 0,
+      probeTimeoutMs: 20,
+      probeFailureThreshold: 2,
+      failureThreshold: 3,
+      failureWindowMs: 1000,
+      reconnectTimeoutMs: 50,
+      reconnectBackoffMinMs: 100,
+      reconnectBackoffMaxMs: 1000,
+      retireGraceMs: 5,
+      exitAfterMs: 10000
+    }, settings),
+    connect: connect || /*#__PURE__*/_asyncToGenerator(function* () {
+      var cluster = fakeCluster("fresh-".concat(opened.length + 1));
+      opened.push(cluster);
+      return cluster;
+    }),
+    now: () => time.now,
+    random: () => 0.5,
+    log,
+    getGlobals: () => globals,
+    terminate: () => terminated.push(time.now)
+  });
+  monitor.register("default", config);
+  return {
+    monitor,
+    initial,
+    globals,
+    opened,
+    time,
+    terminated,
+    logs
+  };
+}
+var sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+test("only connectivity errors count", () => {
+  assert.equal(isConnectivityError(timeoutError()), true);
+  assert.equal(isConnectivityError(new couchbase.AmbiguousTimeoutError()), true);
+  assert.equal(isConnectivityError(new couchbase.RequestCanceledError()), true);
+  assert.equal(isConnectivityError(new couchbase.ServiceNotAvailableError()), true);
+  assert.equal(isConnectivityError(new couchbase.DocumentNotFoundError()), false);
+  assert.equal(isConnectivityError(new Error("syntax error")), false);
+  assert.equal(isConnectivityError(undefined), false);
+});
+test("connect options carry only the timeouts that are set", () => {
+  assert.deepEqual(buildConnectOptions(config), {
+    username: "u",
+    password: "p"
+  });
+  assert.deepEqual(buildConnectOptions(_objectSpread(_objectSpread({}, config), {}, {
+    configProfile: "wanDevelopment",
+    timeouts: {
+      queryTimeout: "30000",
+      kvTimeout: undefined,
+      connectTimeout: ""
+    }
+  })), {
+    username: "u",
+    password: "p",
+    configProfile: "wanDevelopment",
+    timeouts: {
+      queryTimeout: 30000
+    }
+  });
+});
+test("a healthy probe leaves the connection alone", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup = setup(),
+    monitor = _setup.monitor,
+    opened = _setup.opened,
+    globals = _setup.globals,
+    initial = _setup.initial;
+  assert.equal(yield monitor.check("default"), true);
+  assert.equal(opened.length, 0);
+  assert.equal(globals.database.clusters.default, initial);
+  assert.equal(monitor.status().default.healthy, true);
+}));
+test("the probe does not depend on the boot-time collection existing", /*#__PURE__*/_asyncToGenerator(function* () {
+  // The fake collection's KV operations always time out; ping still succeeds.
+  var _setup2 = setup({
+      settings: {
+        probeFailureThreshold: 1
+      }
+    }),
+    monitor = _setup2.monitor,
+    opened = _setup2.opened;
+  assert.equal(yield monitor.check("default"), true);
+  assert.equal(opened.length, 0);
+  assert.equal(monitor.status().default.healthy, true);
+}));
+test("one answering KV node is enough, none is unhealthy", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup3 = setup({
+      settings: {
+        probeFailureThreshold: 1
+      }
+    }),
+    monitor = _setup3.monitor,
+    initial = _setup3.initial,
+    opened = _setup3.opened;
+  var _couchbase$PingState = couchbase.PingState,
+    Ok = _couchbase$PingState.Ok,
+    Timeout = _couchbase$PingState.Timeout,
+    Failed = _couchbase$PingState.Error;
+  initial.kvStates = [Timeout, Ok, Failed];
+  assert.equal(yield monitor.check("default"), true);
+  assert.equal(opened.length, 0);
+  initial.kvStates = [Timeout, Failed];
+  yield monitor.check("default");
+  assert.equal(opened.length, 1);
+}));
+test("probe failures say which check failed", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup4 = setup({
+      settings: {
+        probeFailureThreshold: 1
+      },
+      connect: function () {
+        var _connect = _asyncToGenerator(function* () {
+          return fakeCluster("bad", {
+            healthy: false
+          });
+        });
+        function connect() {
+          return _connect.apply(this, arguments);
+        }
+        return connect;
+      }()
+    }),
+    monitor = _setup4.monitor,
+    initial = _setup4.initial;
+  initial.kvStates = [];
+  yield monitor.check("default");
+  assert.match(monitor.status().default.lastError, /waitUntilReady failed/);
+  var query = setup({
+    settings: {
+      probeFailureThreshold: 1
+    }
+  });
+  query.initial.query = () => Promise.reject(timeoutError());
+  yield query.monitor.check("default");
+  assert.match(query.logs.join("\n"), /unhealthy: Query probe failed/);
+  var kv = setup({
+    settings: {
+      probeFailureThreshold: 1
+    }
+  });
+  kv.initial.kvStates = [];
+  yield kv.monitor.check("default");
+  assert.match(kv.logs.join("\n"), /unhealthy: KV ping failed: no endpoint responded/);
+}));
+test("non-connectivity errors never trigger a probe", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup5 = setup(),
+    monitor = _setup5.monitor,
+    initial = _setup5.initial;
+  for (var i = 0; i < 10; i++) {
+    monitor.recordOutcome("default", new couchbase.DocumentNotFoundError());
+  }
+  yield sleep(5);
+  assert.equal(initial.queries, 0);
+}));
+test("one failed probe is not enough, two trigger a reconnect", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup6 = setup(),
+    monitor = _setup6.monitor,
+    initial = _setup6.initial,
+    opened = _setup6.opened;
+  initial.healthy = false;
+  yield monitor.check("default");
+  assert.equal(opened.length, 0);
+  assert.equal(monitor.status().default.healthy, true);
+  yield monitor.check("default");
+  assert.equal(opened.length, 1);
+  assert.equal(monitor.status().default.healthy, true);
+  assert.equal(monitor.status().default.generation, 2);
+}));
+test("a burst of operation timeouts probes and reconnects at once", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup7 = setup(),
+    monitor = _setup7.monitor,
+    initial = _setup7.initial,
+    opened = _setup7.opened,
+    globals = _setup7.globals;
+  initial.healthy = false;
+  for (var i = 0; i < 3; i++) monitor.recordOutcome("default", timeoutError());
+  yield sleep(10);
+  assert.equal(opened.length, 1);
+  assert.equal(globals.database.clusters.default, opened[0]);
+}));
+test("timeouts outside the window do not add up", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup8 = setup(),
+    monitor = _setup8.monitor,
+    initial = _setup8.initial,
+    opened = _setup8.opened,
+    time = _setup8.time;
+  initial.healthy = false;
+  monitor.recordOutcome("default", timeoutError());
+  monitor.recordOutcome("default", timeoutError());
+  time.now += 2000;
+  monitor.recordOutcome("default", timeoutError());
+  yield sleep(10);
+  assert.equal(initial.queries, 0);
+  assert.equal(opened.length, 0);
+}));
+test("reconnect swaps every global handle and resets collection caches", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup9 = setup(),
+    monitor = _setup9.monitor,
+    initial = _setup9.initial,
+    opened = _setup9.opened,
+    globals = _setup9.globals;
+  initial.healthy = false;
+  yield monitor.check("default");
+  yield monitor.check("default");
+  var fresh = opened[0];
+  assert.equal(globals.database.clusters.default, fresh);
+  assert.equal(globals.database.buckets.default.cluster, fresh);
+  assert.equal(globals.database.scopes.default.bucket.cluster, fresh);
+  assert.equal(globals.database.connections.default.cluster, fresh);
+  assert.deepEqual(globals.checked_collections, {});
+  assert.equal(globals.db.collections.size, 0);
+}));
+test("the replaced cluster is closed after the grace period", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup0 = setup(),
+    monitor = _setup0.monitor,
+    initial = _setup0.initial;
+  initial.healthy = false;
+  yield monitor.check("default");
+  yield monitor.check("default");
+  assert.equal(initial.closed, 0);
+  yield sleep(20);
+  assert.equal(initial.closed, 1);
+}));
+test("concurrent triggers share a single reconnect", /*#__PURE__*/_asyncToGenerator(function* () {
+  var release;
+  var opened = [];
+  var _setup1 = setup({
+      settings: {
+        probeFailureThreshold: 1,
+        reconnectTimeoutMs: 1000
+      },
+      connect: () => new Promise(resolve => {
+        release = () => {
+          var cluster = fakeCluster("fresh");
+          opened.push(cluster);
+          resolve(cluster);
+        };
+      })
+    }),
+    monitor = _setup1.monitor,
+    initial = _setup1.initial;
+  initial.healthy = false;
+  var first = monitor.check("default");
+  yield sleep(5);
+  var others = [monitor.check("default"), monitor.reconnect("default")];
+  for (var i = 0; i < 5; i++) monitor.recordOutcome("default", timeoutError());
+  release();
+  yield Promise.all([first, ...others]);
+  assert.equal(opened.length, 1);
+  assert.equal(monitor.status().default.generation, 2);
+}));
+test("a probe that hangs is cut off by the hard deadline", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup10 = setup({
+      settings: {
+        probeFailureThreshold: 1
+      }
+    }),
+    monitor = _setup10.monitor,
+    initial = _setup10.initial,
+    opened = _setup10.opened;
+  initial.hang = true;
+  // The deadline timer is unref'd; keep the loop alive while the probe hangs.
+  var keepAlive = setInterval(() => {}, 1000);
+  try {
+    var started = Date.now();
+    yield monitor.check("default");
+    assert.ok(Date.now() - started < 1500);
+    assert.equal(opened.length, 1);
+  } finally {
+    clearInterval(keepAlive);
+  }
+}));
+test("a failed reconnect backs off and keeps the old cluster", /*#__PURE__*/_asyncToGenerator(function* () {
+  var opened = [];
+  var _setup11 = setup({
+      settings: {
+        probeFailureThreshold: 1
+      },
+      connect: function () {
+        var _connect2 = _asyncToGenerator(function* () {
+          var cluster = fakeCluster("bad-".concat(opened.length + 1), {
+            healthy: false
+          });
+          opened.push(cluster);
+          return cluster;
+        });
+        function connect() {
+          return _connect2.apply(this, arguments);
+        }
+        return connect;
+      }()
+    }),
+    monitor = _setup11.monitor,
+    initial = _setup11.initial,
+    globals = _setup11.globals,
+    time = _setup11.time;
+  initial.healthy = false;
+  yield monitor.check("default");
+  assert.equal(opened.length, 1);
+  assert.equal(opened[0].closed, 1, "unverified cluster is closed");
+  assert.equal(globals.database.clusters.default, initial);
+  assert.equal(monitor.status().default.reconnectAttempts, 1);
+
+  // Inside the backoff window: probe only, no new connect.
+  yield monitor.check("default");
+  assert.equal(opened.length, 1);
+  time.now += 150;
+  yield monitor.check("default");
+  assert.equal(opened.length, 2);
+  assert.equal(monitor.status().default.reconnectAttempts, 2);
+}));
+test("a connection that recovers on its own is marked healthy again", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup12 = setup({
+      settings: {
+        probeFailureThreshold: 1
+      },
+      connect: function () {
+        var _connect3 = _asyncToGenerator(function* () {
+          return fakeCluster("bad", {
+            healthy: false
+          });
+        });
+        function connect() {
+          return _connect3.apply(this, arguments);
+        }
+        return connect;
+      }()
+    }),
+    monitor = _setup12.monitor,
+    initial = _setup12.initial,
+    time = _setup12.time;
+  initial.healthy = false;
+  yield monitor.check("default");
+  assert.equal(monitor.status().default.healthy, false);
+  initial.healthy = true;
+  time.now += 5000;
+  yield monitor.check("default");
+  var status = monitor.status().default;
+  assert.equal(status.healthy, true);
+  assert.equal(status.reconnectAttempts, 0);
+  assert.equal(status.unhealthySince, null);
+}));
+test("staying unhealthy past the limit terminates the process once", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup13 = setup({
+      settings: {
+        probeFailureThreshold: 1,
+        exitAfterMs: 1000
+      },
+      connect: function () {
+        var _connect4 = _asyncToGenerator(function* () {
+          return fakeCluster("bad", {
+            healthy: false
+          });
+        });
+        function connect() {
+          return _connect4.apply(this, arguments);
+        }
+        return connect;
+      }()
+    }),
+    monitor = _setup13.monitor,
+    initial = _setup13.initial,
+    time = _setup13.time,
+    terminated = _setup13.terminated;
+  initial.healthy = false;
+  yield monitor.check("default");
+  assert.equal(terminated.length, 0);
+  time.now += 1500;
+  yield monitor.check("default");
+  time.now += 1500;
+  yield monitor.check("default");
+  assert.equal(terminated.length, 1);
+}));
+test("exitAfterMs of 0 never terminates", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup14 = setup({
+      settings: {
+        probeFailureThreshold: 1,
+        exitAfterMs: 0
+      },
+      connect: function () {
+        var _connect5 = _asyncToGenerator(function* () {
+          return fakeCluster("bad", {
+            healthy: false
+          });
+        });
+        function connect() {
+          return _connect5.apply(this, arguments);
+        }
+        return connect;
+      }()
+    }),
+    monitor = _setup14.monitor,
+    initial = _setup14.initial,
+    time = _setup14.time,
+    terminated = _setup14.terminated;
+  initial.healthy = false;
+  for (var i = 0; i < 5; i++) {
+    time.now += 100000;
+    yield monitor.check("default");
+  }
+  assert.equal(terminated.length, 0);
+}));
+test("stop() halts reconnects and closes retired clusters", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup15 = setup({
+      settings: {
+        probeFailureThreshold: 1,
+        retireGraceMs: 60000
+      }
+    }),
+    monitor = _setup15.monitor,
+    initial = _setup15.initial,
+    opened = _setup15.opened;
+  initial.healthy = false;
+  yield monitor.check("default");
+  assert.equal(opened.length, 1);
+  yield monitor.stop();
+  assert.equal(initial.closed, 1);
+  opened[0].healthy = false;
+  yield monitor.check("default");
+  yield monitor.reconnect("default");
+  assert.equal(opened.length, 1);
+}));
+test("the interval probe runs when enabled", /*#__PURE__*/_asyncToGenerator(function* () {
+  var _setup16 = setup({
+      settings: {
+        healthIntervalMs: 5,
+        probeFailureThreshold: 1
+      }
+    }),
+    monitor = _setup16.monitor,
+    initial = _setup16.initial,
+    opened = _setup16.opened;
+  initial.healthy = false;
+  yield sleep(40);
+  assert.ok(opened.length >= 1);
+  yield monitor.stop();
+}));
+test("a Database instance follows the swapped globals", /*#__PURE__*/_asyncToGenerator(function* () {
+  var path = require.resolve("./index");
+  delete require.cache[path];
+  var previousSpice = global.spice;
+  var _setup17 = setup({
+      settings: {
+        probeFailureThreshold: 1
+      }
+    }),
+    monitor = _setup17.monitor,
+    initial = _setup17.initial,
+    opened = _setup17.opened,
+    globals = _setup17.globals;
+  global.spice = _objectSpread(_objectSpread({}, globals), {}, {
+    config: {
+      database: {
+        connections: {
+          default: _objectSpread({}, config)
+        }
+      }
+    }
+  });
+  // Point the shared module-level monitor at this test's monitor.
+  var key = Symbol.for("spice-couchbase.health-monitor");
+  var previousMonitor = globalThis[key];
+  globalThis[key] = monitor;
+  try {
+    // Keep a reference to the same globals object the monitor mutates.
+    global.spice.database = globals.database;
+    // The constructor reads the bucket name off the boot-time collection.
+    globals.database.connections.default._scope = {
+      _bucket: {
+        _name: "saturn"
+      }
+    };
+    var Database = require("./index");
+    var db = new Database("default");
+    assert.equal(db.cluster, initial);
+    var oldCollection = db.collection;
+    assert.equal(oldCollection.cluster, initial);
+    initial.healthy = false;
+    yield monitor.check("default");
+    assert.equal(opened.length, 1);
+    assert.equal(db.cluster, opened[0]);
+    assert.equal(db.collection.cluster, opened[0]);
+    assert.notEqual(db.collection, oldCollection);
+    assert.deepEqual(Database.healthStatus().default.generation, 2);
+  } finally {
+    globalThis[key] = previousMonitor;
+    global.spice = previousSpice;
+    delete require.cache[path];
+  }
+}));