spice-couchbase 2.0.62 → 2.0.64

package/src/connectionHealth.test.js

@@ -0,0 +1,461 @@
+"use strict";
+
+const assert = require("node:assert/strict");
+const { test } = require("node:test");
+const couchbase = require("couchbase");
+const {
+  createHealthMonitor,
+  buildConnectOptions,
+  isConnectivityError,
+} = require("./connectionHealth");
+
+const config = {
+  nodes: "couchbase://db",
+  bucket: "saturn",
+  scope: "app",
+  username: "u",
+  password: "p",
+};
+
+const timeoutError = () => new couchbase.UnambiguousTimeoutError();
+
+// A fake cluster whose probe behaviour can be flipped between healthy and
+// wedged. `hang` makes operations never settle, like a stuck SDK.
+function fakeCluster(id, { healthy = true, hang = false } = {}) {
+  const cluster = {
+    id,
+    healthy,
+    hang,
+    closed: 0,
+    queries: 0,
+    kvStates: [couchbase.PingState.Ok],
+    async close() {
+      cluster.closed += 1;
+    },
+    query() {
+      cluster.queries += 1;
+      if (cluster.hang) return new Promise(() => {});
+      return cluster.healthy ?
+          Promise.resolve({ rows: [{ $1: 1 }] })
+        : Promise.reject(timeoutError());
+    },
+    bucket(name) {
+      const bucket = {
+        name,
+        cluster,
+        async waitUntilReady() {
+          if (!cluster.healthy) throw timeoutError();
+        },
+        ping() {
+          if (cluster.hang) return new Promise(() => {});
+          if (!cluster.healthy) return Promise.reject(timeoutError());
+          return Promise.resolve({
+            services: { kv: cluster.kvStates.map((state) => ({ state })) },
+          });
+        },
+        scope(scopeName) {
+          return {
+            name: scopeName,
+            bucket,
+            collection(collectionName) {
+              return {
+                name: collectionName,
+                cluster,
+                // Like `_default` inside a named scope: the collection does
+                // not exist, so KV operations on it always time out.
+                exists() {
+                  return Promise.reject(timeoutError());
+                },
+              };
+            },
+          };
+        },
+      };
+      return bucket;
+    },
+  };
+  return cluster;
+}
+
+function handlesFor(cluster) {
+  const bucket = cluster.bucket(config.bucket);
+  const scope = bucket.scope(config.scope);
+  return { cluster, bucket, scope, collection: scope.collection("_default") };
+}
+
+function setup({ settings = {}, connect, clock } = {}) {
+  const initial = fakeCluster("initial");
+  const handles = handlesFor(initial);
+  const globals = {
+    database: {
+      clusters: { default: handles.cluster },
+      buckets: { default: handles.bucket },
+      scopes: { default: handles.scope },
+      connections: { default: handles.collection },
+    },
+    checked_collections: { users: true },
+    db: { collections: new Set(["users"]) },
+  };
+  const opened = [];
+  const time = clock || { now: 1000 };
+  const terminated = [];
+  const logs = [];
+  const log = {
+    log: (m) => logs.push(m),
+    warn: (m) => logs.push(m),
+    error: (m) => logs.push(m),
+  };
+  const monitor = createHealthMonitor({
+    settings: {
+      healthIntervalMs: 0,
+      probeTimeoutMs: 20,
+      probeFailureThreshold: 2,
+      failureThreshold: 3,
+      failureWindowMs: 1000,
+      reconnectTimeoutMs: 50,
+      reconnectBackoffMinMs: 100,
+      reconnectBackoffMaxMs: 1000,
+      retireGraceMs: 5,
+      exitAfterMs: 10000,
+      ...settings,
+    },
+    connect:
+      connect ||
+      (async () => {
+        const cluster = fakeCluster(`fresh-${opened.length + 1}`);
+        opened.push(cluster);
+        return cluster;
+      }),
+    now: () => time.now,
+    random: () => 0.5,
+    log,
+    getGlobals: () => globals,
+    terminate: () => terminated.push(time.now),
+  });
+  monitor.register("default", config);
+  return { monitor, initial, globals, opened, time, terminated, logs };
+}
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+test("only connectivity errors count", () => {
+  assert.equal(isConnectivityError(timeoutError()), true);
+  assert.equal(isConnectivityError(new couchbase.AmbiguousTimeoutError()), true);
+  assert.equal(isConnectivityError(new couchbase.RequestCanceledError()), true);
+  assert.equal(isConnectivityError(new couchbase.ServiceNotAvailableError()), true);
+  assert.equal(isConnectivityError(new couchbase.DocumentNotFoundError()), false);
+  assert.equal(isConnectivityError(new Error("syntax error")), false);
+  assert.equal(isConnectivityError(undefined), false);
+});
+
+test("connect options carry only the timeouts that are set", () => {
+  assert.deepEqual(buildConnectOptions(config), { username: "u", password: "p" });
+  assert.deepEqual(
+    buildConnectOptions({
+      ...config,
+      configProfile: "wanDevelopment",
+      timeouts: { queryTimeout: "30000", kvTimeout: undefined, connectTimeout: "" },
+    }),
+    {
+      username: "u",
+      password: "p",
+      configProfile: "wanDevelopment",
+      timeouts: { queryTimeout: 30000 },
+    },
+  );
+});
+
+test("a healthy probe leaves the connection alone", async () => {
+  const { monitor, opened, globals, initial } = setup();
+  assert.equal(await monitor.check("default"), true);
+  assert.equal(opened.length, 0);
+  assert.equal(globals.database.clusters.default, initial);
+  assert.equal(monitor.status().default.healthy, true);
+});
+
+test("the probe does not depend on the boot-time collection existing", async () => {
+  // The fake collection's KV operations always time out; ping still succeeds.
+  const { monitor, opened } = setup({ settings: { probeFailureThreshold: 1 } });
+  assert.equal(await monitor.check("default"), true);
+  assert.equal(opened.length, 0);
+  assert.equal(monitor.status().default.healthy, true);
+});
+
+test("one answering KV node is enough, none is unhealthy", async () => {
+  const { monitor, initial, opened } = setup({
+    settings: { probeFailureThreshold: 1 },
+  });
+  const { Ok, Timeout, Error: Failed } = couchbase.PingState;
+  initial.kvStates = [Timeout, Ok, Failed];
+  assert.equal(await monitor.check("default"), true);
+  assert.equal(opened.length, 0);
+
+  initial.kvStates = [Timeout, Failed];
+  await monitor.check("default");
+  assert.equal(opened.length, 1);
+});
+
+test("probe failures say which check failed", async () => {
+  const { monitor, initial } = setup({
+    settings: { probeFailureThreshold: 1 },
+    connect: async () => fakeCluster("bad", { healthy: false }),
+  });
+  initial.kvStates = [];
+  await monitor.check("default");
+  assert.match(monitor.status().default.lastError, /waitUntilReady failed/);
+
+  const query = setup({ settings: { probeFailureThreshold: 1 } });
+  query.initial.query = () => Promise.reject(timeoutError());
+  await query.monitor.check("default");
+  assert.match(query.logs.join("\n"), /unhealthy: Query probe failed/);
+
+  const kv = setup({ settings: { probeFailureThreshold: 1 } });
+  kv.initial.kvStates = [];
+  await kv.monitor.check("default");
+  assert.match(kv.logs.join("\n"), /unhealthy: KV ping failed: no endpoint responded/);
+});
+
+test("non-connectivity errors never trigger a probe", async () => {
+  const { monitor, initial } = setup();
+  for (let i = 0; i < 10; i++) {
+    monitor.recordOutcome("default", new couchbase.DocumentNotFoundError());
+  }
+  await sleep(5);
+  assert.equal(initial.queries, 0);
+});
+
+test("one failed probe is not enough, two trigger a reconnect", async () => {
+  const { monitor, initial, opened } = setup();
+  initial.healthy = false;
+  await monitor.check("default");
+  assert.equal(opened.length, 0);
+  assert.equal(monitor.status().default.healthy, true);
+  await monitor.check("default");
+  assert.equal(opened.length, 1);
+  assert.equal(monitor.status().default.healthy, true);
+  assert.equal(monitor.status().default.generation, 2);
+});
+
+test("a burst of operation timeouts probes and reconnects at once", async () => {
+  const { monitor, initial, opened, globals } = setup();
+  initial.healthy = false;
+  for (let i = 0; i < 3; i++) monitor.recordOutcome("default", timeoutError());
+  await sleep(10);
+  assert.equal(opened.length, 1);
+  assert.equal(globals.database.clusters.default, opened[0]);
+});
+
+test("timeouts outside the window do not add up", async () => {
+  const { monitor, initial, opened, time } = setup();
+  initial.healthy = false;
+  monitor.recordOutcome("default", timeoutError());
+  monitor.recordOutcome("default", timeoutError());
+  time.now += 2000;
+  monitor.recordOutcome("default", timeoutError());
+  await sleep(10);
+  assert.equal(initial.queries, 0);
+  assert.equal(opened.length, 0);
+});
+
+test("reconnect swaps every global handle and resets collection caches", async () => {
+  const { monitor, initial, opened, globals } = setup();
+  initial.healthy = false;
+  await monitor.check("default");
+  await monitor.check("default");
+  const fresh = opened[0];
+  assert.equal(globals.database.clusters.default, fresh);
+  assert.equal(globals.database.buckets.default.cluster, fresh);
+  assert.equal(globals.database.scopes.default.bucket.cluster, fresh);
+  assert.equal(globals.database.connections.default.cluster, fresh);
+  assert.deepEqual(globals.checked_collections, {});
+  assert.equal(globals.db.collections.size, 0);
+});
+
+test("the replaced cluster is closed after the grace period", async () => {
+  const { monitor, initial } = setup();
+  initial.healthy = false;
+  await monitor.check("default");
+  await monitor.check("default");
+  assert.equal(initial.closed, 0);
+  await sleep(20);
+  assert.equal(initial.closed, 1);
+});
+
+test("concurrent triggers share a single reconnect", async () => {
+  let release;
+  const opened = [];
+  const { monitor, initial } = setup({
+    settings: { probeFailureThreshold: 1, reconnectTimeoutMs: 1000 },
+    connect: () =>
+      new Promise((resolve) => {
+        release = () => {
+          const cluster = fakeCluster("fresh");
+          opened.push(cluster);
+          resolve(cluster);
+        };
+      }),
+  });
+  initial.healthy = false;
+  const first = monitor.check("default");
+  await sleep(5);
+  const others = [monitor.check("default"), monitor.reconnect("default")];
+  for (let i = 0; i < 5; i++) monitor.recordOutcome("default", timeoutError());
+  release();
+  await Promise.all([first, ...others]);
+  assert.equal(opened.length, 1);
+  assert.equal(monitor.status().default.generation, 2);
+});
+
+test("a probe that hangs is cut off by the hard deadline", async () => {
+  const { monitor, initial, opened } = setup({
+    settings: { probeFailureThreshold: 1 },
+  });
+  initial.hang = true;
+  // The deadline timer is unref'd; keep the loop alive while the probe hangs.
+  const keepAlive = setInterval(() => {}, 1000);
+  try {
+    const started = Date.now();
+    await monitor.check("default");
+    assert.ok(Date.now() - started < 1500);
+    assert.equal(opened.length, 1);
+  } finally {
+    clearInterval(keepAlive);
+  }
+});
+
+test("a failed reconnect backs off and keeps the old cluster", async () => {
+  const opened = [];
+  const { monitor, initial, globals, time } = setup({
+    settings: { probeFailureThreshold: 1 },
+    connect: async () => {
+      const cluster = fakeCluster(`bad-${opened.length + 1}`, { healthy: false });
+      opened.push(cluster);
+      return cluster;
+    },
+  });
+  initial.healthy = false;
+  await monitor.check("default");
+  assert.equal(opened.length, 1);
+  assert.equal(opened[0].closed, 1, "unverified cluster is closed");
+  assert.equal(globals.database.clusters.default, initial);
+  assert.equal(monitor.status().default.reconnectAttempts, 1);
+
+  // Inside the backoff window: probe only, no new connect.
+  await monitor.check("default");
+  assert.equal(opened.length, 1);
+
+  time.now += 150;
+  await monitor.check("default");
+  assert.equal(opened.length, 2);
+  assert.equal(monitor.status().default.reconnectAttempts, 2);
+});
+
+test("a connection that recovers on its own is marked healthy again", async () => {
+  const { monitor, initial, time } = setup({
+    settings: { probeFailureThreshold: 1 },
+    connect: async () => fakeCluster("bad", { healthy: false }),
+  });
+  initial.healthy = false;
+  await monitor.check("default");
+  assert.equal(monitor.status().default.healthy, false);
+  initial.healthy = true;
+  time.now += 5000;
+  await monitor.check("default");
+  const status = monitor.status().default;
+  assert.equal(status.healthy, true);
+  assert.equal(status.reconnectAttempts, 0);
+  assert.equal(status.unhealthySince, null);
+});
+
+test("staying unhealthy past the limit terminates the process once", async () => {
+  const { monitor, initial, time, terminated } = setup({
+    settings: { probeFailureThreshold: 1, exitAfterMs: 1000 },
+    connect: async () => fakeCluster("bad", { healthy: false }),
+  });
+  initial.healthy = false;
+  await monitor.check("default");
+  assert.equal(terminated.length, 0);
+  time.now += 1500;
+  await monitor.check("default");
+  time.now += 1500;
+  await monitor.check("default");
+  assert.equal(terminated.length, 1);
+});
+
+test("exitAfterMs of 0 never terminates", async () => {
+  const { monitor, initial, time, terminated } = setup({
+    settings: { probeFailureThreshold: 1, exitAfterMs: 0 },
+    connect: async () => fakeCluster("bad", { healthy: false }),
+  });
+  initial.healthy = false;
+  for (let i = 0; i < 5; i++) {
+    time.now += 100000;
+    await monitor.check("default");
+  }
+  assert.equal(terminated.length, 0);
+});
+
+test("stop() halts reconnects and closes retired clusters", async () => {
+  const { monitor, initial, opened } = setup({
+    settings: { probeFailureThreshold: 1, retireGraceMs: 60000 },
+  });
+  initial.healthy = false;
+  await monitor.check("default");
+  assert.equal(opened.length, 1);
+  await monitor.stop();
+  assert.equal(initial.closed, 1);
+  opened[0].healthy = false;
+  await monitor.check("default");
+  await monitor.reconnect("default");
+  assert.equal(opened.length, 1);
+});
+
+test("the interval probe runs when enabled", async () => {
+  const { monitor, initial, opened } = setup({
+    settings: { healthIntervalMs: 5, probeFailureThreshold: 1 },
+  });
+  initial.healthy = false;
+  await sleep(40);
+  assert.ok(opened.length >= 1);
+  await monitor.stop();
+});
+
+test("a Database instance follows the swapped globals", async () => {
+  const path = require.resolve("./index");
+  delete require.cache[path];
+  const previousSpice = global.spice;
+  const { monitor, initial, opened, globals } = setup({
+    settings: { probeFailureThreshold: 1 },
+  });
+  global.spice = {
+    ...globals,
+    config: { database: { connections: { default: { ...config } } } },
+  };
+  // Point the shared module-level monitor at this test's monitor.
+  const key = Symbol.for("spice-couchbase.health-monitor");
+  const previousMonitor = globalThis[key];
+  globalThis[key] = monitor;
+  try {
+    // Keep a reference to the same globals object the monitor mutates.
+    global.spice.database = globals.database;
+    // The constructor reads the bucket name off the boot-time collection.
+    globals.database.connections.default._scope = { _bucket: { _name: "saturn" } };
+    const Database = require("./index");
+    const db = new Database("default");
+    assert.equal(db.cluster, initial);
+    const oldCollection = db.collection;
+    assert.equal(oldCollection.cluster, initial);
+
+    initial.healthy = false;
+    await monitor.check("default");
+    assert.equal(opened.length, 1);
+    assert.equal(db.cluster, opened[0]);
+    assert.equal(db.collection.cluster, opened[0]);
+    assert.notEqual(db.collection, oldCollection);
+    assert.deepEqual(Database.healthStatus().default.generation, 2);
+  } finally {
+    globalThis[key] = previousMonitor;
+    global.spice = previousSpice;
+    delete require.cache[path];
+  }
+});