spice-couchbase 2.0.61 → 2.0.63

package/src/connectionHealth.js

@@ -0,0 +1,529 @@
+"use strict";
+
+const couchbase = require("couchbase");
+
+// Connection health monitor and self-healing reconnect.
+//
+// The cluster object is created once at boot and stored on the `spice.database`
+// globals. When the SDK wedges (every operation times out, but the SDK never
+// recovers on its own) the only remedy used to be a process restart. This
+// module notices that state and opens a fresh cluster in-process:
+//
+//   1. Passive signal: every driver operation reports its outcome. Enough
+//      connectivity errors (timeouts, cancelled requests, service not
+//      available) inside a sliding window trigger a probe straight away.
+//   2. Active probe: on an interval, a KV `exists` and a `SELECT 1`, each with
+//      a short timeout and a hard deadline in case the SDK ignores it.
+//   3. Reconnect: after repeated probe failures, build a new cluster from the
+//      same config, verify it, then swap it into the globals and retire the old
+//      one after a grace period. Single-flight, with exponential backoff.
+//   4. Escalation: if the connection stays unhealthy for too long, terminate
+//      the process so the supervisor (PM2) restarts it.
+
+function envNumber(name, fallback) {
+  const raw = process.env[name];
+  if (raw === undefined || raw === null || String(raw).trim() === "") {
+    return fallback;
+  }
+  const value = Number(raw);
+  return Number.isFinite(value) && value >= 0 ? value : fallback;
+}
+
+function defaultSettings() {
+  return {
+    // 0 disables the interval probe (the passive signal still works).
+    healthIntervalMs: envNumber("SPICE_COUCHBASE_HEALTH_INTERVAL_MS", 15000),
+    probeTimeoutMs: envNumber("SPICE_COUCHBASE_PROBE_TIMEOUT_MS", 3000),
+    // Consecutive failed probes before the connection is declared unhealthy.
+    probeFailureThreshold: Math.max(
+      1,
+      envNumber("SPICE_COUCHBASE_PROBE_FAILURE_THRESHOLD", 2),
+    ),
+    // Connectivity errors inside the window that trigger an immediate probe.
+    failureThreshold: Math.max(
+      1,
+      envNumber("SPICE_COUCHBASE_FAILURE_THRESHOLD", 5),
+    ),
+    failureWindowMs: envNumber("SPICE_COUCHBASE_FAILURE_WINDOW_MS", 30000),
+    reconnectTimeoutMs: envNumber("SPICE_COUCHBASE_RECONNECT_TIMEOUT_MS", 20000),
+    reconnectBackoffMinMs: envNumber(
+      "SPICE_COUCHBASE_RECONNECT_BACKOFF_MIN_MS",
+      5000,
+    ),
+    reconnectBackoffMaxMs: envNumber(
+      "SPICE_COUCHBASE_RECONNECT_BACKOFF_MAX_MS",
+      60000,
+    ),
+    // How long a replaced cluster stays open so in-flight operations settle.
+    retireGraceMs: envNumber("SPICE_COUCHBASE_RETIRE_GRACE_MS", 30000),
+    // 0 disables the escalation.
+    exitAfterMs: envNumber("SPICE_COUCHBASE_EXIT_AFTER_MS", 300000),
+  };
+}
+
+const CONNECTIVITY_ERRORS = [
+  couchbase.TimeoutError,
+  couchbase.RequestCanceledError,
+  couchbase.ServiceNotAvailableError,
+].filter((type) => typeof type === "function");
+
+function isConnectivityError(error) {
+  if (!error || typeof error !== "object") return false;
+  if (CONNECTIVITY_ERRORS.some((type) => error instanceof type)) return true;
+  // Our own deadlines, and SDK errors that crossed a module boundary.
+  return (
+    error.spiceConnectivity === true ||
+    [
+      "TimeoutError",
+      "AmbiguousTimeoutError",
+      "UnambiguousTimeoutError",
+      "RequestCanceledError",
+      "ServiceNotAvailableError",
+    ].includes(error.name)
+  );
+}
+
+function deadline(promise, ms, label) {
+  let timer;
+  return Promise.race([
+    promise,
+    new Promise((_resolve, reject) => {
+      timer = setTimeout(() => {
+        const error = new Error(`${label} timed out after ${ms}ms`);
+        error.spiceConnectivity = true;
+        reject(error);
+      }, ms);
+      timer.unref?.();
+    }),
+  ]).finally(() => clearTimeout(timer));
+}
+
+function describe(error) {
+  return error?.message || String(error);
+}
+
+async function closeQuietly(cluster, log, label) {
+  try {
+    await cluster?.close?.();
+  } catch (error) {
+    log.warn(`[spice-couchbase] Closing ${label} failed: ${describe(error)}`);
+  }
+}
+
+function buildConnectOptions(config = {}) {
+  const options = {
+    username: config.username,
+    password: config.password,
+  };
+  if (config.configProfile) {
+    options.configProfile = config.configProfile;
+  }
+  const timeouts = {};
+  for (const key of [
+    "kvTimeout",
+    "kvDurableTimeout",
+    "queryTimeout",
+    "searchTimeout",
+    "analyticsTimeout",
+    "managementTimeout",
+    "connectTimeout",
+    "bootstrapTimeout",
+    "resolveTimeout",
+  ]) {
+    const raw = config.timeouts?.[key];
+    if (raw === undefined || raw === null || raw === "") continue;
+    const value = Number(raw);
+    if (Number.isFinite(value) && value > 0) timeouts[key] = value;
+  }
+  if (Object.keys(timeouts).length > 0) {
+    options.timeouts = timeouts;
+  }
+  return options;
+}
+
+/**
+ * Opens a cluster and resolves the bucket, scope and collection handles for a
+ * connection config. Shared by the boot-time connect and the reconnect path.
+ */
+async function openHandles(config, { connect = couchbase.connect } = {}) {
+  const cluster = await connect(config.nodes, buildConnectOptions(config));
+  const bucket = cluster.bucket(config.bucket);
+  const scope = bucket.scope(config.scope || "_default");
+  const collection = scope.collection(config.collection || "_default");
+  return { cluster, bucket, scope, collection };
+}
+
+function createHealthMonitor({
+  settings: overrides = {},
+  connect = couchbase.connect,
+  now = () => Date.now(),
+  random = Math.random,
+  log = console,
+  getGlobals = () => (typeof spice !== "undefined" ? spice : undefined),
+  terminate,
+} = {}) {
+  const settings = { ...defaultSettings(), ...overrides };
+  const states = new Map();
+  let stopped = false;
+
+  const terminateProcess =
+    terminate ||
+    (() => {
+      // SIGTERM runs the application's graceful shutdown when it has one,
+      // and terminates the process when it does not. The fallback covers a
+      // shutdown that hangs on the very connection that is stuck.
+      process.exitCode = 1;
+      const fallback = setTimeout(() => process.exit(1), 15000);
+      fallback.unref?.();
+      process.kill(process.pid, "SIGTERM");
+    });
+
+  function stateFor(name) {
+    let state = states.get(name);
+    if (!state) {
+      state = {
+        name,
+        config: null,
+        generation: 1,
+        healthy: true,
+        unhealthySince: null,
+        failures: [],
+        probeFailures: 0,
+        probing: null,
+        reconnecting: null,
+        reconnectAttempts: 0,
+        nextReconnectAt: 0,
+        lastError: null,
+        lastReconnectAt: null,
+        timer: null,
+        retired: new Set(),
+        terminated: false,
+      };
+      states.set(name, state);
+    }
+    return state;
+  }
+
+  function currentHandles(name) {
+    const database = getGlobals()?.database;
+    if (!database) return null;
+    return {
+      cluster: database.clusters?.[name],
+      bucket: database.buckets?.[name],
+      scope: database.scopes?.[name],
+      collection: database.connections?.[name],
+    };
+  }
+
+  async function probe(handles) {
+    if (!handles?.cluster || !handles?.collection) {
+      const error = new Error("No cluster handles to probe");
+      error.spiceConnectivity = true;
+      throw error;
+    }
+    const timeout = settings.probeTimeoutMs;
+    // The hard deadline is the point: a wedged SDK may never honour its own.
+    await deadline(
+      Promise.all([
+        handles.collection.exists("__spice_health_probe__", { timeout }),
+        handles.cluster.query("SELECT 1", { timeout }),
+      ]),
+      timeout + 1000,
+      "Couchbase health probe",
+    );
+  }
+
+  function markHealthy(state) {
+    state.probeFailures = 0;
+    state.failures = [];
+    state.reconnectAttempts = 0;
+    state.nextReconnectAt = 0;
+    if (!state.healthy) {
+      const downFor = state.unhealthySince ? now() - state.unhealthySince : 0;
+      log.log(
+        `[spice-couchbase] Connection "${state.name}" healthy again ` +
+          `(generation ${state.generation}, unhealthy for ${downFor}ms)`,
+      );
+    }
+    state.healthy = true;
+    state.unhealthySince = null;
+    state.lastError = null;
+  }
+
+  function markUnhealthy(state, error) {
+    state.lastError = describe(error);
+    if (state.healthy) {
+      state.healthy = false;
+      state.unhealthySince = now();
+      log.warn(
+        `[spice-couchbase] Connection "${state.name}" unhealthy: ${state.lastError}`,
+      );
+    }
+  }
+
+  function escalateIfStuck(state) {
+    if (
+      state.terminated ||
+      stopped ||
+      !settings.exitAfterMs ||
+      !state.unhealthySince ||
+      now() - state.unhealthySince < settings.exitAfterMs
+    ) {
+      return;
+    }
+    state.terminated = true;
+    log.error(
+      `[spice-couchbase] Connection "${state.name}" has been unhealthy for ` +
+        `${now() - state.unhealthySince}ms after ${state.reconnectAttempts} ` +
+        `failed reconnect(s). Terminating so the supervisor restarts the process.`,
+    );
+    terminateProcess();
+  }
+
+  function retire(state, cluster) {
+    if (!cluster) return;
+    state.retired.add(cluster);
+    const timer = setTimeout(() => {
+      state.retired.delete(cluster);
+      closeQuietly(cluster, log, `retired cluster for "${state.name}"`);
+    }, settings.retireGraceMs);
+    timer.unref?.();
+  }
+
+  function backoffDelay(attempt) {
+    const base = Math.min(
+      settings.reconnectBackoffMaxMs,
+      settings.reconnectBackoffMinMs * 2 ** Math.max(0, attempt - 1),
+    );
+    // +/- 20% jitter so a fleet of pods does not reconnect in lockstep.
+    return Math.round(base * (0.8 + random() * 0.4));
+  }
+
+  async function doReconnect(state) {
+    const started = now();
+    const attempt = state.reconnectAttempts + 1;
+    log.warn(
+      `[spice-couchbase] Reconnecting "${state.name}" (attempt ${attempt})`,
+    );
+    let abandoned = false;
+    let handles;
+    try {
+      const opening = openHandles(state.config, { connect });
+      // If the connect outlives the deadline, close whatever it produces.
+      opening
+        .then((late) => {
+          if (abandoned) closeQuietly(late.cluster, log, "abandoned cluster");
+        })
+        .catch(() => {});
+      handles = await deadline(
+        opening,
+        settings.reconnectTimeoutMs,
+        "Couchbase reconnect",
+      );
+      if (typeof handles.bucket?.waitUntilReady === "function") {
+        const serviceTypes = [
+          couchbase.ServiceType?.KeyValue,
+          couchbase.ServiceType?.Query,
+        ].filter(Boolean);
+        await deadline(
+          handles.bucket.waitUntilReady(
+            settings.reconnectTimeoutMs,
+            serviceTypes.length ? { serviceTypes } : undefined,
+          ),
+          settings.reconnectTimeoutMs + 1000,
+          "Couchbase waitUntilReady",
+        );
+      }
+      await probe(handles);
+    } catch (error) {
+      abandoned = true;
+      if (handles) closeQuietly(handles.cluster, log, "unverified cluster");
+      state.reconnectAttempts = attempt;
+      const delay = backoffDelay(attempt);
+      state.nextReconnectAt = now() + delay;
+      state.lastError = describe(error);
+      log.error(
+        `[spice-couchbase] Reconnect of "${state.name}" failed: ` +
+          `${state.lastError}. Next attempt in ~${delay}ms`,
+      );
+      return false;
+    }
+
+    if (stopped) {
+      closeQuietly(handles.cluster, log, "cluster opened during shutdown");
+      return false;
+    }
+
+    const globals = getGlobals();
+    const database = globals?.database;
+    const previous = database?.clusters?.[state.name];
+    if (database) {
+      database.clusters[state.name] = handles.cluster;
+      database.buckets[state.name] = handles.bucket;
+      database.scopes[state.name] = handles.scope;
+      database.connections[state.name] = handles.collection;
+    }
+    // Collection existence was learnt through the old cluster; re-learn it.
+    if (globals) {
+      globals.checked_collections = {};
+      globals.db?.collections?.clear?.();
+    }
+    state.generation += 1;
+    state.lastReconnectAt = now();
+    if (previous && previous !== handles.cluster) retire(state, previous);
+    log.log(
+      `[spice-couchbase] Reconnected "${state.name}" ` +
+        `(generation ${state.generation}, took ${now() - started}ms)`,
+    );
+    markHealthy(state);
+    return true;
+  }
+
+  function reconnect(name) {
+    const state = stateFor(name);
+    if (stopped || !state.config) return Promise.resolve(false);
+    if (!state.reconnecting) {
+      state.reconnecting = doReconnect(state)
+        .catch((error) => {
+          log.error(
+            `[spice-couchbase] Reconnect of "${name}" crashed: ${describe(error)}`,
+          );
+          return false;
+        })
+        .finally(() => {
+          state.reconnecting = null;
+        });
+    }
+    return state.reconnecting;
+  }
+
+  async function doCheck(state, { suspected = false } = {}) {
+    try {
+      await probe(currentHandles(state.name));
+      markHealthy(state);
+      return true;
+    } catch (error) {
+      state.probeFailures += 1;
+      // A failed probe that follows a burst of operation timeouts is two
+      // independent signals, so it does not wait for a second probe.
+      if (suspected || state.probeFailures >= settings.probeFailureThreshold) {
+        markUnhealthy(state, error);
+      }
+      if (!state.healthy) {
+        if (!state.reconnecting && now() >= state.nextReconnectAt) {
+          await reconnect(state.name);
+        }
+        escalateIfStuck(state);
+      }
+      return false;
+    }
+  }
+
+  /** Probes the connection now. Single-flight per connection. */
+  function check(name, options) {
+    const state = stateFor(name);
+    if (stopped || !state.config) return Promise.resolve(state.healthy);
+    if (state.reconnecting) return state.reconnecting;
+    if (!state.probing) {
+      state.probing = doCheck(state, options).finally(() => {
+        state.probing = null;
+      });
+    }
+    return state.probing;
+  }
+
+  /**
+   * Called by the driver after an operation. Only connectivity errors count,
+   * and only inside the sliding window, so a busy cluster that answers most
+   * requests is not torn down for one slow statement.
+   */
+  function recordOutcome(name, error) {
+    if (stopped || !name || !isConnectivityError(error)) return;
+    const state = states.get(name);
+    if (!state?.config) return;
+    const at = now();
+    state.failures.push(at);
+    const cutoff = at - settings.failureWindowMs;
+    while (state.failures.length && state.failures[0] < cutoff) {
+      state.failures.shift();
+    }
+    if (state.failures.length >= settings.failureThreshold) {
+      state.failures = [];
+      check(name, { suspected: true }).catch(() => {});
+    }
+  }
+
+  /** Starts monitoring a connection. Called once its first connect succeeds. */
+  function register(name, config) {
+    const state = stateFor(name);
+    state.config = config;
+    if (stopped || state.timer || !settings.healthIntervalMs) return;
+    state.timer = setInterval(() => {
+      check(name).catch(() => {});
+    }, settings.healthIntervalMs);
+    state.timer.unref?.();
+  }
+
+  function generation(name) {
+    return states.get(name)?.generation || 1;
+  }
+
+  function status() {
+    const result = {};
+    for (const [name, state] of states) {
+      result[name] = {
+        healthy: state.healthy,
+        reconnecting: Boolean(state.reconnecting),
+        generation: state.generation,
+        recentFailures: state.failures.length,
+        probeFailures: state.probeFailures,
+        reconnectAttempts: state.reconnectAttempts,
+        unhealthySince: state.unhealthySince
+          ? new Date(state.unhealthySince).toISOString()
+          : null,
+        lastReconnectAt: state.lastReconnectAt
+          ? new Date(state.lastReconnectAt).toISOString()
+          : null,
+        lastError: state.lastError,
+      };
+    }
+    return result;
+  }
+
+  /**
+   * Stops probing and reconnecting, and closes clusters that were replaced
+   * but are still in their grace period. The current clusters stay open: the
+   * application closes those through `spice.database.clusters`.
+   */
+  async function stop() {
+    stopped = true;
+    const closing = [];
+    for (const state of states.values()) {
+      if (state.timer) clearInterval(state.timer);
+      state.timer = null;
+      for (const cluster of state.retired) {
+        closing.push(closeQuietly(cluster, log, `retired cluster for "${state.name}"`));
+      }
+      state.retired.clear();
+    }
+    await Promise.all(closing);
+  }
+
+  return {
+    settings,
+    register,
+    recordOutcome,
+    check,
+    reconnect,
+    generation,
+    status,
+    stop,
+  };
+}
+
+module.exports = {
+  createHealthMonitor,
+  openHandles,
+  buildConnectOptions,
+  isConnectivityError,
+};