spice-couchbase 2.0.61 → 2.0.63

package/build/connectionHealth.js

@@ -0,0 +1,573 @@
+"use strict";
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
+
+function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
+
+function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
+
+var couchbase = require("couchbase"); // Connection health monitor and self-healing reconnect.
+//
+// The cluster object is created once at boot and stored on the `spice.database`
+// globals. When the SDK wedges (every operation times out, but the SDK never
+// recovers on its own) the only remedy used to be a process restart. This
+// module notices that state and opens a fresh cluster in-process:
+//
+//   1. Passive signal: every driver operation reports its outcome. Enough
+//      connectivity errors (timeouts, cancelled requests, service not
+//      available) inside a sliding window trigger a probe straight away.
+//   2. Active probe: on an interval, a KV `exists` and a `SELECT 1`, each with
+//      a short timeout and a hard deadline in case the SDK ignores it.
+//   3. Reconnect: after repeated probe failures, build a new cluster from the
+//      same config, verify it, then swap it into the globals and retire the old
+//      one after a grace period. Single-flight, with exponential backoff.
+//   4. Escalation: if the connection stays unhealthy for too long, terminate
+//      the process so the supervisor (PM2) restarts it.
+
+
+function envNumber(name, fallback) {
+  var raw = process.env[name];
+
+  if (raw === undefined || raw === null || String(raw).trim() === "") {
+    return fallback;
+  }
+
+  var value = Number(raw);
+  return Number.isFinite(value) && value >= 0 ? value : fallback;
+}
+
+function defaultSettings() {
+  return {
+    // 0 disables the interval probe (the passive signal still works).
+    healthIntervalMs: envNumber("SPICE_COUCHBASE_HEALTH_INTERVAL_MS", 15000),
+    probeTimeoutMs: envNumber("SPICE_COUCHBASE_PROBE_TIMEOUT_MS", 3000),
+    // Consecutive failed probes before the connection is declared unhealthy.
+    probeFailureThreshold: Math.max(1, envNumber("SPICE_COUCHBASE_PROBE_FAILURE_THRESHOLD", 2)),
+    // Connectivity errors inside the window that trigger an immediate probe.
+    failureThreshold: Math.max(1, envNumber("SPICE_COUCHBASE_FAILURE_THRESHOLD", 5)),
+    failureWindowMs: envNumber("SPICE_COUCHBASE_FAILURE_WINDOW_MS", 30000),
+    reconnectTimeoutMs: envNumber("SPICE_COUCHBASE_RECONNECT_TIMEOUT_MS", 20000),
+    reconnectBackoffMinMs: envNumber("SPICE_COUCHBASE_RECONNECT_BACKOFF_MIN_MS", 5000),
+    reconnectBackoffMaxMs: envNumber("SPICE_COUCHBASE_RECONNECT_BACKOFF_MAX_MS", 60000),
+    // How long a replaced cluster stays open so in-flight operations settle.
+    retireGraceMs: envNumber("SPICE_COUCHBASE_RETIRE_GRACE_MS", 30000),
+    // 0 disables the escalation.
+    exitAfterMs: envNumber("SPICE_COUCHBASE_EXIT_AFTER_MS", 300000)
+  };
+}
+
+var CONNECTIVITY_ERRORS = [couchbase.TimeoutError, couchbase.RequestCanceledError, couchbase.ServiceNotAvailableError].filter(type => typeof type === "function");
+
+function isConnectivityError(error) {
+  if (!error || typeof error !== "object") return false;
+  if (CONNECTIVITY_ERRORS.some(type => error instanceof type)) return true; // Our own deadlines, and SDK errors that crossed a module boundary.
+
+  return error.spiceConnectivity === true || ["TimeoutError", "AmbiguousTimeoutError", "UnambiguousTimeoutError", "RequestCanceledError", "ServiceNotAvailableError"].includes(error.name);
+}
+
+function deadline(promise, ms, label) {
+  var timer;
+  return Promise.race([promise, new Promise((_resolve, reject) => {
+    var _timer$unref, _timer;
+
+    timer = setTimeout(() => {
+      var error = new Error("".concat(label, " timed out after ").concat(ms, "ms"));
+      error.spiceConnectivity = true;
+      reject(error);
+    }, ms);
+    (_timer$unref = (_timer = timer).unref) === null || _timer$unref === void 0 ? void 0 : _timer$unref.call(_timer);
+  })]).finally(() => clearTimeout(timer));
+}
+
+function describe(error) {
+  return (error === null || error === void 0 ? void 0 : error.message) || String(error);
+}
+
+function closeQuietly(_x, _x2, _x3) {
+  return _closeQuietly.apply(this, arguments);
+}
+
+function _closeQuietly() {
+  _closeQuietly = _asyncToGenerator(function* (cluster, log, label) {
+    try {
+      var _cluster$close;
+
+      yield cluster === null || cluster === void 0 ? void 0 : (_cluster$close = cluster.close) === null || _cluster$close === void 0 ? void 0 : _cluster$close.call(cluster);
+    } catch (error) {
+      log.warn("[spice-couchbase] Closing ".concat(label, " failed: ").concat(describe(error)));
+    }
+  });
+  return _closeQuietly.apply(this, arguments);
+}
+
+function buildConnectOptions() {
+  var config = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
+  var options = {
+    username: config.username,
+    password: config.password
+  };
+
+  if (config.configProfile) {
+    options.configProfile = config.configProfile;
+  }
+
+  var timeouts = {};
+
+  for (var key of ["kvTimeout", "kvDurableTimeout", "queryTimeout", "searchTimeout", "analyticsTimeout", "managementTimeout", "connectTimeout", "bootstrapTimeout", "resolveTimeout"]) {
+    var _config$timeouts;
+
+    var raw = (_config$timeouts = config.timeouts) === null || _config$timeouts === void 0 ? void 0 : _config$timeouts[key];
+    if (raw === undefined || raw === null || raw === "") continue;
+    var value = Number(raw);
+    if (Number.isFinite(value) && value > 0) timeouts[key] = value;
+  }
+
+  if (Object.keys(timeouts).length > 0) {
+    options.timeouts = timeouts;
+  }
+
+  return options;
+}
+/**
+ * Opens a cluster and resolves the bucket, scope and collection handles for a
+ * connection config. Shared by the boot-time connect and the reconnect path.
+ */
+
+
+function openHandles(_x4) {
+  return _openHandles.apply(this, arguments);
+}
+
+function _openHandles() {
+  _openHandles = _asyncToGenerator(function* (config) {
+    var {
+      connect = couchbase.connect
+    } = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
+    var cluster = yield connect(config.nodes, buildConnectOptions(config));
+    var bucket = cluster.bucket(config.bucket);
+    var scope = bucket.scope(config.scope || "_default");
+    var collection = scope.collection(config.collection || "_default");
+    return {
+      cluster,
+      bucket,
+      scope,
+      collection
+    };
+  });
+  return _openHandles.apply(this, arguments);
+}
+
+function createHealthMonitor() {
+  var {
+    settings: overrides = {},
+    connect = couchbase.connect,
+    now = () => Date.now(),
+    random = Math.random,
+    log = console,
+    getGlobals = () => typeof spice !== "undefined" ? spice : undefined,
+    terminate
+  } = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
+
+  var settings = _objectSpread({}, defaultSettings(), {}, overrides);
+
+  var states = new Map();
+  var stopped = false;
+
+  var terminateProcess = terminate || (() => {
+    var _fallback$unref;
+
+    // SIGTERM runs the application's graceful shutdown when it has one,
+    // and terminates the process when it does not. The fallback covers a
+    // shutdown that hangs on the very connection that is stuck.
+    process.exitCode = 1;
+    var fallback = setTimeout(() => process.exit(1), 15000);
+    (_fallback$unref = fallback.unref) === null || _fallback$unref === void 0 ? void 0 : _fallback$unref.call(fallback);
+    process.kill(process.pid, "SIGTERM");
+  });
+
+  function stateFor(name) {
+    var state = states.get(name);
+
+    if (!state) {
+      state = {
+        name,
+        config: null,
+        generation: 1,
+        healthy: true,
+        unhealthySince: null,
+        failures: [],
+        probeFailures: 0,
+        probing: null,
+        reconnecting: null,
+        reconnectAttempts: 0,
+        nextReconnectAt: 0,
+        lastError: null,
+        lastReconnectAt: null,
+        timer: null,
+        retired: new Set(),
+        terminated: false
+      };
+      states.set(name, state);
+    }
+
+    return state;
+  }
+
+  function currentHandles(name) {
+    var _getGlobals, _database$clusters, _database$buckets, _database$scopes, _database$connections;
+
+    var database = (_getGlobals = getGlobals()) === null || _getGlobals === void 0 ? void 0 : _getGlobals.database;
+    if (!database) return null;
+    return {
+      cluster: (_database$clusters = database.clusters) === null || _database$clusters === void 0 ? void 0 : _database$clusters[name],
+      bucket: (_database$buckets = database.buckets) === null || _database$buckets === void 0 ? void 0 : _database$buckets[name],
+      scope: (_database$scopes = database.scopes) === null || _database$scopes === void 0 ? void 0 : _database$scopes[name],
+      collection: (_database$connections = database.connections) === null || _database$connections === void 0 ? void 0 : _database$connections[name]
+    };
+  }
+
+  function probe(_x5) {
+    return _probe.apply(this, arguments);
+  }
+
+  function _probe() {
+    _probe = _asyncToGenerator(function* (handles) {
+      if (!(handles === null || handles === void 0 ? void 0 : handles.cluster) || !(handles === null || handles === void 0 ? void 0 : handles.collection)) {
+        var error = new Error("No cluster handles to probe");
+        error.spiceConnectivity = true;
+        throw error;
+      }
+
+      var timeout = settings.probeTimeoutMs; // The hard deadline is the point: a wedged SDK may never honour its own.
+
+      yield deadline(Promise.all([handles.collection.exists("__spice_health_probe__", {
+        timeout
+      }), handles.cluster.query("SELECT 1", {
+        timeout
+      })]), timeout + 1000, "Couchbase health probe");
+    });
+    return _probe.apply(this, arguments);
+  }
+
+  function markHealthy(state) {
+    state.probeFailures = 0;
+    state.failures = [];
+    state.reconnectAttempts = 0;
+    state.nextReconnectAt = 0;
+
+    if (!state.healthy) {
+      var downFor = state.unhealthySince ? now() - state.unhealthySince : 0;
+      log.log("[spice-couchbase] Connection \"".concat(state.name, "\" healthy again ") + "(generation ".concat(state.generation, ", unhealthy for ").concat(downFor, "ms)"));
+    }
+
+    state.healthy = true;
+    state.unhealthySince = null;
+    state.lastError = null;
+  }
+
+  function markUnhealthy(state, error) {
+    state.lastError = describe(error);
+
+    if (state.healthy) {
+      state.healthy = false;
+      state.unhealthySince = now();
+      log.warn("[spice-couchbase] Connection \"".concat(state.name, "\" unhealthy: ").concat(state.lastError));
+    }
+  }
+
+  function escalateIfStuck(state) {
+    if (state.terminated || stopped || !settings.exitAfterMs || !state.unhealthySince || now() - state.unhealthySince < settings.exitAfterMs) {
+      return;
+    }
+
+    state.terminated = true;
+    log.error("[spice-couchbase] Connection \"".concat(state.name, "\" has been unhealthy for ") + "".concat(now() - state.unhealthySince, "ms after ").concat(state.reconnectAttempts, " ") + "failed reconnect(s). Terminating so the supervisor restarts the process.");
+    terminateProcess();
+  }
+
+  function retire(state, cluster) {
+    var _timer$unref2;
+
+    if (!cluster) return;
+    state.retired.add(cluster);
+    var timer = setTimeout(() => {
+      state.retired.delete(cluster);
+      closeQuietly(cluster, log, "retired cluster for \"".concat(state.name, "\""));
+    }, settings.retireGraceMs);
+    (_timer$unref2 = timer.unref) === null || _timer$unref2 === void 0 ? void 0 : _timer$unref2.call(timer);
+  }
+
+  function backoffDelay(attempt) {
+    var base = Math.min(settings.reconnectBackoffMaxMs, settings.reconnectBackoffMinMs * 2 ** Math.max(0, attempt - 1)); // +/- 20% jitter so a fleet of pods does not reconnect in lockstep.
+
+    return Math.round(base * (0.8 + random() * 0.4));
+  }
+
+  function doReconnect(_x6) {
+    return _doReconnect.apply(this, arguments);
+  }
+
+  function _doReconnect() {
+    _doReconnect = _asyncToGenerator(function* (state) {
+      var _database$clusters2;
+
+      var started = now();
+      var attempt = state.reconnectAttempts + 1;
+      log.warn("[spice-couchbase] Reconnecting \"".concat(state.name, "\" (attempt ").concat(attempt, ")"));
+      var abandoned = false;
+      var handles;
+
+      try {
+        var _handles$bucket;
+
+        var opening = openHandles(state.config, {
+          connect
+        }); // If the connect outlives the deadline, close whatever it produces.
+
+        opening.then(late => {
+          if (abandoned) closeQuietly(late.cluster, log, "abandoned cluster");
+        }).catch(() => {});
+        handles = yield deadline(opening, settings.reconnectTimeoutMs, "Couchbase reconnect");
+
+        if (typeof ((_handles$bucket = handles.bucket) === null || _handles$bucket === void 0 ? void 0 : _handles$bucket.waitUntilReady) === "function") {
+          var _couchbase$ServiceTyp, _couchbase$ServiceTyp2;
+
+          var serviceTypes = [(_couchbase$ServiceTyp = couchbase.ServiceType) === null || _couchbase$ServiceTyp === void 0 ? void 0 : _couchbase$ServiceTyp.KeyValue, (_couchbase$ServiceTyp2 = couchbase.ServiceType) === null || _couchbase$ServiceTyp2 === void 0 ? void 0 : _couchbase$ServiceTyp2.Query].filter(Boolean);
+          yield deadline(handles.bucket.waitUntilReady(settings.reconnectTimeoutMs, serviceTypes.length ? {
+            serviceTypes
+          } : undefined), settings.reconnectTimeoutMs + 1000, "Couchbase waitUntilReady");
+        }
+
+        yield probe(handles);
+      } catch (error) {
+        abandoned = true;
+        if (handles) closeQuietly(handles.cluster, log, "unverified cluster");
+        state.reconnectAttempts = attempt;
+        var delay = backoffDelay(attempt);
+        state.nextReconnectAt = now() + delay;
+        state.lastError = describe(error);
+        log.error("[spice-couchbase] Reconnect of \"".concat(state.name, "\" failed: ") + "".concat(state.lastError, ". Next attempt in ~").concat(delay, "ms"));
+        return false;
+      }
+
+      if (stopped) {
+        closeQuietly(handles.cluster, log, "cluster opened during shutdown");
+        return false;
+      }
+
+      var globals = getGlobals();
+      var database = globals === null || globals === void 0 ? void 0 : globals.database;
+      var previous = database === null || database === void 0 ? void 0 : (_database$clusters2 = database.clusters) === null || _database$clusters2 === void 0 ? void 0 : _database$clusters2[state.name];
+
+      if (database) {
+        database.clusters[state.name] = handles.cluster;
+        database.buckets[state.name] = handles.bucket;
+        database.scopes[state.name] = handles.scope;
+        database.connections[state.name] = handles.collection;
+      } // Collection existence was learnt through the old cluster; re-learn it.
+
+
+      if (globals) {
+        var _globals$db, _globals$db$collectio, _globals$db$collectio2;
+
+        globals.checked_collections = {};
+        (_globals$db = globals.db) === null || _globals$db === void 0 ? void 0 : (_globals$db$collectio = _globals$db.collections) === null || _globals$db$collectio === void 0 ? void 0 : (_globals$db$collectio2 = _globals$db$collectio.clear) === null || _globals$db$collectio2 === void 0 ? void 0 : _globals$db$collectio2.call(_globals$db$collectio);
+      }
+
+      state.generation += 1;
+      state.lastReconnectAt = now();
+      if (previous && previous !== handles.cluster) retire(state, previous);
+      log.log("[spice-couchbase] Reconnected \"".concat(state.name, "\" ") + "(generation ".concat(state.generation, ", took ").concat(now() - started, "ms)"));
+      markHealthy(state);
+      return true;
+    });
+    return _doReconnect.apply(this, arguments);
+  }
+
+  function reconnect(name) {
+    var state = stateFor(name);
+    if (stopped || !state.config) return Promise.resolve(false);
+
+    if (!state.reconnecting) {
+      state.reconnecting = doReconnect(state).catch(error => {
+        log.error("[spice-couchbase] Reconnect of \"".concat(name, "\" crashed: ").concat(describe(error)));
+        return false;
+      }).finally(() => {
+        state.reconnecting = null;
+      });
+    }
+
+    return state.reconnecting;
+  }
+
+  function doCheck(_x7) {
+    return _doCheck.apply(this, arguments);
+  }
+  /** Probes the connection now. Single-flight per connection. */
+
+
+  function _doCheck() {
+    _doCheck = _asyncToGenerator(function* (state) {
+      var {
+        suspected = false
+      } = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
+
+      try {
+        yield probe(currentHandles(state.name));
+        markHealthy(state);
+        return true;
+      } catch (error) {
+        state.probeFailures += 1; // A failed probe that follows a burst of operation timeouts is two
+        // independent signals, so it does not wait for a second probe.
+
+        if (suspected || state.probeFailures >= settings.probeFailureThreshold) {
+          markUnhealthy(state, error);
+        }
+
+        if (!state.healthy) {
+          if (!state.reconnecting && now() >= state.nextReconnectAt) {
+            yield reconnect(state.name);
+          }
+
+          escalateIfStuck(state);
+        }
+
+        return false;
+      }
+    });
+    return _doCheck.apply(this, arguments);
+  }
+
+  function check(name, options) {
+    var state = stateFor(name);
+    if (stopped || !state.config) return Promise.resolve(state.healthy);
+    if (state.reconnecting) return state.reconnecting;
+
+    if (!state.probing) {
+      state.probing = doCheck(state, options).finally(() => {
+        state.probing = null;
+      });
+    }
+
+    return state.probing;
+  }
+  /**
+   * Called by the driver after an operation. Only connectivity errors count,
+   * and only inside the sliding window, so a busy cluster that answers most
+   * requests is not torn down for one slow statement.
+   */
+
+
+  function recordOutcome(name, error) {
+    if (stopped || !name || !isConnectivityError(error)) return;
+    var state = states.get(name);
+    if (!(state === null || state === void 0 ? void 0 : state.config)) return;
+    var at = now();
+    state.failures.push(at);
+    var cutoff = at - settings.failureWindowMs;
+
+    while (state.failures.length && state.failures[0] < cutoff) {
+      state.failures.shift();
+    }
+
+    if (state.failures.length >= settings.failureThreshold) {
+      state.failures = [];
+      check(name, {
+        suspected: true
+      }).catch(() => {});
+    }
+  }
+  /** Starts monitoring a connection. Called once its first connect succeeds. */
+
+
+  function register(name, config) {
+    var _state$timer$unref, _state$timer;
+
+    var state = stateFor(name);
+    state.config = config;
+    if (stopped || state.timer || !settings.healthIntervalMs) return;
+    state.timer = setInterval(() => {
+      check(name).catch(() => {});
+    }, settings.healthIntervalMs);
+    (_state$timer$unref = (_state$timer = state.timer).unref) === null || _state$timer$unref === void 0 ? void 0 : _state$timer$unref.call(_state$timer);
+  }
+
+  function generation(name) {
+    var _states$get;
+
+    return ((_states$get = states.get(name)) === null || _states$get === void 0 ? void 0 : _states$get.generation) || 1;
+  }
+
+  function status() {
+    var result = {};
+
+    for (var [name, state] of states) {
+      result[name] = {
+        healthy: state.healthy,
+        reconnecting: Boolean(state.reconnecting),
+        generation: state.generation,
+        recentFailures: state.failures.length,
+        probeFailures: state.probeFailures,
+        reconnectAttempts: state.reconnectAttempts,
+        unhealthySince: state.unhealthySince ? new Date(state.unhealthySince).toISOString() : null,
+        lastReconnectAt: state.lastReconnectAt ? new Date(state.lastReconnectAt).toISOString() : null,
+        lastError: state.lastError
+      };
+    }
+
+    return result;
+  }
+  /**
+   * Stops probing and reconnecting, and closes clusters that were replaced
+   * but are still in their grace period. The current clusters stay open: the
+   * application closes those through `spice.database.clusters`.
+   */
+
+
+  function stop() {
+    return _stop.apply(this, arguments);
+  }
+
+  function _stop() {
+    _stop = _asyncToGenerator(function* () {
+      stopped = true;
+      var closing = [];
+
+      for (var state of states.values()) {
+        if (state.timer) clearInterval(state.timer);
+        state.timer = null;
+
+        for (var cluster of state.retired) {
+          closing.push(closeQuietly(cluster, log, "retired cluster for \"".concat(state.name, "\"")));
+        }
+
+        state.retired.clear();
+      }
+
+      yield Promise.all(closing);
+    });
+    return _stop.apply(this, arguments);
+  }
+
+  return {
+    settings,
+    register,
+    recordOutcome,
+    check,
+    reconnect,
+    generation,
+    status,
+    stop
+  };
+}
+
+module.exports = {
+  createHealthMonitor,
+  openHandles,
+  buildConnectOptions,
+  isConnectivityError
+};