spice-couchbase 2.0.62 → 2.0.63

package/src/index.js

@@ -5,6 +5,7 @@ let couchbase = require("couchbase");
 const { ConflictResolutionType } = require("couchbase");
 const { QueryScanConsistency } = require("couchbase");
 const { parseResourceList, extractStatementResources } = require("./utility");
+const { createHealthMonitor, openHandles } = require("./connectionHealth");
 
 if (!Promise.allSettled) {
   Promise.allSettled = (promises) =>
@@ -71,6 +72,27 @@ const OPTIMIZE_ENABLED_RESOURCES = parseResourceList(
 );
 let autoOptimizePolicy = null;
 
+// One health monitor per process, shared by every copy of this module that
+// ends up loaded. See connectionHealth.js.
+const HEALTH_MONITOR_KEY = Symbol.for("spice-couchbase.health-monitor");
+const health =
+  globalThis[HEALTH_MONITOR_KEY] ||
+  (globalThis[HEALTH_MONITOR_KEY] = createHealthMonitor());
+
+// Cluster handles live on the `spice.database` globals, which a reconnect
+// replaces. Instances read them at call time instead of copying them.
+const HANDLE_GLOBALS = {
+  cluster: "clusters",
+  connection: "connections",
+  scope: "scopes",
+  bucket_handle: "buckets",
+};
+
+function globalHandle(kind, connection_name) {
+  if (typeof spice === "undefined") return undefined;
+  return spice?.database?.[HANDLE_GLOBALS[kind]]?.[connection_name];
+}
+
 function requestedAutoOptimize(query_options) {
   if (
     !query_options ||
@@ -164,13 +186,9 @@ module.exports = class Database {
   ) {
     if (connection) {
       this.connection_name = connection;
-      this.connection = spice.database.connections[connection];
       this.query_scope = query_scope;
-      this.cluster = spice.database.clusters[connection];
       this.bucket = this.connection._scope._bucket._name;
       this.db_config = spice.config.database.connections[connection];
-      this.collection_manager =
-        spice.database.buckets[connection].collections();
       this.auto_create_collection =
         auto_create_collection ||
         ((collection ||
@@ -196,14 +214,11 @@ module.exports = class Database {
         scope ||
         spice.config.database.connections[connection].scope ||
         "_default";
-      this.scope = spice.database.scopes[connection];
       this.full_text_index =
         spice.config.database.connections[connection].full_text_index ||
         full_text_index;
       this.N1qlQuery = couchbase.N1qlQuery;
       this.SearchQuery = couchbase.SearchQuery;
-      this.collection = this.scope.collection(this.collection_name);
-      this.userManager = this.getUserManager(connection);
       if (!spice?.db?.collections) {
         spice.db = spice.db || {};
         spice.db.collections = new Set();
@@ -230,6 +245,137 @@ module.exports = class Database {
     }
   }
 
+  // Handles resolve from the globals on every access, so an instance created
+  // before a reconnect uses the new cluster afterwards. Assigning one pins it
+  // on that instance instead (tests, custom wiring).
+  pinHandle(kind, value) {
+    if (!Object.prototype.hasOwnProperty.call(this, "_pinned_handles")) {
+      Object.defineProperty(this, "_pinned_handles", {
+        value: {},
+        enumerable: false,
+        writable: true,
+      });
+    }
+    this._pinned_handles[kind] = value;
+  }
+
+  pinnedHandle(kind) {
+    const pinned = this._pinned_handles;
+    return pinned && kind in pinned ? { value: pinned[kind] } : null;
+  }
+
+  handle(kind) {
+    const pinned = this.pinnedHandle(kind);
+    return pinned ? pinned.value : globalHandle(kind, this.connection_name);
+  }
+
+  get cluster() {
+    return this.handle("cluster");
+  }
+  set cluster(value) {
+    this.pinHandle("cluster", value);
+  }
+
+  get connection() {
+    return this.handle("connection");
+  }
+  set connection(value) {
+    this.pinHandle("connection", value);
+  }
+
+  get scope() {
+    return this.handle("scope");
+  }
+  set scope(value) {
+    this.pinHandle("scope", value);
+  }
+
+  get collection() {
+    const pinned = this.pinnedHandle("collection");
+    if (pinned) return pinned.value;
+    const scope = this.scope;
+    if (!scope) return undefined;
+    // Collection handles are cheap but not free; cache one per scope object,
+    // which changes exactly when the connection is replaced.
+    if (this._collection_cache?.scope !== scope) {
+      Object.defineProperty(this, "_collection_cache", {
+        value: { scope, collection: scope.collection(this.collection_name) },
+        enumerable: false,
+        writable: true,
+        configurable: true,
+      });
+    }
+    return this._collection_cache.collection;
+  }
+  set collection(value) {
+    this.pinHandle("collection", value);
+  }
+
+  get collection_manager() {
+    const pinned = this.pinnedHandle("collection_manager");
+    if (pinned) return pinned.value;
+    return this.handle("bucket_handle")?.collections();
+  }
+  set collection_manager(value) {
+    this.pinHandle("collection_manager", value);
+  }
+
+  get userManager() {
+    const pinned = this.pinnedHandle("userManager");
+    if (pinned) return pinned.value;
+    return this.cluster?.users();
+  }
+  set userManager(value) {
+    this.pinHandle("userManager", value);
+  }
+
+  /**
+   * Reports a finished operation to the health monitor. Only connectivity
+   * errors (timeouts, cancelled requests, service not available) count.
+   */
+  recordOutcome(error) {
+    health.recordOutcome(this.connection_name, error);
+  }
+
+  recordSettled(results) {
+    for (const result of results) {
+      if (result?.status === "rejected") this.recordOutcome(result.reason);
+    }
+  }
+
+  async observe(operation) {
+    try {
+      return await operation;
+    } catch (error) {
+      this.recordOutcome(error);
+      throw error;
+    }
+  }
+
+  /** Per-connection health: healthy, reconnecting, generation, last error. */
+  static healthStatus() {
+    return health.status();
+  }
+
+  /** Probes a connection now and reconnects it if the probe fails. */
+  static checkHealth(connection_name = "default") {
+    return health.check(connection_name);
+  }
+
+  /** Opens a fresh cluster for a connection and swaps it in. */
+  static forceReconnect(connection_name = "default") {
+    return health.reconnect(connection_name);
+  }
+
+  /**
+   * Stops probing and reconnecting and closes replaced clusters still in
+   * their grace period. Call before closing `spice.database.clusters` on
+   * shutdown so a reconnect cannot race it.
+   */
+  static stopHealthMonitor() {
+    return health.stop();
+  }
+
   /**
    * Lets the application decide per resource whether slow statements may
    * create covering indexes, e.g. from a per-resource setting an operator
@@ -324,26 +470,9 @@ module.exports = class Database {
   }
 
   async connect(connection_name, config) {
-    let connectOptions = {
-      username: config.username,
-      password: config.password,
-    };
-    if (config.configProfile) {
-      connectOptions.configProfile = config.configProfile;
-    }
-
-    let cluster = await couchbase.connect(config.nodes, connectOptions);
-    let bucket = cluster.bucket(config.bucket);
-    let collection;
-    let scope_name = this.scope_name || config.scope || "_default";
-    let collection_name =
-      this.collection_name || config.collection || "_default";
-    let scope = bucket.scope(scope_name);
-
-    collection = bucket
-      .scope(scope_name)
-      .collection(collection_name || "_default");
-
+    const { cluster, bucket, scope, collection } = await openHandles(config);
+    // Watch the connection from here on and replace it if it wedges.
+    health.register(connection_name, config);
     return {
       connection: connection_name,
       bucket,
@@ -354,7 +483,7 @@ module.exports = class Database {
   }
 
   getUserManager(connection) {
-    return spice.database.clusters[connection].users();
+    return globalHandle("cluster", connection).users();
   }
 
   async assignUserPermission() {
@@ -534,7 +663,7 @@ module.exports = class Database {
         this.use_bucket_name ? this.connection : this.collection;
 
       if (!useLookupIn) {
-        let response = await collection.get(doc_id);
+        let response = await this.observe(collection.get(doc_id));
         return { id: doc_id, ...response.content };
       }
 
@@ -549,6 +678,7 @@ module.exports = class Database {
       });
 
       const batchResults = await Promise.allSettled(batchPromises);
+      this.recordSettled(batchResults);
 
       // Merge results from all batches
       batchResults.forEach((result, batchIndex) => {
@@ -570,12 +700,14 @@ module.exports = class Database {
 
   async insert(doc_id, document, expiry) {
     try {
-      await (this.use_bucket_name ? this.connection : this.collection).insert(
-        doc_id,
-        document,
-        {
-          expiry: expiry,
-        },
+      await this.observe(
+        (this.use_bucket_name ? this.connection : this.collection).insert(
+          doc_id,
+          document,
+          {
+            expiry: expiry,
+          },
+        ),
       );
     } catch (e) {
       console.log(e.message, e, doc_id);
@@ -584,38 +716,45 @@ module.exports = class Database {
   }
 
   async touch(doc_id, expiry = 0) {
-    await (this.use_bucket_name ? this.connection : this.collection).touch(
-      doc_id,
-      expiry,
+    await this.observe(
+      (this.use_bucket_name ? this.connection : this.collection).touch(
+        doc_id,
+        expiry,
+      ),
     );
     return await this.get(doc_id);
   }
 
   async count(doc_id, adjustment, initial_value) {
-    let res = await (
-      this.use_bucket_name ?
-        this.connection
-      : this.collection).counter(doc_id, adjustment, {
-      initial: initial_value,
-    });
+    let res = await this.observe(
+      (this.use_bucket_name ? this.connection : this.collection).counter(
+        doc_id,
+        adjustment,
+        {
+          initial: initial_value,
+        },
+      ),
+    );
     return res.value;
   }
 
   async update(doc_id, document, expiry) {
-    let res = await (
-      this.use_bucket_name ?
-        this.connection
-      : this.collection).replace(doc_id, document, {
-      expiry: expiry,
-    });
+    let res = await this.observe(
+      (this.use_bucket_name ? this.connection : this.collection).replace(
+        doc_id,
+        document,
+        {
+          expiry: expiry,
+        },
+      ),
+    );
     return res.value;
   }
 
   async delete(doc_id) {
-    let res = await (
-      this.use_bucket_name ?
-        this.connection
-      : this.collection).remove(doc_id);
+    let res = await this.observe(
+      (this.use_bucket_name ? this.connection : this.collection).remove(doc_id),
+    );
     return res.value;
   }
 
@@ -632,6 +771,7 @@ module.exports = class Database {
         // Regular get - no batching needed
         let functions = _.map(keys, (key) => collection.get(key));
         let returned_all = await Promise.allSettled(functions);
+        this.recordSettled(returned_all);
 
         let responses = _.flatten(
           _.map(returned_all, (returned_obj) => returned_obj?.value),
@@ -670,6 +810,7 @@ module.exports = class Database {
           });
 
           const batchResults = await Promise.allSettled(batchPromises);
+          this.recordSettled(batchResults);
 
           // Merge results from all batches
           batchResults.forEach((result, batchIndex) => {
@@ -1068,7 +1209,9 @@ module.exports = class Database {
     // Start time
     let startTime = process.hrtime.bigint();
     const options = buildQueryOptions(query_options);
-    let response = await exec_scope.query(sanitizedQuery, options);
+    let response = await this.observe(
+      exec_scope.query(sanitizedQuery, options),
+    );
     // End time
     let endTime = process.hrtime.bigint();
     // Calculate duration in microseconds