spice-couchbase 2.0.61 → 2.0.63

package/src/connectionHealth.test.js

@@ -0,0 +1,412 @@
+"use strict";
+
+const assert = require("node:assert/strict");
+const { test } = require("node:test");
+const couchbase = require("couchbase");
+const {
+  createHealthMonitor,
+  buildConnectOptions,
+  isConnectivityError,
+} = require("./connectionHealth");
+
+const config = {
+  nodes: "couchbase://db",
+  bucket: "saturn",
+  scope: "app",
+  username: "u",
+  password: "p",
+};
+
+const timeoutError = () => new couchbase.UnambiguousTimeoutError();
+
+// A fake cluster whose probe behaviour can be flipped between healthy and
+// wedged. `hang` makes operations never settle, like a stuck SDK.
+function fakeCluster(id, { healthy = true, hang = false } = {}) {
+  const cluster = {
+    id,
+    healthy,
+    hang,
+    closed: 0,
+    queries: 0,
+    async close() {
+      cluster.closed += 1;
+    },
+    query() {
+      cluster.queries += 1;
+      if (cluster.hang) return new Promise(() => {});
+      return cluster.healthy ?
+          Promise.resolve({ rows: [{ $1: 1 }] })
+        : Promise.reject(timeoutError());
+    },
+    bucket(name) {
+      const bucket = {
+        name,
+        cluster,
+        async waitUntilReady() {
+          if (!cluster.healthy) throw timeoutError();
+        },
+        scope(scopeName) {
+          return {
+            name: scopeName,
+            bucket,
+            collection(collectionName) {
+              return {
+                name: collectionName,
+                cluster,
+                exists() {
+                  if (cluster.hang) return new Promise(() => {});
+                  return cluster.healthy ?
+                      Promise.resolve({ exists: false })
+                    : Promise.reject(timeoutError());
+                },
+              };
+            },
+          };
+        },
+      };
+      return bucket;
+    },
+  };
+  return cluster;
+}
+
+function handlesFor(cluster) {
+  const bucket = cluster.bucket(config.bucket);
+  const scope = bucket.scope(config.scope);
+  return { cluster, bucket, scope, collection: scope.collection("_default") };
+}
+
+function setup({ settings = {}, connect, clock } = {}) {
+  const initial = fakeCluster("initial");
+  const handles = handlesFor(initial);
+  const globals = {
+    database: {
+      clusters: { default: handles.cluster },
+      buckets: { default: handles.bucket },
+      scopes: { default: handles.scope },
+      connections: { default: handles.collection },
+    },
+    checked_collections: { users: true },
+    db: { collections: new Set(["users"]) },
+  };
+  const opened = [];
+  const time = clock || { now: 1000 };
+  const terminated = [];
+  const logs = [];
+  const log = {
+    log: (m) => logs.push(m),
+    warn: (m) => logs.push(m),
+    error: (m) => logs.push(m),
+  };
+  const monitor = createHealthMonitor({
+    settings: {
+      healthIntervalMs: 0,
+      probeTimeoutMs: 20,
+      probeFailureThreshold: 2,
+      failureThreshold: 3,
+      failureWindowMs: 1000,
+      reconnectTimeoutMs: 50,
+      reconnectBackoffMinMs: 100,
+      reconnectBackoffMaxMs: 1000,
+      retireGraceMs: 5,
+      exitAfterMs: 10000,
+      ...settings,
+    },
+    connect:
+      connect ||
+      (async () => {
+        const cluster = fakeCluster(`fresh-${opened.length + 1}`);
+        opened.push(cluster);
+        return cluster;
+      }),
+    now: () => time.now,
+    random: () => 0.5,
+    log,
+    getGlobals: () => globals,
+    terminate: () => terminated.push(time.now),
+  });
+  monitor.register("default", config);
+  return { monitor, initial, globals, opened, time, terminated, logs };
+}
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+test("only connectivity errors count", () => {
+  assert.equal(isConnectivityError(timeoutError()), true);
+  assert.equal(isConnectivityError(new couchbase.AmbiguousTimeoutError()), true);
+  assert.equal(isConnectivityError(new couchbase.RequestCanceledError()), true);
+  assert.equal(isConnectivityError(new couchbase.ServiceNotAvailableError()), true);
+  assert.equal(isConnectivityError(new couchbase.DocumentNotFoundError()), false);
+  assert.equal(isConnectivityError(new Error("syntax error")), false);
+  assert.equal(isConnectivityError(undefined), false);
+});
+
+test("connect options carry only the timeouts that are set", () => {
+  assert.deepEqual(buildConnectOptions(config), { username: "u", password: "p" });
+  assert.deepEqual(
+    buildConnectOptions({
+      ...config,
+      configProfile: "wanDevelopment",
+      timeouts: { queryTimeout: "30000", kvTimeout: undefined, connectTimeout: "" },
+    }),
+    {
+      username: "u",
+      password: "p",
+      configProfile: "wanDevelopment",
+      timeouts: { queryTimeout: 30000 },
+    },
+  );
+});
+
+test("a healthy probe leaves the connection alone", async () => {
+  const { monitor, opened, globals, initial } = setup();
+  assert.equal(await monitor.check("default"), true);
+  assert.equal(opened.length, 0);
+  assert.equal(globals.database.clusters.default, initial);
+  assert.equal(monitor.status().default.healthy, true);
+});
+
+test("non-connectivity errors never trigger a probe", async () => {
+  const { monitor, initial } = setup();
+  for (let i = 0; i < 10; i++) {
+    monitor.recordOutcome("default", new couchbase.DocumentNotFoundError());
+  }
+  await sleep(5);
+  assert.equal(initial.queries, 0);
+});
+
+test("one failed probe is not enough, two trigger a reconnect", async () => {
+  const { monitor, initial, opened } = setup();
+  initial.healthy = false;
+  await monitor.check("default");
+  assert.equal(opened.length, 0);
+  assert.equal(monitor.status().default.healthy, true);
+  await monitor.check("default");
+  assert.equal(opened.length, 1);
+  assert.equal(monitor.status().default.healthy, true);
+  assert.equal(monitor.status().default.generation, 2);
+});
+
+test("a burst of operation timeouts probes and reconnects at once", async () => {
+  const { monitor, initial, opened, globals } = setup();
+  initial.healthy = false;
+  for (let i = 0; i < 3; i++) monitor.recordOutcome("default", timeoutError());
+  await sleep(10);
+  assert.equal(opened.length, 1);
+  assert.equal(globals.database.clusters.default, opened[0]);
+});
+
+test("timeouts outside the window do not add up", async () => {
+  const { monitor, initial, opened, time } = setup();
+  initial.healthy = false;
+  monitor.recordOutcome("default", timeoutError());
+  monitor.recordOutcome("default", timeoutError());
+  time.now += 2000;
+  monitor.recordOutcome("default", timeoutError());
+  await sleep(10);
+  assert.equal(initial.queries, 0);
+  assert.equal(opened.length, 0);
+});
+
+test("reconnect swaps every global handle and resets collection caches", async () => {
+  const { monitor, initial, opened, globals } = setup();
+  initial.healthy = false;
+  await monitor.check("default");
+  await monitor.check("default");
+  const fresh = opened[0];
+  assert.equal(globals.database.clusters.default, fresh);
+  assert.equal(globals.database.buckets.default.cluster, fresh);
+  assert.equal(globals.database.scopes.default.bucket.cluster, fresh);
+  assert.equal(globals.database.connections.default.cluster, fresh);
+  assert.deepEqual(globals.checked_collections, {});
+  assert.equal(globals.db.collections.size, 0);
+});
+
+test("the replaced cluster is closed after the grace period", async () => {
+  const { monitor, initial } = setup();
+  initial.healthy = false;
+  await monitor.check("default");
+  await monitor.check("default");
+  assert.equal(initial.closed, 0);
+  await sleep(20);
+  assert.equal(initial.closed, 1);
+});
+
+test("concurrent triggers share a single reconnect", async () => {
+  let release;
+  const opened = [];
+  const { monitor, initial } = setup({
+    settings: { probeFailureThreshold: 1, reconnectTimeoutMs: 1000 },
+    connect: () =>
+      new Promise((resolve) => {
+        release = () => {
+          const cluster = fakeCluster("fresh");
+          opened.push(cluster);
+          resolve(cluster);
+        };
+      }),
+  });
+  initial.healthy = false;
+  const first = monitor.check("default");
+  await sleep(5);
+  const others = [monitor.check("default"), monitor.reconnect("default")];
+  for (let i = 0; i < 5; i++) monitor.recordOutcome("default", timeoutError());
+  release();
+  await Promise.all([first, ...others]);
+  assert.equal(opened.length, 1);
+  assert.equal(monitor.status().default.generation, 2);
+});
+
+test("a probe that hangs is cut off by the hard deadline", async () => {
+  const { monitor, initial, opened } = setup({
+    settings: { probeFailureThreshold: 1 },
+  });
+  initial.hang = true;
+  // The deadline timer is unref'd; keep the loop alive while the probe hangs.
+  const keepAlive = setInterval(() => {}, 1000);
+  try {
+    const started = Date.now();
+    await monitor.check("default");
+    assert.ok(Date.now() - started < 1500);
+    assert.equal(opened.length, 1);
+  } finally {
+    clearInterval(keepAlive);
+  }
+});
+
+test("a failed reconnect backs off and keeps the old cluster", async () => {
+  const opened = [];
+  const { monitor, initial, globals, time } = setup({
+    settings: { probeFailureThreshold: 1 },
+    connect: async () => {
+      const cluster = fakeCluster(`bad-${opened.length + 1}`, { healthy: false });
+      opened.push(cluster);
+      return cluster;
+    },
+  });
+  initial.healthy = false;
+  await monitor.check("default");
+  assert.equal(opened.length, 1);
+  assert.equal(opened[0].closed, 1, "unverified cluster is closed");
+  assert.equal(globals.database.clusters.default, initial);
+  assert.equal(monitor.status().default.reconnectAttempts, 1);
+
+  // Inside the backoff window: probe only, no new connect.
+  await monitor.check("default");
+  assert.equal(opened.length, 1);
+
+  time.now += 150;
+  await monitor.check("default");
+  assert.equal(opened.length, 2);
+  assert.equal(monitor.status().default.reconnectAttempts, 2);
+});
+
+test("a connection that recovers on its own is marked healthy again", async () => {
+  const { monitor, initial, time } = setup({
+    settings: { probeFailureThreshold: 1 },
+    connect: async () => fakeCluster("bad", { healthy: false }),
+  });
+  initial.healthy = false;
+  await monitor.check("default");
+  assert.equal(monitor.status().default.healthy, false);
+  initial.healthy = true;
+  time.now += 5000;
+  await monitor.check("default");
+  const status = monitor.status().default;
+  assert.equal(status.healthy, true);
+  assert.equal(status.reconnectAttempts, 0);
+  assert.equal(status.unhealthySince, null);
+});
+
+test("staying unhealthy past the limit terminates the process once", async () => {
+  const { monitor, initial, time, terminated } = setup({
+    settings: { probeFailureThreshold: 1, exitAfterMs: 1000 },
+    connect: async () => fakeCluster("bad", { healthy: false }),
+  });
+  initial.healthy = false;
+  await monitor.check("default");
+  assert.equal(terminated.length, 0);
+  time.now += 1500;
+  await monitor.check("default");
+  time.now += 1500;
+  await monitor.check("default");
+  assert.equal(terminated.length, 1);
+});
+
+test("exitAfterMs of 0 never terminates", async () => {
+  const { monitor, initial, time, terminated } = setup({
+    settings: { probeFailureThreshold: 1, exitAfterMs: 0 },
+    connect: async () => fakeCluster("bad", { healthy: false }),
+  });
+  initial.healthy = false;
+  for (let i = 0; i < 5; i++) {
+    time.now += 100000;
+    await monitor.check("default");
+  }
+  assert.equal(terminated.length, 0);
+});
+
+test("stop() halts reconnects and closes retired clusters", async () => {
+  const { monitor, initial, opened } = setup({
+    settings: { probeFailureThreshold: 1, retireGraceMs: 60000 },
+  });
+  initial.healthy = false;
+  await monitor.check("default");
+  assert.equal(opened.length, 1);
+  await monitor.stop();
+  assert.equal(initial.closed, 1);
+  opened[0].healthy = false;
+  await monitor.check("default");
+  await monitor.reconnect("default");
+  assert.equal(opened.length, 1);
+});
+
+test("the interval probe runs when enabled", async () => {
+  const { monitor, initial, opened } = setup({
+    settings: { healthIntervalMs: 5, probeFailureThreshold: 1 },
+  });
+  initial.healthy = false;
+  await sleep(40);
+  assert.ok(opened.length >= 1);
+  await monitor.stop();
+});
+
+test("a Database instance follows the swapped globals", async () => {
+  const path = require.resolve("./index");
+  delete require.cache[path];
+  const previousSpice = global.spice;
+  const { monitor, initial, opened, globals } = setup({
+    settings: { probeFailureThreshold: 1 },
+  });
+  global.spice = {
+    ...globals,
+    config: { database: { connections: { default: { ...config } } } },
+  };
+  // Point the shared module-level monitor at this test's monitor.
+  const key = Symbol.for("spice-couchbase.health-monitor");
+  const previousMonitor = globalThis[key];
+  globalThis[key] = monitor;
+  try {
+    // Keep a reference to the same globals object the monitor mutates.
+    global.spice.database = globals.database;
+    // The constructor reads the bucket name off the boot-time collection.
+    globals.database.connections.default._scope = { _bucket: { _name: "saturn" } };
+    const Database = require("./index");
+    const db = new Database("default");
+    assert.equal(db.cluster, initial);
+    const oldCollection = db.collection;
+    assert.equal(oldCollection.cluster, initial);
+
+    initial.healthy = false;
+    await monitor.check("default");
+    assert.equal(opened.length, 1);
+    assert.equal(db.cluster, opened[0]);
+    assert.equal(db.collection.cluster, opened[0]);
+    assert.notEqual(db.collection, oldCollection);
+    assert.deepEqual(Database.healthStatus().default.generation, 2);
+  } finally {
+    globalThis[key] = previousMonitor;
+    global.spice = previousSpice;
+    delete require.cache[path];
+  }
+});

package/src/index.auto-optimize.test.js

@@ -37,7 +37,8 @@ function createStatements(scope) {
 }
 
 // Isolate optimizer state and time without connecting to Couchbase or sleeping.
-function createDatabase(durationMs, env = {}) {
+// Each load is a fresh module: its own seen-set, cooldown and policy.
+function loadDriver(durationMs, env = {}) {
   let tick = BigInt(0);
   const module = { exports: {} };
   vm.runInNewContext(fs.readFileSync(path.join(__dirname, "index.js"), "utf8"), {
@@ -55,7 +56,11 @@ function createDatabase(durationMs, env = {}) {
       },
     },
   });
-  const db = Object.create(module.exports.prototype);
+  return module.exports;
+}
+
+function instanceOf(Database) {
+  const db = Object.create(Database.prototype);
   db.bucket = "saturn";
   db.scope_name = "test";
   db.collection_name = "users";
@@ -63,6 +68,13 @@ function createDatabase(durationMs, env = {}) {
   return db;
 }
 
+// Opted in wholesale: per-resource resolution has its own tests further down.
+function createDatabase(durationMs, env = {}) {
+  const db = instanceOf(loadDriver(durationMs, env));
+  db.auto_optimize = true;
+  return db;
+}
+
 test("auto-optimization only starts above the default 3-second threshold", async () => {
   for (const duration of [1001, 2999, 3000, 3001]) {
     const db = createDatabase(duration);
@@ -235,3 +247,147 @@ test("a default collection uses default names in its inventory lookup", async ()
   }));
   assert.match(createStatements(scope)[0].statement, /`saturn`\.`_default`\.`_default`/);
 });
+
+// ---- Per-resource opt-in for auto cover indexing ----
+
+// Nothing decided at instance level: resolution falls through to the call,
+// the application policy and the static lists.
+function createUnconfiguredDatabase(env = {}) {
+  return instanceOf(loadDriver(3001, env));
+}
+
+async function optimizationsFor(db, statement, ...queryArgs) {
+  let optimizations = 0;
+  db.optimize = async () => { optimizations += 1; };
+  await db.query(statement, ...queryArgs);
+  return optimizations;
+}
+
+const slowLookup = "SELECT name FROM users WHERE status = 'active'";
+const EVERY_RESOURCE = { SPICE_COUCHBASE_OPTIMIZE_RESOURCES: "*" };
+
+test("no resource is auto-indexed unless something opts it in", async () => {
+  assert.equal(await optimizationsFor(createUnconfiguredDatabase(), slowLookup), 0);
+  assert.equal(await optimizationsFor(
+    createUnconfiguredDatabase({ SPICE_COUCHBASE_OPTIMIZE_RESOURCES: "" }), slowLookup), 0);
+});
+
+test("SPICE_COUCHBASE_OPTIMIZE_RESOURCES enables the instance's collection, case-insensitively", async () => {
+  const env = { SPICE_COUCHBASE_OPTIMIZE_RESOURCES: " Users ,audit_log,, `sessions` " };
+  assert.equal(await optimizationsFor(createUnconfiguredDatabase(env), slowLookup), 1);
+
+  const sessions = createUnconfiguredDatabase(env);
+  sessions.collection_name = "sessions";
+  assert.equal(await optimizationsFor(sessions, "SELECT id FROM sessions"), 1);
+
+  const orders = createUnconfiguredDatabase(env);
+  orders.collection_name = "orders";
+  assert.equal(await optimizationsFor(orders, "SELECT total FROM orders WHERE status = 'paid'"), 0);
+});
+
+test("a wildcard restores auto indexing for every resource", async () => {
+  const db = createUnconfiguredDatabase(EVERY_RESOURCE);
+  db.collection_name = "anything";
+  assert.equal(await optimizationsFor(db, "SELECT a FROM anything WHERE b = 1"), 1);
+});
+
+test("legacy bucket mode resolves the resource from the type literal the driver adds", async () => {
+  const env = { SPICE_COUCHBASE_OPTIMIZE_RESOURCES: "audit_log" };
+  for (const [type, expected] of [["audit_log", 1], ["users", 0]]) {
+    const db = createUnconfiguredDatabase(env);
+    db.collection_name = "_default";
+    db.scope_name = "_default";
+    assert.equal(await optimizationsFor(db,
+      `SELECT actor FROM \`saturn\` WHERE type = '${type}' AND deleted = false`), expected, type);
+  }
+});
+
+test("a fully qualified or aliased FROM keyspace matches on its last segment", async () => {
+  const env = { SPICE_COUCHBASE_OPTIMIZE_RESOURCES: "logs" };
+  for (const statement of [
+    "SELECT level FROM `default`:`saturn`.`app`.`logs` WHERE level = 'error'",
+    "SELECT l.level FROM saturn.app.logs AS l WHERE l.level = 'error'",
+    "SELECT u.name FROM users AS u JOIN `logs` AS l ON l.user_id = META(u).id",
+  ]) {
+    const db = createUnconfiguredDatabase(env);
+    db.collection_name = "_default";
+    assert.equal(await optimizationsFor(db, statement), 1, statement);
+  }
+});
+
+test("connection or constructor lists work without the environment variable", async () => {
+  const db = createUnconfiguredDatabase();
+  db.enabled_resources = new Set(["users"]);
+  assert.equal(await optimizationsFor(db, slowLookup), 1);
+});
+
+test("an explicit instance setting overrides every resource list", async () => {
+  const on = createUnconfiguredDatabase();
+  on.auto_optimize = true;
+  assert.equal(await optimizationsFor(on, slowLookup), 1);
+
+  const off = createUnconfiguredDatabase(EVERY_RESOURCE);
+  off.auto_optimize = false;
+  assert.equal(await optimizationsFor(off, slowLookup), 0);
+});
+
+test("a single call can opt in or out through query options", async () => {
+  assert.equal(await optimizationsFor(createUnconfiguredDatabase(), slowLookup, { auto_optimize: true }), 1);
+  assert.equal(await optimizationsFor(createUnconfiguredDatabase(EVERY_RESOURCE), slowLookup, { auto_optimize: false }), 0);
+  assert.equal(await optimizationsFor(createUnconfiguredDatabase(EVERY_RESOURCE), slowLookup, "cluster", { optimize: false }), 0);
+  // The scan-consistency shorthand is not an opt-in or opt-out.
+  assert.equal(await optimizationsFor(createUnconfiguredDatabase(EVERY_RESOURCE), slowLookup, "cluster", true), 1);
+  assert.equal(await optimizationsFor(createUnconfiguredDatabase(), slowLookup, "cluster", true), 0);
+});
+
+test("an application policy decides per resource and sees the statement context", async () => {
+  const Database = loadDriver(3001);
+  const seen = [];
+  Database.setAutoOptimizePolicy((resource, context) => {
+    seen.push([resource, context]);
+    return resource === "users";
+  });
+  assert.equal(Database.getAutoOptimizePolicy().length, 2);
+
+  const users = instanceOf(Database);
+  users.connection_name = "default";
+  assert.equal(await optimizationsFor(users, slowLookup), 1);
+  // Objects cross the vm boundary, so compare by value rather than prototype.
+  assert.equal(JSON.stringify(seen[0]), JSON.stringify(["users", { query: slowLookup, connection: "default", collection: "users" }]));
+
+  const orders = instanceOf(Database);
+  orders.collection_name = "orders";
+  assert.equal(await optimizationsFor(orders, "SELECT total FROM orders WHERE status = 'paid'"), 0);
+});
+
+test("a policy that returns undefined defers to the static lists", async () => {
+  for (const [env, expected] of [[{ SPICE_COUCHBASE_OPTIMIZE_RESOURCES: "users" }, 1], [{}, 0]]) {
+    const Database = loadDriver(3001, env);
+    Database.setAutoOptimizePolicy(() => undefined);
+    assert.equal(await optimizationsFor(instanceOf(Database), slowLookup), expected);
+  }
+});
+
+test("an explicit policy refusal wins over the wildcard, and a throwing policy counts as a refusal", async () => {
+  for (const policy of [() => false, () => { throw new Error("settings unavailable"); }]) {
+    const Database = loadDriver(3001, EVERY_RESOURCE);
+    Database.setAutoOptimizePolicy(policy);
+    assert.equal(await optimizationsFor(instanceOf(Database), slowLookup), 0);
+  }
+});
+
+test("the policy can be cleared and must be a function", async () => {
+  const Database = loadDriver(3001, EVERY_RESOURCE);
+  Database.setAutoOptimizePolicy(() => false);
+  Database.setAutoOptimizePolicy(null);
+  assert.equal(Database.getAutoOptimizePolicy(), null);
+  assert.equal(await optimizationsFor(instanceOf(Database), slowLookup), 1);
+  assert.throws(() => Database.setAutoOptimizePolicy("users"), { name: "TypeError" });
+});
+
+test("a declined statement is not remembered as already optimised", async () => {
+  const db = createUnconfiguredDatabase();
+  assert.equal(await optimizationsFor(db, slowLookup), 0);
+  db.auto_optimize = true;
+  assert.equal(await optimizationsFor(db, slowLookup), 1);
+});