spice-couchbase 2.0.57 → 2.0.59

package/build/index.count-query.test.js

@@ -0,0 +1,163 @@
+"use strict";
+/**
+ * Regression tests for the WHERE clause shared by the data and count queries in `search()`.
+ *
+ * The count query used to be built from a `where_clause` that held the bare literal `" WHERE "`,
+ * while the conditions were appended only to the data query. Every `do_count` request therefore
+ * emitted `... FROM \`bucket\`   WHERE` and Couchbase answered
+ * "syntax error - at end of input" (code 3000) with a 400 — on every resource, for every caller.
+ *
+ * It survived the existing suite because nothing exercised the count statement.
+ */
+
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
+
+function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
+
+var assert = require("assert");
+
+var Database = require("./index");
+
+var passed = 0;
+var failed = 0;
+/** Captures the two statements `search()` produces without touching a cluster. */
+
+function createSearchDb() {
+  var statements = {
+    data: null,
+    count: null
+  };
+  var db = Object.create(Database.prototype);
+  db.use_bucket_name = false;
+  db.bucket = "test";
+  db.scope_name = "scope";
+  db.collection_name = "platformtask";
+
+  db.getData =
+  /*#__PURE__*/
+  function () {
+    var _ref = _asyncToGenerator(function* (statement) {
+      statements.data = statement;
+      return [];
+    });
+
+    return function (_x) {
+      return _ref.apply(this, arguments);
+    };
+  }();
+
+  db.getCount =
+  /*#__PURE__*/
+  function () {
+    var _ref2 = _asyncToGenerator(function* (statement) {
+      statements.count = statement;
+      return 0;
+    });
+
+    return function (_x2) {
+      return _ref2.apply(this, arguments);
+    };
+  }();
+
+  db.optimize = () => {};
+
+  return {
+    db,
+    statements
+  };
+}
+
+function test(_x3, _x4) {
+  return _test.apply(this, arguments);
+}
+
+function _test() {
+  _test = _asyncToGenerator(function* (name, fn) {
+    try {
+      yield fn();
+      console.log("\u2713 ".concat(name));
+      passed += 1;
+    } catch (error) {
+      console.log("\u2717 ".concat(name));
+      console.log("  Error: ".concat(error.stack || error.message));
+      failed += 1;
+    }
+  });
+  return _test.apply(this, arguments);
+}
+
+function run() {
+  return _run.apply(this, arguments);
+}
+
+function _run() {
+  _run = _asyncToGenerator(function* () {
+    console.log("\n--- count query tests ---");
+    yield test("the count query carries the predicates, not a dangling WHERE",
+    /*#__PURE__*/
+    _asyncToGenerator(function* () {
+      var {
+        db,
+        statements
+      } = createSearchDb();
+      yield db.search("platformtask", "id", "status = \"pending\"", 1, 0, null, "true");
+      assert.ok(statements.count.includes("status = \"pending\""), "count query lost its predicate: ".concat(statements.count));
+      assert.ok(!/WHERE\s*$/.test(statements.count.trim()), "count query ends in a dangling WHERE: ".concat(statements.count));
+    }));
+    yield test("multiple conditions are joined with AND in the count query",
+    /*#__PURE__*/
+    _asyncToGenerator(function* () {
+      var {
+        db,
+        statements
+      } = createSearchDb();
+      yield db.search("platformtask", "id", "status = \"pending\" AND assignee_type = \"user\"", 1, 0, null, "true");
+      assert.ok(statements.count.includes("status = \"pending\""));
+      assert.ok(statements.count.includes("assignee_type = \"user\""));
+    }));
+    yield test("an unfiltered count emits no WHERE at all",
+    /*#__PURE__*/
+    _asyncToGenerator(function* () {
+      var {
+        db,
+        statements
+      } = createSearchDb();
+      yield db.search("platformtask", "id", "", 1, 0, null, "true");
+      assert.ok(!/WHERE/i.test(statements.count), "unfiltered count should have no WHERE: ".concat(statements.count));
+    }));
+    yield test("the data and count queries agree on the WHERE clause",
+    /*#__PURE__*/
+    _asyncToGenerator(function* () {
+      var {
+        db,
+        statements
+      } = createSearchDb();
+      yield db.search("platformtask", "id", "status = \"blocked\"", 1, 0, null, "true");
+
+      var whereOf = sql => {
+        var m = sql.match(/WHERE[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+([\s\S]*?)(?:[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+ORDER BY|[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+LIMIT|[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+OFFSET|$)/i);
+        return m ? m[1].trim() : null;
+      };
+
+      assert.strictEqual(whereOf(statements.data), whereOf(statements.count));
+    }));
+    yield test("no count statement is produced when do_count is off",
+    /*#__PURE__*/
+    _asyncToGenerator(function* () {
+      var {
+        db,
+        statements
+      } = createSearchDb();
+      yield db.search("platformtask", "id", "status = \"pending\"", 1, 0, null, "false");
+      assert.strictEqual(statements.count, null);
+      assert.ok(statements.data.includes("status = \"pending\""));
+    }));
+    console.log("\n========================================");
+    console.log("Tests: ".concat(passed + failed, " | Passed: ").concat(passed, " | Failed: ").concat(failed));
+    console.log("========================================");
+    if (failed > 0) process.exit(1);
+  });
+  return _run.apply(this, arguments);
+}
+
+run();

package/build/index.js

@@ -1,5 +1,7 @@
 "use strict";
 
+var _process$env$SPICE_CO;
+
 function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); keys.push.apply(keys, symbols); } return keys; }
 
 function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
@@ -33,7 +35,38 @@ if (!Promise.allSettled) {
 } // Couchbase lookupIn has a limit of 16 operations per call
 
 
-var LOOKUP_IN_MAX_SPECS = 16;
+var LOOKUP_IN_MAX_SPECS = 16; // Auto-optimisation guard rails. `query()` fires `optimize()` (an EXPLAIN plus
+// a possible CREATE INDEX) for any statement slower than SLOW_QUERY_MS. When
+// the cluster is degraded every statement is slow, so without limits each
+// query spawned two more heavy ones and deepened the outage. These caps make it
+// a bounded, once-per-statement background task that backs off on failure.
+
+var SLOW_QUERY_MS = Number(process.env.SPICE_COUCHBASE_SLOW_QUERY_MS || 1000);
+var AUTO_OPTIMIZE_ENABLED = !["0", "false", "no", "off"].includes(String((_process$env$SPICE_CO = process.env.SPICE_COUCHBASE_AUTO_OPTIMIZE) !== null && _process$env$SPICE_CO !== void 0 ? _process$env$SPICE_CO : "true").trim().toLowerCase());
+var OPTIMIZE_MAX_INFLIGHT = Number(process.env.SPICE_COUCHBASE_OPTIMIZE_MAX_INFLIGHT || 1);
+var OPTIMIZE_FAILURE_COOLDOWN_MS = Number(process.env.SPICE_COUCHBASE_OPTIMIZE_COOLDOWN_MS || 5 * 60 * 1000); // Statements already optimised in this process. Bounded so it cannot grow
+// without limit on a workload with many distinct literals.
+
+var OPTIMIZE_SEEN_LIMIT = 1000;
+var optimizeState = {
+  inFlight: 0,
+  seen: new Set(),
+  cooldownUntil: 0
+};
+
+function shouldOptimize(query) {
+  if (!AUTO_OPTIMIZE_ENABLED) return false;
+  if (optimizeState.inFlight >= OPTIMIZE_MAX_INFLIGHT) return false;
+  if (Date.now() < optimizeState.cooldownUntil) return false;
+  if (optimizeState.seen.has(query)) return false;
+
+  if (optimizeState.seen.size >= OPTIMIZE_SEEN_LIMIT) {
+    optimizeState.seen.delete(optimizeState.seen.values().next().value);
+  }
+
+  optimizeState.seen.add(query);
+  return true;
+}
 
 function normalizeScanConsistency(value) {
   if (value === true || value === QueryScanConsistency.RequestPlus || typeof value === "string" && ["true", "request_plus"].includes(value.toLowerCase())) {
@@ -868,6 +901,8 @@ module.exports = class Database {
     var _this19 = this;
 
     return _asyncToGenerator(function* () {
+      optimizeState.inFlight += 1;
+
       try {
         var _explainResponse$rows;
 
@@ -894,9 +929,15 @@ module.exports = class Database {
         }); // 4. Execute the index creation if we got a valid statement
 
         if (indexStatement) {
-          var response = yield exec_scope.query(indexStatement);
+          yield exec_scope.query(indexStatement);
         }
-      } catch (e) {// Silently handle errors (index may already exist, etc.)
+      } catch (e) {
+        // A failure here (timeout, index already exists, cluster busy) almost
+        // always means the cluster is struggling. Stop adding EXPLAIN and
+        // CREATE INDEX load for a while rather than piling on.
+        optimizeState.cooldownUntil = Date.now() + OPTIMIZE_FAILURE_COOLDOWN_MS;
+      } finally {
+        optimizeState.inFlight -= 1;
       }
     })();
   }
@@ -929,10 +970,11 @@ module.exports = class Database {
 
       var endTime = process.hrtime.bigint(); // Calculate duration in microseconds
 
-      var durationInMilliseconds = (endTime - startTime) / BigInt(1000000); // optimize if response time is greater than 1 second
+      var durationInMilliseconds = (endTime - startTime) / BigInt(1000000); // Optimise slow statements in the background, subject to the guard rails
+      // declared at the top of this file. Never awaited, never allowed to reject.
 
-      if (durationInMilliseconds > 1000) {
-        _this20.optimize(query, exec_scope);
+      if (durationInMilliseconds > SLOW_QUERY_MS && shouldOptimize(sanitizedQuery)) {
+        _this20.optimize(sanitizedQuery, exec_scope).catch(() => {});
       }
 
       return response;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spice-couchbase",
-  "version": "2.0.57",
+  "version": "2.0.59",
   "description": "",
   "main": "build/index.js",
   "scripts": {

package/src/index.js

@@ -25,6 +25,44 @@ if (!Promise.allSettled) {
 // Couchbase lookupIn has a limit of 16 operations per call
 const LOOKUP_IN_MAX_SPECS = 16;
 
+// Auto-optimisation guard rails. `query()` fires `optimize()` (an EXPLAIN plus
+// a possible CREATE INDEX) for any statement slower than SLOW_QUERY_MS. When
+// the cluster is degraded every statement is slow, so without limits each
+// query spawned two more heavy ones and deepened the outage. These caps make it
+// a bounded, once-per-statement background task that backs off on failure.
+const SLOW_QUERY_MS = Number(process.env.SPICE_COUCHBASE_SLOW_QUERY_MS || 1000);
+const AUTO_OPTIMIZE_ENABLED = !["0", "false", "no", "off"].includes(
+  String(process.env.SPICE_COUCHBASE_AUTO_OPTIMIZE ?? "true")
+    .trim()
+    .toLowerCase(),
+);
+const OPTIMIZE_MAX_INFLIGHT = Number(
+  process.env.SPICE_COUCHBASE_OPTIMIZE_MAX_INFLIGHT || 1,
+);
+const OPTIMIZE_FAILURE_COOLDOWN_MS = Number(
+  process.env.SPICE_COUCHBASE_OPTIMIZE_COOLDOWN_MS || 5 * 60 * 1000,
+);
+// Statements already optimised in this process. Bounded so it cannot grow
+// without limit on a workload with many distinct literals.
+const OPTIMIZE_SEEN_LIMIT = 1000;
+const optimizeState = {
+  inFlight: 0,
+  seen: new Set(),
+  cooldownUntil: 0,
+};
+
+function shouldOptimize(query) {
+  if (!AUTO_OPTIMIZE_ENABLED) return false;
+  if (optimizeState.inFlight >= OPTIMIZE_MAX_INFLIGHT) return false;
+  if (Date.now() < optimizeState.cooldownUntil) return false;
+  if (optimizeState.seen.has(query)) return false;
+  if (optimizeState.seen.size >= OPTIMIZE_SEEN_LIMIT) {
+    optimizeState.seen.delete(optimizeState.seen.values().next().value);
+  }
+  optimizeState.seen.add(query);
+  return true;
+}
+
 function normalizeScanConsistency(value) {
   if (
     value === true ||
@@ -816,6 +854,7 @@ module.exports = class Database {
   }
 
   async optimize(query, exec_scope) {
+    optimizeState.inFlight += 1;
     try {
       const { buildCoveringIndex } = require("./utility");
 
@@ -839,10 +878,15 @@ module.exports = class Database {
 
       // 4. Execute the index creation if we got a valid statement
       if (indexStatement) {
-        const response = await exec_scope.query(indexStatement);
+        await exec_scope.query(indexStatement);
       }
     } catch (e) {
-      // Silently handle errors (index may already exist, etc.)
+      // A failure here (timeout, index already exists, cluster busy) almost
+      // always means the cluster is struggling. Stop adding EXPLAIN and
+      // CREATE INDEX load for a while rather than piling on.
+      optimizeState.cooldownUntil = Date.now() + OPTIMIZE_FAILURE_COOLDOWN_MS;
+    } finally {
+      optimizeState.inFlight -= 1;
     }
   }
 
@@ -873,9 +917,13 @@ module.exports = class Database {
     let endTime = process.hrtime.bigint();
     // Calculate duration in microseconds
     let durationInMilliseconds = (endTime - startTime) / BigInt(1000000);
-    // optimize if response time is greater than 1 second
-    if (durationInMilliseconds > 1000) {
-      this.optimize(query, exec_scope);
+    // Optimise slow statements in the background, subject to the guard rails
+    // declared at the top of this file. Never awaited, never allowed to reject.
+    if (
+      durationInMilliseconds > SLOW_QUERY_MS &&
+      shouldOptimize(sanitizedQuery)
+    ) {
+      this.optimize(sanitizedQuery, exec_scope).catch(() => {});
     }
     return response;
   }