spice-couchbase 2.0.56 → 2.0.59

package/build/index.count-query.test.js

@@ -0,0 +1,163 @@
+"use strict";
+/**
+ * Regression tests for the WHERE clause shared by the data and count queries in `search()`.
+ *
+ * The count query used to be built from a `where_clause` that held the bare literal `" WHERE "`,
+ * while the conditions were appended only to the data query. Every `do_count` request therefore
+ * emitted `... FROM \`bucket\`   WHERE` and Couchbase answered
+ * "syntax error - at end of input" (code 3000) with a 400 — on every resource, for every caller.
+ *
+ * It survived the existing suite because nothing exercised the count statement.
+ */
+
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { Promise.resolve(value).then(_next, _throw); } }
+
+function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
+
+var assert = require("assert");
+
+var Database = require("./index");
+
+var passed = 0;
+var failed = 0;
+/** Captures the two statements `search()` produces without touching a cluster. */
+
+function createSearchDb() {
+  var statements = {
+    data: null,
+    count: null
+  };
+  var db = Object.create(Database.prototype);
+  db.use_bucket_name = false;
+  db.bucket = "test";
+  db.scope_name = "scope";
+  db.collection_name = "platformtask";
+
+  db.getData =
+  /*#__PURE__*/
+  function () {
+    var _ref = _asyncToGenerator(function* (statement) {
+      statements.data = statement;
+      return [];
+    });
+
+    return function (_x) {
+      return _ref.apply(this, arguments);
+    };
+  }();
+
+  db.getCount =
+  /*#__PURE__*/
+  function () {
+    var _ref2 = _asyncToGenerator(function* (statement) {
+      statements.count = statement;
+      return 0;
+    });
+
+    return function (_x2) {
+      return _ref2.apply(this, arguments);
+    };
+  }();
+
+  db.optimize = () => {};
+
+  return {
+    db,
+    statements
+  };
+}
+
+function test(_x3, _x4) {
+  return _test.apply(this, arguments);
+}
+
+function _test() {
+  _test = _asyncToGenerator(function* (name, fn) {
+    try {
+      yield fn();
+      console.log("\u2713 ".concat(name));
+      passed += 1;
+    } catch (error) {
+      console.log("\u2717 ".concat(name));
+      console.log("  Error: ".concat(error.stack || error.message));
+      failed += 1;
+    }
+  });
+  return _test.apply(this, arguments);
+}
+
+function run() {
+  return _run.apply(this, arguments);
+}
+
+function _run() {
+  _run = _asyncToGenerator(function* () {
+    console.log("\n--- count query tests ---");
+    yield test("the count query carries the predicates, not a dangling WHERE",
+    /*#__PURE__*/
+    _asyncToGenerator(function* () {
+      var {
+        db,
+        statements
+      } = createSearchDb();
+      yield db.search("platformtask", "id", "status = \"pending\"", 1, 0, null, "true");
+      assert.ok(statements.count.includes("status = \"pending\""), "count query lost its predicate: ".concat(statements.count));
+      assert.ok(!/WHERE\s*$/.test(statements.count.trim()), "count query ends in a dangling WHERE: ".concat(statements.count));
+    }));
+    yield test("multiple conditions are joined with AND in the count query",
+    /*#__PURE__*/
+    _asyncToGenerator(function* () {
+      var {
+        db,
+        statements
+      } = createSearchDb();
+      yield db.search("platformtask", "id", "status = \"pending\" AND assignee_type = \"user\"", 1, 0, null, "true");
+      assert.ok(statements.count.includes("status = \"pending\""));
+      assert.ok(statements.count.includes("assignee_type = \"user\""));
+    }));
+    yield test("an unfiltered count emits no WHERE at all",
+    /*#__PURE__*/
+    _asyncToGenerator(function* () {
+      var {
+        db,
+        statements
+      } = createSearchDb();
+      yield db.search("platformtask", "id", "", 1, 0, null, "true");
+      assert.ok(!/WHERE/i.test(statements.count), "unfiltered count should have no WHERE: ".concat(statements.count));
+    }));
+    yield test("the data and count queries agree on the WHERE clause",
+    /*#__PURE__*/
+    _asyncToGenerator(function* () {
+      var {
+        db,
+        statements
+      } = createSearchDb();
+      yield db.search("platformtask", "id", "status = \"blocked\"", 1, 0, null, "true");
+
+      var whereOf = sql => {
+        var m = sql.match(/WHERE[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+([\s\S]*?)(?:[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+ORDER BY|[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+LIMIT|[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]+OFFSET|$)/i);
+        return m ? m[1].trim() : null;
+      };
+
+      assert.strictEqual(whereOf(statements.data), whereOf(statements.count));
+    }));
+    yield test("no count statement is produced when do_count is off",
+    /*#__PURE__*/
+    _asyncToGenerator(function* () {
+      var {
+        db,
+        statements
+      } = createSearchDb();
+      yield db.search("platformtask", "id", "status = \"pending\"", 1, 0, null, "false");
+      assert.strictEqual(statements.count, null);
+      assert.ok(statements.data.includes("status = \"pending\""));
+    }));
+    console.log("\n========================================");
+    console.log("Tests: ".concat(passed + failed, " | Passed: ").concat(passed, " | Failed: ").concat(failed));
+    console.log("========================================");
+    if (failed > 0) process.exit(1);
+  });
+  return _run.apply(this, arguments);
+}
+
+run();

package/build/index.js

@@ -1,5 +1,7 @@
 "use strict";
 
+var _process$env$SPICE_CO;
+
 function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); keys.push.apply(keys, symbols); } return keys; }
 
 function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
@@ -33,7 +35,38 @@ if (!Promise.allSettled) {
 } // Couchbase lookupIn has a limit of 16 operations per call
 
 
-var LOOKUP_IN_MAX_SPECS = 16;
+var LOOKUP_IN_MAX_SPECS = 16; // Auto-optimisation guard rails. `query()` fires `optimize()` (an EXPLAIN plus
+// a possible CREATE INDEX) for any statement slower than SLOW_QUERY_MS. When
+// the cluster is degraded every statement is slow, so without limits each
+// query spawned two more heavy ones and deepened the outage. These caps make it
+// a bounded, once-per-statement background task that backs off on failure.
+
+var SLOW_QUERY_MS = Number(process.env.SPICE_COUCHBASE_SLOW_QUERY_MS || 1000);
+var AUTO_OPTIMIZE_ENABLED = !["0", "false", "no", "off"].includes(String((_process$env$SPICE_CO = process.env.SPICE_COUCHBASE_AUTO_OPTIMIZE) !== null && _process$env$SPICE_CO !== void 0 ? _process$env$SPICE_CO : "true").trim().toLowerCase());
+var OPTIMIZE_MAX_INFLIGHT = Number(process.env.SPICE_COUCHBASE_OPTIMIZE_MAX_INFLIGHT || 1);
+var OPTIMIZE_FAILURE_COOLDOWN_MS = Number(process.env.SPICE_COUCHBASE_OPTIMIZE_COOLDOWN_MS || 5 * 60 * 1000); // Statements already optimised in this process. Bounded so it cannot grow
+// without limit on a workload with many distinct literals.
+
+var OPTIMIZE_SEEN_LIMIT = 1000;
+var optimizeState = {
+  inFlight: 0,
+  seen: new Set(),
+  cooldownUntil: 0
+};
+
+function shouldOptimize(query) {
+  if (!AUTO_OPTIMIZE_ENABLED) return false;
+  if (optimizeState.inFlight >= OPTIMIZE_MAX_INFLIGHT) return false;
+  if (Date.now() < optimizeState.cooldownUntil) return false;
+  if (optimizeState.seen.has(query)) return false;
+
+  if (optimizeState.seen.size >= OPTIMIZE_SEEN_LIMIT) {
+    optimizeState.seen.delete(optimizeState.seen.values().next().value);
+  }
+
+  optimizeState.seen.add(query);
+  return true;
+}
 
 function normalizeScanConsistency(value) {
   if (value === true || value === QueryScanConsistency.RequestPlus || typeof value === "string" && ["true", "request_plus"].includes(value.toLowerCase())) {
@@ -704,10 +737,15 @@ module.exports = class Database {
 
         if (query) {
           conditions.push(query);
-        }
+        } // The conditions are folded into `where_clause` itself so that both queries below are built
+        // from one string. Previously this held the bare literal " WHERE " and only the data query
+        // appended `conditions.join(" AND ")`, which left the count query emitting a dangling
+        // `WHERE` — Couchbase answered "syntax error - at end of input" and every `do_count=true`
+        // request failed with a 400, on every resource.
 
-        var where_clause = conditions.length ? " WHERE " : "";
-        query = front_query + where_clause + conditions.join(" AND ");
+
+        var where_clause = conditions.length ? " WHERE ".concat(conditions.join(" AND ")) : "";
+        query = front_query + where_clause;
         var count_query = "SELECT count(meta(".concat(bucket, ").id) as count FROM ").concat(bucket, " ").concat(join, " ").concat(where_clause); //console.log("Cound Query::", count_query);
 
         var order_by_clause = order_by ? " ORDER BY ".concat(order_by) : "";
@@ -863,6 +901,8 @@ module.exports = class Database {
     var _this19 = this;
 
     return _asyncToGenerator(function* () {
+      optimizeState.inFlight += 1;
+
       try {
         var _explainResponse$rows;
 
@@ -889,9 +929,15 @@ module.exports = class Database {
         }); // 4. Execute the index creation if we got a valid statement
 
         if (indexStatement) {
-          var response = yield exec_scope.query(indexStatement);
+          yield exec_scope.query(indexStatement);
         }
-      } catch (e) {// Silently handle errors (index may already exist, etc.)
+      } catch (e) {
+        // A failure here (timeout, index already exists, cluster busy) almost
+        // always means the cluster is struggling. Stop adding EXPLAIN and
+        // CREATE INDEX load for a while rather than piling on.
+        optimizeState.cooldownUntil = Date.now() + OPTIMIZE_FAILURE_COOLDOWN_MS;
+      } finally {
+        optimizeState.inFlight -= 1;
       }
     })();
   }
@@ -924,10 +970,11 @@ module.exports = class Database {
 
       var endTime = process.hrtime.bigint(); // Calculate duration in microseconds
 
-      var durationInMilliseconds = (endTime - startTime) / BigInt(1000000); // optimize if response time is greater than 1 second
+      var durationInMilliseconds = (endTime - startTime) / BigInt(1000000); // Optimise slow statements in the background, subject to the guard rails
+      // declared at the top of this file. Never awaited, never allowed to reject.
 
-      if (durationInMilliseconds > 1000) {
-        _this20.optimize(query, exec_scope);
+      if (durationInMilliseconds > SLOW_QUERY_MS && shouldOptimize(sanitizedQuery)) {
+        _this20.optimize(sanitizedQuery, exec_scope).catch(() => {});
       }
 
       return response;

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "spice-couchbase",
-  "version": "2.0.56",
+  "version": "2.0.59",
   "description": "",
   "main": "build/index.js",
   "scripts": {
     "test": "node src/utility.test.js",
-    "test:query-consistency": "node src/index.query-consistency.test.js"
+    "test:query-consistency": "node src/index.query-consistency.test.js",
+    "test:count-query": "node src/index.count-query.test.js"
   },
   "author": "Chad Fraser <chad@sonover.com> (http://sonover.com)",
   "license": "ISC",

package/src/index.count-query.test.js

@@ -0,0 +1,129 @@
+"use strict";
+
+/**
+ * Regression tests for the WHERE clause shared by the data and count queries in `search()`.
+ *
+ * The count query used to be built from a `where_clause` that held the bare literal `" WHERE "`,
+ * while the conditions were appended only to the data query. Every `do_count` request therefore
+ * emitted `... FROM \`bucket\`   WHERE` and Couchbase answered
+ * "syntax error - at end of input" (code 3000) with a 400 — on every resource, for every caller.
+ *
+ * It survived the existing suite because nothing exercised the count statement.
+ */
+
+const assert = require("assert");
+const Database = require("./index");
+
+let passed = 0;
+let failed = 0;
+
+/** Captures the two statements `search()` produces without touching a cluster. */
+function createSearchDb() {
+  const statements = { data: null, count: null };
+  const db = Object.create(Database.prototype);
+
+  db.use_bucket_name = false;
+  db.bucket = "test";
+  db.scope_name = "scope";
+  db.collection_name = "platformtask";
+
+  db.getData = async (statement) => {
+    statements.data = statement;
+    return [];
+  };
+  db.getCount = async (statement) => {
+    statements.count = statement;
+    return 0;
+  };
+  db.optimize = () => {};
+
+  return { db, statements };
+}
+
+async function test(name, fn) {
+  try {
+    await fn();
+    console.log(`✓ ${name}`);
+    passed += 1;
+  } catch (error) {
+    console.log(`✗ ${name}`);
+    console.log(`  Error: ${error.stack || error.message}`);
+    failed += 1;
+  }
+}
+
+async function run() {
+  console.log("\n--- count query tests ---");
+
+  await test("the count query carries the predicates, not a dangling WHERE", async () => {
+    const { db, statements } = createSearchDb();
+
+    await db.search("platformtask", "id", `status = "pending"`, 1, 0, null, "true");
+
+    assert.ok(
+      statements.count.includes(`status = "pending"`),
+      `count query lost its predicate: ${statements.count}`
+    );
+    assert.ok(
+      !/WHERE\s*$/.test(statements.count.trim()),
+      `count query ends in a dangling WHERE: ${statements.count}`
+    );
+  });
+
+  await test("multiple conditions are joined with AND in the count query", async () => {
+    const { db, statements } = createSearchDb();
+
+    await db.search(
+      "platformtask",
+      "id",
+      `status = "pending" AND assignee_type = "user"`,
+      1,
+      0,
+      null,
+      "true"
+    );
+
+    assert.ok(statements.count.includes(`status = "pending"`));
+    assert.ok(statements.count.includes(`assignee_type = "user"`));
+  });
+
+  await test("an unfiltered count emits no WHERE at all", async () => {
+    const { db, statements } = createSearchDb();
+
+    await db.search("platformtask", "id", "", 1, 0, null, "true");
+
+    assert.ok(
+      !/WHERE/i.test(statements.count),
+      `unfiltered count should have no WHERE: ${statements.count}`
+    );
+  });
+
+  await test("the data and count queries agree on the WHERE clause", async () => {
+    const { db, statements } = createSearchDb();
+
+    await db.search("platformtask", "id", `status = "blocked"`, 1, 0, null, "true");
+
+    const whereOf = (sql) => {
+      const m = sql.match(/WHERE\s+(.*?)(?:\s+ORDER BY|\s+LIMIT|\s+OFFSET|$)/is);
+      return m ? m[1].trim() : null;
+    };
+    assert.strictEqual(whereOf(statements.data), whereOf(statements.count));
+  });
+
+  await test("no count statement is produced when do_count is off", async () => {
+    const { db, statements } = createSearchDb();
+
+    await db.search("platformtask", "id", `status = "pending"`, 1, 0, null, "false");
+
+    assert.strictEqual(statements.count, null);
+    assert.ok(statements.data.includes(`status = "pending"`));
+  });
+
+  console.log("\n========================================");
+  console.log(`Tests: ${passed + failed} | Passed: ${passed} | Failed: ${failed}`);
+  console.log("========================================");
+
+  if (failed > 0) process.exit(1);
+}
+
+run();

package/src/index.js

@@ -25,6 +25,44 @@ if (!Promise.allSettled) {
 // Couchbase lookupIn has a limit of 16 operations per call
 const LOOKUP_IN_MAX_SPECS = 16;
 
+// Auto-optimisation guard rails. `query()` fires `optimize()` (an EXPLAIN plus
+// a possible CREATE INDEX) for any statement slower than SLOW_QUERY_MS. When
+// the cluster is degraded every statement is slow, so without limits each
+// query spawned two more heavy ones and deepened the outage. These caps make it
+// a bounded, once-per-statement background task that backs off on failure.
+const SLOW_QUERY_MS = Number(process.env.SPICE_COUCHBASE_SLOW_QUERY_MS || 1000);
+const AUTO_OPTIMIZE_ENABLED = !["0", "false", "no", "off"].includes(
+  String(process.env.SPICE_COUCHBASE_AUTO_OPTIMIZE ?? "true")
+    .trim()
+    .toLowerCase(),
+);
+const OPTIMIZE_MAX_INFLIGHT = Number(
+  process.env.SPICE_COUCHBASE_OPTIMIZE_MAX_INFLIGHT || 1,
+);
+const OPTIMIZE_FAILURE_COOLDOWN_MS = Number(
+  process.env.SPICE_COUCHBASE_OPTIMIZE_COOLDOWN_MS || 5 * 60 * 1000,
+);
+// Statements already optimised in this process. Bounded so it cannot grow
+// without limit on a workload with many distinct literals.
+const OPTIMIZE_SEEN_LIMIT = 1000;
+const optimizeState = {
+  inFlight: 0,
+  seen: new Set(),
+  cooldownUntil: 0,
+};
+
+function shouldOptimize(query) {
+  if (!AUTO_OPTIMIZE_ENABLED) return false;
+  if (optimizeState.inFlight >= OPTIMIZE_MAX_INFLIGHT) return false;
+  if (Date.now() < optimizeState.cooldownUntil) return false;
+  if (optimizeState.seen.has(query)) return false;
+  if (optimizeState.seen.size >= OPTIMIZE_SEEN_LIMIT) {
+    optimizeState.seen.delete(optimizeState.seen.values().next().value);
+  }
+  optimizeState.seen.add(query);
+  return true;
+}
+
 function normalizeScanConsistency(value) {
   if (
     value === true ||
@@ -671,8 +709,15 @@ module.exports = class Database {
         conditions.push(query);
       }
 
-      const where_clause = conditions.length ? " WHERE " : "";
-      query = front_query + where_clause + conditions.join(" AND ");
+      // The conditions are folded into `where_clause` itself so that both queries below are built
+      // from one string. Previously this held the bare literal " WHERE " and only the data query
+      // appended `conditions.join(" AND ")`, which left the count query emitting a dangling
+      // `WHERE` — Couchbase answered "syntax error - at end of input" and every `do_count=true`
+      // request failed with a 400, on every resource.
+      const where_clause = conditions.length
+        ? ` WHERE ${conditions.join(" AND ")}`
+        : "";
+      query = front_query + where_clause;
 
       const count_query = `SELECT count(meta(${bucket}).id) as count FROM ${bucket} ${join} ${where_clause}`;
       //console.log("Cound Query::", count_query);
@@ -809,6 +854,7 @@ module.exports = class Database {
   }
 
   async optimize(query, exec_scope) {
+    optimizeState.inFlight += 1;
     try {
       const { buildCoveringIndex } = require("./utility");
 
@@ -832,10 +878,15 @@ module.exports = class Database {
 
       // 4. Execute the index creation if we got a valid statement
       if (indexStatement) {
-        const response = await exec_scope.query(indexStatement);
+        await exec_scope.query(indexStatement);
       }
     } catch (e) {
-      // Silently handle errors (index may already exist, etc.)
+      // A failure here (timeout, index already exists, cluster busy) almost
+      // always means the cluster is struggling. Stop adding EXPLAIN and
+      // CREATE INDEX load for a while rather than piling on.
+      optimizeState.cooldownUntil = Date.now() + OPTIMIZE_FAILURE_COOLDOWN_MS;
+    } finally {
+      optimizeState.inFlight -= 1;
     }
   }
 
@@ -866,9 +917,13 @@ module.exports = class Database {
     let endTime = process.hrtime.bigint();
     // Calculate duration in microseconds
     let durationInMilliseconds = (endTime - startTime) / BigInt(1000000);
-    // optimize if response time is greater than 1 second
-    if (durationInMilliseconds > 1000) {
-      this.optimize(query, exec_scope);
+    // Optimise slow statements in the background, subject to the guard rails
+    // declared at the top of this file. Never awaited, never allowed to reject.
+    if (
+      durationInMilliseconds > SLOW_QUERY_MS &&
+      shouldOptimize(sanitizedQuery)
+    ) {
+      this.optimize(sanitizedQuery, exec_scope).catch(() => {});
     }
     return response;
   }