spice-couchbase 2.0.60 → 2.0.62

package/src/utility.js

@@ -245,22 +245,17 @@ function buildOrderDirectionMap(orderByIndexFields) {
   const directionMap = new Map();
   for (const { name, direction } of orderByIndexFields) {
     if (direction === "DESC") {
-      directionMap.set(name.toLowerCase(), direction);
+      directionMap.set(name, direction);
     }
   }
   return directionMap;
 }
 
 function formatIndexField(field, orderDirectionMap) {
-  const direction = orderDirectionMap.get(field.toLowerCase());
+  const direction = orderDirectionMap.get(field);
   return `\`${field}\`${direction ? ` ${direction}` : ""}`;
 }
 
-function formatIndexKey(field, orderDirectionMap) {
-  const direction = orderDirectionMap.get(field.toLowerCase());
-  return `${field}${direction ? ` ${direction}` : ""}`;
-}
-
 /**
  * Splits SELECT clause by commas, respecting parentheses
  * @param {string} clause - The SELECT clause content
@@ -293,23 +288,8 @@ function splitSelectClause(clause) {
   return parts;
 }
 
-function hasQualifiedWildcardProjection(query) {
-  const selectMatch = query.match(/\bSELECT\b(.+?)\bFROM\b/is);
-  if (!selectMatch) {
-    return false;
-  }
-
-  return splitSelectClause(selectMatch[1].trim()).some((part) => {
-    const trimmedPart = part.trim();
-    return (
-      /^\(\s*`?\w+`?\s*\)\s*\.\s*\*$/.test(trimmedPart) ||
-      /^`?\w+`?\s*\.\s*\*$/.test(trimmedPart)
-    );
-  });
-}
-
 /**
- * Builds a covering or fallback non-covering index CREATE statement from a query
+ * Builds a covering index CREATE statement from a query
  * @param {string} query - The N1QL query to analyze
  * @param {Object} options - Configuration options
  * @param {string} options.bucket - The bucket name
@@ -318,6 +298,12 @@ function hasQualifiedWildcardProjection(query) {
  * @returns {string|null} - The CREATE INDEX statement, or null if cannot build
  */
 function buildCoveringIndex(query, options) {
+  return buildCoveringIndexDefinition(query, options)?.statement ?? null;
+}
+
+// Keep the access keys separate from projection-only keys so an older, wider
+// index can satisfy the same request without depending on its generated name.
+function buildCoveringIndexDefinition(query, options) {
   const { bucket, scope = "_default", collection = "_default" } = options;
 
   if (!bucket) {
@@ -326,9 +312,9 @@ function buildCoveringIndex(query, options) {
 
   // Extract fields from SELECT clause
   const selectFields = extractSelectFields(query);
-  const isNonCoveringFallback =
-    selectFields === null && hasQualifiedWildcardProjection(query);
-  if (selectFields === null && !isNonCoveringFallback) {
+  if (selectFields === null) {
+    // Full-document projections cannot be covered by the extracted fields.
+    // Skip them instead of creating a non-covering fallback index.
     return null;
   }
 
@@ -344,44 +330,41 @@ function buildCoveringIndex(query, options) {
   const orderByFields = orderByIndexFields.map((field) => field.name);
 
   // Remove 'deleted' from whereFields as it will be added first with INCLUDE MISSING
-  const filteredWhereFields = whereFields.filter(
-    (f) => f.toLowerCase() !== "deleted",
-  );
+  const filteredWhereFields = whereFields.filter((f) => f !== "deleted");
 
   // Track fields already added to avoid duplicates
-  const addedFields = new Set(filteredWhereFields.map((f) => f.toLowerCase()));
+  const addedFields = new Set(filteredWhereFields);
 
   // Filter GROUP BY fields (remove duplicates from WHERE, remove deleted)
   const filteredGroupByFields = groupByFields.filter((f) => {
-    const lower = f.toLowerCase();
-    if (addedFields.has(lower) || lower === "deleted") {
+    if (addedFields.has(f) || f === "deleted") {
       return false;
     }
-    addedFields.add(lower);
+    addedFields.add(f);
     return true;
   });
 
   // Filter ORDER BY fields (remove duplicates from WHERE and GROUP BY, remove deleted)
   const filteredOrderByFields = orderByFields.filter((f) => {
-    const lower = f.toLowerCase();
-    if (addedFields.has(lower) || lower === "deleted") {
+    if (addedFields.has(f) || f === "deleted") {
       return false;
     }
-    addedFields.add(lower);
+    addedFields.add(f);
     return true;
   });
 
   // Filter SELECT fields (remove duplicates from WHERE, GROUP BY, and ORDER BY, remove deleted)
-  const selectFieldsFiltered = (selectFields || []).filter((f) => {
-    const lower = f.toLowerCase();
-    if (addedFields.has(lower) || lower === "deleted") {
-      return false;
-    }
-    addedFields.add(lower);
-    return true;
-  });
+  const selectFieldsFiltered = selectFields
+    .filter((f) => {
+      if (addedFields.has(f) || f === "deleted") {
+        return false;
+      }
+      addedFields.add(f);
+      return true;
+    })
+    .sort();
 
-  // All fields for the index: WHERE -> GROUP BY -> ORDER BY -> SELECT
+  // Projection order has no effect on coverage. Keep access-key order intact.
   const allFields = [
     ...filteredWhereFields,
     ...filteredGroupByFields,
@@ -395,33 +378,153 @@ function buildCoveringIndex(query, options) {
   }
 
   // Build the field list with deleted INCLUDE MISSING first
-  const fieldList = [
+  const keys = [
     "`deleted` INCLUDE MISSING",
     ...allFields.map((f) => formatIndexField(f, orderDirectionMap)),
-  ].join(", ");
-
-  // Generate unique index name based on bucket, scope, collection, and fields
-  // This ensures multiple queries covered by the same index get the same hash
-  const indexFieldsKey = allFields
-    .map((field) => formatIndexKey(field, orderDirectionMap))
-    .sort()
-    .join(",");
-  const orderFieldsKey = orderByIndexFields
-    .map((field) => formatIndexKey(field.name, orderDirectionMap))
-    .join(",");
-  const indexMode = isNonCoveringFallback ? "noncovered" : "covered";
-  const indexKey = `${bucket}|${scope}|${collection}|mode:${indexMode}|fields:${indexFieldsKey}|order:${orderFieldsKey}`;
+  ];
+
+  // Hash the actual ordered definition, not query syntax. In particular, an
+  // explicit ORDER BY ... ASC must not rename an otherwise identical index.
+  const indexKey = JSON.stringify([bucket, scope, collection, keys]);
   const hash = generateHash(indexKey);
-  const indexName = `adv_${indexMode}_${hash}`;
+  const indexName = `adv_covered_${hash}`;
 
   // Build the CREATE INDEX statement
-  const indexStatement = `CREATE INDEX \`${indexName}\` ON \`default\`:\`${bucket}\`.\`${scope}\`.\`${collection}\`(${fieldList})`;
+  const quote = (identifier) => `\`${identifier.replace(/`/g, "``")}\``;
+  const indexStatement = `CREATE INDEX \`${indexName}\` ON \`default\`:${quote(bucket)}.${quote(scope)}.${quote(collection)}(${keys.join(", ")})`;
+
+  return {
+    name: indexName,
+    statement: indexStatement,
+    keys,
+    accessKeyCount:
+      1 + filteredWhereFields.length + filteredGroupByFields.length + filteredOrderByFields.length,
+    bucket,
+    scope,
+    collection,
+  };
+}
+
+// Only compare simple scalar keys. Do not treat expressions, array indexes,
+// partial indexes, or case-distinct document fields as equivalent.
+function normalizeIndexKey(key) {
+  if (typeof key !== "string") return null;
+  const match = key.trim().match(
+    /^(\(*\s*(?:`(?:[^`]|``)+`|[A-Za-z_]\w*)\s*\)*)(.*?)$/s,
+  );
+  if (!match) return null;
+  let expression = match[1].trim();
+  while (expression.startsWith("(") && expression.endsWith(")")) {
+    expression = expression.slice(1, -1).trim();
+  }
+  const identifier = expression.match(/^(?:`((?:[^`]|``)+)`|([A-Za-z_]\w*))$/);
+  if (!identifier) return null;
+  const attributes = match[2].trim().toUpperCase();
+  if (!/^(?:(?:ASC|DESC)(?:\s+INCLUDE\s+MISSING)?|INCLUDE\s+MISSING(?:\s+(?:ASC|DESC))?)?$/.test(attributes)) {
+    return null;
+  }
+  return {
+    field: identifier[1] !== undefined
+      ? identifier[1].replace(/``/g, "`")
+      : identifier[2],
+    direction: /\bDESC\b/.test(attributes) ? "DESC" : "ASC",
+    includeMissing: /\bINCLUDE\s+MISSING\b/.test(attributes),
+  };
+}
+
+// Call with indexes from the candidate's exact bucket/scope/collection. State
+// is deliberately not filtered: a matching deferred/building index is already
+// committed work, even though EXPLAIN cannot use it yet.
+function indexSatisfiesCoveringDefinition(index, candidate) {
+  if (
+    index.is_primary || index.condition || index.using !== "gsi" ||
+    !Array.isArray(index.index_key)
+  ) {
+    return false;
+  }
+  const existingKeys = index.index_key.map(normalizeIndexKey);
+  const requiredKeys = candidate.keys.map(normalizeIndexKey);
+  if (existingKeys.some((key) => !key) || requiredKeys.some((key) => !key)) return false;
+  for (let i = 0; i < candidate.accessKeyCount; i++) {
+    const existing = existingKeys[i];
+    const required = requiredKeys[i];
+    if (
+      !existing || existing.field !== required.field ||
+      existing.direction !== required.direction ||
+      existing.includeMissing !== required.includeMissing
+    ) return false;
+  }
+  const existingFields = new Set(existingKeys.map((key) => key.field));
+  return requiredKeys.slice(candidate.accessKeyCount).every((key) => existingFields.has(key.field));
+}
+
+/**
+ * Normalizes an exemption list into a Set of lower-case resource names.
+ * Accepts an array, a Set, or a comma-separated string (the env var form).
+ * Surrounding whitespace and backticks are ignored; empty entries are dropped.
+ * @param {string|string[]|Set<string>} value
+ * @returns {Set<string>}
+ */
+function parseResourceList(value) {
+  const names = new Set();
+  let items = [];
+  if (Array.isArray(value) || value instanceof Set) {
+    items = [...value];
+  } else if (typeof value === "string") {
+    items = value.split(",");
+  }
+  for (const item of items) {
+    if (typeof item !== "string") continue;
+    const name = item.trim().replace(/^`+|`+$/g, "").trim().toLowerCase();
+    if (name) names.add(name);
+  }
+  return names;
+}
+
+/**
+ * Extracts the resource names a statement touches, lower-cased:
+ * - the last path segment of every FROM / JOIN keyspace, unquoted, so
+ *   `default`:`bucket`.`scope`.`users` and users both yield "users";
+ * - every `type = '...'` literal, which is how legacy bucket-mode
+ *   deployments distinguish resources that share one keyspace.
+ * Only simple identifiers are recognised; subqueries and expressions are skipped.
+ * @param {string} query
+ * @returns {Set<string>}
+ */
+function extractStatementResources(query) {
+  const names = new Set();
+  if (typeof query !== "string") return names;
+
+  const identifier = "(?:`(?:[^`]|``)+`|[A-Za-z_][\\w$-]*)";
+  const keyspacePattern = new RegExp(
+    `\\b(?:FROM|JOIN)\\s+(${identifier}(?:\\s*[.:]\\s*${identifier})*)`,
+    "gi",
+  );
+  const segmentPattern = new RegExp(identifier, "g");
+  for (const match of query.matchAll(keyspacePattern)) {
+    const segments = match[1].match(segmentPattern) || [];
+    const last = (segments[segments.length - 1] || "").trim();
+    const name = last.startsWith("`")
+      ? last.slice(1, -1).replace(/``/g, "`")
+      : last;
+    if (name) names.add(name.toLowerCase());
+  }
+
+  const typePattern = /(?:`type`|\btype)\s*=\s*'((?:[^']|'')*)'/gi;
+  for (const match of query.matchAll(typePattern)) {
+    const name = match[1].replace(/''/g, "'").trim();
+    if (name) names.add(name.toLowerCase());
+  }
 
-  return indexStatement;
+  return names;
 }
 
 module.exports = {
   buildCoveringIndex,
+  buildCoveringIndexDefinition,
+  indexSatisfiesCoveringDefinition,
+  parseResourceList,
+  extractStatementResources,
   extractWhereFields,
   extractSelectFields,
   extractGroupByFields,

package/src/utility.test.js

@@ -8,6 +8,8 @@ const {
   extractGroupByFields,
   extractOrderByFields,
   generateHash,
+  parseResourceList,
+  extractStatementResources,
 } = require("./utility");
 
 // Test utilities
@@ -599,16 +601,16 @@ test("returns null when missing bucket option", () => {
   assert.strictEqual(result, null);
 });
 
-test("returns null when missing scope option", () => {
+test("uses the default scope when scope option is missing", () => {
   const query = "SELECT name FROM users WHERE status = 'active'";
   const result = buildCoveringIndex(query, { bucket: "test", collection: "users" });
-  assert.strictEqual(result, null);
+  assert.ok(result.includes("`test`.`_default`.`users`"));
 });
 
-test("returns null when missing collection option", () => {
+test("uses the default collection when collection option is missing", () => {
   const query = "SELECT name FROM users WHERE status = 'active'";
   const result = buildCoveringIndex(query, { bucket: "test", scope: "scope" });
-  assert.strictEqual(result, null);
+  assert.ok(result.includes("`test`.`scope`.`_default`"));
 });
 
 test("generates unique index name from query hash", () => {
@@ -1046,6 +1048,73 @@ test("field order: WHERE -> GROUP BY -> ORDER BY -> SELECT", () => {
   assert.ok(createdAtIndex < emailIndex);  // ORDER BY before SELECT
 });
 
+// ============================================
+// Tests for parseResourceList / extractStatementResources
+// ============================================
+console.log("\n--- parseResourceList tests ---");
+
+test("parses a comma-separated string, trimming, unquoting and lower-casing", () => {
+  const result = parseResourceList(" Users, `Audit_Log` ,, sessions ");
+  assert.deepStrictEqual([...result].sort(), ["audit_log", "sessions", "users"]);
+});
+
+test("accepts arrays and Sets and ignores non-strings", () => {
+  assert.deepStrictEqual([...parseResourceList(["Users", 42, null, " logs "])].sort(), ["logs", "users"]);
+  assert.deepStrictEqual([...parseResourceList(new Set(["A", "b"]))].sort(), ["a", "b"]);
+});
+
+test("returns an empty set for undefined, empty or unsupported input", () => {
+  assert.strictEqual(parseResourceList(undefined).size, 0);
+  assert.strictEqual(parseResourceList("").size, 0);
+  assert.strictEqual(parseResourceList(", ,").size, 0);
+  assert.strictEqual(parseResourceList({ users: true }).size, 0);
+});
+
+console.log("\n--- extractStatementResources tests ---");
+
+test("extracts a bare FROM keyspace", () => {
+  assert.deepStrictEqual([...extractStatementResources("SELECT name FROM users WHERE x = 1")], ["users"]);
+});
+
+test("extracts the last segment of a qualified or quoted keyspace", () => {
+  assert.deepStrictEqual(
+    [...extractStatementResources("SELECT name FROM `default`:`saturn`.`app`.`Users` AS u")],
+    ["users"],
+  );
+  assert.deepStrictEqual(
+    [...extractStatementResources("SELECT name FROM saturn.app.logs")],
+    ["logs"],
+  );
+});
+
+test("extracts JOIN keyspaces alongside FROM", () => {
+  const result = extractStatementResources(
+    "SELECT u.name FROM users AS u JOIN `orders` AS o ON o.user_id = META(u).id",
+  );
+  assert.deepStrictEqual([...result].sort(), ["orders", "users"]);
+});
+
+test("extracts type literals used by legacy bucket mode", () => {
+  const result = extractStatementResources(
+    "SELECT actor FROM `saturn` WHERE type = 'Audit_Log' AND deleted = false",
+  );
+  assert.deepStrictEqual([...result].sort(), ["audit_log", "saturn"]);
+  assert.ok(extractStatementResources("SELECT a FROM b WHERE `type`='x'").has("x"));
+  assert.ok(extractStatementResources("SELECT a FROM b WHERE type = 'it''s'").has("it's"));
+});
+
+test("does not treat META() arguments, subqueries or column names as keyspaces", () => {
+  const result = extractStatementResources(
+    "SELECT META(`users`).id AS id, from_date FROM (SELECT * FROM users) AS sub WHERE status = 'from'",
+  );
+  assert.deepStrictEqual([...result], ["users"]);
+});
+
+test("handles non-string input", () => {
+  assert.strictEqual(extractStatementResources(undefined).size, 0);
+  assert.strictEqual(extractStatementResources(null).size, 0);
+});
+
 // ============================================
 // Summary
 // ============================================