spice-couchbase 2.0.45 → 2.0.47

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "spice-couchbase",
-  "version": "2.0.45",
+  "version": "2.0.47",
   "description": "",
   "main": "build/index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "node src/utility.test.js"
   },
   "author": "Chad Fraser <chad@sonover.com> (http://sonover.com)",
   "license": "ISC",

package/src/index.js

@@ -19,8 +19,8 @@ if (!Promise.allSettled) {
           .catch((reason) => ({
             status: "rejected",
             reason,
-          }))
-      )
+          })),
+      ),
     );
 }
 
@@ -36,7 +36,7 @@ module.exports = class Database {
       scope,
       query_scope = "cluster",
       auto_create_collection,
-    } = {}
+    } = {},
   ) {
     if (connection) {
       this.connection_name = connection;
@@ -151,7 +151,7 @@ module.exports = class Database {
             { name: "scope_admin", bucket: this.db_config.bucket },
             { name: "data_reader", bucket: this.db_config.bucket },
           ],
-        }
+        },
       );
     } catch (e) {
       console.log(e);
@@ -162,7 +162,7 @@ module.exports = class Database {
     return (
       (this.db_config.api_base ||
         `http://${_.last(
-          _.first(this.db_config.nodes.split(",")).split("://")
+          _.first(this.db_config.nodes.split(",")).split("://"),
         )}`) +
       ":" +
       default_port
@@ -179,7 +179,7 @@ module.exports = class Database {
       if (!this.checkGlobelConnectionObject(this.collection_name)) {
         let collection_exists = await this.collectionExists(
           this.collection_name,
-          this.scope_name
+          this.scope_name,
         );
         if (!collection_exists) {
           await this.createCollection(this.collection_name, this.scope_name);
@@ -205,7 +205,7 @@ module.exports = class Database {
               username: that.db_config.username,
               password: that.db_config.password,
             },
-          }
+          },
         );
         let scope = _.find(all_scopes.data.scopes, { name: scope_name });
         let collectionSet = new Set();
@@ -248,7 +248,7 @@ module.exports = class Database {
   async createPrimaryIndex(scope_name, col_name) {
     try {
       let response = await this.query(
-        `CREATE PRIMARY INDEX ON default:\`${this.db_config.bucket}\`.\`${scope_name}\`.\`${col_name}\``
+        `CREATE PRIMARY INDEX ON default:\`${this.db_config.bucket}\`.\`${scope_name}\`.\`${col_name}\``,
       );
       return response;
     } catch (e) {
@@ -271,7 +271,8 @@ module.exports = class Database {
   async get(doc_id, { columns = [] } = {}) {
     try {
       const useLookupIn = columns.length > 0;
-      const collection = this.use_bucket_name ? this.connection : this.collection;
+      const collection =
+        this.use_bucket_name ? this.connection : this.collection;
 
       if (!useLookupIn) {
         let response = await collection.get(doc_id);
@@ -315,7 +316,7 @@ module.exports = class Database {
         document,
         {
           expiry: expiry,
-        }
+        },
       );
     } catch (e) {
       console.log(e.message, e, doc_id);
@@ -326,7 +327,7 @@ module.exports = class Database {
   async touch(doc_id, expiry = 0) {
     await (this.use_bucket_name ? this.connection : this.collection).touch(
       doc_id,
-      expiry
+      expiry,
     );
     return await this.get(doc_id);
   }
@@ -365,7 +366,8 @@ module.exports = class Database {
     }
     if (_.isArray(keys) && keys.length > 0) {
       const useLookupIn = columns.length > 0;
-      const collection = this.use_bucket_name ? this.connection : this.collection;
+      const collection =
+        this.use_bucket_name ? this.connection : this.collection;
 
       if (!useLookupIn) {
         // Regular get - no batching needed
@@ -373,7 +375,7 @@ module.exports = class Database {
         let returned_all = await Promise.allSettled(functions);
 
         let responses = _.flatten(
-          _.map(returned_all, (returned_obj) => returned_obj?.value)
+          _.map(returned_all, (returned_obj) => returned_obj?.value),
         );
 
         let final_responses = _.compact(
@@ -388,7 +390,7 @@ module.exports = class Database {
               }
               return value;
             }
-          })
+          }),
         );
 
         return final_responses;
@@ -440,7 +442,7 @@ module.exports = class Database {
             return Object.keys(result.value).length > 1 ? result.value : null;
           }
           return null;
-        })
+        }),
       );
 
       return final_responses;
@@ -508,7 +510,7 @@ module.exports = class Database {
     order_by,
     do_count = "false",
     query_scope,
-    join = ""
+    join = "",
   ) {
     const is_legacy =
       this.scope_name !== "_default" && this.collection_name !== "_default";
@@ -603,7 +605,7 @@ module.exports = class Database {
         let full_query = search_query + " AND type:" + type;
         var query = that.SearchQuery.new(
           that.full_text_index || full_text_index,
-          that.SearchQuery.queryString(full_query)
+          that.SearchQuery.queryString(full_query),
         );
 
         if (!limit) {
@@ -659,22 +661,69 @@ module.exports = class Database {
     return response.rows;
   }
 
+  /**
+   * Recursively checks if the query plan contains a Fetch operator
+   * @param {Object} plan - The EXPLAIN query plan
+   * @returns {boolean} - True if Fetch operator is found
+   */
+  hasFetchOperator(plan) {
+    if (!plan || typeof plan !== "object") {
+      return false;
+    }
+
+    // Check if this node is a Fetch operator
+    if (plan["#operator"] === "Fetch") {
+      return true;
+    }
+
+    // Recursively check all child properties
+    for (const key of Object.keys(plan)) {
+      const value = plan[key];
+      if (Array.isArray(value)) {
+        for (const item of value) {
+          if (this.hasFetchOperator(item)) {
+            return true;
+          }
+        }
+      } else if (typeof value === "object" && value !== null) {
+        if (this.hasFetchOperator(value)) {
+          return true;
+        }
+      }
+    }
+
+    return false;
+  }
+
   async optimize(query, exec_scope) {
     try {
-      const options = {
-        adhoc: true,
-      };
-      let advise_query = `ADVISE ${query}`;
-      let response = await exec_scope.query(advise_query, options);
-
-      let best_index = _.last(
-        _.first(response?.rows || [])?.advice?.adviseinfo?.recommended_indexes
-          ?.indexes || []
-      )?.index_statement;
-      if (best_index) {
-        await exec_scope.query(best_index);
+      const { buildCoveringIndex } = require("./utility");
+
+      // 1. Run EXPLAIN to get the query plan
+      const explainQuery = `EXPLAIN ${query}`;
+      const explainResponse = await exec_scope.query(explainQuery);
+
+      // 2. Check if query plan contains Fetch operator (not covered)
+      const plan = explainResponse?.rows?.[0];
+      if (!this.hasFetchOperator(plan)) {
+        // Query is already covered, no need to create index
+        return;
       }
-    } catch (e) {}
+
+      // 3. Build covering index from the query
+      const indexStatement = buildCoveringIndex(query, {
+        bucket: this.bucket,
+        scope: this.scope_name,
+        collection: this.collection_name,
+      });
+
+      // 4. Execute the index creation if we got a valid statement
+      if (indexStatement) {
+        const response = await exec_scope.query(indexStatement);
+      }
+    } catch (e) {
+      // Silently handle errors (index may already exist, etc.)
+    }
   }
 
   async query(query, query_scope = "cluster") {

package/src/utility.js

@@ -0,0 +1,370 @@
+"use strict";
+
+const crypto = require("crypto");
+
+/**
+ * Generates a short hash from a string for unique index naming
+ * @param {string} str - The string to hash
+ * @returns {string} - A short hash string
+ */
+function generateHash(str) {
+  return crypto.createHash("md5").update(str).digest("hex").substring(0, 8);
+}
+
+/**
+ * Extracts field names from WHERE clause of a query
+ * @param {string} query - The N1QL query
+ * @returns {string[]} - Array of field names found in WHERE clause
+ */
+function extractWhereFields(query) {
+  const fields = new Set();
+
+  // Find the WHERE clause
+  const whereMatch = query.match(
+    /\bWHERE\b(.+?)(?:\bORDER\b|\bLIMIT\b|\bOFFSET\b|\bGROUP\b|\bHAVING\b|$)/is,
+  );
+  if (!whereMatch) {
+    return [];
+  }
+
+  const whereClause = whereMatch[1];
+
+  // Pattern to match field names in comparisons
+  // Matches: field = value, field > value, field < value, field >= value, field <= value
+  // Also matches: field IN (...), field LIKE ..., field IS NULL, field IS MISSING, field IS NOT NULL
+  // Handles backtick-quoted fields like `field_name`
+  const fieldPattern =
+    /`?(\w+)`?\s*(?:=|!=|<>|>=|<=|>|<|\bIN\b|\bLIKE\b|\bIS\b|\bBETWEEN\b)/gi;
+
+  let match;
+  while ((match = fieldPattern.exec(whereClause)) !== null) {
+    const fieldName = match[1].toLowerCase();
+    // Skip common SQL keywords that might be matched
+    if (
+      !["and", "or", "not", "null", "missing", "true", "false"].includes(
+        fieldName,
+      )
+    ) {
+      fields.add(match[1]); // Keep original case
+    }
+  }
+
+  // Also look for fields in function calls like LOWER(field), UPPER(field)
+  const functionPattern =
+    /(?:LOWER|UPPER|TRIM|LENGTH|SUBSTR)\s*\(\s*`?(\w+)`?/gi;
+  while ((match = functionPattern.exec(whereClause)) !== null) {
+    fields.add(match[1]);
+  }
+
+  // Extract array fields from ANY/EVERY ... IN array_field SATISFIES ... END
+  // Pattern: ANY/EVERY variable IN array_field SATISFIES ... END
+  const anyEveryPattern =
+    /\b(?:ANY|EVERY)\s+\w+\s+IN\s+`?(\w+)`?\s+SATISFIES/gi;
+  while ((match = anyEveryPattern.exec(whereClause)) !== null) {
+    fields.add(match[1]);
+  }
+
+  // Also extract nested property access within SATISFIES: v.property = value
+  // This captures fields accessed through array iteration variables
+  const satisfiesPattern = /SATISFIES\s+(.+?)\s+END/gi;
+  while ((match = satisfiesPattern.exec(whereClause)) !== null) {
+    const satisfiesClause = match[1];
+    // Look for comparisons like: v.property = value, v.field > 10
+    const nestedFieldPattern =
+      /\w+\.`?(\w+)`?\s*(?:=|!=|<>|>=|<=|>|<|\bIN\b|\bLIKE\b|\bIS\b)/gi;
+    let nestedMatch;
+    while ((nestedMatch = nestedFieldPattern.exec(satisfiesClause)) !== null) {
+      // These are nested properties, we might want to track them separately
+      // For now, we'll note them but the array field is what matters for indexing
+    }
+  }
+
+  return Array.from(fields);
+}
+
+/**
+ * Extracts field names from SELECT clause of a query
+ * @param {string} query - The N1QL query
+ * @returns {string[]|null} - Array of field names, or null if SELECT *
+ */
+function extractSelectFields(query) {
+  // Find the SELECT clause (between SELECT and FROM)
+  const selectMatch = query.match(/\bSELECT\b(.+?)\bFROM\b/is);
+  if (!selectMatch) {
+    return null;
+  }
+
+  const selectClause = selectMatch[1].trim();
+
+  // Check for SELECT * patterns
+  if (
+    /^\s*\*\s*$/.test(selectClause) ||
+    /\(\s*\w+\s*\)\s*\.\s*\*/.test(selectClause)
+  ) {
+    return null; // Cannot determine fields for SELECT *
+  }
+
+  const fields = new Set();
+
+  // Split by comma, handling nested parentheses
+  const parts = splitSelectClause(selectClause);
+
+  for (const part of parts) {
+    const trimmedPart = part.trim();
+
+    // Skip if it's a * pattern
+    if (trimmedPart === "*" || trimmedPart.endsWith(".*")) {
+      return null;
+    }
+
+    // Handle AS alias - get the source field
+    const asMatch = trimmedPart.match(/^(.+?)\s+AS\s+\w+$/i);
+    const fieldExpr = asMatch ? asMatch[1].trim() : trimmedPart;
+
+    // Extract field from META().id pattern - skip these
+    if (/META\s*\(/i.test(fieldExpr)) {
+      continue;
+    }
+
+    // Extract field from function calls like COUNT(field), SUM(field)
+    const funcMatch = fieldExpr.match(/^\w+\s*\(\s*`?(\w+)`?\s*\)$/);
+    if (funcMatch && funcMatch[1].toLowerCase() !== "id") {
+      fields.add(funcMatch[1]);
+      continue;
+    }
+
+    // Handle simple field reference: `field` or collection.field or `collection`.`field`
+    const simpleFieldMatch = fieldExpr.match(/(?:`?\w+`?\s*\.\s*)?`?(\w+)`?$/);
+    if (simpleFieldMatch) {
+      const fieldName = simpleFieldMatch[1];
+      if (fieldName.toLowerCase() !== "id") {
+        fields.add(fieldName);
+      }
+    }
+  }
+
+  return Array.from(fields);
+}
+
+/**
+ * Extracts field names from GROUP BY clause of a query
+ * @param {string} query - The N1QL query
+ * @returns {string[]} - Array of field names found in GROUP BY clause
+ */
+function extractGroupByFields(query) {
+  const fields = [];
+
+  // Find the GROUP BY clause
+  const groupByMatch = query.match(
+    /\bGROUP\s+BY\b(.+?)(?:\bHAVING\b|\bORDER\b|\bLIMIT\b|\bOFFSET\b|$)/is,
+  );
+  if (!groupByMatch) {
+    return [];
+  }
+
+  const groupByClause = groupByMatch[1];
+
+  // Split by comma and extract field names
+  const parts = groupByClause.split(",");
+
+  for (const part of parts) {
+    const trimmedPart = part.trim();
+
+    // Handle collection.field or `collection`.`field` syntax
+    const fieldMatch = trimmedPart.match(/(?:`?\w+`?\s*\.\s*)?`?(\w+)`?$/);
+    if (fieldMatch) {
+      const fieldName = fieldMatch[1];
+      // Skip if it's a keyword or id
+      if (!["id"].includes(fieldName.toLowerCase())) {
+        fields.push(fieldName);
+      }
+    }
+  }
+
+  return fields;
+}
+
+/**
+ * Extracts field names from ORDER BY clause of a query
+ * @param {string} query - The N1QL query
+ * @returns {string[]} - Array of field names found in ORDER BY clause
+ */
+function extractOrderByFields(query) {
+  const fields = [];
+
+  // Find the ORDER BY clause
+  const orderByMatch = query.match(
+    /\bORDER\s+BY\b(.+?)(?:\bLIMIT\b|\bOFFSET\b|$)/is,
+  );
+  if (!orderByMatch) {
+    return [];
+  }
+
+  const orderByClause = orderByMatch[1];
+
+  // Split by comma and extract field names
+  const parts = orderByClause.split(",");
+
+  for (const part of parts) {
+    const trimmedPart = part.trim();
+
+    // Remove ASC/DESC suffix
+    const withoutDirection = trimmedPart
+      .replace(/\s+(ASC|DESC)\s*$/i, "")
+      .trim();
+
+    // Handle collection.field or `collection`.`field` syntax
+    const fieldMatch = withoutDirection.match(/(?:`?\w+`?\s*\.\s*)?`?(\w+)`?$/);
+    if (fieldMatch) {
+      const fieldName = fieldMatch[1];
+      // Skip if it's a keyword or id
+      if (!["id", "asc", "desc"].includes(fieldName.toLowerCase())) {
+        fields.push(fieldName);
+      }
+    }
+  }
+
+  return fields;
+}
+
+/**
+ * Splits SELECT clause by commas, respecting parentheses
+ * @param {string} clause - The SELECT clause content
+ * @returns {string[]} - Array of individual select expressions
+ */
+function splitSelectClause(clause) {
+  const parts = [];
+  let current = "";
+  let depth = 0;
+
+  for (const char of clause) {
+    if (char === "(") {
+      depth++;
+      current += char;
+    } else if (char === ")") {
+      depth--;
+      current += char;
+    } else if (char === "," && depth === 0) {
+      parts.push(current);
+      current = "";
+    } else {
+      current += char;
+    }
+  }
+
+  if (current.trim()) {
+    parts.push(current);
+  }
+
+  return parts;
+}
+
+/**
+ * Builds a covering index CREATE statement from a query
+ * @param {string} query - The N1QL query to analyze
+ * @param {Object} options - Configuration options
+ * @param {string} options.bucket - The bucket name
+ * @param {string} options.scope - The scope name
+ * @param {string} options.collection - The collection name
+ * @returns {string|null} - The CREATE INDEX statement, or null if cannot build
+ */
+function buildCoveringIndex(query, options) {
+  const { bucket, scope, collection } = options;
+
+  if (!bucket || !scope || !collection) {
+    return null;
+  }
+
+  // Extract fields from SELECT clause
+  const selectFields = extractSelectFields(query);
+  if (selectFields === null) {
+    // SELECT * query - cannot create covering index
+    return null;
+  }
+
+  // Extract fields from WHERE clause
+  const whereFields = extractWhereFields(query);
+
+  // Extract fields from GROUP BY clause
+  const groupByFields = extractGroupByFields(query);
+
+  // Extract fields from ORDER BY clause
+  const orderByFields = extractOrderByFields(query);
+
+  // Remove 'deleted' from whereFields as it will be added first with INCLUDE MISSING
+  const filteredWhereFields = whereFields.filter(
+    (f) => f.toLowerCase() !== "deleted",
+  );
+
+  // Track fields already added to avoid duplicates
+  const addedFields = new Set(filteredWhereFields.map((f) => f.toLowerCase()));
+
+  // Filter GROUP BY fields (remove duplicates from WHERE, remove deleted)
+  const filteredGroupByFields = groupByFields.filter((f) => {
+    const lower = f.toLowerCase();
+    if (addedFields.has(lower) || lower === "deleted") {
+      return false;
+    }
+    addedFields.add(lower);
+    return true;
+  });
+
+  // Filter ORDER BY fields (remove duplicates from WHERE and GROUP BY, remove deleted)
+  const filteredOrderByFields = orderByFields.filter((f) => {
+    const lower = f.toLowerCase();
+    if (addedFields.has(lower) || lower === "deleted") {
+      return false;
+    }
+    addedFields.add(lower);
+    return true;
+  });
+
+  // Filter SELECT fields (remove duplicates from WHERE, GROUP BY, and ORDER BY, remove deleted)
+  const selectFieldsFiltered = selectFields.filter((f) => {
+    const lower = f.toLowerCase();
+    if (addedFields.has(lower) || lower === "deleted") {
+      return false;
+    }
+    addedFields.add(lower);
+    return true;
+  });
+
+  // All fields for the index: WHERE -> GROUP BY -> ORDER BY -> SELECT
+  const allFields = [
+    ...filteredWhereFields,
+    ...filteredGroupByFields,
+    ...filteredOrderByFields,
+    ...selectFieldsFiltered,
+  ];
+
+  if (allFields.length === 0) {
+    // No fields to index besides deleted
+    return null;
+  }
+
+  // Build the field list with deleted INCLUDE MISSING first
+  const fieldList = [
+    "`deleted` INCLUDE MISSING",
+    ...allFields.map((f) => `\`${f}\``),
+  ].join(", ");
+
+  // Generate unique index name based on bucket, scope, collection, and fields
+  // This ensures multiple queries covered by the same index get the same hash
+  const indexKey = `${bucket}|${scope}|${collection}|${allFields.sort().join(",")}`;
+  const hash = generateHash(indexKey);
+  const indexName = `adv_covered_${hash}`;
+
+  // Build the CREATE INDEX statement
+  const indexStatement = `CREATE INDEX \`${indexName}\` ON \`default\`:\`${bucket}\`.\`${scope}\`.\`${collection}\`(${fieldList})`;
+
+  return indexStatement;
+}
+
+module.exports = {
+  buildCoveringIndex,
+  extractWhereFields,
+  extractSelectFields,
+  extractGroupByFields,
+  extractOrderByFields,
+  generateHash,
+};