spice-couchbase 2.0.44 → 2.0.46

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "spice-couchbase",
-  "version": "2.0.44",
+  "version": "2.0.46",
   "description": "",
   "main": "build/index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "node src/utility.test.js"
   },
   "author": "Chad Fraser <chad@sonover.com> (http://sonover.com)",
   "license": "ISC",

package/src/index.js

@@ -19,11 +19,14 @@ if (!Promise.allSettled) {
           .catch((reason) => ({
             status: "rejected",
             reason,
-          }))
-      )
+          })),
+      ),
     );
 }
 
+// Couchbase lookupIn has a limit of 16 operations per call
+const LOOKUP_IN_MAX_SPECS = 16;
+
 module.exports = class Database {
   constructor(
     connection,
@@ -33,7 +36,7 @@ module.exports = class Database {
       scope,
       query_scope = "cluster",
       auto_create_collection,
-    } = {}
+    } = {},
   ) {
     if (connection) {
       this.connection_name = connection;
@@ -148,7 +151,7 @@ module.exports = class Database {
             { name: "scope_admin", bucket: this.db_config.bucket },
             { name: "data_reader", bucket: this.db_config.bucket },
           ],
-        }
+        },
       );
     } catch (e) {
       console.log(e);
@@ -159,7 +162,7 @@ module.exports = class Database {
     return (
       (this.db_config.api_base ||
         `http://${_.last(
-          _.first(this.db_config.nodes.split(",")).split("://")
+          _.first(this.db_config.nodes.split(",")).split("://"),
         )}`) +
       ":" +
       default_port
@@ -176,7 +179,7 @@ module.exports = class Database {
       if (!this.checkGlobelConnectionObject(this.collection_name)) {
         let collection_exists = await this.collectionExists(
           this.collection_name,
-          this.scope_name
+          this.scope_name,
         );
         if (!collection_exists) {
           await this.createCollection(this.collection_name, this.scope_name);
@@ -202,7 +205,7 @@ module.exports = class Database {
               username: that.db_config.username,
               password: that.db_config.password,
             },
-          }
+          },
         );
         let scope = _.find(all_scopes.data.scopes, { name: scope_name });
         let collectionSet = new Set();
@@ -245,7 +248,7 @@ module.exports = class Database {
   async createPrimaryIndex(scope_name, col_name) {
     try {
       let response = await this.query(
-        `CREATE PRIMARY INDEX ON default:\`${this.db_config.bucket}\`.\`${scope_name}\`.\`${col_name}\``
+        `CREATE PRIMARY INDEX ON default:\`${this.db_config.bucket}\`.\`${scope_name}\`.\`${col_name}\``,
       );
       return response;
     } catch (e) {
@@ -268,27 +271,39 @@ module.exports = class Database {
   async get(doc_id, { columns = [] } = {}) {
     try {
       const useLookupIn = columns.length > 0;
-      let response = await (
-        useLookupIn ?
-          (this.use_bucket_name ? this.connection : this.collection).lookupIn(
-            doc_id,
-            columns.map((col) => couchbase.LookupInSpec.get(col))
-          )
-        : (this.use_bucket_name ? this.connection : this.collection).get(doc_id)
-      );
+      const collection =
+        this.use_bucket_name ? this.connection : this.collection;
 
-      if (useLookupIn) {
-        // lookupIn returns content array with each spec result
-        let value = { id: doc_id };
-        columns.forEach((col, colIndex) => {
-          if (response.content && response.content[colIndex]) {
-            value[col] = response.content[colIndex].value;
-          }
-        });
-        return value;
-      } else {
+      if (!useLookupIn) {
+        let response = await collection.get(doc_id);
         return { id: doc_id, ...response.content };
       }
+
+      // Batch columns into chunks of LOOKUP_IN_MAX_SPECS (16)
+      let value = { id: doc_id };
+      const columnBatches = _.chunk(columns, LOOKUP_IN_MAX_SPECS);
+
+      // Execute all batch requests in parallel
+      const batchPromises = columnBatches.map((batch) => {
+        const specs = batch.map((col) => couchbase.LookupInSpec.get(col));
+        return collection.lookupIn(doc_id, specs);
+      });
+
+      const batchResults = await Promise.allSettled(batchPromises);
+
+      // Merge results from all batches
+      batchResults.forEach((result, batchIndex) => {
+        if (result.status === "fulfilled" && result.value) {
+          const batch = columnBatches[batchIndex];
+          batch.forEach((col, colIndex) => {
+            if (result.value.content && result.value.content[colIndex]) {
+              value[col] = result.value.content[colIndex].value;
+            }
+          });
+        }
+      });
+
+      return value;
     } catch (e) {
       console.error(e);
     }
@@ -301,7 +316,7 @@ module.exports = class Database {
         document,
         {
           expiry: expiry,
-        }
+        },
       );
     } catch (e) {
       console.log(e.message, e, doc_id);
@@ -312,7 +327,7 @@ module.exports = class Database {
   async touch(doc_id, expiry = 0) {
     await (this.use_bucket_name ? this.connection : this.collection).touch(
       doc_id,
-      expiry
+      expiry,
     );
     return await this.get(doc_id);
   }
@@ -351,51 +366,83 @@ module.exports = class Database {
     }
     if (_.isArray(keys) && keys.length > 0) {
       const useLookupIn = columns.length > 0;
-      let functions = _.map(keys, (key) =>
-        useLookupIn ?
-          (this.use_bucket_name ? this.connection : this.collection).lookupIn(
-            key,
-            columns.map((col) => couchbase.LookupInSpec.get(col))
-          )
-        : (this.use_bucket_name ? this.connection : this.collection).get(key)
-      );
-      let returned_all = await Promise.allSettled(functions);
+      const collection =
+        this.use_bucket_name ? this.connection : this.collection;
 
-      
+      if (!useLookupIn) {
+        // Regular get - no batching needed
+        let functions = _.map(keys, (key) => collection.get(key));
+        let returned_all = await Promise.allSettled(functions);
 
-      let responses = _.flatten(
-        _.map(returned_all, (returned_obj) => {
-          return returned_obj?.value;
-        })
-      );
+        let responses = _.flatten(
+          _.map(returned_all, (returned_obj) => returned_obj?.value),
+        );
 
+        let final_responses = _.compact(
+          _.map(responses, (res, index) => {
+            if (res) {
+              let value = res.value;
+              if (!keep_type && value) {
+                delete value.type;
+              }
+              if (value) {
+                value.id = keys[index];
+              }
+              return value;
+            }
+          }),
+        );
 
-      let final_responses = _.compact(
-        _.map(responses, (res, index) => {
-          if (res) {
-            let value;
-            if (useLookupIn) {
-              // lookupIn returns content array with each spec result
-              value = {};
-              columns.forEach((col, colIndex) => {
-                if (res.content && res.content[colIndex]) {
-                  value[col] = res.content[colIndex].value;
+        return final_responses;
+      }
+
+      // lookupIn with batching for columns > 16
+      const columnBatches = _.chunk(columns, LOOKUP_IN_MAX_SPECS);
+
+      // For each key, we need to fetch all column batches and merge the results
+      const keyPromises = _.map(keys, async (key) => {
+        try {
+          let value = { id: key };
+
+          // Execute all batch requests for this key in parallel
+          const batchPromises = columnBatches.map((batch) => {
+            const specs = batch.map((col) => couchbase.LookupInSpec.get(col));
+            return collection.lookupIn(key, specs);
+          });
+
+          const batchResults = await Promise.allSettled(batchPromises);
+
+          // Merge results from all batches
+          batchResults.forEach((result, batchIndex) => {
+            if (result.status === "fulfilled" && result.value) {
+              const batch = columnBatches[batchIndex];
+              batch.forEach((col, colIndex) => {
+                if (result.value.content && result.value.content[colIndex]) {
+                  value[col] = result.value.content[colIndex].value;
                 }
               });
-            } else {
-              // regular get returns the full document in value
-              value = res.value;
             }
+          });
 
-            if (!keep_type && value) {
-              delete value.type;
-            }
-            if (value) {
-              value.id = keys[index];
-            }
-            return value;
+          if (!keep_type && value) {
+            delete value.type;
           }
-        })
+
+          return value;
+        } catch (e) {
+          return null;
+        }
+      });
+
+      let results = await Promise.allSettled(keyPromises);
+      let final_responses = _.compact(
+        _.map(results, (result) => {
+          if (result.status === "fulfilled" && result.value) {
+            // Only return if we got at least some data beyond just the id
+            return Object.keys(result.value).length > 1 ? result.value : null;
+          }
+          return null;
+        }),
       );
 
       return final_responses;
@@ -463,7 +510,7 @@ module.exports = class Database {
     order_by,
     do_count = "false",
     query_scope,
-    join = ""
+    join = "",
   ) {
     const is_legacy =
       this.scope_name !== "_default" && this.collection_name !== "_default";
@@ -558,7 +605,7 @@ module.exports = class Database {
         let full_query = search_query + " AND type:" + type;
         var query = that.SearchQuery.new(
           that.full_text_index || full_text_index,
-          that.SearchQuery.queryString(full_query)
+          that.SearchQuery.queryString(full_query),
         );
 
         if (!limit) {
@@ -614,22 +661,69 @@ module.exports = class Database {
     return response.rows;
   }
 
+  /**
+   * Recursively checks if the query plan contains a Fetch operator
+   * @param {Object} plan - The EXPLAIN query plan
+   * @returns {boolean} - True if Fetch operator is found
+   */
+  hasFetchOperator(plan) {
+    if (!plan || typeof plan !== "object") {
+      return false;
+    }
+
+    // Check if this node is a Fetch operator
+    if (plan["#operator"] === "Fetch") {
+      return true;
+    }
+
+    // Recursively check all child properties
+    for (const key of Object.keys(plan)) {
+      const value = plan[key];
+      if (Array.isArray(value)) {
+        for (const item of value) {
+          if (this.hasFetchOperator(item)) {
+            return true;
+          }
+        }
+      } else if (typeof value === "object" && value !== null) {
+        if (this.hasFetchOperator(value)) {
+          return true;
+        }
+      }
+    }
+
+    return false;
+  }
+
   async optimize(query, exec_scope) {
     try {
-      const options = {
-        adhoc: true,
-      };
-      let advise_query = `ADVISE ${query}`;
-      let response = await exec_scope.query(advise_query, options);
-
-      let best_index = _.last(
-        _.first(response?.rows || [])?.advice?.adviseinfo?.recommended_indexes
-          ?.indexes || []
-      )?.index_statement;
-      if (best_index) {
-        await exec_scope.query(best_index);
+      const { buildCoveringIndex } = require("./utility");
+
+      // 1. Run EXPLAIN to get the query plan
+      const explainQuery = `EXPLAIN ${query}`;
+      const explainResponse = await exec_scope.query(explainQuery);
+
+      // 2. Check if query plan contains Fetch operator (not covered)
+      const plan = explainResponse?.rows?.[0];
+      if (!this.hasFetchOperator(plan)) {
+        // Query is already covered, no need to create index
+        return;
       }
-    } catch (e) {}
+
+      // 3. Build covering index from the query
+      const indexStatement = buildCoveringIndex(query, {
+        bucket: this.bucket,
+        scope: this.scope_name,
+        collection: this.collection_name,
+      });
+
+      // 4. Execute the index creation if we got a valid statement
+      if (indexStatement) {
+        const response = await exec_scope.query(indexStatement);
+      }
+    } catch (e) {
+      // Silently handle errors (index may already exist, etc.)
+    }
   }
 
   async query(query, query_scope = "cluster") {