spice-js 2.7.29 → 2.7.31

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spice-js",
-  "version": "2.7.29",
+  "version": "2.7.31",
   "description": "spice",
   "main": "build/index.js",
   "repository": {

package/src/index.js

@@ -2,7 +2,9 @@
 var path = require("path");
 //import IO from "koa-socket-2";
 const cors = require("kcors");
-global.spice = {};
+// Preserve an existing host/test configuration when the package is imported.
+// Production boot still starts from an empty object because spice is undefined.
+global.spice = global.spice || {};
 
 require("dotenv").config({ path: path.resolve("./") });
 

package/src/models/SpiceModel.js

@@ -22,9 +22,66 @@ var SDate = require("sonover-date"),
   _serializers = Symbol(),
   _level = Symbol(),
   _mapping_dept = Symbol(),
+  _mapping_dept_scalar = Symbol(),
+  _mapping_dept_array = Symbol(),
+  _mapping_concurrency = Symbol(),
+  _skip_mapped_serialization = Symbol(),
   _mapping_dept_exempt = Symbol(),
   _current_path = Symbol();
 
+const GLOBAL_MAPPING_CONCURRENCY = Math.max(
+  1,
+  Number(process.env.SPICE_MAPPING_GLOBAL_CONCURRENCY) || 32,
+);
+let activeGlobalMappings = 0;
+const globalMappingWaiters = [];
+
+function normalizeConcurrency(value, fallback = 1) {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback;
+}
+
+async function acquireGlobalMappingSlot() {
+  const queuedAt = process.hrtime.bigint();
+  if (activeGlobalMappings < GLOBAL_MAPPING_CONCURRENCY) {
+    activeGlobalMappings += 1;
+  } else {
+    await new Promise((resolve) => globalMappingWaiters.push(resolve));
+  }
+
+  return {
+    active: activeGlobalMappings,
+    waitMs: Number(process.hrtime.bigint() - queuedAt) / 1e6,
+    release() {
+      const next = globalMappingWaiters.shift();
+      if (next) {
+        next();
+      } else {
+        activeGlobalMappings -= 1;
+      }
+    },
+  };
+}
+
+async function runWithConcurrency(items, concurrency, worker) {
+  const results = new Array(items.length);
+  let nextIndex = 0;
+
+  async function runWorker() {
+    while (nextIndex < items.length) {
+      const index = nextIndex;
+      nextIndex += 1;
+      results[index] = await worker(items[index], index);
+    }
+  }
+
+  const workerCount = Math.min(concurrency, items.length);
+  await Promise.all(
+    Array.from({ length: workerCount }, () => runWorker()),
+  );
+  return results;
+}
+
 //const _type = Symbol("type");
 
 let that;
@@ -47,12 +104,36 @@ if (!Promise.allSettled) {
 export default class SpiceModel {
   constructor(args = {}) {
     try {
+      const modelArgs = args?.args || {};
+      const requestProfile = modelArgs?.ctx?.state?.read_profile || {};
+      const defaultMappingDept = process.env.MAPPING_DEPT || 3;
+      const legacyMappingDept =
+        modelArgs.mapping_dept !== undefined && modelArgs.mapping_dept !== null ?
+          modelArgs.mapping_dept
+        : undefined;
+
       var dbtype =
         spice.config.database.connections[args.connection].type || "couchbase";
       let Database = require(`spice-${dbtype}`);
-      this[_mapping_dept] =
-        args?.args?.mapping_dept || process.env.MAPPING_DEPT || 3;
-      this[_mapping_dept_exempt] = args?.args?.mapping_dept_exempt || [];
+      this[_mapping_dept] = legacyMappingDept ?? defaultMappingDept;
+      this[_mapping_dept_scalar] =
+        modelArgs.mapping_dept_scalar ??
+        legacyMappingDept ??
+        requestProfile.mapping_dept_scalar ??
+        defaultMappingDept;
+      this[_mapping_dept_array] =
+        modelArgs.mapping_dept_array ??
+        legacyMappingDept ??
+        requestProfile.mapping_dept_array ??
+        defaultMappingDept;
+      this[_mapping_concurrency] = normalizeConcurrency(
+        modelArgs.mapping_concurrency ?? requestProfile.mapping_concurrency,
+      );
+      this[_skip_mapped_serialization] =
+        modelArgs.skip_mapped_serialization ??
+        requestProfile.skip_mapped_serialization ??
+        false;
+      this[_mapping_dept_exempt] = modelArgs.mapping_dept_exempt || [];
       this.type = "";
       this.collection = args.connection;
       this[_args] = args.args;
@@ -237,6 +318,41 @@ export default class SpiceModel {
     // }
   }
 
+  applyMappingDepthArgs(args = {}) {
+    const hasLegacyDepth =
+      args.mapping_dept !== undefined && args.mapping_dept !== null;
+
+    if (hasLegacyDepth) {
+      this[_mapping_dept] = args.mapping_dept;
+      this[_mapping_dept_scalar] = args.mapping_dept;
+      this[_mapping_dept_array] = args.mapping_dept;
+    }
+    if (
+      args.mapping_dept_scalar !== undefined &&
+      args.mapping_dept_scalar !== null
+    ) {
+      this[_mapping_dept_scalar] = args.mapping_dept_scalar;
+    }
+    if (
+      args.mapping_dept_array !== undefined &&
+      args.mapping_dept_array !== null
+    ) {
+      this[_mapping_dept_array] = args.mapping_dept_array;
+    }
+    if (args.skip_mapped_serialization !== undefined) {
+      this[_skip_mapped_serialization] = args.skip_mapped_serialization;
+    }
+    if (args.mapping_concurrency !== undefined) {
+      this[_mapping_concurrency] = normalizeConcurrency(
+        args.mapping_concurrency,
+        this[_mapping_concurrency],
+      );
+    }
+    if (args.mapping_dept_exempt) {
+      this[_mapping_dept_exempt] = args.mapping_dept_exempt;
+    }
+  }
+
   get database() {
     return this[_database];
   }
@@ -515,13 +631,10 @@ export default class SpiceModel {
     const p = this[_ctx]?.profiler;
 
     const doGet = async () => {
-      if (args.mapping_dept) this[_mapping_dept] = args.mapping_dept;
-      if (args.mapping_dept_exempt)
-        this[_mapping_dept_exempt] = args.mapping_dept_exempt;
-
       if (!args) {
         args = {};
       }
+      this.applyMappingDepthArgs(args);
       if (_.isString(args.id)) {
         if (args.skip_hooks !== true) {
           await this.run_hook(args, "get", "before");
@@ -687,10 +800,13 @@ export default class SpiceModel {
       if (!args) {
         args = {};
       }
+      this.applyMappingDepthArgs(args);
       if (args.skip_hooks != true) {
         await this.run_hook(this, "list", "before");
       }
-      _.remove(args.ids, (o) => o == undefined);
+      args.ids = _.uniq(
+        _.filter(args.ids || [], (id) => _.isString(id) && id.length > 0),
+      );
       let key = `multi-get::${this.type}::${_.join(args.ids, "|")}:${args.columns}`;
       let results = [];
 
@@ -1384,9 +1500,7 @@ export default class SpiceModel {
     const p = this[_ctx]?.profiler;
 
     const doList = async () => {
-      if (args.mapping_dept) this[_mapping_dept] = args.mapping_dept;
-      if (args.mapping_dept_exempt)
-        this[_mapping_dept_exempt] = args.mapping_dept_exempt;
+      this.applyMappingDepthArgs(args);
 
       // Filter out id, meta().id, and any column ending with .id (e.g., dependents.id)
       if (args.columns) {
@@ -1558,6 +1672,13 @@ export default class SpiceModel {
 
     const doFetch = async () => {
       if (args.is_custom_query === "true" && args.ids.length > 0) {
+        console.log(
+          "Fetching results from database",
+          this.type,
+          args,
+          query,
+          mappedNestings,
+        );
         return await this.database.query(query);
       } else if (args.is_full_text === "true") {
         return await this.database.full_text_search(
@@ -1717,7 +1838,9 @@ export default class SpiceModel {
       data = Array.of(data);
     }
     const isExempt = this.isFieldExempt(source_property);
-    if (isExempt || this[_level] + 1 < this[_mapping_dept]) {
+    const activeMappingDept =
+      type === "read" ? this[_mapping_dept_scalar] : this[_mapping_dept];
+    if (isExempt || this[_level] + 1 < activeMappingDept) {
       let classes = _.compact(_.isArray(Class) ? Class : [Class]);
 
       let ids = [];
@@ -1761,6 +1884,10 @@ export default class SpiceModel {
               skip_cache: this[_skip_cache],
               _level: this[_level] + 1,
               mapping_dept: this[_mapping_dept],
+              mapping_dept_scalar: this[_mapping_dept_scalar],
+              mapping_dept_array: this[_mapping_dept_array],
+              mapping_concurrency: this[_mapping_concurrency],
+              skip_mapped_serialization: this[_skip_mapped_serialization],
               mapping_dept_exempt: this[_mapping_dept_exempt],
               _columns: columnArray.join(","),
               _current_path: childPath,
@@ -1770,6 +1897,7 @@ export default class SpiceModel {
               skip_hooks: true,
               ids: ids,
               columns: columnArray,
+              skip_read_serialize: this[_skip_mapped_serialization],
             });
           }),
         );
@@ -1881,7 +2009,9 @@ export default class SpiceModel {
       data = Array.of(data);
     }
     const isExempt = this.isFieldExempt(source_property);
-    if (isExempt || this[_level] + 1 < this[_mapping_dept]) {
+    const activeMappingDept =
+      type === "read" ? this[_mapping_dept_array] : this[_mapping_dept];
+    if (isExempt || this[_level] + 1 < activeMappingDept) {
       let ids = [];
       _.each(data, (result) => {
         let value = result[source_property];
@@ -1932,6 +2062,10 @@ export default class SpiceModel {
               skip_cache: this[_skip_cache],
               _level: this[_level] + 1,
               mapping_dept: this[_mapping_dept],
+              mapping_dept_scalar: this[_mapping_dept_scalar],
+              mapping_dept_array: this[_mapping_dept_array],
+              mapping_concurrency: this[_mapping_concurrency],
+              skip_mapped_serialization: this[_skip_mapped_serialization],
               mapping_dept_exempt: this[_mapping_dept_exempt],
               _columns: columnArray.join(","),
               _current_path: childPath,
@@ -1939,6 +2073,7 @@ export default class SpiceModel {
               skip_hooks: true,
               ids: ids,
               columns: columnArray,
+              skip_read_serialize: this[_skip_mapped_serialization],
             });
           }),
         );
@@ -2012,8 +2147,21 @@ export default class SpiceModel {
     return original_is_array ? data : data[0];
   }
 
-  addModifier({ when, execute }) {
+  addModifier({
+    when,
+    execute,
+    kind,
+    source_property,
+    store_property,
+  }) {
     if (this[_serializers][when]) {
+      if (kind) {
+        execute.spiceModifier = {
+          kind,
+          source_property,
+          store_property,
+        };
+      }
       this[_serializers][when]["modifiers"].push(execute);
     }
   }
@@ -2087,6 +2235,9 @@ export default class SpiceModel {
               case "string": {
                 this.addModifier({
                   when: properties[i].map.when || "read",
+                  kind: "relationship_map",
+                  source_property: i,
+                  store_property: properties[i].map.destination || i,
                   execute: async (
                     data,
                     old_data,
@@ -2096,6 +2247,7 @@ export default class SpiceModel {
                     mapping_dept,
                     level,
                     columns,
+                    serializationType,
                   ) => {
                     // Skip if columns are specified but this field isn't included
                     if (!this.isFieldInColumns(i, columns)) {
@@ -2110,7 +2262,7 @@ export default class SpiceModel {
                       properties[i].map.destination || i,
                       properties[i],
                       args,
-                      type,
+                      serializationType,
                       mapping_dept,
                       level,
                       columns,
@@ -2123,6 +2275,9 @@ export default class SpiceModel {
               case "array": {
                 this.addModifier({
                   when: properties[i].map.when || "read",
+                  kind: "relationship_map",
+                  source_property: i,
+                  store_property: properties[i].map.destination || i,
                   execute: async (
                     data,
                     old_data,
@@ -2132,6 +2287,7 @@ export default class SpiceModel {
                     mapping_dept,
                     level,
                     columns,
+                    serializationType,
                   ) => {
                     // Skip if columns are specified but this field isn't included
                     if (!this.isFieldInColumns(i, columns)) {
@@ -2147,7 +2303,7 @@ export default class SpiceModel {
                       properties[i].map.destination || i,
                       properties[i],
                       args,
-                      type,
+                      serializationType,
                       mapping_dept,
                       level,
                       columns,
@@ -2185,11 +2341,10 @@ export default class SpiceModel {
 
       // Cache the modifiers lookup for the specified type.
       const modifiers = this[_serializers]?.[type]?.modifiers || [];
-      for (let i = 0; i < modifiers.length; i++) {
-        const modifier = modifiers[i];
+      const invokeModifier = async (modifier, index, inputData) => {
         try {
           const result = await modifier(
-            data,
+            inputData,
             old_data,
             this[_ctx],
             this.type,
@@ -2197,23 +2352,119 @@ export default class SpiceModel {
             this[_mapping_dept],
             this[_level],
             this[_columns],
+            type,
           );
-          // Guard against modifiers that return undefined
           if (result !== undefined) {
-            data = result;
-          } else {
-            console.warn(
-              `Modifier #${i} for type=${this.type} returned undefined, keeping previous data`,
-            );
+            return { hasValue: true, value: result };
           }
+          console.warn(
+            `Modifier #${index} for type=${this.type} returned undefined, keeping previous data`,
+          );
         } catch (error) {
           console.error(
-            `Modifier error in do_serialize (type=${this.type}, modifier #${i}):`,
+            `Modifier error in do_serialize (type=${this.type}, modifier #${index}):`,
             error instanceof Error ?
               error.stack
             : `Non-Error thrown: ${JSON.stringify(error)}`,
           );
         }
+        return { hasValue: false };
+      };
+
+      let modifierIndex = 0;
+      while (modifierIndex < modifiers.length) {
+        const modifier = modifiers[modifierIndex];
+        const canRunRelationshipMapsConcurrently =
+          type === "read" &&
+          this[_level] === 0 &&
+          this[_mapping_concurrency] > 1 &&
+          modifier?.spiceModifier?.kind === "relationship_map";
+
+        if (!canRunRelationshipMapsConcurrently) {
+          const result = await invokeModifier(modifier, modifierIndex, data);
+          if (result.hasValue) data = result.value;
+          modifierIndex += 1;
+          continue;
+        }
+
+        const group = [];
+        const destinations = new Set();
+        while (modifierIndex < modifiers.length) {
+          const candidate = modifiers[modifierIndex];
+          const metadata = candidate?.spiceModifier;
+          if (metadata?.kind !== "relationship_map") break;
+          if (
+            metadata.store_property &&
+            destinations.has(metadata.store_property)
+          ) {
+            break;
+          }
+          if (metadata.store_property) {
+            destinations.add(metadata.store_property);
+          }
+          group.push({ modifier: candidate, index: modifierIndex, metadata });
+          modifierIndex += 1;
+        }
+
+        const groupQueuedAt = process.hrtime.bigint();
+        let activeRequestMappings = 0;
+        const executeGroup = async () =>
+          await runWithConcurrency(
+            group,
+            this[_mapping_concurrency],
+            async (entry) => {
+              const requestQueueWaitMs =
+                Number(process.hrtime.bigint() - groupQueuedAt) / 1e6;
+              const globalSlot = await acquireGlobalMappingSlot();
+              activeRequestMappings += 1;
+              const profileMetadata = {
+                source_property: entry.metadata.source_property,
+                store_property: entry.metadata.store_property,
+                request_queue_wait_ms:
+                  Math.round(requestQueueWaitMs * 100) / 100,
+                global_queue_wait_ms:
+                  Math.round(globalSlot.waitMs * 100) / 100,
+                active_request_mappings: activeRequestMappings,
+                request_limit: this[_mapping_concurrency],
+                active_global_mappings: globalSlot.active,
+                global_limit: GLOBAL_MAPPING_CONCURRENCY,
+              };
+
+              try {
+                const execute = async () =>
+                  await invokeModifier(entry.modifier, entry.index, data);
+                if (p) {
+                  return await p.track(
+                    `${this.type}.map.concurrent.${entry.metadata.source_property}`,
+                    execute,
+                    profileMetadata,
+                  );
+                }
+                return await execute();
+              } finally {
+                activeRequestMappings -= 1;
+                globalSlot.release();
+              }
+            },
+          );
+
+        const groupMetadata = {
+          map_count: group.length,
+          request_limit: this[_mapping_concurrency],
+          global_limit: GLOBAL_MAPPING_CONCURRENCY,
+        };
+        const groupResults =
+          p ?
+            await p.track(
+              `${this.type}.map.concurrent`,
+              executeGroup,
+              groupMetadata,
+            )
+          : await executeGroup();
+
+        for (const result of groupResults) {
+          if (result?.hasValue) data = result.value;
+        }
       }
 
       // Ensure data is always an array for consistent processing.

package/tests/__mocks__/globalMocks.js

@@ -22,8 +22,10 @@ const spiceConfig = {
   cache: {},
   cache_providers: {},
   schemas: {},
+  models: {},
   modifiers: {},
-  schema_extenders: []
+  schema_extenders: [],
+  getModifiers: jest.fn(() => [])
 };
 
 // Set on multiple global scopes to ensure it's available

package/tests/fixtures/testData.js

@@ -29,6 +29,8 @@ const columnTestCases = {
  */
 const sampleDbResults = {
   users: [
+    { id: 'valid-1', type: 'user', name: 'Valid One', email: 'valid1@example.com' },
+    { id: 'valid-2', type: 'user', name: 'Valid Two', email: 'valid2@example.com' },
     { id: 'user-1', type: 'user', name: 'Alice', email: 'alice@example.com' },
     { id: 'user-2', type: 'user', name: 'Bob', email: 'bob@example.com' },
     { id: 'user-3', type: 'user', name: 'Charlie', email: 'charlie@example.com' }
@@ -71,7 +73,7 @@ function generateLargeDataset(size = 100) {
   for (let i = 0; i < size; i++) {
     data.push({
       id: `item-${i}`,
-      type: 'item',
+      type: 'user',
       name: `Item ${i}`,
       value: Math.random()
     });

package/tests/helpers/modelFactory.js

@@ -4,6 +4,12 @@
 
 const SpiceModel = require('../../src/models/SpiceModel').default;
 const MockDatabase = require('../__mocks__/spice-couchbase');
+const spiceConfig = require('../__mocks__/globalMocks');
+
+// Importing SpiceModel loads the package index, which initializes global.spice.
+// Restore the complete test configuration after that package initialization.
+global.spice = spiceConfig;
+globalThis.spice = spiceConfig;
 
 /**
  * Create a test model instance with mocked database
@@ -32,7 +38,7 @@ function createTestModel(config = {}) {
     connection,
     collection: type,
     props,
-    args: { ...args, skip_cache: true }
+    args: { ...args, skip_cache: true, disable_lifecycle_events: true }
   });
 
   // Inject mock database

package/tests/jest-env.js

@@ -21,8 +21,10 @@ class CustomEnvironment extends NodeEnvironment {
       cache: {},
       cache_providers: {},
       schemas: {},
+      models: {},
       modifiers: {},
-      schema_extenders: []
+      schema_extenders: [],
+      getModifiers: () => []
     };
 
     // Set on the global object for the test environment