s3db.js 11.0.5 → 11.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "11.0.5",
+  "version": "11.1.0",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",
@@ -140,6 +140,7 @@
     "test:plugins": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/ --testTimeout=60000",
     "test:full": "pnpm run test:js && pnpm run test:ts",
     "benchmark": "node benchmark-compression.js",
+    "benchmark:partitions": "node docs/benchmarks/partitions-matrix.js",
     "version": "echo 'Use pnpm run release v<version> instead of npm version'",
     "release:check": "./scripts/pre-release-check.sh",
     "release:prepare": "pnpm run build:binaries && echo 'Binaries ready for GitHub release'",

package/src/concerns/crypto.js

@@ -1,19 +1,13 @@
 import { CryptoError } from "../errors.js";
 import tryFn, { tryFnSync } from "./try-fn.js";
+import crypto from 'crypto';
 
 async function dynamicCrypto() {
   let lib;
 
   if (typeof process !== 'undefined') {
-    const [ok, err, result] = await tryFn(async () => {
-      const { webcrypto } = await import('crypto');
-      return webcrypto;
-    });
-    if (ok) {
-      lib = result;
-    } else {
-      throw new CryptoError('Crypto API not available', { original: err, context: 'dynamicCrypto' });
-    }
+    // Use the static import instead of dynamic import
+    lib = crypto.webcrypto;
   } else if (typeof window !== 'undefined') {
     lib = window.crypto;
   }
@@ -86,16 +80,15 @@ export async function md5(data) {
   if (typeof process === 'undefined') {
     throw new CryptoError('MD5 hashing is only available in Node.js environment', { context: 'md5' });
   }
-  
+
   const [ok, err, result] = await tryFn(async () => {
-    const { createHash } = await import('crypto');
-    return createHash('md5').update(data).digest('base64');
+    return crypto.createHash('md5').update(data).digest('base64');
   });
-  
+
   if (!ok) {
     throw new CryptoError('MD5 hashing failed', { original: err, data });
   }
-  
+
   return result;
 }
 

package/src/plugins/eventual-consistency/consolidation.js

@@ -272,6 +272,8 @@ export async function consolidateRecord(
         }
 
         currentValue = 0;
+        // Clear the applied transactions array since we deleted them
+        appliedTransactions.length = 0;
       } else {
         // Record exists - use applied transactions to calculate current value
         // Sort by timestamp to get chronological order
@@ -290,40 +292,44 @@ export async function consolidateRecord(
           // Solution: Get the current record value and create an anchor transaction now
           const recordValue = recordExists[config.field] || 0;
 
-          // Calculate what the base value was by subtracting all applied deltas
-          let appliedDelta = 0;
-          for (const t of appliedTransactions) {
-            if (t.operation === 'add') appliedDelta += t.value;
-            else if (t.operation === 'sub') appliedDelta -= t.value;
-          }
+          // Only create anchor if recordValue is a number (not object/array for nested fields)
+          if (typeof recordValue === 'number') {
+            // Calculate what the base value was by subtracting all applied deltas
+            let appliedDelta = 0;
+            for (const t of appliedTransactions) {
+              if (t.operation === 'add') appliedDelta += t.value;
+              else if (t.operation === 'sub') appliedDelta -= t.value;
+            }
 
-          const baseValue = recordValue - appliedDelta;
-
-          // Create and save anchor transaction with the base value
-          // Only create if baseValue is non-zero AND we don't already have an anchor transaction
-          const hasExistingAnchor = appliedTransactions.some(t => t.source === 'anchor');
-          if (baseValue !== 0 && !hasExistingAnchor) {
-            // Use the timestamp of the first applied transaction for cohort info
-            const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
-            const cohortInfo = getCohortInfo(firstTransactionDate, config.cohort.timezone, config.verbose);
-            const anchorTransaction = {
-              id: idGenerator(),
-              originalId: originalId,
-              field: config.field,
-              value: baseValue,
-              operation: 'set',
-              timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(), // 1ms before first txn to ensure it's first
-              cohortDate: cohortInfo.date,
-              cohortHour: cohortInfo.hour,
-              cohortMonth: cohortInfo.month,
-              source: 'anchor',
-              applied: true
-            };
-
-            await transactionResource.insert(anchorTransaction);
-
-            // Prepend to applied transactions for this consolidation
-            appliedTransactions.unshift(anchorTransaction);
+            const baseValue = recordValue - appliedDelta;
+
+            // Create and save anchor transaction with the base value
+            // Only create if baseValue is non-zero AND we don't already have an anchor transaction
+            const hasExistingAnchor = appliedTransactions.some(t => t.source === 'anchor');
+            if (baseValue !== 0 && typeof baseValue === 'number' && !hasExistingAnchor) {
+              // Use the timestamp of the first applied transaction for cohort info
+              const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
+              const cohortInfo = getCohortInfo(firstTransactionDate, config.cohort.timezone, config.verbose);
+              const anchorTransaction = {
+                id: idGenerator(),
+                originalId: originalId,
+                field: config.field,
+                fieldPath: config.field,  // Add fieldPath for consistency
+                value: baseValue,
+                operation: 'set',
+                timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(), // 1ms before first txn to ensure it's first
+                cohortDate: cohortInfo.date,
+                cohortHour: cohortInfo.hour,
+                cohortMonth: cohortInfo.month,
+                source: 'anchor',
+                applied: true
+              };
+
+              await transactionResource.insert(anchorTransaction);
+
+              // Prepend to applied transactions for this consolidation
+              appliedTransactions.unshift(anchorTransaction);
+            }
           }
         }
 
@@ -340,7 +346,8 @@ export async function consolidateRecord(
 
       // If there's an initial value, create and save an anchor transaction
       // This ensures all future consolidations have a reliable base value
-      if (currentValue !== 0) {
+      // IMPORTANT: Only create anchor if currentValue is a number (not object/array for nested fields)
+      if (currentValue !== 0 && typeof currentValue === 'number') {
         // Use timestamp of the first pending transaction (or current time if none)
         let anchorTimestamp;
         if (transactions && transactions.length > 0) {
@@ -355,6 +362,7 @@ export async function consolidateRecord(
           id: idGenerator(),
           originalId: originalId,
           field: config.field,
+          fieldPath: config.field,  // Add fieldPath for consistency
           value: currentValue,
           operation: 'set',
           timestamp: anchorTimestamp,
@@ -389,22 +397,75 @@ export async function consolidateRecord(
       new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
     );
 
-    // If there's a current value and no 'set' operations in pending transactions,
-    // prepend a synthetic set transaction to preserve the current value
-    const hasSetOperation = transactions.some(t => t.operation === 'set');
-    if (currentValue !== 0 && !hasSetOperation) {
-      transactions.unshift(createSyntheticSetTransaction(currentValue));
+    // Group PENDING transactions by fieldPath to support nested fields
+    const transactionsByPath = {};
+    for (const txn of transactions) {
+      const path = txn.fieldPath || txn.field || config.field;
+      if (!transactionsByPath[path]) {
+        transactionsByPath[path] = [];
+      }
+      transactionsByPath[path].push(txn);
     }
 
-    // Apply reducer to get consolidated value
-    const consolidatedValue = config.reducer(transactions);
+    // For each fieldPath, we need the currentValue from applied transactions
+    // Group APPLIED transactions by fieldPath
+    const appliedByPath = {};
+    if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
+      for (const txn of appliedTransactions) {
+        const path = txn.fieldPath || txn.field || config.field;
+        if (!appliedByPath[path]) {
+          appliedByPath[path] = [];
+        }
+        appliedByPath[path].push(txn);
+      }
+    }
 
-    if (config.verbose) {
-      console.log(
-        `[EventualConsistency] ${config.resource}.${config.field} - ` +
-        `${originalId}: ${currentValue} → ${consolidatedValue} ` +
-        `(${consolidatedValue > currentValue ? '+' : ''}${consolidatedValue - currentValue})`
-      );
+    // Consolidate each fieldPath group separately
+    const consolidatedValues = {};
+    const lodash = await import('lodash-es');
+
+    // Get current record to extract existing values for nested paths
+    const [currentRecordOk, currentRecordErr, currentRecord] = await tryFn(() =>
+      targetResource.get(originalId)
+    );
+
+    for (const [fieldPath, pathTransactions] of Object.entries(transactionsByPath)) {
+      // Calculate current value for this path from applied transactions
+      let pathCurrentValue = 0;
+      if (appliedByPath[fieldPath] && appliedByPath[fieldPath].length > 0) {
+        // Sort applied transactions by timestamp
+        appliedByPath[fieldPath].sort((a, b) =>
+          new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
+        );
+        // Apply reducer to get current value from applied transactions
+        pathCurrentValue = config.reducer(appliedByPath[fieldPath]);
+      } else {
+        // No applied transactions yet - use value from record (first consolidation)
+        // This happens when there's an initial value in the record before any consolidation
+        if (currentRecordOk && currentRecord) {
+          const recordValue = lodash.get(currentRecord, fieldPath, 0);
+          if (typeof recordValue === 'number') {
+            pathCurrentValue = recordValue;
+          }
+        }
+      }
+
+      // Prepend synthetic set transaction with current value
+      if (pathCurrentValue !== 0) {
+        pathTransactions.unshift(createSyntheticSetTransaction(pathCurrentValue));
+      }
+
+      // Apply reducer to get consolidated value for this path
+      const pathConsolidatedValue = config.reducer(pathTransactions);
+      consolidatedValues[fieldPath] = pathConsolidatedValue;
+
+      if (config.verbose) {
+        console.log(
+          `[EventualConsistency] ${config.resource}.${fieldPath} - ` +
+          `${originalId}: ${pathCurrentValue} → ${pathConsolidatedValue} ` +
+          `(${pathTransactions.length - (pathCurrentValue !== 0 ? 1 : 0)} pending txns)`
+        );
+      }
     }
 
     // 🔥 DEBUG: Log BEFORE update
@@ -412,63 +473,105 @@ export async function consolidateRecord(
       console.log(
         `🔥 [DEBUG] BEFORE targetResource.update() {` +
         `\n  originalId: '${originalId}',` +
-        `\n  field: '${config.field}',` +
-        `\n  consolidatedValue: ${consolidatedValue},` +
-        `\n  currentValue: ${currentValue}` +
+        `\n  consolidatedValues: ${JSON.stringify(consolidatedValues, null, 2)}` +
         `\n}`
       );
     }
 
-    // Update the original record
-    // NOTE: We do NOT attempt to insert non-existent records because:
-    // 1. Target resources typically have required fields we don't know about
-    // 2. Record creation should be the application's responsibility
-    // 3. Transactions will remain pending until the record is created
-    const [updateOk, updateErr, updateResult] = await tryFn(() =>
-      targetResource.update(originalId, {
-        [config.field]: consolidatedValue
-      })
+    // Build update object using lodash.set for nested paths
+    // Get fresh record to avoid overwriting other fields
+    const [recordOk, recordErr, record] = await tryFn(() =>
+      targetResource.get(originalId)
     );
 
+    let updateOk, updateErr, updateResult;
+
+    if (!recordOk || !record) {
+      // Record doesn't exist - we'll let the update fail and handle it below
+      // This ensures transactions remain pending until record is created
+      if (config.verbose) {
+        console.log(
+          `[EventualConsistency] ${config.resource}.${config.field} - ` +
+          `Record ${originalId} doesn't exist yet. Will attempt update anyway (expected to fail).`
+        );
+      }
+
+      // Create a minimal record object with just our field
+      const minimalRecord = { id: originalId };
+      for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
+        lodash.set(minimalRecord, fieldPath, value);
+      }
+
+      // Try to update (will fail, handled below)
+      const result = await tryFn(() =>
+        targetResource.update(originalId, minimalRecord)
+      );
+      updateOk = result[0];
+      updateErr = result[1];
+      updateResult = result[2];
+    } else {
+      // Record exists - apply all consolidated values using lodash.set
+      for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
+        lodash.set(record, fieldPath, value);
+      }
+
+      // Update the original record with all changes
+      // NOTE: We update the entire record to preserve nested structures
+      const result = await tryFn(() =>
+        targetResource.update(originalId, record)
+      );
+      updateOk = result[0];
+      updateErr = result[1];
+      updateResult = result[2];
+    }
+
+    // For backward compatibility, return the value of the main field
+    const consolidatedValue = consolidatedValues[config.field] ||
+                             (record ? lodash.get(record, config.field, 0) : 0);
+
     // 🔥 DEBUG: Log AFTER update
     if (config.verbose) {
       console.log(
         `🔥 [DEBUG] AFTER targetResource.update() {` +
         `\n  updateOk: ${updateOk},` +
         `\n  updateErr: ${updateErr?.message || 'undefined'},` +
-        `\n  updateResult: ${JSON.stringify(updateResult, null, 2)},` +
-        `\n  hasField: ${updateResult?.[config.field]}` +
+        `\n  consolidatedValue (main field): ${consolidatedValue}` +
         `\n}`
       );
     }
 
-    // 🔥 VERIFY: Check if update actually persisted
+    // 🔥 VERIFY: Check if update actually persisted for all fieldPaths
     if (updateOk && config.verbose) {
       // Bypass cache to get fresh data
       const [verifyOk, verifyErr, verifiedRecord] = await tryFn(() =>
         targetResource.get(originalId, { skipCache: true })
       );
 
-      console.log(
-        `🔥 [DEBUG] VERIFICATION (fresh from S3, no cache) {` +
-        `\n  verifyOk: ${verifyOk},` +
-        `\n  verifiedRecord[${config.field}]: ${verifiedRecord?.[config.field]},` +
-        `\n  expectedValue: ${consolidatedValue},` +
-        `\n  ✅ MATCH: ${verifiedRecord?.[config.field] === consolidatedValue}` +
-        `\n}`
-      );
+      // Verify each fieldPath
+      for (const [fieldPath, expectedValue] of Object.entries(consolidatedValues)) {
+        const actualValue = lodash.get(verifiedRecord, fieldPath);
+        const match = actualValue === expectedValue;
 
-      // If verification fails, this is a critical bug
-      if (verifyOk && verifiedRecord?.[config.field] !== consolidatedValue) {
-        console.error(
-          `❌ [CRITICAL BUG] Update reported success but value not persisted!` +
-          `\n  Resource: ${config.resource}` +
-          `\n  Field: ${config.field}` +
-          `\n  Record ID: ${originalId}` +
-          `\n  Expected: ${consolidatedValue}` +
-          `\n  Actually got: ${verifiedRecord?.[config.field]}` +
-          `\n  This indicates a bug in s3db.js resource.update()`
+        console.log(
+          `🔥 [DEBUG] VERIFICATION ${fieldPath} {` +
+          `\n  expectedValue: ${expectedValue},` +
+          `\n  actualValue: ${actualValue},` +
+          `\n  ${match ? '✅ MATCH' : '❌ MISMATCH'}` +
+          `\n}`
         );
+
+        // If verification fails, this is a critical bug
+        if (!match) {
+          console.error(
+            `❌ [CRITICAL BUG] Update reported success but value not persisted!` +
+            `\n  Resource: ${config.resource}` +
+            `\n  FieldPath: ${fieldPath}` +
+            `\n  Record ID: ${originalId}` +
+            `\n  Expected: ${expectedValue}` +
+            `\n  Actually got: ${actualValue}` +
+            `\n  This indicates a bug in s3db.js resource.update()`
+          );
+        }
       }
     }
 
@@ -765,6 +868,51 @@ export async function recalculateRecord(
       );
     }
 
+    // Check if there's an anchor transaction
+    const hasAnchor = allTransactions.some(txn => txn.source === 'anchor');
+
+    // If no anchor exists, create one with value 0 to serve as the baseline
+    // This ensures recalculate is idempotent - running it multiple times produces same result
+    if (!hasAnchor) {
+      const now = new Date();
+      const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
+
+      // Create anchor transaction with timestamp before all other transactions
+      const oldestTransaction = allTransactions.sort((a, b) =>
+        new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
+      )[0];
+
+      const anchorTimestamp = oldestTransaction
+        ? new Date(new Date(oldestTransaction.timestamp).getTime() - 1).toISOString()
+        : now.toISOString();
+
+      const anchorCohortInfo = getCohortInfo(new Date(anchorTimestamp), config.cohort.timezone, config.verbose);
+
+      const anchorTransaction = {
+        id: idGenerator(),
+        originalId: originalId,
+        field: config.field,
+        fieldPath: config.field,
+        value: 0,  // Always 0 for recalculate - we start from scratch
+        operation: 'set',
+        timestamp: anchorTimestamp,
+        cohortDate: anchorCohortInfo.date,
+        cohortHour: anchorCohortInfo.hour,
+        cohortMonth: anchorCohortInfo.month,
+        source: 'anchor',
+        applied: true  // Anchor is always applied
+      };
+
+      await transactionResource.insert(anchorTransaction);
+
+      if (config.verbose) {
+        console.log(
+          `[EventualConsistency] ${config.resource}.${config.field} - ` +
+          `Created anchor transaction for ${originalId} with value 0`
+        );
+      }
+    }
+
     // Mark ALL transactions as pending (applied: false)
     // Exclude anchor transactions (they should always be applied)
     const transactionsToReset = allTransactions.filter(txn => txn.source !== 'anchor');

package/src/plugins/eventual-consistency/helpers.js

@@ -18,8 +18,9 @@ import { getCohortInfo, resolveFieldAndPlugin } from "./utils.js";
 export function addHelperMethods(resource, plugin, config) {
   // Add method to set value (replaces current value)
   // Signature: set(id, field, value)
+  // Supports dot notation: set(id, 'utmResults.medium', 10)
   resource.set = async (id, field, value) => {
-    const { plugin: handler } = resolveFieldAndPlugin(resource, field, value);
+    const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, value);
 
     // Create transaction inline
     const now = new Date();
@@ -29,6 +30,7 @@ export function addHelperMethods(resource, plugin, config) {
       id: idGenerator(),
       originalId: id,
       field: handler.field,
+      fieldPath: fieldPath,  // Store full path for nested access
       value: value,
       operation: 'set',
       timestamp: now.toISOString(),
@@ -43,7 +45,7 @@ export function addHelperMethods(resource, plugin, config) {
 
     // In sync mode, immediately consolidate
     if (config.mode === 'sync') {
-      return await plugin._syncModeConsolidate(handler, id, field);
+      return await plugin._syncModeConsolidate(handler, id, fieldPath);
     }
 
     return value;
@@ -51,8 +53,9 @@ export function addHelperMethods(resource, plugin, config) {
 
   // Add method to increment value
   // Signature: add(id, field, amount)
+  // Supports dot notation: add(id, 'utmResults.medium', 5)
   resource.add = async (id, field, amount) => {
-    const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
+    const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
 
     // Create transaction inline
     const now = new Date();
@@ -62,6 +65,7 @@ export function addHelperMethods(resource, plugin, config) {
       id: idGenerator(),
       originalId: id,
       field: handler.field,
+      fieldPath: fieldPath,  // Store full path for nested access
       value: amount,
       operation: 'add',
       timestamp: now.toISOString(),
@@ -76,19 +80,25 @@ export function addHelperMethods(resource, plugin, config) {
 
     // In sync mode, immediately consolidate
     if (config.mode === 'sync') {
-      return await plugin._syncModeConsolidate(handler, id, field);
+      return await plugin._syncModeConsolidate(handler, id, fieldPath);
     }
 
     // Async mode - return current value (optimistic)
+    // Note: For nested paths, we need to use lodash get
     const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
-    const currentValue = (ok && record) ? (record[field] || 0) : 0;
+    if (!ok || !record) return amount;
+
+    // Get current value from nested path
+    const lodash = await import('lodash-es');
+    const currentValue = lodash.get(record, fieldPath, 0);
     return currentValue + amount;
   };
 
   // Add method to decrement value
   // Signature: sub(id, field, amount)
+  // Supports dot notation: sub(id, 'utmResults.medium', 3)
   resource.sub = async (id, field, amount) => {
-    const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
+    const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
 
     // Create transaction inline
     const now = new Date();
@@ -98,6 +108,7 @@ export function addHelperMethods(resource, plugin, config) {
       id: idGenerator(),
       originalId: id,
       field: handler.field,
+      fieldPath: fieldPath,  // Store full path for nested access
       value: amount,
       operation: 'sub',
       timestamp: now.toISOString(),
@@ -112,12 +123,17 @@ export function addHelperMethods(resource, plugin, config) {
 
     // In sync mode, immediately consolidate
     if (config.mode === 'sync') {
-      return await plugin._syncModeConsolidate(handler, id, field);
+      return await plugin._syncModeConsolidate(handler, id, fieldPath);
     }
 
     // Async mode - return current value (optimistic)
+    // Note: For nested paths, we need to use lodash get
     const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
-    const currentValue = (ok && record) ? (record[field] || 0) : 0;
+    if (!ok || !record) return -amount;
+
+    // Get current value from nested path
+    const lodash = await import('lodash-es');
+    const currentValue = lodash.get(record, fieldPath, 0);
     return currentValue - amount;
   };
 

package/src/plugins/eventual-consistency/install.js

@@ -95,6 +95,7 @@ export async function completeFieldSetup(handler, database, config, plugin) {
         id: 'string|required',
         originalId: 'string|required',
         field: 'string|required',
+        fieldPath: 'string|optional',  // Support for nested field paths (e.g., 'utmResults.medium')
         value: 'number|required',
         operation: 'string|required',
         timestamp: 'string|required',