s3db.js 11.0.2 → 11.0.4

package/src/plugins/eventual-consistency/analytics.js

@@ -44,14 +44,16 @@ export async function updateAnalytics(transactions, analyticsResource, config) {
     if (config.verbose) {
       console.log(
         `[EventualConsistency] ${config.resource}.${config.field} - ` +
-        `Updating ${cohortCount} hourly analytics cohorts...`
+        `Updating ${cohortCount} hourly analytics cohorts IN PARALLEL...`
       );
     }
 
-    // Update hourly analytics
-    for (const [cohort, txns] of Object.entries(byHour)) {
-      await upsertAnalytics('hour', cohort, txns, analyticsResource, config);
-    }
+    // ✅ OTIMIZAÇÃO: Update hourly analytics EM PARALELO
+    await Promise.all(
+      Object.entries(byHour).map(([cohort, txns]) =>
+        upsertAnalytics('hour', cohort, txns, analyticsResource, config)
+      )
+    );
 
     // Roll up to daily and monthly if configured
     if (config.analyticsConfig.rollupStrategy === 'incremental') {
@@ -60,13 +62,16 @@ export async function updateAnalytics(transactions, analyticsResource, config) {
       if (config.verbose) {
         console.log(
           `[EventualConsistency] ${config.resource}.${config.field} - ` +
-          `Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
+          `Rolling up ${uniqueHours.length} hours to daily/weekly/monthly analytics IN PARALLEL...`
         );
       }
 
-      for (const cohortHour of uniqueHours) {
-        await rollupAnalytics(cohortHour, analyticsResource, config);
-      }
+      // ✅ OTIMIZAÇÃO: Rollup analytics EM PARALELO
+      await Promise.all(
+        uniqueHours.map(cohortHour =>
+          rollupAnalytics(cohortHour, analyticsResource, config)
+        )
+      );
     }
 
     if (config.verbose) {
@@ -206,7 +211,7 @@ function calculateOperationBreakdown(transactions) {
 }
 
 /**
- * Roll up hourly analytics to daily and monthly
+ * Roll up hourly analytics to daily, weekly, and monthly
  * @private
  */
 async function rollupAnalytics(cohortHour, analyticsResource, config) {
@@ -214,20 +219,58 @@ async function rollupAnalytics(cohortHour, analyticsResource, config) {
   const cohortDate = cohortHour.substring(0, 10); // '2025-10-09'
   const cohortMonth = cohortHour.substring(0, 7);  // '2025-10'
 
+  // Calculate week cohort (ISO 8601 format)
+  const date = new Date(cohortDate);
+  const cohortWeek = getCohortWeekFromDate(date);
+
   // Roll up to day
   await rollupPeriod('day', cohortDate, cohortDate, analyticsResource, config);
 
+  // Roll up to week
+  await rollupPeriod('week', cohortWeek, cohortWeek, analyticsResource, config);
+
   // Roll up to month
   await rollupPeriod('month', cohortMonth, cohortMonth, analyticsResource, config);
 }
 
+/**
+ * Get cohort week string from a date
+ * @private
+ */
+function getCohortWeekFromDate(date) {
+  // ISO week calculation (use UTC methods)
+  const target = new Date(date.valueOf());
+  const dayNr = (date.getUTCDay() + 6) % 7;
+  target.setUTCDate(target.getUTCDate() - dayNr + 3);
+
+  const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
+  const firstThursday = new Date(yearStart.valueOf());
+  if (yearStart.getUTCDay() !== 4) {
+    firstThursday.setUTCDate(yearStart.getUTCDate() + ((4 - yearStart.getUTCDay()) + 7) % 7);
+  }
+
+  const weekNumber = 1 + Math.round((target - firstThursday) / 604800000);
+  const weekYear = target.getUTCFullYear();
+
+  return `${weekYear}-W${String(weekNumber).padStart(2, '0')}`;
+}
+
 /**
  * Roll up analytics for a specific period
  * @private
  */
 async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
-  // Get all source analytics (e.g., all hours for a day)
-  const sourcePeriod = period === 'day' ? 'hour' : 'day';
+  // Get all source analytics (e.g., all hours for a day, all days for a week)
+  let sourcePeriod;
+  if (period === 'day') {
+    sourcePeriod = 'hour';
+  } else if (period === 'week') {
+    sourcePeriod = 'day';  // Week aggregates from days
+  } else if (period === 'month') {
+    sourcePeriod = 'day';  // ✅ Month aggregates from days AND hours (like week)
+  } else {
+    sourcePeriod = 'day'; // Fallback
+  }
 
   const [ok, err, allAnalytics] = await tryFn(() =>
     analyticsResource.list()
@@ -236,9 +279,24 @@ async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, con
   if (!ok || !allAnalytics) return;
 
   // Filter to matching cohorts
-  const sourceAnalytics = allAnalytics.filter(a =>
-    a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
-  );
+  let sourceAnalytics;
+  if (period === 'week') {
+    // For week, we need to find all days that belong to this week
+    sourceAnalytics = allAnalytics.filter(a => {
+      if (a.period !== sourcePeriod) return false;
+      // Check if this day's cohort belongs to the target week
+      const dayDate = new Date(a.cohort);
+      const dayWeek = getCohortWeekFromDate(dayDate);
+      return dayWeek === cohort;
+    });
+  } else {
+    // For day and month, simple prefix matching works
+    // day: aggregates from hours (cohort '2025-10-09' matches '2025-10-09T14', '2025-10-09T15', etc)
+    // month: aggregates from days (cohort '2025-10' matches '2025-10-01', '2025-10-02', etc)
+    sourceAnalytics = allAnalytics.filter(a =>
+      a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
+    );
+  }
 
   if (sourceAnalytics.length === 0) return;
 
@@ -567,6 +625,66 @@ export async function getYearByMonth(resourceName, field, year, options, fieldHa
   return data;
 }
 
+/**
+ * Get analytics for entire year, broken down by weeks
+ *
+ * @param {string} resourceName - Resource name
+ * @param {string} field - Field name
+ * @param {number} year - Year (e.g., 2025)
+ * @param {Object} options - Options
+ * @param {Object} fieldHandlers - Field handlers map
+ * @returns {Promise<Array>} Weekly analytics for the year (up to 53 weeks)
+ */
+export async function getYearByWeek(resourceName, field, year, options, fieldHandlers) {
+  const data = await getAnalytics(resourceName, field, {
+    period: 'week',
+    year
+  }, fieldHandlers);
+
+  // Week data doesn't need gap filling as much as daily/hourly
+  // But we can still provide it if requested
+  if (options.fillGaps) {
+    // ISO weeks: typically 52-53 weeks per year
+    const startWeek = `${year}-W01`;
+    const endWeek = `${year}-W53`;
+    return fillGaps(data, 'week', startWeek, endWeek);
+  }
+
+  return data;
+}
+
+/**
+ * Get analytics for entire month, broken down by weeks
+ *
+ * @param {string} resourceName - Resource name
+ * @param {string} field - Field name
+ * @param {string} month - Month in YYYY-MM format
+ * @param {Object} options - Options
+ * @param {Object} fieldHandlers - Field handlers map
+ * @returns {Promise<Array>} Weekly analytics for the month
+ */
+export async function getMonthByWeek(resourceName, field, month, options, fieldHandlers) {
+  // month format: '2025-10'
+  const year = parseInt(month.substring(0, 4));
+  const monthNum = parseInt(month.substring(5, 7));
+
+  // Get first and last day of month
+  const firstDay = new Date(year, monthNum - 1, 1);
+  const lastDay = new Date(year, monthNum, 0);
+
+  // Find which weeks this month spans
+  const firstWeek = getCohortWeekFromDate(firstDay);
+  const lastWeek = getCohortWeekFromDate(lastDay);
+
+  const data = await getAnalytics(resourceName, field, {
+    period: 'week',
+    startDate: firstWeek,
+    endDate: lastWeek
+  }, fieldHandlers);
+
+  return data;
+}
+
 /**
  * Get analytics for entire month, broken down by hours
  *
@@ -690,3 +808,220 @@ export async function getTopRecords(resourceName, field, options, fieldHandlers)
   // Limit results
   return records.slice(0, limit);
 }
+
+/**
+ * Get analytics for entire year, broken down by days
+ *
+ * @param {string} resourceName - Resource name
+ * @param {string} field - Field name
+ * @param {number} year - Year (e.g., 2025)
+ * @param {Object} options - Options
+ * @param {Object} fieldHandlers - Field handlers map
+ * @returns {Promise<Array>} Daily analytics for the year (up to 365/366 records)
+ */
+export async function getYearByDay(resourceName, field, year, options, fieldHandlers) {
+  const startDate = `${year}-01-01`;
+  const endDate = `${year}-12-31`;
+
+  const data = await getAnalytics(resourceName, field, {
+    period: 'day',
+    startDate,
+    endDate
+  }, fieldHandlers);
+
+  if (options.fillGaps) {
+    return fillGaps(data, 'day', startDate, endDate);
+  }
+
+  return data;
+}
+
+/**
+ * Get analytics for entire week, broken down by days
+ *
+ * @param {string} resourceName - Resource name
+ * @param {string} field - Field name
+ * @param {string} week - Week in YYYY-Www format (e.g., '2025-W42')
+ * @param {Object} options - Options
+ * @param {Object} fieldHandlers - Field handlers map
+ * @returns {Promise<Array>} Daily analytics for the week (7 records)
+ */
+export async function getWeekByDay(resourceName, field, week, options, fieldHandlers) {
+  // week format: '2025-W42'
+  const year = parseInt(week.substring(0, 4));
+  const weekNum = parseInt(week.substring(6, 8));
+
+  // Calculate the first day of the week (Monday)
+  const jan4 = new Date(year, 0, 4);
+  const jan4Day = jan4.getDay() || 7; // Sunday = 7
+  const firstMonday = new Date(year, 0, 4 - jan4Day + 1);
+  const weekStart = new Date(firstMonday);
+  weekStart.setDate(weekStart.getDate() + (weekNum - 1) * 7);
+
+  // Get all 7 days of the week
+  const days = [];
+  for (let i = 0; i < 7; i++) {
+    const day = new Date(weekStart);
+    day.setDate(weekStart.getDate() + i);
+    days.push(day.toISOString().substring(0, 10));
+  }
+
+  const startDate = days[0];
+  const endDate = days[6];
+
+  const data = await getAnalytics(resourceName, field, {
+    period: 'day',
+    startDate,
+    endDate
+  }, fieldHandlers);
+
+  if (options.fillGaps) {
+    return fillGaps(data, 'day', startDate, endDate);
+  }
+
+  return data;
+}
+
+/**
+ * Get analytics for entire week, broken down by hours
+ *
+ * @param {string} resourceName - Resource name
+ * @param {string} field - Field name
+ * @param {string} week - Week in YYYY-Www format (e.g., '2025-W42')
+ * @param {Object} options - Options
+ * @param {Object} fieldHandlers - Field handlers map
+ * @returns {Promise<Array>} Hourly analytics for the week (168 records)
+ */
+export async function getWeekByHour(resourceName, field, week, options, fieldHandlers) {
+  // week format: '2025-W42'
+  const year = parseInt(week.substring(0, 4));
+  const weekNum = parseInt(week.substring(6, 8));
+
+  // Calculate the first day of the week (Monday)
+  const jan4 = new Date(year, 0, 4);
+  const jan4Day = jan4.getDay() || 7; // Sunday = 7
+  const firstMonday = new Date(year, 0, 4 - jan4Day + 1);
+  const weekStart = new Date(firstMonday);
+  weekStart.setDate(weekStart.getDate() + (weekNum - 1) * 7);
+
+  // Get first and last day of week
+  const weekEnd = new Date(weekStart);
+  weekEnd.setDate(weekEnd.getDate() + 6);
+
+  const startDate = weekStart.toISOString().substring(0, 10);
+  const endDate = weekEnd.toISOString().substring(0, 10);
+
+  const data = await getAnalytics(resourceName, field, {
+    period: 'hour',
+    startDate,
+    endDate
+  }, fieldHandlers);
+
+  if (options.fillGaps) {
+    return fillGaps(data, 'hour', startDate, endDate);
+  }
+
+  return data;
+}
+
+/**
+ * Get analytics for last N hours
+ *
+ * @param {string} resourceName - Resource name
+ * @param {string} field - Field name
+ * @param {number} hours - Number of hours to look back (default: 24)
+ * @param {Object} options - Options
+ * @param {Object} fieldHandlers - Field handlers map
+ * @returns {Promise<Array>} Hourly analytics
+ */
+export async function getLastNHours(resourceName, field, hours = 24, options, fieldHandlers) {
+  const now = new Date();
+  const hoursAgo = new Date(now);
+  hoursAgo.setHours(hoursAgo.getHours() - hours);
+
+  const startDate = hoursAgo.toISOString().substring(0, 13); // YYYY-MM-DDTHH
+  const endDate = now.toISOString().substring(0, 13);
+
+  const data = await getAnalytics(resourceName, field, {
+    period: 'hour',
+    startDate,
+    endDate
+  }, fieldHandlers);
+
+  if (options.fillGaps) {
+    const startDay = hoursAgo.toISOString().substring(0, 10);
+    const endDay = now.toISOString().substring(0, 10);
+    return fillGaps(data, 'hour', startDay, endDay);
+  }
+
+  return data;
+}
+
+/**
+ * Get analytics for last N weeks
+ *
+ * @param {string} resourceName - Resource name
+ * @param {string} field - Field name
+ * @param {number} weeks - Number of weeks to look back (default: 4)
+ * @param {Object} options - Options
+ * @param {Object} fieldHandlers - Field handlers map
+ * @returns {Promise<Array>} Weekly analytics
+ */
+export async function getLastNWeeks(resourceName, field, weeks = 4, options, fieldHandlers) {
+  const now = new Date();
+  const weeksAgo = new Date(now);
+  weeksAgo.setDate(weeksAgo.getDate() - (weeks * 7));
+
+  // Get week cohorts for the range
+  const weekCohorts = [];
+  const currentDate = new Date(weeksAgo);
+  while (currentDate <= now) {
+    const weekCohort = getCohortWeekFromDate(currentDate);
+    if (!weekCohorts.includes(weekCohort)) {
+      weekCohorts.push(weekCohort);
+    }
+    currentDate.setDate(currentDate.getDate() + 7);
+  }
+
+  const startWeek = weekCohorts[0];
+  const endWeek = weekCohorts[weekCohorts.length - 1];
+
+  const data = await getAnalytics(resourceName, field, {
+    period: 'week',
+    startDate: startWeek,
+    endDate: endWeek
+  }, fieldHandlers);
+
+  return data;
+}
+
+/**
+ * Get analytics for last N months
+ *
+ * @param {string} resourceName - Resource name
+ * @param {string} field - Field name
+ * @param {number} months - Number of months to look back (default: 12)
+ * @param {Object} options - Options
+ * @param {Object} fieldHandlers - Field handlers map
+ * @returns {Promise<Array>} Monthly analytics
+ */
+export async function getLastNMonths(resourceName, field, months = 12, options, fieldHandlers) {
+  const now = new Date();
+  const monthsAgo = new Date(now);
+  monthsAgo.setMonth(monthsAgo.getMonth() - months);
+
+  const startDate = monthsAgo.toISOString().substring(0, 7); // YYYY-MM
+  const endDate = now.toISOString().substring(0, 7);
+
+  const data = await getAnalytics(resourceName, field, {
+    period: 'month',
+    startDate,
+    endDate
+  }, fieldHandlers);
+
+  if (options.fillGaps) {
+    return fillGaps(data, 'month', startDate, endDate);
+  }
+
+  return data;
+}

package/src/plugins/eventual-consistency/config.js

@@ -47,6 +47,9 @@ export function createConfig(options, detectedTimezone) {
     autoConsolidate: consolidation.auto !== false,
     mode: consolidation.mode || 'async',
 
+    // ✅ NOVO: Performance tuning - Mark applied concurrency (default 50, antes era 10 hardcoded)
+    markAppliedConcurrency: consolidation.markAppliedConcurrency ?? 50,
+
     // Late arrivals
     lateArrivalStrategy: lateArrivals.strategy || 'warn',
 

package/src/plugins/eventual-consistency/consolidation.js

@@ -7,7 +7,6 @@ import tryFn from "../../concerns/try-fn.js";
 import { PromisePool } from "@supercharge/promise-pool";
 import { idGenerator } from "../../concerns/id.js";
 import { getCohortInfo, createSyntheticSetTransaction } from "./utils.js";
-import { cleanupStaleLocks } from "./locks.js";
 
 /**
  * Start consolidation timer for a handler
@@ -169,7 +168,7 @@ export async function runConsolidation(transactionResource, consolidateRecordFn,
  * @param {string} originalId - ID of the record to consolidate
  * @param {Object} transactionResource - Transaction resource
  * @param {Object} targetResource - Target resource
- * @param {Object} lockResource - Lock resource
+ * @param {Object} storage - PluginStorage instance for locks
  * @param {Object} analyticsResource - Analytics resource (optional)
  * @param {Function} updateAnalyticsFn - Function to update analytics (optional)
  * @param {Object} config - Plugin configuration
@@ -179,26 +178,21 @@ export async function consolidateRecord(
   originalId,
   transactionResource,
   targetResource,
-  lockResource,
+  storage,
   analyticsResource,
   updateAnalyticsFn,
   config
 ) {
-  // Clean up stale locks before attempting to acquire
-  await cleanupStaleLocks(lockResource, config);
-
-  // Acquire distributed lock to prevent concurrent consolidation
-  const lockId = `lock-${originalId}`;
-  const [lockAcquired, lockErr, lock] = await tryFn(() =>
-    lockResource.insert({
-      id: lockId,
-      lockedAt: Date.now(),
-      workerId: process.pid ? String(process.pid) : 'unknown'
-    })
-  );
+  // Acquire distributed lock with TTL to prevent concurrent consolidation
+  const lockKey = `consolidation-${config.resource}-${config.field}-${originalId}`;
+  const lock = await storage.acquireLock(lockKey, {
+    ttl: config.lockTimeout || 30,
+    timeout: 0, // Don't wait if locked
+    workerId: process.pid ? String(process.pid) : 'unknown'
+  });
 
   // If lock couldn't be acquired, another worker is consolidating
-  if (!lockAcquired) {
+  if (!lock) {
     if (config.verbose) {
       console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
     }
@@ -508,9 +502,12 @@ export async function consolidateRecord(
       // Mark transactions as applied (skip synthetic ones) - use PromisePool for controlled concurrency
       const transactionsToUpdate = transactions.filter(txn => txn.id !== '__synthetic__');
 
+      // ✅ OTIMIZAÇÃO: Usar concurrency do config (default aumentado de 10 para 50)
+      const markAppliedConcurrency = config.markAppliedConcurrency || 50;
+
       const { results, errors } = await PromisePool
         .for(transactionsToUpdate)
-        .withConcurrency(10) // Limit parallel updates
+        .withConcurrency(markAppliedConcurrency) // ✅ Configurável e maior!
         .process(async (txn) => {
           const [ok, err] = await tryFn(() =>
             transactionResource.update(txn.id, { applied: true })
@@ -576,10 +573,12 @@ export async function consolidateRecord(
     return consolidatedValue;
   } finally {
     // Always release the lock
-    const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
+    const [lockReleased, lockReleaseErr] = await tryFn(() =>
+      storage.releaseLock(lockKey)
+    );
 
     if (!lockReleased && config.verbose) {
-      console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
+      console.warn(`[EventualConsistency] Failed to release lock ${lockKey}:`, lockReleaseErr?.message);
     }
   }
 }
@@ -705,7 +704,7 @@ export async function getCohortStats(cohortDate, transactionResource) {
  * @param {string} originalId - ID of the record to recalculate
  * @param {Object} transactionResource - Transaction resource
  * @param {Object} targetResource - Target resource
- * @param {Object} lockResource - Lock resource
+ * @param {Object} storage - PluginStorage instance for locks
  * @param {Function} consolidateRecordFn - Function to consolidate the record
  * @param {Object} config - Plugin configuration
  * @returns {Promise<number>} Recalculated value
@@ -714,25 +713,20 @@ export async function recalculateRecord(
   originalId,
   transactionResource,
   targetResource,
-  lockResource,
+  storage,
   consolidateRecordFn,
   config
 ) {
-  // Clean up stale locks before attempting to acquire
-  await cleanupStaleLocks(lockResource, config);
-
-  // Acquire distributed lock to prevent concurrent operations
-  const lockId = `lock-recalculate-${originalId}`;
-  const [lockAcquired, lockErr, lock] = await tryFn(() =>
-    lockResource.insert({
-      id: lockId,
-      lockedAt: Date.now(),
-      workerId: process.pid ? String(process.pid) : 'unknown'
-    })
-  );
+  // Acquire distributed lock with TTL to prevent concurrent operations
+  const lockKey = `recalculate-${config.resource}-${config.field}-${originalId}`;
+  const lock = await storage.acquireLock(lockKey, {
+    ttl: config.lockTimeout || 30,
+    timeout: 0, // Don't wait if locked
+    workerId: process.pid ? String(process.pid) : 'unknown'
+  });
 
   // If lock couldn't be acquired, another worker is operating on this record
-  if (!lockAcquired) {
+  if (!lock) {
     if (config.verbose) {
       console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
     }
@@ -832,10 +826,12 @@ export async function recalculateRecord(
     return consolidatedValue;
   } finally {
     // Always release the lock
-    const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
+    const [lockReleased, lockReleaseErr] = await tryFn(() =>
+      storage.releaseLock(lockKey)
+    );
 
     if (!lockReleased && config.verbose) {
-      console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockId}:`, lockReleaseErr?.message);
+      console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockKey}:`, lockReleaseErr?.message);
     }
   }
 }

package/src/plugins/eventual-consistency/garbage-collection.js

@@ -31,24 +31,22 @@ export function startGarbageCollectionTimer(handler, resourceName, fieldName, ru
  * Uses distributed locking to prevent multiple containers from running GC simultaneously
  *
  * @param {Object} transactionResource - Transaction resource
- * @param {Object} lockResource - Lock resource
+ * @param {Object} storage - PluginStorage instance for locks
  * @param {Object} config - Plugin configuration
  * @param {Function} emitFn - Function to emit events
  * @returns {Promise<void>}
  */
-export async function runGarbageCollection(transactionResource, lockResource, config, emitFn) {
-  // Acquire distributed lock for GC operation
-  const gcLockId = `lock-gc-${config.resource}-${config.field}`;
-  const [lockAcquired] = await tryFn(() =>
-    lockResource.insert({
-      id: gcLockId,
-      lockedAt: Date.now(),
-      workerId: process.pid ? String(process.pid) : 'unknown'
-    })
-  );
+export async function runGarbageCollection(transactionResource, storage, config, emitFn) {
+  // Acquire distributed lock with TTL for GC operation
+  const lockKey = `gc-${config.resource}-${config.field}`;
+  const lock = await storage.acquireLock(lockKey, {
+    ttl: 300, // 5 minutes for GC
+    timeout: 0, // Don't wait if locked
+    workerId: process.pid ? String(process.pid) : 'unknown'
+  });
 
   // If another container is already running GC, skip
-  if (!lockAcquired) {
+  if (!lock) {
     if (config.verbose) {
       console.log(`[EventualConsistency] GC already running in another container`);
     }
@@ -121,6 +119,6 @@ export async function runGarbageCollection(transactionResource, lockResource, co
     }
   } finally {
     // Always release GC lock
-    await tryFn(() => lockResource.delete(gcLockId));
+    await tryFn(() => storage.releaseLock(lockKey));
   }
 }