s3db.js 11.2.0 → 11.2.3

package/src/plugins/eventual-consistency/analytics.js

@@ -1209,3 +1209,148 @@ export async function getLastNMonths(resourceName, field, months = 12, options,
 
   return data;
 }
+
+/**
+ * Get raw transaction events for custom aggregation
+ *
+ * This method provides direct access to the underlying transaction events,
+ * allowing developers to perform custom aggregations beyond the pre-built analytics.
+ * Useful for complex queries, custom metrics, or when you need the raw event data.
+ *
+ * @param {string} resourceName - Resource name
+ * @param {string} field - Field name
+ * @param {Object} options - Query options
+ * @param {string} options.recordId - Filter by specific record ID
+ * @param {string} options.startDate - Start date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
+ * @param {string} options.endDate - End date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
+ * @param {string} options.cohortDate - Filter by cohort date (YYYY-MM-DD)
+ * @param {string} options.cohortHour - Filter by cohort hour (YYYY-MM-DDTHH)
+ * @param {string} options.cohortMonth - Filter by cohort month (YYYY-MM)
+ * @param {boolean} options.applied - Filter by applied status (true/false/undefined for both)
+ * @param {string} options.operation - Filter by operation type ('add', 'sub', 'set')
+ * @param {number} options.limit - Maximum number of events to return
+ * @param {Object} fieldHandlers - Field handlers map
+ * @returns {Promise<Array>} Raw transaction events
+ *
+ * @example
+ * // Get all events for a specific record
+ * const events = await plugin.getRawEvents('wallets', 'balance', {
+ *   recordId: 'wallet1'
+ * });
+ *
+ * @example
+ * // Get events for a specific time range
+ * const events = await plugin.getRawEvents('wallets', 'balance', {
+ *   startDate: '2025-10-01',
+ *   endDate: '2025-10-31'
+ * });
+ *
+ * @example
+ * // Get only pending (unapplied) transactions
+ * const pending = await plugin.getRawEvents('wallets', 'balance', {
+ *   applied: false
+ * });
+ */
+export async function getRawEvents(resourceName, field, options, fieldHandlers) {
+  // Get handler for this resource/field combination
+  const resourceHandlers = fieldHandlers.get(resourceName);
+  if (!resourceHandlers) {
+    throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
+  }
+
+  const handler = resourceHandlers.get(field);
+  if (!handler) {
+    throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
+  }
+
+  if (!handler.transactionResource) {
+    throw new Error('Transaction resource not initialized');
+  }
+
+  const {
+    recordId,
+    startDate,
+    endDate,
+    cohortDate,
+    cohortHour,
+    cohortMonth,
+    applied,
+    operation,
+    limit
+  } = options;
+
+  // Build query object for partition-based filtering
+  const query = {};
+
+  // Filter by recordId (uses partition if available)
+  if (recordId !== undefined) {
+    query.originalId = recordId;
+  }
+
+  // Filter by applied status (uses partition)
+  if (applied !== undefined) {
+    query.applied = applied;
+  }
+
+  // Fetch transactions using partition-aware query
+  const [ok, err, allTransactions] = await tryFn(() =>
+    handler.transactionResource.query(query)
+  );
+
+  if (!ok || !allTransactions) {
+    return [];
+  }
+
+  // Ensure all transactions have cohort fields
+  let filtered = allTransactions;
+
+  // Filter by operation type
+  if (operation !== undefined) {
+    filtered = filtered.filter(t => t.operation === operation);
+  }
+
+  // Filter by temporal fields (these are in-memory filters after partition query)
+  if (cohortDate) {
+    filtered = filtered.filter(t => t.cohortDate === cohortDate);
+  }
+
+  if (cohortHour) {
+    filtered = filtered.filter(t => t.cohortHour === cohortHour);
+  }
+
+  if (cohortMonth) {
+    filtered = filtered.filter(t => t.cohortMonth === cohortMonth);
+  }
+
+  if (startDate && endDate) {
+    // Determine which cohort field to use based on format
+    const isHourly = startDate.length > 10; // YYYY-MM-DDTHH vs YYYY-MM-DD
+    const cohortField = isHourly ? 'cohortHour' : 'cohortDate';
+
+    filtered = filtered.filter(t =>
+      t[cohortField] && t[cohortField] >= startDate && t[cohortField] <= endDate
+    );
+  } else if (startDate) {
+    const isHourly = startDate.length > 10;
+    const cohortField = isHourly ? 'cohortHour' : 'cohortDate';
+    filtered = filtered.filter(t => t[cohortField] && t[cohortField] >= startDate);
+  } else if (endDate) {
+    const isHourly = endDate.length > 10;
+    const cohortField = isHourly ? 'cohortHour' : 'cohortDate';
+    filtered = filtered.filter(t => t[cohortField] && t[cohortField] <= endDate);
+  }
+
+  // Sort by timestamp (newest first by default)
+  filtered.sort((a, b) => {
+    const aTime = new Date(a.timestamp || a.createdAt).getTime();
+    const bTime = new Date(b.timestamp || b.createdAt).getTime();
+    return bTime - aTime;
+  });
+
+  // Apply limit
+  if (limit && limit > 0) {
+    filtered = filtered.slice(0, limit);
+  }
+
+  return filtered;
+}

package/src/plugins/eventual-consistency/config.js

@@ -47,9 +47,12 @@ export function createConfig(options, detectedTimezone) {
     autoConsolidate: consolidation.auto !== false,
     mode: consolidation.mode || 'async',
 
-    // ✅ NOVO: Performance tuning - Mark applied concurrency (default 50, antes era 10 hardcoded)
+    // ✅ Performance tuning - Mark applied concurrency (default 50, up from 10)
     markAppliedConcurrency: consolidation.markAppliedConcurrency ?? 50,
 
+    // ✅ Performance tuning - Recalculate concurrency (default 50, up from 10)
+    recalculateConcurrency: consolidation.recalculateConcurrency ?? 50,
+
     // Late arrivals
     lateArrivalStrategy: lateArrivals.strategy || 'warn',
 

package/src/plugins/eventual-consistency/consolidation.js

@@ -6,7 +6,7 @@
 import tryFn from "../../concerns/try-fn.js";
 import { PromisePool } from "@supercharge/promise-pool";
 import { idGenerator } from "../../concerns/id.js";
-import { getCohortInfo, createSyntheticSetTransaction } from "./utils.js";
+import { getCohortInfo, createSyntheticSetTransaction, ensureCohortHour } from "./utils.js";
 
 /**
  * Start consolidation timer for a handler
@@ -612,12 +612,43 @@ export async function consolidateRecord(
         .for(transactionsToUpdate)
         .withConcurrency(markAppliedConcurrency) // ✅ Configurável e maior!
         .process(async (txn) => {
+          // ✅ FIX BUG #3: Ensure cohort fields exist before marking as applied
+          // This handles legacy transactions missing cohortHour, cohortDate, etc.
+          const txnWithCohorts = ensureCohortHour(txn, config.cohort.timezone, false);
+
+          // Build update data with applied flag
+          const updateData = { applied: true };
+
+          // Add missing cohort fields if they were calculated
+          if (txnWithCohorts.cohortHour && !txn.cohortHour) {
+            updateData.cohortHour = txnWithCohorts.cohortHour;
+          }
+          if (txnWithCohorts.cohortDate && !txn.cohortDate) {
+            updateData.cohortDate = txnWithCohorts.cohortDate;
+          }
+          if (txnWithCohorts.cohortWeek && !txn.cohortWeek) {
+            updateData.cohortWeek = txnWithCohorts.cohortWeek;
+          }
+          if (txnWithCohorts.cohortMonth && !txn.cohortMonth) {
+            updateData.cohortMonth = txnWithCohorts.cohortMonth;
+          }
+
+          // Handle null value field (legacy data might have null)
+          if (txn.value === null || txn.value === undefined) {
+            updateData.value = 1; // Default to 1 for backward compatibility
+          }
+
           const [ok, err] = await tryFn(() =>
-            transactionResource.update(txn.id, { applied: true })
+            transactionResource.update(txn.id, updateData)
           );
 
           if (!ok && config.verbose) {
-            console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err?.message);
+            console.warn(
+              `[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`,
+              err?.message,
+              'Update data:',
+              updateData
+            );
           }
 
           return ok;
@@ -917,9 +948,12 @@ export async function recalculateRecord(
     // Exclude anchor transactions (they should always be applied)
     const transactionsToReset = allTransactions.filter(txn => txn.source !== 'anchor');
 
+    // ✅ OPTIMIZATION: Use higher concurrency for recalculate (default 50 vs 10)
+    const recalculateConcurrency = config.recalculateConcurrency || 50;
+
     const { results, errors } = await PromisePool
       .for(transactionsToReset)
-      .withConcurrency(10)
+      .withConcurrency(recalculateConcurrency)
       .process(async (txn) => {
         const [ok, err] = await tryFn(() =>
           transactionResource.update(txn.id, { applied: false })

package/src/plugins/eventual-consistency/index.js

@@ -17,7 +17,7 @@ import {
   runConsolidation
 } from "./consolidation.js";
 import { runGarbageCollection } from "./garbage-collection.js";
-import { updateAnalytics, getAnalytics, getMonthByDay, getDayByHour, getLastNDays, getYearByMonth, getYearByWeek, getMonthByWeek, getMonthByHour, getTopRecords, getYearByDay, getWeekByDay, getWeekByHour, getLastNHours, getLastNWeeks, getLastNMonths } from "./analytics.js";
+import { updateAnalytics, getAnalytics, getMonthByDay, getDayByHour, getLastNDays, getYearByMonth, getYearByWeek, getMonthByWeek, getMonthByHour, getTopRecords, getYearByDay, getWeekByDay, getWeekByHour, getLastNHours, getLastNWeeks, getLastNMonths, getRawEvents } from "./analytics.js";
 import { onInstall, onStart, onStop, watchForResource, completeFieldSetup } from "./install.js";
 
 export class EventualConsistencyPlugin extends Plugin {
@@ -531,6 +531,208 @@ export class EventualConsistencyPlugin extends Plugin {
   async getLastNMonths(resourceName, field, months = 12, options = {}) {
     return await getLastNMonths(resourceName, field, months, options, this.fieldHandlers);
   }
+
+  /**
+   * Get raw transaction events for custom aggregation
+   *
+   * This method provides direct access to the underlying transaction events,
+   * allowing developers to perform custom aggregations beyond the pre-built analytics.
+   * Useful for complex queries, custom metrics, or when you need the raw event data.
+   *
+   * @param {string} resourceName - Resource name
+   * @param {string} field - Field name
+   * @param {Object} options - Query options
+   * @param {string} options.recordId - Filter by specific record ID
+   * @param {string} options.startDate - Start date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
+   * @param {string} options.endDate - End date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
+   * @param {string} options.cohortDate - Filter by cohort date (YYYY-MM-DD)
+   * @param {string} options.cohortHour - Filter by cohort hour (YYYY-MM-DDTHH)
+   * @param {string} options.cohortMonth - Filter by cohort month (YYYY-MM)
+   * @param {boolean} options.applied - Filter by applied status (true/false/undefined for both)
+   * @param {string} options.operation - Filter by operation type ('add', 'sub', 'set')
+   * @param {number} options.limit - Maximum number of events to return
+   * @returns {Promise<Array>} Raw transaction events
+   *
+   * @example
+   * // Get all events for a specific record
+   * const events = await plugin.getRawEvents('wallets', 'balance', {
+   *   recordId: 'wallet1'
+   * });
+   *
+   * @example
+   * // Get events for a specific time range
+   * const events = await plugin.getRawEvents('wallets', 'balance', {
+   *   startDate: '2025-10-01',
+   *   endDate: '2025-10-31'
+   * });
+   *
+   * @example
+   * // Get only pending (unapplied) transactions
+   * const pending = await plugin.getRawEvents('wallets', 'balance', {
+   *   applied: false
+   * });
+   */
+  async getRawEvents(resourceName, field, options = {}) {
+    return await getRawEvents(resourceName, field, options, this.fieldHandlers);
+  }
+
+  /**
+   * Get diagnostics information about the plugin state
+   *
+   * This method provides comprehensive diagnostic information about the EventualConsistencyPlugin,
+   * including configured resources, field handlers, timers, and overall health status.
+   * Useful for debugging initialization issues, configuration problems, or runtime errors.
+   *
+   * @param {Object} options - Diagnostic options
+   * @param {string} options.resourceName - Optional: limit diagnostics to specific resource
+   * @param {string} options.field - Optional: limit diagnostics to specific field
+   * @param {boolean} options.includeStats - Include transaction statistics (default: false)
+   * @returns {Promise<Object>} Diagnostic information
+   *
+   * @example
+   * // Get overall plugin diagnostics
+   * const diagnostics = await plugin.getDiagnostics();
+   * console.log(diagnostics);
+   *
+   * @example
+   * // Get diagnostics for specific resource/field with stats
+   * const diagnostics = await plugin.getDiagnostics({
+   *   resourceName: 'wallets',
+   *   field: 'balance',
+   *   includeStats: true
+   * });
+   */
+  async getDiagnostics(options = {}) {
+    const { resourceName, field, includeStats = false } = options;
+
+    const diagnostics = {
+      plugin: {
+        name: 'EventualConsistencyPlugin',
+        initialized: this.database !== null && this.database !== undefined,
+        verbose: this.config.verbose || false,
+        timezone: this.config.cohort?.timezone || 'UTC',
+        consolidation: {
+          mode: this.config.consolidation?.mode || 'timer',
+          interval: this.config.consolidation?.interval || 60000,
+          batchSize: this.config.consolidation?.batchSize || 100
+        },
+        garbageCollection: {
+          enabled: this.config.garbageCollection?.enabled !== false,
+          retentionDays: this.config.garbageCollection?.retentionDays || 30,
+          interval: this.config.garbageCollection?.interval || 3600000
+        }
+      },
+      resources: [],
+      errors: [],
+      warnings: []
+    };
+
+    // Iterate through configured resources
+    for (const [resName, resourceHandlers] of this.fieldHandlers.entries()) {
+      // Skip if filtering by resource and this isn't it
+      if (resourceName && resName !== resourceName) {
+        continue;
+      }
+
+      const resourceDiag = {
+        name: resName,
+        fields: []
+      };
+
+      for (const [fieldName, handler] of resourceHandlers.entries()) {
+        // Skip if filtering by field and this isn't it
+        if (field && fieldName !== field) {
+          continue;
+        }
+
+        const fieldDiag = {
+          name: fieldName,
+          type: handler.type || 'counter',
+          analyticsEnabled: handler.analyticsResource !== null && handler.analyticsResource !== undefined,
+          resources: {
+            transaction: handler.transactionResource?.name || null,
+            target: handler.targetResource?.name || null,
+            analytics: handler.analyticsResource?.name || null
+          },
+          timers: {
+            consolidation: handler.consolidationTimer !== null && handler.consolidationTimer !== undefined,
+            garbageCollection: handler.garbageCollectionTimer !== null && handler.garbageCollectionTimer !== undefined
+          }
+        };
+
+        // Check for common issues
+        if (!handler.transactionResource) {
+          diagnostics.errors.push({
+            resource: resName,
+            field: fieldName,
+            issue: 'Missing transaction resource',
+            suggestion: 'Ensure plugin is installed and resources are created after plugin installation'
+          });
+        }
+
+        if (!handler.targetResource) {
+          diagnostics.warnings.push({
+            resource: resName,
+            field: fieldName,
+            issue: 'Missing target resource',
+            suggestion: 'Target resource may not have been created yet'
+          });
+        }
+
+        if (handler.analyticsResource && !handler.analyticsResource.name) {
+          diagnostics.errors.push({
+            resource: resName,
+            field: fieldName,
+            issue: 'Invalid analytics resource',
+            suggestion: 'Analytics resource exists but has no name - possible initialization failure'
+          });
+        }
+
+        // Include statistics if requested
+        if (includeStats && handler.transactionResource) {
+          try {
+            const [okPending, errPending, pendingTxns] = await handler.transactionResource.query({ applied: false }).catch(() => [false, null, []]);
+            const [okApplied, errApplied, appliedTxns] = await handler.transactionResource.query({ applied: true }).catch(() => [false, null, []]);
+
+            fieldDiag.stats = {
+              pendingTransactions: okPending ? (pendingTxns?.length || 0) : 'error',
+              appliedTransactions: okApplied ? (appliedTxns?.length || 0) : 'error',
+              totalTransactions: (okPending && okApplied) ? ((pendingTxns?.length || 0) + (appliedTxns?.length || 0)) : 'error'
+            };
+
+            if (handler.analyticsResource) {
+              const [okAnalytics, errAnalytics, analyticsRecords] = await handler.analyticsResource.list().catch(() => [false, null, []]);
+              fieldDiag.stats.analyticsRecords = okAnalytics ? (analyticsRecords?.length || 0) : 'error';
+            }
+          } catch (error) {
+            diagnostics.warnings.push({
+              resource: resName,
+              field: fieldName,
+              issue: 'Failed to fetch statistics',
+              error: error.message
+            });
+          }
+        }
+
+        resourceDiag.fields.push(fieldDiag);
+      }
+
+      if (resourceDiag.fields.length > 0) {
+        diagnostics.resources.push(resourceDiag);
+      }
+    }
+
+    // Overall health check
+    diagnostics.health = {
+      status: diagnostics.errors.length === 0 ? (diagnostics.warnings.length === 0 ? 'healthy' : 'warning') : 'error',
+      totalResources: diagnostics.resources.length,
+      totalFields: diagnostics.resources.reduce((sum, r) => sum + r.fields.length, 0),
+      errorCount: diagnostics.errors.length,
+      warningCount: diagnostics.warnings.length
+    };
+
+    return diagnostics;
+  }
 }
 
 export default EventualConsistencyPlugin;

package/src/plugins/eventual-consistency/install.js

@@ -173,6 +173,28 @@ async function createAnalyticsResource(handler, database, resourceName, fieldNam
       },
       behavior: 'body-overflow',
       timestamps: false,
+      asyncPartitions: true,
+      // ✅ Multi-attribute partitions for optimal analytics query performance
+      partitions: {
+        // Query by period (hour/day/week/month)
+        byPeriod: {
+          fields: { period: 'string' }
+        },
+        // Query by period + cohort (e.g., all hour records for specific hours)
+        byPeriodCohort: {
+          fields: {
+            period: 'string',
+            cohort: 'string'
+          }
+        },
+        // Query by field + period (e.g., all daily analytics for clicks field)
+        byFieldPeriod: {
+          fields: {
+            field: 'string',
+            period: 'string'
+          }
+        }
+      },
       createdBy: 'EventualConsistencyPlugin'
     })
   );

package/src/resource.class.js

@@ -134,6 +134,7 @@ export class Resource extends AsyncEventEmitter {
       idGenerator: customIdGenerator,
       idSize = 22,
       versioningEnabled = false,
+      strictValidation = true,
       events = {},
       asyncEvents = true,
       asyncPartitions = true,
@@ -149,6 +150,7 @@ export class Resource extends AsyncEventEmitter {
     this.parallelism = parallelism;
     this.passphrase = passphrase ?? 'secret';
     this.versioningEnabled = versioningEnabled;
+    this.strictValidation = strictValidation;
     
     // Configure async events mode
     this.setAsyncMode(asyncEvents);
@@ -476,9 +478,14 @@ export class Resource extends AsyncEventEmitter {
 
   /**
    * Validate that all partition fields exist in current resource attributes
-   * @throws {Error} If partition fields don't exist in current schema
+   * @throws {Error} If partition fields don't exist in current schema (only when strictValidation is true)
    */
   validatePartitions() {
+    // Skip validation if strictValidation is disabled
+    if (!this.strictValidation) {
+      return;
+    }
+
     if (!this.config.partitions) {
       return; // No partitions to validate
     }