s3db.js 11.1.0 → 11.2.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "11.1.0",
+  "version": "11.2.2",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",
@@ -55,6 +55,7 @@
     "mcp/server.js",
     "README.md",
     "PLUGINS.md",
+    "SECURITY.md",
     "UNLICENSE"
   ],
   "dependencies": {
@@ -141,10 +142,7 @@
     "test:full": "pnpm run test:js && pnpm run test:ts",
     "benchmark": "node benchmark-compression.js",
     "benchmark:partitions": "node docs/benchmarks/partitions-matrix.js",
-    "version": "echo 'Use pnpm run release v<version> instead of npm version'",
     "release:check": "./scripts/pre-release-check.sh",
-    "release:prepare": "pnpm run build:binaries && echo 'Binaries ready for GitHub release'",
-    "release": "./scripts/release.sh",
     "validate:types": "pnpm run test:ts && echo 'TypeScript definitions are valid!'",
     "test:ts:runtime": "tsx tests/typescript/types-runtime-simple.ts"
   }

package/src/plugins/eventual-consistency/analytics.js

@@ -4,7 +4,7 @@
  */
 
 import tryFn from "../../concerns/try-fn.js";
-import { groupByCohort } from "./utils.js";
+import { groupByCohort, ensureCohortHours } from "./utils.js";
 
 /**
  * Update analytics with consolidated transactions
@@ -459,8 +459,14 @@ export async function getAnalytics(resourceName, field, options, fieldHandlers)
     throw new Error('Analytics not enabled for this plugin');
   }
 
-  const { period = 'day', date, startDate, endDate, month, year, breakdown = false } = options;
+  const { period = 'day', date, startDate, endDate, month, year, breakdown = false, recordId } = options;
 
+  // ✅ FIX BUG #1: If recordId is specified, fetch from transactions directly
+  if (recordId) {
+    return await getAnalyticsForRecord(resourceName, field, recordId, options, handler);
+  }
+
+  // Original behavior: global analytics from pre-aggregated data
   const [ok, err, allAnalytics] = await tryFn(() =>
     handler.analyticsResource.list()
   );
@@ -511,6 +517,156 @@ export async function getAnalytics(resourceName, field, options, fieldHandlers)
   }));
 }
 
+/**
+ * Get analytics for a specific record from transactions
+ * ✅ FIX BUG #1: Filter by recordId
+ *
+ * @param {string} resourceName - Resource name
+ * @param {string} field - Field name
+ * @param {string} recordId - Record ID to filter by
+ * @param {Object} options - Query options
+ * @param {Object} handler - Field handler
+ * @returns {Promise<Array>} Analytics data for specific record
+ * @private
+ */
+async function getAnalyticsForRecord(resourceName, field, recordId, options, handler) {
+  const { period = 'day', date, startDate, endDate, month, year } = options;
+
+  // Fetch transactions for this specific record
+  // Note: We need both applied: true and applied: false transactions for analytics
+  //  Same format as used in consolidation.js - the query() method auto-selects the partition
+  const [okTrue, errTrue, appliedTransactions] = await tryFn(() =>
+    handler.transactionResource.query({
+      originalId: recordId,
+      applied: true
+    })
+  );
+
+  const [okFalse, errFalse, pendingTransactions] = await tryFn(() =>
+    handler.transactionResource.query({
+      originalId: recordId,
+      applied: false
+    })
+  );
+
+  // Combine both applied and pending transactions
+  let allTransactions = [
+    ...(okTrue && appliedTransactions ? appliedTransactions : []),
+    ...(okFalse && pendingTransactions ? pendingTransactions : [])
+  ];
+
+  if (allTransactions.length === 0) {
+    return [];
+  }
+
+  // ✅ FIX BUG #2: Ensure all transactions have cohortHour calculated
+  // This handles legacy data that may be missing cohortHour
+  allTransactions = ensureCohortHours(allTransactions, handler.config?.cohort?.timezone || 'UTC', false);
+
+  // Filter transactions by temporal range
+  let filtered = allTransactions;
+
+  if (date) {
+    if (period === 'hour') {
+      // Match all hours of the date
+      filtered = filtered.filter(t => t.cohortHour && t.cohortHour.startsWith(date));
+    } else if (period === 'day') {
+      filtered = filtered.filter(t => t.cohortDate === date);
+    } else if (period === 'month') {
+      filtered = filtered.filter(t => t.cohortMonth && t.cohortMonth.startsWith(date));
+    }
+  } else if (startDate && endDate) {
+    if (period === 'hour') {
+      filtered = filtered.filter(t => t.cohortHour && t.cohortHour >= startDate && t.cohortHour <= endDate);
+    } else if (period === 'day') {
+      filtered = filtered.filter(t => t.cohortDate && t.cohortDate >= startDate && t.cohortDate <= endDate);
+    } else if (period === 'month') {
+      filtered = filtered.filter(t => t.cohortMonth && t.cohortMonth >= startDate && t.cohortMonth <= endDate);
+    }
+  } else if (month) {
+    if (period === 'hour') {
+      filtered = filtered.filter(t => t.cohortHour && t.cohortHour.startsWith(month));
+    } else if (period === 'day') {
+      filtered = filtered.filter(t => t.cohortDate && t.cohortDate.startsWith(month));
+    }
+  } else if (year) {
+    if (period === 'hour') {
+      filtered = filtered.filter(t => t.cohortHour && t.cohortHour.startsWith(String(year)));
+    } else if (period === 'day') {
+      filtered = filtered.filter(t => t.cohortDate && t.cohortDate.startsWith(String(year)));
+    } else if (period === 'month') {
+      filtered = filtered.filter(t => t.cohortMonth && t.cohortMonth.startsWith(String(year)));
+    }
+  }
+
+  // Aggregate transactions by cohort
+  const cohortField = period === 'hour' ? 'cohortHour' : period === 'day' ? 'cohortDate' : 'cohortMonth';
+  const aggregated = aggregateTransactionsByCohort(filtered, cohortField);
+
+  return aggregated;
+}
+
+/**
+ * Aggregate transactions by cohort field
+ * ✅ Helper for BUG #1 fix
+ *
+ * @param {Array} transactions - Transactions to aggregate
+ * @param {string} cohortField - Cohort field name ('cohortHour', 'cohortDate', 'cohortMonth')
+ * @returns {Array} Aggregated analytics
+ * @private
+ */
+function aggregateTransactionsByCohort(transactions, cohortField) {
+  const groups = {};
+
+  for (const txn of transactions) {
+    const cohort = txn[cohortField];
+    if (!cohort) continue;
+
+    if (!groups[cohort]) {
+      groups[cohort] = {
+        cohort,
+        count: 0,
+        sum: 0,
+        min: Infinity,
+        max: -Infinity,
+        recordCount: new Set(),
+        operations: {}
+      };
+    }
+
+    const group = groups[cohort];
+    const signedValue = txn.operation === 'sub' ? -txn.value : txn.value;
+
+    group.count++;
+    group.sum += signedValue;
+    group.min = Math.min(group.min, signedValue);
+    group.max = Math.max(group.max, signedValue);
+    group.recordCount.add(txn.originalId);
+
+    // Track operation breakdown
+    const op = txn.operation;
+    if (!group.operations[op]) {
+      group.operations[op] = { count: 0, sum: 0 };
+    }
+    group.operations[op].count++;
+    group.operations[op].sum += signedValue;
+  }
+
+  // Convert to array and finalize
+  return Object.values(groups)
+    .map(g => ({
+      cohort: g.cohort,
+      count: g.count,
+      sum: g.sum,
+      avg: g.sum / g.count,
+      min: g.min === Infinity ? 0 : g.min,
+      max: g.max === -Infinity ? 0 : g.max,
+      recordCount: g.recordCount.size,
+      operations: g.operations
+    }))
+    .sort((a, b) => a.cohort.localeCompare(b.cohort));
+}
+
 /**
  * Get analytics for entire month, broken down by days
  *
@@ -587,7 +743,9 @@ export async function getLastNDays(resourceName, field, days, options, fieldHand
     return date.toISOString().substring(0, 10);
   }).reverse();
 
+  // ✅ FIX BUG #1: Pass through recordId and other options
   const data = await getAnalytics(resourceName, field, {
+    ...options,  // ✅ Include all options (recordId, etc.)
     period: 'day',
     startDate: dates[0],
     endDate: dates[dates.length - 1]
@@ -942,7 +1100,9 @@ export async function getLastNHours(resourceName, field, hours = 24, options, fi
   const startHour = hoursAgo.toISOString().substring(0, 13); // YYYY-MM-DDTHH
   const endHour = now.toISOString().substring(0, 13);
 
+  // ✅ FIX BUG #1: Pass through recordId and other options
   const data = await getAnalytics(resourceName, field, {
+    ...options,  // ✅ Include all options (recordId, etc.)
     period: 'hour',
     startDate: startHour,
     endDate: endHour
@@ -1023,7 +1183,9 @@ export async function getLastNMonths(resourceName, field, months = 12, options,
   const startDate = monthsAgo.toISOString().substring(0, 7); // YYYY-MM
   const endDate = now.toISOString().substring(0, 7);
 
+  // ✅ FIX BUG #1: Pass through recordId and other options
   const data = await getAnalytics(resourceName, field, {
+    ...options,  // ✅ Include all options (recordId, etc.)
     period: 'month',
     startDate,
     endDate

package/src/plugins/eventual-consistency/config.js

@@ -47,9 +47,12 @@ export function createConfig(options, detectedTimezone) {
     autoConsolidate: consolidation.auto !== false,
     mode: consolidation.mode || 'async',
 
-    // ✅ NOVO: Performance tuning - Mark applied concurrency (default 50, antes era 10 hardcoded)
+    // ✅ Performance tuning - Mark applied concurrency (default 50, up from 10)
     markAppliedConcurrency: consolidation.markAppliedConcurrency ?? 50,
 
+    // ✅ Performance tuning - Recalculate concurrency (default 50, up from 10)
+    recalculateConcurrency: consolidation.recalculateConcurrency ?? 50,
+
     // Late arrivals
     lateArrivalStrategy: lateArrivals.strategy || 'warn',
 

package/src/plugins/eventual-consistency/consolidation.js

@@ -6,7 +6,7 @@
 import tryFn from "../../concerns/try-fn.js";
 import { PromisePool } from "@supercharge/promise-pool";
 import { idGenerator } from "../../concerns/id.js";
-import { getCohortInfo, createSyntheticSetTransaction } from "./utils.js";
+import { getCohortInfo, createSyntheticSetTransaction, ensureCohortHour } from "./utils.js";
 
 /**
  * Start consolidation timer for a handler
@@ -612,12 +612,43 @@ export async function consolidateRecord(
         .for(transactionsToUpdate)
         .withConcurrency(markAppliedConcurrency) // ✅ Configurável e maior!
         .process(async (txn) => {
+          // ✅ FIX BUG #3: Ensure cohort fields exist before marking as applied
+          // This handles legacy transactions missing cohortHour, cohortDate, etc.
+          const txnWithCohorts = ensureCohortHour(txn, config.cohort.timezone, false);
+
+          // Build update data with applied flag
+          const updateData = { applied: true };
+
+          // Add missing cohort fields if they were calculated
+          if (txnWithCohorts.cohortHour && !txn.cohortHour) {
+            updateData.cohortHour = txnWithCohorts.cohortHour;
+          }
+          if (txnWithCohorts.cohortDate && !txn.cohortDate) {
+            updateData.cohortDate = txnWithCohorts.cohortDate;
+          }
+          if (txnWithCohorts.cohortWeek && !txn.cohortWeek) {
+            updateData.cohortWeek = txnWithCohorts.cohortWeek;
+          }
+          if (txnWithCohorts.cohortMonth && !txn.cohortMonth) {
+            updateData.cohortMonth = txnWithCohorts.cohortMonth;
+          }
+
+          // Handle null value field (legacy data might have null)
+          if (txn.value === null || txn.value === undefined) {
+            updateData.value = 1; // Default to 1 for backward compatibility
+          }
+
           const [ok, err] = await tryFn(() =>
-            transactionResource.update(txn.id, { applied: true })
+            transactionResource.update(txn.id, updateData)
           );
 
           if (!ok && config.verbose) {
-            console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err?.message);
+            console.warn(
+              `[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`,
+              err?.message,
+              'Update data:',
+              updateData
+            );
           }
 
           return ok;
@@ -917,9 +948,12 @@ export async function recalculateRecord(
     // Exclude anchor transactions (they should always be applied)
     const transactionsToReset = allTransactions.filter(txn => txn.source !== 'anchor');
 
+    // ✅ OPTIMIZATION: Use higher concurrency for recalculate (default 50 vs 10)
+    const recalculateConcurrency = config.recalculateConcurrency || 50;
+
     const { results, errors } = await PromisePool
       .for(transactionsToReset)
-      .withConcurrency(10)
+      .withConcurrency(recalculateConcurrency)
       .process(async (txn) => {
         const [ok, err] = await tryFn(() =>
           transactionResource.update(txn.id, { applied: false })

package/src/plugins/eventual-consistency/install.js

@@ -100,7 +100,7 @@ export async function completeFieldSetup(handler, database, config, plugin) {
         operation: 'string|required',
         timestamp: 'string|required',
         cohortDate: 'string|required',
-        cohortHour: 'string|required',
+        cohortHour: 'string|optional',  // ✅ FIX BUG #2: Changed from required to optional for migration compatibility
         cohortWeek: 'string|optional',
         cohortMonth: 'string|optional',
         source: 'string|optional',
@@ -173,6 +173,28 @@ async function createAnalyticsResource(handler, database, resourceName, fieldNam
       },
       behavior: 'body-overflow',
       timestamps: false,
+      asyncPartitions: true,
+      // ✅ Multi-attribute partitions for optimal analytics query performance
+      partitions: {
+        // Query by period (hour/day/week/month)
+        byPeriod: {
+          fields: { period: 'string' }
+        },
+        // Query by period + cohort (e.g., all hour records for specific hours)
+        byPeriodCohort: {
+          fields: {
+            period: 'string',
+            cohort: 'string'
+          }
+        },
+        // Query by field + period (e.g., all daily analytics for clicks field)
+        byFieldPeriod: {
+          fields: {
+            field: 'string',
+            period: 'string'
+          }
+        }
+      },
       createdBy: 'EventualConsistencyPlugin'
     })
   );

package/src/plugins/eventual-consistency/utils.js

@@ -355,3 +355,67 @@ export function groupByCohort(transactions, cohortField) {
   }
   return groups;
 }
+
+/**
+ * Ensure transaction has cohortHour field
+ * ✅ FIX BUG #2: Calculate cohortHour from timestamp if missing
+ *
+ * @param {Object} transaction - Transaction to check/fix
+ * @param {string} timezone - Timezone to use for cohort calculation
+ * @param {boolean} verbose - Whether to log warnings
+ * @returns {Object} Transaction with cohortHour populated
+ */
+export function ensureCohortHour(transaction, timezone = 'UTC', verbose = false) {
+  // If cohortHour already exists, return as-is
+  if (transaction.cohortHour) {
+    return transaction;
+  }
+
+  // Calculate cohortHour from timestamp
+  if (transaction.timestamp) {
+    const date = new Date(transaction.timestamp);
+    const cohortInfo = getCohortInfo(date, timezone, verbose);
+
+    if (verbose) {
+      console.log(
+        `[EventualConsistency] Transaction ${transaction.id} missing cohortHour, ` +
+        `calculated from timestamp: ${cohortInfo.hour}`
+      );
+    }
+
+    // Add cohortHour (and other cohort fields if missing)
+    transaction.cohortHour = cohortInfo.hour;
+
+    if (!transaction.cohortWeek) {
+      transaction.cohortWeek = cohortInfo.week;
+    }
+
+    if (!transaction.cohortMonth) {
+      transaction.cohortMonth = cohortInfo.month;
+    }
+  } else if (verbose) {
+    console.warn(
+      `[EventualConsistency] Transaction ${transaction.id} missing both cohortHour and timestamp, ` +
+      `cannot calculate cohort`
+    );
+  }
+
+  return transaction;
+}
+
+/**
+ * Ensure all transactions in array have cohortHour
+ * ✅ FIX BUG #2: Batch version of ensureCohortHour
+ *
+ * @param {Array} transactions - Transactions to check/fix
+ * @param {string} timezone - Timezone to use for cohort calculation
+ * @param {boolean} verbose - Whether to log warnings
+ * @returns {Array} Transactions with cohortHour populated
+ */
+export function ensureCohortHours(transactions, timezone = 'UTC', verbose = false) {
+  if (!transactions || !Array.isArray(transactions)) {
+    return transactions;
+  }
+
+  return transactions.map(txn => ensureCohortHour(txn, timezone, verbose));
+}