s3db.js 10.0.16 → 10.0.17

package/src/plugins/eventual-consistency/setup.js

@@ -0,0 +1,298 @@
+/**
+ * Setup logic for EventualConsistencyPlugin
+ * @module eventual-consistency/setup
+ */
+
+import tryFn from "../../concerns/try-fn.js";
+import { createPartitionConfig } from "./partitions.js";
+import { addHelperMethods } from "./helpers.js";
+import { flushPendingTransactions } from "./transactions.js";
+import { startConsolidationTimer } from "./consolidation.js";
+import { startGarbageCollectionTimer } from "./garbage-collection.js";
+
+/**
+ * Setup plugin for all configured resources
+ *
+ * @param {Object} database - Database instance
+ * @param {Map} fieldHandlers - Field handlers map
+ * @param {Function} completeFieldSetupFn - Function to complete setup for a field
+ * @param {Function} watchForResourceFn - Function to watch for resource creation
+ */
+export async function onSetup(database, fieldHandlers, completeFieldSetupFn, watchForResourceFn) {
+  // Iterate over all resource/field combinations
+  for (const [resourceName, resourceHandlers] of fieldHandlers) {
+    const targetResource = database.resources[resourceName];
+
+    if (!targetResource) {
+      // Resource doesn't exist yet - mark for deferred setup
+      for (const handler of resourceHandlers.values()) {
+        handler.deferredSetup = true;
+      }
+      // Watch for this resource to be created
+      watchForResourceFn(resourceName);
+      continue;
+    }
+
+    // Resource exists - setup all fields for this resource
+    for (const [fieldName, handler] of resourceHandlers) {
+      handler.targetResource = targetResource;
+      await completeFieldSetupFn(handler);
+    }
+  }
+}
+
+/**
+ * Watch for a specific resource creation
+ *
+ * @param {string} resourceName - Resource name to watch for
+ * @param {Object} database - Database instance
+ * @param {Map} fieldHandlers - Field handlers map
+ * @param {Function} completeFieldSetupFn - Function to complete setup for a field
+ */
+export function watchForResource(resourceName, database, fieldHandlers, completeFieldSetupFn) {
+  const hookCallback = async ({ resource, config }) => {
+    if (config.name === resourceName) {
+      const resourceHandlers = fieldHandlers.get(resourceName);
+      if (!resourceHandlers) return;
+
+      // Setup all fields for this resource
+      for (const [fieldName, handler] of resourceHandlers) {
+        if (handler.deferredSetup) {
+          handler.targetResource = resource;
+          handler.deferredSetup = false;
+          await completeFieldSetupFn(handler);
+        }
+      }
+    }
+  };
+
+  database.addHook('afterCreateResource', hookCallback);
+}
+
+/**
+ * Complete setup for a single field handler
+ *
+ * @param {Object} handler - Field handler
+ * @param {Object} database - Database instance
+ * @param {Object} config - Plugin configuration
+ * @param {Object} plugin - Plugin instance (for adding helper methods)
+ * @returns {Promise<void>}
+ */
+export async function completeFieldSetup(handler, database, config, plugin) {
+  if (!handler.targetResource) return;
+
+  const resourceName = handler.resource;
+  const fieldName = handler.field;
+
+  // Create transaction resource with partitions
+  const transactionResourceName = `${resourceName}_transactions_${fieldName}`;
+  const partitionConfig = createPartitionConfig();
+
+  const [ok, err, transactionResource] = await tryFn(() =>
+    database.createResource({
+      name: transactionResourceName,
+      attributes: {
+        id: 'string|required',
+        originalId: 'string|required',
+        field: 'string|required',
+        value: 'number|required',
+        operation: 'string|required',
+        timestamp: 'string|required',
+        cohortDate: 'string|required',
+        cohortHour: 'string|required',
+        cohortMonth: 'string|optional',
+        source: 'string|optional',
+        applied: 'boolean|optional'
+      },
+      behavior: 'body-overflow',
+      timestamps: true,
+      partitions: partitionConfig,
+      asyncPartitions: true,
+      createdBy: 'EventualConsistencyPlugin'
+    })
+  );
+
+  if (!ok && !database.resources[transactionResourceName]) {
+    throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
+  }
+
+  handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
+
+  // Create lock resource
+  const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
+  const [lockOk, lockErr, lockResource] = await tryFn(() =>
+    database.createResource({
+      name: lockResourceName,
+      attributes: {
+        id: 'string|required',
+        lockedAt: 'number|required',
+        workerId: 'string|optional'
+      },
+      behavior: 'body-only',
+      timestamps: false,
+      createdBy: 'EventualConsistencyPlugin'
+    })
+  );
+
+  if (!lockOk && !database.resources[lockResourceName]) {
+    throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
+  }
+
+  handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
+
+  // Create analytics resource if enabled
+  if (config.enableAnalytics) {
+    await createAnalyticsResource(handler, database, resourceName, fieldName);
+  }
+
+  // Add helper methods to the target resource
+  addHelperMethodsForHandler(handler, plugin, config);
+
+  if (config.verbose) {
+    console.log(
+      `[EventualConsistency] ${resourceName}.${fieldName} - ` +
+      `Setup complete. Resources: ${transactionResourceName}, ${lockResourceName}` +
+      `${config.enableAnalytics ? `, ${resourceName}_analytics_${fieldName}` : ''}`
+    );
+  }
+}
+
+/**
+ * Create analytics resource for a field handler
+ *
+ * @param {Object} handler - Field handler
+ * @param {Object} database - Database instance
+ * @param {string} resourceName - Resource name
+ * @param {string} fieldName - Field name
+ * @returns {Promise<void>}
+ */
+async function createAnalyticsResource(handler, database, resourceName, fieldName) {
+  const analyticsResourceName = `${resourceName}_analytics_${fieldName}`;
+
+  const [ok, err, analyticsResource] = await tryFn(() =>
+    database.createResource({
+      name: analyticsResourceName,
+      attributes: {
+        id: 'string|required',
+        period: 'string|required',
+        cohort: 'string|required',
+        transactionCount: 'number|required',
+        totalValue: 'number|required',
+        avgValue: 'number|required',
+        minValue: 'number|required',
+        maxValue: 'number|required',
+        operations: 'object|optional',
+        recordCount: 'number|required',
+        consolidatedAt: 'string|required',
+        updatedAt: 'string|required'
+      },
+      behavior: 'body-overflow',
+      timestamps: false,
+      createdBy: 'EventualConsistencyPlugin'
+    })
+  );
+
+  if (!ok && !database.resources[analyticsResourceName]) {
+    throw new Error(`Failed to create analytics resource for ${resourceName}.${fieldName}: ${err?.message}`);
+  }
+
+  handler.analyticsResource = ok ? analyticsResource : database.resources[analyticsResourceName];
+}
+
+/**
+ * Add helper methods to the target resource for a field handler
+ *
+ * @param {Object} handler - Field handler
+ * @param {Object} plugin - Plugin instance
+ * @param {Object} config - Plugin configuration
+ */
+function addHelperMethodsForHandler(handler, plugin, config) {
+  const resource = handler.targetResource;
+  const fieldName = handler.field;
+
+  // Store handler reference on the resource for later access
+  if (!resource._eventualConsistencyPlugins) {
+    resource._eventualConsistencyPlugins = {};
+  }
+  resource._eventualConsistencyPlugins[fieldName] = handler;
+
+  // Add helper methods if not already added
+  if (!resource.add) {
+    addHelperMethods(resource, plugin, config);
+  }
+}
+
+/**
+ * Start timers and emit events for all field handlers
+ *
+ * @param {Map} fieldHandlers - Field handlers map
+ * @param {Object} config - Plugin configuration
+ * @param {Function} runConsolidationFn - Function to run consolidation for a handler
+ * @param {Function} runGCFn - Function to run GC for a handler
+ * @param {Function} emitFn - Function to emit events
+ * @returns {Promise<void>}
+ */
+export async function onStart(fieldHandlers, config, runConsolidationFn, runGCFn, emitFn) {
+  // Start timers and emit events for all field handlers
+  for (const [resourceName, resourceHandlers] of fieldHandlers) {
+    for (const [fieldName, handler] of resourceHandlers) {
+      if (!handler.deferredSetup) {
+        // Start auto-consolidation timer if enabled
+        if (config.autoConsolidate && config.mode === 'async') {
+          startConsolidationTimer(handler, resourceName, fieldName, runConsolidationFn, config);
+        }
+
+        // Start garbage collection timer
+        if (config.transactionRetention && config.transactionRetention > 0) {
+          startGarbageCollectionTimer(handler, resourceName, fieldName, runGCFn, config);
+        }
+
+        if (emitFn) {
+          emitFn('eventual-consistency.started', {
+            resource: resourceName,
+            field: fieldName,
+            cohort: config.cohort
+          });
+        }
+      }
+    }
+  }
+}
+
+/**
+ * Stop all timers and flush pending transactions
+ *
+ * @param {Map} fieldHandlers - Field handlers map
+ * @param {Function} emitFn - Function to emit events
+ * @returns {Promise<void>}
+ */
+export async function onStop(fieldHandlers, emitFn) {
+  // Stop all timers for all handlers
+  for (const [resourceName, resourceHandlers] of fieldHandlers) {
+    for (const [fieldName, handler] of resourceHandlers) {
+      // Stop consolidation timer
+      if (handler.consolidationTimer) {
+        clearInterval(handler.consolidationTimer);
+        handler.consolidationTimer = null;
+      }
+
+      // Stop garbage collection timer
+      if (handler.gcTimer) {
+        clearInterval(handler.gcTimer);
+        handler.gcTimer = null;
+      }
+
+      // Flush pending transactions
+      if (handler.pendingTransactions && handler.pendingTransactions.size > 0) {
+        await flushPendingTransactions(handler);
+      }
+
+      if (emitFn) {
+        emitFn('eventual-consistency.stopped', {
+          resource: resourceName,
+          field: fieldName
+        });
+      }
+    }
+  }
+}

package/src/plugins/eventual-consistency/transactions.js

@@ -0,0 +1,119 @@
+/**
+ * Transaction management for EventualConsistencyPlugin
+ * @module eventual-consistency/transactions
+ */
+
+import { idGenerator } from "../../concerns/id.js";
+import { getCohortInfo } from "./utils.js";
+
+/**
+ * Create a transaction for a field handler
+ *
+ * @param {Object} handler - Field handler
+ * @param {Object} data - Transaction data
+ * @param {Object} config - Plugin configuration
+ * @returns {Promise<Object|null>} Created transaction or null if ignored
+ */
+export async function createTransaction(handler, data, config) {
+  const now = new Date();
+  const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
+
+  // Check for late arrivals (transaction older than watermark)
+  const watermarkMs = config.consolidationWindow * 60 * 60 * 1000;
+  const watermarkTime = now.getTime() - watermarkMs;
+  const cohortHourDate = new Date(cohortInfo.hour + ':00:00Z');
+
+  if (cohortHourDate.getTime() < watermarkTime) {
+    // Late arrival detected!
+    const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1000));
+
+    if (config.lateArrivalStrategy === 'ignore') {
+      if (config.verbose) {
+        console.warn(
+          `[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} ` +
+          `is ${hoursLate}h late (watermark: ${config.consolidationWindow}h)`
+        );
+      }
+      return null;
+    } else if (config.lateArrivalStrategy === 'warn') {
+      console.warn(
+        `[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} ` +
+        `is ${hoursLate}h late (watermark: ${config.consolidationWindow}h). ` +
+        `Processing anyway, but consolidation may not pick it up.`
+      );
+    }
+    // 'process' strategy: continue normally
+  }
+
+  const transaction = {
+    id: idGenerator(),
+    originalId: data.originalId,
+    field: handler.field,
+    value: data.value || 0,
+    operation: data.operation || 'set',
+    timestamp: now.toISOString(),
+    cohortDate: cohortInfo.date,
+    cohortHour: cohortInfo.hour,
+    cohortMonth: cohortInfo.month,
+    source: data.source || 'unknown',
+    applied: false
+  };
+
+  // Batch transactions if configured
+  if (config.batchTransactions) {
+    handler.pendingTransactions.set(transaction.id, transaction);
+
+    if (config.verbose) {
+      console.log(
+        `[EventualConsistency] ${handler.resource}.${handler.field} - ` +
+        `Transaction batched: ${data.operation} ${data.value} for ${data.originalId} ` +
+        `(batch: ${handler.pendingTransactions.size}/${config.batchSize})`
+      );
+    }
+
+    // Flush if batch size reached
+    if (handler.pendingTransactions.size >= config.batchSize) {
+      await flushPendingTransactions(handler);
+    }
+  } else {
+    await handler.transactionResource.insert(transaction);
+
+    if (config.verbose) {
+      console.log(
+        `[EventualConsistency] ${handler.resource}.${handler.field} - ` +
+        `Transaction created: ${data.operation} ${data.value} for ${data.originalId} ` +
+        `(cohort: ${cohortInfo.hour}, applied: false)`
+      );
+    }
+  }
+
+  return transaction;
+}
+
+/**
+ * Flush pending transactions for a handler
+ *
+ * @param {Object} handler - Field handler with pending transactions
+ * @throws {Error} If flush fails
+ */
+export async function flushPendingTransactions(handler) {
+  if (handler.pendingTransactions.size === 0) return;
+
+  const transactions = Array.from(handler.pendingTransactions.values());
+
+  try {
+    // Insert all pending transactions in parallel
+    await Promise.all(
+      transactions.map(transaction =>
+        handler.transactionResource.insert(transaction)
+      )
+    );
+
+    // Only clear after successful inserts (prevents data loss on crashes)
+    handler.pendingTransactions.clear();
+  } catch (error) {
+    // Keep pending transactions for retry on next flush
+    console.error('Failed to flush pending transactions:', error);
+    throw error;
+  }
+}

package/src/plugins/eventual-consistency/utils.js

@@ -0,0 +1,182 @@
+/**
+ * Utility functions for EventualConsistencyPlugin
+ * @module eventual-consistency/utils
+ */
+
+/**
+ * Auto-detect timezone from environment or system
+ * @returns {string} Detected timezone (defaults to 'UTC')
+ */
+export function detectTimezone() {
+  // 1. Try TZ environment variable (common in Docker/K8s)
+  if (process.env.TZ) {
+    return process.env.TZ;
+  }
+
+  // 2. Try Intl API (works in Node.js and browsers)
+  try {
+    const systemTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+    if (systemTimezone) {
+      return systemTimezone;
+    }
+  } catch (err) {
+    // Intl API not available or failed
+  }
+
+  // 3. Fallback to UTC
+  return 'UTC';
+}
+
+/**
+ * Get timezone offset in milliseconds
+ * @param {string} timezone - IANA timezone name
+ * @param {boolean} verbose - Whether to log warnings
+ * @returns {number} Offset in milliseconds
+ */
+export function getTimezoneOffset(timezone, verbose = false) {
+  // Try to calculate offset using Intl API (handles DST automatically)
+  try {
+    const now = new Date();
+
+    // Get UTC time
+    const utcDate = new Date(now.toLocaleString('en-US', { timeZone: 'UTC' }));
+
+    // Get time in target timezone
+    const tzDate = new Date(now.toLocaleString('en-US', { timeZone: timezone }));
+
+    // Calculate offset in milliseconds
+    return tzDate.getTime() - utcDate.getTime();
+  } catch (err) {
+    // Intl API failed, fallback to manual offsets (without DST support)
+    const offsets = {
+      'UTC': 0,
+      'America/New_York': -5 * 3600000,
+      'America/Chicago': -6 * 3600000,
+      'America/Denver': -7 * 3600000,
+      'America/Los_Angeles': -8 * 3600000,
+      'America/Sao_Paulo': -3 * 3600000,
+      'Europe/London': 0,
+      'Europe/Paris': 1 * 3600000,
+      'Europe/Berlin': 1 * 3600000,
+      'Asia/Tokyo': 9 * 3600000,
+      'Asia/Shanghai': 8 * 3600000,
+      'Australia/Sydney': 10 * 3600000
+    };
+
+    if (verbose && !offsets[timezone]) {
+      console.warn(
+        `[EventualConsistency] Unknown timezone '${timezone}', using UTC. ` +
+        `Consider using a valid IANA timezone (e.g., 'America/New_York')`
+      );
+    }
+
+    return offsets[timezone] || 0;
+  }
+}
+
+/**
+ * Get cohort information for a date
+ * @param {Date} date - Date to get cohort info for
+ * @param {string} timezone - IANA timezone name
+ * @param {boolean} verbose - Whether to log warnings
+ * @returns {Object} Cohort information (date, hour, month)
+ */
+export function getCohortInfo(date, timezone, verbose = false) {
+  // Simple timezone offset calculation
+  const offset = getTimezoneOffset(timezone, verbose);
+  const localDate = new Date(date.getTime() + offset);
+
+  const year = localDate.getFullYear();
+  const month = String(localDate.getMonth() + 1).padStart(2, '0');
+  const day = String(localDate.getDate()).padStart(2, '0');
+  const hour = String(localDate.getHours()).padStart(2, '0');
+
+  return {
+    date: `${year}-${month}-${day}`,
+    hour: `${year}-${month}-${day}T${hour}`, // ISO-like format for hour partition
+    month: `${year}-${month}`
+  };
+}
+
+/**
+ * Create synthetic 'set' transaction from current value
+ * @param {number} currentValue - Current value to create transaction for
+ * @returns {Object} Synthetic transaction object
+ */
+export function createSyntheticSetTransaction(currentValue) {
+  return {
+    id: '__synthetic__',
+    operation: 'set',
+    value: currentValue,
+    timestamp: new Date(0).toISOString(),
+    synthetic: true
+  };
+}
+
+/**
+ * Create a field handler for a specific resource/field combination
+ * @param {string} resourceName - Resource name
+ * @param {string} fieldName - Field name
+ * @returns {Object} Field handler object
+ */
+export function createFieldHandler(resourceName, fieldName) {
+  return {
+    resource: resourceName,
+    field: fieldName,
+    transactionResource: null,
+    targetResource: null,
+    analyticsResource: null,
+    lockResource: null,
+    checkpointResource: null,
+    consolidationTimer: null,
+    gcTimer: null,
+    pendingTransactions: new Map(),
+    deferredSetup: false
+  };
+}
+
+/**
+ * Resolve field and plugin from arguments
+ * @param {Object} resource - Resource object
+ * @param {string} field - Field name
+ * @param {*} value - Value (for error reporting)
+ * @returns {Object} Resolved field and plugin handler
+ * @throws {Error} If field or plugin not found
+ */
+export function resolveFieldAndPlugin(resource, field, value) {
+  if (!resource._eventualConsistencyPlugins) {
+    throw new Error(`No eventual consistency plugins configured for this resource`);
+  }
+
+  const fieldPlugin = resource._eventualConsistencyPlugins[field];
+
+  if (!fieldPlugin) {
+    const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
+    throw new Error(
+      `No eventual consistency plugin found for field "${field}". ` +
+      `Available fields: ${availableFields}`
+    );
+  }
+
+  return { field, value, plugin: fieldPlugin };
+}
+
+/**
+ * Group transactions by cohort field
+ * @param {Array} transactions - Transactions to group
+ * @param {string} cohortField - Field to group by (e.g., 'cohortHour')
+ * @returns {Object} Grouped transactions
+ */
+export function groupByCohort(transactions, cohortField) {
+  const groups = {};
+  for (const txn of transactions) {
+    const cohort = txn[cohortField];
+    if (!cohort) continue;
+
+    if (!groups[cohort]) {
+      groups[cohort] = [];
+    }
+    groups[cohort].push(txn);
+  }
+  return groups;
+}