s3db.js 10.0.16 → 10.0.17

package/src/plugins/eventual-consistency/garbage-collection.js

@@ -0,0 +1,126 @@
+/**
+ * Garbage collection for EventualConsistencyPlugin
+ * @module eventual-consistency/garbage-collection
+ */
+
+import tryFn from "../../concerns/try-fn.js";
+import { PromisePool } from "@supercharge/promise-pool";
+
+/**
+ * Start garbage collection timer for a handler
+ *
+ * @param {Object} handler - Field handler
+ * @param {string} resourceName - Resource name
+ * @param {string} fieldName - Field name
+ * @param {Function} runGCCallback - Callback to run GC
+ * @param {Object} config - Plugin configuration
+ * @returns {NodeJS.Timeout} GC timer
+ */
+export function startGarbageCollectionTimer(handler, resourceName, fieldName, runGCCallback, config) {
+  const gcIntervalMs = config.gcInterval * 1000; // Convert seconds to ms
+
+  handler.gcTimer = setInterval(async () => {
+    await runGCCallback(handler, resourceName, fieldName);
+  }, gcIntervalMs);
+
+  return handler.gcTimer;
+}
+
+/**
+ * Delete old applied transactions based on retention policy
+ * Uses distributed locking to prevent multiple containers from running GC simultaneously
+ *
+ * @param {Object} transactionResource - Transaction resource
+ * @param {Object} lockResource - Lock resource
+ * @param {Object} config - Plugin configuration
+ * @param {Function} emitFn - Function to emit events
+ * @returns {Promise<void>}
+ */
+export async function runGarbageCollection(transactionResource, lockResource, config, emitFn) {
+  // Acquire distributed lock for GC operation
+  const gcLockId = `lock-gc-${config.resource}-${config.field}`;
+  const [lockAcquired] = await tryFn(() =>
+    lockResource.insert({
+      id: gcLockId,
+      lockedAt: Date.now(),
+      workerId: process.pid ? String(process.pid) : 'unknown'
+    })
+  );
+
+  // If another container is already running GC, skip
+  if (!lockAcquired) {
+    if (config.verbose) {
+      console.log(`[EventualConsistency] GC already running in another container`);
+    }
+    return;
+  }
+
+  try {
+    const now = Date.now();
+    const retentionMs = config.transactionRetention * 24 * 60 * 60 * 1000; // Days to ms
+    const cutoffDate = new Date(now - retentionMs);
+    const cutoffIso = cutoffDate.toISOString();
+
+    if (config.verbose) {
+      console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${config.transactionRetention} days)`);
+    }
+
+    // Query old applied transactions
+    const [ok, err, oldTransactions] = await tryFn(() =>
+      transactionResource.query({
+        applied: true,
+        timestamp: { '<': cutoffIso }
+      })
+    );
+
+    if (!ok) {
+      if (config.verbose) {
+        console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
+      }
+      return;
+    }
+
+    if (!oldTransactions || oldTransactions.length === 0) {
+      if (config.verbose) {
+        console.log(`[EventualConsistency] No old transactions to clean up`);
+      }
+      return;
+    }
+
+    if (config.verbose) {
+      console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
+    }
+
+    // Delete old transactions using PromisePool
+    const { results, errors } = await PromisePool
+      .for(oldTransactions)
+      .withConcurrency(10)
+      .process(async (txn) => {
+        const [deleted] = await tryFn(() => transactionResource.delete(txn.id));
+        return deleted;
+      });
+
+    if (config.verbose) {
+      console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
+    }
+
+    if (emitFn) {
+      emitFn('eventual-consistency.gc-completed', {
+        resource: config.resource,
+        field: config.field,
+        deletedCount: results.length,
+        errorCount: errors.length
+      });
+    }
+  } catch (error) {
+    if (config.verbose) {
+      console.warn(`[EventualConsistency] GC error:`, error.message);
+    }
+    if (emitFn) {
+      emitFn('eventual-consistency.gc-error', error);
+    }
+  } finally {
+    // Always release GC lock
+    await tryFn(() => lockResource.delete(gcLockId));
+  }
+}

package/src/plugins/eventual-consistency/helpers.js

@@ -0,0 +1,179 @@
+/**
+ * Helper methods added to resources for EventualConsistencyPlugin
+ * @module eventual-consistency/helpers
+ */
+
+import { idGenerator } from "../../concerns/id.js";
+import tryFn from "../../concerns/try-fn.js";
+import { getCohortInfo, resolveFieldAndPlugin } from "./utils.js";
+
+/**
+ * Add helper methods to resources
+ * This adds: set(), add(), sub(), consolidate(), getConsolidatedValue(), recalculate()
+ *
+ * @param {Object} resource - Resource to add methods to
+ * @param {Object} plugin - Plugin instance
+ * @param {Object} config - Plugin configuration
+ */
+export function addHelperMethods(resource, plugin, config) {
+  // Add method to set value (replaces current value)
+  // Signature: set(id, field, value)
+  resource.set = async (id, field, value) => {
+    const { plugin: handler } = resolveFieldAndPlugin(resource, field, value);
+
+    // Create transaction inline
+    const now = new Date();
+    const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
+
+    const transaction = {
+      id: idGenerator(),
+      originalId: id,
+      field: handler.field,
+      value: value,
+      operation: 'set',
+      timestamp: now.toISOString(),
+      cohortDate: cohortInfo.date,
+      cohortHour: cohortInfo.hour,
+      cohortMonth: cohortInfo.month,
+      source: 'set',
+      applied: false
+    };
+
+    await handler.transactionResource.insert(transaction);
+
+    // In sync mode, immediately consolidate
+    if (config.mode === 'sync') {
+      return await plugin._syncModeConsolidate(handler, id, field);
+    }
+
+    return value;
+  };
+
+  // Add method to increment value
+  // Signature: add(id, field, amount)
+  resource.add = async (id, field, amount) => {
+    const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
+
+    // Create transaction inline
+    const now = new Date();
+    const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
+
+    const transaction = {
+      id: idGenerator(),
+      originalId: id,
+      field: handler.field,
+      value: amount,
+      operation: 'add',
+      timestamp: now.toISOString(),
+      cohortDate: cohortInfo.date,
+      cohortHour: cohortInfo.hour,
+      cohortMonth: cohortInfo.month,
+      source: 'add',
+      applied: false
+    };
+
+    await handler.transactionResource.insert(transaction);
+
+    // In sync mode, immediately consolidate
+    if (config.mode === 'sync') {
+      return await plugin._syncModeConsolidate(handler, id, field);
+    }
+
+    // Async mode - return current value (optimistic)
+    const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
+    const currentValue = (ok && record) ? (record[field] || 0) : 0;
+    return currentValue + amount;
+  };
+
+  // Add method to decrement value
+  // Signature: sub(id, field, amount)
+  resource.sub = async (id, field, amount) => {
+    const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
+
+    // Create transaction inline
+    const now = new Date();
+    const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
+
+    const transaction = {
+      id: idGenerator(),
+      originalId: id,
+      field: handler.field,
+      value: amount,
+      operation: 'sub',
+      timestamp: now.toISOString(),
+      cohortDate: cohortInfo.date,
+      cohortHour: cohortInfo.hour,
+      cohortMonth: cohortInfo.month,
+      source: 'sub',
+      applied: false
+    };
+
+    await handler.transactionResource.insert(transaction);
+
+    // In sync mode, immediately consolidate
+    if (config.mode === 'sync') {
+      return await plugin._syncModeConsolidate(handler, id, field);
+    }
+
+    // Async mode - return current value (optimistic)
+    const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
+    const currentValue = (ok && record) ? (record[field] || 0) : 0;
+    return currentValue - amount;
+  };
+
+  // Add method to manually trigger consolidation
+  // Signature: consolidate(id, field)
+  resource.consolidate = async (id, field) => {
+    if (!field) {
+      throw new Error(`Field parameter is required: consolidate(id, field)`);
+    }
+
+    const handler = resource._eventualConsistencyPlugins[field];
+
+    if (!handler) {
+      const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
+      throw new Error(
+        `No eventual consistency plugin found for field "${field}". ` +
+        `Available fields: ${availableFields}`
+      );
+    }
+
+    return await plugin._consolidateWithHandler(handler, id);
+  };
+
+  // Add method to get consolidated value without applying
+  // Signature: getConsolidatedValue(id, field, options)
+  resource.getConsolidatedValue = async (id, field, options = {}) => {
+    const handler = resource._eventualConsistencyPlugins[field];
+
+    if (!handler) {
+      const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
+      throw new Error(
+        `No eventual consistency plugin found for field "${field}". ` +
+        `Available fields: ${availableFields}`
+      );
+    }
+
+    return await plugin._getConsolidatedValueWithHandler(handler, id, options);
+  };
+
+  // Add method to recalculate from scratch
+  // Signature: recalculate(id, field)
+  resource.recalculate = async (id, field) => {
+    if (!field) {
+      throw new Error(`Field parameter is required: recalculate(id, field)`);
+    }
+
+    const handler = resource._eventualConsistencyPlugins[field];
+
+    if (!handler) {
+      const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
+      throw new Error(
+        `No eventual consistency plugin found for field "${field}". ` +
+        `Available fields: ${availableFields}`
+      );
+    }
+
+    return await plugin._recalculateWithHandler(handler, id);
+  };
+}