s3db.js 10.0.12 → 10.0.14

package/dist/s3db.cjs.js

@@ -4549,6 +4549,12 @@ class EventualConsistencyPlugin extends Plugin {
     for (const [resourceName, fieldHandlers] of this.fieldHandlers) {
       for (const [fieldName, handler] of fieldHandlers) {
         if (!handler.deferredSetup) {
+          if (this.config.autoConsolidate && this.config.mode === "async") {
+            this.startConsolidationTimerForHandler(handler, resourceName, fieldName);
+          }
+          if (this.config.transactionRetention && this.config.transactionRetention > 0) {
+            this.startGarbageCollectionTimerForHandler(handler, resourceName, fieldName);
+          }
           this.emit("eventual-consistency.started", {
             resource: resourceName,
             field: fieldName,
@@ -4662,68 +4668,189 @@ class EventualConsistencyPlugin extends Plugin {
     const resource = firstHandler.targetResource;
     const plugin = this;
     resource.set = async (id, field, value) => {
-      const { plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, field, value);
-      await fieldPlugin.createTransaction(fieldPlugin, {
+      const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, value);
+      const now = /* @__PURE__ */ new Date();
+      const cohortInfo = plugin.getCohortInfo(now);
+      const transaction = {
+        id: idGenerator(),
         originalId: id,
-        operation: "set",
+        field: handler.field,
         value,
-        source: "set"
-      });
-      if (fieldPlugin.config.mode === "sync") {
-        return await fieldPlugin._syncModeConsolidate(fieldPlugin, id, field);
+        operation: "set",
+        timestamp: now.toISOString(),
+        cohortDate: cohortInfo.date,
+        cohortHour: cohortInfo.hour,
+        cohortMonth: cohortInfo.month,
+        source: "set",
+        applied: false
+      };
+      await handler.transactionResource.insert(transaction);
+      if (plugin.config.mode === "sync") {
+        const oldResource = plugin.config.resource;
+        const oldField = plugin.config.field;
+        const oldTransactionResource = plugin.transactionResource;
+        const oldTargetResource = plugin.targetResource;
+        const oldLockResource = plugin.lockResource;
+        const oldAnalyticsResource = plugin.analyticsResource;
+        plugin.config.resource = handler.resource;
+        plugin.config.field = handler.field;
+        plugin.transactionResource = handler.transactionResource;
+        plugin.targetResource = handler.targetResource;
+        plugin.lockResource = handler.lockResource;
+        plugin.analyticsResource = handler.analyticsResource;
+        const result = await plugin._syncModeConsolidate(id, field);
+        plugin.config.resource = oldResource;
+        plugin.config.field = oldField;
+        plugin.transactionResource = oldTransactionResource;
+        plugin.targetResource = oldTargetResource;
+        plugin.lockResource = oldLockResource;
+        plugin.analyticsResource = oldAnalyticsResource;
+        return result;
       }
       return value;
     };
     resource.add = async (id, field, amount) => {
-      const { plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, field, amount);
-      await fieldPlugin.createTransaction(fieldPlugin, {
+      const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, amount);
+      const now = /* @__PURE__ */ new Date();
+      const cohortInfo = plugin.getCohortInfo(now);
+      const transaction = {
+        id: idGenerator(),
         originalId: id,
-        operation: "add",
+        field: handler.field,
         value: amount,
-        source: "add"
-      });
-      if (fieldPlugin.config.mode === "sync") {
-        return await fieldPlugin._syncModeConsolidate(fieldPlugin, id, field);
+        operation: "add",
+        timestamp: now.toISOString(),
+        cohortDate: cohortInfo.date,
+        cohortHour: cohortInfo.hour,
+        cohortMonth: cohortInfo.month,
+        source: "add",
+        applied: false
+      };
+      await handler.transactionResource.insert(transaction);
+      if (plugin.config.mode === "sync") {
+        const oldResource = plugin.config.resource;
+        const oldField = plugin.config.field;
+        const oldTransactionResource = plugin.transactionResource;
+        const oldTargetResource = plugin.targetResource;
+        const oldLockResource = plugin.lockResource;
+        const oldAnalyticsResource = plugin.analyticsResource;
+        plugin.config.resource = handler.resource;
+        plugin.config.field = handler.field;
+        plugin.transactionResource = handler.transactionResource;
+        plugin.targetResource = handler.targetResource;
+        plugin.lockResource = handler.lockResource;
+        plugin.analyticsResource = handler.analyticsResource;
+        const result = await plugin._syncModeConsolidate(id, field);
+        plugin.config.resource = oldResource;
+        plugin.config.field = oldField;
+        plugin.transactionResource = oldTransactionResource;
+        plugin.targetResource = oldTargetResource;
+        plugin.lockResource = oldLockResource;
+        plugin.analyticsResource = oldAnalyticsResource;
+        return result;
       }
-      const currentValue = await fieldPlugin.getConsolidatedValue(fieldPlugin, id);
+      const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
+      const currentValue = ok && record ? record[field] || 0 : 0;
       return currentValue + amount;
     };
     resource.sub = async (id, field, amount) => {
-      const { plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, field, amount);
-      await fieldPlugin.createTransaction(fieldPlugin, {
+      const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, amount);
+      const now = /* @__PURE__ */ new Date();
+      const cohortInfo = plugin.getCohortInfo(now);
+      const transaction = {
+        id: idGenerator(),
         originalId: id,
-        operation: "sub",
+        field: handler.field,
         value: amount,
-        source: "sub"
-      });
-      if (fieldPlugin.config.mode === "sync") {
-        return await fieldPlugin._syncModeConsolidate(fieldPlugin, id, field);
+        operation: "sub",
+        timestamp: now.toISOString(),
+        cohortDate: cohortInfo.date,
+        cohortHour: cohortInfo.hour,
+        cohortMonth: cohortInfo.month,
+        source: "sub",
+        applied: false
+      };
+      await handler.transactionResource.insert(transaction);
+      if (plugin.config.mode === "sync") {
+        const oldResource = plugin.config.resource;
+        const oldField = plugin.config.field;
+        const oldTransactionResource = plugin.transactionResource;
+        const oldTargetResource = plugin.targetResource;
+        const oldLockResource = plugin.lockResource;
+        const oldAnalyticsResource = plugin.analyticsResource;
+        plugin.config.resource = handler.resource;
+        plugin.config.field = handler.field;
+        plugin.transactionResource = handler.transactionResource;
+        plugin.targetResource = handler.targetResource;
+        plugin.lockResource = handler.lockResource;
+        plugin.analyticsResource = handler.analyticsResource;
+        const result = await plugin._syncModeConsolidate(id, field);
+        plugin.config.resource = oldResource;
+        plugin.config.field = oldField;
+        plugin.transactionResource = oldTransactionResource;
+        plugin.targetResource = oldTargetResource;
+        plugin.lockResource = oldLockResource;
+        plugin.analyticsResource = oldAnalyticsResource;
+        return result;
       }
-      const currentValue = await fieldPlugin.getConsolidatedValue(fieldPlugin, id);
+      const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
+      const currentValue = ok && record ? record[field] || 0 : 0;
       return currentValue - amount;
     };
     resource.consolidate = async (id, field) => {
       if (!field) {
         throw new Error(`Field parameter is required: consolidate(id, field)`);
       }
-      const fieldPlugin = resource._eventualConsistencyPlugins[field];
-      if (!fieldPlugin) {
+      const handler = resource._eventualConsistencyPlugins[field];
+      if (!handler) {
         const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
         throw new Error(
           `No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
         );
       }
-      return await fieldPlugin.consolidateRecord(fieldPlugin, id);
+      const oldResource = plugin.config.resource;
+      const oldField = plugin.config.field;
+      const oldTransactionResource = plugin.transactionResource;
+      const oldTargetResource = plugin.targetResource;
+      const oldLockResource = plugin.lockResource;
+      const oldAnalyticsResource = plugin.analyticsResource;
+      plugin.config.resource = handler.resource;
+      plugin.config.field = handler.field;
+      plugin.transactionResource = handler.transactionResource;
+      plugin.targetResource = handler.targetResource;
+      plugin.lockResource = handler.lockResource;
+      plugin.analyticsResource = handler.analyticsResource;
+      const result = await plugin.consolidateRecord(id);
+      plugin.config.resource = oldResource;
+      plugin.config.field = oldField;
+      plugin.transactionResource = oldTransactionResource;
+      plugin.targetResource = oldTargetResource;
+      plugin.lockResource = oldLockResource;
+      plugin.analyticsResource = oldAnalyticsResource;
+      return result;
     };
     resource.getConsolidatedValue = async (id, field, options = {}) => {
-      const fieldPlugin = resource._eventualConsistencyPlugins[field];
-      if (!fieldPlugin) {
+      const handler = resource._eventualConsistencyPlugins[field];
+      if (!handler) {
         const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
         throw new Error(
           `No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
         );
       }
-      return await fieldPlugin.getConsolidatedValue(fieldPlugin, id, options);
+      const oldResource = plugin.config.resource;
+      const oldField = plugin.config.field;
+      const oldTransactionResource = plugin.transactionResource;
+      const oldTargetResource = plugin.targetResource;
+      plugin.config.resource = handler.resource;
+      plugin.config.field = handler.field;
+      plugin.transactionResource = handler.transactionResource;
+      plugin.targetResource = handler.targetResource;
+      const result = await plugin.getConsolidatedValue(id, options);
+      plugin.config.resource = oldResource;
+      plugin.config.field = oldField;
+      plugin.transactionResource = oldTransactionResource;
+      plugin.targetResource = oldTargetResource;
+      return result;
     };
   }
   async createTransaction(handler, data) {
@@ -4851,6 +4978,42 @@ class EventualConsistencyPlugin extends Plugin {
       await this.runConsolidation();
     }, intervalMs);
   }
+  startConsolidationTimerForHandler(handler, resourceName, fieldName) {
+    const intervalMs = this.config.consolidationInterval * 1e3;
+    if (this.config.verbose) {
+      const nextRun = new Date(Date.now() + intervalMs);
+      console.log(
+        `[EventualConsistency] ${resourceName}.${fieldName} - Consolidation timer started. Next run at ${nextRun.toISOString()} (every ${this.config.consolidationInterval}s)`
+      );
+    }
+    handler.consolidationTimer = setInterval(async () => {
+      await this.runConsolidationForHandler(handler, resourceName, fieldName);
+    }, intervalMs);
+  }
+  async runConsolidationForHandler(handler, resourceName, fieldName) {
+    const oldResource = this.config.resource;
+    const oldField = this.config.field;
+    const oldTransactionResource = this.transactionResource;
+    const oldTargetResource = this.targetResource;
+    const oldLockResource = this.lockResource;
+    const oldAnalyticsResource = this.analyticsResource;
+    this.config.resource = resourceName;
+    this.config.field = fieldName;
+    this.transactionResource = handler.transactionResource;
+    this.targetResource = handler.targetResource;
+    this.lockResource = handler.lockResource;
+    this.analyticsResource = handler.analyticsResource;
+    try {
+      await this.runConsolidation();
+    } finally {
+      this.config.resource = oldResource;
+      this.config.field = oldField;
+      this.transactionResource = oldTransactionResource;
+      this.targetResource = oldTargetResource;
+      this.lockResource = oldLockResource;
+      this.analyticsResource = oldAnalyticsResource;
+    }
+  }
   async runConsolidation() {
     const startTime = Date.now();
     if (this.config.verbose) {
@@ -4950,10 +5113,6 @@ class EventualConsistencyPlugin extends Plugin {
       return recordOk && record ? record[this.config.field] || 0 : 0;
     }
     try {
-      const [recordOk, recordErr, record] = await tryFn(
-        () => this.targetResource.get(originalId)
-      );
-      const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
       const [ok, err, transactions] = await tryFn(
         () => this.transactionResource.query({
           originalId,
@@ -4961,16 +5120,119 @@ class EventualConsistencyPlugin extends Plugin {
         })
       );
       if (!ok || !transactions || transactions.length === 0) {
+        const [recordOk, recordErr, record] = await tryFn(
+          () => this.targetResource.get(originalId)
+        );
+        const currentValue2 = recordOk && record ? record[this.config.field] || 0 : 0;
         if (this.config.verbose) {
           console.log(
             `[EventualConsistency] ${this.config.resource}.${this.config.field} - No pending transactions for ${originalId}, skipping`
           );
         }
-        return currentValue;
+        return currentValue2;
+      }
+      const [appliedOk, appliedErr, appliedTransactions] = await tryFn(
+        () => this.transactionResource.query({
+          originalId,
+          applied: true
+        })
+      );
+      let currentValue = 0;
+      if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
+        const [recordExistsOk, recordExistsErr, recordExists] = await tryFn(
+          () => this.targetResource.get(originalId)
+        );
+        if (!recordExistsOk || !recordExists) {
+          if (this.config.verbose) {
+            console.log(
+              `[EventualConsistency] ${this.config.resource}.${this.config.field} - Record ${originalId} doesn't exist, deleting ${appliedTransactions.length} old applied transactions`
+            );
+          }
+          const { results, errors } = await promisePool.PromisePool.for(appliedTransactions).withConcurrency(10).process(async (txn) => {
+            const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
+            return deleted;
+          });
+          if (this.config.verbose && errors && errors.length > 0) {
+            console.warn(
+              `[EventualConsistency] ${this.config.resource}.${this.config.field} - Failed to delete ${errors.length} old applied transactions`
+            );
+          }
+          currentValue = 0;
+        } else {
+          appliedTransactions.sort(
+            (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
+          );
+          const hasSetInApplied = appliedTransactions.some((t) => t.operation === "set");
+          if (!hasSetInApplied) {
+            const recordValue = recordExists[this.config.field] || 0;
+            let appliedDelta = 0;
+            for (const t of appliedTransactions) {
+              if (t.operation === "add") appliedDelta += t.value;
+              else if (t.operation === "sub") appliedDelta -= t.value;
+            }
+            const baseValue = recordValue - appliedDelta;
+            const hasExistingAnchor = appliedTransactions.some((t) => t.source === "anchor");
+            if (baseValue !== 0 && !hasExistingAnchor) {
+              const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
+              const cohortInfo = this.getCohortInfo(firstTransactionDate);
+              const anchorTransaction = {
+                id: idGenerator(),
+                originalId,
+                field: this.config.field,
+                value: baseValue,
+                operation: "set",
+                timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(),
+                // 1ms before first txn to ensure it's first
+                cohortDate: cohortInfo.date,
+                cohortHour: cohortInfo.hour,
+                cohortMonth: cohortInfo.month,
+                source: "anchor",
+                applied: true
+              };
+              await this.transactionResource.insert(anchorTransaction);
+              appliedTransactions.unshift(anchorTransaction);
+            }
+          }
+          currentValue = this.config.reducer(appliedTransactions);
+        }
+      } else {
+        const [recordOk, recordErr, record] = await tryFn(
+          () => this.targetResource.get(originalId)
+        );
+        currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
+        if (currentValue !== 0) {
+          let anchorTimestamp;
+          if (transactions && transactions.length > 0) {
+            const firstPendingDate = new Date(transactions[0].timestamp);
+            anchorTimestamp = new Date(firstPendingDate.getTime() - 1).toISOString();
+          } else {
+            anchorTimestamp = (/* @__PURE__ */ new Date()).toISOString();
+          }
+          const cohortInfo = this.getCohortInfo(new Date(anchorTimestamp));
+          const anchorTransaction = {
+            id: idGenerator(),
+            originalId,
+            field: this.config.field,
+            value: currentValue,
+            operation: "set",
+            timestamp: anchorTimestamp,
+            cohortDate: cohortInfo.date,
+            cohortHour: cohortInfo.hour,
+            cohortMonth: cohortInfo.month,
+            source: "anchor",
+            applied: true
+          };
+          await this.transactionResource.insert(anchorTransaction);
+          if (this.config.verbose) {
+            console.log(
+              `[EventualConsistency] ${this.config.resource}.${this.config.field} - Created anchor transaction for ${originalId} with base value ${currentValue}`
+            );
+          }
+        }
       }
       if (this.config.verbose) {
         console.log(
-          `[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidating ${originalId}: ${transactions.length} pending transactions (current: ${currentValue})`
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidating ${originalId}: ${transactions.length} pending transactions (current: ${currentValue} from ${appliedOk && appliedTransactions?.length > 0 ? "applied transactions" : "record"})`
         );
       }
       transactions.sort(
@@ -5048,7 +5310,7 @@ class EventualConsistencyPlugin extends Plugin {
         if (this.config.enableAnalytics && transactionsToUpdate.length > 0) {
           await this.updateAnalytics(transactionsToUpdate);
         }
-        if (this.targetResource.cache && typeof this.targetResource.cache.delete === "function") {
+        if (this.targetResource && this.targetResource.cache && typeof this.targetResource.cache.delete === "function") {
           try {
             const cacheKey = await this.targetResource.cacheKeyFor({ id: originalId });
             await this.targetResource.cache.delete(cacheKey);
@@ -5201,6 +5463,33 @@ class EventualConsistencyPlugin extends Plugin {
       await this.runGarbageCollection();
     }, gcIntervalMs);
   }
+  startGarbageCollectionTimerForHandler(handler, resourceName, fieldName) {
+    const gcIntervalMs = this.config.gcInterval * 1e3;
+    handler.gcTimer = setInterval(async () => {
+      await this.runGarbageCollectionForHandler(handler, resourceName, fieldName);
+    }, gcIntervalMs);
+  }
+  async runGarbageCollectionForHandler(handler, resourceName, fieldName) {
+    const oldResource = this.config.resource;
+    const oldField = this.config.field;
+    const oldTransactionResource = this.transactionResource;
+    const oldTargetResource = this.targetResource;
+    const oldLockResource = this.lockResource;
+    this.config.resource = resourceName;
+    this.config.field = fieldName;
+    this.transactionResource = handler.transactionResource;
+    this.targetResource = handler.targetResource;
+    this.lockResource = handler.lockResource;
+    try {
+      await this.runGarbageCollection();
+    } finally {
+      this.config.resource = oldResource;
+      this.config.field = oldField;
+      this.transactionResource = oldTransactionResource;
+      this.targetResource = oldTargetResource;
+      this.lockResource = oldLockResource;
+    }
+  }
   /**
    * Delete old applied transactions based on retention policy
    * Uses distributed locking to prevent multiple containers from running GC simultaneously
@@ -5503,12 +5792,20 @@ class EventualConsistencyPlugin extends Plugin {
    * @returns {Promise<Array>} Analytics data
    */
   async getAnalytics(resourceName, field, options = {}) {
-    if (!this.analyticsResource) {
+    const fieldHandlers = this.fieldHandlers.get(resourceName);
+    if (!fieldHandlers) {
+      throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
+    }
+    const handler = fieldHandlers.get(field);
+    if (!handler) {
+      throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
+    }
+    if (!handler.analyticsResource) {
       throw new Error("Analytics not enabled for this plugin");
     }
     const { period = "day", date, startDate, endDate, month, year, breakdown = false } = options;
     const [ok, err, allAnalytics] = await tryFn(
-      () => this.analyticsResource.list()
+      () => handler.analyticsResource.list()
     );
     if (!ok || !allAnalytics) {
       return [];
@@ -5734,12 +6031,20 @@ class EventualConsistencyPlugin extends Plugin {
    * @returns {Promise<Array>} Top records
    */
   async getTopRecords(resourceName, field, options = {}) {
-    if (!this.transactionResource) {
+    const fieldHandlers = this.fieldHandlers.get(resourceName);
+    if (!fieldHandlers) {
+      throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
+    }
+    const handler = fieldHandlers.get(field);
+    if (!handler) {
+      throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
+    }
+    if (!handler.transactionResource) {
       throw new Error("Transaction resource not initialized");
     }
     const { period = "day", date, metric = "transactionCount", limit = 10 } = options;
     const [ok, err, transactions] = await tryFn(
-      () => this.transactionResource.list()
+      () => handler.transactionResource.list()
     );
     if (!ok || !transactions) {
       return [];
@@ -11847,7 +12152,7 @@ class Database extends EventEmitter {
     this.id = idGenerator(7);
     this.version = "1";
     this.s3dbVersion = (() => {
-      const [ok, err, version] = tryFn(() => true ? "10.0.12" : "latest");
+      const [ok, err, version] = tryFn(() => true ? "10.0.14" : "latest");
       return ok ? version : "latest";
     })();
     this.resources = {};