s3db.js 10.0.12 → 10.0.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "10.0.12",
+  "version": "10.0.13",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",
@@ -131,11 +131,11 @@
     "build:binaries": "./scripts/scripts/build-binaries.sh",
     "dev": "rollup -c -w",
     "test": "pnpm run test:js && pnpm run test:ts",
-    "test:js": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --testTimeout=10000",
+    "test:js": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js",
     "test:ts": "tsc --noEmit --project tests/typescript/tsconfig.json",
-    "test:coverage": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --detectOpenHandles --coverage --runInBand",
-    "test:quick": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --runInBand --testTimeout=10000",
-    "test:plugins": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/ --runInBand --testTimeout=60000",
+    "test:coverage": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --maxWorkers=1",
+    "test:serial": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --runInBand",
+    "test:plugins": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/ --testTimeout=60000",
     "test:full": "pnpm run test:js && pnpm run test:ts",
     "benchmark": "node benchmark-compression.js",
     "version": "echo 'Use pnpm run release v<version> instead of npm version'",

package/src/plugins/eventual-consistency.plugin.js

@@ -317,10 +317,20 @@ export class EventualConsistencyPlugin extends Plugin {
   }
 
   async onStart() {
-    // Emit events for all field handlers
+    // Start timers and emit events for all field handlers
     for (const [resourceName, fieldHandlers] of this.fieldHandlers) {
       for (const [fieldName, handler] of fieldHandlers) {
         if (!handler.deferredSetup) {
+          // Start auto-consolidation timer if enabled
+          if (this.config.autoConsolidate && this.config.mode === 'async') {
+            this.startConsolidationTimerForHandler(handler, resourceName, fieldName);
+          }
+
+          // Start garbage collection timer
+          if (this.config.transactionRetention && this.config.transactionRetention > 0) {
+            this.startGarbageCollectionTimerForHandler(handler, resourceName, fieldName);
+          }
+
           this.emit('eventual-consistency.started', {
             resource: resourceName,
             field: fieldName,
@@ -467,20 +477,57 @@ export class EventualConsistencyPlugin extends Plugin {
     // Add method to set value (replaces current value)
     // Signature: set(id, field, value)
     resource.set = async (id, field, value) => {
-      const { plugin: fieldPlugin } =
+      const { plugin: handler } =
         plugin._resolveFieldAndPlugin(resource, field, value);
 
-      // Create set transaction
-      await fieldPlugin.createTransaction(fieldPlugin, {
+      // Create transaction inline
+      const now = new Date();
+      const cohortInfo = plugin.getCohortInfo(now);
+
+      const transaction = {
+        id: idGenerator(),
         originalId: id,
-        operation: 'set',
+        field: handler.field,
         value: value,
-        source: 'set'
-      });
+        operation: 'set',
+        timestamp: now.toISOString(),
+        cohortDate: cohortInfo.date,
+        cohortHour: cohortInfo.hour,
+        cohortMonth: cohortInfo.month,
+        source: 'set',
+        applied: false
+      };
 
-      // In sync mode, immediately consolidate and update (atomic with locking)
-      if (fieldPlugin.config.mode === 'sync') {
-        return await fieldPlugin._syncModeConsolidate(fieldPlugin, id, field);
+      await handler.transactionResource.insert(transaction);
+
+      // In sync mode, immediately consolidate
+      if (plugin.config.mode === 'sync') {
+        // Temporarily set config for legacy methods
+        const oldResource = plugin.config.resource;
+        const oldField = plugin.config.field;
+        const oldTransactionResource = plugin.transactionResource;
+        const oldTargetResource = plugin.targetResource;
+        const oldLockResource = plugin.lockResource;
+        const oldAnalyticsResource = plugin.analyticsResource;
+
+        plugin.config.resource = handler.resource;
+        plugin.config.field = handler.field;
+        plugin.transactionResource = handler.transactionResource;
+        plugin.targetResource = handler.targetResource;
+        plugin.lockResource = handler.lockResource;
+        plugin.analyticsResource = handler.analyticsResource;
+
+        const result = await plugin._syncModeConsolidate(id, field);
+
+        // Restore
+        plugin.config.resource = oldResource;
+        plugin.config.field = oldField;
+        plugin.transactionResource = oldTransactionResource;
+        plugin.targetResource = oldTargetResource;
+        plugin.lockResource = oldLockResource;
+        plugin.analyticsResource = oldAnalyticsResource;
+
+        return result;
       }
 
       return value;
@@ -489,48 +536,120 @@ export class EventualConsistencyPlugin extends Plugin {
     // Add method to increment value
     // Signature: add(id, field, amount)
     resource.add = async (id, field, amount) => {
-      const { plugin: fieldPlugin } =
+      const { plugin: handler } =
         plugin._resolveFieldAndPlugin(resource, field, amount);
 
-      // Create add transaction
-      await fieldPlugin.createTransaction(fieldPlugin, {
+      // Create transaction inline
+      const now = new Date();
+      const cohortInfo = plugin.getCohortInfo(now);
+
+      const transaction = {
+        id: idGenerator(),
         originalId: id,
-        operation: 'add',
+        field: handler.field,
         value: amount,
-        source: 'add'
-      });
+        operation: 'add',
+        timestamp: now.toISOString(),
+        cohortDate: cohortInfo.date,
+        cohortHour: cohortInfo.hour,
+        cohortMonth: cohortInfo.month,
+        source: 'add',
+        applied: false
+      };
 
-      // In sync mode, immediately consolidate and update (atomic with locking)
-      if (fieldPlugin.config.mode === 'sync') {
-        return await fieldPlugin._syncModeConsolidate(fieldPlugin, id, field);
-      }
+      await handler.transactionResource.insert(transaction);
 
-      // In async mode, return expected value (for user feedback)
-      const currentValue = await fieldPlugin.getConsolidatedValue(fieldPlugin, id);
+      // In sync mode, immediately consolidate
+      if (plugin.config.mode === 'sync') {
+        const oldResource = plugin.config.resource;
+        const oldField = plugin.config.field;
+        const oldTransactionResource = plugin.transactionResource;
+        const oldTargetResource = plugin.targetResource;
+        const oldLockResource = plugin.lockResource;
+        const oldAnalyticsResource = plugin.analyticsResource;
+
+        plugin.config.resource = handler.resource;
+        plugin.config.field = handler.field;
+        plugin.transactionResource = handler.transactionResource;
+        plugin.targetResource = handler.targetResource;
+        plugin.lockResource = handler.lockResource;
+        plugin.analyticsResource = handler.analyticsResource;
+
+        const result = await plugin._syncModeConsolidate(id, field);
+
+        plugin.config.resource = oldResource;
+        plugin.config.field = oldField;
+        plugin.transactionResource = oldTransactionResource;
+        plugin.targetResource = oldTargetResource;
+        plugin.lockResource = oldLockResource;
+        plugin.analyticsResource = oldAnalyticsResource;
+
+        return result;
+      }
+
+      // Async mode - return current value (optimistic)
+      const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
+      const currentValue = (ok && record) ? (record[field] || 0) : 0;
       return currentValue + amount;
     };
 
     // Add method to decrement value
     // Signature: sub(id, field, amount)
     resource.sub = async (id, field, amount) => {
-      const { plugin: fieldPlugin } =
+      const { plugin: handler } =
         plugin._resolveFieldAndPlugin(resource, field, amount);
 
-      // Create sub transaction
-      await fieldPlugin.createTransaction(fieldPlugin, {
+      // Create transaction inline
+      const now = new Date();
+      const cohortInfo = plugin.getCohortInfo(now);
+
+      const transaction = {
+        id: idGenerator(),
         originalId: id,
-        operation: 'sub',
+        field: handler.field,
         value: amount,
-        source: 'sub'
-      });
+        operation: 'sub',
+        timestamp: now.toISOString(),
+        cohortDate: cohortInfo.date,
+        cohortHour: cohortInfo.hour,
+        cohortMonth: cohortInfo.month,
+        source: 'sub',
+        applied: false
+      };
 
-      // In sync mode, immediately consolidate and update (atomic with locking)
-      if (fieldPlugin.config.mode === 'sync') {
-        return await fieldPlugin._syncModeConsolidate(fieldPlugin, id, field);
-      }
+      await handler.transactionResource.insert(transaction);
 
-      // In async mode, return expected value (for user feedback)
-      const currentValue = await fieldPlugin.getConsolidatedValue(fieldPlugin, id);
+      // In sync mode, immediately consolidate
+      if (plugin.config.mode === 'sync') {
+        const oldResource = plugin.config.resource;
+        const oldField = plugin.config.field;
+        const oldTransactionResource = plugin.transactionResource;
+        const oldTargetResource = plugin.targetResource;
+        const oldLockResource = plugin.lockResource;
+        const oldAnalyticsResource = plugin.analyticsResource;
+
+        plugin.config.resource = handler.resource;
+        plugin.config.field = handler.field;
+        plugin.transactionResource = handler.transactionResource;
+        plugin.targetResource = handler.targetResource;
+        plugin.lockResource = handler.lockResource;
+        plugin.analyticsResource = handler.analyticsResource;
+
+        const result = await plugin._syncModeConsolidate(id, field);
+
+        plugin.config.resource = oldResource;
+        plugin.config.field = oldField;
+        plugin.transactionResource = oldTransactionResource;
+        plugin.targetResource = oldTargetResource;
+        plugin.lockResource = oldLockResource;
+        plugin.analyticsResource = oldAnalyticsResource;
+
+        return result;
+      }
+
+      // Async mode - return current value (optimistic)
+      const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
+      const currentValue = (ok && record) ? (record[field] || 0) : 0;
       return currentValue - amount;
     };
 
@@ -541,9 +660,9 @@ export class EventualConsistencyPlugin extends Plugin {
         throw new Error(`Field parameter is required: consolidate(id, field)`);
       }
 
-      const fieldPlugin = resource._eventualConsistencyPlugins[field];
+      const handler = resource._eventualConsistencyPlugins[field];
 
-      if (!fieldPlugin) {
+      if (!handler) {
         const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
         throw new Error(
           `No eventual consistency plugin found for field "${field}". ` +
@@ -551,15 +670,39 @@ export class EventualConsistencyPlugin extends Plugin {
         );
       }
 
-      return await fieldPlugin.consolidateRecord(fieldPlugin, id);
+      // Temporarily set config for legacy methods
+      const oldResource = plugin.config.resource;
+      const oldField = plugin.config.field;
+      const oldTransactionResource = plugin.transactionResource;
+      const oldTargetResource = plugin.targetResource;
+      const oldLockResource = plugin.lockResource;
+      const oldAnalyticsResource = plugin.analyticsResource;
+
+      plugin.config.resource = handler.resource;
+      plugin.config.field = handler.field;
+      plugin.transactionResource = handler.transactionResource;
+      plugin.targetResource = handler.targetResource;
+      plugin.lockResource = handler.lockResource;
+      plugin.analyticsResource = handler.analyticsResource;
+
+      const result = await plugin.consolidateRecord(id);
+
+      plugin.config.resource = oldResource;
+      plugin.config.field = oldField;
+      plugin.transactionResource = oldTransactionResource;
+      plugin.targetResource = oldTargetResource;
+      plugin.lockResource = oldLockResource;
+      plugin.analyticsResource = oldAnalyticsResource;
+
+      return result;
     };
 
     // Add method to get consolidated value without applying
     // Signature: getConsolidatedValue(id, field, options)
     resource.getConsolidatedValue = async (id, field, options = {}) => {
-      const fieldPlugin = resource._eventualConsistencyPlugins[field];
+      const handler = resource._eventualConsistencyPlugins[field];
 
-      if (!fieldPlugin) {
+      if (!handler) {
         const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
         throw new Error(
           `No eventual consistency plugin found for field "${field}". ` +
@@ -567,7 +710,25 @@ export class EventualConsistencyPlugin extends Plugin {
         );
       }
 
-      return await fieldPlugin.getConsolidatedValue(fieldPlugin, id, options);
+      // Temporarily set config for legacy methods
+      const oldResource = plugin.config.resource;
+      const oldField = plugin.config.field;
+      const oldTransactionResource = plugin.transactionResource;
+      const oldTargetResource = plugin.targetResource;
+
+      plugin.config.resource = handler.resource;
+      plugin.config.field = handler.field;
+      plugin.transactionResource = handler.transactionResource;
+      plugin.targetResource = handler.targetResource;
+
+      const result = await plugin.getConsolidatedValue(id, options);
+
+      plugin.config.resource = oldResource;
+      plugin.config.field = oldField;
+      plugin.transactionResource = oldTransactionResource;
+      plugin.targetResource = oldTargetResource;
+
+      return result;
     };
   }
 
@@ -746,6 +907,52 @@ export class EventualConsistencyPlugin extends Plugin {
     }, intervalMs);
   }
 
+  startConsolidationTimerForHandler(handler, resourceName, fieldName) {
+    const intervalMs = this.config.consolidationInterval * 1000; // Convert seconds to ms
+
+    if (this.config.verbose) {
+      const nextRun = new Date(Date.now() + intervalMs);
+      console.log(
+        `[EventualConsistency] ${resourceName}.${fieldName} - ` +
+        `Consolidation timer started. Next run at ${nextRun.toISOString()} ` +
+        `(every ${this.config.consolidationInterval}s)`
+      );
+    }
+
+    handler.consolidationTimer = setInterval(async () => {
+      await this.runConsolidationForHandler(handler, resourceName, fieldName);
+    }, intervalMs);
+  }
+
+  async runConsolidationForHandler(handler, resourceName, fieldName) {
+    // Temporarily swap config to use this handler
+    const oldResource = this.config.resource;
+    const oldField = this.config.field;
+    const oldTransactionResource = this.transactionResource;
+    const oldTargetResource = this.targetResource;
+    const oldLockResource = this.lockResource;
+    const oldAnalyticsResource = this.analyticsResource;
+
+    this.config.resource = resourceName;
+    this.config.field = fieldName;
+    this.transactionResource = handler.transactionResource;
+    this.targetResource = handler.targetResource;
+    this.lockResource = handler.lockResource;
+    this.analyticsResource = handler.analyticsResource;
+
+    try {
+      await this.runConsolidation();
+    } finally {
+      // Restore
+      this.config.resource = oldResource;
+      this.config.field = oldField;
+      this.transactionResource = oldTransactionResource;
+      this.targetResource = oldTargetResource;
+      this.lockResource = oldLockResource;
+      this.analyticsResource = oldAnalyticsResource;
+    }
+  }
+
   async runConsolidation() {
     const startTime = Date.now();
 
@@ -1035,7 +1242,7 @@ export class EventualConsistencyPlugin extends Plugin {
         }
 
         // Invalidate cache for this record after consolidation
-        if (this.targetResource.cache && typeof this.targetResource.cache.delete === 'function') {
+        if (this.targetResource && this.targetResource.cache && typeof this.targetResource.cache.delete === 'function') {
           try {
             const cacheKey = await this.targetResource.cacheKeyFor({ id: originalId });
             await this.targetResource.cache.delete(cacheKey);
@@ -1244,6 +1451,40 @@ export class EventualConsistencyPlugin extends Plugin {
     }, gcIntervalMs);
   }
 
+  startGarbageCollectionTimerForHandler(handler, resourceName, fieldName) {
+    const gcIntervalMs = this.config.gcInterval * 1000; // Convert seconds to ms
+
+    handler.gcTimer = setInterval(async () => {
+      await this.runGarbageCollectionForHandler(handler, resourceName, fieldName);
+    }, gcIntervalMs);
+  }
+
+  async runGarbageCollectionForHandler(handler, resourceName, fieldName) {
+    // Temporarily swap config to use this handler
+    const oldResource = this.config.resource;
+    const oldField = this.config.field;
+    const oldTransactionResource = this.transactionResource;
+    const oldTargetResource = this.targetResource;
+    const oldLockResource = this.lockResource;
+
+    this.config.resource = resourceName;
+    this.config.field = fieldName;
+    this.transactionResource = handler.transactionResource;
+    this.targetResource = handler.targetResource;
+    this.lockResource = handler.lockResource;
+
+    try {
+      await this.runGarbageCollection();
+    } finally {
+      // Restore
+      this.config.resource = oldResource;
+      this.config.field = oldField;
+      this.transactionResource = oldTransactionResource;
+      this.targetResource = oldTargetResource;
+      this.lockResource = oldLockResource;
+    }
+  }
+
   /**
    * Delete old applied transactions based on retention policy
    * Uses distributed locking to prevent multiple containers from running GC simultaneously
@@ -1634,14 +1875,25 @@ export class EventualConsistencyPlugin extends Plugin {
    * @returns {Promise<Array>} Analytics data
    */
   async getAnalytics(resourceName, field, options = {}) {
-    if (!this.analyticsResource) {
+    // Get handler for this resource/field combination
+    const fieldHandlers = this.fieldHandlers.get(resourceName);
+    if (!fieldHandlers) {
+      throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
+    }
+
+    const handler = fieldHandlers.get(field);
+    if (!handler) {
+      throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
+    }
+
+    if (!handler.analyticsResource) {
       throw new Error('Analytics not enabled for this plugin');
     }
 
     const { period = 'day', date, startDate, endDate, month, year, breakdown = false } = options;
 
     const [ok, err, allAnalytics] = await tryFn(() =>
-      this.analyticsResource.list()
+      handler.analyticsResource.list()
     );
 
     if (!ok || !allAnalytics) {
@@ -1921,7 +2173,18 @@ export class EventualConsistencyPlugin extends Plugin {
    * @returns {Promise<Array>} Top records
    */
   async getTopRecords(resourceName, field, options = {}) {
-    if (!this.transactionResource) {
+    // Get handler for this resource/field combination
+    const fieldHandlers = this.fieldHandlers.get(resourceName);
+    if (!fieldHandlers) {
+      throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
+    }
+
+    const handler = fieldHandlers.get(field);
+    if (!handler) {
+      throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
+    }
+
+    if (!handler.transactionResource) {
       throw new Error('Transaction resource not initialized');
     }
 
@@ -1929,7 +2192,7 @@ export class EventualConsistencyPlugin extends Plugin {
 
     // Get all transactions for the period
     const [ok, err, transactions] = await tryFn(() =>
-      this.transactionResource.list()
+      handler.transactionResource.list()
     );
 
     if (!ok || !transactions) {