s3db.js 10.0.4 → 10.0.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "10.0.4",
+  "version": "10.0.6",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",
@@ -58,16 +58,16 @@
     "UNLICENSE"
   ],
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.873.0",
-    "@modelcontextprotocol/sdk": "^1.17.4",
-    "@smithy/node-http-handler": "^4.1.1",
+    "@aws-sdk/client-s3": "^3.906.0",
+    "@modelcontextprotocol/sdk": "^1.19.1",
+    "@smithy/node-http-handler": "^4.3.0",
     "@supercharge/promise-pool": "^3.2.0",
-    "dotenv": "^17.2.1",
+    "dotenv": "^17.2.3",
     "fastest-validator": "^1.19.1",
     "flat": "^6.0.1",
     "json-stable-stringify": "^1.3.0",
     "lodash-es": "^4.17.21",
-    "nanoid": "5.1.5"
+    "nanoid": "5.1.6"
   },
   "peerDependencies": {
     "@aws-sdk/client-sqs": "^3.0.0",
@@ -94,32 +94,32 @@
     }
   },
   "devDependencies": {
-    "@babel/core": "^7.28.3",
+    "@babel/core": "^7.28.4",
     "@babel/preset-env": "^7.28.3",
     "@rollup/plugin-commonjs": "^28.0.6",
     "@rollup/plugin-json": "^6.1.0",
-    "@rollup/plugin-node-resolve": "^16.0.1",
+    "@rollup/plugin-node-resolve": "^16.0.2",
     "@rollup/plugin-replace": "^6.0.2",
     "@rollup/plugin-terser": "^0.4.4",
-    "@types/node": "24.3.0",
+    "@types/node": "24.7.0",
     "babel-loader": "^10.0.0",
-    "chalk": "^5.6.0",
+    "chalk": "^5.6.2",
     "cli-table3": "^0.6.5",
-    "commander": "^14.0.0",
-    "esbuild": "^0.25.9",
-    "inquirer": "^12.9.3",
-    "jest": "^30.0.5",
+    "commander": "^14.0.1",
+    "esbuild": "^0.25.10",
+    "inquirer": "^12.9.6",
+    "jest": "^30.2.0",
     "node-loader": "^2.1.0",
-    "ora": "^8.2.0",
+    "ora": "^9.0.0",
     "pkg": "^5.8.1",
-    "rollup": "^4.48.0",
+    "rollup": "^4.52.4",
     "rollup-plugin-copy": "^3.5.0",
     "rollup-plugin-esbuild": "^6.2.1",
     "rollup-plugin-polyfill-node": "^0.13.0",
     "rollup-plugin-shebang-bin": "^0.1.0",
     "rollup-plugin-terser": "^7.0.2",
-    "typescript": "5.9.2",
-    "webpack": "^5.101.3",
+    "typescript": "5.9.3",
+    "webpack": "^5.102.1",
     "webpack-cli": "^6.0.1"
   },
   "funding": [

package/src/plugins/eventual-consistency.plugin.js

@@ -186,10 +186,33 @@ export class EventualConsistencyPlugin extends Plugin {
     // Setup consolidation if enabled
     if (this.config.autoConsolidate) {
       this.startConsolidationTimer();
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Auto-consolidation ENABLED (interval: ${this.config.consolidationInterval}s, ` +
+          `window: ${this.config.consolidationWindow}h, mode: ${this.config.mode})`
+        );
+      }
+    } else {
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Auto-consolidation DISABLED (manual consolidation only)`
+        );
+      }
     }
 
     // Setup garbage collection timer
     this.startGarbageCollectionTimer();
+
+    if (this.config.verbose) {
+      console.log(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Setup complete. Resources: ${this.config.resource}_transactions_${this.config.field}, ` +
+        `${this.config.resource}_consolidation_locks_${this.config.field}` +
+        `${this.config.enableAnalytics ? `, ${this.config.resource}_analytics_${this.config.field}` : ''}`
+      );
+    }
   }
 
   async onStart() {
@@ -539,15 +562,31 @@ export class EventualConsistencyPlugin extends Plugin {
     // Batch transactions if configured
     if (this.config.batchTransactions) {
       this.pendingTransactions.set(transaction.id, transaction);
-      
+
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Transaction batched: ${data.operation} ${data.value} for ${data.originalId} ` +
+          `(batch: ${this.pendingTransactions.size}/${this.config.batchSize})`
+        );
+      }
+
       // Flush if batch size reached
       if (this.pendingTransactions.size >= this.config.batchSize) {
         await this.flushPendingTransactions();
       }
     } else {
       await this.transactionResource.insert(transaction);
+
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Transaction created: ${data.operation} ${data.value} for ${data.originalId} ` +
+          `(cohort: ${cohortInfo.hour}, applied: false)`
+        );
+      }
     }
-    
+
     return transaction;
   }
 
@@ -636,12 +675,30 @@ export class EventualConsistencyPlugin extends Plugin {
   startConsolidationTimer() {
     const intervalMs = this.config.consolidationInterval * 1000; // Convert seconds to ms
 
+    if (this.config.verbose) {
+      const nextRun = new Date(Date.now() + intervalMs);
+      console.log(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Consolidation timer started. Next run at ${nextRun.toISOString()} ` +
+        `(every ${this.config.consolidationInterval}s)`
+      );
+    }
+
     this.consolidationTimer = setInterval(async () => {
       await this.runConsolidation();
     }, intervalMs);
   }
 
   async runConsolidation() {
+    const startTime = Date.now();
+
+    if (this.config.verbose) {
+      console.log(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Starting consolidation run at ${new Date().toISOString()}`
+      );
+    }
+
     try {
       // Query unapplied transactions from recent cohorts (last 24 hours by default)
       // This uses hourly partition for O(1) performance instead of full scan
@@ -655,6 +712,13 @@ export class EventualConsistencyPlugin extends Plugin {
         cohortHours.push(cohortInfo.hour);
       }
 
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Querying ${hoursToCheck} hour partitions for pending transactions...`
+        );
+      }
+
       // Query transactions by partition for each hour (parallel for speed)
       const transactionsByHour = await Promise.all(
         cohortHours.map(async (cohortHour) => {
@@ -673,7 +737,10 @@ export class EventualConsistencyPlugin extends Plugin {
 
       if (transactions.length === 0) {
         if (this.config.verbose) {
-          console.log(`[EventualConsistency] No pending transactions to consolidate`);
+          console.log(
+            `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+            `No pending transactions found. Next run in ${this.config.consolidationInterval}s`
+          );
         }
         return;
       }
@@ -681,6 +748,14 @@ export class EventualConsistencyPlugin extends Plugin {
       // Get unique originalIds
       const uniqueIds = [...new Set(transactions.map(t => t.originalId))];
 
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Found ${transactions.length} pending transactions for ${uniqueIds.length} records. ` +
+          `Consolidating with concurrency=${this.config.consolidationConcurrency}...`
+        );
+      }
+
       // Consolidate each record in parallel with concurrency limit
       const { results, errors } = await PromisePool
         .for(uniqueIds)
@@ -689,8 +764,22 @@ export class EventualConsistencyPlugin extends Plugin {
           return await this.consolidateRecord(id);
         });
 
+      const duration = Date.now() - startTime;
+
       if (errors && errors.length > 0) {
-        console.error(`Consolidation completed with ${errors.length} errors:`, errors);
+        console.error(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Consolidation completed with ${errors.length} errors in ${duration}ms:`,
+          errors
+        );
+      }
+
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Consolidation complete: ${results.length} records consolidated in ${duration}ms ` +
+          `(${errors.length} errors). Next run in ${this.config.consolidationInterval}s`
+        );
       }
 
       this.emit('eventual-consistency.consolidated', {
@@ -698,10 +787,16 @@ export class EventualConsistencyPlugin extends Plugin {
         field: this.config.field,
         recordCount: uniqueIds.length,
         successCount: results.length,
-        errorCount: errors.length
+        errorCount: errors.length,
+        duration
       });
     } catch (error) {
-      console.error('Consolidation error:', error);
+      const duration = Date.now() - startTime;
+      console.error(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Consolidation error after ${duration}ms:`,
+        error
+      );
       this.emit('eventual-consistency.consolidation-error', error);
     }
   }
@@ -749,9 +844,23 @@ export class EventualConsistencyPlugin extends Plugin {
       );
 
       if (!ok || !transactions || transactions.length === 0) {
+        if (this.config.verbose) {
+          console.log(
+            `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+            `No pending transactions for ${originalId}, skipping`
+          );
+        }
         return currentValue;
       }
 
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Consolidating ${originalId}: ${transactions.length} pending transactions ` +
+          `(current: ${currentValue})`
+        );
+      }
+
       // Sort transactions by timestamp
       transactions.sort((a, b) =>
         new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
@@ -766,6 +875,14 @@ export class EventualConsistencyPlugin extends Plugin {
       // Apply reducer to get consolidated value
       const consolidatedValue = this.config.reducer(transactions);
 
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `${originalId}: ${currentValue} → ${consolidatedValue} ` +
+          `(${consolidatedValue > currentValue ? '+' : ''}${consolidatedValue - currentValue})`
+        );
+      }
+
       // Update the original record
       const [updateOk, updateErr] = await tryFn(() =>
         this.targetResource.update(originalId, {
@@ -1087,9 +1204,24 @@ export class EventualConsistencyPlugin extends Plugin {
   async updateAnalytics(transactions) {
     if (!this.analyticsResource || transactions.length === 0) return;
 
+    if (this.config.verbose) {
+      console.log(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Updating analytics for ${transactions.length} transactions...`
+      );
+    }
+
     try {
       // Group transactions by cohort hour
       const byHour = this._groupByCohort(transactions, 'cohortHour');
+      const cohortCount = Object.keys(byHour).length;
+
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Updating ${cohortCount} hourly analytics cohorts...`
+        );
+      }
 
       // Update hourly analytics
       for (const [cohort, txns] of Object.entries(byHour)) {
@@ -1099,14 +1231,31 @@ export class EventualConsistencyPlugin extends Plugin {
       // Roll up to daily and monthly if configured
       if (this.config.analyticsConfig.rollupStrategy === 'incremental') {
         const uniqueHours = Object.keys(byHour);
+
+        if (this.config.verbose) {
+          console.log(
+            `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+            `Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
+          );
+        }
+
         for (const cohortHour of uniqueHours) {
           await this._rollupAnalytics(cohortHour);
         }
       }
-    } catch (error) {
+
       if (this.config.verbose) {
-        console.warn(`[EventualConsistency] Analytics update error:`, error.message);
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Analytics update complete for ${cohortCount} cohorts`
+        );
       }
+    } catch (error) {
+      console.warn(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Analytics update error:`,
+        error.message
+      );
     }
   }
 
@@ -1402,14 +1551,86 @@ export class EventualConsistencyPlugin extends Plugin {
     }));
   }
 
+  /**
+   * Fill gaps in analytics data with zeros for continuous time series
+   * @private
+   * @param {Array} data - Sparse analytics data
+   * @param {string} period - Period type ('hour', 'day', 'month')
+   * @param {string} startDate - Start date (ISO format)
+   * @param {string} endDate - End date (ISO format)
+   * @returns {Array} Complete time series with gaps filled
+   */
+  _fillGaps(data, period, startDate, endDate) {
+    if (!data || data.length === 0) {
+      // If no data, still generate empty series
+      data = [];
+    }
+
+    // Create a map of existing data by cohort
+    const dataMap = new Map();
+    data.forEach(item => {
+      dataMap.set(item.cohort, item);
+    });
+
+    const result = [];
+    const emptyRecord = {
+      count: 0,
+      sum: 0,
+      avg: 0,
+      min: 0,
+      max: 0,
+      recordCount: 0
+    };
+
+    if (period === 'hour') {
+      // Generate all hours between startDate and endDate
+      const start = new Date(startDate + 'T00:00:00Z');
+      const end = new Date(endDate + 'T23:59:59Z');
+
+      for (let dt = new Date(start); dt <= end; dt.setHours(dt.getHours() + 1)) {
+        const cohort = dt.toISOString().substring(0, 13); // YYYY-MM-DDTHH
+        result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
+      }
+    } else if (period === 'day') {
+      // Generate all days between startDate and endDate
+      const start = new Date(startDate);
+      const end = new Date(endDate);
+
+      for (let dt = new Date(start); dt <= end; dt.setDate(dt.getDate() + 1)) {
+        const cohort = dt.toISOString().substring(0, 10); // YYYY-MM-DD
+        result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
+      }
+    } else if (period === 'month') {
+      // Generate all months between startDate and endDate
+      const startYear = parseInt(startDate.substring(0, 4));
+      const startMonth = parseInt(startDate.substring(5, 7));
+      const endYear = parseInt(endDate.substring(0, 4));
+      const endMonth = parseInt(endDate.substring(5, 7));
+
+      for (let year = startYear; year <= endYear; year++) {
+        const firstMonth = (year === startYear) ? startMonth : 1;
+        const lastMonth = (year === endYear) ? endMonth : 12;
+
+        for (let month = firstMonth; month <= lastMonth; month++) {
+          const cohort = `${year}-${month.toString().padStart(2, '0')}`;
+          result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
+        }
+      }
+    }
+
+    return result;
+  }
+
   /**
    * Get analytics for entire month, broken down by days
    * @param {string} resourceName - Resource name
    * @param {string} field - Field name
    * @param {string} month - Month in YYYY-MM format
+   * @param {Object} options - Options
+   * @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
    * @returns {Promise<Array>} Daily analytics for the month
    */
-  async getMonthByDay(resourceName, field, month) {
+  async getMonthByDay(resourceName, field, month, options = {}) {
     // month format: '2025-10'
     const year = parseInt(month.substring(0, 4));
     const monthNum = parseInt(month.substring(5, 7));
@@ -1421,11 +1642,17 @@ export class EventualConsistencyPlugin extends Plugin {
     const startDate = firstDay.toISOString().substring(0, 10);
     const endDate = lastDay.toISOString().substring(0, 10);
 
-    return await this.getAnalytics(resourceName, field, {
+    const data = await this.getAnalytics(resourceName, field, {
       period: 'day',
       startDate,
       endDate
     });
+
+    if (options.fillGaps) {
+      return this._fillGaps(data, 'day', startDate, endDate);
+    }
+
+    return data;
   }
 
   /**
@@ -1433,14 +1660,22 @@ export class EventualConsistencyPlugin extends Plugin {
    * @param {string} resourceName - Resource name
    * @param {string} field - Field name
    * @param {string} date - Date in YYYY-MM-DD format
+   * @param {Object} options - Options
+   * @param {boolean} options.fillGaps - Fill missing hours with zeros (default: false)
    * @returns {Promise<Array>} Hourly analytics for the day
    */
-  async getDayByHour(resourceName, field, date) {
+  async getDayByHour(resourceName, field, date, options = {}) {
     // date format: '2025-10-09'
-    return await this.getAnalytics(resourceName, field, {
+    const data = await this.getAnalytics(resourceName, field, {
       period: 'hour',
       date
     });
+
+    if (options.fillGaps) {
+      return this._fillGaps(data, 'hour', date, date);
+    }
+
+    return data;
   }
 
   /**
@@ -1448,20 +1683,28 @@ export class EventualConsistencyPlugin extends Plugin {
    * @param {string} resourceName - Resource name
    * @param {string} field - Field name
    * @param {number} days - Number of days to look back (default: 7)
+   * @param {Object} options - Options
+   * @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
    * @returns {Promise<Array>} Daily analytics
    */
-  async getLastNDays(resourceName, field, days = 7) {
+  async getLastNDays(resourceName, field, days = 7, options = {}) {
     const dates = Array.from({ length: days }, (_, i) => {
       const date = new Date();
       date.setDate(date.getDate() - i);
       return date.toISOString().substring(0, 10);
     }).reverse();
 
-    return await this.getAnalytics(resourceName, field, {
+    const data = await this.getAnalytics(resourceName, field, {
       period: 'day',
       startDate: dates[0],
       endDate: dates[dates.length - 1]
     });
+
+    if (options.fillGaps) {
+      return this._fillGaps(data, 'day', dates[0], dates[dates.length - 1]);
+    }
+
+    return data;
   }
 
   /**
@@ -1469,13 +1712,23 @@ export class EventualConsistencyPlugin extends Plugin {
    * @param {string} resourceName - Resource name
    * @param {string} field - Field name
    * @param {number} year - Year (e.g., 2025)
+   * @param {Object} options - Options
+   * @param {boolean} options.fillGaps - Fill missing months with zeros (default: false)
    * @returns {Promise<Array>} Monthly analytics for the year
    */
-  async getYearByMonth(resourceName, field, year) {
-    return await this.getAnalytics(resourceName, field, {
+  async getYearByMonth(resourceName, field, year, options = {}) {
+    const data = await this.getAnalytics(resourceName, field, {
       period: 'month',
       year
     });
+
+    if (options.fillGaps) {
+      const startDate = `${year}-01`;
+      const endDate = `${year}-12`;
+      return this._fillGaps(data, 'month', startDate, endDate);
+    }
+
+    return data;
   }
 
   /**
@@ -1483,9 +1736,11 @@ export class EventualConsistencyPlugin extends Plugin {
    * @param {string} resourceName - Resource name
    * @param {string} field - Field name
    * @param {string} month - Month in YYYY-MM format (or 'last' for previous month)
+   * @param {Object} options - Options
+   * @param {boolean} options.fillGaps - Fill missing hours with zeros (default: false)
    * @returns {Promise<Array>} Hourly analytics for the month (up to 24*31=744 records)
    */
-  async getMonthByHour(resourceName, field, month) {
+  async getMonthByHour(resourceName, field, month, options = {}) {
     // month format: '2025-10' or 'last'
     let year, monthNum;
 
@@ -1506,11 +1761,17 @@ export class EventualConsistencyPlugin extends Plugin {
     const startDate = firstDay.toISOString().substring(0, 10);
     const endDate = lastDay.toISOString().substring(0, 10);
 
-    return await this.getAnalytics(resourceName, field, {
+    const data = await this.getAnalytics(resourceName, field, {
       period: 'hour',
       startDate,
       endDate
     });
+
+    if (options.fillGaps) {
+      return this._fillGaps(data, 'hour', startDate, endDate);
+    }
+
+    return data;
   }
 
   /**