s3db.js 11.0.2 → 11.0.4

package/src/plugins/eventual-consistency/index.js

@@ -17,7 +17,7 @@ import {
   runConsolidation
 } from "./consolidation.js";
 import { runGarbageCollection } from "./garbage-collection.js";
-import { updateAnalytics, getAnalytics, getMonthByDay, getDayByHour, getLastNDays, getYearByMonth, getMonthByHour, getTopRecords } from "./analytics.js";
+import { updateAnalytics, getAnalytics, getMonthByDay, getDayByHour, getLastNDays, getYearByMonth, getYearByWeek, getMonthByWeek, getMonthByHour, getTopRecords } from "./analytics.js";
 import { onInstall, onStart, onStop, watchForResource, completeFieldSetup } from "./install.js";
 
 export class EventualConsistencyPlugin extends Plugin {
@@ -125,7 +125,7 @@ export class EventualConsistencyPlugin extends Plugin {
       originalId,
       this.transactionResource,
       this.targetResource,
-      this.lockResource,
+      this.getStorage(),
       this.analyticsResource,
       (transactions) => this.updateAnalytics(transactions),
       this.config
@@ -164,7 +164,7 @@ export class EventualConsistencyPlugin extends Plugin {
       originalId,
       this.transactionResource,
       this.targetResource,
-      this.lockResource,
+      this.getStorage(),
       (id) => this.consolidateRecord(id),
       this.config
     );
@@ -188,14 +188,12 @@ export class EventualConsistencyPlugin extends Plugin {
     const oldField = this.config.field;
     const oldTransactionResource = this.transactionResource;
     const oldTargetResource = this.targetResource;
-    const oldLockResource = this.lockResource;
     const oldAnalyticsResource = this.analyticsResource;
 
     this.config.resource = handler.resource;
     this.config.field = handler.field;
     this.transactionResource = handler.transactionResource;
     this.targetResource = handler.targetResource;
-    this.lockResource = handler.lockResource;
     this.analyticsResource = handler.analyticsResource;
 
     const result = await this.consolidateRecord(id);
@@ -205,7 +203,6 @@ export class EventualConsistencyPlugin extends Plugin {
     this.config.field = oldField;
     this.transactionResource = oldTransactionResource;
     this.targetResource = oldTargetResource;
-    this.lockResource = oldLockResource;
     this.analyticsResource = oldAnalyticsResource;
 
     return result;
@@ -220,14 +217,12 @@ export class EventualConsistencyPlugin extends Plugin {
     const oldField = this.config.field;
     const oldTransactionResource = this.transactionResource;
     const oldTargetResource = this.targetResource;
-    const oldLockResource = this.lockResource;
     const oldAnalyticsResource = this.analyticsResource;
 
     this.config.resource = handler.resource;
     this.config.field = handler.field;
     this.transactionResource = handler.transactionResource;
     this.targetResource = handler.targetResource;
-    this.lockResource = handler.lockResource;
     this.analyticsResource = handler.analyticsResource;
 
     const result = await this.consolidateRecord(id);
@@ -236,7 +231,6 @@ export class EventualConsistencyPlugin extends Plugin {
     this.config.field = oldField;
     this.transactionResource = oldTransactionResource;
     this.targetResource = oldTargetResource;
-    this.lockResource = oldLockResource;
     this.analyticsResource = oldAnalyticsResource;
 
     return result;
@@ -276,14 +270,12 @@ export class EventualConsistencyPlugin extends Plugin {
     const oldField = this.config.field;
     const oldTransactionResource = this.transactionResource;
     const oldTargetResource = this.targetResource;
-    const oldLockResource = this.lockResource;
     const oldAnalyticsResource = this.analyticsResource;
 
     this.config.resource = handler.resource;
     this.config.field = handler.field;
     this.transactionResource = handler.transactionResource;
     this.targetResource = handler.targetResource;
-    this.lockResource = handler.lockResource;
     this.analyticsResource = handler.analyticsResource;
 
     const result = await this.recalculateRecord(id);
@@ -292,7 +284,6 @@ export class EventualConsistencyPlugin extends Plugin {
     this.config.field = oldField;
     this.transactionResource = oldTransactionResource;
     this.targetResource = oldTargetResource;
-    this.lockResource = oldLockResource;
     this.analyticsResource = oldAnalyticsResource;
 
     return result;
@@ -307,14 +298,12 @@ export class EventualConsistencyPlugin extends Plugin {
     const oldField = this.config.field;
     const oldTransactionResource = this.transactionResource;
     const oldTargetResource = this.targetResource;
-    const oldLockResource = this.lockResource;
     const oldAnalyticsResource = this.analyticsResource;
 
     this.config.resource = resourceName;
     this.config.field = fieldName;
     this.transactionResource = handler.transactionResource;
     this.targetResource = handler.targetResource;
-    this.lockResource = handler.lockResource;
     this.analyticsResource = handler.analyticsResource;
 
     try {
@@ -329,7 +318,6 @@ export class EventualConsistencyPlugin extends Plugin {
       this.config.field = oldField;
       this.transactionResource = oldTransactionResource;
       this.targetResource = oldTargetResource;
-      this.lockResource = oldLockResource;
       this.analyticsResource = oldAnalyticsResource;
     }
   }
@@ -343,18 +331,16 @@ export class EventualConsistencyPlugin extends Plugin {
     const oldField = this.config.field;
     const oldTransactionResource = this.transactionResource;
     const oldTargetResource = this.targetResource;
-    const oldLockResource = this.lockResource;
 
     this.config.resource = resourceName;
     this.config.field = fieldName;
     this.transactionResource = handler.transactionResource;
     this.targetResource = handler.targetResource;
-    this.lockResource = handler.lockResource;
 
     try {
       await runGarbageCollection(
         this.transactionResource,
-        this.lockResource,
+        this.getStorage(),
         this.config,
         (event, data) => this.emit(event, data)
       );
@@ -363,7 +349,6 @@ export class EventualConsistencyPlugin extends Plugin {
       this.config.field = oldField;
       this.transactionResource = oldTransactionResource;
       this.targetResource = oldTargetResource;
-      this.lockResource = oldLockResource;
     }
   }
 
@@ -440,6 +425,30 @@ export class EventualConsistencyPlugin extends Plugin {
     return await getMonthByHour(resourceName, field, month, options, this.fieldHandlers);
   }
 
+  /**
+   * Get analytics for entire year, broken down by weeks
+   * @param {string} resourceName - Resource name
+   * @param {string} field - Field name
+   * @param {number} year - Year (e.g., 2025)
+   * @param {Object} options - Options
+   * @returns {Promise<Array>} Weekly analytics for the year (up to 53 weeks)
+   */
+  async getYearByWeek(resourceName, field, year, options = {}) {
+    return await getYearByWeek(resourceName, field, year, options, this.fieldHandlers);
+  }
+
+  /**
+   * Get analytics for entire month, broken down by weeks
+   * @param {string} resourceName - Resource name
+   * @param {string} field - Field name
+   * @param {string} month - Month in YYYY-MM format
+   * @param {Object} options - Options
+   * @returns {Promise<Array>} Weekly analytics for the month
+   */
+  async getMonthByWeek(resourceName, field, month, options = {}) {
+    return await getMonthByWeek(resourceName, field, month, options, this.fieldHandlers);
+  }
+
   /**
    * Get top records by volume
    * @param {string} resourceName - Resource name

package/src/plugins/eventual-consistency/install.js

@@ -84,8 +84,8 @@ export async function completeFieldSetup(handler, database, config, plugin) {
   const resourceName = handler.resource;
   const fieldName = handler.field;
 
-  // Create transaction resource with partitions
-  const transactionResourceName = `${resourceName}_transactions_${fieldName}`;
+  // Create transaction resource with partitions (plg_ prefix for plugin resources)
+  const transactionResourceName = `plg_${resourceName}_tx_${fieldName}`;
   const partitionConfig = createPartitionConfig();
 
   const [ok, err, transactionResource] = await tryFn(() =>
@@ -100,6 +100,7 @@ export async function completeFieldSetup(handler, database, config, plugin) {
         timestamp: 'string|required',
         cohortDate: 'string|required',
         cohortHour: 'string|required',
+        cohortWeek: 'string|optional',
         cohortMonth: 'string|optional',
         source: 'string|optional',
         applied: 'boolean|optional'
@@ -118,27 +119,8 @@ export async function completeFieldSetup(handler, database, config, plugin) {
 
   handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
 
-  // Create lock resource
-  const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
-  const [lockOk, lockErr, lockResource] = await tryFn(() =>
-    database.createResource({
-      name: lockResourceName,
-      attributes: {
-        id: 'string|required',
-        lockedAt: 'number|required',
-        workerId: 'string|optional'
-      },
-      behavior: 'body-only',
-      timestamps: false,
-      createdBy: 'EventualConsistencyPlugin'
-    })
-  );
-
-  if (!lockOk && !database.resources[lockResourceName]) {
-    throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
-  }
-
-  handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
+  // Locks are now managed by PluginStorage with TTL - no Resource needed
+  // Lock acquisition is handled via storage.acquireLock() with automatic expiration
 
   // Create analytics resource if enabled
   if (config.enableAnalytics) {
@@ -151,8 +133,9 @@ export async function completeFieldSetup(handler, database, config, plugin) {
   if (config.verbose) {
     console.log(
       `[EventualConsistency] ${resourceName}.${fieldName} - ` +
-      `Setup complete. Resources: ${transactionResourceName}, ${lockResourceName}` +
-      `${config.enableAnalytics ? `, ${resourceName}_analytics_${fieldName}` : ''}`
+      `Setup complete. Resources: ${transactionResourceName}` +
+      `${config.enableAnalytics ? `, plg_${resourceName}_an_${fieldName}` : ''}` +
+      ` (locks via PluginStorage TTL)`
     );
   }
 }
@@ -167,7 +150,7 @@ export async function completeFieldSetup(handler, database, config, plugin) {
  * @returns {Promise<void>}
  */
 async function createAnalyticsResource(handler, database, resourceName, fieldName) {
-  const analyticsResourceName = `${resourceName}_analytics_${fieldName}`;
+  const analyticsResourceName = `plg_${resourceName}_an_${fieldName}`;
 
   const [ok, err, analyticsResource] = await tryFn(() =>
     database.createResource({

package/src/plugins/eventual-consistency/partitions.js

@@ -34,6 +34,11 @@ export function createPartitionConfig() {
         cohortDate: 'string'
       }
     },
+    byWeek: {
+      fields: {
+        cohortWeek: 'string'
+      }
+    },
     byMonth: {
       fields: {
         cohortMonth: 'string'

package/src/plugins/eventual-consistency/transactions.js

@@ -54,6 +54,7 @@ export async function createTransaction(handler, data, config) {
     timestamp: now.toISOString(),
     cohortDate: cohortInfo.date,
     cohortHour: cohortInfo.hour,
+    cohortWeek: cohortInfo.week,
     cohortMonth: cohortInfo.month,
     source: data.source || 'unknown',
     applied: false

package/src/plugins/eventual-consistency/utils.js

@@ -64,12 +64,42 @@ export function getTimezoneOffset(timezone, verbose = false) {
   }
 }
 
+/**
+ * Calculate ISO 8601 week number for a date
+ * @param {Date} date - Date to get week number for
+ * @returns {Object} Year and week number { year, week }
+ */
+function getISOWeek(date) {
+  // Copy date to avoid mutating original
+  const target = new Date(date.valueOf());
+
+  // ISO week starts on Monday (day 1)
+  // Find Thursday of this week (ISO week contains Jan 4th)
+  const dayNr = (date.getUTCDay() + 6) % 7; // Make Monday = 0 (use UTC)
+  target.setUTCDate(target.getUTCDate() - dayNr + 3); // Thursday of this week
+
+  // Get first Thursday of the year (use UTC)
+  const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
+  const firstThursday = new Date(yearStart.valueOf());
+  if (yearStart.getUTCDay() !== 4) {
+    firstThursday.setUTCDate(yearStart.getUTCDate() + ((4 - yearStart.getUTCDay()) + 7) % 7);
+  }
+
+  // Calculate week number
+  const weekNumber = 1 + Math.round((target - firstThursday) / 604800000);
+
+  return {
+    year: target.getUTCFullYear(),
+    week: weekNumber
+  };
+}
+
 /**
  * Get cohort information for a date
  * @param {Date} date - Date to get cohort info for
  * @param {string} timezone - IANA timezone name
  * @param {boolean} verbose - Whether to log warnings
- * @returns {Object} Cohort information (date, hour, month)
+ * @returns {Object} Cohort information (date, hour, week, month)
  */
 export function getCohortInfo(date, timezone, verbose = false) {
   // Simple timezone offset calculation
@@ -81,9 +111,14 @@ export function getCohortInfo(date, timezone, verbose = false) {
   const day = String(localDate.getDate()).padStart(2, '0');
   const hour = String(localDate.getHours()).padStart(2, '0');
 
+  // Calculate ISO week
+  const { year: weekYear, week: weekNumber } = getISOWeek(localDate);
+  const week = `${weekYear}-W${String(weekNumber).padStart(2, '0')}`;
+
   return {
     date: `${year}-${month}-${day}`,
     hour: `${year}-${month}-${day}T${hour}`, // ISO-like format for hour partition
+    week: week, // ISO 8601 week format (e.g., '2025-W42')
     month: `${year}-${month}`
   };
 }

package/src/plugins/fulltext.plugin.js

@@ -11,6 +11,8 @@ export class FullTextPlugin extends Plugin {
       ...options
     };
     this.indexes = new Map(); // In-memory index for simplicity
+    this.dirtyIndexes = new Set(); // Track changed index keys for incremental saves
+    this.deletedIndexes = new Set(); // Track deleted index keys
   }
 
   async onInstall() {
@@ -26,7 +28,11 @@ export class FullTextPlugin extends Plugin {
           recordIds: 'json|required', // Array of record IDs containing this word
           count: 'number|required',
           lastUpdated: 'string|required'
-        }
+        },
+        partitions: {
+          byResource: { fields: { resourceName: 'string' } }
+        },
+        behavior: 'body-overflow'
       }));
     this.indexResource = ok ? indexResource : this.database.resources.fulltext_indexes;
 
@@ -69,26 +75,71 @@ export class FullTextPlugin extends Plugin {
 
   async saveIndexes() {
     if (!this.indexResource) return;
-    
+
     const [ok, err] = await tryFn(async () => {
-      // Clear existing indexes
-      const existingIndexes = await this.indexResource.getAll();
-      for (const index of existingIndexes) {
-        await this.indexResource.delete(index.id);
+      // Delete indexes that were removed
+      for (const key of this.deletedIndexes) {
+        // Find and delete the index record using partition-aware query
+        const [resourceName] = key.split(':');
+        const [queryOk, queryErr, results] = await tryFn(() =>
+          this.indexResource.query({ resourceName })
+        );
+
+        if (queryOk && results) {
+          for (const index of results) {
+            const indexKey = `${index.resourceName}:${index.fieldName}:${index.word}`;
+            if (indexKey === key) {
+              await this.indexResource.delete(index.id);
+            }
+          }
+        }
       }
-      // Save current indexes
-      for (const [key, data] of this.indexes.entries()) {
+
+      // Save or update dirty indexes
+      for (const key of this.dirtyIndexes) {
         const [resourceName, fieldName, word] = key.split(':');
-        await this.indexResource.insert({
-          id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
-          resourceName,
-          fieldName,
-          word,
-          recordIds: data.recordIds,
-          count: data.count,
-          lastUpdated: new Date().toISOString()
-        });
+        const data = this.indexes.get(key);
+
+        if (!data) continue; // Skip if index was deleted
+
+        // Try to find existing index record
+        const [queryOk, queryErr, results] = await tryFn(() =>
+          this.indexResource.query({ resourceName })
+        );
+
+        let existingRecord = null;
+        if (queryOk && results) {
+          existingRecord = results.find(
+            (index) => index.resourceName === resourceName &&
+                      index.fieldName === fieldName &&
+                      index.word === word
+          );
+        }
+
+        if (existingRecord) {
+          // Update existing record
+          await this.indexResource.update(existingRecord.id, {
+            recordIds: data.recordIds,
+            count: data.count,
+            lastUpdated: new Date().toISOString()
+          });
+        } else {
+          // Insert new record
+          await this.indexResource.insert({
+            id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
+            resourceName,
+            fieldName,
+            word,
+            recordIds: data.recordIds,
+            count: data.count,
+            lastUpdated: new Date().toISOString()
+          });
+        }
       }
+
+      // Clear tracking sets after successful save
+      this.dirtyIndexes.clear();
+      this.deletedIndexes.clear();
     });
   }
 
@@ -195,21 +246,22 @@ export class FullTextPlugin extends Plugin {
       }
 
       const words = this.tokenize(fieldValue);
-      
+
       for (const word of words) {
         if (word.length < this.config.minWordLength) {
           continue;
         }
-        
+
         const key = `${resourceName}:${fieldName}:${word.toLowerCase()}`;
         const existing = this.indexes.get(key) || { recordIds: [], count: 0 };
-        
+
         if (!existing.recordIds.includes(recordId)) {
           existing.recordIds.push(recordId);
           existing.count = existing.recordIds.length;
         }
-        
+
         this.indexes.set(key, existing);
+        this.dirtyIndexes.add(key); // Mark as dirty for incremental save
       }
     }
   }
@@ -221,11 +273,13 @@ export class FullTextPlugin extends Plugin {
         if (index > -1) {
           data.recordIds.splice(index, 1);
           data.count = data.recordIds.length;
-          
+
           if (data.recordIds.length === 0) {
             this.indexes.delete(key);
+            this.deletedIndexes.add(key); // Track deletion for incremental save
           } else {
             this.indexes.set(key, data);
+            this.dirtyIndexes.add(key); // Mark as dirty for incremental save
           }
         }
       }

package/src/plugins/metrics.plugin.js

@@ -47,8 +47,13 @@ export class MetricsPlugin extends Plugin {
           errors: 'number|required',
           avgTime: 'number|required',
           timestamp: 'string|required',
-          metadata: 'json'
-        }
+          metadata: 'json',
+          createdAt: 'string|required' // YYYY-MM-DD for partitioning
+        },
+        partitions: {
+          byDate: { fields: { createdAt: 'string|maxlength:10' } }
+        },
+        behavior: 'body-overflow'
       }));
       this.metricsResource = ok1 ? metricsResource : this.database.resources.plg_metrics;
 
@@ -60,8 +65,13 @@ export class MetricsPlugin extends Plugin {
           operation: 'string|required',
           error: 'string|required',
           timestamp: 'string|required',
-          metadata: 'json'
-        }
+          metadata: 'json',
+          createdAt: 'string|required' // YYYY-MM-DD for partitioning
+        },
+        partitions: {
+          byDate: { fields: { createdAt: 'string|maxlength:10' } }
+        },
+        behavior: 'body-overflow'
       }));
       this.errorsResource = ok2 ? errorsResource : this.database.resources.plg_error_logs;
 
@@ -73,8 +83,13 @@ export class MetricsPlugin extends Plugin {
           operation: 'string|required',
           duration: 'number|required',
           timestamp: 'string|required',
-          metadata: 'json'
-        }
+          metadata: 'json',
+          createdAt: 'string|required' // YYYY-MM-DD for partitioning
+        },
+        partitions: {
+          byDate: { fields: { createdAt: 'string|maxlength:10' } }
+        },
+        behavior: 'body-overflow'
       }));
       this.performanceResource = ok3 ? performanceResource : this.database.resources.plg_performance_logs;
     });
@@ -359,6 +374,9 @@ export class MetricsPlugin extends Plugin {
       }
 
       // Flush operation metrics
+      const now = new Date();
+      const createdAt = now.toISOString().slice(0, 10); // YYYY-MM-DD
+
       for (const [operation, data] of Object.entries(this.metrics.operations)) {
         if (data.count > 0) {
           await this.metricsResource.insert({
@@ -370,7 +388,8 @@ export class MetricsPlugin extends Plugin {
             totalTime: data.totalTime,
             errors: data.errors,
             avgTime: data.count > 0 ? data.totalTime / data.count : 0,
-            timestamp: new Date().toISOString(),
+            timestamp: now.toISOString(),
+            createdAt,
             metadata
           });
         }
@@ -389,7 +408,8 @@ export class MetricsPlugin extends Plugin {
               totalTime: data.totalTime,
               errors: data.errors,
               avgTime: data.count > 0 ? data.totalTime / data.count : 0,
-              timestamp: new Date().toISOString(),
+              timestamp: now.toISOString(),
+              createdAt,
               metadata: resourceMetadata
             });
           }
@@ -405,6 +425,7 @@ export class MetricsPlugin extends Plugin {
             operation: perf.operation,
             duration: perf.duration,
             timestamp: perf.timestamp,
+            createdAt: perf.timestamp.slice(0, 10), // YYYY-MM-DD from timestamp
             metadata: perfMetadata
           });
         }
@@ -420,6 +441,7 @@ export class MetricsPlugin extends Plugin {
             error: error.error,
             stack: error.stack,
             timestamp: error.timestamp,
+            createdAt: error.timestamp.slice(0, 10), // YYYY-MM-DD from timestamp
             metadata: errorMetadata
           });
         }
@@ -597,28 +619,56 @@ export class MetricsPlugin extends Plugin {
   async cleanupOldData() {
     const cutoffDate = new Date();
     cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionDays);
+    const cutoffDateStr = cutoffDate.toISOString().slice(0, 10); // YYYY-MM-DD
+
+    // Generate list of dates to delete (all dates before cutoff)
+    const datesToDelete = [];
+    const startDate = new Date(cutoffDate);
+    startDate.setDate(startDate.getDate() - 365); // Go back up to 1 year to catch old data
+
+    for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
+      datesToDelete.push(d.toISOString().slice(0, 10));
+    }
 
-    // Clean up old metrics
+    // Clean up old metrics using partition-aware deletion
     if (this.metricsResource) {
-      const oldMetrics = await this.getMetrics({ endDate: cutoffDate.toISOString() });
-      for (const metric of oldMetrics) {
-        await this.metricsResource.delete(metric.id);
+      for (const dateStr of datesToDelete) {
+        const [ok, err, oldMetrics] = await tryFn(() =>
+          this.metricsResource.query({ createdAt: dateStr })
+        );
+        if (ok && oldMetrics) {
+          for (const metric of oldMetrics) {
+            await tryFn(() => this.metricsResource.delete(metric.id));
+          }
+        }
       }
     }
 
-    // Clean up old error logs
+    // Clean up old error logs using partition-aware deletion
     if (this.errorsResource) {
-      const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
-      for (const error of oldErrors) {
-        await this.errorsResource.delete(error.id);
+      for (const dateStr of datesToDelete) {
+        const [ok, err, oldErrors] = await tryFn(() =>
+          this.errorsResource.query({ createdAt: dateStr })
+        );
+        if (ok && oldErrors) {
+          for (const error of oldErrors) {
+            await tryFn(() => this.errorsResource.delete(error.id));
+          }
+        }
       }
     }
 
-    // Clean up old performance logs
+    // Clean up old performance logs using partition-aware deletion
     if (this.performanceResource) {
-      const oldPerformance = await this.getPerformanceLogs({ endDate: cutoffDate.toISOString() });
-      for (const perf of oldPerformance) {
-        await this.performanceResource.delete(perf.id);
+      for (const dateStr of datesToDelete) {
+        const [ok, err, oldPerformance] = await tryFn(() =>
+          this.performanceResource.query({ createdAt: dateStr })
+        );
+        if (ok && oldPerformance) {
+          for (const perf of oldPerformance) {
+            await tryFn(() => this.performanceResource.delete(perf.id));
+          }
+        }
       }
     }
   }