s3db.js 11.0.2 → 11.0.3

package/src/plugins/eventual-consistency/utils.js

@@ -64,12 +64,42 @@ export function getTimezoneOffset(timezone, verbose = false) {
   }
 }
 
+/**
+ * Calculate ISO 8601 week number for a date
+ * @param {Date} date - Date to get week number for
+ * @returns {Object} Year and week number { year, week }
+ */
+function getISOWeek(date) {
+  // Copy date to avoid mutating original
+  const target = new Date(date.valueOf());
+
+  // ISO week starts on Monday (day 1)
+  // Find Thursday of this week (ISO week contains Jan 4th)
+  const dayNr = (date.getUTCDay() + 6) % 7; // Make Monday = 0 (use UTC)
+  target.setUTCDate(target.getUTCDate() - dayNr + 3); // Thursday of this week
+
+  // Get first Thursday of the year (use UTC)
+  const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
+  const firstThursday = new Date(yearStart.valueOf());
+  if (yearStart.getUTCDay() !== 4) {
+    firstThursday.setUTCDate(yearStart.getUTCDate() + ((4 - yearStart.getUTCDay()) + 7) % 7);
+  }
+
+  // Calculate week number
+  const weekNumber = 1 + Math.round((target - firstThursday) / 604800000);
+
+  return {
+    year: target.getUTCFullYear(),
+    week: weekNumber
+  };
+}
+
 /**
  * Get cohort information for a date
  * @param {Date} date - Date to get cohort info for
  * @param {string} timezone - IANA timezone name
  * @param {boolean} verbose - Whether to log warnings
- * @returns {Object} Cohort information (date, hour, month)
+ * @returns {Object} Cohort information (date, hour, week, month)
  */
 export function getCohortInfo(date, timezone, verbose = false) {
   // Simple timezone offset calculation
@@ -81,9 +111,14 @@ export function getCohortInfo(date, timezone, verbose = false) {
   const day = String(localDate.getDate()).padStart(2, '0');
   const hour = String(localDate.getHours()).padStart(2, '0');
 
+  // Calculate ISO week
+  const { year: weekYear, week: weekNumber } = getISOWeek(localDate);
+  const week = `${weekYear}-W${String(weekNumber).padStart(2, '0')}`;
+
   return {
     date: `${year}-${month}-${day}`,
     hour: `${year}-${month}-${day}T${hour}`, // ISO-like format for hour partition
+    week: week, // ISO 8601 week format (e.g., '2025-W42')
     month: `${year}-${month}`
   };
 }

package/src/plugins/fulltext.plugin.js

@@ -11,6 +11,8 @@ export class FullTextPlugin extends Plugin {
       ...options
     };
     this.indexes = new Map(); // In-memory index for simplicity
+    this.dirtyIndexes = new Set(); // Track changed index keys for incremental saves
+    this.deletedIndexes = new Set(); // Track deleted index keys
   }
 
   async onInstall() {
@@ -26,7 +28,11 @@ export class FullTextPlugin extends Plugin {
           recordIds: 'json|required', // Array of record IDs containing this word
           count: 'number|required',
           lastUpdated: 'string|required'
-        }
+        },
+        partitions: {
+          byResource: { fields: { resourceName: 'string' } }
+        },
+        behavior: 'body-overflow'
       }));
     this.indexResource = ok ? indexResource : this.database.resources.fulltext_indexes;
 
@@ -69,26 +75,71 @@ export class FullTextPlugin extends Plugin {
 
   async saveIndexes() {
     if (!this.indexResource) return;
-    
+
     const [ok, err] = await tryFn(async () => {
-      // Clear existing indexes
-      const existingIndexes = await this.indexResource.getAll();
-      for (const index of existingIndexes) {
-        await this.indexResource.delete(index.id);
+      // Delete indexes that were removed
+      for (const key of this.deletedIndexes) {
+        // Find and delete the index record using partition-aware query
+        const [resourceName] = key.split(':');
+        const [queryOk, queryErr, results] = await tryFn(() =>
+          this.indexResource.query({ resourceName })
+        );
+
+        if (queryOk && results) {
+          for (const index of results) {
+            const indexKey = `${index.resourceName}:${index.fieldName}:${index.word}`;
+            if (indexKey === key) {
+              await this.indexResource.delete(index.id);
+            }
+          }
+        }
       }
-      // Save current indexes
-      for (const [key, data] of this.indexes.entries()) {
+
+      // Save or update dirty indexes
+      for (const key of this.dirtyIndexes) {
         const [resourceName, fieldName, word] = key.split(':');
-        await this.indexResource.insert({
-          id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
-          resourceName,
-          fieldName,
-          word,
-          recordIds: data.recordIds,
-          count: data.count,
-          lastUpdated: new Date().toISOString()
-        });
+        const data = this.indexes.get(key);
+
+        if (!data) continue; // Skip if index was deleted
+
+        // Try to find existing index record
+        const [queryOk, queryErr, results] = await tryFn(() =>
+          this.indexResource.query({ resourceName })
+        );
+
+        let existingRecord = null;
+        if (queryOk && results) {
+          existingRecord = results.find(
+            (index) => index.resourceName === resourceName &&
+                      index.fieldName === fieldName &&
+                      index.word === word
+          );
+        }
+
+        if (existingRecord) {
+          // Update existing record
+          await this.indexResource.update(existingRecord.id, {
+            recordIds: data.recordIds,
+            count: data.count,
+            lastUpdated: new Date().toISOString()
+          });
+        } else {
+          // Insert new record
+          await this.indexResource.insert({
+            id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
+            resourceName,
+            fieldName,
+            word,
+            recordIds: data.recordIds,
+            count: data.count,
+            lastUpdated: new Date().toISOString()
+          });
+        }
       }
+
+      // Clear tracking sets after successful save
+      this.dirtyIndexes.clear();
+      this.deletedIndexes.clear();
     });
   }
 
@@ -195,21 +246,22 @@ export class FullTextPlugin extends Plugin {
       }
 
       const words = this.tokenize(fieldValue);
-      
+
       for (const word of words) {
         if (word.length < this.config.minWordLength) {
           continue;
         }
-        
+
         const key = `${resourceName}:${fieldName}:${word.toLowerCase()}`;
         const existing = this.indexes.get(key) || { recordIds: [], count: 0 };
-        
+
         if (!existing.recordIds.includes(recordId)) {
           existing.recordIds.push(recordId);
           existing.count = existing.recordIds.length;
         }
-        
+
         this.indexes.set(key, existing);
+        this.dirtyIndexes.add(key); // Mark as dirty for incremental save
       }
     }
   }
@@ -221,11 +273,13 @@ export class FullTextPlugin extends Plugin {
         if (index > -1) {
           data.recordIds.splice(index, 1);
           data.count = data.recordIds.length;
-          
+
           if (data.recordIds.length === 0) {
             this.indexes.delete(key);
+            this.deletedIndexes.add(key); // Track deletion for incremental save
           } else {
             this.indexes.set(key, data);
+            this.dirtyIndexes.add(key); // Mark as dirty for incremental save
           }
         }
       }

package/src/plugins/metrics.plugin.js

@@ -47,8 +47,13 @@ export class MetricsPlugin extends Plugin {
           errors: 'number|required',
           avgTime: 'number|required',
           timestamp: 'string|required',
-          metadata: 'json'
-        }
+          metadata: 'json',
+          createdAt: 'string|required' // YYYY-MM-DD for partitioning
+        },
+        partitions: {
+          byDate: { fields: { createdAt: 'string|maxlength:10' } }
+        },
+        behavior: 'body-overflow'
       }));
       this.metricsResource = ok1 ? metricsResource : this.database.resources.plg_metrics;
 
@@ -60,8 +65,13 @@ export class MetricsPlugin extends Plugin {
           operation: 'string|required',
           error: 'string|required',
           timestamp: 'string|required',
-          metadata: 'json'
-        }
+          metadata: 'json',
+          createdAt: 'string|required' // YYYY-MM-DD for partitioning
+        },
+        partitions: {
+          byDate: { fields: { createdAt: 'string|maxlength:10' } }
+        },
+        behavior: 'body-overflow'
       }));
       this.errorsResource = ok2 ? errorsResource : this.database.resources.plg_error_logs;
 
@@ -73,8 +83,13 @@ export class MetricsPlugin extends Plugin {
           operation: 'string|required',
           duration: 'number|required',
           timestamp: 'string|required',
-          metadata: 'json'
-        }
+          metadata: 'json',
+          createdAt: 'string|required' // YYYY-MM-DD for partitioning
+        },
+        partitions: {
+          byDate: { fields: { createdAt: 'string|maxlength:10' } }
+        },
+        behavior: 'body-overflow'
       }));
       this.performanceResource = ok3 ? performanceResource : this.database.resources.plg_performance_logs;
     });
@@ -359,6 +374,9 @@ export class MetricsPlugin extends Plugin {
       }
 
       // Flush operation metrics
+      const now = new Date();
+      const createdAt = now.toISOString().slice(0, 10); // YYYY-MM-DD
+
       for (const [operation, data] of Object.entries(this.metrics.operations)) {
         if (data.count > 0) {
           await this.metricsResource.insert({
@@ -370,7 +388,8 @@ export class MetricsPlugin extends Plugin {
             totalTime: data.totalTime,
             errors: data.errors,
             avgTime: data.count > 0 ? data.totalTime / data.count : 0,
-            timestamp: new Date().toISOString(),
+            timestamp: now.toISOString(),
+            createdAt,
             metadata
           });
         }
@@ -389,7 +408,8 @@ export class MetricsPlugin extends Plugin {
               totalTime: data.totalTime,
               errors: data.errors,
               avgTime: data.count > 0 ? data.totalTime / data.count : 0,
-              timestamp: new Date().toISOString(),
+              timestamp: now.toISOString(),
+              createdAt,
               metadata: resourceMetadata
             });
           }
@@ -405,6 +425,7 @@ export class MetricsPlugin extends Plugin {
             operation: perf.operation,
             duration: perf.duration,
             timestamp: perf.timestamp,
+            createdAt: perf.timestamp.slice(0, 10), // YYYY-MM-DD from timestamp
             metadata: perfMetadata
           });
         }
@@ -420,6 +441,7 @@ export class MetricsPlugin extends Plugin {
             error: error.error,
             stack: error.stack,
             timestamp: error.timestamp,
+            createdAt: error.timestamp.slice(0, 10), // YYYY-MM-DD from timestamp
             metadata: errorMetadata
           });
         }
@@ -597,28 +619,56 @@ export class MetricsPlugin extends Plugin {
   async cleanupOldData() {
     const cutoffDate = new Date();
     cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionDays);
+    const cutoffDateStr = cutoffDate.toISOString().slice(0, 10); // YYYY-MM-DD
+
+    // Generate list of dates to delete (all dates before cutoff)
+    const datesToDelete = [];
+    const startDate = new Date(cutoffDate);
+    startDate.setDate(startDate.getDate() - 365); // Go back up to 1 year to catch old data
+
+    for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
+      datesToDelete.push(d.toISOString().slice(0, 10));
+    }
 
-    // Clean up old metrics
+    // Clean up old metrics using partition-aware deletion
     if (this.metricsResource) {
-      const oldMetrics = await this.getMetrics({ endDate: cutoffDate.toISOString() });
-      for (const metric of oldMetrics) {
-        await this.metricsResource.delete(metric.id);
+      for (const dateStr of datesToDelete) {
+        const [ok, err, oldMetrics] = await tryFn(() =>
+          this.metricsResource.query({ createdAt: dateStr })
+        );
+        if (ok && oldMetrics) {
+          for (const metric of oldMetrics) {
+            await tryFn(() => this.metricsResource.delete(metric.id));
+          }
+        }
       }
     }
 
-    // Clean up old error logs
+    // Clean up old error logs using partition-aware deletion
     if (this.errorsResource) {
-      const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
-      for (const error of oldErrors) {
-        await this.errorsResource.delete(error.id);
+      for (const dateStr of datesToDelete) {
+        const [ok, err, oldErrors] = await tryFn(() =>
+          this.errorsResource.query({ createdAt: dateStr })
+        );
+        if (ok && oldErrors) {
+          for (const error of oldErrors) {
+            await tryFn(() => this.errorsResource.delete(error.id));
+          }
+        }
       }
     }
 
-    // Clean up old performance logs
+    // Clean up old performance logs using partition-aware deletion
     if (this.performanceResource) {
-      const oldPerformance = await this.getPerformanceLogs({ endDate: cutoffDate.toISOString() });
-      for (const perf of oldPerformance) {
-        await this.performanceResource.delete(perf.id);
+      for (const dateStr of datesToDelete) {
+        const [ok, err, oldPerformance] = await tryFn(() =>
+          this.performanceResource.query({ createdAt: dateStr })
+        );
+        if (ok && oldPerformance) {
+          for (const perf of oldPerformance) {
+            await tryFn(() => this.performanceResource.delete(perf.id));
+          }
+        }
       }
     }
   }

package/src/plugins/s3-queue.plugin.js

@@ -139,31 +139,8 @@ export class S3QueuePlugin extends Plugin {
 
     this.queueResource = this.database.resources[queueName];
 
-    // Create lock resource for distributed locking (enabled by default)
-    const lockName = `${this.config.resource}_locks`;
-    const [okLock, errLock] = await tryFn(() =>
-      this.database.createResource({
-        name: lockName,
-        attributes: {
-          id: 'string|required',
-          workerId: 'string|required',
-          timestamp: 'number|required',
-          ttl: 'number|default:5000'
-        },
-        behavior: 'body-overflow',
-        timestamps: false
-      })
-    );
-
-    if (okLock || this.database.resources[lockName]) {
-      this.lockResource = this.database.resources[lockName];
-    } else {
-      // Locks disabled if creation fails
-      this.lockResource = null;
-      if (this.config.verbose) {
-        console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
-      }
-    }
+    // Locks are now managed by PluginStorage with TTL - no Resource needed
+    // Lock acquisition is handled via storage.acquireLock() with automatic expiration
 
     // Add helper methods to target resource
     this.addHelperMethods();
@@ -273,14 +250,7 @@ export class S3QueuePlugin extends Plugin {
       }
     }, 5000);
 
-    // Start lock cleanup (every 10 seconds, remove expired locks)
-    this.lockCleanupInterval = setInterval(() => {
-      this.cleanupStaleLocks().catch(err => {
-        if (this.config.verbose) {
-          console.log(`[lockCleanup] Error: ${err.message}`);
-        }
-      });
-    }, 10000);
+    // Lock cleanup no longer needed - TTL handles expiration automatically
 
     // Start N workers
     for (let i = 0; i < concurrency; i++) {
@@ -306,11 +276,7 @@ export class S3QueuePlugin extends Plugin {
       this.cacheCleanupInterval = null;
     }
 
-    // Stop lock cleanup
-    if (this.lockCleanupInterval) {
-      clearInterval(this.lockCleanupInterval);
-      this.lockCleanupInterval = null;
-    }
+    // Lock cleanup interval no longer exists (TTL handles it)
 
     // Wait for workers to finish current tasks
     await Promise.all(this.workers);
@@ -383,59 +349,21 @@ export class S3QueuePlugin extends Plugin {
   }
 
   /**
-   * Acquire a distributed lock using ETag-based conditional updates
+   * Acquire a distributed lock using PluginStorage TTL
    * This ensures only one worker can claim a message at a time
-   *
-   * Uses a two-step process:
-   * 1. Create lock resource (similar to queue resource) if not exists
-   * 2. Try to claim lock using ETag-based conditional update
    */
   async acquireLock(messageId) {
-    if (!this.lockResource) {
-      return true; // Locks disabled
-    }
-
-    const lockId = `lock-${messageId}`;
-    const now = Date.now();
+    const storage = this.getStorage();
+    const lockKey = `msg-${messageId}`;
 
     try {
-      // Try to get existing lock
-      const [okGet, errGet, existingLock] = await tryFn(() =>
-        this.lockResource.get(lockId)
-      );
-
-      if (existingLock) {
-        // Lock exists - check if expired
-        const lockAge = now - existingLock.timestamp;
-        if (lockAge < existingLock.ttl) {
-          // Lock still valid, owned by another worker
-          return false;
-        }
-        // Lock expired - try to claim it with ETag
-        const [ok, err, result] = await tryFn(() =>
-          this.lockResource.updateConditional(lockId, {
-            workerId: this.workerId,
-            timestamp: now,
-            ttl: 5000
-          }, {
-            ifMatch: existingLock._etag
-          })
-        );
-
-        return ok && result.success;
-      }
+      const lock = await storage.acquireLock(lockKey, {
+        ttl: 5, // 5 seconds
+        timeout: 0, // Don't wait if locked
+        workerId: this.workerId
+      });
 
-      // Lock doesn't exist - create it
-      const [okCreate, errCreate] = await tryFn(() =>
-        this.lockResource.insert({
-          id: lockId,
-          workerId: this.workerId,
-          timestamp: now,
-          ttl: 5000
-        })
-      );
-
-      return okCreate;
+      return lock !== null;
     } catch (error) {
       // On any error, skip this message
       if (this.config.verbose) {
@@ -446,17 +374,14 @@ export class S3QueuePlugin extends Plugin {
   }
 
   /**
-   * Release a distributed lock by deleting the lock record
+   * Release a distributed lock via PluginStorage
    */
   async releaseLock(messageId) {
-    if (!this.lockResource) {
-      return; // Locks disabled
-    }
-
-    const lockId = `lock-${messageId}`;
+    const storage = this.getStorage();
+    const lockKey = `msg-${messageId}`;
 
     try {
-      await this.lockResource.delete(lockId);
+      await storage.releaseLock(lockKey);
     } catch (error) {
       // Ignore errors on release (lock may have expired or been cleaned up)
       if (this.config.verbose) {
@@ -466,36 +391,12 @@ export class S3QueuePlugin extends Plugin {
   }
 
   /**
-   * Clean up stale locks (older than TTL)
-   * This prevents deadlocks if a worker crashes while holding a lock
+   * Clean up stale locks - NO LONGER NEEDED
+   * TTL handles automatic expiration, no manual cleanup required
    */
   async cleanupStaleLocks() {
-    if (!this.lockResource) {
-      return; // Locks disabled
-    }
-
-    const now = Date.now();
-
-    try {
-      // List all locks
-      const locks = await this.lockResource.list();
-
-      // Delete expired locks
-      for (const lock of locks) {
-        const lockAge = now - lock.timestamp;
-        if (lockAge > lock.ttl) {
-          await this.lockResource.delete(lock.id);
-          if (this.config.verbose) {
-            console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
-          }
-        }
-      }
-    } catch (error) {
-      // Ignore errors in cleanup (non-critical)
-      if (this.config.verbose) {
-        console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
-      }
-    }
+    // TTL automatically expires locks - no manual cleanup needed! ✨
+    return;
   }
 
   async attemptClaim(msg) {

package/src/plugins/scheduler.plugin.js

@@ -163,7 +163,6 @@ export class SchedulerPlugin extends Plugin {
     };
     
     this.database = null;
-    this.lockResource = null;
     this.jobs = new Map();
     this.activeJobs = new Map();
     this.timers = new Map();
@@ -218,9 +217,7 @@ export class SchedulerPlugin extends Plugin {
   }
 
   async onInstall() {
-
-    // Create lock resource for distributed locking
-    await this._createLockResource();
+    // Locks are now managed by PluginStorage with TTL - no Resource needed
 
     // Create job execution history resource
     if (this.config.persistJobs) {
@@ -258,27 +255,6 @@ export class SchedulerPlugin extends Plugin {
     this.emit('initialized', { jobs: this.jobs.size });
   }
 
-  async _createLockResource() {
-    const [ok, err, lockResource] = await tryFn(() =>
-      this.database.createResource({
-        name: 'plg_scheduler_job_locks',
-        attributes: {
-          id: 'string|required',
-          jobName: 'string|required',
-          lockedAt: 'number|required',
-          instanceId: 'string|optional'
-        },
-        behavior: 'body-only',
-        timestamps: false
-      })
-    );
-
-    if (!ok && !this.database.resources.plg_scheduler_job_locks) {
-      throw new Error(`Failed to create lock resource: ${err?.message}`);
-    }
-
-    this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
-  }
 
   async _createJobHistoryResource() {
     const [ok] = await tryFn(() => this.database.createResource({
@@ -416,19 +392,17 @@ export class SchedulerPlugin extends Plugin {
     // Mark as active immediately (will be updated with executionId later)
     this.activeJobs.set(jobName, 'acquiring-lock');
 
-    // Acquire distributed lock to prevent concurrent execution across instances
-    const lockId = `lock-${jobName}`;
-    const [lockAcquired, lockErr] = await tryFn(() =>
-      this.lockResource.insert({
-        id: lockId,
-        jobName,
-        lockedAt: Date.now(),
-        instanceId: process.pid ? String(process.pid) : 'unknown'
-      })
-    );
+    // Acquire distributed lock with TTL to prevent concurrent execution across instances
+    const storage = this.getStorage();
+    const lockKey = `job-${jobName}`;
+    const lock = await storage.acquireLock(lockKey, {
+      ttl: Math.ceil(job.timeout / 1000) + 60, // Job timeout + 60 seconds buffer
+      timeout: 0, // Don't wait if locked
+      workerId: process.pid ? String(process.pid) : 'unknown'
+    });
 
     // If lock couldn't be acquired, another instance is executing this job
-    if (!lockAcquired) {
+    if (!lock) {
       if (this.config.verbose) {
         console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
       }
@@ -577,7 +551,7 @@ export class SchedulerPlugin extends Plugin {
       }
     } finally {
       // Always release the distributed lock
-      await tryFn(() => this.lockResource.delete(lockId));
+      await tryFn(() => storage.releaseLock(lockKey));
     }
   }