s3db.js 11.0.2 → 11.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "11.0.2",
+  "version": "11.0.4",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",

package/src/concerns/plugin-storage.js

@@ -73,20 +73,29 @@ export class PluginStorage {
   }
 
   /**
-   * Save data with metadata encoding and behavior support
+   * Save data with metadata encoding, behavior support, and optional TTL
    *
    * @param {string} key - S3 key
    * @param {Object} data - Data to save
    * @param {Object} options - Options
+   * @param {number} options.ttl - Time-to-live in seconds (optional)
    * @param {string} options.behavior - 'body-overflow' | 'body-only' | 'enforce-limits'
    * @param {string} options.contentType - Content type (default: application/json)
    * @returns {Promise<void>}
    */
-  async put(key, data, options = {}) {
-    const { behavior = 'body-overflow', contentType = 'application/json' } = options;
+  async set(key, data, options = {}) {
+    const { ttl, behavior = 'body-overflow', contentType = 'application/json' } = options;
+
+    // Clone data to avoid mutating original
+    const dataToSave = { ...data };
+
+    // Add TTL expiration timestamp if provided
+    if (ttl && typeof ttl === 'number' && ttl > 0) {
+      dataToSave._expiresAt = Date.now() + (ttl * 1000);
+    }
 
     // Apply behavior to split data between metadata and body
-    const { metadata, body } = this._applyBehavior(data, behavior);
+    const { metadata, body } = this._applyBehavior(dataToSave, behavior);
 
     // Prepare putObject parameters
     const putParams = {
@@ -104,15 +113,23 @@ export class PluginStorage {
     const [ok, err] = await tryFn(() => this.client.putObject(putParams));
 
     if (!ok) {
-      throw new Error(`PluginStorage.put failed for key ${key}: ${err.message}`);
+      throw new Error(`PluginStorage.set failed for key ${key}: ${err.message}`);
     }
   }
 
   /**
-   * Get data with automatic metadata decoding
+   * Alias for set() to maintain backward compatibility
+   * @deprecated Use set() instead
+   */
+  async put(key, data, options = {}) {
+    return this.set(key, data, options);
+  }
+
+  /**
+   * Get data with automatic metadata decoding and TTL check
    *
    * @param {string} key - S3 key
-   * @returns {Promise<Object|null>} Data or null if not found
+   * @returns {Promise<Object|null>} Data or null if not found/expired
    */
   async get(key) {
     const [ok, err, response] = await tryFn(() => this.client.getObject(key));
@@ -130,6 +147,9 @@ export class PluginStorage {
     const metadata = response.Metadata || {};
     const parsedMetadata = this._parseMetadataValues(metadata);
 
+    // Build final data object
+    let data = parsedMetadata;
+
     // If has body, merge with metadata
     if (response.Body) {
       try {
@@ -139,14 +159,27 @@ export class PluginStorage {
         if (bodyContent && bodyContent.trim()) {
           const body = JSON.parse(bodyContent);
           // Body takes precedence over metadata for same keys
-          return { ...parsedMetadata, ...body };
+          data = { ...parsedMetadata, ...body };
         }
       } catch (parseErr) {
         throw new Error(`PluginStorage.get failed to parse body for key ${key}: ${parseErr.message}`);
       }
     }
 
-    return parsedMetadata;
+    // Check TTL expiration (S3 lowercases metadata keys)
+    const expiresAt = data._expiresat || data._expiresAt;
+    if (expiresAt) {
+      if (Date.now() > expiresAt) {
+        // Expired - delete and return null
+        await this.delete(key);
+        return null;
+      }
+      // Remove internal fields before returning
+      delete data._expiresat;
+      delete data._expiresAt;
+    }
+
+    return data;
   }
 
   /**
@@ -265,6 +298,154 @@ export class PluginStorage {
       .map(key => (key.startsWith('/') ? key.replace('/', '') : key));
   }
 
+  /**
+   * Check if a key exists (not expired)
+   *
+   * @param {string} key - S3 key
+   * @returns {Promise<boolean>} True if exists and not expired
+   */
+  async has(key) {
+    const data = await this.get(key);
+    return data !== null;
+  }
+
+  /**
+   * Check if a key is expired
+   *
+   * @param {string} key - S3 key
+   * @returns {Promise<boolean>} True if expired or not found
+   */
+  async isExpired(key) {
+    const [ok, err, response] = await tryFn(() => this.client.getObject(key));
+
+    if (!ok) {
+      return true; // Not found = expired
+    }
+
+    const metadata = response.Metadata || {};
+    const parsedMetadata = this._parseMetadataValues(metadata);
+
+    let data = parsedMetadata;
+
+    if (response.Body) {
+      try {
+        const bodyContent = await response.Body.transformToString();
+        if (bodyContent && bodyContent.trim()) {
+          const body = JSON.parse(bodyContent);
+          data = { ...parsedMetadata, ...body };
+        }
+      } catch {
+        return true;
+      }
+    }
+
+    // S3 lowercases metadata keys
+    const expiresAt = data._expiresat || data._expiresAt;
+    if (!expiresAt) {
+      return false; // No TTL = not expired
+    }
+
+    return Date.now() > expiresAt;
+  }
+
+  /**
+   * Get remaining TTL in seconds
+   *
+   * @param {string} key - S3 key
+   * @returns {Promise<number|null>} Remaining seconds or null if no TTL/not found
+   */
+  async getTTL(key) {
+    const [ok, err, response] = await tryFn(() => this.client.getObject(key));
+
+    if (!ok) {
+      return null;
+    }
+
+    const metadata = response.Metadata || {};
+    const parsedMetadata = this._parseMetadataValues(metadata);
+
+    let data = parsedMetadata;
+
+    if (response.Body) {
+      try {
+        const bodyContent = await response.Body.transformToString();
+        if (bodyContent && bodyContent.trim()) {
+          const body = JSON.parse(bodyContent);
+          data = { ...parsedMetadata, ...body };
+        }
+      } catch {
+        return null;
+      }
+    }
+
+    // S3 lowercases metadata keys
+    const expiresAt = data._expiresat || data._expiresAt;
+    if (!expiresAt) {
+      return null; // No TTL
+    }
+
+    const remaining = Math.max(0, expiresAt - Date.now());
+    return Math.floor(remaining / 1000); // Convert to seconds
+  }
+
+  /**
+   * Extend TTL by adding additional seconds
+   *
+   * @param {string} key - S3 key
+   * @param {number} additionalSeconds - Seconds to add to current TTL
+   * @returns {Promise<boolean>} True if extended, false if not found or no TTL
+   */
+  async touch(key, additionalSeconds) {
+    const [ok, err, response] = await tryFn(() => this.client.getObject(key));
+
+    if (!ok) {
+      return false;
+    }
+
+    const metadata = response.Metadata || {};
+    const parsedMetadata = this._parseMetadataValues(metadata);
+
+    let data = parsedMetadata;
+
+    if (response.Body) {
+      try {
+        const bodyContent = await response.Body.transformToString();
+        if (bodyContent && bodyContent.trim()) {
+          const body = JSON.parse(bodyContent);
+          data = { ...parsedMetadata, ...body };
+        }
+      } catch {
+        return false;
+      }
+    }
+
+    // S3 lowercases metadata keys
+    const expiresAt = data._expiresat || data._expiresAt;
+    if (!expiresAt) {
+      return false; // No TTL to extend
+    }
+
+    // Extend TTL - use the standard field name (will be lowercased by S3)
+    data._expiresAt = expiresAt + (additionalSeconds * 1000);
+    delete data._expiresat; // Remove lowercased version
+
+    // Save back (reuse same behavior)
+    const { metadata: newMetadata, body: newBody } = this._applyBehavior(data, 'body-overflow');
+
+    const putParams = {
+      key,
+      metadata: newMetadata,
+      contentType: 'application/json'
+    };
+
+    if (newBody !== null) {
+      putParams.body = JSON.stringify(newBody);
+    }
+
+    const [putOk] = await tryFn(() => this.client.putObject(putParams));
+    return putOk;
+  }
+
   /**
    * Delete a single object
    *
@@ -363,6 +544,90 @@ export class PluginStorage {
     return results;
   }
 
+  /**
+   * Acquire a distributed lock with TTL and retry logic
+   *
+   * @param {string} lockName - Lock identifier
+   * @param {Object} options - Lock options
+   * @param {number} options.ttl - Lock TTL in seconds (default: 30)
+   * @param {number} options.timeout - Max wait time in ms (default: 0, no wait)
+   * @param {string} options.workerId - Worker identifier (default: 'unknown')
+   * @returns {Promise<Object|null>} Lock object or null if couldn't acquire
+   */
+  async acquireLock(lockName, options = {}) {
+    const { ttl = 30, timeout = 0, workerId = 'unknown' } = options;
+    const key = this.getPluginKey(null, 'locks', lockName);
+
+    const startTime = Date.now();
+
+    while (true) {
+      // Try to acquire
+      const existing = await this.get(key);
+      if (!existing) {
+        await this.set(key, { workerId, acquiredAt: Date.now() }, { ttl });
+        return { key, workerId };
+      }
+
+      // Check timeout
+      if (Date.now() - startTime >= timeout) {
+        return null; // Could not acquire
+      }
+
+      // Wait and retry (100ms intervals)
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+  }
+
+  /**
+   * Release a distributed lock
+   *
+   * @param {string} lockName - Lock identifier
+   * @returns {Promise<void>}
+   */
+  async releaseLock(lockName) {
+    const key = this.getPluginKey(null, 'locks', lockName);
+    await this.delete(key);
+  }
+
+  /**
+   * Check if a lock is currently held
+   *
+   * @param {string} lockName - Lock identifier
+   * @returns {Promise<boolean>} True if locked
+   */
+  async isLocked(lockName) {
+    const key = this.getPluginKey(null, 'locks', lockName);
+    const lock = await this.get(key);
+    return lock !== null;
+  }
+
+  /**
+   * Increment a counter value
+   *
+   * @param {string} key - S3 key
+   * @param {number} amount - Amount to increment (default: 1)
+   * @param {Object} options - Options (e.g., ttl)
+   * @returns {Promise<number>} New value
+   */
+  async increment(key, amount = 1, options = {}) {
+    const data = await this.get(key);
+    const value = (data?.value || 0) + amount;
+    await this.set(key, { value }, options);
+    return value;
+  }
+
+  /**
+   * Decrement a counter value
+   *
+   * @param {string} key - S3 key
+   * @param {number} amount - Amount to decrement (default: 1)
+   * @param {Object} options - Options (e.g., ttl)
+   * @returns {Promise<number>} New value
+   */
+  async decrement(key, amount = 1, options = {}) {
+    return this.increment(key, -amount, options);
+  }
+
   /**
    * Apply behavior to split data between metadata and body
    *

package/src/plugins/audit.plugin.js

@@ -24,12 +24,17 @@ export class AuditPlugin extends Plugin {
         recordId: 'string|required',
         userId: 'string|optional',
         timestamp: 'string|required',
+        createdAt: 'string|required', // YYYY-MM-DD for partitioning
         oldData: 'string|optional',
         newData: 'string|optional',
         partition: 'string|optional',
         partitionValues: 'string|optional',
         metadata: 'string|optional'
       },
+      partitions: {
+        byDate: { fields: { createdAt: 'string|maxlength:10' } },
+        byResource: { fields: { resourceName: 'string' } }
+      },
       behavior: 'body-overflow'
     }));
     this.auditResource = ok ? auditResource : (this.database.resources.plg_audits || null);
@@ -162,10 +167,12 @@ export class AuditPlugin extends Plugin {
       return;
     }
 
+    const now = new Date();
     const auditRecord = {
       id: `audit-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`,
       userId: this.getCurrentUserId?.() || 'system',
-      timestamp: new Date().toISOString(),
+      timestamp: now.toISOString(),
+      createdAt: now.toISOString().slice(0, 10), // YYYY-MM-DD for partitioning
       metadata: JSON.stringify({ source: 'audit-plugin', version: '2.0' }),
       resourceName: auditData.resourceName,
       operation: auditData.operation,
@@ -253,20 +260,37 @@ export class AuditPlugin extends Plugin {
 
   async getAuditLogs(options = {}) {
     if (!this.auditResource) return [];
-    
+
     const { resourceName, operation, recordId, partition, startDate, endDate, limit = 100, offset = 0 } = options;
-    
-    // If we have specific filters, we need to fetch more items to ensure proper pagination after filtering
-    const hasFilters = resourceName || operation || recordId || partition || startDate || endDate;
-    
+
     let items = [];
-    
-    if (hasFilters) {
-      // Fetch enough items to handle filtering
+
+    // Use partition-aware queries when possible
+    if (resourceName && !operation && !recordId && !partition && !startDate && !endDate) {
+      // Query by resource partition directly (most efficient)
+      const [ok, err, result] = await tryFn(() =>
+        this.auditResource.query({ resourceName }, { limit: limit + offset })
+      );
+      items = ok && result ? result : [];
+      return items.slice(offset, offset + limit);
+    } else if (startDate && !resourceName && !operation && !recordId && !partition) {
+      // Query by date partition (efficient for date ranges)
+      const dates = this._generateDateRange(startDate, endDate);
+      for (const date of dates) {
+        const [ok, err, result] = await tryFn(() =>
+          this.auditResource.query({ createdAt: date })
+        );
+        if (ok && result) {
+          items.push(...result);
+        }
+      }
+      return items.slice(offset, offset + limit);
+    } else if (resourceName || operation || recordId || partition || startDate || endDate) {
+      // Fetch with filters (less efficient, but necessary)
       const fetchSize = Math.min(10000, Math.max(1000, (limit + offset) * 20));
       const result = await this.auditResource.list({ limit: fetchSize });
       items = result || [];
-      
+
       // Apply filters
       if (resourceName) {
         items = items.filter(log => log.resourceName === resourceName);
@@ -288,8 +312,7 @@ export class AuditPlugin extends Plugin {
           return true;
         });
       }
-      
-      // Apply offset and limit after filtering
+
       return items.slice(offset, offset + limit);
     } else {
       // No filters, use direct pagination
@@ -298,6 +321,18 @@ export class AuditPlugin extends Plugin {
     }
   }
 
+  _generateDateRange(startDate, endDate) {
+    const dates = [];
+    const start = new Date(startDate);
+    const end = endDate ? new Date(endDate) : new Date();
+
+    for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) {
+      dates.push(d.toISOString().slice(0, 10));
+    }
+
+    return dates;
+  }
+
   async getRecordHistory(resourceName, recordId) {
     return await this.getAuditLogs({ resourceName, recordId });
   }
@@ -312,7 +347,7 @@ export class AuditPlugin extends Plugin {
 
   async getAuditStats(options = {}) {
     const logs = await this.getAuditLogs(options);
-    
+
     const stats = {
       total: logs.length,
       byOperation: {},
@@ -321,22 +356,22 @@ export class AuditPlugin extends Plugin {
       byUser: {},
       timeline: {}
     };
-    
+
     for (const log of logs) {
       // Count by operation
       stats.byOperation[log.operation] = (stats.byOperation[log.operation] || 0) + 1;
-      
+
       // Count by resource
       stats.byResource[log.resourceName] = (stats.byResource[log.resourceName] || 0) + 1;
-      
+
       // Count by partition
       if (log.partition) {
         stats.byPartition[log.partition] = (stats.byPartition[log.partition] || 0) + 1;
       }
-      
+
       // Count by user
       stats.byUser[log.userId] = (stats.byUser[log.userId] || 0) + 1;
-      
+
       // Timeline by date
       const date = log.timestamp.split('T')[0];
       stats.timeline[date] = (stats.timeline[date] || 0) + 1;
@@ -344,6 +379,47 @@ export class AuditPlugin extends Plugin {
 
     return stats;
   }
+
+  /**
+   * Clean up audit logs older than retention period
+   * @param {number} retentionDays - Number of days to retain (default: 90)
+   * @returns {Promise<number>} Number of records deleted
+   */
+  async cleanupOldAudits(retentionDays = 90) {
+    if (!this.auditResource) return 0;
+
+    const cutoffDate = new Date();
+    cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
+
+    // Generate list of dates to delete (all dates before cutoff)
+    const datesToDelete = [];
+    const startDate = new Date(cutoffDate);
+    startDate.setDate(startDate.getDate() - 365); // Go back up to 1 year to catch old data
+
+    for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
+      datesToDelete.push(d.toISOString().slice(0, 10));
+    }
+
+    let deletedCount = 0;
+
+    // Clean up using partition-aware queries
+    for (const dateStr of datesToDelete) {
+      const [ok, err, oldAudits] = await tryFn(() =>
+        this.auditResource.query({ createdAt: dateStr })
+      );
+
+      if (ok && oldAudits) {
+        for (const audit of oldAudits) {
+          const [delOk] = await tryFn(() => this.auditResource.delete(audit.id));
+          if (delOk) {
+            deletedCount++;
+          }
+        }
+      }
+    }
+
+    return deletedCount;
+  }
 }
 
 export default AuditPlugin;