s3db.js 10.0.0 → 10.0.3

package/src/plugins/fulltext.plugin.js

@@ -18,7 +18,7 @@ export class FullTextPlugin extends Plugin {
     
     // Create index resource if it doesn't exist
     const [ok, err, indexResource] = await tryFn(() => database.createResource({
-        name: 'fulltext_indexes',
+        name: 'plg_fulltext_indexes',
         attributes: {
           id: 'string|required',
           resourceName: 'string|required',
@@ -96,7 +96,7 @@ export class FullTextPlugin extends Plugin {
   installDatabaseHooks() {
     // Use the new database hooks system for automatic resource discovery
     this.database.addHook('afterCreateResource', (resource) => {
-      if (resource.name !== 'fulltext_indexes') {
+      if (resource.name !== 'plg_fulltext_indexes') {
         this.installResourceHooks(resource);
       }
     });
@@ -115,7 +115,7 @@ export class FullTextPlugin extends Plugin {
     this.database.plugins.fulltext = this;
 
     for (const resource of Object.values(this.database.resources)) {
-      if (resource.name === 'fulltext_indexes') continue;
+      if (resource.name === 'plg_fulltext_indexes') continue;
       
       this.installResourceHooks(resource);
     }
@@ -126,7 +126,7 @@ export class FullTextPlugin extends Plugin {
       this.database._previousCreateResourceForFullText = this.database.createResource;
       this.database.createResource = async function (...args) {
         const resource = await this._previousCreateResourceForFullText(...args);
-        if (this.plugins?.fulltext && resource.name !== 'fulltext_indexes') {
+        if (this.plugins?.fulltext && resource.name !== 'plg_fulltext_indexes') {
           this.plugins.fulltext.installResourceHooks(resource);
         }
         return resource;
@@ -136,7 +136,7 @@ export class FullTextPlugin extends Plugin {
 
     // Ensure all existing resources have hooks (even if created before plugin setup)
     for (const resource of Object.values(this.database.resources)) {
-      if (resource.name !== 'fulltext_indexes') {
+      if (resource.name !== 'plg_fulltext_indexes') {
         this.installResourceHooks(resource);
       }
     }
@@ -460,7 +460,7 @@ export class FullTextPlugin extends Plugin {
   }
 
   async _rebuildAllIndexesInternal() {
-    const resourceNames = Object.keys(this.database.resources).filter(name => name !== 'fulltext_indexes');
+    const resourceNames = Object.keys(this.database.resources).filter(name => name !== 'plg_fulltext_indexes');
     
     // Process resources sequentially to avoid overwhelming the system
     for (const resourceName of resourceNames) {

package/src/plugins/index.js

@@ -12,5 +12,6 @@ export * from './fulltext.plugin.js'
 export * from './metrics.plugin.js'
 export * from './queue-consumer.plugin.js'
 export * from './replicator.plugin.js'
+export * from './s3-queue.plugin.js'
 export * from './scheduler.plugin.js'
 export * from './state-machine.plugin.js'

package/src/plugins/metrics.plugin.js

@@ -37,7 +37,7 @@ export class MetricsPlugin extends Plugin {
 
     const [ok, err] = await tryFn(async () => {
       const [ok1, err1, metricsResource] = await tryFn(() => database.createResource({
-        name: 'metrics',
+        name: 'plg_metrics',
         attributes: {
           id: 'string|required',
           type: 'string|required', // 'operation', 'error', 'performance'
@@ -51,10 +51,10 @@ export class MetricsPlugin extends Plugin {
           metadata: 'json'
         }
       }));
-      this.metricsResource = ok1 ? metricsResource : database.resources.metrics;
+      this.metricsResource = ok1 ? metricsResource : database.resources.plg_metrics;
 
       const [ok2, err2, errorsResource] = await tryFn(() => database.createResource({
-        name: 'error_logs',
+        name: 'plg_error_logs',
         attributes: {
           id: 'string|required',
           resourceName: 'string|required',
@@ -64,10 +64,10 @@ export class MetricsPlugin extends Plugin {
           metadata: 'json'
         }
       }));
-      this.errorsResource = ok2 ? errorsResource : database.resources.error_logs;
+      this.errorsResource = ok2 ? errorsResource : database.resources.plg_error_logs;
 
       const [ok3, err3, performanceResource] = await tryFn(() => database.createResource({
-        name: 'performance_logs',
+        name: 'plg_performance_logs',
         attributes: {
           id: 'string|required',
           resourceName: 'string|required',
@@ -77,13 +77,13 @@ export class MetricsPlugin extends Plugin {
           metadata: 'json'
         }
       }));
-      this.performanceResource = ok3 ? performanceResource : database.resources.performance_logs;
+      this.performanceResource = ok3 ? performanceResource : database.resources.plg_performance_logs;
     });
     if (!ok) {
       // Resources might already exist
-      this.metricsResource = database.resources.metrics;
-      this.errorsResource = database.resources.error_logs;
-      this.performanceResource = database.resources.performance_logs;
+      this.metricsResource = database.resources.plg_metrics;
+      this.errorsResource = database.resources.plg_error_logs;
+      this.performanceResource = database.resources.plg_performance_logs;
     }
 
     // Use database hooks for automatic resource discovery
@@ -116,7 +116,7 @@ export class MetricsPlugin extends Plugin {
   installDatabaseHooks() {
     // Use the new database hooks system for automatic resource discovery
     this.database.addHook('afterCreateResource', (resource) => {
-      if (resource.name !== 'metrics' && resource.name !== 'error_logs' && resource.name !== 'performance_logs') {
+      if (resource.name !== 'plg_metrics' && resource.name !== 'plg_error_logs' && resource.name !== 'plg_performance_logs') {
         this.installResourceHooks(resource);
       }
     });
@@ -130,10 +130,10 @@ export class MetricsPlugin extends Plugin {
   installMetricsHooks() {
     // Only hook into non-metrics resources
     for (const resource of Object.values(this.database.resources)) {
-      if (['metrics', 'error_logs', 'performance_logs'].includes(resource.name)) {
+      if (['plg_metrics', 'plg_error_logs', 'plg_performance_logs'].includes(resource.name)) {
         continue; // Skip metrics resources to avoid recursion
       }
-      
+
       this.installResourceHooks(resource);
     }
 
@@ -141,7 +141,7 @@ export class MetricsPlugin extends Plugin {
     this.database._createResource = this.database.createResource;
     this.database.createResource = async function (...args) {
       const resource = await this._createResource(...args);
-      if (this.plugins?.metrics && !['metrics', 'error_logs', 'performance_logs'].includes(resource.name)) {
+      if (this.plugins?.metrics && !['plg_metrics', 'plg_error_logs', 'plg_performance_logs'].includes(resource.name)) {
         this.plugins.metrics.installResourceHooks(resource);
       }
       return resource;

package/src/plugins/queue-consumer.plugin.js

@@ -20,7 +20,7 @@ import tryFn from "../concerns/try-fn.js";
 //   reconnectInterval: 2000,
 // });
 
-export default class QueueConsumerPlugin {
+export class QueueConsumerPlugin {
   constructor(options = {}) {
     this.options = options;
     // New pattern: consumers = [{ driver, config, consumers: [{ queueUrl, resources, ... }] }]
@@ -131,4 +131,6 @@ export default class QueueConsumerPlugin {
 
   _handleError(err, raw, resourceName) {
   }
-} 
+}
+
+export default QueueConsumerPlugin;

package/src/plugins/replicator.plugin.js

@@ -17,7 +17,7 @@ function normalizeResourceName(name) {
  *     If true, the plugin will persist all replicator events to a log resource.
  *     If false, no replicator log resource is created or used.
  *
- * - replicatorLogResource (string, default: 'replicator_logs')
+ * - replicatorLogResource (string, default: 'plg_replicator_logs')
  *     The name of the resource used to store replicator logs.
  *
  * === replicator Log Resource Structure ===
@@ -132,24 +132,24 @@ export class ReplicatorPlugin extends Plugin {
       replicators: options.replicators || [],
       logErrors: options.logErrors !== false,
       replicatorLogResource: options.replicatorLogResource || 'replicator_log',
+      persistReplicatorLog: options.persistReplicatorLog || false,
       enabled: options.enabled !== false,
       batchSize: options.batchSize || 100,
       maxRetries: options.maxRetries || 3,
       timeout: options.timeout || 30000,
-      verbose: options.verbose || false,
-      ...options
+      verbose: options.verbose || false
     };
-    
+
     this.replicators = [];
     this.database = null;
     this.eventListenersInstalled = new Set();
-  }
-
-  /**
-   * Decompress data if it was compressed
-   */
-  async decompressData(data) {
-    return data;
+    this.eventHandlers = new Map(); // Map<resourceName, {insert, update, delete}>
+    this.stats = {
+      totalReplications: 0,
+      totalErrors: 0,
+      lastSync: null
+    };
+    this._afterCreateResourceHook = null;
   }
 
   // Helper to filter out internal S3DB fields
@@ -172,54 +172,67 @@ export class ReplicatorPlugin extends Plugin {
   }
 
   installEventListeners(resource, database, plugin) {
-    if (!resource || this.eventListenersInstalled.has(resource.name) || 
+    if (!resource || this.eventListenersInstalled.has(resource.name) ||
         resource.name === this.config.replicatorLogResource) {
       return;
     }
 
-    resource.on('insert', async (data) => {
+    // Create handler functions and save references for later removal
+    const insertHandler = async (data) => {
       const [ok, error] = await tryFn(async () => {
         const completeData = { ...data, createdAt: new Date().toISOString() };
         await plugin.processReplicatorEvent('insert', resource.name, completeData.id, completeData);
       });
-      
+
       if (!ok) {
         if (this.config.verbose) {
           console.warn(`[ReplicatorPlugin] Insert event failed for resource ${resource.name}: ${error.message}`);
         }
         this.emit('error', { operation: 'insert', error: error.message, resource: resource.name });
       }
-    });
+    };
 
-    resource.on('update', async (data, beforeData) => {
+    const updateHandler = async (data, beforeData) => {
       const [ok, error] = await tryFn(async () => {
         // For updates, we need to get the complete updated record, not just the changed fields
         const completeData = await plugin.getCompleteData(resource, data);
         const dataWithTimestamp = { ...completeData, updatedAt: new Date().toISOString() };
         await plugin.processReplicatorEvent('update', resource.name, completeData.id, dataWithTimestamp, beforeData);
       });
-      
+
       if (!ok) {
         if (this.config.verbose) {
           console.warn(`[ReplicatorPlugin] Update event failed for resource ${resource.name}: ${error.message}`);
         }
         this.emit('error', { operation: 'update', error: error.message, resource: resource.name });
       }
-    });
+    };
 
-    resource.on('delete', async (data) => {
+    const deleteHandler = async (data) => {
       const [ok, error] = await tryFn(async () => {
         await plugin.processReplicatorEvent('delete', resource.name, data.id, data);
       });
-      
+
       if (!ok) {
         if (this.config.verbose) {
           console.warn(`[ReplicatorPlugin] Delete event failed for resource ${resource.name}: ${error.message}`);
         }
         this.emit('error', { operation: 'delete', error: error.message, resource: resource.name });
       }
+    };
+
+    // Save handler references
+    this.eventHandlers.set(resource.name, {
+      insert: insertHandler,
+      update: updateHandler,
+      delete: deleteHandler
     });
 
+    // Attach listeners
+    resource.on('insert', insertHandler);
+    resource.on('update', updateHandler);
+    resource.on('delete', deleteHandler);
+
     this.eventListenersInstalled.add(resource.name);
   }
 
@@ -229,7 +242,7 @@ export class ReplicatorPlugin extends Plugin {
     // Create replicator log resource if enabled
     if (this.config.persistReplicatorLog) {
       const [ok, err, logResource] = await tryFn(() => database.createResource({
-        name: this.config.replicatorLogResource || 'replicator_logs',
+        name: this.config.replicatorLogResource || 'plg_replicator_logs',
         attributes: {
           id: 'string|required',
           resource: 'string|required',
@@ -244,7 +257,7 @@ export class ReplicatorPlugin extends Plugin {
       if (ok) {
         this.replicatorLogResource = logResource;
       } else {
-        this.replicatorLogResource = database.resources[this.config.replicatorLogResource || 'replicator_logs'];
+        this.replicatorLogResource = database.resources[this.config.replicatorLogResource || 'plg_replicator_logs'];
       }
     }
 
@@ -256,7 +269,7 @@ export class ReplicatorPlugin extends Plugin {
     
     // Install event listeners for existing resources
     for (const resource of Object.values(database.resources)) {
-      if (resource.name !== (this.config.replicatorLogResource || 'replicator_logs')) {
+      if (resource.name !== (this.config.replicatorLogResource || 'plg_replicator_logs')) {
         this.installEventListeners(resource, database, this);
       }
     }
@@ -279,17 +292,22 @@ export class ReplicatorPlugin extends Plugin {
   }
 
   installDatabaseHooks() {
-    // Use the new database hooks system for automatic resource discovery
-    this.database.addHook('afterCreateResource', (resource) => {
-      if (resource.name !== (this.config.replicatorLogResource || 'replicator_logs')) {
+    // Store hook reference for later removal
+    this._afterCreateResourceHook = (resource) => {
+      if (resource.name !== (this.config.replicatorLogResource || 'plg_replicator_logs')) {
         this.installEventListeners(resource, this.database, this);
       }
-    });
+    };
+
+    this.database.addHook('afterCreateResource', this._afterCreateResourceHook);
   }
 
   removeDatabaseHooks() {
-    // Remove the hook we added
-    this.database.removeHook('afterCreateResource', this.installEventListeners.bind(this));
+    // Remove the hook we added using stored reference
+    if (this._afterCreateResourceHook) {
+      this.database.removeHook('afterCreateResource', this._afterCreateResourceHook);
+      this._afterCreateResourceHook = null;
+    }
   }
 
   createReplicator(driver, config, resources, client) {
@@ -324,16 +342,16 @@ export class ReplicatorPlugin extends Plugin {
   async retryWithBackoff(operation, maxRetries = 3) {
     let lastError;
     for (let attempt = 1; attempt <= maxRetries; attempt++) {
-      const [ok, error] = await tryFn(operation);
-      
+      const [ok, error, result] = await tryFn(operation);
+
       if (ok) {
-        return ok;
+        return result;
       } else {
         lastError = error;
         if (this.config.verbose) {
           console.warn(`[ReplicatorPlugin] Retry attempt ${attempt}/${maxRetries} failed: ${error.message}`);
         }
-        
+
         if (attempt === maxRetries) {
           throw error;
         }
@@ -438,7 +456,7 @@ export class ReplicatorPlugin extends Plugin {
     return Promise.allSettled(promises);
   }
 
-  async processreplicatorItem(item) {
+  async processReplicatorItem(item) {
     const applicableReplicators = this.replicators.filter(replicator => {
       const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(item.resourceName, item.operation);
       return should;
@@ -512,14 +530,10 @@ export class ReplicatorPlugin extends Plugin {
     return Promise.allSettled(promises);
   }
 
-  async logreplicator(item) {
-            // Always use the saved reference
+  async logReplicator(item) {
+    // Always use the saved reference
     const logRes = this.replicatorLog || this.database.resources[normalizeResourceName(this.config.replicatorLogResource)];
     if (!logRes) {
-      if (this.database) {
-        if (this.database.options && this.database.options.connectionString) {
-        }
-      }
       this.emit('replicator.log.failed', { error: 'replicator log resource not found', item });
       return;
     }
@@ -544,7 +558,7 @@ export class ReplicatorPlugin extends Plugin {
     }
   }
 
-  async updatereplicatorLog(logId, updates) {
+  async updateReplicatorLog(logId, updates) {
     if (!this.replicatorLog) return;
 
     const [ok, err] = await tryFn(async () => {
@@ -559,7 +573,7 @@ export class ReplicatorPlugin extends Plugin {
   }
 
   // Utility methods
-  async getreplicatorStats() {
+  async getReplicatorStats() {
     const replicatorStats = await Promise.all(
       this.replicators.map(async (replicator) => {
         const status = await replicator.getStatus();
@@ -574,16 +588,12 @@ export class ReplicatorPlugin extends Plugin {
 
     return {
       replicators: replicatorStats,
-      queue: {
-        length: this.queue.length,
-        isProcessing: this.isProcessing
-      },
       stats: this.stats,
       lastSync: this.stats.lastSync
     };
   }
 
-  async getreplicatorLogs(options = {}) {
+  async getReplicatorLogs(options = {}) {
     if (!this.replicatorLog) {
       return [];
     }
@@ -596,43 +606,42 @@ export class ReplicatorPlugin extends Plugin {
       offset = 0
     } = options;
 
-    let query = {};
-    
+    const filter = {};
+
     if (resourceName) {
-      query.resourceName = resourceName;
+      filter.resourceName = resourceName;
     }
-    
+
     if (operation) {
-      query.operation = operation;
+      filter.operation = operation;
     }
-    
+
     if (status) {
-      query.status = status;
+      filter.status = status;
     }
 
-    const logs = await this.replicatorLog.list(query);
-    
-    // Apply pagination
-    return logs.slice(offset, offset + limit);
+    const logs = await this.replicatorLog.query(filter, { limit, offset });
+
+    return logs || [];
   }
 
-  async retryFailedreplicators() {
+  async retryFailedReplicators() {
     if (!this.replicatorLog) {
       return { retried: 0 };
     }
 
-    const failedLogs = await this.replicatorLog.list({
+    const failedLogs = await this.replicatorLog.query({
       status: 'failed'
     });
 
     let retried = 0;
-    
-    for (const log of failedLogs) {
+
+    for (const log of failedLogs || []) {
       const [ok, err] = await tryFn(async () => {
         // Re-queue the replicator
         await this.processReplicatorEvent(
-          log.resourceName,
           log.operation,
+          log.resourceName,
           log.recordId,
           log.data
         );
@@ -656,16 +665,30 @@ export class ReplicatorPlugin extends Plugin {
     this.stats.lastSync = new Date().toISOString();
 
     for (const resourceName in this.database.resources) {
-      if (normalizeResourceName(resourceName) === normalizeResourceName('replicator_logs')) continue;
+      if (normalizeResourceName(resourceName) === normalizeResourceName('plg_replicator_logs')) continue;
 
       if (replicator.shouldReplicateResource(resourceName)) {
         this.emit('replicator.sync.resource', { resourceName, replicatorId });
-        
+
         const resource = this.database.resources[resourceName];
-      const allRecords = await resource.getAll();
-      
-      for (const record of allRecords) {
-          await replicator.replicate(resourceName, 'insert', record, record.id);
+
+        // Use pagination to avoid memory issues
+        let offset = 0;
+        const pageSize = this.config.batchSize || 100;
+
+        while (true) {
+          const [ok, err, page] = await tryFn(() => resource.page({ offset, size: pageSize }));
+
+          if (!ok || !page) break;
+
+          const records = Array.isArray(page) ? page : (page.items || []);
+          if (records.length === 0) break;
+
+          for (const record of records) {
+            await replicator.replicate(resourceName, 'insert', record, record.id);
+          }
+
+          offset += pageSize;
         }
       }
     }
@@ -698,10 +721,25 @@ export class ReplicatorPlugin extends Plugin {
         await Promise.allSettled(cleanupPromises);
       }
       
+      // Remove event listeners from resources to prevent memory leaks
+      if (this.database && this.database.resources) {
+        for (const resourceName of this.eventListenersInstalled) {
+          const resource = this.database.resources[resourceName];
+          const handlers = this.eventHandlers.get(resourceName);
+
+          if (resource && handlers) {
+            resource.off('insert', handlers.insert);
+            resource.off('update', handlers.update);
+            resource.off('delete', handlers.delete);
+          }
+        }
+      }
+
       this.replicators = [];
       this.database = null;
       this.eventListenersInstalled.clear();
-      
+      this.eventHandlers.clear();
+
       this.removeAllListeners();
     });
     

package/src/plugins/replicators/s3db-replicator.class.js

@@ -328,17 +328,21 @@ class S3dbReplicator extends BaseReplicator {
   }
 
   _getDestResourceObj(resource) {
-    const available = Object.keys(this.client.resources || {});
+    const db = this.targetDatabase || this.client;
+    const available = Object.keys(db.resources || {});
     const norm = normalizeResourceName(resource);
     const found = available.find(r => normalizeResourceName(r) === norm);
     if (!found) {
       throw new Error(`[S3dbReplicator] Destination resource not found: ${resource}. Available: ${available.join(', ')}`);
     }
-    return this.client.resources[found];
+    return db.resources[found];
   }
 
   async replicateBatch(resourceName, records) {
-    if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
+    if (this.enabled === false) {
+      return { skipped: true, reason: 'replicator_disabled' };
+    }
+    if (!this.shouldReplicateResource(resourceName)) {
       return { skipped: true, reason: 'resource_not_included' };
     }
 

package/src/plugins/replicators/sqs-replicator.class.js

@@ -32,11 +32,13 @@ class SqsReplicator extends BaseReplicator {
     this.client = client;
     this.queueUrl = config.queueUrl;
     this.queues = config.queues || {};
-    this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault;
+    // Support legacy names but prefer defaultQueue
+    this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault || null;
     this.region = config.region || 'us-east-1';
     this.sqsClient = client || null;
     this.messageGroupId = config.messageGroupId;
     this.deduplicationId = config.deduplicationId;
+    this.resourceQueueMap = config.resourceQueueMap || null;
     
     // Normalize resources to object format
     if (Array.isArray(resources)) {
@@ -188,7 +190,10 @@ class SqsReplicator extends BaseReplicator {
   }
 
   async replicate(resource, operation, data, id, beforeData = null) {
-    if (!this.enabled || !this.shouldReplicateResource(resource)) {
+    if (this.enabled === false) {
+      return { skipped: true, reason: 'replicator_disabled' };
+    }
+    if (!this.shouldReplicateResource(resource)) {
       return { skipped: true, reason: 'resource_not_included' };
     }
     const [ok, err, result] = await tryFn(async () => {
@@ -234,7 +239,10 @@ class SqsReplicator extends BaseReplicator {
   }
 
   async replicateBatch(resource, records) {
-    if (!this.enabled || !this.shouldReplicateResource(resource)) {
+    if (this.enabled === false) {
+      return { skipped: true, reason: 'replicator_disabled' };
+    }
+    if (!this.shouldReplicateResource(resource)) {
       return { skipped: true, reason: 'resource_not_included' };
     }
     const [ok, err, result] = await tryFn(async () => {