s3db.js 13.6.1 → 14.0.2

package/src/plugins/replicators/base-replicator.class.js

@@ -1,4 +1,5 @@
 import EventEmitter from 'events';
+import { PromisePool } from '@supercharge/promise-pool';
 import { ReplicationError } from '../replicator.errors.js';
 
 /**
@@ -11,6 +12,7 @@ export class BaseReplicator extends EventEmitter {
     this.config = config;
     this.name = this.constructor.name;
     this.enabled = config.enabled !== false; // Default to enabled unless explicitly disabled
+    this.batchConcurrency = Math.max(1, config.batchConcurrency ?? 5);
   }
 
   /**
@@ -89,6 +91,92 @@ export class BaseReplicator extends EventEmitter {
     this.emit('cleanup', { replicator: this.name });
   }
 
+  /**
+   * Update the default batch concurrency at runtime
+   * @param {number} value
+   */
+  setBatchConcurrency(value) {
+    if (!Number.isFinite(value) || value <= 0) {
+      throw new ReplicationError('Batch concurrency must be a positive number', {
+        operation: 'setBatchConcurrency',
+        replicatorClass: this.name,
+        providedValue: value,
+        suggestion: 'Provide a positive integer value greater than zero.'
+      });
+    }
+    this.batchConcurrency = Math.floor(value);
+  }
+
+  /**
+   * Generic helper to process batches with controlled concurrency
+   * @param {Array} records - Items to process
+   * @param {Function} handler - Async handler executed for each record
+   * @param {Object} options
+   * @param {number} [options.concurrency] - Concurrency override
+  * @param {Function} [options.mapError] - Maps thrown errors before collection
+   * @returns {Promise<{results: Array, errors: Array}>}
+   */
+  async processBatch(records = [], handler, { concurrency, mapError } = {}) {
+    if (!Array.isArray(records) || records.length === 0) {
+      return { results: [], errors: [] };
+    }
+
+    if (typeof handler !== 'function') {
+      throw new ReplicationError('processBatch requires an async handler function', {
+        operation: 'processBatch',
+        replicatorClass: this.name,
+        suggestion: 'Provide an async handler: async (record) => { ... }'
+      });
+    }
+
+    const limit = Math.max(1, concurrency ?? this.batchConcurrency ?? 5);
+    const results = [];
+    const errors = [];
+
+    await PromisePool
+      .withConcurrency(limit)
+      .for(records)
+      .process(async record => {
+        try {
+          const result = await handler(record);
+          results.push(result);
+        } catch (error) {
+          if (typeof mapError === 'function') {
+            const mapped = mapError(error, record);
+            if (mapped !== undefined) {
+              errors.push(mapped);
+            }
+          } else {
+            errors.push({ record, error });
+          }
+        }
+      });
+
+    return { results, errors };
+  }
+
+  /**
+   * Helper to build replication errors with contextual metadata
+   * @param {string} message
+   * @param {Object} details
+   * @returns {ReplicationError}
+   */
+  createError(message, details = {}) {
+    return new ReplicationError(message, {
+      replicatorClass: this.name,
+      operation: details.operation || 'unknown',
+      resourceName: details.resourceName,
+      statusCode: details.statusCode ?? 500,
+      retriable: details.retriable ?? false,
+      suggestion: details.suggestion,
+      description: details.description,
+      docs: details.docs,
+      hint: details.hint,
+      metadata: details.metadata,
+      ...details
+    });
+  }
+
   /**
    * Validate replicator configuration
    * @returns {Object} Validation result
@@ -98,4 +186,4 @@ export class BaseReplicator extends EventEmitter {
   }
 }
 
-export default BaseReplicator; 
+export default BaseReplicator; 

package/src/plugins/replicators/bigquery-replicator.class.js

@@ -87,7 +87,12 @@ class BigqueryReplicator extends BaseReplicator {
   _validateMutability(mutability) {
     const validModes = ['append-only', 'mutable', 'immutable'];
     if (!validModes.includes(mutability)) {
-      throw new Error(`Invalid mutability mode: ${mutability}. Must be one of: ${validModes.join(', ')}`);
+      throw this.createError(`Invalid mutability mode: ${mutability}`, {
+        operation: 'config',
+        statusCode: 400,
+        retriable: false,
+        suggestion: `Use one of the supported mutability modes: ${validModes.join(', ')}.`
+      });
     }
   }
 
@@ -213,11 +218,12 @@ class BigqueryReplicator extends BaseReplicator {
         continue;
       }
 
-      const allAttributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+      // Use $schema for reliable access to resource definition
+      const allAttributes = resource.$schema.attributes || {};
 
       // Filter out plugin attributes - they are internal and should not be replicated
-      const pluginAttrNames = resource.schema?._pluginAttributes
-        ? Object.values(resource.schema._pluginAttributes).flat()
+      const pluginAttrNames = resource.$schema._pluginAttributes
+        ? Object.values(resource.$schema._pluginAttributes).flat()
         : [];
       const attributes = Object.fromEntries(
         Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
@@ -235,7 +241,15 @@ class BigqueryReplicator extends BaseReplicator {
           const message = `Schema sync failed for table ${tableName}: ${errSync.message}`;
 
           if (this.schemaSync.onMismatch === 'error') {
-            throw new Error(message);
+            throw this.createError(message, {
+              operation: 'schemaSync',
+              resourceName,
+              tableName,
+              statusCode: 409,
+              retriable: errSync?.retriable ?? false,
+              suggestion: 'Review the BigQuery table schema and align it with the S3DB resource definition or relax schemaSync.onMismatch.',
+              docs: 'docs/plugins/replicator.md'
+            });
           } else if (this.schemaSync.onMismatch === 'warn') {
             console.warn(`[BigQueryReplicator] ${message}`);
           }
@@ -261,11 +275,23 @@ class BigqueryReplicator extends BaseReplicator {
 
     if (!exists) {
       if (!this.schemaSync.autoCreateTable) {
-        throw new Error(`Table ${tableName} does not exist and autoCreateTable is disabled`);
+        throw this.createError(`Table ${tableName} does not exist and autoCreateTable is disabled`, {
+          operation: 'schemaSync',
+          tableName,
+          statusCode: 404,
+          retriable: false,
+          suggestion: 'Create the BigQuery table manually or enable schemaSync.autoCreateTable.'
+        });
       }
 
       if (this.schemaSync.strategy === 'validate-only') {
-        throw new Error(`Table ${tableName} does not exist (validate-only mode)`);
+        throw this.createError(`Table ${tableName} does not exist (validate-only mode)`, {
+          operation: 'schemaSync',
+          tableName,
+          statusCode: 404,
+          retriable: false,
+          suggestion: 'Provision the table before running validate-only checks or switch the schemaSync.strategy to alter.'
+        });
       }
 
       // Create table with schema (including tracking fields based on mutability)
@@ -339,7 +365,13 @@ class BigqueryReplicator extends BaseReplicator {
       const newFields = generateBigQuerySchemaUpdate(attributes, existingSchema, mutability);
 
       if (newFields.length > 0) {
-        throw new Error(`Table ${tableName} schema mismatch. Missing columns: ${newFields.length}`);
+        throw this.createError(`Table ${tableName} schema mismatch. Missing columns: ${newFields.length}`, {
+          operation: 'schemaValidation',
+          tableName,
+          statusCode: 409,
+          retriable: false,
+          suggestion: 'Update the BigQuery table schema to include the missing columns or enable schemaSync.autoCreateColumns.'
+        });
       }
     }
   }
@@ -555,7 +587,14 @@ class BigqueryReplicator extends BaseReplicator {
               throw error;
             }
           } else {
-            throw new Error(`Unsupported operation: ${operation}`);
+            throw this.createError(`Unsupported operation: ${operation}`, {
+              operation: 'replicate',
+              resourceName,
+              tableName: tableConfig.table,
+              statusCode: 400,
+              retriable: false,
+              suggestion: 'Replicator supports insert, update, or delete actions. Adjust the resources configuration accordingly.'
+            });
           }
 
           results.push({
@@ -634,26 +673,31 @@ class BigqueryReplicator extends BaseReplicator {
   }
 
   async replicateBatch(resourceName, records) {
-    const results = [];
-    const errors = [];
-
-    for (const record of records) {
-      const [ok, err, res] = await tryFn(() => this.replicate(
+    const { results, errors } = await this.processBatch(
+      records,
+      async (record) => {
+        const [ok, err, res] = await tryFn(() => this.replicate(
         resourceName,
         record.operation,
         record.data,
         record.id,
         record.beforeData
       ));
-      if (ok) {
-        results.push(res);
-      } else {
-        if (this.config.verbose) {
-          console.warn(`[BigqueryReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
+        if (!ok) {
+          throw err;
+        }
+        return res;
+      },
+      {
+        concurrency: this.config.batchConcurrency,
+        mapError: (error, record) => {
+          if (this.config.verbose) {
+            console.warn(`[BigqueryReplicator] Batch replication failed for record ${record.id}: ${error.message}`);
+          }
+          return { id: record.id, error: error.message };
         }
-        errors.push({ id: record.id, error: err.message });
       }
-    }
+    );
 
     // Log errors if any occurred during batch processing
     if (errors.length > 0) {
@@ -699,4 +743,4 @@ class BigqueryReplicator extends BaseReplicator {
   }
 }
 
-export default BigqueryReplicator; 
+export default BigqueryReplicator; 

package/src/plugins/replicators/dynamodb-replicator.class.js

@@ -315,22 +315,24 @@ class DynamoDBReplicator extends BaseReplicator {
   }
 
   async replicateBatch(resourceName, records) {
-    const results = [];
-    const errors = [];
-
-    // DynamoDB batch operations (up to 25 items)
-    // For now, process sequentially (can be optimized with BatchWriteItem)
-    for (const record of records) {
-      const [ok, err, result] = await tryFn(() =>
-        this.replicate(resourceName, record.operation, record.data, record.id)
-      );
+    const { results, errors } = await this.processBatch(
+      records,
+      async (record) => {
+        const [ok, err, result] = await tryFn(() =>
+          this.replicate(resourceName, record.operation, record.data, record.id)
+        );
+
+        if (!ok) {
+          throw err;
+        }
 
-      if (ok) {
-        results.push(result);
-      } else {
-        errors.push({ id: record.id, error: err.message });
+        return result;
+      },
+      {
+        concurrency: this.config.batchConcurrency,
+        mapError: (error, record) => ({ id: record.id, error: error.message })
       }
-    }
+    );
 
     return {
       success: errors.length === 0,
@@ -343,7 +345,12 @@ class DynamoDBReplicator extends BaseReplicator {
   async testConnection() {
     const [ok, err] = await tryFn(async () => {
       if (!this.client) {
-        throw new Error('Client not initialized');
+        throw this.createError('Client not initialized', {
+          operation: 'testConnection',
+          statusCode: 503,
+          retriable: true,
+          suggestion: 'Call initialize() before testing connectivity or ensure the DynamoDB client was created successfully.'
+        });
       }
 
       const { ListTablesCommand } = requirePluginDependency('@aws-sdk/client-dynamodb', 'DynamoDBReplicator');

package/src/plugins/replicators/mongodb-replicator.class.js

@@ -324,22 +324,24 @@ class MongoDBReplicator extends BaseReplicator {
   }
 
   async replicateBatch(resourceName, records) {
-    const results = [];
-    const errors = [];
-
-    // MongoDB supports bulk operations, but for consistency with other replicators
-    // and error handling, we process sequentially
-    for (const record of records) {
-      const [ok, err, result] = await tryFn(() =>
-        this.replicate(resourceName, record.operation, record.data, record.id)
-      );
+    const { results, errors } = await this.processBatch(
+      records,
+      async (record) => {
+        const [ok, err, result] = await tryFn(() =>
+          this.replicate(resourceName, record.operation, record.data, record.id)
+        );
+
+        if (!ok) {
+          throw err;
+        }
 
-      if (ok) {
-        results.push(result);
-      } else {
-        errors.push({ id: record.id, error: err.message });
+        return result;
+      },
+      {
+        concurrency: this.config.batchConcurrency,
+        mapError: (error, record) => ({ id: record.id, error: error.message })
       }
-    }
+    );
 
     return {
       success: errors.length === 0,
@@ -352,7 +354,12 @@ class MongoDBReplicator extends BaseReplicator {
   async testConnection() {
     const [ok, err] = await tryFn(async () => {
       if (!this.client) {
-        throw new Error('Client not initialized');
+        throw this.createError('Client not initialized', {
+          operation: 'testConnection',
+          statusCode: 503,
+          retriable: true,
+          suggestion: 'Ensure initialize() was called and the MongoDB client connected before testing connectivity.'
+        });
       }
 
       await this.db.admin().ping();

package/src/plugins/replicators/mysql-replicator.class.js

@@ -242,7 +242,15 @@ class MySQLReplicator extends BaseReplicator {
           const message = `Schema sync failed for table ${tableName}: ${errSync.message}`;
 
           if (this.schemaSync.onMismatch === 'error') {
-            throw new Error(message);
+            throw this.createError(message, {
+              operation: 'schemaSync',
+              resourceName,
+              tableName,
+              statusCode: 409,
+              retriable: errSync?.retriable ?? false,
+              suggestion: 'Update the MySQL table schema to match the resource definition or adjust schemaSync.onMismatch.',
+              docs: 'docs/plugins/replicator.md'
+            });
           } else if (this.schemaSync.onMismatch === 'warn') {
             console.warn(`[MySQLReplicator] ${message}`);
           }
@@ -268,11 +276,23 @@ class MySQLReplicator extends BaseReplicator {
 
       if (!existingSchema) {
         if (!this.schemaSync.autoCreateTable) {
-          throw new Error(`Table ${tableName} does not exist and autoCreateTable is disabled`);
+          throw this.createError(`Table ${tableName} does not exist and autoCreateTable is disabled`, {
+            operation: 'schemaSync',
+            tableName,
+            statusCode: 404,
+            retriable: false,
+            suggestion: 'Provision the table manually or enable schemaSync.autoCreateTable.'
+          });
         }
 
         if (this.schemaSync.strategy === 'validate-only') {
-          throw new Error(`Table ${tableName} does not exist (validate-only mode)`);
+          throw this.createError(`Table ${tableName} does not exist (validate-only mode)`, {
+            operation: 'schemaSync',
+            tableName,
+            statusCode: 404,
+            retriable: false,
+            suggestion: 'Create the table before running validate-only checks or choose the alter strategy.'
+          });
         }
 
         // Create table
@@ -336,7 +356,13 @@ class MySQLReplicator extends BaseReplicator {
         const alterStatements = generateMySQLAlterTable(tableName, attributes, existingSchema);
 
         if (alterStatements.length > 0) {
-          throw new Error(`Table ${tableName} schema mismatch. Missing columns: ${alterStatements.length}`);
+          throw this.createError(`Table ${tableName} schema mismatch. Missing columns: ${alterStatements.length}`, {
+            operation: 'schemaValidation',
+            tableName,
+            statusCode: 409,
+            retriable: false,
+            suggestion: 'Add the missing columns to the MySQL table or enable schemaSync.autoCreateColumns.'
+          });
         }
       }
     } finally {
@@ -498,20 +524,24 @@ class MySQLReplicator extends BaseReplicator {
   }
 
   async replicateBatch(resourceName, records) {
-    const results = [];
-    const errors = [];
-
-    for (const record of records) {
-      const [ok, err, result] = await tryFn(() =>
-        this.replicate(resourceName, record.operation, record.data, record.id)
-      );
+    const { results, errors } = await this.processBatch(
+      records,
+      async (record) => {
+        const [ok, err, result] = await tryFn(() =>
+          this.replicate(resourceName, record.operation, record.data, record.id)
+        );
+
+        if (!ok) {
+          throw err;
+        }
 
-      if (ok) {
-        results.push(result);
-      } else {
-        errors.push({ id: record.id, error: err.message });
+        return result;
+      },
+      {
+        concurrency: this.config.batchConcurrency,
+        mapError: (error, record) => ({ id: record.id, error: error.message })
       }
-    }
+    );
 
     return {
       success: errors.length === 0,
@@ -524,7 +554,12 @@ class MySQLReplicator extends BaseReplicator {
   async testConnection() {
     const [ok, err] = await tryFn(async () => {
       if (!this.pool) {
-        throw new Error('Pool not initialized');
+        throw this.createError('Pool not initialized', {
+          operation: 'testConnection',
+          statusCode: 503,
+          retriable: true,
+          suggestion: 'Call initialize() before testing the connection or ensure the pool was not cleaned up prematurely.'
+        });
       }
 
       const connection = await this.pool.promise().getConnection();

package/src/plugins/replicators/planetscale-replicator.class.js

@@ -204,7 +204,15 @@ class PlanetScaleReplicator extends BaseReplicator {
           const message = `Schema sync failed for table ${tableName}: ${errSync.message}`;
 
           if (this.schemaSync.onMismatch === 'error') {
-            throw new Error(message);
+            throw this.createError(message, {
+              operation: 'schemaSync',
+              resourceName,
+              tableName,
+              statusCode: 409,
+              retriable: errSync?.retriable ?? false,
+              suggestion: 'Align the PlanetScale table schema with the resource definition or relax schemaSync.onMismatch.',
+              docs: 'docs/plugins/replicator.md'
+            });
           } else if (this.schemaSync.onMismatch === 'warn') {
             console.warn(`[PlanetScaleReplicator] ${message}`);
           }
@@ -227,11 +235,23 @@ class PlanetScaleReplicator extends BaseReplicator {
 
     if (!existingSchema) {
       if (!this.schemaSync.autoCreateTable) {
-        throw new Error(`Table ${tableName} does not exist and autoCreateTable is disabled`);
+        throw this.createError(`Table ${tableName} does not exist and autoCreateTable is disabled`, {
+          operation: 'schemaSync',
+          tableName,
+          statusCode: 404,
+          retriable: false,
+          suggestion: 'Create the table manually or enable schemaSync.autoCreateTable.'
+        });
       }
 
       if (this.schemaSync.strategy === 'validate-only') {
-        throw new Error(`Table ${tableName} does not exist (validate-only mode)`);
+        throw this.createError(`Table ${tableName} does not exist (validate-only mode)`, {
+          operation: 'schemaSync',
+          tableName,
+          statusCode: 404,
+          retriable: false,
+          suggestion: 'Provision the table before running validate-only checks or choose the alter strategy.'
+        });
       }
 
       // Create table
@@ -295,7 +315,13 @@ class PlanetScaleReplicator extends BaseReplicator {
       const alterStatements = generateMySQLAlterTable(tableName, attributes, existingSchema);
 
       if (alterStatements.length > 0) {
-        throw new Error(`Table ${tableName} schema mismatch. Missing columns: ${alterStatements.length}`);
+        throw this.createError(`Table ${tableName} schema mismatch. Missing columns: ${alterStatements.length}`, {
+          operation: 'schemaValidation',
+          tableName,
+          statusCode: 409,
+          retriable: false,
+          suggestion: 'Add the missing columns to the PlanetScale table or enable schemaSync.autoCreateColumns.'
+        });
       }
     }
   }

package/src/plugins/replicators/postgres-replicator.class.js

@@ -247,7 +247,15 @@ class PostgresReplicator extends BaseReplicator {
           const message = `Schema sync failed for table ${tableName}: ${errSync.message}`;
 
           if (this.schemaSync.onMismatch === 'error') {
-            throw new Error(message);
+            throw this.createError(message, {
+              operation: 'schemaSync',
+              resourceName,
+              tableName,
+              statusCode: 409,
+              retriable: errSync?.retriable ?? false,
+              suggestion: 'Align the PostgreSQL table schema with the resource definition or relax schemaSync.onMismatch.',
+              docs: 'docs/plugins/replicator.md'
+            });
           } else if (this.schemaSync.onMismatch === 'warn') {
             console.warn(`[PostgresReplicator] ${message}`);
           }
@@ -272,11 +280,23 @@ class PostgresReplicator extends BaseReplicator {
     if (!existingSchema) {
       // Table doesn't exist
       if (!this.schemaSync.autoCreateTable) {
-        throw new Error(`Table ${tableName} does not exist and autoCreateTable is disabled`);
+        throw this.createError(`Table ${tableName} does not exist and autoCreateTable is disabled`, {
+          operation: 'schemaSync',
+          tableName,
+          statusCode: 404,
+          retriable: false,
+          suggestion: 'Create the table manually or enable schemaSync.autoCreateTable to let the replicator provision it.'
+        });
       }
 
       if (this.schemaSync.strategy === 'validate-only') {
-        throw new Error(`Table ${tableName} does not exist (validate-only mode)`);
+        throw this.createError(`Table ${tableName} does not exist (validate-only mode)`, {
+          operation: 'schemaSync',
+          tableName,
+          statusCode: 404,
+          retriable: false,
+          suggestion: 'Provision the destination table before running validate-only schema checks or switch to the alter strategy.'
+        });
       }
 
       // Create table
@@ -343,7 +363,13 @@ class PostgresReplicator extends BaseReplicator {
       const alterStatements = generatePostgresAlterTable(tableName, attributes, existingSchema);
 
       if (alterStatements.length > 0) {
-        throw new Error(`Table ${tableName} schema mismatch. Missing columns: ${alterStatements.length}`);
+        throw this.createError(`Table ${tableName} schema mismatch. Missing columns: ${alterStatements.length}`, {
+          operation: 'schemaValidation',
+          tableName,
+          statusCode: 409,
+          retriable: false,
+          suggestion: 'Update the PostgreSQL table to include the missing columns or allow the replicator to manage schema changes.'
+        });
       }
     }
   }
@@ -416,7 +442,14 @@ class PostgresReplicator extends BaseReplicator {
             const sql = `DELETE FROM ${table} WHERE id=$1 RETURNING *`;
             result = await this.client.query(sql, [id]);
           } else {
-            throw new Error(`Unsupported operation: ${operation}`);
+            throw this.createError(`Unsupported operation: ${operation}`, {
+              operation: 'replicate',
+              resourceName,
+              tableName: table,
+              statusCode: 400,
+              retriable: false,
+              suggestion: 'Use one of the supported actions: insert, update, or delete.'
+            });
           }
 
           results.push({
@@ -484,36 +517,38 @@ class PostgresReplicator extends BaseReplicator {
   }
 
   async replicateBatch(resourceName, records) {
-    const results = [];
-    const errors = [];
-    
-    for (const record of records) {
+    const { results, errors } = await this.processBatch(records, async (record) => {
       const [ok, err, res] = await tryFn(() => this.replicate(
-        resourceName, 
-        record.operation, 
-        record.data, 
-        record.id, 
+        resourceName,
+        record.operation,
+        record.data,
+        record.id,
         record.beforeData
       ));
-      if (ok) {
-        results.push(res);
-      } else {
+
+      if (!ok) {
+        throw err;
+      }
+
+      return res;
+    }, {
+      concurrency: this.config.batchConcurrency,
+      mapError: (error, record) => {
         if (this.config.verbose) {
-          console.warn(`[PostgresReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
+          console.warn(`[PostgresReplicator] Batch replication failed for record ${record.id}: ${error.message}`);
         }
-        errors.push({ id: record.id, error: err.message });
+        return { id: record.id, error: error.message };
       }
-    }
-    
-    // Log errors if any occurred during batch processing
+    });
+
     if (errors.length > 0) {
       console.warn(`[PostgresReplicator] Batch replication completed with ${errors.length} error(s) for ${resourceName}:`, errors);
     }
-    
-    return { 
-      success: errors.length === 0, 
-      results, 
-      errors 
+
+    return {
+      success: errors.length === 0,
+      results,
+      errors
     };
   }
 
@@ -562,4 +597,4 @@ class PostgresReplicator extends BaseReplicator {
 }
 
 
-export default PostgresReplicator; 
+export default PostgresReplicator;