s3db.js 11.3.2 → 12.0.0

package/src/plugins/replicators/bigquery-replicator.class.js

@@ -1,27 +1,43 @@
 import tryFn from "#src/concerns/try-fn.js";
-
+import requirePluginDependency from "#src/plugins/concerns/plugin-dependencies.js";
 import BaseReplicator from './base-replicator.class.js';
+import {
+  generateBigQuerySchema,
+  getBigQueryTableSchema,
+  generateBigQuerySchemaUpdate
+} from './schema-sync.helper.js';
 
 /**
  * BigQuery Replicator - Replicate data to Google BigQuery tables
- * 
+ *
  * ⚠️  REQUIRED DEPENDENCY: You must install the Google Cloud BigQuery SDK:
  * ```bash
  * pnpm add @google-cloud/bigquery
  * ```
- * 
+ *
  * Configuration:
  * @param {string} projectId - Google Cloud project ID (required)
- * @param {string} datasetId - BigQuery dataset ID (required) 
+ * @param {string} datasetId - BigQuery dataset ID (required)
  * @param {Object} credentials - Service account credentials object (optional)
  * @param {string} location - BigQuery dataset location/region (default: 'US')
  * @param {string} logTable - Table name for operation logging (optional)
- * 
+ * @param {Object} schemaSync - Schema synchronization configuration
+ * @param {boolean} schemaSync.enabled - Enable automatic schema management (default: false)
+ * @param {string} schemaSync.strategy - Sync strategy: 'alter' | 'drop-create' | 'validate-only' (default: 'alter')
+ * @param {string} schemaSync.onMismatch - Action on schema mismatch: 'error' | 'warn' | 'ignore' (default: 'error')
+ * @param {boolean} schemaSync.autoCreateTable - Auto-create table if not exists (default: true)
+ * @param {boolean} schemaSync.autoCreateColumns - Auto-add missing columns (default: true, only with strategy: 'alter')
+ *
  * @example
  * new BigqueryReplicator({
  *   projectId: 'my-gcp-project',
  *   datasetId: 'analytics',
- *   credentials: JSON.parse(Buffer.from(GOOGLE_CREDENTIALS, 'base64').toString())
+ *   credentials: JSON.parse(Buffer.from(GOOGLE_CREDENTIALS, 'base64').toString()),
+ *   schemaSync: {
+ *     enabled: true,
+ *     strategy: 'alter',
+ *     onMismatch: 'error'
+ *   }
  * }, {
  *   users: {
  *     table: 'users_table',
@@ -29,7 +45,7 @@ import BaseReplicator from './base-replicator.class.js';
  *   },
  *   orders: 'orders_table'
  * })
- * 
+ *
  * See PLUGINS.md for comprehensive configuration documentation.
  */
 class BigqueryReplicator extends BaseReplicator {
@@ -42,6 +58,15 @@ class BigqueryReplicator extends BaseReplicator {
     this.location = config.location || 'US';
     this.logTable = config.logTable;
 
+    // Schema sync configuration
+    this.schemaSync = {
+      enabled: config.schemaSync?.enabled || false,
+      strategy: config.schemaSync?.strategy || 'alter',
+      onMismatch: config.schemaSync?.onMismatch || 'error',
+      autoCreateTable: config.schemaSync?.autoCreateTable !== false,
+      autoCreateColumns: config.schemaSync?.autoCreateColumns !== false
+    };
+
     // Parse resources configuration
     this.resources = this.parseResourcesConfig(resources);
   }
@@ -113,6 +138,10 @@ class BigqueryReplicator extends BaseReplicator {
 
   async initialize(database) {
     await super.initialize(database);
+
+    // Validate plugin dependencies are installed
+    await requirePluginDependency('bigquery-replicator');
+
     const [ok, err, sdk] = await tryFn(() => import('@google-cloud/bigquery'));
     if (!ok) {
       if (this.config.verbose) {
@@ -127,6 +156,12 @@ class BigqueryReplicator extends BaseReplicator {
       credentials: this.credentials,
       location: this.location
     });
+
+    // Sync schemas if enabled
+    if (this.schemaSync.enabled) {
+      await this.syncSchemas(database);
+    }
+
     this.emit('initialized', {
       replicator: this.name,
       projectId: this.projectId,
@@ -135,6 +170,142 @@ class BigqueryReplicator extends BaseReplicator {
     });
   }
 
+  /**
+   * Sync table schemas based on S3DB resource definitions
+   */
+  async syncSchemas(database) {
+    for (const [resourceName, tableConfigs] of Object.entries(this.resources)) {
+      const [okRes, errRes, resource] = await tryFn(async () => {
+        return await database.getResource(resourceName);
+      });
+
+      if (!okRes) {
+        if (this.config.verbose) {
+          console.warn(`[BigQueryReplicator] Could not get resource ${resourceName} for schema sync: ${errRes.message}`);
+        }
+        continue;
+      }
+
+      const attributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+
+      for (const tableConfig of tableConfigs) {
+        const tableName = tableConfig.table;
+
+        const [okSync, errSync] = await tryFn(async () => {
+          await this.syncTableSchema(tableName, attributes);
+        });
+
+        if (!okSync) {
+          const message = `Schema sync failed for table ${tableName}: ${errSync.message}`;
+
+          if (this.schemaSync.onMismatch === 'error') {
+            throw new Error(message);
+          } else if (this.schemaSync.onMismatch === 'warn') {
+            console.warn(`[BigQueryReplicator] ${message}`);
+          }
+        }
+      }
+    }
+
+    this.emit('schema_sync_completed', {
+      replicator: this.name,
+      resources: Object.keys(this.resources)
+    });
+  }
+
+  /**
+   * Sync a single table schema in BigQuery
+   */
+  async syncTableSchema(tableName, attributes) {
+    const dataset = this.bigqueryClient.dataset(this.datasetId);
+    const table = dataset.table(tableName);
+
+    // Check if table exists
+    const [exists] = await table.exists();
+
+    if (!exists) {
+      if (!this.schemaSync.autoCreateTable) {
+        throw new Error(`Table ${tableName} does not exist and autoCreateTable is disabled`);
+      }
+
+      if (this.schemaSync.strategy === 'validate-only') {
+        throw new Error(`Table ${tableName} does not exist (validate-only mode)`);
+      }
+
+      // Create table with schema
+      const schema = generateBigQuerySchema(attributes);
+
+      if (this.config.verbose) {
+        console.log(`[BigQueryReplicator] Creating table ${tableName} with schema:`, schema);
+      }
+
+      await dataset.createTable(tableName, { schema });
+
+      this.emit('table_created', {
+        replicator: this.name,
+        tableName,
+        attributes: Object.keys(attributes)
+      });
+
+      return;
+    }
+
+    // Table exists - check for schema changes
+    if (this.schemaSync.strategy === 'drop-create') {
+      if (this.config.verbose) {
+        console.warn(`[BigQueryReplicator] Dropping and recreating table ${tableName}`);
+      }
+
+      await table.delete();
+      const schema = generateBigQuerySchema(attributes);
+      await dataset.createTable(tableName, { schema });
+
+      this.emit('table_recreated', {
+        replicator: this.name,
+        tableName,
+        attributes: Object.keys(attributes)
+      });
+
+      return;
+    }
+
+    if (this.schemaSync.strategy === 'alter' && this.schemaSync.autoCreateColumns) {
+      const existingSchema = await getBigQueryTableSchema(this.bigqueryClient, this.datasetId, tableName);
+      const newFields = generateBigQuerySchemaUpdate(attributes, existingSchema);
+
+      if (newFields.length > 0) {
+        if (this.config.verbose) {
+          console.log(`[BigQueryReplicator] Adding ${newFields.length} field(s) to table ${tableName}:`, newFields);
+        }
+
+        // Get current schema
+        const [metadata] = await table.getMetadata();
+        const currentSchema = metadata.schema.fields;
+
+        // Add new fields to existing schema
+        const updatedSchema = [...currentSchema, ...newFields];
+
+        // Update table schema
+        await table.setMetadata({ schema: updatedSchema });
+
+        this.emit('table_altered', {
+          replicator: this.name,
+          tableName,
+          addedColumns: newFields.length
+        });
+      }
+    }
+
+    if (this.schemaSync.strategy === 'validate-only') {
+      const existingSchema = await getBigQueryTableSchema(this.bigqueryClient, this.datasetId, tableName);
+      const newFields = generateBigQuerySchemaUpdate(attributes, existingSchema);
+
+      if (newFields.length > 0) {
+        throw new Error(`Table ${tableName} schema mismatch. Missing columns: ${newFields.length}`);
+      }
+    }
+  }
+
   shouldReplicateResource(resourceName) {
     return this.resources.hasOwnProperty(resourceName);
   }
@@ -434,7 +605,8 @@ class BigqueryReplicator extends BaseReplicator {
       projectId: this.projectId,
       datasetId: this.datasetId,
       resources: this.resources,
-      logTable: this.logTable
+      logTable: this.logTable,
+      schemaSync: this.schemaSync
     };
   }
 }

package/src/plugins/replicators/dynamodb-replicator.class.js

@@ -0,0 +1,383 @@
+import tryFn from "#src/concerns/try-fn.js";
+import requirePluginDependency from "#src/plugins/concerns/plugin-dependencies.js";
+import BaseReplicator from './base-replicator.class.js';
+import { ReplicationError } from '../replicator.errors.js';
+
+/**
+ * DynamoDB Replicator - Replicate data to AWS DynamoDB tables
+ *
+ * ⚠️  REQUIRED DEPENDENCY: You must install the AWS SDK:
+ * ```bash
+ * pnpm add @aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb
+ * ```
+ *
+ * Configuration:
+ * @param {string} region - AWS region (required)
+ * @param {string} accessKeyId - AWS access key (optional, uses AWS SDK default chain)
+ * @param {string} secretAccessKey - AWS secret key (optional)
+ * @param {string} endpoint - Custom endpoint for DynamoDB Local (optional)
+ * @param {Object} credentials - AWS credentials object (optional)
+ *
+ * @example
+ * new DynamoDBReplicator({
+ *   region: 'us-east-1',
+ *   accessKeyId: 'YOUR_ACCESS_KEY',
+ *   secretAccessKey: 'YOUR_SECRET_KEY'
+ * }, {
+ *   users: [{ actions: ['insert', 'update', 'delete'], table: 'UsersTable' }],
+ *   orders: 'OrdersTable'
+ * })
+ *
+ * // DynamoDB Local example
+ * new DynamoDBReplicator({
+ *   region: 'us-east-1',
+ *   endpoint: 'http://localhost:8000'
+ * }, {
+ *   users: 'Users'
+ * })
+ *
+ * See PLUGINS.md for comprehensive configuration documentation.
+ */
+class DynamoDBReplicator extends BaseReplicator {
+  constructor(config = {}, resources = {}) {
+    super(config);
+    this.region = config.region || 'us-east-1';
+    this.accessKeyId = config.accessKeyId;
+    this.secretAccessKey = config.secretAccessKey;
+    this.endpoint = config.endpoint;
+    this.credentials = config.credentials;
+    this.client = null;
+    this.docClient = null;
+
+    // Parse resources configuration
+    this.resources = this.parseResourcesConfig(resources);
+  }
+
+  parseResourcesConfig(resources) {
+    const parsed = {};
+
+    for (const [resourceName, config] of Object.entries(resources)) {
+      if (typeof config === 'string') {
+        // Short form: just table name
+        parsed[resourceName] = [{
+          table: config,
+          actions: ['insert'],
+          primaryKey: 'id' // Default primary key
+        }];
+      } else if (Array.isArray(config)) {
+        // Array form: multiple table mappings
+        parsed[resourceName] = config.map(item => {
+          if (typeof item === 'string') {
+            return { table: item, actions: ['insert'], primaryKey: 'id' };
+          }
+          return {
+            table: item.table,
+            actions: item.actions || ['insert'],
+            primaryKey: item.primaryKey || 'id',
+            sortKey: item.sortKey // Optional sort key
+          };
+        });
+      } else if (typeof config === 'object') {
+        // Single object form
+        parsed[resourceName] = [{
+          table: config.table,
+          actions: config.actions || ['insert'],
+          primaryKey: config.primaryKey || 'id',
+          sortKey: config.sortKey
+        }];
+      }
+    }
+
+    return parsed;
+  }
+
+  validateConfig() {
+    const errors = [];
+    // Region defaults to us-east-1, so it's always set
+    // Only validate if explicitly set to empty string
+    if (this.region === '') {
+      errors.push('AWS region is required');
+    }
+    if (Object.keys(this.resources).length === 0) {
+      errors.push('At least one resource must be configured');
+    }
+
+    // Validate resource configurations
+    for (const [resourceName, tables] of Object.entries(this.resources)) {
+      for (const tableConfig of tables) {
+        if (!tableConfig.table) {
+          errors.push(`Table name is required for resource '${resourceName}'`);
+        }
+        if (!Array.isArray(tableConfig.actions) || tableConfig.actions.length === 0) {
+          errors.push(`Actions array is required for resource '${resourceName}'`);
+        }
+      }
+    }
+
+    return { isValid: errors.length === 0, errors };
+  }
+
+  async initialize(database) {
+    await super.initialize(database);
+
+    // Load AWS SDK dependencies
+    const { DynamoDBClient } = requirePluginDependency('@aws-sdk/client-dynamodb', 'DynamoDBReplicator');
+    const { DynamoDBDocumentClient, PutCommand, UpdateCommand, DeleteCommand } = requirePluginDependency('@aws-sdk/lib-dynamodb', 'DynamoDBReplicator');
+
+    // Store command constructors for later use
+    this.PutCommand = PutCommand;
+    this.UpdateCommand = UpdateCommand;
+    this.DeleteCommand = DeleteCommand;
+
+    const [ok, err] = await tryFn(async () => {
+      const clientConfig = {
+        region: this.region
+      };
+
+      if (this.endpoint) {
+        clientConfig.endpoint = this.endpoint;
+      }
+
+      if (this.credentials) {
+        clientConfig.credentials = this.credentials;
+      } else if (this.accessKeyId && this.secretAccessKey) {
+        clientConfig.credentials = {
+          accessKeyId: this.accessKeyId,
+          secretAccessKey: this.secretAccessKey
+        };
+      }
+
+      this.client = new DynamoDBClient(clientConfig);
+      this.docClient = DynamoDBDocumentClient.from(this.client);
+
+      // Test connection by listing tables
+      const { ListTablesCommand } = requirePluginDependency('@aws-sdk/client-dynamodb', 'DynamoDBReplicator');
+      await this.client.send(new ListTablesCommand({ Limit: 1 }));
+    });
+
+    if (!ok) {
+      throw new ReplicationError('Failed to connect to DynamoDB', {
+        operation: 'initialize',
+        replicatorClass: 'DynamoDBReplicator',
+        region: this.region,
+        endpoint: this.endpoint,
+        original: err,
+        suggestion: 'Check AWS credentials and ensure DynamoDB is accessible'
+      });
+    }
+
+    this.emit('connected', {
+      replicator: 'DynamoDBReplicator',
+      region: this.region,
+      endpoint: this.endpoint || 'default'
+    });
+  }
+
+  shouldReplicateResource(resourceName) {
+    return this.resources.hasOwnProperty(resourceName);
+  }
+
+  async replicate(resourceName, operation, data, id) {
+    if (!this.resources[resourceName]) {
+      throw new ReplicationError('Resource not configured for replication', {
+        operation: 'replicate',
+        replicatorClass: 'DynamoDBReplicator',
+        resourceName,
+        configuredResources: Object.keys(this.resources),
+        suggestion: 'Add resource to replicator resources configuration'
+      });
+    }
+
+    const results = [];
+
+    for (const tableConfig of this.resources[resourceName]) {
+      if (!tableConfig.actions.includes(operation)) {
+        continue; // Skip if operation not allowed for this table
+      }
+
+      const [ok, error, result] = await tryFn(async () => {
+        switch (operation) {
+          case 'insert':
+            return await this._putItem(tableConfig.table, data);
+          case 'update':
+            return await this._updateItem(tableConfig.table, id, data, tableConfig);
+          case 'delete':
+            return await this._deleteItem(tableConfig.table, id, tableConfig);
+          default:
+            throw new ReplicationError(`Unsupported operation: ${operation}`, {
+              operation: 'replicate',
+              replicatorClass: 'DynamoDBReplicator',
+              invalidOperation: operation,
+              supportedOperations: ['insert', 'update', 'delete']
+            });
+        }
+      });
+
+      if (ok) {
+        results.push(result);
+      } else {
+        this.emit('replication_error', {
+          resource: resourceName,
+          operation,
+          table: tableConfig.table,
+          error: error.message
+        });
+
+        if (this.config.verbose) {
+          console.error(`[DynamoDBReplicator] Failed to replicate ${operation} for ${resourceName}:`, error);
+        }
+      }
+    }
+
+    return results.length > 0 ? results[0] : null;
+  }
+
+  async _putItem(table, data) {
+    const cleanData = this._cleanInternalFields(data);
+
+    const command = new this.PutCommand({
+      TableName: table,
+      Item: cleanData
+    });
+
+    const result = await this.docClient.send(command);
+    return result;
+  }
+
+  async _updateItem(table, id, data, tableConfig) {
+    const cleanData = this._cleanInternalFields(data);
+
+    // Build update expression
+    const updateExpressions = [];
+    const expressionAttributeNames = {};
+    const expressionAttributeValues = {};
+
+    let index = 0;
+    for (const [key, value] of Object.entries(cleanData)) {
+      // Skip primary key and sort key
+      if (key === tableConfig.primaryKey || key === tableConfig.sortKey) {
+        continue;
+      }
+
+      const attrName = `#attr${index}`;
+      const attrValue = `:val${index}`;
+
+      expressionAttributeNames[attrName] = key;
+      expressionAttributeValues[attrValue] = value;
+      updateExpressions.push(`${attrName} = ${attrValue}`);
+      index++;
+    }
+
+    // Build key
+    const key = { [tableConfig.primaryKey]: id };
+    if (tableConfig.sortKey && cleanData[tableConfig.sortKey]) {
+      key[tableConfig.sortKey] = cleanData[tableConfig.sortKey];
+    }
+
+    const command = new this.UpdateCommand({
+      TableName: table,
+      Key: key,
+      UpdateExpression: `SET ${updateExpressions.join(', ')}`,
+      ExpressionAttributeNames: expressionAttributeNames,
+      ExpressionAttributeValues: expressionAttributeValues,
+      ReturnValues: 'ALL_NEW'
+    });
+
+    const result = await this.docClient.send(command);
+    return result;
+  }
+
+  async _deleteItem(table, id, tableConfig) {
+    const key = { [tableConfig.primaryKey]: id };
+
+    const command = new this.DeleteCommand({
+      TableName: table,
+      Key: key
+    });
+
+    const result = await this.docClient.send(command);
+    return result;
+  }
+
+  _cleanInternalFields(data) {
+    if (!data || typeof data !== 'object') return data;
+
+    const cleanData = { ...data };
+
+    // Remove internal s3db fields
+    Object.keys(cleanData).forEach(key => {
+      if (key.startsWith('$') || key.startsWith('_')) {
+        delete cleanData[key];
+      }
+    });
+
+    return cleanData;
+  }
+
+  async replicateBatch(resourceName, records) {
+    const results = [];
+    const errors = [];
+
+    // DynamoDB batch operations (up to 25 items)
+    // For now, process sequentially (can be optimized with BatchWriteItem)
+    for (const record of records) {
+      const [ok, err, result] = await tryFn(() =>
+        this.replicate(resourceName, record.operation, record.data, record.id)
+      );
+
+      if (ok) {
+        results.push(result);
+      } else {
+        errors.push({ id: record.id, error: err.message });
+      }
+    }
+
+    return {
+      success: errors.length === 0,
+      results,
+      errors,
+      total: records.length
+    };
+  }
+
+  async testConnection() {
+    const [ok, err] = await tryFn(async () => {
+      if (!this.client) {
+        throw new Error('Client not initialized');
+      }
+
+      const { ListTablesCommand } = requirePluginDependency('@aws-sdk/client-dynamodb', 'DynamoDBReplicator');
+      await this.client.send(new ListTablesCommand({ Limit: 1 }));
+      return true;
+    });
+
+    if (!ok) {
+      this.emit('connection_error', { replicator: 'DynamoDBReplicator', error: err.message });
+      return false;
+    }
+
+    return true;
+  }
+
+  async getStatus() {
+    const baseStatus = await super.getStatus();
+    return {
+      ...baseStatus,
+      connected: !!this.client,
+      region: this.region,
+      endpoint: this.endpoint || 'default',
+      resources: Object.keys(this.resources)
+    };
+  }
+
+  async cleanup() {
+    if (this.client) {
+      this.client.destroy();
+      this.client = null;
+      this.docClient = null;
+    }
+    await super.cleanup();
+  }
+}
+
+export default DynamoDBReplicator;

package/src/plugins/replicators/index.js

@@ -1,11 +1,29 @@
 import BaseReplicator from './base-replicator.class.js';
 import BigqueryReplicator from './bigquery-replicator.class.js';
+import DynamoDBReplicator from './dynamodb-replicator.class.js';
+import MongoDBReplicator from './mongodb-replicator.class.js';
+import MySQLReplicator from './mysql-replicator.class.js';
+import PlanetScaleReplicator from './planetscale-replicator.class.js';
 import PostgresReplicator from './postgres-replicator.class.js';
 import S3dbReplicator from './s3db-replicator.class.js';
 import SqsReplicator from './sqs-replicator.class.js';
+import TursoReplicator from './turso-replicator.class.js';
+import WebhookReplicator from './webhook-replicator.class.js';
 import { ReplicationError } from '../replicator.errors.js';
 
-export { BaseReplicator, BigqueryReplicator, PostgresReplicator, S3dbReplicator, SqsReplicator };
+export {
+  BaseReplicator,
+  BigqueryReplicator,
+  DynamoDBReplicator,
+  MongoDBReplicator,
+  MySQLReplicator,
+  PlanetScaleReplicator,
+  PostgresReplicator,
+  S3dbReplicator,
+  SqsReplicator,
+  TursoReplicator,
+  WebhookReplicator
+};
 
 /**
  * Available replicator drivers
@@ -14,12 +32,19 @@ export const REPLICATOR_DRIVERS = {
   s3db: S3dbReplicator,
   sqs: SqsReplicator,
   bigquery: BigqueryReplicator,
-  postgres: PostgresReplicator
+  postgres: PostgresReplicator,
+  mysql: MySQLReplicator,
+  mariadb: MySQLReplicator, // MariaDB uses the same driver as MySQL
+  planetscale: PlanetScaleReplicator,
+  turso: TursoReplicator,
+  dynamodb: DynamoDBReplicator,
+  mongodb: MongoDBReplicator,
+  webhook: WebhookReplicator
 };
 
 /**
  * Create a replicator instance based on driver type
- * @param {string} driver - Driver type (s3db, sqs, bigquery, postgres)
+ * @param {string} driver - Driver type (s3db, sqs, bigquery, postgres, mysql, mariadb, planetscale, turso, dynamodb, mongodb, webhook)
  * @param {Object} config - Replicator configuration
  * @returns {BaseReplicator} Replicator instance
  */