s3db.js 7.2.0 → 7.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "7.2.0",
+  "version": "7.2.1",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",
@@ -25,6 +25,10 @@
   ],
   "type": "module",
   "sideEffects": false,
+  "imports": {
+    "#src/*": "./src/*",
+    "#tests/*": "./tests/*"
+  },
   "exports": {
     ".": {
       "import": "./dist/s3db.es.js",

package/src/plugins/replicators/bigquery-replicator.class.js

@@ -1,55 +1,36 @@
+import tryFn from "#src/concerns/try-fn.js";
+
 import BaseReplicator from './base-replicator.class.js';
-import tryFn from "../../concerns/try-fn.js";
 
 /**
- * BigQuery Replicator
- *
- * Replicates data to Google BigQuery tables, supporting per-resource table mapping and action filtering.
- *
- * ⚠️  REQUIRED DEPENDENCY: You must install the Google Cloud BigQuery SDK to use this replicator:
- *
+ * BigQuery Replicator - Replicate data to Google BigQuery tables
+ * 
+ * ⚠️  REQUIRED DEPENDENCY: You must install the Google Cloud BigQuery SDK:
  * ```bash
- * npm install @google-cloud/bigquery
- * # or
- * yarn add @google-cloud/bigquery
- * # or
  * pnpm add @google-cloud/bigquery
  * ```
- *
- * @config {Object} config - Configuration object for the replicator
- * @config {string} config.projectId - (Required) Google Cloud project ID
- * @config {string} config.datasetId - (Required) BigQuery dataset ID
- * @config {Object} [config.credentials] - (Optional) Google service account credentials object (JSON). If omitted, uses default credentials.
- * @config {string} [config.location='US'] - (Optional) BigQuery dataset location/region
- * @config {string} [config.logTable] - (Optional) Table name for operation logging. If omitted, no logging is performed.
- * @config {Object} resources - Resource configuration mapping
- * @config {Object|string} resources[resourceName] - Resource configuration
- * @config {string} resources[resourceName].table - Table name for this resource
- * @config {Array} resources[resourceName].actions - Array of actions to replicate (insert, update, delete)
- * @config {string} resources[resourceName] - Short form: just the table name (equivalent to { actions: ['insert'], table: tableName })
- *
+ * 
+ * Configuration:
+ * @param {string} projectId - Google Cloud project ID (required)
+ * @param {string} datasetId - BigQuery dataset ID (required) 
+ * @param {Object} credentials - Service account credentials object (optional)
+ * @param {string} location - BigQuery dataset location/region (default: 'US')
+ * @param {string} logTable - Table name for operation logging (optional)
+ * 
  * @example
  * new BigqueryReplicator({
  *   projectId: 'my-gcp-project',
  *   datasetId: 'analytics',
- *   location: 'US',
- *   credentials: require('./gcp-service-account.json'),
- *   logTable: 's3db_replicator_log'
+ *   credentials: JSON.parse(Buffer.from(GOOGLE_CREDENTIALS, 'base64').toString())
  * }, {
- *   users: [
- *     { actions: ['insert', 'update', 'delete'], table: 'users_table' },
- *   ],
- *   urls: [
- *     { actions: ['insert'], table: 'urls_table' },
- *     { actions: ['insert'], table: 'urls_table_v2' },
- *   ],
- *   clicks: 'clicks_table' // equivalent to { actions: ['insert'], table: 'clicks_table' }
+ *   users: {
+ *     table: 'users_table',
+ *     transform: (data) => ({ ...data, ip: data.ip || 'unknown' })
+ *   },
+ *   orders: 'orders_table'
  * })
- *
- * Notes:
- * - The target tables must exist and have columns matching the resource attributes (id is required as primary key)
- * - The log table must have columns: resource_name, operation, record_id, data, timestamp, source
- * - Uses @google-cloud/bigquery SDK
+ * 
+ * See PLUGINS.md for comprehensive configuration documentation.
  */
 class BigqueryReplicator extends BaseReplicator {
   constructor(config = {}, resources = {}) {
@@ -73,24 +54,27 @@ class BigqueryReplicator extends BaseReplicator {
         // Short form: just table name
         parsed[resourceName] = [{
           table: config,
-          actions: ['insert']
+          actions: ['insert'],
+          transform: null
         }];
       } else if (Array.isArray(config)) {
         // Array form: multiple table mappings
         parsed[resourceName] = config.map(item => {
           if (typeof item === 'string') {
-            return { table: item, actions: ['insert'] };
+            return { table: item, actions: ['insert'], transform: null };
           }
           return {
             table: item.table,
-            actions: item.actions || ['insert']
+            actions: item.actions || ['insert'],
+            transform: item.transform || null
           };
         });
       } else if (typeof config === 'object') {
         // Single object form
         parsed[resourceName] = [{
           table: config.table,
-          actions: config.actions || ['insert']
+          actions: config.actions || ['insert'],
+          transform: config.transform || null
         }];
       }
     }
@@ -118,6 +102,9 @@ class BigqueryReplicator extends BaseReplicator {
         if (invalidActions.length > 0) {
           errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(', ')}. Valid actions: ${validActions.join(', ')}`);
         }
+        if (tableConfig.transform && typeof tableConfig.transform !== 'function') {
+          errors.push(`Transform must be a function for resource '${resourceName}'`);
+        }
       }
     }
     
@@ -162,7 +149,19 @@ class BigqueryReplicator extends BaseReplicator {
     
     return this.resources[resourceName]
       .filter(tableConfig => tableConfig.actions.includes(operation))
-      .map(tableConfig => tableConfig.table);
+      .map(tableConfig => ({
+        table: tableConfig.table,
+        transform: tableConfig.transform
+      }));
+  }
+
+  applyTransform(data, transformFn) {
+    if (!transformFn) return data;
+    
+    let transformedData = JSON.parse(JSON.stringify(data));
+    if (transformedData._length) delete transformedData._length;
+    
+    return transformFn(transformedData);
   }
 
   async replicate(resourceName, operation, data, id, beforeData = null) {
@@ -175,8 +174,8 @@ class BigqueryReplicator extends BaseReplicator {
       return { skipped: true, reason: 'action_not_included' };
     }
 
-    const tables = this.getTablesForResource(resourceName, operation);
-    if (tables.length === 0) {
+    const tableConfigs = this.getTablesForResource(resourceName, operation);
+    if (tableConfigs.length === 0) {
       return { skipped: true, reason: 'no_tables_for_action' };
     }
 
@@ -185,50 +184,80 @@ class BigqueryReplicator extends BaseReplicator {
 
     const [ok, err, result] = await tryFn(async () => {
       const dataset = this.bigqueryClient.dataset(this.datasetId);
+      
       // Replicate to all applicable tables
-      for (const tableId of tables) {
+      for (const tableConfig of tableConfigs) {
         const [okTable, errTable] = await tryFn(async () => {
-          const table = dataset.table(tableId);
+          const table = dataset.table(tableConfig.table);
           let job;
+          
           if (operation === 'insert') {
-            const row = { ...data };
-            job = await table.insert([row]);
+            const transformedData = this.applyTransform(data, tableConfig.transform);
+            job = await table.insert([transformedData]);
           } else if (operation === 'update') {
-            const keys = Object.keys(data).filter(k => k !== 'id');
-            const setClause = keys.map(k => `${k}=@${k}`).join(', ');
-            const params = { id };
-            keys.forEach(k => { params[k] = data[k]; });
-            const query = `UPDATE \`${this.projectId}.${this.datasetId}.${tableId}\` SET ${setClause} WHERE id=@id`;
-            const [updateJob] = await this.bigqueryClient.createQueryJob({
-              query,
-              params
-            });
-            await updateJob.getQueryResults();
-            job = [updateJob];
+            const transformedData = this.applyTransform(data, tableConfig.transform);
+            const keys = Object.keys(transformedData).filter(k => k !== 'id');
+            const setClause = keys.map(k => `${k} = @${k}`).join(', ');
+            const params = { id, ...transformedData };
+            const query = `UPDATE \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` SET ${setClause} WHERE id = @id`;
+            
+            // Retry logic for streaming buffer issues
+            const maxRetries = 2;
+            let lastError = null;
+            
+            for (let attempt = 1; attempt <= maxRetries; attempt++) {
+              try {
+                const [updateJob] = await this.bigqueryClient.createQueryJob({
+                  query,
+                  params,
+                  location: this.location
+                });
+                await updateJob.getQueryResults();
+                job = [updateJob];
+                break;
+              } catch (error) {
+                lastError = error;
+                
+                // If it's streaming buffer error and not the last attempt
+                if (error?.message?.includes('streaming buffer') && attempt < maxRetries) {
+                  const delaySeconds = 30;
+                  await new Promise(resolve => setTimeout(resolve, delaySeconds * 1000));
+                  continue;
+                }
+                
+                throw error;
+              }
+            }
+            
+            if (!job) throw lastError;
           } else if (operation === 'delete') {
-            const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableId}\` WHERE id=@id`;
+            const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` WHERE id = @id`;
             const [deleteJob] = await this.bigqueryClient.createQueryJob({
               query,
-              params: { id }
+              params: { id },
+              location: this.location
             });
             await deleteJob.getQueryResults();
             job = [deleteJob];
           } else {
             throw new Error(`Unsupported operation: ${operation}`);
           }
+          
           results.push({
-            table: tableId,
+            table: tableConfig.table,
             success: true,
             jobId: job[0]?.id
           });
         });
+        
         if (!okTable) {
           errors.push({
-            table: tableId,
+            table: tableConfig.table,
             error: errTable.message
           });
         }
       }
+      
       // Log operation if logTable is configured
       if (this.logTable) {
         const [okLog, errLog] = await tryFn(async () => {
@@ -246,25 +275,29 @@ class BigqueryReplicator extends BaseReplicator {
           // Don't fail the main operation if logging fails
         }
       }
+      
       const success = errors.length === 0;
       this.emit('replicated', {
         replicator: this.name,
         resourceName,
         operation,
         id,
-        tables,
+        tables: tableConfigs.map(t => t.table),
         results,
         errors,
         success
       });
+      
       return { 
         success, 
         results, 
         errors,
-        tables 
+        tables: tableConfigs.map(t => t.table)
       };
     });
+    
     if (ok) return result;
+    
     this.emit('replicator_error', {
       replicator: this.name,
       resourceName,
@@ -272,6 +305,7 @@ class BigqueryReplicator extends BaseReplicator {
       id,
       error: err.message
     });
+    
     return { success: false, error: err.message };
   }
 

package/src/plugins/replicators/postgres-replicator.class.js

@@ -1,124 +1,34 @@
+import tryFn from "#src/concerns/try-fn.js";
+import BaseReplicator from './base-replicator.class.js';
+
 /**
- * Postgres Replicator Configuration Documentation
- * 
- * This replicator executes real SQL operations (INSERT, UPDATE, DELETE) on PostgreSQL tables
- * using the official pg (node-postgres) library. It maps s3db resources to database tables
- * and performs actual database operations for each replicator event.
- * 
- * ⚠️  REQUIRED DEPENDENCY: You must install the PostgreSQL client library to use this replicator:
+ * PostgreSQL Replicator - Replicate data to PostgreSQL tables
  * 
+ * ⚠️  REQUIRED DEPENDENCY: You must install the PostgreSQL client library:
  * ```bash
- * npm install pg
- * # or
- * yarn add pg
- * # or
  * pnpm add pg
  * ```
  * 
- * @typedef {Object} PostgresReplicatorConfig
- * @property {string} database - The name of the PostgreSQL database to connect to
- * @property {string} resourceArn - The ARN of the Aurora Serverless cluster or RDS instance
- * @property {string} secretArn - The ARN of the Secrets Manager secret containing database credentials
- * @property {string} [region='us-east-1'] - AWS region where the database is located
- * @property {Object.<string, string>} [tableMapping] - Maps s3db resource names to PostgreSQL table names
- *   - Key: s3db resource name (e.g., 'users', 'orders')
- *   - Value: PostgreSQL table name (e.g., 'public.users', 'analytics.orders')
- *   - If not provided, resource names are used as table names
- * @property {boolean} [logOperations=false] - Whether to log SQL operations to console for debugging
- * @property {string} [schema='public'] - Default database schema to use when tableMapping doesn't specify schema
- * @property {number} [maxRetries=3] - Maximum number of retry attempts for failed operations
- * @property {number} [retryDelay=1000] - Delay in milliseconds between retry attempts
- * @property {boolean} [useUpsert=true] - Whether to use UPSERT (INSERT ... ON CONFLICT) for updates
- * @property {string} [conflictColumn='id'] - Column name to use for conflict resolution in UPSERT operations
- * 
- * @example
- * // Basic configuration with table mapping
- * {
- *   database: 'analytics_db',
- *   resourceArn: 'arn:aws:rds:us-east-1:123456789012:cluster:my-aurora-cluster',
- *   secretArn: 'arn:aws:secretsmanager:us-east-1:123456789012:secret:db-credentials',
- *   region: 'us-east-1',
- *   tableMapping: {
- *     'users': 'public.users',
- *     'orders': 'analytics.orders',
- *     'products': 'inventory.products'
- *   },
- *   logOperations: true,
- *   useUpsert: true,
- *   conflictColumn: 'id'
- * }
- * 
- * @example
- * // Minimal configuration using default settings
- * {
- *   database: 'my_database',
- *   resourceArn: 'arn:aws:rds:us-east-1:123456789012:cluster:my-cluster',
- *   secretArn: 'arn:aws:secretsmanager:us-east-1:123456789012:secret:db-secret'
- * }
+ * Configuration:
+ * @param {string} connectionString - PostgreSQL connection string (required)
+ * @param {string} host - Database host (alternative to connectionString)
+ * @param {number} port - Database port (default: 5432)
+ * @param {string} database - Database name
+ * @param {string} user - Database user
+ * @param {string} password - Database password
+ * @param {Object} ssl - SSL configuration (optional)
+ * @param {string} logTable - Table name for operation logging (optional)
  * 
- * @notes
- * - Requires AWS credentials with RDS Data Service permissions
- * - Database tables must exist before replicator starts
- * - For UPSERT operations, the conflict column must have a unique constraint
- * - All data is automatically converted to JSON format for storage
- * - Timestamps are stored as ISO strings in the database
- * - Failed operations are retried with exponential backoff
- * - Operations are executed within database transactions for consistency
- */
-import BaseReplicator from './base-replicator.class.js';
-import tryFn from "../../concerns/try-fn.js";
-
-/**
- * PostgreSQL Replicator
- *
- * Replicates data to PostgreSQL tables, supporting per-resource table mapping and action filtering.
- *
- * ⚠️  REQUIRED DEPENDENCY: You must install the PostgreSQL client library to use this replicator:
- *
- * ```bash
- * npm install pg
- * # or
- * yarn add pg
- * # or
- * pnpm add pg
- * ```
- *
- * @config {Object} config - Configuration object for the replicator
- * @config {string} [config.connectionString] - PostgreSQL connection string (alternative to individual params)
- * @config {string} [config.host] - Database host (required if not using connectionString)
- * @config {number} [config.port=5432] - Database port
- * @config {string} [config.database] - Database name (required if not using connectionString)
- * @config {string} [config.user] - Database user (required if not using connectionString)
- * @config {string} [config.password] - Database password (required if not using connectionString)
- * @config {Object} [config.ssl] - SSL configuration
- * @config {string} [config.logTable] - Table name for operation logging. If omitted, no logging is performed.
- * @config {Object} resources - Resource configuration mapping
- * @config {Object|string} resources[resourceName] - Resource configuration
- * @config {string} resources[resourceName].table - Table name for this resource
- * @config {Array} resources[resourceName].actions - Array of actions to replicate (insert, update, delete)
- * @config {string} resources[resourceName] - Short form: just the table name (equivalent to { actions: ['insert'], table: tableName })
- *
  * @example
  * new PostgresReplicator({
  *   connectionString: 'postgresql://user:password@localhost:5432/analytics',
- *   ssl: false,
- *   logTable: 's3db_replicator_log'
+ *   logTable: 'replication_log'
  * }, {
- *   users: [
- *     { actions: ['insert', 'update', 'delete'], table: 'users_table' },
- *   ],
- *   orders: [
- *     { actions: ['insert'], table: 'orders_table' },
- *     { actions: ['insert'], table: 'orders_analytics' }, // Also replicate to analytics table
- *   ],
- *   products: 'products_table' // Short form: equivalent to { actions: ['insert'], table: 'products_table' }
+ *   users: [{ actions: ['insert', 'update'], table: 'users_table' }],
+ *   orders: 'orders_table'
  * })
- *
- * Notes:
- * - The target tables must exist and have columns matching the resource attributes (id is required as primary key)
- * - The log table must have columns: resource_name, operation, record_id, data, timestamp, source
- * - Uses pg (node-postgres) library
- * - Supports UPSERT operations with ON CONFLICT handling
+ * 
+ * See PLUGINS.md for comprehensive configuration documentation.
  */
 class PostgresReplicator extends BaseReplicator {
   constructor(config = {}, resources = {}) {

package/src/plugins/replicators/s3db-replicator.class.js

@@ -1,59 +1,31 @@
-/**
- * S3DB Replicator Configuration Documentation
- * 
- * This replicator supports highly flexible resource mapping and transformer configuration. You can specify the resources to replicate using any of the following syntaxes:
- *
- * 1. Array of resource names (replicate resource to itself):
- *    resources: ['users']
- *    // Replicates 'users' to 'users' in the destination
- *
- * 2. Map: source resource → destination resource name:
- *    resources: { users: 'people' }
- *    // Replicates 'users' to 'people' in the destination
- *
- * 3. Map: source resource → array of destination resource names and/or transformers:
- *    resources: { users: ['people', (el) => ({ ...el, fullName: el.name })] }
- *    // Replicates 'users' to 'people' and also applies the transformer
- *
- * 4. Map: source resource → object with resource and transformer:
- *    resources: { users: { resource: 'people', transformer: (el) => ({ ...el, fullName: el.name }) } }
- *    // Replicates 'users' to 'people' with a custom transformer
- *
- * 5. Map: source resource → array of objects with resource and transformer (multi-destination):
- *    resources: { users: [ { resource: 'people', transformer: (el) => ({ ...el, fullName: el.name }) } ] }
- *    // Replicates 'users' to multiple destinations, each with its own transformer
- *
- * 6. Map: source resource → function (rare, but supported):
- *    resources: { users: (el) => ... }
- *    // Replicates 'users' to 'users' with a custom transformer
- * 
- * All forms can be mixed and matched for different resources. The transformer is always available (default: identity function).
- *
- * Example:
- *   resources: {
- *     users: [
- *       'people',
- *       { resource: 'people', transformer: (el) => ({ ...el, fullName: el.name }) },
- *       (el) => ({ ...el, fullName: el.name })
- *     ],
- *     orders: 'orders_copy',
- *     products: { resource: 'products_copy' }
- *   }
- *
- * The replicator always uses the provided client as the destination.
- *
- * See tests/examples for all supported syntaxes.
- */
+import tryFn from "#src/concerns/try-fn.js";
+import { S3db } from '#src/database.class.js';
 import BaseReplicator from './base-replicator.class.js';
-import { S3db } from '../../database.class.js';
-import tryFn from "../../concerns/try-fn.js";
 
 function normalizeResourceName(name) {
   return typeof name === 'string' ? name.trim().toLowerCase() : name;
 }
 
 /**
- * S3DB Replicator - Replicates data to another s3db instance
+ * S3DB Replicator - Replicate data to another S3DB instance
+ * 
+ * Configuration:
+ * @param {string} connectionString - S3DB connection string for destination database (required)
+ * @param {Object} client - Pre-configured S3DB client instance (alternative to connectionString)
+ * @param {Object} resources - Resource mapping configuration
+ * 
+ * @example
+ * new S3dbReplicator({
+ *   connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup"
+ * }, {
+ *   users: 'backup_users',
+ *   orders: {
+ *     resource: 'order_backup',
+ *     transformer: (data) => ({ ...data, backup_timestamp: new Date().toISOString() })
+ *   }
+ * })
+ * 
+ * See PLUGINS.md for comprehensive configuration documentation.
  */
 class S3dbReplicator extends BaseReplicator {
   constructor(config = {}, resources = [], client = null) {