s3db.js 7.1.0 → 7.2.1

package/PLUGINS.md

@@ -1608,18 +1608,147 @@ await users.insert({ name: 'John', email: 'john@example.com' });
 
 #### S3DB Replicator
 
-Replicate to another S3DB instance:
+Replicate to another S3DB instance with flexible resource mapping and transformation capabilities:
 
 ```javascript
 {
   driver: 's3db',
-  resources: ['users', 'products'],
   config: {
     connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup"
+  },
+  resources: {
+    // Simple resource mapping (replicate to same name)
+    users: 'users',
+    
+    // Map source → destination resource name
+    products: 'backup_products',
+    
+    // Advanced mapping with transformer
+    orders: {
+      resource: 'order_backup',
+      transformer: (data) => ({
+        ...data,
+        backup_timestamp: new Date().toISOString(),
+        original_source: 'production'
+      })
+    },
+    
+    // Multi-destination replication
+    analytics: [
+      'analytics_backup',
+      { 
+        resource: 'analytics_processed', 
+        transformer: (data) => ({
+          ...data,
+          processed_date: data.createdAt?.split('T')[0],
+          data_source: 'analytics'
+        })
+      }
+    ]
+  }
+}
+```
+
+**Resource Configuration Syntaxes:**
+
+The S3DB replicator supports highly flexible resource mapping configurations:
+
+**1. Array of resource names** (replicate to same name):
+```javascript
+resources: ['users', 'products', 'orders']
+// Replicates each resource to itself in the destination database
+```
+
+**2. Object mapping** (source → destination):
+```javascript
+resources: { 
+  users: 'people',           // users → people
+  products: 'items',         // products → items  
+  orders: 'order_history'    // orders → order_history
+}
+```
+
+**3. Array with transformer** (resource + transformation):
+```javascript
+resources: { 
+  users: ['people', (data) => ({ ...data, fullName: `${data.firstName} ${data.lastName}` })]
+  // Replicates 'users' to 'people' with transformation
+}
+```
+
+**4. Object with resource and transformer**:
+```javascript
+resources: { 
+  users: { 
+    resource: 'people', 
+    transformer: (data) => ({ 
+      ...data, 
+      fullName: `${data.firstName} ${data.lastName}`,
+      migrated_at: new Date().toISOString()
+    })
   }
 }
 ```
 
+**5. Multi-destination arrays**:
+```javascript
+resources: { 
+  users: [
+    'people',                    // Simple copy
+    { 
+      resource: 'user_analytics', 
+      transformer: (data) => ({
+        id: data.id,
+        signup_date: data.createdAt,
+        user_type: data.role || 'standard'
+      })
+    }
+  ]
+}
+```
+
+**6. Function-only transformation** (transform to same resource):
+```javascript
+resources: { 
+  users: (data) => ({ 
+    ...data, 
+    processed: true,
+    backup_date: new Date().toISOString()
+  })
+}
+```
+
+**Mixed Configuration Example:**
+```javascript
+resources: {
+  users: [
+    'people',                                    // Simple copy
+    { 
+      resource: 'user_profiles', 
+      transformer: (data) => ({ 
+        ...data, 
+        profile_complete: !!(data.name && data.email)
+      })
+    }
+  ],
+  orders: 'order_backup',                        // Rename only
+  products: { resource: 'product_catalog' },     // Object form
+  analytics: (data) => ({ ...data, processed: true })  // Transform only
+}
+```
+
+**Configuration Options:**
+- `connectionString`: S3DB connection string for destination database (required)
+- `client`: Pre-configured S3DB client instance (alternative to connectionString)
+- `resources`: Resource mapping configuration (see syntaxes above)
+
+**Transformer Features:**
+- **Data Transformation**: Apply custom logic before replication
+- **Field Mapping**: Rename, combine, or derive new fields  
+- **Data Enrichment**: Add metadata, timestamps, or computed values
+- **Conditional Logic**: Apply transformations based on data content
+- **Multi-destination**: Send different transformed versions to multiple targets
+
 #### SQS Replicator
 
 Send changes to AWS SQS queues:
@@ -1639,23 +1768,84 @@ Send changes to AWS SQS queues:
 
 #### BigQuery Replicator
 
-Replicate to Google BigQuery:
+Replicate to Google BigQuery with advanced data transformation capabilities:
 
 ```javascript
 {
   driver: 'bigquery',
-  resources: {
-    users: [{ actions: ['insert', 'update'], table: 'users_table' }],
-    orders: 'orders_table'
-  },
   config: {
-    projectId: 'my-project',
-    datasetId: 'analytics',
-    credentials: { /* service account */ }
+    location: 'US',
+    projectId: 'my-gcp-project',
+    datasetId: 'analytics', 
+    credentials: JSON.parse(Buffer.from(GOOGLE_CREDENTIALS, 'base64').toString()),
+    logTable: 'replication_log'  // Optional: table for operation logging
+  },
+  resources: {
+    // Simple table mapping
+    clicks: 'mrt-shortner__clicks',
+    fingerprints: 'mrt-shortner__fingerprints',
+    scans: 'mrt-shortner__scans',
+    sessions: 'mrt-shortner__sessions',
+    shares: 'mrt-shortner__shares', 
+    urls: 'mrt-shortner__urls',
+    views: 'mrt-shortner__views',
+    
+    // Advanced configuration with transform functions
+    users: {
+      table: 'mrt-shortner__users',
+      actions: ['insert', 'update'],
+      transform: (data) => {
+        return {
+          ...data,
+          ip: data.ip || 'unknown',
+          userIp: data.userIp || 'unknown',
+        }
+      }
+    },
+    
+    // Multiple destinations for a single resource
+    orders: [
+      { actions: ['insert'], table: 'fact_orders' },
+      { 
+        actions: ['insert'], 
+        table: 'daily_revenue',
+        transform: (data) => ({
+          date: data.createdAt?.split('T')[0],
+          revenue: data.amount,
+          customer_id: data.userId,
+          order_count: 1
+        })
+      }
+    ]
   }
 }
 ```
 
+**Transform Function Features:**
+- **Data Transformation**: Apply custom logic before sending to BigQuery
+- **Field Mapping**: Rename, combine, or derive new fields
+- **Data Enrichment**: Add computed fields, defaults, or metadata
+- **Format Conversion**: Convert data types or formats for BigQuery compatibility
+- **Multiple Destinations**: Send transformed data to different tables
+
+**Configuration Options:**
+- `projectId`: Google Cloud project ID (required)
+- `datasetId`: BigQuery dataset ID (required) 
+- `credentials`: Service account credentials object (optional, uses default if omitted)
+- `location`: BigQuery dataset location/region (default: 'US')
+- `logTable`: Table name for operation logging (optional)
+
+**Resource Configuration:**
+- **String**: Simple table mapping (e.g., `'table_name'`)
+- **Object**: Advanced configuration with actions and transforms
+- **Array**: Multiple destination tables with different configurations
+
+**Automatic Features:**
+- **Retry Logic**: Handles BigQuery streaming buffer limitations with 30-second retry delays
+- **Error Handling**: Graceful handling of schema mismatches and quota limits
+- **Operation Logging**: Optional audit trail of all replication operations
+- **Schema Compatibility**: Automatic handling of missing fields
+
 #### PostgreSQL Replicator
 
 Replicate to PostgreSQL database:

package/dist/s3db.cjs.js

@@ -8006,22 +8006,25 @@ class BigqueryReplicator extends base_replicator_class_default {
       if (typeof config === "string") {
         parsed[resourceName] = [{
           table: config,
-          actions: ["insert"]
+          actions: ["insert"],
+          transform: null
         }];
       } else if (Array.isArray(config)) {
         parsed[resourceName] = config.map((item) => {
           if (typeof item === "string") {
-            return { table: item, actions: ["insert"] };
+            return { table: item, actions: ["insert"], transform: null };
           }
           return {
             table: item.table,
-            actions: item.actions || ["insert"]
+            actions: item.actions || ["insert"],
+            transform: item.transform || null
           };
         });
       } else if (typeof config === "object") {
         parsed[resourceName] = [{
           table: config.table,
-          actions: config.actions || ["insert"]
+          actions: config.actions || ["insert"],
+          transform: config.transform || null
         }];
       }
     }
@@ -8045,6 +8048,9 @@ class BigqueryReplicator extends base_replicator_class_default {
         if (invalidActions.length > 0) {
           errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(", ")}. Valid actions: ${validActions.join(", ")}`);
         }
+        if (tableConfig.transform && typeof tableConfig.transform !== "function") {
+          errors.push(`Transform must be a function for resource '${resourceName}'`);
+        }
       }
     }
     return { isValid: errors.length === 0, errors };
@@ -8080,7 +8086,16 @@ class BigqueryReplicator extends base_replicator_class_default {
   }
   getTablesForResource(resourceName, operation) {
     if (!this.resources[resourceName]) return [];
-    return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => tableConfig.table);
+    return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => ({
+      table: tableConfig.table,
+      transform: tableConfig.transform
+    }));
+  }
+  applyTransform(data, transformFn) {
+    if (!transformFn) return data;
+    let transformedData = JSON.parse(JSON.stringify(data));
+    if (transformedData._length) delete transformedData._length;
+    return transformFn(transformedData);
   }
   async replicate(resourceName, operation, data, id, beforeData = null) {
     if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
@@ -8089,40 +8104,56 @@ class BigqueryReplicator extends base_replicator_class_default {
     if (!this.shouldReplicateAction(resourceName, operation)) {
       return { skipped: true, reason: "action_not_included" };
     }
-    const tables = this.getTablesForResource(resourceName, operation);
-    if (tables.length === 0) {
+    const tableConfigs = this.getTablesForResource(resourceName, operation);
+    if (tableConfigs.length === 0) {
       return { skipped: true, reason: "no_tables_for_action" };
     }
     const results = [];
     const errors = [];
     const [ok, err, result] = await try_fn_default(async () => {
       const dataset = this.bigqueryClient.dataset(this.datasetId);
-      for (const tableId of tables) {
+      for (const tableConfig of tableConfigs) {
         const [okTable, errTable] = await try_fn_default(async () => {
-          const table = dataset.table(tableId);
+          const table = dataset.table(tableConfig.table);
           let job;
           if (operation === "insert") {
-            const row = { ...data };
-            job = await table.insert([row]);
+            const transformedData = this.applyTransform(data, tableConfig.transform);
+            job = await table.insert([transformedData]);
           } else if (operation === "update") {
-            const keys = Object.keys(data).filter((k) => k !== "id");
-            const setClause = keys.map((k) => `${k}=@${k}`).join(", ");
-            const params = { id };
-            keys.forEach((k) => {
-              params[k] = data[k];
-            });
-            const query = `UPDATE \`${this.projectId}.${this.datasetId}.${tableId}\` SET ${setClause} WHERE id=@id`;
-            const [updateJob] = await this.bigqueryClient.createQueryJob({
-              query,
-              params
-            });
-            await updateJob.getQueryResults();
-            job = [updateJob];
+            const transformedData = this.applyTransform(data, tableConfig.transform);
+            const keys = Object.keys(transformedData).filter((k) => k !== "id");
+            const setClause = keys.map((k) => `${k} = @${k}`).join(", ");
+            const params = { id, ...transformedData };
+            const query = `UPDATE \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` SET ${setClause} WHERE id = @id`;
+            const maxRetries = 2;
+            let lastError = null;
+            for (let attempt = 1; attempt <= maxRetries; attempt++) {
+              try {
+                const [updateJob] = await this.bigqueryClient.createQueryJob({
+                  query,
+                  params,
+                  location: this.location
+                });
+                await updateJob.getQueryResults();
+                job = [updateJob];
+                break;
+              } catch (error) {
+                lastError = error;
+                if (error?.message?.includes("streaming buffer") && attempt < maxRetries) {
+                  const delaySeconds = 30;
+                  await new Promise((resolve) => setTimeout(resolve, delaySeconds * 1e3));
+                  continue;
+                }
+                throw error;
+              }
+            }
+            if (!job) throw lastError;
           } else if (operation === "delete") {
-            const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableId}\` WHERE id=@id`;
+            const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` WHERE id = @id`;
             const [deleteJob] = await this.bigqueryClient.createQueryJob({
               query,
-              params: { id }
+              params: { id },
+              location: this.location
             });
             await deleteJob.getQueryResults();
             job = [deleteJob];
@@ -8130,14 +8161,14 @@ class BigqueryReplicator extends base_replicator_class_default {
             throw new Error(`Unsupported operation: ${operation}`);
           }
           results.push({
-            table: tableId,
+            table: tableConfig.table,
             success: true,
             jobId: job[0]?.id
           });
         });
         if (!okTable) {
           errors.push({
-            table: tableId,
+            table: tableConfig.table,
             error: errTable.message
           });
         }
@@ -8163,7 +8194,7 @@ class BigqueryReplicator extends base_replicator_class_default {
         resourceName,
         operation,
         id,
-        tables,
+        tables: tableConfigs.map((t) => t.table),
         results,
         errors,
         success
@@ -8172,7 +8203,7 @@ class BigqueryReplicator extends base_replicator_class_default {
         success,
         results,
         errors,
-        tables
+        tables: tableConfigs.map((t) => t.table)
       };
     });
     if (ok) return result;
@@ -12211,7 +12242,7 @@ class Database extends EventEmitter {
     super();
     this.version = "1";
     this.s3dbVersion = (() => {
-      const [ok, err, version] = try_fn_default(() => true ? "7.1.0" : "latest");
+      const [ok, err, version] = try_fn_default(() => true ? "7.2.0" : "latest");
       return ok ? version : "latest";
     })();
     this.resources = {};