s3db.js 7.2.0 → 7.3.1

package/src/plugins/replicators/sqs-replicator.class.js

@@ -1,134 +1,34 @@
+import tryFn from "#src/concerns/try-fn.js";
+import BaseReplicator from './base-replicator.class.js';
+
 /**
- * SQS Replicator Configuration Documentation
- * 
- * This replicator sends replicator events to Amazon SQS queues. It supports both
- * resource-specific queues and a single queue for all events, with a flexible message
- * structure that includes operation details and data.
- * 
- * ⚠️  REQUIRED DEPENDENCY: You must install the AWS SQS SDK to use this replicator:
+ * SQS Replicator - Send data changes to AWS SQS queues
  * 
+ * ⚠️  REQUIRED DEPENDENCY: You must install the AWS SQS SDK:
  * ```bash
- * npm install @aws-sdk/client-sqs
- * # or
- * yarn add @aws-sdk/client-sqs
- * # or
  * pnpm add @aws-sdk/client-sqs
  * ```
  * 
- * @typedef {Object} SQSReplicatorConfig
- * @property {string} region - AWS region where the SQS queues are located
- * @property {string} [accessKeyId] - AWS access key ID (if not using IAM roles)
- * @property {string} [secretAccessKey] - AWS secret access key (if not using IAM roles)
- * @property {string} [sessionToken] - AWS session token for temporary credentials
- * @property {string} [defaultQueueUrl] - Default SQS queue URL for all events when resource-specific queues are not configured
- * @property {Object.<string, string>} [resourceQueues] - Maps s3db resource names to specific SQS queue URLs
- *   - Key: s3db resource name (e.g., 'users', 'orders')
- *   - Value: SQS queue URL (e.g., 'https://sqs.us-east-1.amazonaws.com/123456789012/users-queue')
- *   - If not provided, defaultQueueUrl is used for all resources
- * @property {number} [maxRetries=3] - Maximum number of retry attempts for failed message sends
- * @property {number} [retryDelay=1000] - Delay in milliseconds between retry attempts
- * @property {boolean} [logMessages=false] - Whether to log message details to console for debugging
- * @property {number} [messageDelaySeconds=0] - Delay in seconds before messages become visible in queue
- * @property {Object} [messageAttributes] - Additional attributes to include with every SQS message
- *   - Key: attribute name (e.g., 'environment', 'version')
- *   - Value: attribute value (e.g., 'production', '1.0.0')
- * @property {string} [messageGroupId] - Message group ID for FIFO queues (required for FIFO queues)
- * @property {boolean} [useFIFO=false] - Whether the target queues are FIFO queues
- * @property {number} [batchSize=10] - Number of messages to send in a single batch (for batch operations)
- * @property {boolean} [compressMessages=false] - Whether to compress message bodies using gzip
- * @property {string} [messageFormat='json'] - Format for message body: 'json' or 'stringified'
- * @property {Object} [sqsClientOptions] - Additional options to pass to the SQS client constructor
- * 
- * @example
- * // Configuration with resource-specific queues
- * {
- *   region: 'us-east-1',
- *   accessKeyId: 'AKIAIOSFODNN7EXAMPLE',
- *   secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
- *   resourceQueues: {
- *     'users': 'https://sqs.us-east-1.amazonaws.com/123456789012/users-events',
- *     'orders': 'https://sqs.us-east-1.amazonaws.com/123456789012/orders-events',
- *     'products': 'https://sqs.us-east-1.amazonaws.com/123456789012/products-events'
- *   },
- *   logMessages: true,
- *   messageAttributes: {
- *     'environment': 'production',
- *     'source': 's3db-replicator'
- *   }
- * }
- * 
- * @example
- * // Configuration with single default queue
- * {
- *   region: 'us-west-2',
- *   defaultQueueUrl: 'https://sqs.us-west-2.amazonaws.com/123456789012/all-events',
- *   maxRetries: 5,
- *   retryDelay: 2000,
- *   compressMessages: true
- * }
- * 
- * @example
- * // FIFO queue configuration
- * {
- *   region: 'eu-west-1',
- *   defaultQueueUrl: 'https://sqs.eu-west-1.amazonaws.com/123456789012/events.fifo',
- *   useFIFO: true,
- *   messageGroupId: 's3db-events',
- *   messageDelaySeconds: 5
- * }
+ * Configuration:
+ * @param {string} region - AWS region (required)
+ * @param {string} queueUrl - Single queue URL for all resources
+ * @param {Object} queues - Resource-specific queue mapping { resource: queueUrl }
+ * @param {string} defaultQueueUrl - Fallback queue URL
+ * @param {string} messageGroupId - Message group ID for FIFO queues
+ * @param {boolean} deduplicationId - Enable deduplication for FIFO queues
+ * @param {Object} credentials - AWS credentials (optional, uses default if omitted)
  * 
  * @example
- * // Minimal configuration using IAM roles
- * {
+ * new SqsReplicator({
  *   region: 'us-east-1',
- *   defaultQueueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue'
- * }
+ *   queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/events-queue'
+ * }, ['users', 'orders'])
  * 
- * @notes
- * - Requires AWS credentials with SQS SendMessage permissions
- * - Resource-specific queues take precedence over defaultQueueUrl
- * - Message structure includes: resource, action, data, before (for updates), timestamp, source
- * - FIFO queues require messageGroupId and ensure strict ordering
- * - Message compression reduces bandwidth but increases CPU usage
- * - Batch operations improve performance but may fail if any message in batch fails
- * - Retry mechanism uses exponential backoff for failed sends
- * - Message attributes are useful for filtering and routing in SQS
- * - Message delay is useful for implementing eventual consistency patterns
- * - SQS client options allow for custom endpoint, credentials, etc.
- */
-import BaseReplicator from './base-replicator.class.js';
-import tryFn from "../../concerns/try-fn.js";
-
-/**
- * SQS Replicator - Sends data to AWS SQS queues with support for resource-specific queues
- * 
- * Configuration options:
- * - queueUrl: Single queue URL for all resources
- * - queues: Object mapping resource names to specific queue URLs
- * - defaultQueueUrl: Fallback queue URL when resource-specific queue is not found
- * - messageGroupId: For FIFO queues
- * - deduplicationId: For FIFO queues
- * 
- * Example configurations:
- * 
- * // Single queue for all resources
- * {
- *   queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue'
- * }
- * 
- * // Resource-specific queues
- * {
- *   queues: {
- *     users: 'https://sqs.us-east-1.amazonaws.com/123456789012/users-queue',
- *     orders: 'https://sqs.us-east-1.amazonaws.com/123456789012/orders-queue'
- *   },
- *   defaultQueueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/default-queue'
- * }
+ * See PLUGINS.md for comprehensive configuration documentation.
  */
 class SqsReplicator extends BaseReplicator {
   constructor(config = {}, resources = [], client = null) {
     super(config);
-    this.resources = resources;
     this.client = client;
     this.queueUrl = config.queueUrl;
     this.queues = config.queues || {};
@@ -138,13 +38,26 @@ class SqsReplicator extends BaseReplicator {
     this.messageGroupId = config.messageGroupId;
     this.deduplicationId = config.deduplicationId;
     
-    // Build queues from resources configuration
-    if (resources && typeof resources === 'object') {
+    // Normalize resources to object format
+    if (Array.isArray(resources)) {
+      this.resources = {};
+      for (const resource of resources) {
+        if (typeof resource === 'string') {
+          this.resources[resource] = true;
+        } else if (typeof resource === 'object' && resource.name) {
+          this.resources[resource.name] = resource;
+        }
+      }
+    } else if (typeof resources === 'object') {
+      this.resources = resources;
+      // Build queues from resources configuration
       for (const [resourceName, resourceConfig] of Object.entries(resources)) {
-        if (resourceConfig.queueUrl) {
+        if (resourceConfig && resourceConfig.queueUrl) {
           this.queues[resourceName] = resourceConfig.queueUrl;
         }
       }
+    } else {
+      this.resources = {};
     }
   }
 
@@ -396,7 +309,7 @@ class SqsReplicator extends BaseReplicator {
       connected: !!this.sqsClient,
       queueUrl: this.queueUrl,
       region: this.region,
-      resources: this.resources,
+      resources: Object.keys(this.resources || {}),
       totalreplicators: this.listenerCount('replicated'),
       totalErrors: this.listenerCount('replicator_error')
     };

package/src/resource.class.js

@@ -171,7 +171,7 @@ export class Resource extends EventEmitter {
             if (typeof fn === 'function') {
               this.hooks[event].push(fn.bind(this));
             }
-            // Se não for função, ignore silenciosamente
+            // If not a function, ignore silently
           }
         }
       }
@@ -704,7 +704,7 @@ export class Resource extends EventEmitter {
     // LOG: body e contentType antes do putObject
     // Only throw if behavior is 'body-only' and body is empty
     if (this.behavior === 'body-only' && (!body || body === "")) {
-      throw new Error(`[Resource.insert] Tentativa de gravar objeto sem body! Dados: id=${finalId}, resource=${this.name}`);
+      throw new Error(`[Resource.insert] Attempt to save object without body! Data: id=${finalId}, resource=${this.name}`);
     }
     // For other behaviors, allow empty body (all data in metadata)
     // Before putObject in insert
@@ -753,7 +753,7 @@ export class Resource extends EventEmitter {
 
     // Execute afterInsert hooks
     const finalResult = await this.executeHooks('afterInsert', insertedData);
-    // Emit event com dados antes dos hooks afterInsert
+    // Emit event with data before afterInsert hooks
     this.emit("insert", {
       ...insertedData,
       $before: { ...completeData },
@@ -774,7 +774,7 @@ export class Resource extends EventEmitter {
     if (isEmpty(id)) throw new Error('id cannot be empty');
     
     const key = this.getResourceKey(id);
-    // LOG: início do get
+    // LOG: start of get
     // eslint-disable-next-line no-console
     const [ok, err, request] = await tryFn(() => this.client.getObject(key));
     // LOG: resultado do headObject
@@ -788,7 +788,7 @@ export class Resource extends EventEmitter {
         id
       });
     }
-    // Se o objeto existe mas não tem conteúdo, lançar erro NoSuchKey
+    // If object exists but has no content, throw NoSuchKey error
     if (request.ContentLength === 0) {
       const noContentErr = new Error(`No such key: ${key} [bucket:${this.client.config.bucket}]`);
       noContentErr.name = 'NoSuchKey';
@@ -1346,7 +1346,7 @@ export class Resource extends EventEmitter {
         prefix = `resource=${this.name}/partition=${partition}`;
       }
     } else {
-      // List from main resource (sem versão no path)
+      // List from main resource (without version in path)
       prefix = `resource=${this.name}/data`;
     }
     // Use getKeysPage for real pagination support
@@ -1892,7 +1892,7 @@ export class Resource extends EventEmitter {
     for (const [partitionName, partition] of Object.entries(partitions)) {
       const partitionKey = this.getPartitionKey({ partitionName, id: data.id, data });
       if (partitionKey) {
-        // Salvar apenas a versão como metadado, nunca atributos do objeto
+        // Save only version as metadata, never object attributes
         const partitionMetadata = {
           _v: String(this.version)
         };
@@ -2082,7 +2082,7 @@ export class Resource extends EventEmitter {
       // Create new partition reference if new key exists
       if (newPartitionKey) {
         const [ok, err] = await tryFn(async () => {
-          // Salvar apenas a versão como metadado
+          // Save only version as metadata
           const partitionMetadata = {
             _v: String(this.version)
           };
@@ -2101,7 +2101,7 @@ export class Resource extends EventEmitter {
     } else if (newPartitionKey) {
       // If partition keys are the same, just update the existing reference
       const [ok, err] = await tryFn(async () => {
-        // Salvar apenas a versão como metadado
+        // Save only version as metadata
         const partitionMetadata = {
           _v: String(this.version)
         };
@@ -2138,7 +2138,7 @@ export class Resource extends EventEmitter {
       }
       const partitionKey = this.getPartitionKey({ partitionName, id: data.id, data });
       if (partitionKey) {
-        // Salvar apenas a versão como metadado
+        // Save only version as metadata
         const partitionMetadata = {
           _v: String(this.version)
         };
@@ -2306,8 +2306,8 @@ export class Resource extends EventEmitter {
   }
 
   /**
-   * Compose the full object (metadata + body) as retornado por .get(),
-   * usando os dados em memória após insert/update, de acordo com o behavior
+   * Compose the full object (metadata + body) as returned by .get(),
+   * using in-memory data after insert/update, according to behavior
    */
   async composeFullObjectFromWrite({ id, metadata, body, behavior }) {
     // Preserve behavior flags before unmapping
@@ -2464,7 +2464,7 @@ export class Resource extends EventEmitter {
     this._middlewares.get(method).push(fn);
   }
 
-  // Utilitário para aplicar valores default do schema
+  // Utility to apply schema default values
   applyDefaults(data) {
     const out = { ...data };
     for (const [key, def] of Object.entries(this.attributes)) {
@@ -2473,7 +2473,7 @@ export class Resource extends EventEmitter {
           const match = def.match(/default:([^|]+)/);
           if (match) {
             let val = match[1];
-            // Conversão para boolean/number se necessário
+            // Convert to boolean/number if necessary
             if (def.includes('boolean')) val = val === 'true';
             else if (def.includes('number')) val = Number(val);
             out[key] = val;

package/src/s3db.d.ts

@@ -418,23 +418,22 @@ declare module 's3db.js' {
   export interface BigQueryReplicatorConfig {
     projectId: string;
     datasetId: string;
-    keyFilename?: string;
     credentials?: Record<string, any>;
-    tableMapping?: Record<string, string>;
-    logOperations?: boolean;
+    location?: string;
+    logTable?: string;
     batchSize?: number;
     maxRetries?: number;
-    retryDelay?: number;
-    writeDisposition?: 'WRITE_TRUNCATE' | 'WRITE_APPEND' | 'WRITE_EMPTY';
-    createDisposition?: 'CREATE_IF_NEEDED' | 'CREATE_NEVER';
-    schema?: Record<string, any>[];
-    location?: string;
-    clustering?: string[];
-    partitioning?: {
-      type: 'DAY' | 'HOUR' | 'MONTH' | 'YEAR';
-      field?: string;
-    };
-    labels?: Record<string, string>;
+    writeDisposition?: string;
+    createDisposition?: string;
+    tableMapping?: Record<string, string>;
+    logOperations?: boolean;
+  }
+
+  /** BigQuery Resource Configuration */
+  export interface BigQueryResourceConfig {
+    table: string;
+    actions?: ('insert' | 'update' | 'delete')[];
+    transform?: (data: any) => any;
   }
 
   /** Postgres Replicator config */
@@ -1041,7 +1040,7 @@ declare module 's3db.js' {
 
   /** BigQuery Replicator class */
   export class BigqueryReplicator extends BaseReplicator {
-    constructor(config: BigQueryReplicatorConfig);
+    constructor(config: BigQueryReplicatorConfig, resources: Record<string, string | BigQueryResourceConfig | BigQueryResourceConfig[]>);
   }
 
   /** Postgres Replicator class */

package/src/schema.class.js

@@ -484,7 +484,7 @@ export class Schema {
    */
   static _importAttributes(attrs) {
     if (typeof attrs === 'string') {
-      // Tenta detectar se é um objeto serializado como string JSON
+      // Try to detect if it's an object serialized as JSON string
       const [ok, err, parsed] = tryFnSync(() => JSON.parse(attrs));
       if (ok && typeof parsed === 'object' && parsed !== null) {
         const [okNested, errNested, nested] = tryFnSync(() => Schema._importAttributes(parsed));
@@ -687,9 +687,9 @@ export class Schema {
           properties: this.preprocessAttributesForValidation(value),
           strict: false
         };
-        // Se for explicitamente required, não marca como opcional
+        // If explicitly required, don't mark as optional
         if (isExplicitRequired) {
-          // nada
+          // nothing
         } else if (isExplicitOptional || this.allNestedObjectsOptional) {
           objectConfig.optional = true;
         }