unnbound-events 1.0.15 → 1.0.16

package/README.md

@@ -156,6 +156,75 @@ interface QueueEnvelope {
 }
 ```
 
+## S3 Payload Offloading for Large Messages
+
+The SDK supports automatic handling of large payloads (>256KB) stored in S3, which is necessary since SQS has a 256KB message size limit.
+
+### Configuration
+
+Configure S3 support when starting the client:
+
+```ts
+await client.start({
+  sqs: { queueUrl: 'https://sqs.us-west-2.amazonaws.com/123456789012/my-queue' },
+  s3: {
+    bucketName: 'my-payload-bucket',
+    region: 'us-west-2',
+  },
+});
+```
+
+Or with `sqsListen()`:
+
+```ts
+await client.sqsListen({
+  queueUrl: 'https://sqs.us-west-2.amazonaws.com/123456789012/my-queue',
+  s3: {
+    bucketName: 'my-payload-bucket',
+    region: 'us-west-2',
+  },
+});
+```
+
+### Supported Formats
+
+The SDK supports two S3 payload formats:
+
+1. **ExtendedSQSClient Pointer Format** (AWS Extended Client Library compatible):
+   ```json
+   ["software.amazon.payloadoffloading.PayloadS3Pointer", {
+     "s3BucketName": "bucket-name",
+     "s3Key": "key-path"
+   }]
+   ```
+
+2. **Envelope-based S3 Payloads** (Unnbound format):
+   ```json
+   {
+     "payload": {
+       "type": "s3-single",
+       "size": 512000,
+       "compressed": true,
+       "compressionType": "gzip",
+       "location": {
+         "bucket": "bucket-name",
+         "key": "key-path",
+         "region": "us-west-2",
+         "versionId": "optional-version-id"
+       }
+     },
+     "request": { ... }
+   }
+   ```
+
+### Compression Support
+
+The SDK automatically decompresses payloads with:
+- `brotli` compression
+- `gzip` compression
+
+When the `compressed` flag is `true` and `compressionType` is specified, the SDK will automatically decompress the payload after fetching from S3.
+
 ## Environment Variables
 
 ### SQS Configuration
@@ -184,6 +253,26 @@ export UNNBOUND_SQS_QUEUE_URL="https://sqs.us-west-2.amazonaws.com/123456789012/
 export AWS_REGION="us-west-2"
 ```
 
+### S3 Configuration
+
+For S3 payload support, configure:
+
+**Required:**
+
+- `UNNBOUND_S3_BUCKET` or `S3_BUCKET` - S3 bucket name for payload storage
+
+**Optional:**
+
+- `AWS_REGION` - AWS region (defaults to `us-east-1`)
+- `AWS_S3_ENDPOINT` - Custom S3 endpoint (for LocalStack, etc.)
+
+**Example:**
+
+```bash
+export UNNBOUND_S3_BUCKET="my-payload-bucket"
+export AWS_REGION="us-west-2"
+```
+
 ### AWS Credentials
 
 When running in ECS with a Task Role, no additional credentials are needed. The AWS SDK will automatically use the ECS Task Role credentials.

package/dist/lib/client.js

@@ -46,9 +46,91 @@ const shouldIgnorePath = (path, patterns) => {
 };
 exports.shouldIgnorePath = shouldIgnorePath;
 exports.defaultIgnoreTraceRoutes = ['/health', '/healthcheck'];
+// Helper function to create S3 client
+function createS3Client(options) {
+    const awsS3 = (() => {
+        try {
+            return require('@aws-sdk/client-s3');
+        }
+        catch {
+            return null;
+        }
+    })();
+    if (!awsS3) {
+        throw new Error('@aws-sdk/client-s3 is required for S3 payload support');
+    }
+    const env = globalThis.process
+        ?.env ?? {};
+    const region = options.region || env.AWS_REGION || 'us-east-1';
+    const endpoint = options.endpoint || env.AWS_S3_ENDPOINT || env.AWS_ENDPOINT_URL || undefined;
+    return new awsS3.S3Client({ region, endpoint });
+}
+// Helper function to fetch payload from S3
+async function fetchS3Payload(s3Client, bucket, key, versionId) {
+    const awsS3 = require('@aws-sdk/client-s3');
+    const params = {
+        Bucket: bucket,
+        Key: key,
+    };
+    if (versionId) {
+        params.VersionId = versionId;
+    }
+    try {
+        const response = await s3Client.send(new awsS3.GetObjectCommand(params));
+        const body = response.Body;
+        // Convert stream to string
+        const chunks = [];
+        for await (const chunk of body) {
+            chunks.push(chunk);
+        }
+        const buffer = Buffer.concat(chunks);
+        return buffer.toString('utf-8');
+    }
+    catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        unnbound_logger_sdk_1.logger.error({ err, bucket, key }, 'Failed to fetch S3 payload');
+        throw new Error(`Failed to fetch S3 payload: ${err.message}`);
+    }
+}
+// Helper function to decompress payload
+async function decompressPayload(compressed, compressionType) {
+    const zlib = (() => {
+        try {
+            return require('zlib');
+        }
+        catch {
+            return null;
+        }
+    })();
+    if (!zlib) {
+        throw new Error('zlib is required for decompression');
+    }
+    return new Promise((resolve, reject) => {
+        const buffer = Buffer.from(compressed);
+        const callback = (err, result) => {
+            if (err) {
+                reject(err);
+            }
+            else {
+                resolve(result.toString('utf-8'));
+            }
+        };
+        if (compressionType === 'brotli') {
+            zlib.brotliDecompress(buffer, callback);
+        }
+        else if (compressionType === 'gzip') {
+            zlib.gunzip(buffer, callback);
+        }
+        else {
+            reject(new Error(`Unsupported compression type: ${compressionType}`));
+        }
+    });
+}
 function createEventsClient(ignoreTraceRoutes = exports.defaultIgnoreTraceRoutes) {
     const routes = [];
     const middlewares = [];
+    let s3Client = null;
+    let s3Options = null;
     routes.push({
         method: 'GET',
         matcher: '/healthcheck',
@@ -265,7 +347,59 @@ function createEventsClient(ignoreTraceRoutes = exports.defaultIgnoreTraceRoutes
                 const failures = [];
                 await Promise.all(event.Records.map(async (record) => {
                     try {
-                        const envelope = JSON.parse(record.body);
+                        // Check if this is an S3 pointer message (ExtendedSQSClient format)
+                        let envelope;
+                        let recordBody = record.body;
+                        try {
+                            const parsed = JSON.parse(recordBody);
+                            // Detect S3 pointer format: ["software.amazon.payloadoffloading.PayloadS3Pointer", {...}]
+                            if (Array.isArray(parsed) && parsed.length === 2 &&
+                                parsed[0] === 'software.amazon.payloadoffloading.PayloadS3Pointer') {
+                                const pointer = parsed[1];
+                                if (!s3Client) {
+                                    throw new Error('S3 client not initialized but S3 pointer message received');
+                                }
+                                unnbound_logger_sdk_1.logger.info({ bucket: pointer.s3BucketName, key: pointer.s3Key }, 'Fetching S3 pointer payload');
+                                recordBody = await fetchS3Payload(s3Client, pointer.s3BucketName, pointer.s3Key);
+                                envelope = JSON.parse(recordBody);
+                            }
+                            else {
+                                envelope = parsed;
+                            }
+                        }
+                        catch (error) {
+                            const err = error instanceof Error ? error : new Error(String(error));
+                            unnbound_logger_sdk_1.logger.error({ err }, 'Failed to parse SQS record body');
+                            throw error;
+                        }
+                        // Check if envelope payload is stored in S3
+                        if (envelope.payload.type === 's3-single' || envelope.payload.type === 's3-multipart') {
+                            if (!envelope.payload.location) {
+                                throw new Error(`Payload type is ${envelope.payload.type} but no location specified`);
+                            }
+                            if (!s3Client) {
+                                throw new Error('S3 client not initialized but S3 payload message received');
+                            }
+                            unnbound_logger_sdk_1.logger.info({
+                                type: envelope.payload.type,
+                                bucket: envelope.payload.location.bucket,
+                                key: envelope.payload.location.key
+                            }, 'Fetching S3 envelope payload');
+                            let payloadData = await fetchS3Payload(s3Client, envelope.payload.location.bucket, envelope.payload.location.key, envelope.payload.location.versionId);
+                            // Handle decompression if needed
+                            if (envelope.payload.compressed && envelope.payload.compressionType) {
+                                unnbound_logger_sdk_1.logger.info({ compressionType: envelope.payload.compressionType }, 'Decompressing payload');
+                                payloadData = await decompressPayload(new TextEncoder().encode(payloadData), envelope.payload.compressionType);
+                            }
+                            // Parse the payload and set it as the request body
+                            try {
+                                envelope.request.body = JSON.parse(payloadData);
+                            }
+                            catch {
+                                envelope.request.body = payloadData;
+                            }
+                        }
+                        // Continue with standard envelope processing
                         const headerMessageId = getHeaderValue(envelope.request.headers, MESSAGE_HEADER_KEY);
                         const envelopeMessageId = typeof envelope.metadata?.messageId === 'string'
                             ? envelope.metadata.messageId
@@ -332,6 +466,27 @@ function createEventsClient(ignoreTraceRoutes = exports.defaultIgnoreTraceRoutes
             };
         },
         sqsListen(options) {
+            // Initialize S3 client if options provided
+            if (options?.s3) {
+                s3Options = options.s3;
+                const env = globalThis
+                    .process?.env ?? {};
+                const bucketName = options.s3.bucketName || env.UNNBOUND_S3_BUCKET || env.S3_BUCKET;
+                if (!bucketName) {
+                    unnbound_logger_sdk_1.logger.warn('S3 options provided but no bucket name specified. S3 payload support disabled.');
+                }
+                else {
+                    try {
+                        s3Client = createS3Client({ ...options.s3, bucketName });
+                        unnbound_logger_sdk_1.logger.info({ bucket: bucketName }, 'S3 client initialized for payload offloading');
+                    }
+                    catch (error) {
+                        const err = error instanceof Error ? error : new Error(String(error));
+                        unnbound_logger_sdk_1.logger.error({ err }, 'Failed to initialize S3 client');
+                        throw error;
+                    }
+                }
+            }
             const awsSqs = (() => {
                 try {
                     return require('@aws-sdk/client-sqs');
@@ -425,6 +580,10 @@ function createEventsClient(ignoreTraceRoutes = exports.defaultIgnoreTraceRoutes
             // Default to starting both HTTP and SQS if no options provided
             const httpOptions = options?.http ?? { port: 3000 };
             const sqsOptions = options?.sqs ?? {};
+            // Pass S3 options through to SQS listener if provided
+            const sqsListenOptions = options?.s3
+                ? { ...sqsOptions, s3: options.s3 }
+                : sqsOptions;
             // Start HTTP server
             let httpServer;
             try {
@@ -436,7 +595,7 @@ function createEventsClient(ignoreTraceRoutes = exports.defaultIgnoreTraceRoutes
             // Start SQS listener
             let sqsListener;
             try {
-                sqsListener = await this.sqsListen(sqsOptions);
+                sqsListener = await this.sqsListen(sqsListenOptions);
             }
             catch (error) {
                 unnbound_logger_sdk_1.logger.warn({ error }, 'Failed to start SQS listener, continuing without it');

package/dist/lib/types.d.ts

@@ -32,6 +32,11 @@ export type SqsRecord = {
 export interface SqsBatchEvent {
     Records: SqsRecord[];
 }
+export interface S3PayloadOptions {
+    bucketName?: string;
+    region?: string;
+    endpoint?: string;
+}
 export interface StartOptions {
     http?: {
         port?: number;
@@ -43,6 +48,7 @@ export interface StartOptions {
         maxMessages?: number;
         visibilityTimeoutSeconds?: number;
     };
+    s3?: S3PayloadOptions;
 }
 export interface EventsClient {
     on: (method: HttpMethod, matcher: RouteMatcher, handler: RouteHandler<any, any>) => void;
@@ -76,6 +82,7 @@ export interface EventsClient {
         waitTimeSeconds?: number;
         maxMessages?: number;
         visibilityTimeoutSeconds?: number;
+        s3?: S3PayloadOptions;
     }) => Promise<{
         stop: () => Promise<void>;
     }>;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "unnbound-events",
   "description": "Unified events SDK to handle HTTP routes and SQS messages with a single routing API.",
-  "version": "1.0.15",
+  "version": "1.0.16",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
@@ -36,11 +36,15 @@
     "unnbound-logger-sdk": "^3.0.22"
   },
   "peerDependencies": {
-    "express": "^4.0.0 || ^5.0.0"
+    "express": "^4.0.0 || ^5.0.0",
+    "@aws-sdk/client-s3": "^3.0.0"
   },
   "peerDependenciesMeta": {
     "express": {
       "optional": true
+    },
+    "@aws-sdk/client-s3": {
+      "optional": true
     }
   },
   "devDependencies": {