serverless-plugin-module-registry 1.0.12 → 1.0.13

package/src/internal-infrastructure/handlers/stream-processor.ts

@@ -0,0 +1,371 @@
+import { DynamoDBStreamEvent, DynamoDBRecord } from 'aws-lambda'
+import { SQSClient, SendMessageCommand } from '@aws-sdk/client-sqs'
+import { EventBridgeClient, PutEventsCommand } from '@aws-sdk/client-eventbridge'
+import { createLogger } from './shared/logger'
+import { RoleUpdateMessage, FeatureChangeEvent } from './shared/types'
+
+const logger = createLogger('stream-processor')
+const sqsClient = new SQSClient({})
+const eventBridgeClient = new EventBridgeClient({})
+
+/**
+ * Parse SK to extract module name and feature ID
+ * SK Format: MODULE#{moduleName}#FEATURE#{featureId}
+ */
+function parseModuleAndFeature(sk: string): { moduleName: string; featureId: string } | null {
+  const parts = sk.split('#')
+  if (parts.length !== 4 || parts[0] !== 'MODULE' || parts[2] !== 'FEATURE') {
+    return null
+  }
+  return {
+    moduleName: parts[1],
+    featureId: parts[3],
+  }
+}
+
+/**
+ * Extract role name from PK
+ * PK Format: ROLE#{roleName}
+ */
+function extractRoleName(pk: string): string {
+  return pk.replace('ROLE#', '')
+}
+
+/**
+ * Filter DynamoDB Stream records for ROLE# items
+ */
+function filterRoleRecords(records: DynamoDBRecord[]): DynamoDBRecord[] {
+  return records.filter(record => {
+    const pk = record.dynamodb?.Keys?.PK?.S || record.dynamodb?.Keys?.pk?.S
+    return pk?.startsWith('ROLE#') ?? false
+  })
+}
+
+/**
+ * Filter DynamoDB Stream records for FEATURE# items
+ * PK Format: MODULE#{moduleName}
+ * SK Format: Must contain #FEATURE# pattern
+ */
+function filterFeatureRecords(records: DynamoDBRecord[]): DynamoDBRecord[] {
+  return records.filter(record => {
+    const pk = record.dynamodb?.Keys?.PK?.S || record.dynamodb?.Keys?.pk?.S
+    const sk = record.dynamodb?.Keys?.SK?.S || record.dynamodb?.Keys?.sk?.S
+
+    // PK must start with MODULE# (not ROLE#)
+    const isModulePK = pk?.startsWith('MODULE#') ?? false
+    // SK must contain #FEATURE# pattern
+    const isFeatureSK = sk?.includes('#FEATURE#') ?? false
+
+    return isModulePK && isFeatureSK
+  })
+}
+
+/**
+ * Parse feature event from DynamoDB record
+ * PK Format: MODULE#{moduleName}
+ * SK Format: MODULE#*#FEATURE#{featureId} or similar patterns containing #FEATURE#
+ * Returns null for invalid patterns
+ */
+function parseFeatureEvent(record: DynamoDBRecord): FeatureChangeEvent | null {
+  try {
+    // Extract PK (moduleName)
+    const pk = record.dynamodb?.Keys?.PK?.S || record.dynamodb?.Keys?.pk?.S
+    if (!pk || !pk.startsWith('MODULE#')) {
+      logger.warn('Invalid PK format for feature record', { eventID: record.eventID })
+      return null
+    }
+
+    const moduleName = pk.replace('MODULE#', '')
+    if (!moduleName) {
+      logger.warn('Empty moduleName in feature record', { eventID: record.eventID })
+      return null
+    }
+
+    // Extract SK from NewImage (for INSERT/MODIFY) or OldImage (for REMOVE)
+    const image = record.dynamodb?.NewImage || record.dynamodb?.OldImage
+    const sk = image?.SK?.S || image?.sk?.S
+
+    if (!sk || !sk.includes('#FEATURE#')) {
+      logger.warn('Invalid SK format for feature record', { eventID: record.eventID, sk })
+      return null
+    }
+
+    // Extract featureId from SK
+    // SK patterns can vary: MODULE#*#FEATURE#{featureId} or other patterns containing #FEATURE#
+    const featureMatch = sk.match(/#FEATURE#([^#]+)/)
+    if (!featureMatch || !featureMatch[1]) {
+      logger.warn('Failed to extract featureId from SK', { eventID: record.eventID, sk })
+      return null
+    }
+
+    const featureId = featureMatch[1]
+
+    // Extract eventType
+    const eventType = record.eventName as 'INSERT' | 'MODIFY' | 'REMOVE'
+
+    return {
+      moduleName,
+      featureId,
+      eventType,
+      timestamp: new Date().toISOString(),
+    }
+  } catch (error) {
+    logger.error('Error parsing feature event', error as Error, { eventID: record.eventID })
+    return null
+  }
+}
+
+/**
+ * Deduplicate messages by roleName + moduleName
+ * Keep only the latest event for each combination
+ */
+function deduplicateMessages(messages: RoleUpdateMessage[]): RoleUpdateMessage[] {
+  const messageMap = new Map<string, RoleUpdateMessage>()
+
+  for (const message of messages) {
+    const key = `${message.roleName}:${message.moduleName}`
+    const existing = messageMap.get(key)
+
+    // Keep the latest message (comparing timestamps)
+    if (!existing || new Date(message.timestamp) > new Date(existing.timestamp)) {
+      messageMap.set(key, message)
+    }
+  }
+
+  return Array.from(messageMap.values())
+}
+
+/**
+ * Publish feature change events to EventBridge
+ * Wrapped in try/catch to isolate failures and prevent breaking ROLE# processing
+ */
+async function publishFeatureChangeEvents(
+  events: FeatureChangeEvent[],
+  eventBusName: string
+): Promise<number> {
+  if (events.length === 0) {
+    return 0
+  }
+
+  let publishedCount = 0
+
+  try {
+    // Batch publish events to EventBridge (max 10 entries per PutEvents call)
+    const batchSize = 10
+    for (let i = 0; i < events.length; i += batchSize) {
+      const batch = events.slice(i, i + batchSize)
+
+      const putEventsCommand = new PutEventsCommand({
+        Entries: batch.map(event => ({
+          Source: 'module-registry',
+          DetailType: 'FeatureChange',
+          Detail: JSON.stringify({
+            moduleName: event.moduleName,
+            featureId: event.featureId,
+            eventType: event.eventType,
+            timestamp: event.timestamp,
+          }),
+          EventBusName: eventBusName,
+        })),
+      })
+
+      await eventBridgeClient.send(putEventsCommand)
+      publishedCount += batch.length
+
+      for (const event of batch) {
+        logger.info('Published feature change event', {
+          moduleName: event.moduleName,
+          featureId: event.featureId,
+          eventType: event.eventType,
+        })
+      }
+    }
+  } catch (error) {
+    // Log error but don't throw - feature event publishing failures should not break ROLE# processing
+    logger.error('Failed to publish feature change events to EventBridge', error as Error, {
+      eventCount: events.length,
+      publishedCount,
+    })
+  }
+
+  return publishedCount
+}
+
+/**
+ * Main Lambda handler for processing DynamoDB Stream events
+ */
+export async function handler(event: DynamoDBStreamEvent): Promise<void> {
+  const startTime = Date.now()
+  const QUEUE_URL = process.env.QUEUE_URL
+
+  logger.info('Processing DynamoDB Stream event', {
+    recordCount: event.Records.length,
+  })
+
+  if (!QUEUE_URL) {
+    logger.error('QUEUE_URL environment variable not set')
+    throw new Error('QUEUE_URL environment variable not set')
+  }
+
+  // Filter for ROLE# records only
+  const roleRecords = filterRoleRecords(event.Records)
+
+  logger.info('Filtered ROLE# records', {
+    originalCount: event.Records.length,
+    filteredCount: roleRecords.length,
+  })
+
+  // Build messages from filtered records
+  const messages: RoleUpdateMessage[] = []
+
+  for (const record of roleRecords) {
+    try {
+      // Extract PK (role name)
+      const pk = record.dynamodb?.Keys?.PK?.S || record.dynamodb?.Keys?.pk?.S
+      if (!pk) {
+        logger.warn('Record missing PK', { eventID: record.eventID })
+        continue
+      }
+
+      const roleName = extractRoleName(pk)
+
+      // Extract SK from NewImage (for INSERT/MODIFY) or OldImage (for REMOVE)
+      const image = record.dynamodb?.NewImage || record.dynamodb?.OldImage
+      const sk = image?.SK?.S || image?.sk?.S
+
+      if (!sk) {
+        logger.warn('Record missing SK', { eventID: record.eventID, roleName })
+        continue
+      }
+
+      // Parse module name from SK
+      const parsed = parseModuleAndFeature(sk)
+      if (!parsed) {
+        logger.warn('Failed to parse SK', { eventID: record.eventID, sk })
+        continue
+      }
+
+      // Build message payload
+      const message: RoleUpdateMessage = {
+        roleName,
+        moduleName: parsed.moduleName,
+        eventType: record.eventName as 'INSERT' | 'MODIFY' | 'REMOVE',
+        timestamp: new Date().toISOString(),
+      }
+
+      messages.push(message)
+
+      logger.info('Parsed Stream record', {
+        eventID: record.eventID,
+        roleName,
+        moduleName: parsed.moduleName,
+        featureId: parsed.featureId,
+        eventType: record.eventName,
+      })
+    } catch (error) {
+      logger.error('Error processing record', error as Error, {
+        eventID: record.eventID,
+      })
+      // Continue processing other records
+    }
+  }
+
+  // Deduplicate messages
+  const originalCount = messages.length
+  const dedupedMessages = deduplicateMessages(messages)
+
+  logger.info('Deduplicated messages', {
+    originalCount,
+    deduplicatedCount: dedupedMessages.length,
+  })
+
+  // Publish messages to SQS
+  let sqsPublishedCount = 0
+  if (dedupedMessages.length > 0) {
+    for (const message of dedupedMessages) {
+      try {
+        await sqsClient.send(new SendMessageCommand({
+          QueueUrl: QUEUE_URL,
+          MessageBody: JSON.stringify(message),
+          MessageGroupId: message.roleName, // For FIFO queues
+          MessageDeduplicationId: `${message.roleName}-${message.moduleName}-${message.timestamp}`,
+        }))
+
+        sqsPublishedCount++
+
+        logger.info('Published SQS message', {
+          roleName: message.roleName,
+          moduleName: message.moduleName,
+          eventType: message.eventType,
+        })
+      } catch (error) {
+        logger.error('Failed to publish SQS message', error as Error, {
+          roleName: message.roleName,
+          moduleName: message.moduleName,
+        })
+        // Continue trying to publish other messages
+      }
+    }
+  }
+
+  // Process FEATURE# records (separate from ROLE# processing)
+  // Wrapped in try/catch to isolate failures and prevent breaking ROLE# processing
+  let featurePublishedCount = 0
+  try {
+    const EVENT_BUS_NAME = process.env.EVENT_BUS_NAME
+
+    if (!EVENT_BUS_NAME) {
+      logger.warn('EVENT_BUS_NAME environment variable not set, skipping feature event processing')
+    } else {
+      const featureStartTime = Date.now()
+
+      // Filter for FEATURE# records
+      const featureRecords = filterFeatureRecords(event.Records)
+
+      logger.info('Filtered FEATURE# records', {
+        originalCount: event.Records.length,
+        filteredCount: featureRecords.length,
+      })
+
+      if (featureRecords.length > 0) {
+        // Parse feature events
+        const featureEvents: FeatureChangeEvent[] = []
+        for (const record of featureRecords) {
+          const parsedEvent = parseFeatureEvent(record)
+          if (parsedEvent) {
+            featureEvents.push(parsedEvent)
+          }
+        }
+
+        logger.info('Parsed feature events', {
+          filteredCount: featureRecords.length,
+          parsedCount: featureEvents.length,
+        })
+
+        // Publish to EventBridge
+        if (featureEvents.length > 0) {
+          featurePublishedCount = await publishFeatureChangeEvents(featureEvents, EVENT_BUS_NAME)
+        }
+
+        const featureDuration = Date.now() - featureStartTime
+        logger.info('Completed feature event processing', {
+          filteredRecords: featureRecords.length,
+          publishedEvents: featurePublishedCount,
+          duration: featureDuration,
+        })
+      }
+    }
+  } catch (error) {
+    // Log error but don't throw - feature processing failures should not break ROLE# processing
+    logger.error('Error during feature event processing', error as Error)
+  }
+
+  const duration = Date.now() - startTime
+
+  logger.info('Completed Stream processing', {
+    processedRecords: event.Records.length,
+    roleMessages: sqsPublishedCount,
+    featureEvents: featurePublishedCount,
+    duration,
+  })
+}

package/src/internal-infrastructure/resources/cloudwatch.yml

@@ -0,0 +1,36 @@
+Resources:
+  # CloudWatch Alarm for DLQ Messages
+  DLQAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Properties:
+      AlarmName: ${self:provider.stackName}-dlq-messages
+      AlarmDescription: Alert when messages appear in the DLQ
+      MetricName: ApproximateNumberOfMessagesVisible
+      Namespace: AWS/SQS
+      Statistic: Average
+      Period: 300
+      EvaluationPeriods: 1
+      Threshold: 1
+      ComparisonOperator: GreaterThanOrEqualToThreshold
+      Dimensions:
+        - Name: QueueName
+          Value: !GetAtt RoleUpdateDLQ.QueueName
+      TreatMissingData: notBreaching
+
+  # CloudWatch Alarm for Role Updater Lambda Errors
+  RoleUpdaterErrorAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Properties:
+      AlarmName: ${self:provider.stackName}-role-updater-errors
+      AlarmDescription: Alert on Role Updater Lambda errors
+      MetricName: Errors
+      Namespace: AWS/Lambda
+      Statistic: Sum
+      Period: 300
+      EvaluationPeriods: 1
+      Threshold: 5
+      ComparisonOperator: GreaterThanThreshold
+      Dimensions:
+        - Name: FunctionName
+          Value: !Ref RoleUpdaterLambdaFunction
+      TreatMissingData: notBreaching

package/src/internal-infrastructure/resources/custom-stream-enabler.yml

@@ -0,0 +1,106 @@
+Resources:
+  # Custom CloudFormation Resource to enable DynamoDB Streams
+  EnableTableStreams:
+    Type: Custom::DynamoDBStreams
+    Properties:
+      ServiceToken: !GetAtt StreamEnablerLambda.Arn
+      TableName: ${param:tableName}
+      StreamViewType: NEW_AND_OLD_IMAGES
+
+  # Lambda function for the custom resource
+  StreamEnablerLambda:
+    Type: AWS::Lambda::Function
+    Properties:
+      FunctionName: ${self:provider.stackName}-stream-enabler
+      Handler: index.handler
+      Runtime: nodejs20.x
+      Role: !GetAtt StreamEnablerRole.Arn
+      Timeout: 60
+      Code:
+        ZipFile: |
+          const { DynamoDBClient, DescribeTableCommand, UpdateTableCommand } = require('@aws-sdk/client-dynamodb');
+          const https = require('https');
+          const url = require('url');
+
+          const client = new DynamoDBClient({});
+
+          async function sendResponse(event, context, status, data, physicalResourceId) {
+            const responseBody = JSON.stringify({
+              Status: status,
+              Reason: `See CloudWatch Log Stream: ${context.logStreamName}`,
+              PhysicalResourceId: physicalResourceId || context.logStreamName,
+              StackId: event.StackId,
+              RequestId: event.RequestId,
+              LogicalResourceId: event.LogicalResourceId,
+              Data: data
+            });
+
+            const parsedUrl = url.parse(event.ResponseURL);
+            const options = {
+              hostname: parsedUrl.hostname,
+              port: 443,
+              path: parsedUrl.path,
+              method: 'PUT',
+              headers: {
+                'Content-Type': '',
+                'Content-Length': responseBody.length
+              }
+            };
+
+            return new Promise((resolve, reject) => {
+              const req = https.request(options, (res) => {
+                resolve();
+              });
+              req.on('error', reject);
+              req.write(responseBody);
+              req.end();
+            });
+          }
+
+          exports.handler = async (event, context) => {
+            console.log('Event:', JSON.stringify(event, null, 2));
+
+            try {
+              const { TableName, StreamViewType } = event.ResourceProperties;
+              const requestType = event.RequestType;
+
+              if (requestType === 'Delete') {
+                // On delete, we optionally disable streams (or leave enabled for safety)
+                // For safety, we'll leave the stream enabled
+                await sendResponse(event, context, 'SUCCESS', {}, TableName);
+                return;
+              }
+
+              // Check if streams are already enabled (idempotent)
+              const describeResult = await client.send(new DescribeTableCommand({ TableName }));
+              const streamEnabled = describeResult.Table?.StreamSpecification?.StreamEnabled;
+              const currentStreamArn = describeResult.Table?.LatestStreamArn;
+
+              if (!streamEnabled) {
+                console.log('Enabling streams on table:', TableName);
+                // Enable streams
+                await client.send(new UpdateTableCommand({
+                  TableName,
+                  StreamSpecification: {
+                    StreamEnabled: true,
+                    StreamViewType
+                  }
+                }));
+
+                // Wait a bit for the stream to be created
+                await new Promise(resolve => setTimeout(resolve, 2000));
+
+                // Get updated table info
+                const updatedTable = await client.send(new DescribeTableCommand({ TableName }));
+                const streamArn = updatedTable.Table?.LatestStreamArn;
+
+                await sendResponse(event, context, 'SUCCESS', { StreamArn: streamArn }, TableName);
+              } else {
+                console.log('Streams already enabled on table:', TableName);
+                await sendResponse(event, context, 'SUCCESS', { StreamArn: currentStreamArn }, TableName);
+              }
+            } catch (error) {
+              console.error('Error:', error);
+              await sendResponse(event, context, 'FAILED', {}, event.PhysicalResourceId);
+            }
+          };

package/src/internal-infrastructure/resources/dynamodb.yml

@@ -0,0 +1,74 @@
+Resources:
+  # DynamoDB Table for Module Registry
+  ModuleRegistryTable:
+    Type: AWS::DynamoDB::Table
+    DeletionPolicy: ${param:deletionPolicy}
+    UpdateReplacePolicy: ${param:deletionPolicy}
+    Properties:
+      TableName: ${param:tableName}
+      BillingMode: PAY_PER_REQUEST
+      PointInTimeRecoverySpecification:
+        PointInTimeRecoveryEnabled: true
+      StreamSpecification:
+        StreamViewType: NEW_AND_OLD_IMAGES
+      AttributeDefinitions:
+        - AttributeName: pk
+          AttributeType: S
+        - AttributeName: sk
+          AttributeType: S
+        - AttributeName: gsi1pk
+          AttributeType: S
+        - AttributeName: gsi1sk
+          AttributeType: S
+        - AttributeName: gsi2pk
+          AttributeType: S
+        - AttributeName: gsi2sk
+          AttributeType: S
+      KeySchema:
+        - AttributeName: pk
+          KeyType: HASH
+        - AttributeName: sk
+          KeyType: RANGE
+      GlobalSecondaryIndexes:
+        - IndexName: GSI1
+          KeySchema:
+            - AttributeName: gsi1pk
+              KeyType: HASH
+            - AttributeName: gsi1sk
+              KeyType: RANGE
+          Projection:
+            ProjectionType: ALL
+        - IndexName: GSI2
+          KeySchema:
+            - AttributeName: gsi2pk
+              KeyType: HASH
+            - AttributeName: gsi2sk
+              KeyType: RANGE
+          Projection:
+            ProjectionType: ALL
+      Tags:
+        - Key: Module
+          Value: module-registry
+        - Key: Purpose
+          Value: StateStore
+        - Key: ManagedBy
+          Value: Serverless
+
+Outputs:
+  ModuleRegistryTableName:
+    Description: Module Registry DynamoDB Table Name
+    Value: !Ref ModuleRegistryTable
+    Export:
+      Name: ${self:provider.stackName}-TableName
+
+  ModuleRegistryTableArn:
+    Description: Module Registry DynamoDB Table ARN
+    Value: !GetAtt ModuleRegistryTable.Arn
+    Export:
+      Name: ${self:provider.stackName}-TableArn
+
+  ModuleRegistryTableStreamArn:
+    Description: Module Registry DynamoDB Table Stream ARN
+    Value: !GetAtt ModuleRegistryTable.StreamArn
+    Export:
+      Name: ${self:provider.stackName}-TableStreamArn

package/src/internal-infrastructure/resources/eventbridge.yml

@@ -0,0 +1,26 @@
+Resources:
+  # EventBridge Bus for Module Registry Feature Events
+  ModuleRegistryEventBus:
+    Type: AWS::Events::EventBus
+    Properties:
+      Name: module-registry-events-${self:provider.stage}
+      Tags:
+        - Key: Module
+          Value: module-registry
+        - Key: Purpose
+          Value: EventBus
+        - Key: ManagedBy
+          Value: Serverless
+
+Outputs:
+  EventBusArn:
+    Description: Module Registry EventBridge EventBus ARN
+    Value: !GetAtt ModuleRegistryEventBus.Arn
+    Export:
+      Name: ${self:provider.stackName}-EventBusArn
+
+  EventBusName:
+    Description: Module Registry EventBridge EventBus Name
+    Value: !Ref ModuleRegistryEventBus
+    Export:
+      Name: ${self:provider.stackName}-EventBusName