fdc-shared-kernel 0.0.89__tar.gz → 0.0.91__tar.gz

{fdc_shared_kernel-0.0.89 → fdc_shared_kernel-0.0.91}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.89
+Version: 0.0.91
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.89 → fdc_shared_kernel-0.0.91}/fdc_shared_kernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fdc_shared_kernel
-Version: 0.0.89
+Version: 0.0.91
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.89 → fdc_shared_kernel-0.0.91}/fdc_shared_kernel.egg-info/SOURCES.txt

@@ -89,4 +89,8 @@ shared_kernel/utils/data_validators_utils.py
 shared_kernel/utils/date_format_utils.py
 shared_kernel/utils/string_utils.py
 shared_kernel/utils/template_renderer.py
-shared_kernel/utils/thread_local_util.py
+shared_kernel/utils/thread_local_util.py
+tests/__init__.py
+tests/messaging/__init__.py
+tests/messaging/utils/__init__.py
+tests/messaging/utils/test_aws_utility.py

{fdc_shared_kernel-0.0.89 → fdc_shared_kernel-0.0.91}/fdc_shared_kernel.egg-info/top_level.txt

@@ -3,3 +3,4 @@ dist
 docs
 release_notes
 shared_kernel
+tests

{fdc_shared_kernel-0.0.89 → fdc_shared_kernel-0.0.91}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fdc_shared_kernel"
-version = "0.0.89"
+version = "0.0.91"
 requires-python = ">=3.7"
 readme = "README_pypi.md"
 description = "Shared library for microservice"

{fdc_shared_kernel-0.0.89 → fdc_shared_kernel-0.0.91}/shared_kernel/event_executor/event_executor.py

@@ -206,7 +206,7 @@ class EventExecutor:
             else:
                 self._stats[event_name].failed_events += 1
 
-    def _callback_wrapper(self, callback: Callable[[Any], None], message: dict) -> None:
+    def _callback_wrapper(self, callback: Callable[[Any], None], event_msg: AWSEventMessage) -> None:
         """
         Wrapper around message processing to handle cleanup and status updates.
         """
@@ -214,8 +214,7 @@ class EventExecutor:
         event_name = None
 
         try:
-            logger.info(f"Initiating callback for message: {message}")
-            event_msg = AWSEventMessage(message)
+            logger.info(f"Initiating callback for message: {event_msg}")
             event_name = event_msg.event_name
 
             # Set thread-local context

{fdc_shared_kernel-0.0.89 → fdc_shared_kernel-0.0.91}/shared_kernel/messaging/aws_databus.py

@@ -120,7 +120,8 @@ class AWSDataBus(DataBus):
     def get_async_message(self, event_name):
         queue = self.event_queue_mapper[event_name]
         # get_message_from_queue should be blocking
-        message = self.aws_utility.get_message_from_queue(queue)
+        message = self.aws_utility.recieve_event(queue)
+        # message = self.aws_utility.get_message_from_queue(queue)
         return message
 
     def delete_message(self, message: AWSEventMessage):

{fdc_shared_kernel-0.0.89 → fdc_shared_kernel-0.0.91}/shared_kernel/messaging/utils/aws_utility.py

@@ -1,12 +1,13 @@
 import json
 from typing import Dict, Optional, Union
+import uuid
 
 import boto3
 from botocore.exceptions import ClientError
 
 from shared_kernel.config import Config
 from shared_kernel.logger import Logger
-from shared_kernel.messaging.utils.event_messages import PublishEventMessage
+from shared_kernel.messaging.utils.event_messages import AWSEventMessage, PublishEventMessage
 
 app_config = Config()
 logger = Logger(app_config.get("APP_NAME"))
@@ -29,11 +30,13 @@ class AWSMessagingUtility:
     def __init__(self):
         self.event_bridge = boto3.client("events")
         self.sqs = boto3.client("sqs")
+        self.s3 = boto3.client("s3")
         self.event_bus_name = app_config.get("EVENT_BUS_NAME")
         self.service_name = app_config.get("APP_NAME")
         self.account_id = app_config.get("ACCOUNT_ID")
         self.region = app_config.get("AWS_REGION")
         self.queue_visiblity_timeout = app_config.get("QUEUE_VISIBLITY_TIMEOUT")
+        self.s3_bucket = app_config.get("AWS_EVENT_PAYLOAD_BUCKET")
 
     def create_queue(self, event_name: str) -> AWSQueue:
         """
@@ -216,6 +219,49 @@ class AWSMessagingUtility:
             f"Message deleted from queue with receipt handle '{receipt_handle}'."
         )
 
+    def _check_payload_size(self, json_payload: str) -> bool:
+        """
+        Check if the payload size exceeds 256 KB.
+        
+        Args:
+            json_payload (str): The JSON payload to check
+            
+        Returns:
+            bool: True if payload size exceeds 256 KB, False otherwise
+        """
+        return len(json_payload.encode('utf-8')) > 262144  # 256 KB in bytes
+
+    def _upload_payload_to_s3(self, json_payload: str) -> str:
+        """
+        Upload large payload to S3 and return the S3 path.
+        
+        Args:
+            json_payload (str): The JSON payload to upload
+            
+        Returns:
+            str: The S3 path where the payload is stored
+        """
+        try:
+            # Generate a unique filename using UUID
+            file_name = f"event_payload/{str(uuid.uuid4())}.txt"
+            
+            # Upload to S3
+            self.s3.put_object(
+                Bucket=self.s3_bucket,
+                Key=file_name,
+                Body=json_payload,
+                ContentType='application/json'
+            )
+            
+            # Return the S3 path
+            s3_path = f"s3://{self.s3_bucket}/{file_name}"
+            logger.info(f"Uploaded large payload to S3: {s3_path}")
+            return s3_path
+            
+        except Exception as e:
+            logger.error(f"Failed to upload payload to S3: {e}")
+            raise
+
     def publish_event(self, event_name: str, event_payload: dict) -> Union[bool, Optional[dict]]:
         """
         Publish an event to EventBridge.
@@ -227,12 +273,26 @@ class AWSMessagingUtility:
         """
         try:
             publish_message = PublishEventMessage(event_payload)
+            json_payload = publish_message.to_json()
+            
+            # Check if payload size exceeds 256 KB
+            if self._check_payload_size(json_payload):
+                # Upload to S3 and get the path
+                s3_path = self._upload_payload_to_s3(json_payload)
+                
+                # Replace the payload with the S3 path and remove event_payload
+                event_payload['event_payload_path'] = s3_path
+                del event_payload['event_payload']
+                publish_message = PublishEventMessage(event_payload)
+                json_payload = publish_message.to_json()
+                logger.warning(f"Event payload size exceeded 256 KB limit. Using S3 path: {s3_path}")
+            
             response: dict = self.event_bridge.put_events(
                 Entries=[
                     {
                         "Source": self.service_name,
                         "DetailType": event_name,
-                        "Detail": publish_message.to_json(),
+                        "Detail": json_payload,
                         "EventBusName": self.event_bus_name,
                     }
                 ]
@@ -256,7 +316,60 @@ class AWSMessagingUtility:
         except Exception as e:
             logger.error(f"Failed to publish event: {e}")
             return False, None
-         
+        
+    def _get_payload_from_s3(self, s3_path: str) -> str:
+        """
+        Retrieve and decode payload from S3 bucket.
+        
+        Args:
+            s3_path (str): The S3 path where the payload is stored
+            
+        Returns:
+            str: The decoded payload content
+            
+        Raises:
+            ClientError: If there's an error accessing the S3 bucket
+            ValueError: If the S3 path is invalid
+        """
+        try:
+            # Extract bucket and key from s3:// path
+            if not s3_path.startswith('s3://'):
+                raise ValueError(f"Invalid S3 path format: {s3_path}")
+                
+            path_parts = s3_path.replace('s3://', '').split('/', 1)
+            if len(path_parts) != 2:
+                raise ValueError(f"Invalid S3 path format: {s3_path}")
+                
+            bucket, key = path_parts
+            
+            s3_object = self.s3.get_object(Bucket=bucket, Key=key)
+            return s3_object["Body"].read().decode("utf-8")
+            
+        except ClientError as e:
+            logger.error(f"Failed to retrieve payload from S3: {e}")
+            raise
+        except Exception as e:
+            logger.error(f"Unexpected error while retrieving payload from S3: {e}")
+            raise
+
+    def recieve_event(self, queue):
+        """
+        Receive and process an event from the queue, handling both direct payloads and S3-stored payloads.
+        
+        Args:
+            queue (AWSQueue): The queue to receive the message from
+            
+        Returns:
+            AWSEventMessage: The processed event message
+        """
+        deserialized_message = self.get_message_from_queue(queue)
+        event_msg = AWSEventMessage(deserialized_message)
+        
+        if event_msg.event_payload_path:
+            event_msg.event_payload = json.loads(self._get_payload_from_s3(event_msg.event_payload_path))["event_payload"]
+            
+        return event_msg
+
     def get_queue_url_arn(self, event_name: str) -> dict:
         queue_name = f"{self.service_name}-{event_name}"
         queue_arn = f"arn:aws:sqs:{self.region}:{self.account_id}:{queue_name}"

{fdc_shared_kernel-0.0.89 → fdc_shared_kernel-0.0.91}/shared_kernel/messaging/utils/event_messages.py

@@ -158,6 +158,7 @@ class EventMessage:
         # event payload will contain entity_id, connector_id and other info
         # needed for specific events
         self.event_payload: dict = raw_message.get("event_payload", {})
+        self.event_payload_path = raw_message.get("event_payload_path", False)
 
         # initialize EventMeta object
         self.event_meta = EventMeta(raw_message.get("event_meta", {}))

fdc_shared_kernel-0.0.91/tests/__init__.py

@@ -0,0 +1,3 @@
+"""
+Test package for shared_kernel.
+""" 

fdc_shared_kernel-0.0.91/tests/messaging/__init__.py

@@ -0,0 +1,3 @@
+"""
+Test package for messaging module.
+""" 

fdc_shared_kernel-0.0.91/tests/messaging/utils/__init__.py

@@ -0,0 +1,3 @@
+"""
+Test package for messaging utils module.
+""" 

fdc_shared_kernel-0.0.91/tests/messaging/utils/test_aws_utility.py

@@ -0,0 +1,247 @@
+import json
+import pytest
+from unittest.mock import patch, MagicMock
+from moto.sqs import mock_sqs
+from moto.events import mock_events
+from moto.s3 import mock_s3
+import boto3
+
+from shared_kernel.messaging.utils.aws_utility import AWSMessagingUtility, AWSQueue
+from shared_kernel.messaging.utils.event_messages import AWSEventMessage, PublishEventMessage
+
+@pytest.fixture
+def aws_utility():
+    mock_config = MagicMock()
+    mock_config.get.side_effect = lambda key: {
+        "APP_NAME": "test-app",
+        "EVENT_BUS_NAME": "test-bus",
+        "ACCOUNT_ID": "123456789012",
+        "AWS_REGION": "us-east-1",
+        "QUEUE_VISIBLITY_TIMEOUT": "30",
+        "AWS_EVENT_PAYLOAD_BUCKET": "test-bucket"
+    }[key]
+    
+    with patch('shared_kernel.messaging.utils.aws_utility.app_config', mock_config):
+        utility = AWSMessagingUtility()
+        return utility
+
+@pytest.fixture
+def sample_event_payload():
+    return {
+        "event_name": "test-event",
+        "event_id": "test-event-id",
+        "event_payload": {"key": "value"},
+        "event_meta": {
+            "trace_id": "test-trace-id",
+            "span_id": "test-span-id",
+            "timestamp": "2024-04-17T13:54:14.479Z"
+        }
+    }
+
+@pytest.fixture
+def setup_s3(aws_utility):
+    with mock_s3():
+        s3 = boto3.client('s3', region_name='us-east-1')
+        s3.create_bucket(Bucket='test-bucket')
+        yield s3
+
+@pytest.fixture
+def setup_sqs(aws_utility):
+    with mock_sqs():
+        sqs = boto3.client('sqs', region_name='us-east-1')
+        yield sqs
+
+@pytest.fixture
+def setup_events(aws_utility):
+    with mock_events():
+        events = boto3.client('events', region_name='us-east-1')
+        events.create_event_bus(Name='test-bus')
+        yield events
+
+class TestAWSQueue:
+    def test_aws_queue_initialization(self):
+        queue = AWSQueue("test-arn", "test-url", "test-event")
+        assert queue.arn == "test-arn"
+        assert queue.url == "test-url"
+        assert queue.event_name == "test-event"
+
+    def test_aws_queue_is_valid(self):
+        valid_queue = AWSQueue("test-arn", "test-url", "test-event")
+        assert valid_queue.is_valid()
+
+        invalid_queue = AWSQueue(None, "test-url", "test-event")
+        assert not invalid_queue.is_valid()
+
+        invalid_queue = AWSQueue("test-arn", None, "test-event")
+        assert not invalid_queue.is_valid()
+
+class TestQueueOperations:
+    def test_create_queue(self, aws_utility, setup_sqs):
+        queue = aws_utility.create_queue("test-event")
+        assert isinstance(queue, AWSQueue)
+        assert queue.event_name == "test-event"
+        assert "test-app-test-event" in queue.url
+
+    def test_get_queue(self, aws_utility, setup_sqs):
+        # First create a queue
+        created_queue = aws_utility.create_queue("test-event")
+        # Then get it
+        queue = aws_utility.get_queue("test-event")
+        assert queue.url == created_queue.url
+        assert queue.arn == created_queue.arn
+
+    def test_check_if_queue_exist(self, aws_utility, setup_sqs):
+        # Queue doesn't exist initially
+        assert aws_utility.check_if_queue_exist("test-event") is False
+        
+        # Create queue
+        aws_utility.create_queue("test-event")
+        assert aws_utility.check_if_queue_exist("test-event") is True
+
+    def test_check_and_update_queue_config(self, aws_utility, setup_sqs):
+        queue = aws_utility.create_queue("test-event")
+        aws_utility.check_and_update_queue_config(queue)
+        # No assertion needed as we're just ensuring no exceptions are raised
+
+class TestEventBridgeOperations:
+    def test_add_event_bridge_rule(self, aws_utility, setup_sqs, setup_events):
+        queue = aws_utility.create_queue("test-event")
+        aws_utility.add_event_bridge_rule(queue)
+        # Verify rule was created by checking if we can get it
+        rules = aws_utility.event_bridge.list_rules(EventBusName='test-bus', NamePrefix="test-event_rule")
+        assert len(rules['Rules']) == 1
+        rule = rules['Rules'][0]
+        assert rule['Name'] == "test-event_rule"
+        assert rule['State'] == "ENABLED"
+
+    def test_publish_event_success(self, aws_utility, setup_events, sample_event_payload):
+        success, result = aws_utility.publish_event("test-event", sample_event_payload)
+        assert success is True
+        assert result is not None
+
+    def test_publish_event_failure(self, aws_utility):
+        with patch('boto3.client') as mock_client:
+            mock_client.return_value.put_events.return_value = {
+                "FailedEntryCount": 1,
+                "Entries": [{"ErrorMessage": "Test error"}]
+            }
+            success, result = aws_utility.publish_event("test-event", {})
+            assert success is False
+            assert result is None
+
+class TestS3Operations:
+    def test_check_payload_size(self, aws_utility):
+        small_payload = json.dumps({"key": "value"})
+        large_payload = json.dumps({"key": "x" * 300000})  # > 256KB
+        
+        assert aws_utility._check_payload_size(small_payload) is False
+        assert aws_utility._check_payload_size(large_payload) is True
+
+    def test_upload_payload_to_s3(self, aws_utility, setup_s3):
+        payload = json.dumps({"key": "value"})
+        s3_path = aws_utility._upload_payload_to_s3(payload)
+        
+        assert s3_path.startswith("s3://")
+        assert aws_utility.s3_bucket in s3_path
+
+    def test_get_payload_from_s3(self, aws_utility, setup_s3):
+        # First upload a payload
+        original_payload = json.dumps({"key": "value"})
+        s3_path = aws_utility._upload_payload_to_s3(original_payload)
+        
+        # Then retrieve it
+        retrieved_payload = aws_utility._get_payload_from_s3(s3_path)
+        assert retrieved_payload == original_payload
+
+    def test_get_payload_from_s3_invalid_path(self, aws_utility):
+        with pytest.raises(ValueError):
+            aws_utility._get_payload_from_s3("invalid-path")
+
+class TestMessageOperations:
+    def test_get_message_from_queue(self, aws_utility, setup_sqs):
+        queue = aws_utility.create_queue("test-event")
+        # Send a message to the queue
+        aws_utility.sqs.send_message(
+            QueueUrl=queue.url,
+            MessageBody=json.dumps({"test": "message"})
+        )
+        message = aws_utility.get_message_from_queue(queue)
+        assert message is not None
+        assert "test" in json.loads(message["Body"])
+
+    def test_delete_message_from_queue(self, aws_utility, setup_sqs):
+        queue = aws_utility.create_queue("test-event")
+        # Send and receive a message
+        aws_utility.sqs.send_message(
+            QueueUrl=queue.url,
+            MessageBody=json.dumps({"test": "message"})
+        )
+        message = aws_utility.get_message_from_queue(queue)
+        # Delete the message
+        aws_utility.delete_message_from_queue(queue, message["ReceiptHandle"])
+        # Try to receive messages again, should be empty
+        response = aws_utility.sqs.receive_message(QueueUrl=queue.url)
+        assert "Messages" not in response
+
+    def test_recieve_event_with_direct_payload(self, aws_utility, setup_sqs):
+        queue = aws_utility.create_queue("test-event")
+        # Send a message with the correct event structure
+        aws_utility.sqs.send_message(
+            QueueUrl=queue.url,
+            MessageBody=json.dumps({
+                "detail": {
+                    "event_name": "test-event",
+                    "event_payload": "test",
+                    "metadata": {
+                        "trace_id": "test-trace-id",
+                        "timestamp": "2024-04-17T13:54:14.479Z"
+                    }
+                }
+            })
+        )
+        event_msg = aws_utility.recieve_event(queue)
+        assert event_msg is not None
+        assert event_msg.event_name == "test-event"
+        assert event_msg.event_payload == "test"
+
+    def test_recieve_event_with_s3_payload(self, aws_utility, setup_sqs, setup_s3):
+        queue = aws_utility.create_queue("test-event")
+        # Upload payload to S3 with proper JSON format
+        payload = json.dumps({"event_payload": "test payload"})
+        s3_path = aws_utility._upload_payload_to_s3(payload)
+        # Send a message with S3 path
+        aws_utility.sqs.send_message(
+            QueueUrl=queue.url,
+            MessageBody=json.dumps({
+                "detail": {
+                    "event_name": "test-event",
+                    "event_payload_path": s3_path,
+                    "event_meta": {
+                        "trace_id": "test-trace-id",
+                        "span_id": "test-span-id",
+                        "timestamp": "2024-04-17T13:54:14.479Z"
+                    }
+                }
+            })
+        )
+        event_msg = aws_utility.recieve_event(queue)
+        assert event_msg.event_payload == "test payload"
+
+    def test_update_visibility_timeout(self, aws_utility, setup_sqs):
+        queue = aws_utility.create_queue("test-event")
+        # Send and receive a message
+        aws_utility.sqs.send_message(
+            QueueUrl=queue.url,
+            MessageBody=json.dumps({"test": "message"})
+        )
+        message = aws_utility.get_message_from_queue(queue)
+        # Update visibility timeout
+        aws_utility.update_visibility_timeout(queue.url, message["ReceiptHandle"], 30)
+        # No assertion needed as we're just ensuring no exceptions are raised
+
+    def test_get_queue_url_arn(self, aws_utility):
+        result = aws_utility.get_queue_url_arn("test-event")
+        assert "queue_arn" in result
+        assert "queue_url" in result
+        assert "test-event" in result["queue_arn"]
+        assert "test-event" in result["queue_url"]