fdc-shared-kernel 0.0.71__tar.gz → 0.0.73__tar.gz

{fdc_shared_kernel-0.0.71 → fdc_shared_kernel-0.0.73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fdc_shared_kernel
-Version: 0.0.71
+Version: 0.0.73
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.71 → fdc_shared_kernel-0.0.73}/fdc_shared_kernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fdc_shared_kernel
-Version: 0.0.71
+Version: 0.0.73
 Summary: Shared library for microservice
 Author-email: Shikhil S <shikhil.s@dbizsolution.com>, Ahammed Akdham N <ahammedakdham.n@dbizsolution.com>
 Classifier: Programming Language :: Python :: 3

{fdc_shared_kernel-0.0.71 → fdc_shared_kernel-0.0.73}/fdc_shared_kernel.egg-info/SOURCES.txt

@@ -73,4 +73,5 @@ shared_kernel/utils/__init__.py
 shared_kernel/utils/data_validators_utils.py
 shared_kernel/utils/date_format_utils.py
 shared_kernel/utils/string_utils.py
-shared_kernel/utils/template_renderer.py
+shared_kernel/utils/template_renderer.py
+shared_kernel/utils/thread_local_util.py

{fdc_shared_kernel-0.0.71 → fdc_shared_kernel-0.0.73}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fdc_shared_kernel"
-version = "0.0.71"
+version = "0.0.73"
 requires-python = ">=3.7"
 readme = "README_pypi.md"
 description = "Shared library for microservice"

{fdc_shared_kernel-0.0.71 → fdc_shared_kernel-0.0.73}/shared_kernel/event_executor/event_executor.py

@@ -11,10 +11,14 @@ from shared_kernel.logger import Logger
 from shared_kernel.messaging.utils.event_messages import AWSEventMessage, EventMessage
 from shared_kernel.status_tracker import StatusTracker
 from shared_kernel.enums import TaskStatus
+from shared_kernel.utils.thread_local_util import ThreadLocalStorage
+
 
 app_config = Config()
 logger = Logger(app_config.get("APP_NAME"))
 
+thread_local_storage = ThreadLocalStorage()
+
 class EventExecutor:
 
     _instance = None
@@ -81,10 +85,14 @@ class EventExecutor:
             logger.info(
                 f"Processing event {event_msg.event_name}. trace-id: {event_msg.event_meta.trace_id}. span-id: {event_msg.event_meta.trace_id}."
             )
-            task: dict = self.status_tracker.get_task(
-                task=event_msg.event_name, task_id=event_msg.event_meta.job_id
-            )
 
+            task_details = self.status_tracker.get_task(task_details = event_msg)
+            if task_details is not None and task_details["is_duplicate"]:
+                logger.info(
+                    f"Duplicate task {event_msg.event_name} is already in progress. trace-id: {event_msg.event_meta.trace_id}. span-id: {event_msg.event_meta.trace_id}."
+                )
+                return
+            task = task_details.get("task_details") if task_details is not None else None
             if task is None:
                 logger.info(
                     f"Creating new task for event {event_msg.event_name}. trace-id: {event_msg.event_meta.trace_id}. span-id: {event_msg.event_meta.trace_id}."
@@ -95,6 +103,8 @@ class EventExecutor:
                     task=event_msg.event_name,
                     status=TaskStatus.PROCESSING.value,
                     task_id=event_msg.event_meta.job_id,
+                    org_id=event_msg.event_meta.org_id,
+                    entity_id=event_msg.event_meta.entity_id,
                 )
                 
                 # setting tracking payload without the time taken and end time
@@ -143,6 +153,8 @@ class EventExecutor:
             logger.error(
                 f"Error processing event {event_msg.event_name} trace-id: {event_msg.event_meta.trace_id}. span-id: {event_msg.event_meta.trace_id} : {str(e)}"
             )
+            logger.info("Error processing event",type="distributed_trace",is_success="False",failure_reason=str(e))
+
             # adding the failure reason to the event meta
             event_msg.event_meta.failure_reason = str(e)
 
@@ -203,16 +215,28 @@ class EventExecutor:
             logger.info(f"Initiating callback for message: {message}")
             event_msg = AWSEventMessage(message)
             event_name = event_msg.event_name
+
+            # Set thread-local context
+            thread_local_storage.set_all({
+                "trace_id": event_msg.event_meta.trace_id,
+                "span_id": event_msg.event_meta.span_id,
+                "org_id": event_msg.event_meta.org_id if hasattr(event_msg.event_meta, 'org_id') else None,
+                "event_name": event_msg.event_name,
+                "event_payload": json.dumps(event_msg.event_payload)
+            })
+            logger.info("Event recieved",type="distributed_trace")
+            # Process the message
             success = self._process_message(event_msg, callback)
-        finally:
 
-            # update the event stats whether its successful or failure
+        finally:
+            # Update the event stats whether it succeeded or failed
             if event_name:
                 self._update_event_stats(event_name, success)
 
             if success:
                 logger.info(
-                    f"Event {event_msg.event_name} completed successfully. trace-id: {event_msg.event_meta.trace_id}. span-id: {event_msg.event_meta.span_id}."
+                    f"Event {event_msg.event_name} completed successfully. "
+                    f"trace-id: {event_msg.event_meta.trace_id}. span-id: {event_msg.event_meta.span_id}."
                 )
                 self.status_tracker.update_task(
                     span_id=event_msg.event_meta.span_id,
@@ -221,10 +245,17 @@ class EventExecutor:
                     status=TaskStatus.COMPLETED.value,
                     task_id=event_msg.event_meta.job_id,
                 )
+                logger.info("Event processed successfully",type="distributed_trace",is_success="True",time_in_seconds= event_msg.event_meta.time_taken)
             else:
                 logger.warning(
-                    f"Event {event_msg.event_name} failed during processing. trace-id: {event_msg.event_meta.trace_id}. span-id: {event_msg.event_meta.span_id}."
+                    f"Event {event_msg.event_name} failed during processing. "
+                    f"trace-id: {event_msg.event_meta.trace_id}. span-id: {event_msg.event_meta.span_id}."
                 )
+
+            # Clean up thread-local storage
+            thread_local_storage.clear()
+
+            # Delete the message from the queue
             self.databus.delete_message(event_msg)
 
     def _task_done_callback(self, future: Future, event_name: str, event_semaphore: Semaphore) -> None:

{fdc_shared_kernel-0.0.71 → fdc_shared_kernel-0.0.73}/shared_kernel/logger/__init__.py

@@ -2,6 +2,7 @@ import logging
 import os
 import json
 from shared_kernel.config import Config
+from shared_kernel.utils.thread_local_util import ThreadLocalStorage
 class JSONFormatter(logging.Formatter):
     """
     Custom JSON formatter to structure log records as JSON.
@@ -16,10 +17,24 @@ class JSONFormatter(logging.Formatter):
             "level": record.levelname,
             "message": record.getMessage(),
         }
-        if hasattr(record, 'extra_data'):
-            log_record.update(record.extra_data)
+        # Get meta_data if present
+        meta_data = getattr(record, 'meta_data', {})
+
+        # Check if 'type' is in meta_data and is 'distributed_trace'
+        if meta_data.get("type") == "distributed_trace":
+            thread_context = ThreadLocalStorage.get_all()
+            # Add each data from thread local storage
+            log_record["trace_id"] = thread_context.get("trace_id")
+            log_record["span_id"] = thread_context.get("span_id")
+            log_record["org_id"] = thread_context.get("org_id")
+            log_record["event_name"] = thread_context.get("event_name")
+            log_record["event_payload"] = thread_context.get("event_payload")
+            log_record.update(meta_data)
+        if hasattr(record, 'meta_data'):
+            log_record.update(record.meta_data)
         if record.exc_info:
-            log_record["exception"] = self.formatException(record.exc_info)
+            log_record["stack_trace"] = self.formatException(record.exc_info)
+            log_record["exception"] = str(record.exc_info[1])
         return json.dumps(log_record)
 class Logger:
     """
@@ -100,17 +115,17 @@ class Logger:
         self.logger.addHandler(json_file_handler)
 
     def info(self, message, *args, **kwargs):
-        extra_data = {"extra_data": kwargs}
-        self.logger.info(message, *args, extra=extra_data, stacklevel=2)
+        meta_data = {"meta_data": kwargs}
+        self.logger.info(message, *args, extra=meta_data, stacklevel=2)
 
     def error(self, message, *args, **kwargs):
-        extra_data = {"extra_data": kwargs}
-        self.logger.error(message, exc_info=True, *args, extra=extra_data, stacklevel=2)
+        meta_data = {"meta_data": kwargs}
+        self.logger.error(message, exc_info=True, *args, extra=meta_data, stacklevel=2)
 
     def debug(self, message, *args, **kwargs):
-        extra_data = {"extra_data": kwargs}
-        self.logger.debug(message, *args, extra=extra_data, stacklevel=2)
+        meta_data = {"meta_data": kwargs}
+        self.logger.debug(message, *args, extra=meta_data, stacklevel=2)
 
     def warning(self, message, *args, **kwargs):
-        extra_data = {"extra_data": kwargs}
-        self.logger.warning(message, *args, extra=extra_data, stacklevel=2)
+        meta_data = {"meta_data": kwargs}
+        self.logger.warning(message, *args, extra=meta_data, stacklevel=2)

{fdc_shared_kernel-0.0.71 → fdc_shared_kernel-0.0.73}/shared_kernel/messaging/aws_databus.py

@@ -70,6 +70,8 @@ class AWSDataBus(DataBus):
                 task=validated_event_dict["event_name"],
                 status=TaskStatus.QUEUED.value,
                 task_id=validated_event_dict["event_meta"]["job_id"],
+                org_id=validated_event_dict["event_meta"]["org_id"],
+                entity_id=validated_event_dict["event_meta"].get("entity_id", None),
             )
             
             self.status_tracker.set_event_meta_and_message_receipt_handle(

{fdc_shared_kernel-0.0.71 → fdc_shared_kernel-0.0.73}/shared_kernel/messaging/utils/event_messages.py

@@ -19,6 +19,9 @@ class EventMeta:
         self.trace_id: str = meta_data.get("trace_id")
         self.job_id: str = meta_data.get("job_id")
         self.trigger = meta_data.get("trigger")
+        self.org_id = meta_data.get("org_id")
+        # entity_id is an optional field as some events may not be associated with an entity
+        self.entity_id = meta_data.get("entity_id", None)
         # generate a new span_id for each event if not present already
         self.span_id: str = meta_data.get("span_id", str(uuid.uuid4()))
 

{fdc_shared_kernel-0.0.71 → fdc_shared_kernel-0.0.73}/shared_kernel/registries/service_event_registry.py

@@ -20,9 +20,9 @@ class ServiceEventRegistry:
         + "/event/subscribe-sync-event/MARK_TASK_AS_FAILURE/"
     )
 
-    GET_TASK = (
+    PROCESS_TASK = (
         config.get("MASTER_SERVICE_BASE_ENDPOINT")
-        + "/event/subscribe-sync-event/GET_TASK/"
+        + "/event/subscribe-sync-event/PROCESS_TASK/"
     )
 
     GET_IN_PROGRESS_TASK = (

{fdc_shared_kernel-0.0.71 → fdc_shared_kernel-0.0.73}/shared_kernel/status_tracker/status_tracker.py

@@ -1,6 +1,7 @@
 from shared_kernel.exceptions.custom_exceptions import StatusTrackerException
 from shared_kernel.interfaces.databus import DataBus
 from shared_kernel.messaging import DataBusFactory
+from shared_kernel.messaging.utils.event_messages import EventMessage
 from shared_kernel.registries.service_event_registry import ServiceEventRegistry
 
 service_event_registry = ServiceEventRegistry()
@@ -32,7 +33,7 @@ class StatusTracker:
         )
 
 
-    def create_task(self, span_id, trace_id, task, status, task_id):
+    def create_task(self, span_id, trace_id, task, status, task_id, org_id, entity_id=None):
         """Publishes a synchronous event to create a task"""
         try:
             payload = {
@@ -41,7 +42,10 @@ class StatusTracker:
                 "task": task,
                 "status": status,
                 "task_id": task_id,
+                "org_id": org_id,
             }
+            if entity_id:
+                payload["entity_id"] = entity_id
             response = self.databus.request_event(
                 getattr(service_event_registry, "CREATE_TASK"), payload
             )
@@ -89,15 +93,19 @@ class StatusTracker:
             raise StatusTrackerException(e)
         
     
-    def get_task(self, task, task_id):
+    def get_task(self, task_details: EventMessage):
         """Publishes a synchronous event to retrieve a task"""
         try:
             payload = {
-                "task": task,
-                "task_id": task_id,
+                "task": task_details.event_name,
+                "task_id": task_details.event_meta.job_id,
+                "entity_id": task_details.event_meta.entity_id if task_details.event_meta.entity_id else None,
+                "trace_id": task_details.event_meta.trace_id,
+                "span_id": task_details.event_meta.span_id,
+                "org_id": task_details.event_meta.org_id,
             }
             response: dict = self.databus.request_event(
-                getattr(service_event_registry, "GET_TASK"), payload
+                getattr(service_event_registry, "PROCESS_TASK"), payload
             )
             return response.get("data")
 
@@ -155,4 +163,3 @@ class StatusTracker:
 
         except Exception as e:
             raise StatusTrackerException(e)
-        

fdc_shared_kernel-0.0.73/shared_kernel/utils/thread_local_util.py

@@ -0,0 +1,63 @@
+import threading
+
+
+class ThreadLocalStorage:
+    """
+    Utility class for managing thread-local storage with class-level methods and variables.
+    """
+
+    # Class-level thread-local storage
+    _storage = threading.local()
+
+    @classmethod
+    def set(cls, key: str, value: any) -> None:
+        """
+        Sets a value in the thread-local storage.
+
+        :param key: The key for the value.
+        :param value: The value to be stored.
+        """
+        if not hasattr(cls._storage, "data"):
+            cls._storage.data = {}
+        cls._storage.data[key] = value
+
+    @classmethod
+    def set_all(cls, values: dict) -> None:
+        """
+        Sets multiple key-value pairs in the thread-local storage.
+
+        :param values: A dictionary of key-value pairs to store.
+        """
+        if not hasattr(cls._storage, "data"):
+            cls._storage.data = {}
+        cls._storage.data.update(values)
+
+    @classmethod
+    def get(cls, key: str, default=None) -> any:
+        """
+        Retrieves a value from the thread-local storage.
+
+        :param key: The key for the value.
+        :param default: The default value to return if the key is not found.
+        :return: The value associated with the key, or the default value.
+        """
+        if hasattr(cls._storage, "data") and key in cls._storage.data:
+            return cls._storage.data[key]
+        return default
+
+    @classmethod
+    def get_all(cls) -> dict:
+        """
+        Retrieves all key-value pairs from the thread-local storage.
+
+        :return: A dictionary of all thread-local data.
+        """
+        return getattr(cls._storage, "data", {}).copy() if hasattr(cls._storage, "data") else {}
+
+    @classmethod
+    def clear(cls) -> None:
+        """
+        Clears all values from the thread-local storage.
+        """
+        if hasattr(cls._storage, "data"):
+            cls._storage.data.clear()