ipulse-shared-core-ftredge 2.54__tar.gz → 2.55__tar.gz

{ipulse_shared_core_ftredge-2.54/src/ipulse_shared_core_ftredge.egg-info → ipulse_shared_core_ftredge-2.55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.54
+Version: 2.55
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.54 → ipulse_shared_core_ftredge-2.55}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='ipulse_shared_core_ftredge',
-    version='2.54',
+    version='2.55',
     package_dir={'': 'src'},  # Specify the source directory
     packages=find_packages(where='src'),  # Look for packages in 'src'
     install_requires=[

{ipulse_shared_core_ftredge-2.54 → ipulse_shared_core_ftredge-2.55}/src/ipulse_shared_core_ftredge/__init__.py

@@ -5,7 +5,7 @@ from .utils_gcp import (setup_gcp_logger_and_error_report,
                         write_csv_to_gcs, write_json_to_gcs)
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
                                           check_format_against_schema_template)
-from .utils_common import (ContextLog,  PipelineWatcher)
+from .utils_common import (ContextLog,  Pipelinemon)
 
 from .enums import (TargetLogs, LogLevel, Unit, Frequency,
                     Module, SubModule, BaseDataCategory,

{ipulse_shared_core_ftredge-2.54 → ipulse_shared_core_ftredge-2.55}/src/ipulse_shared_core_ftredge/utils_common.py

@@ -17,11 +17,12 @@ from ipulse_shared_core_ftredge.utils_gcp import write_json_to_gcs
 # ["data_import","data_quality", "data_processing","data_general","data_persistance","metadata_quality", "metadata_processing", "metadata_persistance","metadata_general"]
 
 class ContextLog:
-    MAX_TRACEBACK_LINES = 14  # Define the maximum number of traceback lines to include
+    MAX_TRACEBACK_LINES = 24  # Define the maximum number of traceback lines to include
     def __init__(self, level: LogLevel, base_context: str = None, collector_id: str = None,
-                 e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
-                 subject: str = None, description: str = None, context: str = None,
-                 log_status: LogStatus = LogStatus.OPEN):
+                context: str = None, description: str = None,
+                e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
+                log_status: LogStatus = LogStatus.OPEN, subject: str = None
+                ):
         if e is not None:
             e_type = type(e).__name__ if e_type is None else e_type
             e_message = str(e) if e_message is None else e_message
@@ -46,7 +47,7 @@ class ContextLog:
             return None
 
         traceback_lines = e_traceback.splitlines()
-        
+
         # Remove lines that are part of the exception message if they are present in traceback
         message_lines = e_message.splitlines() if e_message else []
         if message_lines:
@@ -56,7 +57,7 @@ class ContextLog:
 
         # Filter out lines from third-party libraries (like site-packages)
         filtered_lines = [line for line in traceback_lines if "site-packages" not in line]
-        
+
         # If filtering results in too few lines, revert to original traceback
         if len(filtered_lines) < 2:
             filtered_lines = traceback_lines
@@ -101,23 +102,108 @@ class ContextLog:
     def context(self, value):
         self._context = value
 
-    def to_dict(self):
-        return {
-            "base_context": self.base_context,
-            "context": self.context,
-            "level_code": self.level.value,
-            "level_name": self.level.name,
-            "subject": self.subject,
-            "description": self.description,
-            "exception_type": self.exception_type,
-            "exception_message": self.exception_message,
-            "exception_traceback": self.exception_traceback,
-            "log_status": self.log_status.value,
-            "collector_id": self.collector_id,
-            "timestamp": self.timestamp
-        }
-
-class PipelineWatcher:
+
+    def to_dict(self, size_limit=256 * 1024 * 0.80):
+        size_limit = int(size_limit)  # Ensure size_limit is an integer
+
+        # Define the priority order of the fields
+        priority_fields = [
+            ("base_context", self.base_context),
+            ("level_code", self.level.value),
+            ("level_name", self.level.name),
+            ("log_status", self.log_status.value),
+            ("collector_id", self.collector_id),
+            ("timestamp", self.timestamp),
+        ]
+
+        # Additional fields to be truncated if necessary. Shorter fields are truncated first so that remaining size can increase for longer fields.
+        additional_fields = [
+            ("subject", self.subject),
+            ("description", self.description),
+            ("exception_type", self.exception_type),
+            ("exception_message", self.exception_message),
+            ("context", self.context),  # special sizing rules apply to it
+            ("exception_traceback", self.exception_traceback)
+        ]
+
+        all_fields = priority_fields + additional_fields
+        non_zero_fields = [(key, value) for key, value in all_fields if value is not None]
+
+        total_size = 0
+        truncated = False  # Flag to indicate if truncation happened
+
+        # Function to calculate the byte size of a JSON-encoded field
+        def field_size(key, value):
+            return len(json.dumps({key: value}).encode('utf-8'))
+
+        # Function to truncate a value based on its type
+        def truncate_value(value, max_size):
+            if isinstance(value, str):
+                half_size = max_size // 2
+                return value[:half_size] + '...' + value[-(max_size - half_size - 3):]
+            elif isinstance(value, (list, tuple)):
+                half_size = max_size // 2
+                return list(value[:half_size]) + ['...'] + list(value[-(max_size - half_size - 1):])
+            elif isinstance(value, set):
+                truncated_set = set(list(value)[:max_size // 2]) | set(list(value)[-(max_size // 2):])
+                return truncated_set
+            elif isinstance(value, dict):
+                truncated_dict = {k: truncate_value(v, max_size // len(value)) for k, v in list(value.items())}
+                return truncated_dict
+            else:
+                return value
+
+        # Calculate the initial total size
+        for key, value in non_zero_fields:
+            total_size += field_size(key, value)
+
+        log_dict = {}
+        # Check if total size exceeds the size limit
+        if total_size > size_limit:
+            truncated = True  # Set the truncation flag
+            # Calculate max size per field based on all non-zero fields
+            max_size_per_field = size_limit // len(non_zero_fields)
+
+            # Reset total_size to recompute with truncation
+            total_size = 0
+
+            # Add priority fields first with possible truncation
+            for key, value in priority_fields:
+                if value is not None:
+                    truncated_value = value
+                    if isinstance(value, (str, list, tuple, set, dict)) and field_size(key, value) > max_size_per_field:
+                        truncated_value = truncate_value(value, max_size_per_field)
+                    log_dict[key] = truncated_value
+                    total_size += field_size(key, truncated_value)
+                else:
+                    log_dict[key] = value
+
+            # Calculate remaining size for additional fields
+            remaining_size = size_limit - total_size
+
+            # Handle remaining additional fields
+            non_zero_additional_fields = [field for field in additional_fields[1:] if field[1]]
+            remaining_field_size = remaining_size // len(non_zero_additional_fields) if non_zero_additional_fields else 0
+
+            for key, value in additional_fields[1:]:
+                if value is not None:
+                    if field_size(key, value) > remaining_field_size:
+                        truncated_value = truncate_value(value, remaining_field_size)
+                    else:
+                        truncated_value = value
+                    log_dict[key] = truncated_value
+                    remaining_size -= field_size(key, truncated_value)
+                else:
+                    log_dict[key] = value
+        else:
+            log_dict = dict(all_fields)
+
+        # Add trunc flag to the log dictionary
+        log_dict['trunc'] = truncated
+
+        return log_dict
+
+class Pipelinemon:
     ERROR_START_CODE = LogLevel.ERROR.value
     WARNING_START_CODE = LogLevel.WARNING.value
     NOTICE_START_CODE = LogLevel.NOTICE.value
@@ -192,7 +278,7 @@ class PipelineWatcher:
     def add_log(self, log: ContextLog):
         if (self._target_logs == TargetLogs.SUCCESSES and log.level >=self.NOTICE_START_CODE) or \
            (self._target_logs == TargetLogs.WARNINGS_AND_ERRORS and log.level.value < self.WARNING_START_CODE):
-            raise ValueError(f"Invalid log level {log.level.name} for Pipeline Watcher target logs setup: {self._target_logs}")
+            raise ValueError(f"Invalid log level {log.level.name} for Pipelinemon target logs setup: {self._target_logs}")
         log.base_context = self.base_context
         log.context = self.current_context
         log.collector_id = self.id
@@ -201,7 +287,7 @@ class PipelineWatcher:
         self._update_counts(log_dict)
 
         if self._logger:
-            # We specifically want to avoid having an ERROR log level for this structured Pipeline Watcher reporting, to ensure Errors are alerting on Critical Application Services.
+            # We specifically want to avoid having an ERROR log level for this structured Pipelinemon reporting, to ensure Errors are alerting on Critical Application Services.
             # A single ERROR log level can be used for the entire pipeline, which shall be used at the end of the pipeline
             if log.level.value >= self.WARNING_START_CODE:
                 self._logger.log_struct(log_dict, severity="WARNING")

{ipulse_shared_core_ftredge-2.54 → ipulse_shared_core_ftredge-2.55/src/ipulse_shared_core_ftredge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.54
+Version: 2.55
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar