ipulse-shared-core-ftredge 2.54__tar.gz → 2.56__tar.gz

{ipulse_shared_core_ftredge-2.54/src/ipulse_shared_core_ftredge.egg-info → ipulse_shared_core_ftredge-2.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.54
+Version: 2.56
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.54 → ipulse_shared_core_ftredge-2.56}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='ipulse_shared_core_ftredge',
-    version='2.54',
+    version='2.56',
     package_dir={'': 'src'},  # Specify the source directory
     packages=find_packages(where='src'),  # Look for packages in 'src'
     install_requires=[

{ipulse_shared_core_ftredge-2.54 → ipulse_shared_core_ftredge-2.56}/src/ipulse_shared_core_ftredge/__init__.py

@@ -5,7 +5,7 @@ from .utils_gcp import (setup_gcp_logger_and_error_report,
                         write_csv_to_gcs, write_json_to_gcs)
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
                                           check_format_against_schema_template)
-from .utils_common import (ContextLog,  PipelineWatcher)
+from .utils_common import (ContextLog,  Pipelinemon)
 
 from .enums import (TargetLogs, LogLevel, Unit, Frequency,
                     Module, SubModule, BaseDataCategory,

{ipulse_shared_core_ftredge-2.54 → ipulse_shared_core_ftredge-2.56}/src/ipulse_shared_core_ftredge/enums/__init__.py

@@ -4,6 +4,7 @@
 # pylint: disable=missing-class-docstring
 
 from .enums_common_utils import (LogLevel,
+                                SystemsImpacted,
                                 TargetLogs,
                                 Unit,
                                 Frequency)

{ipulse_shared_core_ftredge-2.54 → ipulse_shared_core_ftredge-2.56}/src/ipulse_shared_core_ftredge/enums/enums_common_utils.py

@@ -2,10 +2,28 @@
 # pylint: disable=missing-module-docstring
 # pylint: disable=missing-function-docstring
 # pylint: disable=missing-class-docstring
+# pylint: disable=line-too-long
 
 from enum import Enum
 
 
+class SystemsImpacted(Enum):
+    NO = "__no"
+    YES = "__yes"
+    INVESTIGATE = "__investigate"
+    MULTIPLE = "__multiple"
+    DB = "db"
+    BQ_TABLE= "bq_table"
+    BQ_TABLES = "bq_tables"
+    GCS_BUCKET = "gcs_bucket"
+    GCS_BUCKETS = "gcs_buckets"
+    GCS_BUCKET_FILE = "gcs_bucket_file"
+    GCS_BUCKET_FILES = "gcs_bucket_files"
+    API = "api"
+    APIS = "apis"
+    LOCAL_FILE = "local_file"
+    LOCAL_FILES = "local_files"
+
 class TargetLogs(Enum):
     MIXED="mixed_logs"
     SUCCESSES = "success_logs"
@@ -15,22 +33,27 @@ class TargetLogs(Enum):
     WARNINGS_AND_ERRORS = "warn_n_err_logs"
     ERRORS = "error_logs"
 
-
 class LogLevel(Enum):
     """
     Standardized notice levels for data engineering pipelines,
     designed for easy analysis and identification of manual 
     intervention needs.
     """
-    DEBUG = 100  # Detailed debug information (for development/troubleshooting)
+    DEBUG = 10  # Detailed debug information (for development/troubleshooting)
+
+    INFO = 100
+    INFO_PERSISTNACE_COMPLETE= 101
+    INFO_UPDATE_COMPLETE = 102
+    INFO_DELETE_COMPLETE = 103
 
-    INFO = 200
     SUCCESS = 201
+    SUCCESS_WITH_NOTICES = 211
+    SUCCESS_WITH_WARNINGS = 212
 
     NOTICE = 300  # Maybe same file or data already fully or partially exists
     NOTICE_ALREADY_EXISTS = 301 # Data already exists, no action required
     NOTICE_PARTIAL_EXISTS = 302 # Partial data exists, no action required
-    NOTICE_CANCELLED = 303 # Data processing cancelled, no action required
+    NOTICE_ACTION_CANCELLED = 303 # Data processing cancelled, no action required
 
      # Warnings indicate potential issues that might require attention:
     WARNING = 400 # General warning, no immediate action required
@@ -40,18 +63,22 @@ class LogLevel(Enum):
     WARNING_FIX_REQUIRED = 404  # Action required, pipeline can likely continue
 
     ERROR = 500 # General error, no immediate action required
-    # Errors indicate a problem that disrupts normal pipeline execution:
-    ERROR_EXCEPTION_REDO = 501
-    ERROR_CUSTOM_REDO = 502 # Temporary error, automatic retry likely to succeed
-    
-    ERROR_EXCEPTION_INVESTIGATE = 601 # Exception occured after some data was likely persisted (e.g., to GCS or BQ)
-    ERROR_CUSTOM_INVESTIGATE= 602
-    ERROR_EXCEPTION_PERSTISTANCE = 603 # Exception occured after data was persisted (e.g., to GCS or BQ)
-    ERROR_CUSTOM_PERSTISTANCE = 604
 
+    ERROR_EXCEPTION = 501
+    ERROR_CUSTOM = 502 # Temporary error, automatic retry likely to succeed
+    ERROR_OPERATION_PARTIALLY_FAILED = 511 # Partial or full failure, manual intervention required
+    ERROR_OPERATION_FAILED = 512 # Operation failed, manual intervention required
+    ERORR_OPERATION_WITH_WARNINGS = 513 # Partial or full failure, manual intervention required
+    ERORR_OPERATION_WITH_ERRORS = 514 # Partial or full failure, manual intervention required
+    ERORR_OPERATION_WITH_WARNINGS_OR_ERRORS = 515 # Partial or full failure, manual intervention required
+
+    ERROR_THRESHOLD_REACHED = 551
+    ERROR_PIPELINE_THRESHOLD_REACHED = 552 # Error due to threshold reached, no immediate action required
+    ERROR_SUBTHRESHOLD_REACHED = 553 # Error due to threshold reached, no immediate action required
+    ERROR_DATA_QUALITY_THRESHOLD_REACHED = 554 # Error due to threshold reached, no immediate action required
     # Critical errors indicate severe failures requiring immediate attention:
-    CRITICAL_SYSTEM_FAILURE = 701 # System-level failure (e.g., infrastructure), requires immediate action
-    CRITICAL_PIPELINE_FAILURE = 702 # Complete pipeline failure, requires investigation and potential rollback
+    CRITICAL=600 # General critical error, requires immediate action
+    CRITICAL_SYSTEM_FAILURE = 601 # System-level failure (e.g., infrastructure, stackoverflow ), requires immediate action
 
     UNKNOWN=1001 # Unknown error, should not be used in normal operation
 
@@ -63,8 +90,6 @@ class LogStatus(Enum):
     RESOLVED = "resolved"
     IGNORED = "ignored"
     CANCELLED = "cancelled"
-  
-
 
 ### Exception during full exection, partially saved
 # Exception during ensemble pipeline; modifications collected in local object , nothing persisted
@@ -143,4 +168,4 @@ class Frequency(Enum):
     THREE_M="3m"
     SIX_M="6m"
     ONE_Y="1y"
-    THREE_Y="3y"
+    THREE_Y="3y"

{ipulse_shared_core_ftredge-2.54 → ipulse_shared_core_ftredge-2.56}/src/ipulse_shared_core_ftredge/utils_common.py

@@ -3,6 +3,7 @@
 # pylint: disable=logging-fstring-interpolation
 # pylint: disable=line-too-long
 # pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
 import traceback
 import json
 import uuid
@@ -17,11 +18,14 @@ from ipulse_shared_core_ftredge.utils_gcp import write_json_to_gcs
 # ["data_import","data_quality", "data_processing","data_general","data_persistance","metadata_quality", "metadata_processing", "metadata_persistance","metadata_general"]
 
 class ContextLog:
-    MAX_TRACEBACK_LINES = 14  # Define the maximum number of traceback lines to include
+    MAX_FIELD_LINES = 26  # Define the maximum number of traceback lines to include
+    MAX_FIELD_LENGTH=10000
+
     def __init__(self, level: LogLevel, base_context: str = None, collector_id: str = None,
-                 e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
-                 subject: str = None, description: str = None, context: str = None,
-                 log_status: LogStatus = LogStatus.OPEN):
+                context: str = None, description: str = None,
+                e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
+                log_status: LogStatus = LogStatus.OPEN, subject: str = None, systems_impacted: List[str] = None
+                ):
         if e is not None:
             e_type = type(e).__name__ if e_type is None else e_type
             e_message = str(e) if e_message is None else e_message
@@ -34,19 +38,60 @@ class ContextLog:
         self.description = description
         self._base_context = base_context
         self._context = context
+        self._systems_impacted = systems_impacted if systems_impacted else []
         self.collector_id = collector_id
         self.exception_type = e_type
         self.exception_message = e_message
-        self.exception_traceback = self._format_traceback(e_traceback,e_message)
+        self.exception_traceback = e_traceback
         self.log_status = log_status
         self.timestamp = datetime.now(timezone.utc).isoformat()
 
+    @property
+    def base_context(self):
+        return self._base_context
+
+    @base_context.setter
+    def base_context(self, value):
+        self._base_context = value
+
+    @property
+    def context(self):
+        return self._context
+
+    @context.setter
+    def context(self, value):
+        self._context = value
+
+    @property
+    def systems_impacted(self):
+        return self._systems_impacted
+
+    @systems_impacted.setter
+    def systems_impacted(self, list_of_si: List[str]):
+        self._systems_impacted = list_of_si
+
+    def add_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is None:
+            self._systems_impacted = []
+        self._systems_impacted.append(system_impacted)
+
+    def remove_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is not None:
+            self._systems_impacted.remove(system_impacted)
+
+    def clear_systems_impacted(self):
+        self._systems_impacted = []
+
     def _format_traceback(self, e_traceback, e_message):
         if not e_traceback or e_traceback == 'None\n':
             return None
 
         traceback_lines = e_traceback.splitlines()
-        
+
+            # Check if the traceback is within the limits
+        if len(traceback_lines) <= self.MAX_FIELD_LINES and len(e_traceback) <= self.MAX_FIELD_LENGTH:
+            return e_traceback
+
         # Remove lines that are part of the exception message if they are present in traceback
         message_lines = e_message.splitlines() if e_message else []
         if message_lines:
@@ -56,7 +101,7 @@ class ContextLog:
 
         # Filter out lines from third-party libraries (like site-packages)
         filtered_lines = [line for line in traceback_lines if "site-packages" not in line]
-        
+
         # If filtering results in too few lines, revert to original traceback
         if len(filtered_lines) < 2:
             filtered_lines = traceback_lines
@@ -69,61 +114,106 @@ class ContextLog:
             else:
                 combined_lines.append(line)
 
-        # Determine the number of lines to keep from the start and end
-        keep_lines_start = min(self.MAX_TRACEBACK_LINES // 2, len(combined_lines))
-        keep_lines_end = min(self.MAX_TRACEBACK_LINES // 2, len(combined_lines) - keep_lines_start)
-
-        if len(combined_lines) > self.MAX_TRACEBACK_LINES:
-            # Include the first few and last few lines, and an indicator of truncation
-            formatted_traceback = '\n'.join(
-                combined_lines[:keep_lines_start] + 
-                ['... (truncated) ...'] + 
+            # Ensure the number of lines doesn't exceed MAX_TRACEBACK_LINES
+        if len(combined_lines) > self.MAX_FIELD_LINES:
+            keep_lines_start = min(self.MAX_FIELD_LINES // 2, len(combined_lines))
+            keep_lines_end = min(self.MAX_FIELD_LINES // 2, len(combined_lines) - keep_lines_start)
+            combined_lines = (
+                combined_lines[:keep_lines_start] +
+                ['... (truncated) ...'] +
                 combined_lines[-keep_lines_end:]
             )
-        else:
-            formatted_traceback = '\n'.join(combined_lines)
 
+        formatted_traceback = '\n'.join(combined_lines)
+
+        # Ensure the total length doesn't exceed MAX_TRACEBACK_LENGTH
+        if len(formatted_traceback) > self.MAX_FIELD_LENGTH:
+            truncated_length = self.MAX_FIELD_LENGTH - len('... (truncated) ...')
+            half_truncated_length = truncated_length // 2
+            formatted_traceback = (
+                formatted_traceback[:half_truncated_length] +
+                '\n... (truncated) ...\n' +
+                formatted_traceback[-half_truncated_length:]
+            )
         return formatted_traceback
 
-    @property
-    def base_context(self):
-        return self._base_context
+    def to_dict(self, max_field_len:int =10000, size_limit:float=256 * 1024 * 0.80):
+        size_limit = int(size_limit)  # Ensure size_limit is an integer
+
+        # Unified list of all fields
+        systems_impacted_str = f"{len(self.systems_impacted)} system(s): " + " ,,, ".join(self.systems_impacted) if self.systems_impacted else None
+        fields = [
+            ("log_status", str(self.log_status.name)),
+            ("level_code", self.level.value),
+            ("level_name", str(self.level.name)),
+            ("base_context", str(self.base_context)),
+            ("timestamp", str(self.timestamp)),
+            ("collector_id", str(self.collector_id)),
+            ("systems_impacted", systems_impacted_str),
+            ("context", str(self.context)),  # special sizing rules apply to it
+            ("subject", str(self.subject)),
+            ("description", str(self.description)),
+            ("exception_type", str(self.exception_type)),
+            ("exception_message", str(self.exception_message)),
+            ("exception_traceback", str(self._format_traceback(self.exception_traceback,self.exception_message)))
+        ]
 
-    @base_context.setter
-    def base_context(self, value):
-        self._base_context = value
+        # Function to calculate the byte size of a JSON-encoded field
+        def field_size(key, value):
+            return len(json.dumps({key: value}).encode('utf-8'))
+
+        # Function to truncate a value based on its type
+        # Function to truncate a value based on its type
+        def truncate_value(value, max_size):
+            if isinstance(value, str):
+                half_size = max_size // 2
+                return value[:half_size] + '...' + value[-(max_size - half_size - 3):]
+            return value
+
+         # Ensure no field exceeds max_field_len
+        for i, (key, value) in enumerate(fields):
+            if isinstance(value, str) and len(value) > max_field_len:
+                fields[i] = (key, truncate_value(value, max_field_len))
+
+        # Ensure total size of the dict doesn't exceed size_limit
+        total_size = sum(field_size(key, value) for key, value in fields)
+        log_dict = {}
+        truncated = False
+
+        if total_size > size_limit:
+            truncated = True
+            remaining_size = size_limit
+            remaining_fields = len(fields)
+
+            for key, value in fields:
+                if remaining_fields > 0:
+                    max_size_per_field = remaining_size // remaining_fields
+                else:
+                    max_size_per_field = 0
+
+                field_sz = field_size(key, value)
+                if field_sz > max_size_per_field:
+                    value = truncate_value(value, max_size_per_field)
+                    field_sz = field_size(key, value)
+
+                log_dict[key] = value
+                remaining_size -= field_sz
+                remaining_fields -= 1
+        else:
+            log_dict = dict(fields)
 
-    @property
-    def context(self):
-        return self._context
+        log_dict['trunc'] = truncated
 
-    @context.setter
-    def context(self, value):
-        self._context = value
+        return log_dict
 
-    def to_dict(self):
-        return {
-            "base_context": self.base_context,
-            "context": self.context,
-            "level_code": self.level.value,
-            "level_name": self.level.name,
-            "subject": self.subject,
-            "description": self.description,
-            "exception_type": self.exception_type,
-            "exception_message": self.exception_message,
-            "exception_traceback": self.exception_traceback,
-            "log_status": self.log_status.value,
-            "collector_id": self.collector_id,
-            "timestamp": self.timestamp
-        }
-
-class PipelineWatcher:
+class Pipelinemon:
     ERROR_START_CODE = LogLevel.ERROR.value
     WARNING_START_CODE = LogLevel.WARNING.value
     NOTICE_START_CODE = LogLevel.NOTICE.value
     SUCCESS_START_CODE = LogLevel.SUCCESS.value
+    INFO_START_CODE = LogLevel.INFO.value
 
-    def __init__(self, base_context: str, target_logs: TargetLogs = TargetLogs.MIXED, logger_name=None):
+    def __init__(self, base_context: str, target_logs: TargetLogs = TargetLogs.MIXED, logger_name=None, max_log_field_size:int =10000, max_log_dict_size:float=256 * 1024 * 0.80):
         self._id = str(uuid.uuid4())
         self._logs = []
         self._early_stop = False
@@ -131,11 +221,15 @@ class PipelineWatcher:
         self._warnings_count = 0
         self._notices_count = 0
         self._successes_count = 0
+        self._infos_count = 0
+        self._systems_impacted = []
         self._level_counts = {level.name: 0 for level in LogLevel}
         self._base_context = base_context
         self._context_stack = []
         self._target_logs = target_logs.value
         self._logger = self._initialize_logger(logger_name)
+        self._max_log_field_size = max_log_field_size
+        self._max_log_dict_size = max_log_dict_size
 
     def _initialize_logger(self, logger_name):
         if logger_name:
@@ -170,6 +264,38 @@ class PipelineWatcher:
     def id(self):
         return self._id
 
+    @property
+    def systems_impacted(self):
+        return self._systems_impacted
+
+    @systems_impacted.setter
+    def systems_impacted(self, list_of_si: List[str]):
+        self._systems_impacted = list_of_si
+
+    def add_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is None:
+            self._systems_impacted = []
+        self._systems_impacted.append(system_impacted)
+
+    def clear_systems_impacted(self):
+        self._systems_impacted = []
+
+    @property
+    def max_log_field_size(self):
+        return self._max_log_field_size
+
+    @max_log_field_size.setter
+    def max_log_field_size(self, value):
+        self._max_log_field_size = value
+
+    @property
+    def max_log_dict_size(self):
+        return self._max_log_dict_size
+
+    @max_log_dict_size.setter
+    def max_log_dict_size(self, value):
+        self._max_log_dict_size = value
+
     @property
     def early_stop(self):
         return self._early_stop
@@ -179,29 +305,28 @@ class PipelineWatcher:
         if create_error_log:
             if pop_context:
                 self.pop_context()
-            self.add_log(ContextLog(level=LogLevel.ERROR,
+            self.add_log(ContextLog(level=LogLevel.ERROR_PIPELINE_THRESHOLD_REACHED,
                                     subject="EARLY_STOP",
                                     description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
 
     def reset_early_stop(self):
         self._early_stop = False
 
-    def get_early_stop(self):
-        return self._early_stop
 
-    def add_log(self, log: ContextLog):
+    def add_log(self, log: ContextLog, ):
         if (self._target_logs == TargetLogs.SUCCESSES and log.level >=self.NOTICE_START_CODE) or \
            (self._target_logs == TargetLogs.WARNINGS_AND_ERRORS and log.level.value < self.WARNING_START_CODE):
-            raise ValueError(f"Invalid log level {log.level.name} for Pipeline Watcher target logs setup: {self._target_logs}")
+            raise ValueError(f"Invalid log level {log.level.name} for Pipelinemon target logs setup: {self._target_logs}")
         log.base_context = self.base_context
         log.context = self.current_context
         log.collector_id = self.id
-        log_dict = log.to_dict()
+        log.systems_impacted = self.systems_impacted
+        log_dict = log.to_dict(max_field_len=self.max_log_field_size, size_limit=self.max_log_dict_size)
         self._logs.append(log_dict)
         self._update_counts(log_dict)
 
         if self._logger:
-            # We specifically want to avoid having an ERROR log level for this structured Pipeline Watcher reporting, to ensure Errors are alerting on Critical Application Services.
+            # We specifically want to avoid having an ERROR log level for this structured Pipelinemon reporting, to ensure Errors are alerting on Critical Application Services.
             # A single ERROR log level can be used for the entire pipeline, which shall be used at the end of the pipeline
             if log.level.value >= self.WARNING_START_CODE:
                 self._logger.log_struct(log_dict, severity="WARNING")
@@ -220,6 +345,7 @@ class PipelineWatcher:
         self._warnings_count = 0
         self._notices_count = 0
         self._successes_count = 0
+        self._infos_count = 0
         self._level_counts = {level.name: 0 for level in LogLevel}
 
     def clear_logs(self):
@@ -258,6 +384,15 @@ class PipelineWatcher:
     def count_successes(self):
         return self._successes_count
 
+    def count_successes_with_notice(self):
+        return self.count_logs_by_level(LogLevel.SUCCESS_WITH_NOTICES)
+
+    def count_successes_no_notice(self):
+        return self.count_logs_by_level(LogLevel.SUCCESS)
+
+    def count_infos(self):
+        return self._infos_count
+
     def count_all_logs(self):
         return len(self._logs)
 
@@ -314,7 +449,13 @@ class PipelineWatcher:
     def count_successes_for_current_and_nested_contexts(self):
         return self._count_logs(self.current_context, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.NOTICE_START_CODE-1)
 
-    def export_logs_to_gcs_file(self, bucket_name, storage_client, file_prefix=None, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
+    def count_infos_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.INFO_START_CODE, level_code_max=self.SUCCESS_START_CODE-1)
+
+    def count_infos_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.INFO_START_CODE, level_code_max=self.SUCCESS_START_CODE-1)
+
+    def export_logs_to_gcs_file(self, bucket_name, storage_client, file_prefix=None, file_name=None, top_level_context=None, save_locally=False, overwrite_if_exists=False, increment_if_exists=True, local_path=None, logger=None, max_retries=2):
         def log_message(message):
             if logger:
                 logger.info(message)
@@ -343,9 +484,11 @@ class PipelineWatcher:
                 local_path=local_path,
                 logger=logger,
                 max_retries=max_retries,
-                overwrite_if_exists=False
+                overwrite_if_exists=overwrite_if_exists,
+                increment_if_exists=increment_if_exists
+
             )
-            log_message(f"{file_prefix} successfully saved (overwritten={result.get('gcs_file_overwritten')}) to GCS at {result.get('gcs_path')} and locally at {result.get('local_path')}.")
+            log_message(f"{file_prefix} successfully saved (overwritten={result.get('gcs_file_overwritten')}, incremented={result.get('gcs_file_saved_with_increment')}) to GCS at {result.get('gcs_path')} and locally at {result.get('local_path')}.")
         except Exception as e:
             log_error(f"Failed at export_logs_to_gcs_file for {file_prefix} for file {file_name} to bucket {bucket_name}: {type(e).__name__} - {str(e)}")
 
@@ -383,6 +526,8 @@ class PipelineWatcher:
                 self._notices_count -= 1
             elif self.SUCCESS_START_CODE <= level_code < self.NOTICE_START_CODE:
                 self._successes_count -= 1
+            elif self.INFO_START_CODE <= level_code < self.SUCCESS_START_CODE:
+                self._infos_count -= 1
             self._level_counts[level_name] -= 1
         else:
             if level_code >= self.ERROR_START_CODE:
@@ -393,4 +538,6 @@ class PipelineWatcher:
                 self._notices_count += 1
             elif self.SUCCESS_START_CODE <= level_code < self.NOTICE_START_CODE:
                 self._successes_count += 1
-            self._level_counts[level_name] += 1
+            elif self.INFO_START_CODE <= level_code < self.SUCCESS_START_CODE:
+                self._infos_count += 1
+            self._level_counts[level_name] += 1

{ipulse_shared_core_ftredge-2.54 → ipulse_shared_core_ftredge-2.56}/src/ipulse_shared_core_ftredge/utils_gcp.py

@@ -1,6 +1,9 @@
 # pylint: disable=missing-module-docstring
 # pylint: disable=missing-function-docstring
 # pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
 import json
 import csv
 from io import StringIO
@@ -23,7 +26,6 @@ from google.api_core.exceptions import NotFound
 
 
 ##### THIS APPROACH IS USED NOW ########
-## TODO Fix the issue with POST 0B Nan.... printed in Cloud Logging , which is referring to posting to Cloud Logging probably.
 ENV = os.getenv('ENV', 'LOCAL').strip("'")
 
 def setup_gcp_logger_and_error_report(logger_name,level=logging.INFO, use_cloud_logging=True):
@@ -130,11 +132,7 @@ def write_json_to_gcs(bucket_name, storage_client, data, file_name,
     This function attempts to upload data to GCS. If the upload fails after
     retries and `save_locally` is True or `local_path` is provided, it attempts
     to save the data locally.
-
-    Returns:
-        dict: A dictionary containing the GCS path (or None if upload failed),
-              the local path (or None if not saved locally), a boolean indicating if the file was overwritten,
-              a boolean indicating if the file already existed, and a boolean indicating if the file was saved with an incremented name.
+    It also tries to handle file name conflicts by overwriting or incrementing. If both are provided as Ture, an exception will be raised.
     """
 
     def log_message(message):
@@ -214,14 +212,13 @@ def write_json_to_gcs(bucket_name, storage_client, data, file_name,
                 local_path_final = os.path.join("/tmp", file_name)
             else:
                 local_path_final = os.path.join(local_path, file_name)
-            
+
             if os.path.exists(local_path_final):
                 if increment_if_exists:
                     increment = 0
                     while os.path.exists(local_path_final):
                         increment += 1
                         local_path_final = os.path.join(local_path, f"{base_file_name}_{increment}{ext}")
-                    gcs_file_saved_with_increment = True
                 elif not overwrite_if_exists:
                     log_message(f"File {file_name} already exists locally at {local_path_final} and overwrite is set to False. Skipping save.")
                     success = True
@@ -267,4 +264,4 @@ def write_csv_to_gcs(bucket_name, file_name, data, storage_client, logger,log_in
     except ValueError as e:
         logger.error(f"ValueError: {e}")
     except Exception as e:
-     logger.error(f"An unexpected error occurred while writing CSV to GCS: {e}", exc_info=True)
+        logger.error(f"An unexpected error occurred while writing CSV to GCS: {e}", exc_info=True)

{ipulse_shared_core_ftredge-2.54 → ipulse_shared_core_ftredge-2.56/src/ipulse_shared_core_ftredge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.54
+Version: 2.56
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar