ipulse-shared-core-ftredge 2.50__py3-none-any.whl → 2.52__py3-none-any.whl

ipulse_shared_core_ftredge/__init__.py

@@ -2,12 +2,12 @@ from .models import (Organisation, UserAuth, UserProfile,
                      UserStatus, UserProfileUpdate, pulse_enums)
 from .utils_gcp import (setup_gcp_logger_and_error_report, 
                         read_csv_from_gcs, read_json_from_gcs, 
-                        write_csv_to_gcs, write_data_to_gcs)
+                        write_csv_to_gcs, write_json_to_gcs)
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
                                           update_check_with_schema_template)
-from .utils_common import (Notice, NoticeSeverity, NoticesManager,SuccessLog, SuccessLogManager)
+from .utils_common import (Notice,  NoticesManager)
 
-from .enums import (NoticeSeverity, Unit, Frequency,
+from .enums import (NoticeManagerCategory, NoticeLevel, Unit, Frequency,
                     Module, SubModule, BaseDataCategory,
                     FinCoreCategory, FincCoreSubCategory,
                     FinCoreRecordsCategory, ExchangeOrPublisher,

ipulse_shared_core_ftredge/enums/__init__.py

@@ -3,7 +3,8 @@
 # pylint: disable=missing-function-docstring
 # pylint: disable=missing-class-docstring
 
-from .enums_common_utils import (NoticeSeverity,
+from .enums_common_utils import (NoticeLevel,
+                                NoticeManagerCategory,
                                 Unit,
                                 Frequency)
 

ipulse_shared_core_ftredge/enums/enums_common_utils.py

@@ -5,27 +5,35 @@
 
 from enum import Enum
 
-class NoticeSeverity(Enum):
+
+class NoticeManagerCategory(Enum):
+    NOTICES = "notices"
+    WARN_ERRS = "warn_errs"
+    SUCCESSES = "successes"
+class NoticeLevel(Enum):
     """
-    Standardized logging levels for data engineering pipelines,
+    Standardized notice levels for data engineering pipelines,
     designed for easy analysis and identification of manual 
     intervention needs.
     """
     DEBUG = 100  # Detailed debug information (for development/troubleshooting)
-    INFO = 200   # Normal pipeline execution information 
-    NOTICE = 300  # Events requiring attention, but not necessarily errors
+
+    INFO = 200
+
+    SUCCESS = 300  # Events requiring attention, but not necessarily errors
 
      # Warnings indicate potential issues that might require attention:
+    WARNING = 400 # General warning, no immediate action required
     WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
     WARNING_REVIEW_RECOMMENDED = 402 # Action recommended to prevent potential future issues
     WARNING_FIX_RECOMMENDED = 403 # Action recommended to prevent potential future issues
     WARNING_FIX_REQUIRED = 404  # Action required, pipeline can likely continue
 
+    ERROR = 500 # General error, no immediate action required
     # Errors indicate a problem that disrupts normal pipeline execution:
-    ERROR_EXCEPTION_REDO = 502
-    ERROR_CUSTOM_REDO = 503 # Temporary error, automatic retry likely to succeed
+    ERROR_EXCEPTION_REDO = 501
+    ERROR_CUSTOM_REDO = 502 # Temporary error, automatic retry likely to succeed
     
-
     ERROR_EXCEPTION_INVESTIGATE = 601 # Exception occured after some data was likely persisted (e.g., to GCS or BQ)
     ERROR_CUSTOM_INVESTIGATE= 602
     ERROR_EXCEPTION_PERSTISTANCE = 603 # Exception occured after data was persisted (e.g., to GCS or BQ)
@@ -37,6 +45,17 @@ class NoticeSeverity(Enum):
 
     UNKNOWN=1001 # Unknown error, should not be used in normal operation
 
+
+class NoticeStatus(Enum):
+    OPEN = "open"
+    ACKNOWLEDGED = "acknowledged"
+    IN_PROGRESS = "in_progress"
+    RESOLVED = "resolved"
+    IGNORED = "ignored"
+  
+
+
+
 ### Exception during full exection, partially saved
 # Exception during ensemble pipeline; modifications collected in local object , nothing persisted
 # Exception during ensemble pipeline; modifications persisted , metadata failed

ipulse_shared_core_ftredge/utils_common.py

@@ -2,116 +2,148 @@
 # pylint: disable=missing-function-docstring
 # pylint: disable=logging-fstring-interpolation
 # pylint: disable=line-too-long
+# pylint: disable=missing-class-docstring
 import traceback
 import json
-import os
-import time
+import uuid
 from datetime import datetime, timezone
 from contextlib import contextmanager
 from typing import List
-from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeSeverity
-from ipulse_shared_core_ftredge.utils_gcp import write_data_to_gcs
-
-def create_notice(severity, e=None, e_type=None, e_message=None, e_traceback=None, subject=None, message=None,context=None):
-    # Validate input: ensure severity is provided, use a default if not
-    if severity is None:
-        severity = NoticeSeverity.UNKNOWN  # Assume Severity.UNKNOWN is a default fallback
-
-    # If an exception object is provided, use it to extract details
-    if e is not None:
-        e_type = type(e).__name__ if e_type is None else e_type
-        e_message = str(e) if e_message is None else e_message
-        e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
-    else:
-        # Calculate traceback if not provided and if exception details are partially present
-        if e_traceback is None and (e_type or e_message):
-            e_traceback = traceback.format_exc()
-
-    # Prepare the base notice dictionary with all fields
-    notice = {
-        "severity_code": severity.value,
-        "severity_name": severity.name,
-        "subject": subject,
-        "message": message,
-        "exception_code": e_type,
-        "exception_message": e_message,
-        "exception_traceback": e_traceback or None,  # Ensure field is present even if traceback isn't calculated
-        "context": context or ""
-    }
-    return notice
-
-
-
-
-def merge_notices_dicts(dict1, dict2):
-    """
-    Merge two dictionaries of lists, combining lists for overlapping keys.
-
-    Parameters:
-    dict1 (dict): The first dictionary of lists.
-    dict2 (dict): The second dictionary of lists.
-
-    Returns:
-    dict: A new dictionary with combined lists for overlapping keys.
-    """
-    merged_dict = {}
-
-    # Get all unique keys from both dictionaries
-    all_keys = set(dict1) | set(dict2)
-
-    for key in all_keys:
-        # Combine lists from both dictionaries for each key
-        merged_dict[key] = dict1.get(key, []) + dict2.get(key, [])
-
-    return merged_dict
+from google.cloud import logging as cloudlogging
+from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeLevel, NoticeManagerCategory, NoticeStatus
+from ipulse_shared_core_ftredge.utils_gcp import write_json_to_gcs
 
 
 # ["data_import","data_quality", "data_processing","data_general","data_persistance","metadata_quality", "metadata_processing", "metadata_persistance","metadata_general"]
 
 class Notice:
-    def __init__(self, severity: NoticeSeverity, e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None, subject: str = None, message: str = None, context: str = None):
-   
-        # If an exception object is provided, use it to extract details
+    MAX_TRACEBACK_LINES = 14  # Define the maximum number of traceback lines to include
+    def __init__(self, level: NoticeLevel, start_context: str = None, notice_manager_id: str = None,
+                 e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
+                 subject: str = None, description: str = None, context: str = None,
+                 notice_status: NoticeStatus = NoticeStatus.OPEN):
         if e is not None:
             e_type = type(e).__name__ if e_type is None else e_type
             e_message = str(e) if e_message is None else e_message
             e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
-         # If exception details are provided but not from an exception object
         elif e_traceback is None and (e_type or e_message):
             e_traceback = traceback.format_exc()
 
-        self.timestamp = datetime.now(timezone.utc).isoformat()
-        self.severity = severity
+        self.level = level
         self.subject = subject
-        self.message = message
-        self.context = context
+        self.description = description
+        self._start_context = start_context
+        self._context = context
+        self.notice_manager_id = notice_manager_id
         self.exception_type = e_type
         self.exception_message = e_message
-        self.exception_traceback = e_traceback
+        self.exception_traceback = self._format_traceback(e_traceback,e_message)
+        self.notice_status = notice_status
+        self.timestamp = datetime.now(timezone.utc).isoformat()
+
+    def _format_traceback(self, e_traceback, e_message):
+        if not e_traceback or e_traceback == 'None\n':
+            return None
+
+        traceback_lines = e_traceback.splitlines()
+        
+        # Remove lines that are part of the exception message if they are present in traceback
+        message_lines = e_message.splitlines() if e_message else []
+        if message_lines:
+            for message_line in message_lines:
+                if message_line in traceback_lines:
+                    traceback_lines.remove(message_line)
+
+        # Filter out lines from third-party libraries (like site-packages)
+        filtered_lines = [line for line in traceback_lines if "site-packages" not in line]
+        
+        # If filtering results in too few lines, revert to original traceback
+        if len(filtered_lines) < 2:
+            filtered_lines = traceback_lines
+
+        # Combine standalone bracket lines with previous or next lines
+        combined_lines = []
+        for line in filtered_lines:
+            if line.strip() in {"(", ")", "{", "}", "[", "]"} and combined_lines:
+                combined_lines[-1] += " " + line.strip()
+            else:
+                combined_lines.append(line)
+
+        # Determine the number of lines to keep from the start and end
+        keep_lines_start = min(self.MAX_TRACEBACK_LINES // 2, len(combined_lines))
+        keep_lines_end = min(self.MAX_TRACEBACK_LINES // 2, len(combined_lines) - keep_lines_start)
+
+        if len(combined_lines) > self.MAX_TRACEBACK_LINES:
+            # Include the first few and last few lines, and an indicator of truncation
+            formatted_traceback = '\n'.join(
+                combined_lines[:keep_lines_start] + 
+                ['... (truncated) ...'] + 
+                combined_lines[-keep_lines_end:]
+            )
+        else:
+            formatted_traceback = '\n'.join(combined_lines)
+
+        return formatted_traceback 
+
+    @property
+    def start_context(self):
+        return self._start_context
+
+    @start_context.setter
+    def start_context(self, value):
+        self._start_context = value
+
+    @property
+    def context(self):
+        return self._context
+
+    @context.setter
+    def context(self, value):
+        self._context = value
 
     def to_dict(self):
         return {
+            "start_context": self.start_context,
             "context": self.context,
-            "severity_code": self.severity.value,
-            "severity_name": self.severity.name,
+            "level_code": self.level.value,
+            "level_name": self.level.name,
             "subject": self.subject,
-            "message": self.message,
+            "description": self.description,
             "exception_type": self.exception_type,
             "exception_message": self.exception_message,
             "exception_traceback": self.exception_traceback,
+            "notice_status": self.notice_status.value,
+            "notice_manager_id": self.notice_manager_id,
+            "timestamp": self.timestamp
         }
 
 class NoticesManager:
-    ERROR_CODE_START_VALUE = 500
+    ERROR_CODE_START_VALUE = NoticeLevel.ERROR.value
+    WARNING_CODE_START_VALUE = NoticeLevel.WARNING.value
+    SUCCESS_CODE_START_VALUE = NoticeLevel.SUCCESS.value
+
+    def __init__(self, start_context: str, category: NoticeManagerCategory = NoticeManagerCategory.NOTICES, logger_name=None):
+        self._id = str(uuid.uuid4())
+        self._notices = []
+        self._early_stop = False
+        self._errors_count = 0
+        self._warnings_count = 0
+        self._successes_count = 0
+        self._level_counts = {level.name: 0 for level in NoticeLevel}
+        self._start_context = start_context
+        self._context_stack = []
+        self._category = category.value
+        self._logger = self._initialize_logger(logger_name)
+
+    def _initialize_logger(self, logger_name):
+        if logger_name:
+            logging_client = cloudlogging.Client()
+            return logging_client.logger(logger_name)
+        return None
 
-    def __init__(self):
-        self.notices = []
-        self.error_count = 0
-        self.severity_counts = {severity.name: 0 for severity in NoticeSeverity}
-        self.context_stack = []
 
     @contextmanager
-    def notice_context(self, context):
+    def context(self, context):
         self.push_context(context)
         try:
             yield
@@ -119,245 +151,155 @@ class NoticesManager:
             self.pop_context()
 
     def push_context(self, context):
-        self.context_stack.append(context)
+        self._context_stack.append(context)
 
     def pop_context(self):
-        if self.context_stack:
-            self.context_stack.pop()
+        if self._context_stack:
+            self._context_stack.pop()
 
-    def get_notices_by_context(self, context_substring: str):
-        return [
-            notice for notice in self.notices
-            if context_substring in notice["context"]
-        ]
+    @property
+    def current_context(self):
+        return " >> ".join(self._context_stack)
 
-    def get_current_context(self):
-        return " >> ".join(self.context_stack)
+    @property
+    def start_context(self):
+        return self._start_context
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def early_stop(self):
+        return self._early_stop
+
+    def set_early_stop(self, max_errors_tolerance:int, create_error_notice=True,pop_context=False):
+        self.early_stop = True
+        if create_error_notice:
+            if pop_context:
+                self.pop_context()
+            self.add_notice(Notice(level=NoticeLevel.ERROR,
+                    subject="EARLY_STOP",
+                    description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
+
+    def reset_early_stop(self):
+        self._early_stop = False
+
+    def get_early_stop(self):
+        return self._early_stop
 
-    def get_all_notices(self):
-        return self.notices
     def add_notice(self, notice: Notice):
-        notice.context = self.get_current_context()
+        if (self._category == NoticeManagerCategory.SUCCESSES.value and notice.level != NoticeLevel.SUCCESS) or \
+           (self._category == NoticeManagerCategory.WARN_ERRS.value and notice.level.value < self.WARNING_CODE_START_VALUE):
+            raise ValueError(f"Invalid notice level {notice.level.name} for category {self._category}")
+        notice.start_context = self.start_context
+        notice.context = self.current_context
+        notice.notice_manager_id = self.id
         notice_dict = notice.to_dict()
-        self.notices.append(notice_dict)
+        self._notices.append(notice_dict)
         self._update_counts(notice_dict)
 
+        if self._logger:
+            if notice.level.value >= self.WARNING_CODE_START_VALUE:
+                self._logger.log_struct(notice_dict, severity="WARNING")
+            else:
+                self._logger.log_struct(notice_dict, severity="INFO")
+
     def add_notices(self, notices: List[Notice]):
         for notice in notices:
-            notice.context = self.get_current_context()
-            notice_dict = notice.to_dict()
-            self.notices.append(notice_dict)
-            self._update_counts(notice_dict)
-   
-    def remove_notice(self, notice: Notice):
-        notice_dict = notice.to_dict()
-        if notice_dict in self.notices:
-            self.notices.remove(notice_dict)
-            self._update_counts(notice_dict, remove=True)
+            self.add_notice(notice)
 
-    def clear_notices(self):
-        self.notices = []
-        self.error_count = 0
-        self.severity_counts = {severity.name: 0 for severity in NoticeSeverity}
+    def clear_notices_and_counts(self):
+        self._notices = []
+        self._errors_count = 0
+        self._warnings_count = 0
+        self._successes_count = 0
+        self._level_counts = {level.name: 0 for level in NoticeLevel}
 
-    def contains_errors(self):
-        return self.error_count > 0
+    def clear_notices(self):
+        self._notices = []
 
-    def count_errors(self):
-        return self.error_count
+    def get_all_notices(self):
+        return self._notices
 
-    def count_notices_by_severity(self, severity: NoticeSeverity):
-        return self.severity_counts.get(severity.name, 0)
-    
-    def count_errors_for_current_context(self):
-        current_context = self.get_current_context()
-        return sum(
-            1 for notice in self.notices
-            if notice["context"] == current_context and notice["severity_code"] >= self.ERROR_CODE_START_VALUE
-        )
-    def count_all_notices(self):
-        return len(self.notices)
+    def get_notices_for_level(self, level: NoticeLevel):
+        return [notice for notice in self._notices if notice["level_code"] == level.value]
 
-    def count_notices_for_current_context(self):
-        current_context = self.get_current_context()
-        return sum(
-            1 for notice in self.notices
-            if notice["context"] == current_context
-        )
-    
-    def count_notices_by_severity_for_current_context(self, severity: NoticeSeverity):
-        current_context = self.get_current_context()
-        return sum(
-            1 for notice in self.notices
-            if notice["context"] == current_context and notice["severity_code"] == severity.value
-        )
-    def count_notices_for_current_and_nested_contexts(self):
-        current_context = self.get_current_context()
-        return sum(
-            1 for notice in self.notices
-            if current_context in notice["context"]
-        )
-    def count_errors_for_current_and_nested_contexts(self):
-        current_context = self.get_current_context()
-        return sum(
-            1 for notice in self.notices
-            if current_context in notice["context"] and notice["severity_code"] >= self.ERROR_CODE_START_VALUE
-        )
-    def count_notices_by_severity_for_current_and_nested_contexts(self, severity: NoticeSeverity):
-        current_context = self.get_current_context()
-        return sum(
-            1 for notice in self.notices
-            if current_context in notice["context"] and notice["severity_code"] == severity.value
-        )
+    def get_notices_by_str_in_context(self, context_substring: str):
+        return [
+            notice for notice in self._notices
+            if context_substring in notice["context"]
+        ]
     
-    def export_notices_to_gcs_file(self, bucket_name, storage_client, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
-        def log_message(message):
-            if logger:
-                logger.info(message)
-
-        def log_error(message, exc_info=False):
-            if logger:
-                logger.error(message, exc_info=exc_info)
-
-        if not file_name:
-            timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
-            if top_level_context:
-                file_name = f"notices_{timestamp}_{top_level_context}_len{len(self.notices)}.json"
-            else:
-                file_name = f"notices_{timestamp}_len{len(self.notices)}.json"
-
-        cloud_path = None  # Initialize cloud_path here
-        local_path = None  # Initialize local_path here
-        try:
-            cloud_path, local_path = write_data_to_gcs(
-                bucket_name=bucket_name,
-                storage_client=storage_client,
-                data=self.notices,
-                file_name=file_name,
-                save_locally=save_locally,
-                local_path=local_path,
-                logger=logger,
-                max_retries=max_retries
-            )
-            log_message(f"Notices successfully saved to GCS at {cloud_path} and locally at {local_path}.")
-        except Exception as e:
-            log_error(f"Failed to export notices: {type(e).__name__} - {str(e)}", exc_info=True)
-
-        return cloud_path , local_path
-
-    def import_notices_from_json(self, json_or_file, logger=None):
-        def log_message(message):
-            if logger:
-                logger.info(message)
-            else:
-                print(message)
+    def contains_errors(self):
+        return self._errors_count > 0
 
-        def log_error(message, exc_info=False):
-            if logger:
-                logger.error(message, exc_info=exc_info)
-            else:
-                print(message)
-        try:
-            if isinstance(json_or_file, str):  # Load from string
-                imported_notices = json.loads(json_or_file)
-            elif hasattr(json_or_file, 'read'):  # Load from file-like object
-                imported_notices = json.load(json_or_file)
-            self.add_notice(imported_notices)
-            log_message("Successfully imported notices from json.")
-        except Exception as e:
-            log_error(f"Failed to import notices from json: {type(e).__name__} - {str(e)}", exc_info=True)
+    def count_errors(self):
+        return self._errors_count
 
-    def _update_counts(self, notice, remove=False):
-        if remove:
-            if notice["severity_code"] >= self.ERROR_CODE_START_VALUE:
-                self.error_count -= 1
-            self.severity_counts[notice["severity_name"]] -= 1
-        else:
-            if notice["severity_code"] >= self.ERROR_CODE_START_VALUE:
-                self.error_count += 1
-            self.severity_counts[notice["severity_name"]] += 1
+    def contains_warnings_or_errors(self):
+        return self._warnings_count > 0 or self._errors_count > 0
 
+    def count_warnings_and_errors(self):
+        return self._warnings_count + self._errors_count
 
-class SuccessLog:
-    def __init__(self, subject:str, description:str=None, context:str=None):
-        self.context = context
-        self.subject = subject
-        self.timestamp = datetime.now(timezone.utc).isoformat()
-        self.description = description
+    def count_warnings(self):
+        return self._warnings_count
 
-    def to_dict(self):
-        return {
-            "context": self.context or "",
-            "subject": self.subject,
-            "timestamp": self.timestamp,
-            "description": self.description or ""
-        }
+    def count_successes(self):
+        return self._successes_count
 
+    def count_all_notices(self):
+        return len(self._notices)
 
-class SuccessLogManager:
-    def __init__(self):
-        self.successlogs = []
-        self.context_stack = []
+    def count_notices_by_level(self, level: NoticeLevel):
+        return self._level_counts.get(level.name, 0)
 
-    @contextmanager
-    def successlog_context(self, context):
-        self.push_context(context)
-        try:
-            yield
-        finally:
-            self.pop_context()
+    def _count_notices(self, context_substring: str, exact_match=False, level_code_min=None, level_code_max=None):
+        return sum(
+            1 for notice in self._notices
+            if (notice["context"] == context_substring if exact_match else context_substring in notice["context"]) and
+               (level_code_min is None or notice["level_code"] >= level_code_min) and
+               (level_code_max is None or notice["level_code"] <= level_code_max)
+        )
 
-    def push_context(self, context):
-        self.context_stack.append(context)
+    def count_notices_for_current_context(self):
+        return self._count_notices(self.current_context, exact_match=True)
 
-    def pop_context(self):
-        if self.context_stack:
-            self.context_stack.pop()
+    def count_notices_for_current_and_nested_contexts(self):
+        return self._count_notices(self.current_context)
 
-    def get_current_context(self):
-        return " >> ".join(self.context_stack)
+    def count_notices_by_level_for_current_context(self, level: NoticeLevel):
+        return self._count_notices(self.current_context, exact_match=True, level_code_min=level.value, level_code_max=level.value)
 
-    def get_all_successlogs(self):
-        return self.successlogs
+    def count_notices_by_level_for_current_and_nested_contexts(self, level: NoticeLevel):
+        return self._count_notices(self.current_context, level_code_min=level.value, level_code_max=level.value)
 
-    def add_successlog(self, successlog: SuccessLog):
-        successlog.context = self.get_current_context()
-        successlog_dict = successlog.to_dict()
-        self.successlogs.append(successlog_dict)
+    def count_errors_for_current_context(self):
+        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.ERROR_CODE_START_VALUE)
 
-    def add_successlogs(self, successlogs: List[SuccessLog]):
-        for successlog in successlogs:
-            successlog.context = self.get_current_context()
-            successlog_dict = successlog.to_dict()
-            self.successlogs.append(successlog_dict)
+    def count_errors_for_current_and_nested_contexts(self):
+        return self._count_notices(self.current_context, level_code_min=self.ERROR_CODE_START_VALUE)
 
-    def remove_successlog(self, successlog: SuccessLog):
-        successlog_dict = successlog.to_dict()
-        if successlog_dict in self.successlogs:
-            self.successlogs.remove(successlog_dict)
+    def count_warnings_and_errors_for_current_context(self):
+        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE)
 
-    def clear_successlogs(self):
-        self.successlogs = []
+    def count_warnings_and_errors_for_current_and_nested_contexts(self):
+        return self._count_notices(self.current_context, level_code_min=self.WARNING_CODE_START_VALUE)
 
-    def count_all_successlogs(self):
-        return len(self.successlogs)
+    def count_warnings_for_current_context(self):
+        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
 
-    def count_successlogs_for_current_context(self):
-        current_context = self.get_current_context()
-        return sum(
-            1 for successlog in self.successlogs
-            if successlog["context"] == current_context
-        )
+    def count_warnings_for_current_and_nested_contexts(self):
+        return self._count_notices(self.current_context, level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
 
-    def count_successlogs_for_current_and_nested_contexts(self):
-        current_context = self.get_current_context()
-        return sum(
-            1 for successlog in self.successlogs
-            if current_context in successlog["context"]
-        )
+    def count_successes_for_current_context(self):
+        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
 
+    def count_successes_for_current_and_nested_contexts(self):
+        return self._count_notices(self.current_context, level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
 
-    def export_successlogs_to_gcs_file(self, bucket_name, storage_client, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=3):
+    def export_notices_to_gcs_file(self, bucket_name, storage_client, file_prefix=None, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
         def log_message(message):
             if logger:
                 logger.info(message)
@@ -366,50 +308,70 @@ class SuccessLogManager:
             if logger:
                 logger.error(message, exc_info=exc_info)
 
+        if not file_prefix:
+            file_prefix = self._category
         if not file_name:
             timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
             if top_level_context:
-                file_name = f"successlogs_{timestamp}_{top_level_context}_len{len(self.successlogs)}.json"
+                file_name = f"{file_prefix}_{timestamp}_{top_level_context}_len{len(self._notices)}.json"
             else:
-                file_name = f"successlogs_{timestamp}_len{len(self.successlogs)}.json"
+                file_name = f"{file_prefix}_{timestamp}_len{len(self._notices)}.json"
 
-        cloud_path=None
-        local_path=None
+        result=None
         try:
-            cloud_path, local_path = write_data_to_gcs(
+            result= write_json_to_gcs(
                 bucket_name=bucket_name,
                 storage_client=storage_client,
-                data=self.successlogs,
+                data=self._notices,
                 file_name=file_name,
                 save_locally=save_locally,
                 local_path=local_path,
                 logger=logger,
-                max_retries=max_retries
+                max_retries=max_retries,
+                overwrite=True
             )
-            log_message(f"Success logs successfully saved to GCS at {cloud_path} and locally at {local_path}.")
+            log_message(f"{file_prefix} successfully saved (ovewritten={result.get("ovewritten")}) to GCS at {result.get("gcs_path")} and locally at {result.get("local_path")}.")
         except Exception as e:
-            log_error(f"Failed to export success logs: {type(e).__name__} - {str(e)}", exc_info=True)
+            log_error(f"Failed at export_notices_to_gcs_file for {file_prefix} for file {file_name} to bucket {bucket_name}: {type(e).__name__} - {str(e)}")
 
-        return cloud_path, local_path
-    
-    def import_successlogs_from_json(self, json_or_file, logger=None):
+        return result
+
+    def import_notices_from_json(self, json_or_file, logger=None):
         def log_message(message):
             if logger:
                 logger.info(message)
-            else:
-                print(message)
 
-        def log_error(message, exc_info=False):
+        def log_warning(message, exc_info=False):
             if logger:
-                logger.error(message, exc_info=exc_info)
-            else:
-                print(message)
+                logger.warning(message, exc_info=exc_info)
+
         try:
             if isinstance(json_or_file, str):  # Load from string
-                imported_success_logs = json.loads(json_or_file)
+                imported_notices = json.loads(json_or_file)
             elif hasattr(json_or_file, 'read'):  # Load from file-like object
-                imported_success_logs = json.load(json_or_file)
-            self.add_successlog(imported_success_logs)
-            log_message("Successfully imported success logs from json.")
+                imported_notices = json.load(json_or_file)
+            self.add_notices(imported_notices)
+            log_message("Successfully imported notices from json.")
         except Exception as e:
-            log_error(f"Failed to import success logs from json: {type(e).__name__} - {str(e)}", exc_info=True)
+            log_warning(f"Failed to import notices from json: {type(e).__name__} - {str(e)}", exc_info=True)
+
+    def _update_counts(self, notice, remove=False):
+        level_code = notice["level_code"]
+        level_name = notice["level_name"]
+
+        if remove:
+            if level_code >= self.ERROR_CODE_START_VALUE:
+                self._errors_count -= 1
+            elif level_code >= self.WARNING_CODE_START_VALUE:
+                self._warnings_count -= 1
+            elif level_code >= self.SUCCESS_CODE_START_VALUE:
+                self._successes_count -= 1
+            self._level_counts[level_name] -= 1
+        else:
+            if level_code >= self.ERROR_CODE_START_VALUE:
+                self._errors_count += 1
+            elif level_code >= self.WARNING_CODE_START_VALUE:
+                self._warnings_count += 1
+            elif level_code == self.SUCCESS_CODE_START_VALUE:
+                self._successes_count += 1
+            self._level_counts[level_name] += 1

ipulse_shared_core_ftredge/utils_gcp.py

@@ -7,7 +7,6 @@ from io import StringIO
 import logging
 import os
 import time
-from datetime import datetime, timezone
 import traceback
 from google.cloud import error_reporting, logging as cloud_logging
 from google.api_core.exceptions import NotFound
@@ -123,47 +122,32 @@ def read_csv_from_gcs(bucket_name, file_name, storage_client, logger):
 
 
 
-def write_data_to_gcs(bucket_name, storage_client, data, file_name=None, 
-                      save_locally=False, local_path=None, logger=None, max_retries=3):
+def write_json_to_gcs(bucket_name, storage_client, data, file_name=None, 
+                      save_locally=False, local_path=None, logger=None, max_retries=3, overwrite=True):
     """Saves data to Google Cloud Storage and optionally locally.
 
     This function attempts to upload data to GCS. If the upload fails after
     retries and `save_locally` is True or `local_path` is provided, it attempts
     to save the data locally.
 
-    Args:
-        bucket_name (str): Name of the GCS bucket.
-        storage_client (google.cloud.storage.Client): GCS client object.
-        data (list, dict, or str): Data to be saved.
-        file_name (str, optional): File name for GCS and local. Defaults to None.
-        save_locally (bool, optional): Save locally if GCS fails. Defaults to False.
-        local_path (str, optional): Local directory to save. Defaults to None.
-        logger (logging.Logger, optional): Logger for messages. Defaults to None.
-        max_retries (int, optional): Number of GCS upload retries. Defaults to 3.
-
     Returns:
-        tuple: A tuple containing the GCS path (or None if upload failed) and
-               the local path (or None if not saved locally).
-
-    Raises:
-        ValueError: If data is not a list, dict, or str.
-        Exception: If GCS upload fails after retries and local saving fails or
-                   is not requested. If GCS upload fails after retries and
-                   local saving is requested but unsuccessful.
+        dict: A dictionary containing the GCS path (or None if upload failed),
+              the local path (or None if not saved locally), and a boolean indicating if the file was overwritten.
     """
 
     def log_message(message):
         if logger:
             logger.info(message)
 
-    def log_error(message, exc_info=False):
+    def log_error(message,exc_info=False):
         if logger:
             logger.error(message, exc_info=exc_info)
 
     attempts = 0
     success = False
-    cloud_path = None
+    gcs_path = None
     local_path_final = None
+    overwritten = False
     gcs_upload_exception = None  # Store potential GCS exception
 
     if isinstance(data, (list, dict)):
@@ -177,17 +161,25 @@ def write_data_to_gcs(bucket_name, storage_client, data, file_name=None,
         try:
             bucket = storage_client.bucket(bucket_name)
             blob = bucket.blob(file_name)
+
+            # Check if the file exists and if we should overwrite it
+            if blob.exists():
+                if not overwrite:
+                    raise FileExistsError(f"File {file_name} already exists in bucket {bucket_name} and overwrite is set to False.")
+                else:
+                    overwritten = True
+
             blob.upload_from_string(data_str, content_type='application/json')
-            cloud_path = f"{bucket_name}/{file_name}"
-            log_message(f"Successfully saved file to GCS {cloud_path}.")
+            gcs_path = f"gs://{bucket_name}/{file_name}"
+            log_message(f"Successfully saved file to GCS {gcs_path}.")
             success = True
         except Exception as e:
-            gcs_upload_exception = e 
+            gcs_upload_exception = e
             attempts += 1
-            log_error(f"Attempt {attempts} - Failed to write {file_name} "
-                        f"to GCS bucket '{bucket_name}': {e}")  # Log with full traceback
             if attempts < max_retries:
                 time.sleep(2 ** attempts)
+            else:
+                log_error(f"Failed to write {file_name} to GCS bucket {bucket_name} after {max_retries} attempts: {e}")
 
     if not success and (save_locally or local_path):
         try:
@@ -199,14 +191,16 @@ def write_data_to_gcs(bucket_name, storage_client, data, file_name=None,
                 f.write(data_str)
             log_message(f"Saved {file_name} locally at {local_path_final}.")
         except Exception as local_e:
-            log_error(f"Failed to write {file_name} locally: {local_e}",exc_info=True)
-
-        # If GCS upload failed, raise a single exception here
+            log_error(f"Failed to write {file_name} locally: {local_e}", exc_info=True)
 
-        if gcs_upload_exception:
-            raise gcs_upload_exception from None  # Propagate without nesting
+    if gcs_upload_exception is not None:
+        raise gcs_upload_exception  # Propagate without nesting
 
-    return cloud_path, local_path_final
+    return {
+        "gcs_path": gcs_path,
+        "local_path": local_path_final,
+        "overwritten": overwritten
+    }
 
 
 def write_csv_to_gcs(bucket_name, file_name, data, storage_client, logger,log_info_verbose=True):

ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -5,7 +5,7 @@
 
 import datetime
 from google.cloud import bigquery
-from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeSeverity
+from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeLevel
 from ipulse_shared_core_ftredge.utils_common import Notice
 
 
@@ -61,9 +61,9 @@ def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_
                 valid_updates[field_name] = value
 
         elif mode == "REQUIRED":
-            notice=Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+            notice=Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                              subject=field_name,
-                             message=f"Required field '{field_name}' is missing in the updates.")
+                             description=f"Required field '{field_name}' is missing in the updates.")
 
             notices.append(notice)
 
@@ -82,13 +82,13 @@ def handle_date_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_date, None
         except ValueError:
-            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
-                                                   message=f"Expected a DATE in YYYY-MM-DD format but got {value}.")
+                                                   description=f"Expected a DATE in YYYY-MM-DD format but got {value}.")
     else:
-        return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+        return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                                               subject=field_name,
-                                              message= f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
+                                              description= f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
 
 
 def handle_timestamp_fields(field_name, value, dt_ts_to_str):
@@ -104,21 +104,21 @@ def handle_timestamp_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_datetime, None
         except ValueError:
-            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
-                                                  message= f"Expected ISO format TIMESTAMP but got {value}.")
+                                                  description= f"Expected ISO format TIMESTAMP but got {value}.")
     else:
-        return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+        return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                                               subject=field_name,
-                                              message= f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
+                                              description= f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
 
 
 def check_and_truncate_length(field_name, value, max_length):
     """Checks and truncates the length of string fields if they exceed the max length."""
     if isinstance(value, str) and len(value) > max_length:
-        return value[:max_length], Notice(severity=NoticeSeverity.WARNING_FIX_RECOMMENDED,
+        return value[:max_length], Notice(level=NoticeLevel.WARNING_FIX_RECOMMENDED,
                             subject= field_name,
-                             message= f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
+                             description= f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
 
     return value, None
 
@@ -126,27 +126,27 @@ def check_and_truncate_length(field_name, value, max_length):
 
 def handle_type_conversion(field_type, field_name, value):
     if field_type == "STRING" and not isinstance(value, str):
-        return str(value), Notice(severity=NoticeSeverity.WARNING_REVIEW_RECOMMENDED,
+        return str(value), Notice(level=NoticeLevel.WARNING_REVIEW_RECOMMENDED,
                              subject=field_name,
-                             message= f"Expected STRING but got {value} of type {type(value).__name__}.")
+                             description= f"Expected STRING but got {value} of type {type(value).__name__}.")
 
     if field_type == "INT64" and not isinstance(value, int):
         try:
             return int(value), None
         except ValueError:
-            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                                                 subject= field_name,
-                                                message=f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
+                                                description=f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
     if field_type == "FLOAT64" and not isinstance(value, float):
         try:
             return float(value), None
         except ValueError:
-            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED, 
+            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED, 
                                                 subject=field_name,
-                                                message=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
+                                                description=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
     if field_type == "BOOL" and not isinstance(value, bool):
-        return bool(value), Notice(severity=NoticeSeverity.WARNING_REVIEW_RECOMMENDED, 
+        return bool(value), Notice(level=NoticeLevel.WARNING_REVIEW_RECOMMENDED, 
                                                 subject=field_name,
-                                                message=f"Expected BOOL, but got  {value}. Converting as {bool(value)}.")
+                                                description=f"Expected BOOL, but got  {value}. Converting as {bool(value)}.")
 
     return value, None

{ipulse_shared_core_ftredge-2.50.dist-info → ipulse_shared_core_ftredge-2.52.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.50
+Version: 2.52
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.50.dist-info → ipulse_shared_core_ftredge-2.52.dist-info}/RECORD

@@ -1,9 +1,9 @@
-ipulse_shared_core_ftredge/__init__.py,sha256=jAtj9nifLy5nOgpVuc5JeAQAkyF8a_5IzWHKexFZT2Q,904
-ipulse_shared_core_ftredge/utils_common.py,sha256=w038BKQZqqndM-9hYT-tZ0opw3QbVgBemJxug8d6HK0,15565
-ipulse_shared_core_ftredge/utils_gcp.py,sha256=OuR0tMGDSn69kGWhrDP1URuItrtGLu6T3vQlQS65IMk,9389
-ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=67WaefoBXFiRKoYKCo3XcThODJyW7YoSWhsZOj1nTlo,7107
-ipulse_shared_core_ftredge/enums/__init__.py,sha256=PjxJiUConI2TuaG_Ushe2BaFVjBDw1rbq1E9Vt9nXvE,801
-ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=aUH5SJOInEvVA_jtKwaBYNc1_5E3ud5IWCVSEiK1cCI,4895
+ipulse_shared_core_ftredge/__init__.py,sha256=g3PRhnLbDFPVAFPIpL6tQgiEJFbPGPmeleHoxPNaQzE,879
+ipulse_shared_core_ftredge/utils_common.py,sha256=AT8R7BfEbOvMztgxQKv6d6jwGfTCJMO5FDz9idTSE4A,15564
+ipulse_shared_core_ftredge/utils_gcp.py,sha256=KZSuugt-746lfSAlNi8ks1llxxPzTskmtZwnm18SnhQ,8873
+ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=OriQHxM4AU6T3kGwwhjRdMt3ZYGmMJe0B5PLcHyzgXk,7084
+ipulse_shared_core_ftredge/enums/__init__.py,sha256=Pg8LUhBb7PJAHULoM13TrFEzG9wgCmw-ZuOdN3Rw6Og,853
+ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=oW0zhmJZfeYycVXWDCede1_Vaa0Q-KClp_KOK4kzIj8,5261
 ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=2i6Qo6Yi_j_O9xxnOD6QA-r0Cv7mWAUaKUx907XMRio,1825
 ipulse_shared_core_ftredge/enums/enums_module_fincore.py,sha256=MuqQg249clrWUOBb1S-iPsoOldN2_F3ohRQizbjhwG0,1374
 ipulse_shared_core_ftredge/enums/enums_modules.py,sha256=AyXUoNmR75DZLaEHi3snV6LngR25LeZRqzrLDaAupbY,1244
@@ -18,8 +18,8 @@ ipulse_shared_core_ftredge/models/user_profile_update.py,sha256=oKK0XsQDKkgDvjFP
 ipulse_shared_core_ftredge/models/user_status.py,sha256=8TyRd8tBK9_xb0MPKbI5pn9-lX7ovKbeiuWYYPtIOiw,3202
 ipulse_shared_core_ftredge/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ipulse_shared_core_ftredge/tests/test.py,sha256=0lS8HP5Quo_BqNoscU40qOH9aJRaa1Pfam5VUBmdld8,682
-ipulse_shared_core_ftredge-2.50.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
-ipulse_shared_core_ftredge-2.50.dist-info/METADATA,sha256=Y00Y547YKmK5-I0zlA522urqyihif5GUVgCG5yAXZIg,561
-ipulse_shared_core_ftredge-2.50.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-ipulse_shared_core_ftredge-2.50.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
-ipulse_shared_core_ftredge-2.50.dist-info/RECORD,,
+ipulse_shared_core_ftredge-2.52.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
+ipulse_shared_core_ftredge-2.52.dist-info/METADATA,sha256=TKMk8quvkJXQbkQmBUcANVJfow831f3dHL2QGqwU9IM,561
+ipulse_shared_core_ftredge-2.52.dist-info/WHEEL,sha256=rWxmBtp7hEUqVLOnTaDOPpR-cZpCDkzhhcBce-Zyd5k,91
+ipulse_shared_core_ftredge-2.52.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
+ipulse_shared_core_ftredge-2.52.dist-info/RECORD,,

{ipulse_shared_core_ftredge-2.50.dist-info → ipulse_shared_core_ftredge-2.52.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.3.0)
+Generator: setuptools (71.0.4)
 Root-Is-Purelib: true
 Tag: py3-none-any