ipulse-shared-core-ftredge 2.51__py3-none-any.whl → 2.53__py3-none-any.whl

ipulse_shared_core_ftredge/__init__.py

@@ -2,12 +2,12 @@ from .models import (Organisation, UserAuth, UserProfile,
                      UserStatus, UserProfileUpdate, pulse_enums)
 from .utils_gcp import (setup_gcp_logger_and_error_report, 
                         read_csv_from_gcs, read_json_from_gcs, 
-                        write_csv_to_gcs, write_data_to_gcs)
+                        write_csv_to_gcs, write_json_to_gcs)
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
-                                          update_check_with_schema_template)
-from .utils_common import (Notice,  NoticesManager)
+                                          check_format_against_schema_template)
+from .utils_common import (SmartLog,  Watcher)
 
-from .enums import (NoticeManagerCategory, NoticeLevel, Unit, Frequency,
+from .enums import (WatcherCategory, LogLevel, Unit, Frequency,
                     Module, SubModule, BaseDataCategory,
                     FinCoreCategory, FincCoreSubCategory,
                     FinCoreRecordsCategory, ExchangeOrPublisher,

ipulse_shared_core_ftredge/enums/__init__.py

@@ -3,8 +3,8 @@
 # pylint: disable=missing-function-docstring
 # pylint: disable=missing-class-docstring
 
-from .enums_common_utils import (NoticeLevel,
-                                NoticeManagerCategory,
+from .enums_common_utils import (LogLevel,
+                                WatcherCategory,
                                 Unit,
                                 Frequency)
 

ipulse_shared_core_ftredge/enums/enums_common_utils.py

@@ -6,11 +6,17 @@
 from enum import Enum
 
 
-class NoticeManagerCategory(Enum):
-    NOTICES = "notices"
-    WARN_ERRS = "warn_errs"
-    SUCCESSES = "successes"
-class NoticeLevel(Enum):
+class WatcherCategory(Enum):
+    MIXED="watcher_mixed"
+    SUCCESSES = "watcher_successes"
+    NOTICES = "watcher_notices"
+    SUCCESSES_AND_NOTICES = "watcher_succs_n_notcs"
+    WARNINGS = "watcher_warnings"
+    WARNINGS_AND_ERRORS = "watcher_warns_n_errs"
+    ERRORS = "watcher_errors"
+
+
+class LogLevel(Enum):
     """
     Standardized notice levels for data engineering pipelines,
     designed for easy analysis and identification of manual 
@@ -19,12 +25,16 @@ class NoticeLevel(Enum):
     DEBUG = 100  # Detailed debug information (for development/troubleshooting)
 
     INFO = 200
+    SUCCESS = 201
 
-    SUCCESS = 300  # Events requiring attention, but not necessarily errors
+    NOTICE = 300  # Maybe same file or data already fully or partially exists
+    NOTICE_ALREADY_EXISTS = 301 # Data already exists, no action required
+    NOTICE_PARTIAL_EXISTS = 302 # Partial data exists, no action required
+    NOTICE_CANCELLED = 303 # Data processing cancelled, no action required
 
      # Warnings indicate potential issues that might require attention:
     WARNING = 400 # General warning, no immediate action required
-    WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
+    # WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
     WARNING_REVIEW_RECOMMENDED = 402 # Action recommended to prevent potential future issues
     WARNING_FIX_RECOMMENDED = 403 # Action recommended to prevent potential future issues
     WARNING_FIX_REQUIRED = 404  # Action required, pipeline can likely continue
@@ -46,16 +56,16 @@ class NoticeLevel(Enum):
     UNKNOWN=1001 # Unknown error, should not be used in normal operation
 
 
-class NoticeStatus(Enum):
+class LogStatus(Enum):
     OPEN = "open"
     ACKNOWLEDGED = "acknowledged"
     IN_PROGRESS = "in_progress"
     RESOLVED = "resolved"
     IGNORED = "ignored"
+    CANCELLED = "cancelled"
   
 
 
-
 ### Exception during full exection, partially saved
 # Exception during ensemble pipeline; modifications collected in local object , nothing persisted
 # Exception during ensemble pipeline; modifications persisted , metadata failed

ipulse_shared_core_ftredge/utils_common.py

@@ -10,18 +10,18 @@ from datetime import datetime, timezone
 from contextlib import contextmanager
 from typing import List
 from google.cloud import logging as cloudlogging
-from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeLevel, NoticeManagerCategory, NoticeStatus
-from ipulse_shared_core_ftredge.utils_gcp import write_data_to_gcs
+from ipulse_shared_core_ftredge.enums.enums_common_utils import WatcherCategory, LogLevel, LogStatus
+from ipulse_shared_core_ftredge.utils_gcp import write_json_to_gcs
 
 
 # ["data_import","data_quality", "data_processing","data_general","data_persistance","metadata_quality", "metadata_processing", "metadata_persistance","metadata_general"]
 
-class Notice:
+class SmartLog:
     MAX_TRACEBACK_LINES = 14  # Define the maximum number of traceback lines to include
-    def __init__(self, level: NoticeLevel, start_context: str = None, notice_manager_id: str = None,
+    def __init__(self, level: LogLevel, start_context: str = None, collector_id: str = None,
                  e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
                  subject: str = None, description: str = None, context: str = None,
-                 notice_status: NoticeStatus = NoticeStatus.OPEN):
+                 log_status: LogStatus = LogStatus.OPEN):
         if e is not None:
             e_type = type(e).__name__ if e_type is None else e_type
             e_message = str(e) if e_message is None else e_message
@@ -34,11 +34,11 @@ class Notice:
         self.description = description
         self._start_context = start_context
         self._context = context
-        self.notice_manager_id = notice_manager_id
+        self.collector_id = collector_id
         self.exception_type = e_type
         self.exception_message = e_message
         self.exception_traceback = self._format_traceback(e_traceback,e_message)
-        self.notice_status = notice_status
+        self.log_status = log_status
         self.timestamp = datetime.now(timezone.utc).isoformat()
 
     def _format_traceback(self, e_traceback, e_message):
@@ -72,7 +72,7 @@ class Notice:
         # Determine the number of lines to keep from the start and end
         keep_lines_start = min(self.MAX_TRACEBACK_LINES // 2, len(combined_lines))
         keep_lines_end = min(self.MAX_TRACEBACK_LINES // 2, len(combined_lines) - keep_lines_start)
-        
+
         if len(combined_lines) > self.MAX_TRACEBACK_LINES:
             # Include the first few and last few lines, and an indicator of truncation
             formatted_traceback = '\n'.join(
@@ -82,8 +82,8 @@ class Notice:
             )
         else:
             formatted_traceback = '\n'.join(combined_lines)
-        
-        return formatted_traceback  
+
+        return formatted_traceback
 
     @property
     def start_context(self):
@@ -112,24 +112,26 @@ class Notice:
             "exception_type": self.exception_type,
             "exception_message": self.exception_message,
             "exception_traceback": self.exception_traceback,
-            "notice_status": self.notice_status.value,
-            "notice_manager_id": self.notice_manager_id,
+            "log_status": self.log_status.value,
+            "collector_id": self.collector_id,
             "timestamp": self.timestamp
         }
 
-class NoticesManager:
-    ERROR_CODE_START_VALUE = NoticeLevel.ERROR.value
-    WARNING_CODE_START_VALUE = NoticeLevel.WARNING.value
-    SUCCESS_CODE_START_VALUE = NoticeLevel.SUCCESS.value
+class Watcher:
+    ERROR_START_CODE = LogLevel.ERROR.value
+    WARNING_START_CODE = LogLevel.WARNING.value
+    NOTICE_START_CODE = LogLevel.NOTICE.value
+    SUCCESS_START_CODE = LogLevel.SUCCESS.value
 
-    def __init__(self, start_context: str, category: NoticeManagerCategory = NoticeManagerCategory.NOTICES, logger_name=None):
-        self._notice_manager_id = str(uuid.uuid4())
-        self._notices = []
+    def __init__(self, start_context: str, category: WatcherCategory = WatcherCategory.MIXED, logger_name=None):
+        self._id = str(uuid.uuid4())
+        self._logs = []
         self._early_stop = False
-        self._error_count = 0
-        self._warning_count = 0
-        self._success_count = 0
-        self._level_counts = {level.name: 0 for level in NoticeLevel}
+        self._errors_count = 0
+        self._warnings_count = 0
+        self._notices_count = 0
+        self._successes_count = 0
+        self._level_counts = {level.name: 0 for level in LogLevel}
         self._start_context = start_context
         self._context_stack = []
         self._category = category.value
@@ -141,7 +143,6 @@ class NoticesManager:
             return logging_client.logger(logger_name)
         return None
 
-
     @contextmanager
     def context(self, context):
         self.push_context(context)
@@ -157,141 +158,163 @@ class NoticesManager:
         if self._context_stack:
             self._context_stack.pop()
 
-    def get_current_context(self):
+    @property
+    def current_context(self):
         return " >> ".join(self._context_stack)
 
-    def get_start_context(self):
+    @property
+    def start_context(self):
         return self._start_context
-    
-    def get_notice_manager_id(self):
-        return self._notice_manager_id
-    
-    def set_early_stop(self, max_errors_tolerance:int, create_error_notice=True,pop_context=False):
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def early_stop(self):
+        return self._early_stop
+
+    def set_early_stop(self, max_errors_tolerance: int, create_error_log=True, pop_context=False):
         self._early_stop = True
-        if create_error_notice:
+        if create_error_log:
             if pop_context:
                 self.pop_context()
-            self.add_notice(Notice(level=NoticeLevel.ERROR,
-                    subject="EARLY_STOP",
-                    description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
+            self.add_log(SmartLog(level=LogLevel.ERROR,
+                                    subject="EARLY_STOP",
+                                    description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
+
     def reset_early_stop(self):
         self._early_stop = False
 
     def get_early_stop(self):
         return self._early_stop
 
-    def add_notice(self, notice: Notice):
-        if (self._category == NoticeManagerCategory.SUCCESSES.value and notice.level != NoticeLevel.SUCCESS) or \
-           (self._category == NoticeManagerCategory.WARN_ERRS.value and notice.level.value < self.WARNING_CODE_START_VALUE):
-            raise ValueError(f"Invalid notice level {notice.level.name} for category {self._category}")
-        notice.start_context = self.get_start_context()
-        notice.context = self.get_current_context()
-        notice.notice_manager_id = self._notice_manager_id
-        notice_dict = notice.to_dict()
-        self._notices.append(notice_dict)
-        self._update_counts(notice_dict)
+    def add_log(self, log: SmartLog):
+        if (self._category == WatcherCategory.SUCCESSES and log.level >=self.NOTICE_START_CODE) or \
+           (self._category == WatcherCategory.WARNINGS_AND_ERRORS and log.level.value < self.WARNING_START_CODE):
+            raise ValueError(f"Invalid log level {log.level.name} for category {self._category}")
+        log.start_context = self.start_context
+        log.context = self.current_context
+        log.collector_id = self.id
+        log_dict = log.to_dict()
+        self._logs.append(log_dict)
+        self._update_counts(log_dict)
 
         if self._logger:
-            if notice.level.value >= self.WARNING_CODE_START_VALUE:
-                self._logger.log_struct(notice_dict, severity="WARNING")
+            # We specifically want to avoid having an ERROR log level for this structured Watcher reporting, to ensure Errors are alerting on Critical Application Services.
+            # A single ERROR log level can be used for the entire pipeline, which shall be used at the end of the pipeline
+            if log.level.value >= self.WARNING_START_CODE:
+                self._logger.log_struct(log_dict, severity="WARNING")
+            elif log.level.value >= self.NOTICE_START_CODE:
+                self._logger.log_struct(log_dict, severity="NOTICE")
             else:
-                self._logger.log_struct(notice_dict, severity="INFO")
+                self._logger.log_struct(log_dict, severity="INFO")
 
-    def add_notices(self, notices: List[Notice]):
-        for notice in notices:
-            self.add_notice(notice)
+    def add_logs(self, logs: List[SmartLog]):
+        for log in logs:
+            self.add_log(log)
 
-    def clear_notices_and_counts(self):
-        self._notices = []
-        self._error_count = 0
-        self._warning_count = 0
-        self._success_count = 0
-        self._level_counts = {level.name: 0 for level in NoticeLevel}
+    def clear_logs_and_counts(self):
+        self._logs = []
+        self._errors_count = 0
+        self._warnings_count = 0
+        self._notices_count = 0
+        self._successes_count = 0
+        self._level_counts = {level.name: 0 for level in LogLevel}
 
-    def clear_notices(self):
-        self._notices = []
+    def clear_logs(self):
+        self._logs = []
 
-    def get_all_notices(self):
-        return self._notices
+    def get_all_logs(self):
+        return self._logs
 
-    def get_notices_for_level(self, level: NoticeLevel):
-        return [notice for notice in self._notices if notice["level_code"] == level.value]
+    def get_logs_for_level(self, level: LogLevel):
+        return [log for log in self._logs if log["level_code"] == level.value]
 
-    def get_notices_by_str_in_context(self, context_substring: str):
+    def get_logs_by_str_in_context(self, context_substring: str):
         return [
-            notice for notice in self._notices
-            if context_substring in notice["context"]
+            log for log in self._logs
+            if context_substring in log["context"]
         ]
-
+    
     def contains_errors(self):
-        return self._error_count > 0
+        return self._errors_count > 0
 
     def count_errors(self):
-        return self._error_count
+        return self._errors_count
 
     def contains_warnings_or_errors(self):
-        return self._warning_count > 0 or self._error_count > 0
+        return self._warnings_count > 0 or self._errors_count > 0
 
     def count_warnings_and_errors(self):
-        return self._warning_count + self._error_count
-
+        return self._warnings_count + self._errors_count
+    
     def count_warnings(self):
-        return self._warning_count
+        return self._warnings_count
+
+    def count_notices(self):
+        return self._notices_count
 
     def count_successes(self):
-        return self._success_count
+        return self._successes_count
 
-    def count_all_notices(self):
-        return len(self._notices)
+    def count_all_logs(self):
+        return len(self._logs)
 
-    def count_notices_by_level(self, level: NoticeLevel):
+    def count_logs_by_level(self, level: LogLevel):
         return self._level_counts.get(level.name, 0)
 
-    def _count_notices(self, context_substring: str, exact_match=False, level_code_min=None, level_code_max=None):
+    def _count_logs(self, context_substring: str, exact_match=False, level_code_min=None, level_code_max=None):
         return sum(
-            1 for notice in self._notices
-            if (notice["context"] == context_substring if exact_match else context_substring in notice["context"]) and
-               (level_code_min is None or notice["level_code"] >= level_code_min) and
-               (level_code_max is None or notice["level_code"] <= level_code_max)
+            1 for log in self._logs
+            if (log["context"] == context_substring if exact_match else context_substring in log["context"]) and
+               (level_code_min is None or log["level_code"] >= level_code_min) and
+               (level_code_max is None or log["level_code"] <= level_code_max)
         )
 
-    def count_notices_for_current_context(self):
-        return self._count_notices(self.get_current_context(), exact_match=True)
+    def count_logs_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True)
 
-    def count_notices_for_current_and_nested_contexts(self):
-        return self._count_notices(self.get_current_context())
+    def count_logs_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context)
 
-    def count_notices_by_level_for_current_context(self, level: NoticeLevel):
-        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=level.value, level_code_max=level.value)
+    def count_logs_by_level_for_current_context(self, level: LogLevel):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=level.value, level_code_max=level.value)
 
-    def count_notices_by_level_for_current_and_nested_contexts(self, level: NoticeLevel):
-        return self._count_notices(self.get_current_context(), level_code_min=level.value, level_code_max=level.value)
+    def count_logs_by_level_for_current_and_nested_contexts(self, level: LogLevel):
+        return self._count_logs(self.current_context, level_code_min=level.value, level_code_max=level.value)
 
     def count_errors_for_current_context(self):
-        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.ERROR_CODE_START_VALUE)
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.ERROR_START_CODE)
 
     def count_errors_for_current_and_nested_contexts(self):
-        return self._count_notices(self.get_current_context(), level_code_min=self.ERROR_CODE_START_VALUE)
+        return self._count_logs(self.current_context, level_code_min=self.ERROR_START_CODE)
 
     def count_warnings_and_errors_for_current_context(self):
-        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE)
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE)
 
     def count_warnings_and_errors_for_current_and_nested_contexts(self):
-        return self._count_notices(self.get_current_context(), level_code_min=self.WARNING_CODE_START_VALUE)
+        return self._count_logs(self.current_context, level_code_min=self.WARNING_START_CODE)
 
     def count_warnings_for_current_context(self):
-        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
 
     def count_warnings_for_current_and_nested_contexts(self):
-        return self._count_notices(self.get_current_context(), level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
+        return self._count_logs(self.current_context, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
+
+    def count_notices_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.NOTICE_START_CODE, level_code_max=self.WARNING_START_CODE-1)
+
+    def count_notices_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.NOTICE_START_CODE, level_code_max=self.WARNING_START_CODE-1)
 
     def count_successes_for_current_context(self):
-        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.NOTICE_START_CODE-1)
 
     def count_successes_for_current_and_nested_contexts(self):
-        return self._count_notices(self.get_current_context(), level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
+        return self._count_logs(self.current_context, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.NOTICE_START_CODE-1)
 
-    def export_notices_to_gcs_file(self, bucket_name, storage_client, file_prefix=None, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
+    def export_logs_to_gcs_file(self, bucket_name, storage_client, file_prefix=None, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
         def log_message(message):
             if logger:
                 logger.info(message)
@@ -305,30 +328,30 @@ class NoticesManager:
         if not file_name:
             timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
             if top_level_context:
-                file_name = f"{file_prefix}_{timestamp}_{top_level_context}_len{len(self._notices)}.json"
+                file_name = f"{file_prefix}_{timestamp}_{top_level_context}_len{len(self._logs)}.json"
             else:
-                file_name = f"{file_prefix}_{timestamp}_len{len(self._notices)}.json"
+                file_name = f"{file_prefix}_{timestamp}_len{len(self._logs)}.json"
 
-        cloud_path = None
-        local_path = None
+        result = None
         try:
-            cloud_path, local_path = write_data_to_gcs(
+            result = write_json_to_gcs(
                 bucket_name=bucket_name,
                 storage_client=storage_client,
-                data=self._notices,
+                data=self._logs,
                 file_name=file_name,
                 save_locally=save_locally,
                 local_path=local_path,
                 logger=logger,
-                max_retries=max_retries
+                max_retries=max_retries,
+                overwrite_if_exists=False
             )
-            log_message(f"{file_prefix} successfully saved to GCS at {cloud_path} and locally at {local_path}.")
+            log_message(f"{file_prefix} successfully saved (overwritten={result.get('gcs_file_overwritten')}) to GCS at {result.get('gcs_path')} and locally at {result.get('local_path')}.")
         except Exception as e:
-            log_error(f"Failed at export_notices_to_gcs_file for {file_prefix} for file {file_name} to bucket {bucket_name}: {type(e).__name__} - {str(e)}")
+            log_error(f"Failed at export_logs_to_gcs_file for {file_prefix} for file {file_name} to bucket {bucket_name}: {type(e).__name__} - {str(e)}")
 
-        return cloud_path, local_path
+        return result
 
-    def import_notices_from_json(self, json_or_file, logger=None):
+    def import_logs_from_json(self, json_or_file, logger=None):
         def log_message(message):
             if logger:
                 logger.info(message)
@@ -339,31 +362,296 @@ class NoticesManager:
 
         try:
             if isinstance(json_or_file, str):  # Load from string
-                imported_notices = json.loads(json_or_file)
+                imported_logs = json.loads(json_or_file)
             elif hasattr(json_or_file, 'read'):  # Load from file-like object
-                imported_notices = json.load(json_or_file)
-            self.add_notices(imported_notices)
-            log_message("Successfully imported notices from json.")
+                imported_logs = json.load(json_or_file)
+            self.add_logs(imported_logs)
+            log_message("Successfully imported logs from json.")
         except Exception as e:
-            log_warning(f"Failed to import notices from json: {type(e).__name__} - {str(e)}", exc_info=True)
+            log_warning(f"Failed to import logs from json: {type(e).__name__} - {str(e)}", exc_info=True)
 
-    def _update_counts(self, notice, remove=False):
-        level_code = notice["level_code"]
-        level_name = notice["level_name"]
+    def _update_counts(self, log, remove=False):
+        level_code = log["level_code"]
+        level_name = log["level_name"]
 
         if remove:
-            if level_code >= self.ERROR_CODE_START_VALUE:
-                self._error_count -= 1
-            elif level_code >= self.WARNING_CODE_START_VALUE:
-                self._warning_count -= 1
-            elif level_code >= self.SUCCESS_CODE_START_VALUE:
-                self._success_count -= 1
+            if level_code >= self.ERROR_START_CODE:
+                self._errors_count -= 1
+            elif self.WARNING_START_CODE <= level_code < self.ERROR_START_CODE:
+                self._warnings_count -= 1
+            elif self.NOTICE_START_CODE <= level_code < self.WARNING_START_CODE:
+                self._notices_count -= 1
+            elif self.SUCCESS_START_CODE <= level_code < self.NOTICE_START_CODE:
+                self._successes_count -= 1
             self._level_counts[level_name] -= 1
         else:
-            if level_code >= self.ERROR_CODE_START_VALUE:
-                self._error_count += 1
-            elif level_code >= self.WARNING_CODE_START_VALUE:
-                self._warning_count += 1
-            elif level_code == self.SUCCESS_CODE_START_VALUE:
-                self._success_count += 1
-            self._level_counts[level_name] += 1
+            if level_code >= self.ERROR_START_CODE:
+                self._errors_count += 1
+            elif self.WARNING_START_CODE <= level_code < self.ERROR_START_CODE:
+                self._warnings_count += 1
+            elif self.NOTICE_START_CODE <= level_code < self.WARNING_START_CODE:
+                self._notices_count += 1
+            elif self.SUCCESS_START_CODE <= level_code < self.NOTICE_START_CODE:
+                self._successes_count += 1
+            self._level_counts[level_name] += 1
+
+
+# class Watcher:
+#     ERROR_START_CODE = WatcherLogLevel.ERROR.value
+#     WARNING_START_CODE = WatcherLogLevel.WARNING.value
+#     NOTICE_START_CODE = WatcherLogLevel.NOTICE.value
+#     SUCCESS_START_CODE = WatcherLogLevel.SUCCESS.value
+
+#     def __init__(self, start_context: str, category: WatcherCategory = WatcherCategory.MIXED, logger_name=None):
+#         self._id = str(uuid.uuid4())
+#         self._logs = []
+#         self._early_stop = False
+#         self._errors_count = 0
+#         self._warnings_count = 0
+#         self._successes_count = 0
+#         self._level_counts = {level.name: 0 for level in WatcherLogLevel}
+#         self._start_context = start_context
+#         self._context_stack = []
+#         self._category = category.value
+#         self._logger = self._initialize_logger(logger_name)
+
+#     def _initialize_logger(self, logger_name):
+#         if logger_name:
+#             logging_client = cloudlogging.Client()
+#             return logging_client.logger(logger_name)
+#         return None
+
+
+#     @contextmanager
+#     def context(self, context):
+#         self.push_context(context)
+#         try:
+#             yield
+#         finally:
+#             self.pop_context()
+
+#     def push_context(self, context):
+#         self._context_stack.append(context)
+
+#     def pop_context(self):
+#         if self._context_stack:
+#             self._context_stack.pop()
+
+#     @property
+#     def current_context(self):
+#         return " >> ".join(self._context_stack)
+
+#     @property
+#     def start_context(self):
+#         return self._start_context
+
+#     @property
+#     def id(self):
+#         return self._id
+
+#     @property
+#     def early_stop(self):
+#         return self._early_stop
+
+#     def set_early_stop(self, max_errors_tolerance:int, create_error_notice=True,pop_context=False):
+#         self.early_stop = True
+#         if create_error_notice:
+#             if pop_context:
+#                 self.pop_context()
+#             self.add_notice(WatcherLog(level=WatcherLogLevel.ERROR,
+#                     subject="EARLY_STOP",
+#                     description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
+
+#     def reset_early_stop(self):
+#         self._early_stop = False
+
+#     def get_early_stop(self):
+#         return self._early_stop
+
+#     def add_notice(self, notice: WatcherLog):
+#         if (self._category == WatcherCategory.SUCCESSES.value and notice.level != WatcherLogLevel.SUCCESS) or \
+#            (self._category == WatcherCategory.WARNINGS_AND_ERRORS.value and notice.level.value < self.WARNING_START_CODE):
+#             raise ValueError(f"Invalid notice level {notice.level.name} for category {self._category}")
+#         notice.start_context = self.start_context
+#         notice.context = self.current_context
+#         notice.watcher_id = self.id
+#         notice_dict = notice.to_dict()
+#         self._logs.append(notice_dict)
+#         self._update_counts(notice_dict)
+
+#         if self._logger:
+#             if notice.level.value >= self.WARNING_START_CODE:
+#                 self._logger.log_struct(notice_dict, severity="WARNING")
+#             else:
+#                 self._logger.log_struct(notice_dict, severity="INFO")
+
+#     def add_notices(self, notices: List[WatcherLog]):
+#         for notice in notices:
+#             self.add_notice(notice)
+
+#     def clear_notices_and_counts(self):
+#         self._logs = []
+#         self._errors_count = 0
+#         self._warnings_count = 0
+#         self._successes_count = 0
+#         self._level_counts = {level.name: 0 for level in WatcherLogLevel}
+
+#     def clear_notices(self):
+#         self._logs = []
+
+#     def get_all_notices(self):
+#         return self._logs
+
+#     def get_notices_for_level(self, level: WatcherLogLevel):
+#         return [notice for notice in self._logs if notice["level_code"] == level.value]
+
+#     def get_notices_by_str_in_context(self, context_substring: str):
+#         return [
+#             notice for notice in self._logs
+#             if context_substring in notice["context"]
+#         ]
+    
+#     def contains_errors(self):
+#         return self._errors_count > 0
+
+#     def count_errors(self):
+#         return self._errors_count
+
+#     def contains_warnings_or_errors(self):
+#         return self._warnings_count > 0 or self._errors_count > 0
+
+#     def count_warnings_and_errors(self):
+#         return self._warnings_count + self._errors_count
+
+#     def count_warnings(self):
+#         return self._warnings_count
+
+#     def count_successes(self):
+#         return self._successes_count
+
+#     def count_all_notices(self):
+#         return len(self._logs)
+
+#     def count_notices_by_level(self, level: WatcherLogLevel):
+#         return self._level_counts.get(level.name, 0)
+
+#     def _count_notices(self, context_substring: str, exact_match=False, level_code_min=None, level_code_max=None):
+#         return sum(
+#             1 for notice in self._logs
+#             if (notice["context"] == context_substring if exact_match else context_substring in notice["context"]) and
+#                (level_code_min is None or notice["level_code"] >= level_code_min) and
+#                (level_code_max is None or notice["level_code"] <= level_code_max)
+#         )
+
+#     def count_notices_for_current_context(self):
+#         return self._count_notices(self.current_context, exact_match=True)
+
+#     def count_notices_for_current_and_nested_contexts(self):
+#         return self._count_notices(self.current_context)
+
+#     def count_notices_by_level_for_current_context(self, level: WatcherLogLevel):
+#         return self._count_notices(self.current_context, exact_match=True, level_code_min=level.value, level_code_max=level.value)
+
+#     def count_notices_by_level_for_current_and_nested_contexts(self, level: WatcherLogLevel):
+#         return self._count_notices(self.current_context, level_code_min=level.value, level_code_max=level.value)
+
+#     def count_errors_for_current_context(self):
+#         return self._count_notices(self.current_context, exact_match=True, level_code_min=self.ERROR_START_CODE)
+
+#     def count_errors_for_current_and_nested_contexts(self):
+#         return self._count_notices(self.current_context, level_code_min=self.ERROR_START_CODE)
+
+#     def count_warnings_and_errors_for_current_context(self):
+#         return self._count_notices(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE)
+
+#     def count_warnings_and_errors_for_current_and_nested_contexts(self):
+#         return self._count_notices(self.current_context, level_code_min=self.WARNING_START_CODE)
+
+#     def count_warnings_for_current_context(self):
+#         return self._count_notices(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
+
+#     def count_warnings_for_current_and_nested_contexts(self):
+#         return self._count_notices(self.current_context, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
+
+#     def count_successes_for_current_context(self):
+#         return self._count_notices(self.current_context, exact_match=True, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.SUCCESS_START_CODE)
+
+#     def count_successes_for_current_and_nested_contexts(self):
+#         return self._count_notices(self.current_context, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.SUCCESS_START_CODE)
+
+#     def export_notices_to_gcs_file(self, bucket_name, storage_client, file_prefix=None, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
+#         def log_message(message):
+#             if logger:
+#                 logger.info(message)
+
+#         def log_error(message, exc_info=False):
+#             if logger:
+#                 logger.error(message, exc_info=exc_info)
+
+#         if not file_prefix:
+#             file_prefix = self._category
+#         if not file_name:
+#             timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+#             if top_level_context:
+#                 file_name = f"{file_prefix}_{timestamp}_{top_level_context}_len{len(self._logs)}.json"
+#             else:
+#                 file_name = f"{file_prefix}_{timestamp}_len{len(self._logs)}.json"
+
+#         result=None
+#         try:
+#             result= write_json_to_gcs(
+#                 bucket_name=bucket_name,
+#                 storage_client=storage_client,
+#                 data=self._logs,
+#                 file_name=file_name,
+#                 save_locally=save_locally,
+#                 local_path=local_path,
+#                 logger=logger,
+#                 max_retries=max_retries,
+#                 overwrite_gcs=False
+#             )
+#             log_message(f"{file_prefix} successfully saved (ovewritten={result.get("gcs_file_overwritten")}) to GCS at {result.get("gcs_path")} and locally at {result.get("local_path")}.")
+#         except Exception as e:
+#             log_error(f"Failed at export_notices_to_gcs_file for {file_prefix} for file {file_name} to bucket {bucket_name}: {type(e).__name__} - {str(e)}")
+
+#         return result
+
+#     def import_notices_from_json(self, json_or_file, logger=None):
+#         def log_message(message):
+#             if logger:
+#                 logger.info(message)
+
+#         def log_warning(message, exc_info=False):
+#             if logger:
+#                 logger.warning(message, exc_info=exc_info)
+
+#         try:
+#             if isinstance(json_or_file, str):  # Load from string
+#                 imported_notices = json.loads(json_or_file)
+#             elif hasattr(json_or_file, 'read'):  # Load from file-like object
+#                 imported_notices = json.load(json_or_file)
+#             self.add_notices(imported_notices)
+#             log_message("Successfully imported notices from json.")
+#         except Exception as e:
+#             log_warning(f"Failed to import notices from json: {type(e).__name__} - {str(e)}", exc_info=True)
+
+#     def _update_counts(self, notice, remove=False):
+#         level_code = notice["level_code"]
+#         level_name = notice["level_name"]
+
+#         if remove:
+#             if level_code >= self.ERROR_START_CODE:
+#                 self._errors_count -= 1
+#             elif level_code >= self.WARNING_START_CODE:
+#                 self._warnings_count -= 1
+#             elif level_code >= self.SUCCESS_START_CODE:
+#                 self._successes_count -= 1
+#             self._level_counts[level_name] -= 1
+#         else:
+#             if level_code >= self.ERROR_START_CODE:
+#                 self._errors_count += 1
+#             elif level_code >= self.WARNING_START_CODE:
+#                 self._warnings_count += 1
+#             elif level_code == self.SUCCESS_START_CODE:
+#                 self._successes_count += 1
+#             self._level_counts[level_name] += 1

ipulse_shared_core_ftredge/utils_gcp.py

@@ -122,33 +122,19 @@ def read_csv_from_gcs(bucket_name, file_name, storage_client, logger):
 
 
 
-def write_data_to_gcs(bucket_name, storage_client, data, file_name=None, 
-                      save_locally=False, local_path=None, logger=None, max_retries=3):
+def write_json_to_gcs(bucket_name, storage_client, data, file_name,
+                      save_locally=False, local_path=None, logger=None, max_retries=2,
+                      overwrite_if_exists=False, increment_if_exists=False):
     """Saves data to Google Cloud Storage and optionally locally.
 
     This function attempts to upload data to GCS. If the upload fails after
     retries and `save_locally` is True or `local_path` is provided, it attempts
     to save the data locally.
 
-    Args:
-        bucket_name (str): Name of the GCS bucket.
-        storage_client (google.cloud.storage.Client): GCS client object.
-        data (list, dict, or str): Data to be saved.
-        file_name (str, optional): File name for GCS and local. Defaults to None.
-        save_locally (bool, optional): Save locally if GCS fails. Defaults to False.
-        local_path (str, optional): Local directory to save. Defaults to None.
-        logger (logging.Logger, optional): Logger for messages. Defaults to None.
-        max_retries (int, optional): Number of GCS upload retries. Defaults to 3.
-
     Returns:
-        tuple: A tuple containing the GCS path (or None if upload failed) and
-               the local path (or None if not saved locally).
-
-    Raises:
-        ValueError: If data is not a list, dict, or str.
-        Exception: If GCS upload fails after retries and local saving fails or
-                   is not requested. If GCS upload fails after retries and
-                   local saving is requested but unsuccessful.
+        dict: A dictionary containing the GCS path (or None if upload failed),
+              the local path (or None if not saved locally), a boolean indicating if the file was overwritten,
+              a boolean indicating if the file already existed, and a boolean indicating if the file was saved with an incremented name.
     """
 
     def log_message(message):
@@ -159,12 +145,23 @@ def write_data_to_gcs(bucket_name, storage_client, data, file_name=None,
         if logger:
             logger.error(message, exc_info=exc_info)
 
+    def log_warning(message):
+        if logger:
+            logger.warning(message)
+
     attempts = 0
     success = False
-    cloud_path = None
+    gcs_path = None
     local_path_final = None
+    gcs_file_overwritten = False
+    gcs_file_already_exists = False
+    gcs_file_saved_with_increment = False
     gcs_upload_exception = None  # Store potential GCS exception
 
+    # Check for conflicting options
+    if overwrite_if_exists and increment_if_exists:
+        raise ValueError("When writing JSON to GCS, both overwrite and increment_if_exists cannot be True at the same time.")
+
     if isinstance(data, (list, dict)):
         data_str = json.dumps(data, indent=2)
     elif isinstance(data, str):
@@ -172,13 +169,36 @@ def write_data_to_gcs(bucket_name, storage_client, data, file_name=None,
     else:
         raise ValueError("Unsupported data type. It should be a list, dict, or str.")
 
+    bucket = storage_client.bucket(bucket_name)
+    base_file_name, ext = os.path.splitext(file_name)
+    increment = 0
+
     while attempts < max_retries and not success:
         try:
-            bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(file_name)
+            if increment_if_exists:
+                while bucket.blob(file_name).exists():
+                    gcs_file_already_exists = True
+                    increment += 1
+                    file_name = f"{base_file_name}_{increment}{ext}"
+                    gcs_file_saved_with_increment = True
+                    log_warning(f"File {file_name} already exists in bucket {bucket_name}. Writing with increment: {increment_if_exists}")
+            else:
+                blob = bucket.blob(file_name)
+
+                # Check if the file exists
+                if blob.exists():
+                    gcs_file_already_exists = True
+                    gcs_path = f"gs://{bucket_name}/{file_name}"
+                    log_message(f"File {file_name} already exists in bucket {bucket_name}. Overwriting: {overwrite_if_exists}")
+                    if not overwrite_if_exists:
+                        log_warning(f"File {file_name} already exists and overwrite is set to False. Skipping save to GCS.")
+                        break
+                    else:
+                        gcs_file_overwritten = True
+
             blob.upload_from_string(data_str, content_type='application/json')
-            cloud_path = f"{bucket_name}/{file_name}"
-            log_message(f"Successfully saved file to GCS {cloud_path}.")
+            gcs_path = f"gs://{bucket_name}/{file_name}"
+            log_message(f"Successfully saved file to GCS {gcs_path}.")
             success = True
         except Exception as e:
             gcs_upload_exception = e
@@ -186,26 +206,46 @@ def write_data_to_gcs(bucket_name, storage_client, data, file_name=None,
             if attempts < max_retries:
                 time.sleep(2 ** attempts)
             else:
-                log_error(f"Failed to write {file_name} to GCS bucket {bucket_name} after {max_retries} attempts :{e}")
+                log_error(f"Failed to write {file_name} to GCS bucket {bucket_name} after {max_retries} attempts: {e}")
 
-    if not success and (save_locally or local_path):
+    if not success or save_locally or local_path:
         try:
             if not local_path:
                 local_path_final = os.path.join("/tmp", file_name)
             else:
                 local_path_final = os.path.join(local_path, file_name)
-            with open(local_path_final, 'w', encoding='utf-8') as f:
-                f.write(data_str)
-            log_message(f"Saved {file_name} locally at {local_path_final}.")
+            
+            if os.path.exists(local_path_final):
+                if increment_if_exists:
+                    increment = 0
+                    while os.path.exists(local_path_final):
+                        increment += 1
+                        local_path_final = os.path.join(local_path, f"{base_file_name}_{increment}{ext}")
+                    gcs_file_saved_with_increment = True
+                elif not overwrite_if_exists:
+                    log_message(f"File {file_name} already exists locally at {local_path_final} and overwrite is set to False. Skipping save.")
+                    success = True
+                else:
+                    log_message(f"File {file_name} already exists locally at {local_path_final}. Overwriting: {overwrite_if_exists}")
+
+            if not success:
+                with open(local_path_final, 'w', encoding='utf-8') as f:
+                    f.write(data_str)
+                log_message(f"Saved {file_name} locally at {local_path_final}. Overwritten: {overwrite_if_exists}")
+                success = True
         except Exception as local_e:
-            log_error(f"Failed to write {file_name} locally: {local_e}",exc_info=True)
-
-        # If GCS upload failed, raise a single exception here
+            log_error(f"Failed to write {file_name} locally: {local_e}", exc_info=True)
 
     if gcs_upload_exception is not None:
-        raise gcs_upload_exception # Propagate without nesting
-
-    return cloud_path, local_path_final
+        raise gcs_upload_exception  # Propagate without nesting
+
+    return {
+        "gcs_path": gcs_path,
+        "local_path": local_path_final,
+        "gcs_file_already_exists": gcs_file_already_exists,
+        "gcs_file_overwritten": gcs_file_overwritten,
+        "gcs_file_saved_with_increment": gcs_file_saved_with_increment
+    }
 
 
 def write_csv_to_gcs(bucket_name, file_name, data, storage_client, logger,log_info_verbose=True):

ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -5,8 +5,8 @@
 
 import datetime
 from google.cloud import bigquery
-from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeLevel
-from ipulse_shared_core_ftredge.utils_common import Notice
+from ipulse_shared_core_ftredge.enums.enums_common_utils import LogLevel
+from ipulse_shared_core_ftredge.utils_common import SmartLog
 
 
 def create_bigquery_schema_from_json(json_schema):
@@ -19,55 +19,58 @@ def create_bigquery_schema_from_json(json_schema):
     return schema
 
 
-def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_max_length=True):
-
+def check_format_against_schema_template(data_to_check, schema, dt_ts_to_str=True, check_max_length=True):
     """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
-    valid_updates = {}
-    notices=[] ### THIS IS TO AVOID LOGGING A WARNING RANDOMLY, INSTEAD GROUPPING FOR A GIVEN RUN
-
-    # Process updates to conform to the schema
-    for field in schema:
-        field_name = field["name"]
-        field_type = field["type"]
-        mode = field["mode"]
-
-        # Initialize notice to None at the start of each field processing
-        notice = None
-
-        if field_name in updates:
-            value = updates[field_name]
-
-            # Handle date and timestamp formatting
-
-            # Validate and potentially convert date and timestamp fields
-            if field_type == "DATE":
-                value, notice = handle_date_fields(field_name, value, dt_ts_to_str)
-            elif field_type == "TIMESTAMP":
-                value, notice = handle_timestamp_fields(field_name, value, dt_ts_to_str)
-            elif field_type in ["STRING", "INT64", "FLOAT64", "BOOL"]:
-                value, notice = handle_type_conversion(field_type, field_name, value )
-
-            if notice:
-                notices.append(notice)
-
-            # Check and handle max length restriction
-            if check_max_length and "max_length" in field:
-                value,notice = check_and_truncate_length(field_name, value, field["max_length"])
-                if notice:
-                    notices.append(notice)
-
-            # Only add to the dictionary if value is not None or the field is required
-            if value is not None or mode == "REQUIRED":
-                valid_updates[field_name] = value
-
-        elif mode == "REQUIRED":
-            notice=Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
-                             subject=field_name,
-                             description=f"Required field '{field_name}' is missing in the updates.")
+    checked_data = {}
+    warnings_or_error = []  # Group warnings and errors for a given run
+
+    try:
+        # Process updates to conform to the schema
+        for field in schema:
+            field_name = field["name"]
+            field_type = field["type"]
+            mode = field["mode"]
+
+            # Initialize notice to None at the start of each field processing
+            warning = None
+
+            if field_name in data_to_check:
+                value = data_to_check[field_name]
+
+                # Handle date and timestamp formatting
+                if field_type == "DATE":
+                    value, warning = handle_date_fields(field_name, value, dt_ts_to_str)
+                elif field_type == "TIMESTAMP":
+                    value, warning = handle_timestamp_fields(field_name, value, dt_ts_to_str)
+                elif field_type in ["STRING", "INT64", "FLOAT64", "BOOL"]:
+                    value, warning = handle_type_conversion(field_type, field_name, value)
+
+                if warning:
+                    warnings_or_error.append(warning)
+
+                # Check and handle max length restriction
+                if check_max_length and "max_length" in field:
+                    value, warning = check_and_truncate_length(field_name, value, field["max_length"])
+                    if warning:
+                        warnings_or_error.append(warning)
+
+                # Only add to the dictionary if value is not None or the field is required
+                if value is not None or mode == "REQUIRED":
+                    checked_data[field_name] = value
+
+            elif mode == "REQUIRED":
+                warning = SmartLog(level=LogLevel.WARNING,
+                                              subject=field_name,
+                                              description=f"Required field '{field_name}' is missing in the updates.")
+                warnings_or_error.append(warning)
 
-            notices.append(notice)
+    except Exception as e:
+        error_log = SmartLog(level=LogLevel.ERROR_EXCEPTION_REDO,
+                               subject=data_to_check,
+                               description=f"An error occurred during update check: {str(e)}")
+        warnings_or_error.append(error_log)
 
-    return valid_updates, notices
+    return checked_data, warnings_or_error
 
 def handle_date_fields(field_name, value, dt_ts_to_str):
     """Handles date fields, ensuring they are in the correct format and optionally converts them to string."""
@@ -82,11 +85,11 @@ def handle_date_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_date, None
         except ValueError:
-            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
+            return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
                                                    description=f"Expected a DATE in YYYY-MM-DD format but got {value}.")
     else:
-        return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
+        return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                               subject=field_name,
                                               description= f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
 
@@ -104,11 +107,11 @@ def handle_timestamp_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_datetime, None
         except ValueError:
-            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
+            return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
                                                   description= f"Expected ISO format TIMESTAMP but got {value}.")
     else:
-        return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
+        return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                               subject=field_name,
                                               description= f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
 
@@ -116,7 +119,7 @@ def handle_timestamp_fields(field_name, value, dt_ts_to_str):
 def check_and_truncate_length(field_name, value, max_length):
     """Checks and truncates the length of string fields if they exceed the max length."""
     if isinstance(value, str) and len(value) > max_length:
-        return value[:max_length], Notice(level=NoticeLevel.WARNING_FIX_RECOMMENDED,
+        return value[:max_length], SmartLog(level=LogLevel.WARNING_FIX_RECOMMENDED,
                             subject= field_name,
                              description= f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
 
@@ -126,7 +129,7 @@ def check_and_truncate_length(field_name, value, max_length):
 
 def handle_type_conversion(field_type, field_name, value):
     if field_type == "STRING" and not isinstance(value, str):
-        return str(value), Notice(level=NoticeLevel.WARNING_REVIEW_RECOMMENDED,
+        return str(value), SmartLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED,
                              subject=field_name,
                              description= f"Expected STRING but got {value} of type {type(value).__name__}.")
 
@@ -134,18 +137,18 @@ def handle_type_conversion(field_type, field_name, value):
         try:
             return int(value), None
         except ValueError:
-            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
+            return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                                 subject= field_name,
                                                 description=f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
     if field_type == "FLOAT64" and not isinstance(value, float):
         try:
             return float(value), None
         except ValueError:
-            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED, 
+            return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED, 
                                                 subject=field_name,
                                                 description=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
     if field_type == "BOOL" and not isinstance(value, bool):
-        return bool(value), Notice(level=NoticeLevel.WARNING_REVIEW_RECOMMENDED, 
+        return bool(value), SmartLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED, 
                                                 subject=field_name,
                                                 description=f"Expected BOOL, but got  {value}. Converting as {bool(value)}.")
 

{ipulse_shared_core_ftredge-2.51.dist-info → ipulse_shared_core_ftredge-2.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.51
+Version: 2.53
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.51.dist-info → ipulse_shared_core_ftredge-2.53.dist-info}/RECORD

@@ -1,9 +1,9 @@
-ipulse_shared_core_ftredge/__init__.py,sha256=nE8rDjbqHYVCMp3MNFO0gQQGVUuEcglvNdkghY7Tb_0,879
-ipulse_shared_core_ftredge/utils_common.py,sha256=QZqM8c7ka0uQRhv0rsV620-lgk8BDffz_pHS3qExRoE,15569
-ipulse_shared_core_ftredge/utils_gcp.py,sha256=U0UoAqdhkYv6RCTjxC98qRW5922BjHV0h88PEOBGYaE,9314
-ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=OriQHxM4AU6T3kGwwhjRdMt3ZYGmMJe0B5PLcHyzgXk,7084
-ipulse_shared_core_ftredge/enums/__init__.py,sha256=Pg8LUhBb7PJAHULoM13TrFEzG9wgCmw-ZuOdN3Rw6Og,853
-ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=oW0zhmJZfeYycVXWDCede1_Vaa0Q-KClp_KOK4kzIj8,5261
+ipulse_shared_core_ftredge/__init__.py,sha256=CcHx8XkC7YJ5pOxsOpZJrTuxweN1ya1WlQJZjOTwrBY,868
+ipulse_shared_core_ftredge/utils_common.py,sha256=GEo4Xilh9quDdUh_ppOVO6G7ustHWkSaxuILKC_FLNo,27406
+ipulse_shared_core_ftredge/utils_gcp.py,sha256=8KgsOPkLe1-1i3M_UX5niKg_CjjiNoUhZXiWFIHJdmY,11286
+ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=CHrFbhRVrXlqDzGdPe9nujn5uFQtIN2xW7RBTiHYFBc,7475
+ipulse_shared_core_ftredge/enums/__init__.py,sha256=PT8Ig7hcx_hhVlsfun24H0pFjbdfQb201ZtJplQ9uAE,844
+ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=CB0IMW5aer-n50G3AM6Fz-NrN85mJkvZhSrnuUb7EMs,5702
 ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=2i6Qo6Yi_j_O9xxnOD6QA-r0Cv7mWAUaKUx907XMRio,1825
 ipulse_shared_core_ftredge/enums/enums_module_fincore.py,sha256=MuqQg249clrWUOBb1S-iPsoOldN2_F3ohRQizbjhwG0,1374
 ipulse_shared_core_ftredge/enums/enums_modules.py,sha256=AyXUoNmR75DZLaEHi3snV6LngR25LeZRqzrLDaAupbY,1244
@@ -18,8 +18,8 @@ ipulse_shared_core_ftredge/models/user_profile_update.py,sha256=oKK0XsQDKkgDvjFP
 ipulse_shared_core_ftredge/models/user_status.py,sha256=8TyRd8tBK9_xb0MPKbI5pn9-lX7ovKbeiuWYYPtIOiw,3202
 ipulse_shared_core_ftredge/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ipulse_shared_core_ftredge/tests/test.py,sha256=0lS8HP5Quo_BqNoscU40qOH9aJRaa1Pfam5VUBmdld8,682
-ipulse_shared_core_ftredge-2.51.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
-ipulse_shared_core_ftredge-2.51.dist-info/METADATA,sha256=i2-lZRb1HCejuLEHmxDekWk_WWJG2AW8mpnad78fjqc,561
-ipulse_shared_core_ftredge-2.51.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-ipulse_shared_core_ftredge-2.51.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
-ipulse_shared_core_ftredge-2.51.dist-info/RECORD,,
+ipulse_shared_core_ftredge-2.53.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
+ipulse_shared_core_ftredge-2.53.dist-info/METADATA,sha256=9JixJKcqPsiCzRQR6ZpOiKwDIOcjEHFY6OG-VILJ_zg,561
+ipulse_shared_core_ftredge-2.53.dist-info/WHEEL,sha256=rWxmBtp7hEUqVLOnTaDOPpR-cZpCDkzhhcBce-Zyd5k,91
+ipulse_shared_core_ftredge-2.53.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
+ipulse_shared_core_ftredge-2.53.dist-info/RECORD,,

{ipulse_shared_core_ftredge-2.51.dist-info → ipulse_shared_core_ftredge-2.53.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.3.0)
+Generator: setuptools (71.0.4)
 Root-Is-Purelib: true
 Tag: py3-none-any