ipulse-shared-core-ftredge 2.52__py3-none-any.whl → 2.54__py3-none-any.whl

ipulse_shared_core_ftredge/__init__.py

@@ -4,10 +4,10 @@ from .utils_gcp import (setup_gcp_logger_and_error_report,
                         read_csv_from_gcs, read_json_from_gcs, 
                         write_csv_to_gcs, write_json_to_gcs)
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
-                                          update_check_with_schema_template)
-from .utils_common import (Notice,  NoticesManager)
+                                          check_format_against_schema_template)
+from .utils_common import (ContextLog,  PipelineWatcher)
 
-from .enums import (NoticeManagerCategory, NoticeLevel, Unit, Frequency,
+from .enums import (TargetLogs, LogLevel, Unit, Frequency,
                     Module, SubModule, BaseDataCategory,
                     FinCoreCategory, FincCoreSubCategory,
                     FinCoreRecordsCategory, ExchangeOrPublisher,

ipulse_shared_core_ftredge/enums/__init__.py

@@ -3,8 +3,8 @@
 # pylint: disable=missing-function-docstring
 # pylint: disable=missing-class-docstring
 
-from .enums_common_utils import (NoticeLevel,
-                                NoticeManagerCategory,
+from .enums_common_utils import (LogLevel,
+                                TargetLogs,
                                 Unit,
                                 Frequency)
 

ipulse_shared_core_ftredge/enums/enums_common_utils.py

@@ -6,11 +6,17 @@
 from enum import Enum
 
 
-class NoticeManagerCategory(Enum):
-    NOTICES = "notices"
-    WARN_ERRS = "warn_errs"
-    SUCCESSES = "successes"
-class NoticeLevel(Enum):
+class TargetLogs(Enum):
+    MIXED="mixed_logs"
+    SUCCESSES = "success_logs"
+    NOTICES = "notice_logs"
+    SUCCESSES_AND_NOTICES = "succs_n_notc_logs"
+    WARNINGS = "warning_logs"
+    WARNINGS_AND_ERRORS = "warn_n_err_logs"
+    ERRORS = "error_logs"
+
+
+class LogLevel(Enum):
     """
     Standardized notice levels for data engineering pipelines,
     designed for easy analysis and identification of manual 
@@ -19,12 +25,16 @@ class NoticeLevel(Enum):
     DEBUG = 100  # Detailed debug information (for development/troubleshooting)
 
     INFO = 200
+    SUCCESS = 201
 
-    SUCCESS = 300  # Events requiring attention, but not necessarily errors
+    NOTICE = 300  # Maybe same file or data already fully or partially exists
+    NOTICE_ALREADY_EXISTS = 301 # Data already exists, no action required
+    NOTICE_PARTIAL_EXISTS = 302 # Partial data exists, no action required
+    NOTICE_CANCELLED = 303 # Data processing cancelled, no action required
 
      # Warnings indicate potential issues that might require attention:
     WARNING = 400 # General warning, no immediate action required
-    WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
+    # WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
     WARNING_REVIEW_RECOMMENDED = 402 # Action recommended to prevent potential future issues
     WARNING_FIX_RECOMMENDED = 403 # Action recommended to prevent potential future issues
     WARNING_FIX_REQUIRED = 404  # Action required, pipeline can likely continue
@@ -46,16 +56,16 @@ class NoticeLevel(Enum):
     UNKNOWN=1001 # Unknown error, should not be used in normal operation
 
 
-class NoticeStatus(Enum):
+class LogStatus(Enum):
     OPEN = "open"
     ACKNOWLEDGED = "acknowledged"
     IN_PROGRESS = "in_progress"
     RESOLVED = "resolved"
     IGNORED = "ignored"
+    CANCELLED = "cancelled"
   
 
 
-
 ### Exception during full exection, partially saved
 # Exception during ensemble pipeline; modifications collected in local object , nothing persisted
 # Exception during ensemble pipeline; modifications persisted , metadata failed
@@ -88,17 +98,17 @@ class Unit(Enum):
     CURRENCY = "currency"    # General currency, when specific currency is not needed
 
     # Stock Market and Investments
-    SHARES = "shars"        # Number of shares
+    SHARES = "shares"        # Number of shares
     PERCENT = "prcnt"      # Percentage, used for rates and ratios
     BPS = "bps"              # Basis points, often used for interest rates and financial ratios
 
     # Volume and Quantitative Measurements
-    VOLUME = "vol"        # Trading volume in units
+    VOLUME = "volume"        # Trading volume in units
     MILLIONS = "mills"    # Millions, used for large quantities or sums
     BILLIONS = "bills"    # Billions, used for very large quantities or sums
 
     # Commodity Specific Units
-    BARRELS = "barrls"      # Barrels, specifically for oil and similar liquids
+    BARRELS = "barrels"      # Barrels, specifically for oil and similar liquids
     TONNES = "tonnes"        # Tonnes, for bulk materials like metals or grains
     TROY_OUNCES = "troy_oz" # Troy ounces, specifically for precious metals
 

ipulse_shared_core_ftredge/enums/enums_data_eng.py

@@ -4,11 +4,12 @@
 from enum import Enum
 
 class SourcingTriggerType(Enum):
-    HISTORIC_MANUAL = "historic_manual"
-    LIVE_SCHEDULED = "live_scheduled"
+    BULK_MANUAL = "bulk_manual"
+    BULK_SCHEDULED = "bulk_scheduled" # almost always historic bulk is manual
+    RECENT_SCHEDULED = "recent_scheduled"
+    RECENT_MANUAL = "recent_manual"
     ADHOC_MANUAL = "adhoc_manual"
     ADHOC_SCHEDULED = "adhoc_scheduled"
-    LIVE_MANUAL = "live_manual"
 
 class SourcingPipelineType(Enum):
     LOCAL_GET_API_TO_GCS = "local_get_api_to_gcs"

ipulse_shared_core_ftredge/enums/enums_module_fincore.py

@@ -5,14 +5,14 @@ from enum import Enum
 
 
 class FinCoreCategory(Enum):
-    MARKET="market"
-    ECONOMY="economy"
-    POLITICS="poltcs"
-    CORPORATE="corp"
+    MARKET="market" # Market prices data
+    CORPORATE="corp" # Corporate data such as financial statements and earnings, similar to fundamental data
     FUNDAMENTAL="fundam"
-    SENTIMENT="sntmnt"
+    ECONOMY="economy"
     NEWS="news"
+    SENTIMENT="sntmnt"
     SOCIAL="social"
+    POLITICS="poltcs"
     OTHER="other"
 
 class FincCoreSubCategory(Enum):
@@ -32,16 +32,16 @@ class FincCoreSubCategory(Enum):
 
 class FinCoreRecordsCategory(Enum):
     PRICE="pric"
-    PRICE_SPOT= "pric.s"
-    PRICE_OHLCVA="pric.ohlcva"
-    PRICE_OHLCV="pric.ohlcv"
-    PRICE_OPEN="pric.o"
-    PRICE_HIGH="pric.h"
-    PRICE_LOW="pric.l"
-    PRICE_CLOSE="pric.c"
-    PRICE_VOLUME="pric.v"
-    PRICE_ADJC="pric.a"
-    FUNDAMENTAL="fundam" # treat this differently 
+    SPOT= "spot"
+    OHLCVA="ohlcva"
+    OHLCV="ohlcv"
+    OPEN="open"
+    HIGH="high"
+    LOW="low"
+    CLOSE="close"
+    VOLUME="volume"
+    ADJC="adjc"
+    FUNDAMENTAL="fundam" # treat this differently
     EARNINGS="earnings"
     CASH_FLOW="cashflw"
     BALANCE_SHEET="blnce_sht"

ipulse_shared_core_ftredge/utils_common.py

@@ -10,18 +10,18 @@ from datetime import datetime, timezone
 from contextlib import contextmanager
 from typing import List
 from google.cloud import logging as cloudlogging
-from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeLevel, NoticeManagerCategory, NoticeStatus
+from ipulse_shared_core_ftredge.enums.enums_common_utils import TargetLogs, LogLevel, LogStatus
 from ipulse_shared_core_ftredge.utils_gcp import write_json_to_gcs
 
 
 # ["data_import","data_quality", "data_processing","data_general","data_persistance","metadata_quality", "metadata_processing", "metadata_persistance","metadata_general"]
 
-class Notice:
+class ContextLog:
     MAX_TRACEBACK_LINES = 14  # Define the maximum number of traceback lines to include
-    def __init__(self, level: NoticeLevel, start_context: str = None, notice_manager_id: str = None,
+    def __init__(self, level: LogLevel, base_context: str = None, collector_id: str = None,
                  e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
                  subject: str = None, description: str = None, context: str = None,
-                 notice_status: NoticeStatus = NoticeStatus.OPEN):
+                 log_status: LogStatus = LogStatus.OPEN):
         if e is not None:
             e_type = type(e).__name__ if e_type is None else e_type
             e_message = str(e) if e_message is None else e_message
@@ -32,13 +32,13 @@ class Notice:
         self.level = level
         self.subject = subject
         self.description = description
-        self._start_context = start_context
+        self._base_context = base_context
         self._context = context
-        self.notice_manager_id = notice_manager_id
+        self.collector_id = collector_id
         self.exception_type = e_type
         self.exception_message = e_message
         self.exception_traceback = self._format_traceback(e_traceback,e_message)
-        self.notice_status = notice_status
+        self.log_status = log_status
         self.timestamp = datetime.now(timezone.utc).isoformat()
 
     def _format_traceback(self, e_traceback, e_message):
@@ -83,15 +83,15 @@ class Notice:
         else:
             formatted_traceback = '\n'.join(combined_lines)
 
-        return formatted_traceback 
+        return formatted_traceback
 
     @property
-    def start_context(self):
-        return self._start_context
+    def base_context(self):
+        return self._base_context
 
-    @start_context.setter
-    def start_context(self, value):
-        self._start_context = value
+    @base_context.setter
+    def base_context(self, value):
+        self._base_context = value
 
     @property
     def context(self):
@@ -103,7 +103,7 @@ class Notice:
 
     def to_dict(self):
         return {
-            "start_context": self.start_context,
+            "base_context": self.base_context,
             "context": self.context,
             "level_code": self.level.value,
             "level_name": self.level.name,
@@ -112,27 +112,29 @@ class Notice:
             "exception_type": self.exception_type,
             "exception_message": self.exception_message,
             "exception_traceback": self.exception_traceback,
-            "notice_status": self.notice_status.value,
-            "notice_manager_id": self.notice_manager_id,
+            "log_status": self.log_status.value,
+            "collector_id": self.collector_id,
             "timestamp": self.timestamp
         }
 
-class NoticesManager:
-    ERROR_CODE_START_VALUE = NoticeLevel.ERROR.value
-    WARNING_CODE_START_VALUE = NoticeLevel.WARNING.value
-    SUCCESS_CODE_START_VALUE = NoticeLevel.SUCCESS.value
+class PipelineWatcher:
+    ERROR_START_CODE = LogLevel.ERROR.value
+    WARNING_START_CODE = LogLevel.WARNING.value
+    NOTICE_START_CODE = LogLevel.NOTICE.value
+    SUCCESS_START_CODE = LogLevel.SUCCESS.value
 
-    def __init__(self, start_context: str, category: NoticeManagerCategory = NoticeManagerCategory.NOTICES, logger_name=None):
+    def __init__(self, base_context: str, target_logs: TargetLogs = TargetLogs.MIXED, logger_name=None):
         self._id = str(uuid.uuid4())
-        self._notices = []
+        self._logs = []
         self._early_stop = False
         self._errors_count = 0
         self._warnings_count = 0
+        self._notices_count = 0
         self._successes_count = 0
-        self._level_counts = {level.name: 0 for level in NoticeLevel}
-        self._start_context = start_context
+        self._level_counts = {level.name: 0 for level in LogLevel}
+        self._base_context = base_context
         self._context_stack = []
-        self._category = category.value
+        self._target_logs = target_logs.value
         self._logger = self._initialize_logger(logger_name)
 
     def _initialize_logger(self, logger_name):
@@ -141,7 +143,6 @@ class NoticesManager:
             return logging_client.logger(logger_name)
         return None
 
-
     @contextmanager
     def context(self, context):
         self.push_context(context)
@@ -162,8 +163,8 @@ class NoticesManager:
         return " >> ".join(self._context_stack)
 
     @property
-    def start_context(self):
-        return self._start_context
+    def base_context(self):
+        return self._base_context
 
     @property
     def id(self):
@@ -173,14 +174,14 @@ class NoticesManager:
     def early_stop(self):
         return self._early_stop
 
-    def set_early_stop(self, max_errors_tolerance:int, create_error_notice=True,pop_context=False):
-        self.early_stop = True
-        if create_error_notice:
+    def set_early_stop(self, max_errors_tolerance: int, create_error_log=True, pop_context=False):
+        self._early_stop = True
+        if create_error_log:
             if pop_context:
                 self.pop_context()
-            self.add_notice(Notice(level=NoticeLevel.ERROR,
-                    subject="EARLY_STOP",
-                    description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
+            self.add_log(ContextLog(level=LogLevel.ERROR,
+                                    subject="EARLY_STOP",
+                                    description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
 
     def reset_early_stop(self):
         self._early_stop = False
@@ -188,49 +189,54 @@ class NoticesManager:
     def get_early_stop(self):
         return self._early_stop
 
-    def add_notice(self, notice: Notice):
-        if (self._category == NoticeManagerCategory.SUCCESSES.value and notice.level != NoticeLevel.SUCCESS) or \
-           (self._category == NoticeManagerCategory.WARN_ERRS.value and notice.level.value < self.WARNING_CODE_START_VALUE):
-            raise ValueError(f"Invalid notice level {notice.level.name} for category {self._category}")
-        notice.start_context = self.start_context
-        notice.context = self.current_context
-        notice.notice_manager_id = self.id
-        notice_dict = notice.to_dict()
-        self._notices.append(notice_dict)
-        self._update_counts(notice_dict)
+    def add_log(self, log: ContextLog):
+        if (self._target_logs == TargetLogs.SUCCESSES and log.level >=self.NOTICE_START_CODE) or \
+           (self._target_logs == TargetLogs.WARNINGS_AND_ERRORS and log.level.value < self.WARNING_START_CODE):
+            raise ValueError(f"Invalid log level {log.level.name} for Pipeline Watcher target logs setup: {self._target_logs}")
+        log.base_context = self.base_context
+        log.context = self.current_context
+        log.collector_id = self.id
+        log_dict = log.to_dict()
+        self._logs.append(log_dict)
+        self._update_counts(log_dict)
 
         if self._logger:
-            if notice.level.value >= self.WARNING_CODE_START_VALUE:
-                self._logger.log_struct(notice_dict, severity="WARNING")
+            # We specifically want to avoid having an ERROR log level for this structured Pipeline Watcher reporting, to ensure Errors are alerting on Critical Application Services.
+            # A single ERROR log level can be used for the entire pipeline, which shall be used at the end of the pipeline
+            if log.level.value >= self.WARNING_START_CODE:
+                self._logger.log_struct(log_dict, severity="WARNING")
+            elif log.level.value >= self.NOTICE_START_CODE:
+                self._logger.log_struct(log_dict, severity="NOTICE")
             else:
-                self._logger.log_struct(notice_dict, severity="INFO")
+                self._logger.log_struct(log_dict, severity="INFO")
 
-    def add_notices(self, notices: List[Notice]):
-        for notice in notices:
-            self.add_notice(notice)
+    def add_logs(self, logs: List[ContextLog]):
+        for log in logs:
+            self.add_log(log)
 
-    def clear_notices_and_counts(self):
-        self._notices = []
+    def clear_logs_and_counts(self):
+        self._logs = []
         self._errors_count = 0
         self._warnings_count = 0
+        self._notices_count = 0
         self._successes_count = 0
-        self._level_counts = {level.name: 0 for level in NoticeLevel}
+        self._level_counts = {level.name: 0 for level in LogLevel}
 
-    def clear_notices(self):
-        self._notices = []
+    def clear_logs(self):
+        self._logs = []
 
-    def get_all_notices(self):
-        return self._notices
+    def get_all_logs(self):
+        return self._logs
 
-    def get_notices_for_level(self, level: NoticeLevel):
-        return [notice for notice in self._notices if notice["level_code"] == level.value]
+    def get_logs_for_level(self, level: LogLevel):
+        return [log for log in self._logs if log["level_code"] == level.value]
 
-    def get_notices_by_str_in_context(self, context_substring: str):
+    def get_logs_by_str_in_context(self, context_substring: str):
         return [
-            notice for notice in self._notices
-            if context_substring in notice["context"]
+            log for log in self._logs
+            if context_substring in log["context"]
         ]
-    
+
     def contains_errors(self):
         return self._errors_count > 0
 
@@ -246,60 +252,69 @@ class NoticesManager:
     def count_warnings(self):
         return self._warnings_count
 
+    def count_notices(self):
+        return self._notices_count
+
     def count_successes(self):
         return self._successes_count
 
-    def count_all_notices(self):
-        return len(self._notices)
+    def count_all_logs(self):
+        return len(self._logs)
 
-    def count_notices_by_level(self, level: NoticeLevel):
+    def count_logs_by_level(self, level: LogLevel):
         return self._level_counts.get(level.name, 0)
 
-    def _count_notices(self, context_substring: str, exact_match=False, level_code_min=None, level_code_max=None):
+    def _count_logs(self, context_substring: str, exact_match=False, level_code_min=None, level_code_max=None):
         return sum(
-            1 for notice in self._notices
-            if (notice["context"] == context_substring if exact_match else context_substring in notice["context"]) and
-               (level_code_min is None or notice["level_code"] >= level_code_min) and
-               (level_code_max is None or notice["level_code"] <= level_code_max)
+            1 for log in self._logs
+            if (log["context"] == context_substring if exact_match else context_substring in log["context"]) and
+               (level_code_min is None or log["level_code"] >= level_code_min) and
+               (level_code_max is None or log["level_code"] <= level_code_max)
         )
 
-    def count_notices_for_current_context(self):
-        return self._count_notices(self.current_context, exact_match=True)
+    def count_logs_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True)
 
-    def count_notices_for_current_and_nested_contexts(self):
-        return self._count_notices(self.current_context)
+    def count_logs_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context)
 
-    def count_notices_by_level_for_current_context(self, level: NoticeLevel):
-        return self._count_notices(self.current_context, exact_match=True, level_code_min=level.value, level_code_max=level.value)
+    def count_logs_by_level_for_current_context(self, level: LogLevel):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=level.value, level_code_max=level.value)
 
-    def count_notices_by_level_for_current_and_nested_contexts(self, level: NoticeLevel):
-        return self._count_notices(self.current_context, level_code_min=level.value, level_code_max=level.value)
+    def count_logs_by_level_for_current_and_nested_contexts(self, level: LogLevel):
+        return self._count_logs(self.current_context, level_code_min=level.value, level_code_max=level.value)
 
     def count_errors_for_current_context(self):
-        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.ERROR_CODE_START_VALUE)
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.ERROR_START_CODE)
 
     def count_errors_for_current_and_nested_contexts(self):
-        return self._count_notices(self.current_context, level_code_min=self.ERROR_CODE_START_VALUE)
+        return self._count_logs(self.current_context, level_code_min=self.ERROR_START_CODE)
 
     def count_warnings_and_errors_for_current_context(self):
-        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE)
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE)
 
     def count_warnings_and_errors_for_current_and_nested_contexts(self):
-        return self._count_notices(self.current_context, level_code_min=self.WARNING_CODE_START_VALUE)
+        return self._count_logs(self.current_context, level_code_min=self.WARNING_START_CODE)
 
     def count_warnings_for_current_context(self):
-        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
 
     def count_warnings_for_current_and_nested_contexts(self):
-        return self._count_notices(self.current_context, level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
+        return self._count_logs(self.current_context, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
+
+    def count_notices_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.NOTICE_START_CODE, level_code_max=self.WARNING_START_CODE-1)
+
+    def count_notices_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.NOTICE_START_CODE, level_code_max=self.WARNING_START_CODE-1)
 
     def count_successes_for_current_context(self):
-        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.NOTICE_START_CODE-1)
 
     def count_successes_for_current_and_nested_contexts(self):
-        return self._count_notices(self.current_context, level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
+        return self._count_logs(self.current_context, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.NOTICE_START_CODE-1)
 
-    def export_notices_to_gcs_file(self, bucket_name, storage_client, file_prefix=None, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
+    def export_logs_to_gcs_file(self, bucket_name, storage_client, file_prefix=None, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
         def log_message(message):
             if logger:
                 logger.info(message)
@@ -309,34 +324,34 @@ class NoticesManager:
                 logger.error(message, exc_info=exc_info)
 
         if not file_prefix:
-            file_prefix = self._category
+            file_prefix = self._target_logs
         if not file_name:
             timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
             if top_level_context:
-                file_name = f"{file_prefix}_{timestamp}_{top_level_context}_len{len(self._notices)}.json"
+                file_name = f"{file_prefix}_{timestamp}_{top_level_context}_len{len(self._logs)}.json"
             else:
-                file_name = f"{file_prefix}_{timestamp}_len{len(self._notices)}.json"
+                file_name = f"{file_prefix}_{timestamp}_len{len(self._logs)}.json"
 
-        result=None
+        result = None
         try:
-            result= write_json_to_gcs(
+            result = write_json_to_gcs(
                 bucket_name=bucket_name,
                 storage_client=storage_client,
-                data=self._notices,
+                data=self._logs,
                 file_name=file_name,
                 save_locally=save_locally,
                 local_path=local_path,
                 logger=logger,
                 max_retries=max_retries,
-                overwrite=True
+                overwrite_if_exists=False
             )
-            log_message(f"{file_prefix} successfully saved (ovewritten={result.get("ovewritten")}) to GCS at {result.get("gcs_path")} and locally at {result.get("local_path")}.")
+            log_message(f"{file_prefix} successfully saved (overwritten={result.get('gcs_file_overwritten')}) to GCS at {result.get('gcs_path')} and locally at {result.get('local_path')}.")
         except Exception as e:
-            log_error(f"Failed at export_notices_to_gcs_file for {file_prefix} for file {file_name} to bucket {bucket_name}: {type(e).__name__} - {str(e)}")
+            log_error(f"Failed at export_logs_to_gcs_file for {file_prefix} for file {file_name} to bucket {bucket_name}: {type(e).__name__} - {str(e)}")
 
         return result
 
-    def import_notices_from_json(self, json_or_file, logger=None):
+    def import_logs_from_json(self, json_or_file, logger=None):
         def log_message(message):
             if logger:
                 logger.info(message)
@@ -347,31 +362,35 @@ class NoticesManager:
 
         try:
             if isinstance(json_or_file, str):  # Load from string
-                imported_notices = json.loads(json_or_file)
+                imported_logs = json.loads(json_or_file)
             elif hasattr(json_or_file, 'read'):  # Load from file-like object
-                imported_notices = json.load(json_or_file)
-            self.add_notices(imported_notices)
-            log_message("Successfully imported notices from json.")
+                imported_logs = json.load(json_or_file)
+            self.add_logs(imported_logs)
+            log_message("Successfully imported logs from json.")
         except Exception as e:
-            log_warning(f"Failed to import notices from json: {type(e).__name__} - {str(e)}", exc_info=True)
+            log_warning(f"Failed to import logs from json: {type(e).__name__} - {str(e)}", exc_info=True)
 
-    def _update_counts(self, notice, remove=False):
-        level_code = notice["level_code"]
-        level_name = notice["level_name"]
+    def _update_counts(self, log, remove=False):
+        level_code = log["level_code"]
+        level_name = log["level_name"]
 
         if remove:
-            if level_code >= self.ERROR_CODE_START_VALUE:
+            if level_code >= self.ERROR_START_CODE:
                 self._errors_count -= 1
-            elif level_code >= self.WARNING_CODE_START_VALUE:
+            elif self.WARNING_START_CODE <= level_code < self.ERROR_START_CODE:
                 self._warnings_count -= 1
-            elif level_code >= self.SUCCESS_CODE_START_VALUE:
+            elif self.NOTICE_START_CODE <= level_code < self.WARNING_START_CODE:
+                self._notices_count -= 1
+            elif self.SUCCESS_START_CODE <= level_code < self.NOTICE_START_CODE:
                 self._successes_count -= 1
             self._level_counts[level_name] -= 1
         else:
-            if level_code >= self.ERROR_CODE_START_VALUE:
+            if level_code >= self.ERROR_START_CODE:
                 self._errors_count += 1
-            elif level_code >= self.WARNING_CODE_START_VALUE:
+            elif self.WARNING_START_CODE <= level_code < self.ERROR_START_CODE:
                 self._warnings_count += 1
-            elif level_code == self.SUCCESS_CODE_START_VALUE:
+            elif self.NOTICE_START_CODE <= level_code < self.WARNING_START_CODE:
+                self._notices_count += 1
+            elif self.SUCCESS_START_CODE <= level_code < self.NOTICE_START_CODE:
                 self._successes_count += 1
             self._level_counts[level_name] += 1

ipulse_shared_core_ftredge/utils_gcp.py

@@ -122,8 +122,9 @@ def read_csv_from_gcs(bucket_name, file_name, storage_client, logger):
 
 
 
-def write_json_to_gcs(bucket_name, storage_client, data, file_name=None, 
-                      save_locally=False, local_path=None, logger=None, max_retries=3, overwrite=True):
+def write_json_to_gcs(bucket_name, storage_client, data, file_name,
+                      save_locally=False, local_path=None, logger=None, max_retries=2,
+                      overwrite_if_exists=False, increment_if_exists=False):
     """Saves data to Google Cloud Storage and optionally locally.
 
     This function attempts to upload data to GCS. If the upload fails after
@@ -132,24 +133,35 @@ def write_json_to_gcs(bucket_name, storage_client, data, file_name=None,
 
     Returns:
         dict: A dictionary containing the GCS path (or None if upload failed),
-              the local path (or None if not saved locally), and a boolean indicating if the file was overwritten.
+              the local path (or None if not saved locally), a boolean indicating if the file was overwritten,
+              a boolean indicating if the file already existed, and a boolean indicating if the file was saved with an incremented name.
     """
 
     def log_message(message):
         if logger:
             logger.info(message)
 
-    def log_error(message,exc_info=False):
+    def log_error(message, exc_info=False):
         if logger:
             logger.error(message, exc_info=exc_info)
 
+    def log_warning(message):
+        if logger:
+            logger.warning(message)
+
     attempts = 0
     success = False
     gcs_path = None
     local_path_final = None
-    overwritten = False
+    gcs_file_overwritten = False
+    gcs_file_already_exists = False
+    gcs_file_saved_with_increment = False
     gcs_upload_exception = None  # Store potential GCS exception
 
+    # Check for conflicting options
+    if overwrite_if_exists and increment_if_exists:
+        raise ValueError("When writing JSON to GCS, both overwrite and increment_if_exists cannot be True at the same time.")
+
     if isinstance(data, (list, dict)):
         data_str = json.dumps(data, indent=2)
     elif isinstance(data, str):
@@ -157,17 +169,32 @@ def write_json_to_gcs(bucket_name, storage_client, data, file_name=None,
     else:
         raise ValueError("Unsupported data type. It should be a list, dict, or str.")
 
+    bucket = storage_client.bucket(bucket_name)
+    base_file_name, ext = os.path.splitext(file_name)
+    increment = 0
+
     while attempts < max_retries and not success:
         try:
-            bucket = storage_client.bucket(bucket_name)
-            blob = bucket.blob(file_name)
-
-            # Check if the file exists and if we should overwrite it
-            if blob.exists():
-                if not overwrite:
-                    raise FileExistsError(f"File {file_name} already exists in bucket {bucket_name} and overwrite is set to False.")
-                else:
-                    overwritten = True
+            if increment_if_exists:
+                while bucket.blob(file_name).exists():
+                    gcs_file_already_exists = True
+                    increment += 1
+                    file_name = f"{base_file_name}_{increment}{ext}"
+                    gcs_file_saved_with_increment = True
+                    log_warning(f"File {file_name} already exists in bucket {bucket_name}. Writing with increment: {increment_if_exists}")
+            else:
+                blob = bucket.blob(file_name)
+
+                # Check if the file exists
+                if blob.exists():
+                    gcs_file_already_exists = True
+                    gcs_path = f"gs://{bucket_name}/{file_name}"
+                    log_message(f"File {file_name} already exists in bucket {bucket_name}. Overwriting: {overwrite_if_exists}")
+                    if not overwrite_if_exists:
+                        log_warning(f"File {file_name} already exists and overwrite is set to False. Skipping save to GCS.")
+                        break
+                    else:
+                        gcs_file_overwritten = True
 
             blob.upload_from_string(data_str, content_type='application/json')
             gcs_path = f"gs://{bucket_name}/{file_name}"
@@ -181,15 +208,31 @@ def write_json_to_gcs(bucket_name, storage_client, data, file_name=None,
             else:
                 log_error(f"Failed to write {file_name} to GCS bucket {bucket_name} after {max_retries} attempts: {e}")
 
-    if not success and (save_locally or local_path):
+    if not success or save_locally or local_path:
         try:
             if not local_path:
                 local_path_final = os.path.join("/tmp", file_name)
             else:
                 local_path_final = os.path.join(local_path, file_name)
-            with open(local_path_final, 'w', encoding='utf-8') as f:
-                f.write(data_str)
-            log_message(f"Saved {file_name} locally at {local_path_final}.")
+            
+            if os.path.exists(local_path_final):
+                if increment_if_exists:
+                    increment = 0
+                    while os.path.exists(local_path_final):
+                        increment += 1
+                        local_path_final = os.path.join(local_path, f"{base_file_name}_{increment}{ext}")
+                    gcs_file_saved_with_increment = True
+                elif not overwrite_if_exists:
+                    log_message(f"File {file_name} already exists locally at {local_path_final} and overwrite is set to False. Skipping save.")
+                    success = True
+                else:
+                    log_message(f"File {file_name} already exists locally at {local_path_final}. Overwriting: {overwrite_if_exists}")
+
+            if not success:
+                with open(local_path_final, 'w', encoding='utf-8') as f:
+                    f.write(data_str)
+                log_message(f"Saved {file_name} locally at {local_path_final}. Overwritten: {overwrite_if_exists}")
+                success = True
         except Exception as local_e:
             log_error(f"Failed to write {file_name} locally: {local_e}", exc_info=True)
 
@@ -199,7 +242,9 @@ def write_json_to_gcs(bucket_name, storage_client, data, file_name=None,
     return {
         "gcs_path": gcs_path,
         "local_path": local_path_final,
-        "overwritten": overwritten
+        "gcs_file_already_exists": gcs_file_already_exists,
+        "gcs_file_overwritten": gcs_file_overwritten,
+        "gcs_file_saved_with_increment": gcs_file_saved_with_increment
     }
 
 

ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -5,8 +5,8 @@
 
 import datetime
 from google.cloud import bigquery
-from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeLevel
-from ipulse_shared_core_ftredge.utils_common import Notice
+from ipulse_shared_core_ftredge.enums.enums_common_utils import LogLevel
+from ipulse_shared_core_ftredge.utils_common import ContextLog
 
 
 def create_bigquery_schema_from_json(json_schema):
@@ -19,55 +19,58 @@ def create_bigquery_schema_from_json(json_schema):
     return schema
 
 
-def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_max_length=True):
-
+def check_format_against_schema_template(data_to_check, schema, dt_ts_to_str=True, check_max_length=True):
     """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
-    valid_updates = {}
-    notices=[] ### THIS IS TO AVOID LOGGING A WARNING RANDOMLY, INSTEAD GROUPPING FOR A GIVEN RUN
-
-    # Process updates to conform to the schema
-    for field in schema:
-        field_name = field["name"]
-        field_type = field["type"]
-        mode = field["mode"]
-
-        # Initialize notice to None at the start of each field processing
-        notice = None
-
-        if field_name in updates:
-            value = updates[field_name]
-
-            # Handle date and timestamp formatting
-
-            # Validate and potentially convert date and timestamp fields
-            if field_type == "DATE":
-                value, notice = handle_date_fields(field_name, value, dt_ts_to_str)
-            elif field_type == "TIMESTAMP":
-                value, notice = handle_timestamp_fields(field_name, value, dt_ts_to_str)
-            elif field_type in ["STRING", "INT64", "FLOAT64", "BOOL"]:
-                value, notice = handle_type_conversion(field_type, field_name, value )
-
-            if notice:
-                notices.append(notice)
-
-            # Check and handle max length restriction
-            if check_max_length and "max_length" in field:
-                value,notice = check_and_truncate_length(field_name, value, field["max_length"])
-                if notice:
-                    notices.append(notice)
-
-            # Only add to the dictionary if value is not None or the field is required
-            if value is not None or mode == "REQUIRED":
-                valid_updates[field_name] = value
-
-        elif mode == "REQUIRED":
-            notice=Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
-                             subject=field_name,
-                             description=f"Required field '{field_name}' is missing in the updates.")
+    checked_data = {}
+    warnings_or_error = []  # Group warnings and errors for a given run
+
+    try:
+        # Process updates to conform to the schema
+        for field in schema:
+            field_name = field["name"]
+            field_type = field["type"]
+            mode = field["mode"]
+
+            # Initialize notice to None at the start of each field processing
+            warning = None
+
+            if field_name in data_to_check:
+                value = data_to_check[field_name]
+
+                # Handle date and timestamp formatting
+                if field_type == "DATE":
+                    value, warning = handle_date_fields(field_name, value, dt_ts_to_str)
+                elif field_type == "TIMESTAMP":
+                    value, warning = handle_timestamp_fields(field_name, value, dt_ts_to_str)
+                elif field_type in ["STRING", "INT64", "FLOAT64", "BOOL"]:
+                    value, warning = handle_type_conversion(field_type, field_name, value)
+
+                if warning:
+                    warnings_or_error.append(warning)
+
+                # Check and handle max length restriction
+                if check_max_length and "max_length" in field:
+                    value, warning = check_and_truncate_length(field_name, value, field["max_length"])
+                    if warning:
+                        warnings_or_error.append(warning)
+
+                # Only add to the dictionary if value is not None or the field is required
+                if value is not None or mode == "REQUIRED":
+                    checked_data[field_name] = value
+
+            elif mode == "REQUIRED":
+                warning = ContextLog(level=LogLevel.WARNING,
+                                              subject=field_name,
+                                              description=f"Required field '{field_name}' is missing in the updates.")
+                warnings_or_error.append(warning)
 
-            notices.append(notice)
+    except Exception as e:
+        error_log = ContextLog(level=LogLevel.ERROR_EXCEPTION_REDO,
+                               subject=data_to_check,
+                               description=f"An error occurred during update check: {str(e)}")
+        warnings_or_error.append(error_log)
 
-    return valid_updates, notices
+    return checked_data, warnings_or_error
 
 def handle_date_fields(field_name, value, dt_ts_to_str):
     """Handles date fields, ensuring they are in the correct format and optionally converts them to string."""
@@ -82,11 +85,11 @@ def handle_date_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_date, None
         except ValueError:
-            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
+            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
                                                    description=f"Expected a DATE in YYYY-MM-DD format but got {value}.")
     else:
-        return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
+        return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                               subject=field_name,
                                               description= f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
 
@@ -104,11 +107,11 @@ def handle_timestamp_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_datetime, None
         except ValueError:
-            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
+            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
                                                   description= f"Expected ISO format TIMESTAMP but got {value}.")
     else:
-        return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
+        return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                               subject=field_name,
                                               description= f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
 
@@ -116,7 +119,7 @@ def handle_timestamp_fields(field_name, value, dt_ts_to_str):
 def check_and_truncate_length(field_name, value, max_length):
     """Checks and truncates the length of string fields if they exceed the max length."""
     if isinstance(value, str) and len(value) > max_length:
-        return value[:max_length], Notice(level=NoticeLevel.WARNING_FIX_RECOMMENDED,
+        return value[:max_length], ContextLog(level=LogLevel.WARNING_FIX_RECOMMENDED,
                             subject= field_name,
                              description= f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
 
@@ -126,7 +129,7 @@ def check_and_truncate_length(field_name, value, max_length):
 
 def handle_type_conversion(field_type, field_name, value):
     if field_type == "STRING" and not isinstance(value, str):
-        return str(value), Notice(level=NoticeLevel.WARNING_REVIEW_RECOMMENDED,
+        return str(value), ContextLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED,
                              subject=field_name,
                              description= f"Expected STRING but got {value} of type {type(value).__name__}.")
 
@@ -134,18 +137,18 @@ def handle_type_conversion(field_type, field_name, value):
         try:
             return int(value), None
         except ValueError:
-            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
+            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                                 subject= field_name,
                                                 description=f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
     if field_type == "FLOAT64" and not isinstance(value, float):
         try:
             return float(value), None
         except ValueError:
-            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED, 
+            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED, 
                                                 subject=field_name,
                                                 description=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
     if field_type == "BOOL" and not isinstance(value, bool):
-        return bool(value), Notice(level=NoticeLevel.WARNING_REVIEW_RECOMMENDED, 
+        return bool(value), ContextLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED, 
                                                 subject=field_name,
                                                 description=f"Expected BOOL, but got  {value}. Converting as {bool(value)}.")
 

{ipulse_shared_core_ftredge-2.52.dist-info → ipulse_shared_core_ftredge-2.54.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.52
+Version: 2.54
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.52.dist-info → ipulse_shared_core_ftredge-2.54.dist-info}/RECORD

@@ -1,11 +1,11 @@
-ipulse_shared_core_ftredge/__init__.py,sha256=g3PRhnLbDFPVAFPIpL6tQgiEJFbPGPmeleHoxPNaQzE,879
-ipulse_shared_core_ftredge/utils_common.py,sha256=AT8R7BfEbOvMztgxQKv6d6jwGfTCJMO5FDz9idTSE4A,15564
-ipulse_shared_core_ftredge/utils_gcp.py,sha256=KZSuugt-746lfSAlNi8ks1llxxPzTskmtZwnm18SnhQ,8873
-ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=OriQHxM4AU6T3kGwwhjRdMt3ZYGmMJe0B5PLcHyzgXk,7084
-ipulse_shared_core_ftredge/enums/__init__.py,sha256=Pg8LUhBb7PJAHULoM13TrFEzG9wgCmw-ZuOdN3Rw6Og,853
-ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=oW0zhmJZfeYycVXWDCede1_Vaa0Q-KClp_KOK4kzIj8,5261
-ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=2i6Qo6Yi_j_O9xxnOD6QA-r0Cv7mWAUaKUx907XMRio,1825
-ipulse_shared_core_ftredge/enums/enums_module_fincore.py,sha256=MuqQg249clrWUOBb1S-iPsoOldN2_F3ohRQizbjhwG0,1374
+ipulse_shared_core_ftredge/__init__.py,sha256=BBOoNtI-PdKen1nCcmfBEXJ40X0Jf6b8Yeo6ytJT5rQ,873
+ipulse_shared_core_ftredge/utils_common.py,sha256=r4jYBsu6TnAINTewk72CZebWH_P_oVf5LlkeDy9-ndU,16482
+ipulse_shared_core_ftredge/utils_gcp.py,sha256=8KgsOPkLe1-1i3M_UX5niKg_CjjiNoUhZXiWFIHJdmY,11286
+ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=AwGl9J-XQc_aO_VKWhR_TuA1ML8nWxHuzHtxBH8yfwE,7499
+ipulse_shared_core_ftredge/enums/__init__.py,sha256=reCHJE0j_QTGwag7uo3cQXMTRaGRR_YOwywweb5pFb8,839
+ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=gYckmSAzhh5MA3CyV18Z9-YtHsFJlmEcKC-4nuqMXu4,5673
+ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=7w3Jjmw84Wq22Bb5Qs09Z82Bdf-j8nhRiQJfw60_g80,1903
+ipulse_shared_core_ftredge/enums/enums_module_fincore.py,sha256=W1TkSLu3ryLf_aif2VcKsFznWz0igeMUR_buoGEG6w8,1406
 ipulse_shared_core_ftredge/enums/enums_modules.py,sha256=AyXUoNmR75DZLaEHi3snV6LngR25LeZRqzrLDaAupbY,1244
 ipulse_shared_core_ftredge/models/__init__.py,sha256=gE22Gzhil0RYQa7YLtdtT44_AsWqklcDfRtgLAQc1dI,200
 ipulse_shared_core_ftredge/models/audit_log_firestore.py,sha256=5AwO6NHuOncq65n400eqM8QPrS2EGGaP3Z_6l2rxdBE,261
@@ -18,8 +18,8 @@ ipulse_shared_core_ftredge/models/user_profile_update.py,sha256=oKK0XsQDKkgDvjFP
 ipulse_shared_core_ftredge/models/user_status.py,sha256=8TyRd8tBK9_xb0MPKbI5pn9-lX7ovKbeiuWYYPtIOiw,3202
 ipulse_shared_core_ftredge/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ipulse_shared_core_ftredge/tests/test.py,sha256=0lS8HP5Quo_BqNoscU40qOH9aJRaa1Pfam5VUBmdld8,682
-ipulse_shared_core_ftredge-2.52.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
-ipulse_shared_core_ftredge-2.52.dist-info/METADATA,sha256=TKMk8quvkJXQbkQmBUcANVJfow831f3dHL2QGqwU9IM,561
-ipulse_shared_core_ftredge-2.52.dist-info/WHEEL,sha256=rWxmBtp7hEUqVLOnTaDOPpR-cZpCDkzhhcBce-Zyd5k,91
-ipulse_shared_core_ftredge-2.52.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
-ipulse_shared_core_ftredge-2.52.dist-info/RECORD,,
+ipulse_shared_core_ftredge-2.54.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
+ipulse_shared_core_ftredge-2.54.dist-info/METADATA,sha256=EB0Hd9nuPAFaIClcQR5RIGECC1L9G2ZpVXsZ7gYNFB4,561
+ipulse_shared_core_ftredge-2.54.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
+ipulse_shared_core_ftredge-2.54.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
+ipulse_shared_core_ftredge-2.54.dist-info/RECORD,,

{ipulse_shared_core_ftredge-2.52.dist-info → ipulse_shared_core_ftredge-2.54.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (71.0.4)
+Generator: setuptools (71.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any