ipulse-shared-core-ftredge 2.53__py3-none-any.whl → 2.55__py3-none-any.whl

ipulse_shared_core_ftredge/__init__.py

@@ -5,9 +5,9 @@ from .utils_gcp import (setup_gcp_logger_and_error_report,
                         write_csv_to_gcs, write_json_to_gcs)
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
                                           check_format_against_schema_template)
-from .utils_common import (SmartLog,  Watcher)
+from .utils_common import (ContextLog,  Pipelinemon)
 
-from .enums import (WatcherCategory, LogLevel, Unit, Frequency,
+from .enums import (TargetLogs, LogLevel, Unit, Frequency,
                     Module, SubModule, BaseDataCategory,
                     FinCoreCategory, FincCoreSubCategory,
                     FinCoreRecordsCategory, ExchangeOrPublisher,

ipulse_shared_core_ftredge/enums/__init__.py

@@ -4,7 +4,7 @@
 # pylint: disable=missing-class-docstring
 
 from .enums_common_utils import (LogLevel,
-                                WatcherCategory,
+                                TargetLogs,
                                 Unit,
                                 Frequency)
 

ipulse_shared_core_ftredge/enums/enums_common_utils.py

@@ -6,14 +6,14 @@
 from enum import Enum
 
 
-class WatcherCategory(Enum):
-    MIXED="watcher_mixed"
-    SUCCESSES = "watcher_successes"
-    NOTICES = "watcher_notices"
-    SUCCESSES_AND_NOTICES = "watcher_succs_n_notcs"
-    WARNINGS = "watcher_warnings"
-    WARNINGS_AND_ERRORS = "watcher_warns_n_errs"
-    ERRORS = "watcher_errors"
+class TargetLogs(Enum):
+    MIXED="mixed_logs"
+    SUCCESSES = "success_logs"
+    NOTICES = "notice_logs"
+    SUCCESSES_AND_NOTICES = "succs_n_notc_logs"
+    WARNINGS = "warning_logs"
+    WARNINGS_AND_ERRORS = "warn_n_err_logs"
+    ERRORS = "error_logs"
 
 
 class LogLevel(Enum):
@@ -98,17 +98,17 @@ class Unit(Enum):
     CURRENCY = "currency"    # General currency, when specific currency is not needed
 
     # Stock Market and Investments
-    SHARES = "shars"        # Number of shares
+    SHARES = "shares"        # Number of shares
     PERCENT = "prcnt"      # Percentage, used for rates and ratios
     BPS = "bps"              # Basis points, often used for interest rates and financial ratios
 
     # Volume and Quantitative Measurements
-    VOLUME = "vol"        # Trading volume in units
+    VOLUME = "volume"        # Trading volume in units
     MILLIONS = "mills"    # Millions, used for large quantities or sums
     BILLIONS = "bills"    # Billions, used for very large quantities or sums
 
     # Commodity Specific Units
-    BARRELS = "barrls"      # Barrels, specifically for oil and similar liquids
+    BARRELS = "barrels"      # Barrels, specifically for oil and similar liquids
     TONNES = "tonnes"        # Tonnes, for bulk materials like metals or grains
     TROY_OUNCES = "troy_oz" # Troy ounces, specifically for precious metals
 

ipulse_shared_core_ftredge/enums/enums_data_eng.py

@@ -4,11 +4,12 @@
 from enum import Enum
 
 class SourcingTriggerType(Enum):
-    HISTORIC_MANUAL = "historic_manual"
-    LIVE_SCHEDULED = "live_scheduled"
+    BULK_MANUAL = "bulk_manual"
+    BULK_SCHEDULED = "bulk_scheduled" # almost always historic bulk is manual
+    RECENT_SCHEDULED = "recent_scheduled"
+    RECENT_MANUAL = "recent_manual"
     ADHOC_MANUAL = "adhoc_manual"
     ADHOC_SCHEDULED = "adhoc_scheduled"
-    LIVE_MANUAL = "live_manual"
 
 class SourcingPipelineType(Enum):
     LOCAL_GET_API_TO_GCS = "local_get_api_to_gcs"

ipulse_shared_core_ftredge/enums/enums_module_fincore.py

@@ -5,14 +5,14 @@ from enum import Enum
 
 
 class FinCoreCategory(Enum):
-    MARKET="market"
-    ECONOMY="economy"
-    POLITICS="poltcs"
-    CORPORATE="corp"
+    MARKET="market" # Market prices data
+    CORPORATE="corp" # Corporate data such as financial statements and earnings, similar to fundamental data
     FUNDAMENTAL="fundam"
-    SENTIMENT="sntmnt"
+    ECONOMY="economy"
     NEWS="news"
+    SENTIMENT="sntmnt"
     SOCIAL="social"
+    POLITICS="poltcs"
     OTHER="other"
 
 class FincCoreSubCategory(Enum):
@@ -32,16 +32,16 @@ class FincCoreSubCategory(Enum):
 
 class FinCoreRecordsCategory(Enum):
     PRICE="pric"
-    PRICE_SPOT= "pric.s"
-    PRICE_OHLCVA="pric.ohlcva"
-    PRICE_OHLCV="pric.ohlcv"
-    PRICE_OPEN="pric.o"
-    PRICE_HIGH="pric.h"
-    PRICE_LOW="pric.l"
-    PRICE_CLOSE="pric.c"
-    PRICE_VOLUME="pric.v"
-    PRICE_ADJC="pric.a"
-    FUNDAMENTAL="fundam" # treat this differently 
+    SPOT= "spot"
+    OHLCVA="ohlcva"
+    OHLCV="ohlcv"
+    OPEN="open"
+    HIGH="high"
+    LOW="low"
+    CLOSE="close"
+    VOLUME="volume"
+    ADJC="adjc"
+    FUNDAMENTAL="fundam" # treat this differently
     EARNINGS="earnings"
     CASH_FLOW="cashflw"
     BALANCE_SHEET="blnce_sht"

ipulse_shared_core_ftredge/utils_common.py

@@ -10,18 +10,19 @@ from datetime import datetime, timezone
 from contextlib import contextmanager
 from typing import List
 from google.cloud import logging as cloudlogging
-from ipulse_shared_core_ftredge.enums.enums_common_utils import WatcherCategory, LogLevel, LogStatus
+from ipulse_shared_core_ftredge.enums.enums_common_utils import TargetLogs, LogLevel, LogStatus
 from ipulse_shared_core_ftredge.utils_gcp import write_json_to_gcs
 
 
 # ["data_import","data_quality", "data_processing","data_general","data_persistance","metadata_quality", "metadata_processing", "metadata_persistance","metadata_general"]
 
-class SmartLog:
-    MAX_TRACEBACK_LINES = 14  # Define the maximum number of traceback lines to include
-    def __init__(self, level: LogLevel, start_context: str = None, collector_id: str = None,
-                 e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
-                 subject: str = None, description: str = None, context: str = None,
-                 log_status: LogStatus = LogStatus.OPEN):
+class ContextLog:
+    MAX_TRACEBACK_LINES = 24  # Define the maximum number of traceback lines to include
+    def __init__(self, level: LogLevel, base_context: str = None, collector_id: str = None,
+                context: str = None, description: str = None,
+                e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
+                log_status: LogStatus = LogStatus.OPEN, subject: str = None
+                ):
         if e is not None:
             e_type = type(e).__name__ if e_type is None else e_type
             e_message = str(e) if e_message is None else e_message
@@ -32,7 +33,7 @@ class SmartLog:
         self.level = level
         self.subject = subject
         self.description = description
-        self._start_context = start_context
+        self._base_context = base_context
         self._context = context
         self.collector_id = collector_id
         self.exception_type = e_type
@@ -46,7 +47,7 @@ class SmartLog:
             return None
 
         traceback_lines = e_traceback.splitlines()
-        
+
         # Remove lines that are part of the exception message if they are present in traceback
         message_lines = e_message.splitlines() if e_message else []
         if message_lines:
@@ -56,7 +57,7 @@ class SmartLog:
 
         # Filter out lines from third-party libraries (like site-packages)
         filtered_lines = [line for line in traceback_lines if "site-packages" not in line]
-        
+
         # If filtering results in too few lines, revert to original traceback
         if len(filtered_lines) < 2:
             filtered_lines = traceback_lines
@@ -86,12 +87,12 @@ class SmartLog:
         return formatted_traceback
 
     @property
-    def start_context(self):
-        return self._start_context
+    def base_context(self):
+        return self._base_context
 
-    @start_context.setter
-    def start_context(self, value):
-        self._start_context = value
+    @base_context.setter
+    def base_context(self, value):
+        self._base_context = value
 
     @property
     def context(self):
@@ -101,29 +102,114 @@ class SmartLog:
     def context(self, value):
         self._context = value
 
-    def to_dict(self):
-        return {
-            "start_context": self.start_context,
-            "context": self.context,
-            "level_code": self.level.value,
-            "level_name": self.level.name,
-            "subject": self.subject,
-            "description": self.description,
-            "exception_type": self.exception_type,
-            "exception_message": self.exception_message,
-            "exception_traceback": self.exception_traceback,
-            "log_status": self.log_status.value,
-            "collector_id": self.collector_id,
-            "timestamp": self.timestamp
-        }
-
-class Watcher:
+
+    def to_dict(self, size_limit=256 * 1024 * 0.80):
+        size_limit = int(size_limit)  # Ensure size_limit is an integer
+
+        # Define the priority order of the fields
+        priority_fields = [
+            ("base_context", self.base_context),
+            ("level_code", self.level.value),
+            ("level_name", self.level.name),
+            ("log_status", self.log_status.value),
+            ("collector_id", self.collector_id),
+            ("timestamp", self.timestamp),
+        ]
+
+        # Additional fields to be truncated if necessary. Shorter fields are truncated first so that remaining size can increase for longer fields.
+        additional_fields = [
+            ("subject", self.subject),
+            ("description", self.description),
+            ("exception_type", self.exception_type),
+            ("exception_message", self.exception_message),
+            ("context", self.context),  # special sizing rules apply to it
+            ("exception_traceback", self.exception_traceback)
+        ]
+
+        all_fields = priority_fields + additional_fields
+        non_zero_fields = [(key, value) for key, value in all_fields if value is not None]
+
+        total_size = 0
+        truncated = False  # Flag to indicate if truncation happened
+
+        # Function to calculate the byte size of a JSON-encoded field
+        def field_size(key, value):
+            return len(json.dumps({key: value}).encode('utf-8'))
+
+        # Function to truncate a value based on its type
+        def truncate_value(value, max_size):
+            if isinstance(value, str):
+                half_size = max_size // 2
+                return value[:half_size] + '...' + value[-(max_size - half_size - 3):]
+            elif isinstance(value, (list, tuple)):
+                half_size = max_size // 2
+                return list(value[:half_size]) + ['...'] + list(value[-(max_size - half_size - 1):])
+            elif isinstance(value, set):
+                truncated_set = set(list(value)[:max_size // 2]) | set(list(value)[-(max_size // 2):])
+                return truncated_set
+            elif isinstance(value, dict):
+                truncated_dict = {k: truncate_value(v, max_size // len(value)) for k, v in list(value.items())}
+                return truncated_dict
+            else:
+                return value
+
+        # Calculate the initial total size
+        for key, value in non_zero_fields:
+            total_size += field_size(key, value)
+
+        log_dict = {}
+        # Check if total size exceeds the size limit
+        if total_size > size_limit:
+            truncated = True  # Set the truncation flag
+            # Calculate max size per field based on all non-zero fields
+            max_size_per_field = size_limit // len(non_zero_fields)
+
+            # Reset total_size to recompute with truncation
+            total_size = 0
+
+            # Add priority fields first with possible truncation
+            for key, value in priority_fields:
+                if value is not None:
+                    truncated_value = value
+                    if isinstance(value, (str, list, tuple, set, dict)) and field_size(key, value) > max_size_per_field:
+                        truncated_value = truncate_value(value, max_size_per_field)
+                    log_dict[key] = truncated_value
+                    total_size += field_size(key, truncated_value)
+                else:
+                    log_dict[key] = value
+
+            # Calculate remaining size for additional fields
+            remaining_size = size_limit - total_size
+
+            # Handle remaining additional fields
+            non_zero_additional_fields = [field for field in additional_fields[1:] if field[1]]
+            remaining_field_size = remaining_size // len(non_zero_additional_fields) if non_zero_additional_fields else 0
+
+            for key, value in additional_fields[1:]:
+                if value is not None:
+                    if field_size(key, value) > remaining_field_size:
+                        truncated_value = truncate_value(value, remaining_field_size)
+                    else:
+                        truncated_value = value
+                    log_dict[key] = truncated_value
+                    remaining_size -= field_size(key, truncated_value)
+                else:
+                    log_dict[key] = value
+        else:
+            log_dict = dict(all_fields)
+
+        # Add trunc flag to the log dictionary
+        log_dict['trunc'] = truncated
+
+        return log_dict
+
+class Pipelinemon:
     ERROR_START_CODE = LogLevel.ERROR.value
     WARNING_START_CODE = LogLevel.WARNING.value
     NOTICE_START_CODE = LogLevel.NOTICE.value
     SUCCESS_START_CODE = LogLevel.SUCCESS.value
 
-    def __init__(self, start_context: str, category: WatcherCategory = WatcherCategory.MIXED, logger_name=None):
+    def __init__(self, base_context: str, target_logs: TargetLogs = TargetLogs.MIXED, logger_name=None):
         self._id = str(uuid.uuid4())
         self._logs = []
         self._early_stop = False
@@ -132,9 +218,9 @@ class Watcher:
         self._notices_count = 0
         self._successes_count = 0
         self._level_counts = {level.name: 0 for level in LogLevel}
-        self._start_context = start_context
+        self._base_context = base_context
         self._context_stack = []
-        self._category = category.value
+        self._target_logs = target_logs.value
         self._logger = self._initialize_logger(logger_name)
 
     def _initialize_logger(self, logger_name):
@@ -163,8 +249,8 @@ class Watcher:
         return " >> ".join(self._context_stack)
 
     @property
-    def start_context(self):
-        return self._start_context
+    def base_context(self):
+        return self._base_context
 
     @property
     def id(self):
@@ -179,7 +265,7 @@ class Watcher:
         if create_error_log:
             if pop_context:
                 self.pop_context()
-            self.add_log(SmartLog(level=LogLevel.ERROR,
+            self.add_log(ContextLog(level=LogLevel.ERROR,
                                     subject="EARLY_STOP",
                                     description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
 
@@ -189,11 +275,11 @@ class Watcher:
     def get_early_stop(self):
         return self._early_stop
 
-    def add_log(self, log: SmartLog):
-        if (self._category == WatcherCategory.SUCCESSES and log.level >=self.NOTICE_START_CODE) or \
-           (self._category == WatcherCategory.WARNINGS_AND_ERRORS and log.level.value < self.WARNING_START_CODE):
-            raise ValueError(f"Invalid log level {log.level.name} for category {self._category}")
-        log.start_context = self.start_context
+    def add_log(self, log: ContextLog):
+        if (self._target_logs == TargetLogs.SUCCESSES and log.level >=self.NOTICE_START_CODE) or \
+           (self._target_logs == TargetLogs.WARNINGS_AND_ERRORS and log.level.value < self.WARNING_START_CODE):
+            raise ValueError(f"Invalid log level {log.level.name} for Pipelinemon target logs setup: {self._target_logs}")
+        log.base_context = self.base_context
         log.context = self.current_context
         log.collector_id = self.id
         log_dict = log.to_dict()
@@ -201,7 +287,7 @@ class Watcher:
         self._update_counts(log_dict)
 
         if self._logger:
-            # We specifically want to avoid having an ERROR log level for this structured Watcher reporting, to ensure Errors are alerting on Critical Application Services.
+            # We specifically want to avoid having an ERROR log level for this structured Pipelinemon reporting, to ensure Errors are alerting on Critical Application Services.
             # A single ERROR log level can be used for the entire pipeline, which shall be used at the end of the pipeline
             if log.level.value >= self.WARNING_START_CODE:
                 self._logger.log_struct(log_dict, severity="WARNING")
@@ -210,7 +296,7 @@ class Watcher:
             else:
                 self._logger.log_struct(log_dict, severity="INFO")
 
-    def add_logs(self, logs: List[SmartLog]):
+    def add_logs(self, logs: List[ContextLog]):
         for log in logs:
             self.add_log(log)
 
@@ -236,7 +322,7 @@ class Watcher:
             log for log in self._logs
             if context_substring in log["context"]
         ]
-    
+
     def contains_errors(self):
         return self._errors_count > 0
 
@@ -248,7 +334,7 @@ class Watcher:
 
     def count_warnings_and_errors(self):
         return self._warnings_count + self._errors_count
-    
+
     def count_warnings(self):
         return self._warnings_count
 
@@ -324,7 +410,7 @@ class Watcher:
                 logger.error(message, exc_info=exc_info)
 
         if not file_prefix:
-            file_prefix = self._category
+            file_prefix = self._target_logs
         if not file_name:
             timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
             if top_level_context:
@@ -393,265 +479,4 @@ class Watcher:
                 self._notices_count += 1
             elif self.SUCCESS_START_CODE <= level_code < self.NOTICE_START_CODE:
                 self._successes_count += 1
-            self._level_counts[level_name] += 1
-
-
-# class Watcher:
-#     ERROR_START_CODE = WatcherLogLevel.ERROR.value
-#     WARNING_START_CODE = WatcherLogLevel.WARNING.value
-#     NOTICE_START_CODE = WatcherLogLevel.NOTICE.value
-#     SUCCESS_START_CODE = WatcherLogLevel.SUCCESS.value
-
-#     def __init__(self, start_context: str, category: WatcherCategory = WatcherCategory.MIXED, logger_name=None):
-#         self._id = str(uuid.uuid4())
-#         self._logs = []
-#         self._early_stop = False
-#         self._errors_count = 0
-#         self._warnings_count = 0
-#         self._successes_count = 0
-#         self._level_counts = {level.name: 0 for level in WatcherLogLevel}
-#         self._start_context = start_context
-#         self._context_stack = []
-#         self._category = category.value
-#         self._logger = self._initialize_logger(logger_name)
-
-#     def _initialize_logger(self, logger_name):
-#         if logger_name:
-#             logging_client = cloudlogging.Client()
-#             return logging_client.logger(logger_name)
-#         return None
-
-
-#     @contextmanager
-#     def context(self, context):
-#         self.push_context(context)
-#         try:
-#             yield
-#         finally:
-#             self.pop_context()
-
-#     def push_context(self, context):
-#         self._context_stack.append(context)
-
-#     def pop_context(self):
-#         if self._context_stack:
-#             self._context_stack.pop()
-
-#     @property
-#     def current_context(self):
-#         return " >> ".join(self._context_stack)
-
-#     @property
-#     def start_context(self):
-#         return self._start_context
-
-#     @property
-#     def id(self):
-#         return self._id
-
-#     @property
-#     def early_stop(self):
-#         return self._early_stop
-
-#     def set_early_stop(self, max_errors_tolerance:int, create_error_notice=True,pop_context=False):
-#         self.early_stop = True
-#         if create_error_notice:
-#             if pop_context:
-#                 self.pop_context()
-#             self.add_notice(WatcherLog(level=WatcherLogLevel.ERROR,
-#                     subject="EARLY_STOP",
-#                     description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
-
-#     def reset_early_stop(self):
-#         self._early_stop = False
-
-#     def get_early_stop(self):
-#         return self._early_stop
-
-#     def add_notice(self, notice: WatcherLog):
-#         if (self._category == WatcherCategory.SUCCESSES.value and notice.level != WatcherLogLevel.SUCCESS) or \
-#            (self._category == WatcherCategory.WARNINGS_AND_ERRORS.value and notice.level.value < self.WARNING_START_CODE):
-#             raise ValueError(f"Invalid notice level {notice.level.name} for category {self._category}")
-#         notice.start_context = self.start_context
-#         notice.context = self.current_context
-#         notice.watcher_id = self.id
-#         notice_dict = notice.to_dict()
-#         self._logs.append(notice_dict)
-#         self._update_counts(notice_dict)
-
-#         if self._logger:
-#             if notice.level.value >= self.WARNING_START_CODE:
-#                 self._logger.log_struct(notice_dict, severity="WARNING")
-#             else:
-#                 self._logger.log_struct(notice_dict, severity="INFO")
-
-#     def add_notices(self, notices: List[WatcherLog]):
-#         for notice in notices:
-#             self.add_notice(notice)
-
-#     def clear_notices_and_counts(self):
-#         self._logs = []
-#         self._errors_count = 0
-#         self._warnings_count = 0
-#         self._successes_count = 0
-#         self._level_counts = {level.name: 0 for level in WatcherLogLevel}
-
-#     def clear_notices(self):
-#         self._logs = []
-
-#     def get_all_notices(self):
-#         return self._logs
-
-#     def get_notices_for_level(self, level: WatcherLogLevel):
-#         return [notice for notice in self._logs if notice["level_code"] == level.value]
-
-#     def get_notices_by_str_in_context(self, context_substring: str):
-#         return [
-#             notice for notice in self._logs
-#             if context_substring in notice["context"]
-#         ]
-    
-#     def contains_errors(self):
-#         return self._errors_count > 0
-
-#     def count_errors(self):
-#         return self._errors_count
-
-#     def contains_warnings_or_errors(self):
-#         return self._warnings_count > 0 or self._errors_count > 0
-
-#     def count_warnings_and_errors(self):
-#         return self._warnings_count + self._errors_count
-
-#     def count_warnings(self):
-#         return self._warnings_count
-
-#     def count_successes(self):
-#         return self._successes_count
-
-#     def count_all_notices(self):
-#         return len(self._logs)
-
-#     def count_notices_by_level(self, level: WatcherLogLevel):
-#         return self._level_counts.get(level.name, 0)
-
-#     def _count_notices(self, context_substring: str, exact_match=False, level_code_min=None, level_code_max=None):
-#         return sum(
-#             1 for notice in self._logs
-#             if (notice["context"] == context_substring if exact_match else context_substring in notice["context"]) and
-#                (level_code_min is None or notice["level_code"] >= level_code_min) and
-#                (level_code_max is None or notice["level_code"] <= level_code_max)
-#         )
-
-#     def count_notices_for_current_context(self):
-#         return self._count_notices(self.current_context, exact_match=True)
-
-#     def count_notices_for_current_and_nested_contexts(self):
-#         return self._count_notices(self.current_context)
-
-#     def count_notices_by_level_for_current_context(self, level: WatcherLogLevel):
-#         return self._count_notices(self.current_context, exact_match=True, level_code_min=level.value, level_code_max=level.value)
-
-#     def count_notices_by_level_for_current_and_nested_contexts(self, level: WatcherLogLevel):
-#         return self._count_notices(self.current_context, level_code_min=level.value, level_code_max=level.value)
-
-#     def count_errors_for_current_context(self):
-#         return self._count_notices(self.current_context, exact_match=True, level_code_min=self.ERROR_START_CODE)
-
-#     def count_errors_for_current_and_nested_contexts(self):
-#         return self._count_notices(self.current_context, level_code_min=self.ERROR_START_CODE)
-
-#     def count_warnings_and_errors_for_current_context(self):
-#         return self._count_notices(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE)
-
-#     def count_warnings_and_errors_for_current_and_nested_contexts(self):
-#         return self._count_notices(self.current_context, level_code_min=self.WARNING_START_CODE)
-
-#     def count_warnings_for_current_context(self):
-#         return self._count_notices(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
-
-#     def count_warnings_for_current_and_nested_contexts(self):
-#         return self._count_notices(self.current_context, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
-
-#     def count_successes_for_current_context(self):
-#         return self._count_notices(self.current_context, exact_match=True, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.SUCCESS_START_CODE)
-
-#     def count_successes_for_current_and_nested_contexts(self):
-#         return self._count_notices(self.current_context, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.SUCCESS_START_CODE)
-
-#     def export_notices_to_gcs_file(self, bucket_name, storage_client, file_prefix=None, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
-#         def log_message(message):
-#             if logger:
-#                 logger.info(message)
-
-#         def log_error(message, exc_info=False):
-#             if logger:
-#                 logger.error(message, exc_info=exc_info)
-
-#         if not file_prefix:
-#             file_prefix = self._category
-#         if not file_name:
-#             timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
-#             if top_level_context:
-#                 file_name = f"{file_prefix}_{timestamp}_{top_level_context}_len{len(self._logs)}.json"
-#             else:
-#                 file_name = f"{file_prefix}_{timestamp}_len{len(self._logs)}.json"
-
-#         result=None
-#         try:
-#             result= write_json_to_gcs(
-#                 bucket_name=bucket_name,
-#                 storage_client=storage_client,
-#                 data=self._logs,
-#                 file_name=file_name,
-#                 save_locally=save_locally,
-#                 local_path=local_path,
-#                 logger=logger,
-#                 max_retries=max_retries,
-#                 overwrite_gcs=False
-#             )
-#             log_message(f"{file_prefix} successfully saved (ovewritten={result.get("gcs_file_overwritten")}) to GCS at {result.get("gcs_path")} and locally at {result.get("local_path")}.")
-#         except Exception as e:
-#             log_error(f"Failed at export_notices_to_gcs_file for {file_prefix} for file {file_name} to bucket {bucket_name}: {type(e).__name__} - {str(e)}")
-
-#         return result
-
-#     def import_notices_from_json(self, json_or_file, logger=None):
-#         def log_message(message):
-#             if logger:
-#                 logger.info(message)
-
-#         def log_warning(message, exc_info=False):
-#             if logger:
-#                 logger.warning(message, exc_info=exc_info)
-
-#         try:
-#             if isinstance(json_or_file, str):  # Load from string
-#                 imported_notices = json.loads(json_or_file)
-#             elif hasattr(json_or_file, 'read'):  # Load from file-like object
-#                 imported_notices = json.load(json_or_file)
-#             self.add_notices(imported_notices)
-#             log_message("Successfully imported notices from json.")
-#         except Exception as e:
-#             log_warning(f"Failed to import notices from json: {type(e).__name__} - {str(e)}", exc_info=True)
-
-#     def _update_counts(self, notice, remove=False):
-#         level_code = notice["level_code"]
-#         level_name = notice["level_name"]
-
-#         if remove:
-#             if level_code >= self.ERROR_START_CODE:
-#                 self._errors_count -= 1
-#             elif level_code >= self.WARNING_START_CODE:
-#                 self._warnings_count -= 1
-#             elif level_code >= self.SUCCESS_START_CODE:
-#                 self._successes_count -= 1
-#             self._level_counts[level_name] -= 1
-#         else:
-#             if level_code >= self.ERROR_START_CODE:
-#                 self._errors_count += 1
-#             elif level_code >= self.WARNING_START_CODE:
-#                 self._warnings_count += 1
-#             elif level_code == self.SUCCESS_START_CODE:
-#                 self._successes_count += 1
-#             self._level_counts[level_name] += 1
+            self._level_counts[level_name] += 1

ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -6,7 +6,7 @@
 import datetime
 from google.cloud import bigquery
 from ipulse_shared_core_ftredge.enums.enums_common_utils import LogLevel
-from ipulse_shared_core_ftredge.utils_common import SmartLog
+from ipulse_shared_core_ftredge.utils_common import ContextLog
 
 
 def create_bigquery_schema_from_json(json_schema):
@@ -59,13 +59,13 @@ def check_format_against_schema_template(data_to_check, schema, dt_ts_to_str=Tru
                     checked_data[field_name] = value
 
             elif mode == "REQUIRED":
-                warning = SmartLog(level=LogLevel.WARNING,
+                warning = ContextLog(level=LogLevel.WARNING,
                                               subject=field_name,
                                               description=f"Required field '{field_name}' is missing in the updates.")
                 warnings_or_error.append(warning)
 
     except Exception as e:
-        error_log = SmartLog(level=LogLevel.ERROR_EXCEPTION_REDO,
+        error_log = ContextLog(level=LogLevel.ERROR_EXCEPTION_REDO,
                                subject=data_to_check,
                                description=f"An error occurred during update check: {str(e)}")
         warnings_or_error.append(error_log)
@@ -85,11 +85,11 @@ def handle_date_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_date, None
         except ValueError:
-            return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED,
+            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
                                                    description=f"Expected a DATE in YYYY-MM-DD format but got {value}.")
     else:
-        return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED,
+        return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                               subject=field_name,
                                               description= f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
 
@@ -107,11 +107,11 @@ def handle_timestamp_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_datetime, None
         except ValueError:
-            return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED,
+            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
                                                   description= f"Expected ISO format TIMESTAMP but got {value}.")
     else:
-        return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED,
+        return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                               subject=field_name,
                                               description= f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
 
@@ -119,7 +119,7 @@ def handle_timestamp_fields(field_name, value, dt_ts_to_str):
 def check_and_truncate_length(field_name, value, max_length):
     """Checks and truncates the length of string fields if they exceed the max length."""
     if isinstance(value, str) and len(value) > max_length:
-        return value[:max_length], SmartLog(level=LogLevel.WARNING_FIX_RECOMMENDED,
+        return value[:max_length], ContextLog(level=LogLevel.WARNING_FIX_RECOMMENDED,
                             subject= field_name,
                              description= f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
 
@@ -129,7 +129,7 @@ def check_and_truncate_length(field_name, value, max_length):
 
 def handle_type_conversion(field_type, field_name, value):
     if field_type == "STRING" and not isinstance(value, str):
-        return str(value), SmartLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED,
+        return str(value), ContextLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED,
                              subject=field_name,
                              description= f"Expected STRING but got {value} of type {type(value).__name__}.")
 
@@ -137,18 +137,18 @@ def handle_type_conversion(field_type, field_name, value):
         try:
             return int(value), None
         except ValueError:
-            return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED,
+            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                                 subject= field_name,
                                                 description=f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
     if field_type == "FLOAT64" and not isinstance(value, float):
         try:
             return float(value), None
         except ValueError:
-            return None, SmartLog(level=LogLevel.WARNING_FIX_REQUIRED, 
+            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED, 
                                                 subject=field_name,
                                                 description=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
     if field_type == "BOOL" and not isinstance(value, bool):
-        return bool(value), SmartLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED, 
+        return bool(value), ContextLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED, 
                                                 subject=field_name,
                                                 description=f"Expected BOOL, but got  {value}. Converting as {bool(value)}.")
 

{ipulse_shared_core_ftredge-2.53.dist-info → ipulse_shared_core_ftredge-2.55.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.53
+Version: 2.55
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.53.dist-info → ipulse_shared_core_ftredge-2.55.dist-info}/RECORD

@@ -1,11 +1,11 @@
-ipulse_shared_core_ftredge/__init__.py,sha256=CcHx8XkC7YJ5pOxsOpZJrTuxweN1ya1WlQJZjOTwrBY,868
-ipulse_shared_core_ftredge/utils_common.py,sha256=GEo4Xilh9quDdUh_ppOVO6G7ustHWkSaxuILKC_FLNo,27406
+ipulse_shared_core_ftredge/__init__.py,sha256=gIwh_6xjoG7UIuTKNkq0iGvCuxsbyXNrTyV3vB8YdGc,869
+ipulse_shared_core_ftredge/utils_common.py,sha256=MzOoeCZRibVE8yZd5DLMaEuKa-ed5bb4ihob6wHpXRs,20351
 ipulse_shared_core_ftredge/utils_gcp.py,sha256=8KgsOPkLe1-1i3M_UX5niKg_CjjiNoUhZXiWFIHJdmY,11286
-ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=CHrFbhRVrXlqDzGdPe9nujn5uFQtIN2xW7RBTiHYFBc,7475
-ipulse_shared_core_ftredge/enums/__init__.py,sha256=PT8Ig7hcx_hhVlsfun24H0pFjbdfQb201ZtJplQ9uAE,844
-ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=CB0IMW5aer-n50G3AM6Fz-NrN85mJkvZhSrnuUb7EMs,5702
-ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=2i6Qo6Yi_j_O9xxnOD6QA-r0Cv7mWAUaKUx907XMRio,1825
-ipulse_shared_core_ftredge/enums/enums_module_fincore.py,sha256=MuqQg249clrWUOBb1S-iPsoOldN2_F3ohRQizbjhwG0,1374
+ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=AwGl9J-XQc_aO_VKWhR_TuA1ML8nWxHuzHtxBH8yfwE,7499
+ipulse_shared_core_ftredge/enums/__init__.py,sha256=reCHJE0j_QTGwag7uo3cQXMTRaGRR_YOwywweb5pFb8,839
+ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=gYckmSAzhh5MA3CyV18Z9-YtHsFJlmEcKC-4nuqMXu4,5673
+ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=7w3Jjmw84Wq22Bb5Qs09Z82Bdf-j8nhRiQJfw60_g80,1903
+ipulse_shared_core_ftredge/enums/enums_module_fincore.py,sha256=W1TkSLu3ryLf_aif2VcKsFznWz0igeMUR_buoGEG6w8,1406
 ipulse_shared_core_ftredge/enums/enums_modules.py,sha256=AyXUoNmR75DZLaEHi3snV6LngR25LeZRqzrLDaAupbY,1244
 ipulse_shared_core_ftredge/models/__init__.py,sha256=gE22Gzhil0RYQa7YLtdtT44_AsWqklcDfRtgLAQc1dI,200
 ipulse_shared_core_ftredge/models/audit_log_firestore.py,sha256=5AwO6NHuOncq65n400eqM8QPrS2EGGaP3Z_6l2rxdBE,261
@@ -18,8 +18,8 @@ ipulse_shared_core_ftredge/models/user_profile_update.py,sha256=oKK0XsQDKkgDvjFP
 ipulse_shared_core_ftredge/models/user_status.py,sha256=8TyRd8tBK9_xb0MPKbI5pn9-lX7ovKbeiuWYYPtIOiw,3202
 ipulse_shared_core_ftredge/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ipulse_shared_core_ftredge/tests/test.py,sha256=0lS8HP5Quo_BqNoscU40qOH9aJRaa1Pfam5VUBmdld8,682
-ipulse_shared_core_ftredge-2.53.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
-ipulse_shared_core_ftredge-2.53.dist-info/METADATA,sha256=9JixJKcqPsiCzRQR6ZpOiKwDIOcjEHFY6OG-VILJ_zg,561
-ipulse_shared_core_ftredge-2.53.dist-info/WHEEL,sha256=rWxmBtp7hEUqVLOnTaDOPpR-cZpCDkzhhcBce-Zyd5k,91
-ipulse_shared_core_ftredge-2.53.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
-ipulse_shared_core_ftredge-2.53.dist-info/RECORD,,
+ipulse_shared_core_ftredge-2.55.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
+ipulse_shared_core_ftredge-2.55.dist-info/METADATA,sha256=nZuLld1WcyBi4KcRJGZr6dGRV62pYA1Nh5SkXMW1-0I,561
+ipulse_shared_core_ftredge-2.55.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
+ipulse_shared_core_ftredge-2.55.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
+ipulse_shared_core_ftredge-2.55.dist-info/RECORD,,

{ipulse_shared_core_ftredge-2.53.dist-info → ipulse_shared_core_ftredge-2.55.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (71.0.4)
+Generator: setuptools (71.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any