ipulse-shared-core-ftredge 2.39__tar.gz → 2.51__tar.gz

{ipulse_shared_core_ftredge-2.39/src/ipulse_shared_core_ftredge.egg-info → ipulse_shared_core_ftredge-2.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.39
+Version: 2.51
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.39 → ipulse_shared_core_ftredge-2.51}/setup.py

@@ -1,8 +1,9 @@
+# pylint: disable=import-error
 from setuptools import setup, find_packages
 
 setup(
     name='ipulse_shared_core_ftredge',
-    version='2.39',
+    version='2.51',
     package_dir={'': 'src'},  # Specify the source directory
     packages=find_packages(where='src'),  # Look for packages in 'src'
     install_requires=[

{ipulse_shared_core_ftredge-2.39 → ipulse_shared_core_ftredge-2.51}/src/ipulse_shared_core_ftredge/__init__.py

@@ -2,16 +2,14 @@ from .models import (Organisation, UserAuth, UserProfile,
                      UserStatus, UserProfileUpdate, pulse_enums)
 from .utils_gcp import (setup_gcp_logger_and_error_report, 
                         read_csv_from_gcs, read_json_from_gcs, 
-                        write_csv_to_gcs, write_json_to_gcs)
+                        write_csv_to_gcs, write_data_to_gcs)
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
                                           update_check_with_schema_template)
-from .utils_common import (create_custom_notice, create_exception_notice)
+from .utils_common import (Notice,  NoticesManager)
 
-from .enums import (NoticeSeverity, Unit, Frequency,
+from .enums import (NoticeManagerCategory, NoticeLevel, Unit, Frequency,
                     Module, SubModule, BaseDataCategory,
                     FinCoreCategory, FincCoreSubCategory,
                     FinCoreRecordsCategory, ExchangeOrPublisher,
                     SourcingPipelineType, SourcingTriggerType,
                     DWEvent, DWEventTriggerType)
-
-                    

{ipulse_shared_core_ftredge-2.39 → ipulse_shared_core_ftredge-2.51}/src/ipulse_shared_core_ftredge/enums/__init__.py

@@ -3,7 +3,8 @@
 # pylint: disable=missing-function-docstring
 # pylint: disable=missing-class-docstring
 
-from .enums_common_utils import (NoticeSeverity,
+from .enums_common_utils import (NoticeLevel,
+                                NoticeManagerCategory,
                                 Unit,
                                 Frequency)
 

{ipulse_shared_core_ftredge-2.39 → ipulse_shared_core_ftredge-2.51}/src/ipulse_shared_core_ftredge/enums/enums_common_utils.py

@@ -5,32 +5,62 @@
 
 from enum import Enum
 
-class NoticeSeverity(Enum):
+
+class NoticeManagerCategory(Enum):
+    NOTICES = "notices"
+    WARN_ERRS = "warn_errs"
+    SUCCESSES = "successes"
+class NoticeLevel(Enum):
     """
-    Standardized logging levels for data engineering pipelines,
+    Standardized notice levels for data engineering pipelines,
     designed for easy analysis and identification of manual 
     intervention needs.
     """
     DEBUG = 100  # Detailed debug information (for development/troubleshooting)
-    INFO = 200   # Normal pipeline execution information 
-    NOTICE = 300  # Events requiring attention, but not necessarily errors
+
+    INFO = 200
+
+    SUCCESS = 300  # Events requiring attention, but not necessarily errors
 
      # Warnings indicate potential issues that might require attention:
+    WARNING = 400 # General warning, no immediate action required
     WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
-    WARNING_ACTION_RECOMMENDED = 402 # Action recommended to prevent potential future issues
-    WARNING_ACTION_REQUIRED = 403  # Action required, pipeline can likely continue
+    WARNING_REVIEW_RECOMMENDED = 402 # Action recommended to prevent potential future issues
+    WARNING_FIX_RECOMMENDED = 403 # Action recommended to prevent potential future issues
+    WARNING_FIX_REQUIRED = 404  # Action required, pipeline can likely continue
 
+    ERROR = 500 # General error, no immediate action required
     # Errors indicate a problem that disrupts normal pipeline execution:
-    ERROR_TRANSIENT_RETRY = 501 # Temporary error, automatic retry likely to succeed
-    ERROR_DATA_ISSUE_ISOLATED = 502 # Error likely caused by data issues, manual intervention likely needed
-    ERROR_DATA_ISSUE_WITH_DEPENDENCIES = 503 # Error likely in code/configuration, investigation required
-    ERROR_CONFIG_OR_CODE_ISSUE = 504 # Error likely in code/configuration, investigation required
-    ERROR_UNKNOWN_EXCEPTION = 505
+    ERROR_EXCEPTION_REDO = 501
+    ERROR_CUSTOM_REDO = 502 # Temporary error, automatic retry likely to succeed
+    
+    ERROR_EXCEPTION_INVESTIGATE = 601 # Exception occured after some data was likely persisted (e.g., to GCS or BQ)
+    ERROR_CUSTOM_INVESTIGATE= 602
+    ERROR_EXCEPTION_PERSTISTANCE = 603 # Exception occured after data was persisted (e.g., to GCS or BQ)
+    ERROR_CUSTOM_PERSTISTANCE = 604
 
     # Critical errors indicate severe failures requiring immediate attention:
-    CRITICAL_SYSTEM_FAILURE = 601 # System-level failure (e.g., infrastructure), requires immediate action
-    CRITICAL_PIPELINE_FAILURE = 602 # Complete pipeline failure, requires investigation and potential rollback
+    CRITICAL_SYSTEM_FAILURE = 701 # System-level failure (e.g., infrastructure), requires immediate action
+    CRITICAL_PIPELINE_FAILURE = 702 # Complete pipeline failure, requires investigation and potential rollback
+
+    UNKNOWN=1001 # Unknown error, should not be used in normal operation
+
+
+class NoticeStatus(Enum):
+    OPEN = "open"
+    ACKNOWLEDGED = "acknowledged"
+    IN_PROGRESS = "in_progress"
+    RESOLVED = "resolved"
+    IGNORED = "ignored"
+  
+
+
 
+### Exception during full exection, partially saved
+# Exception during ensemble pipeline; modifications collected in local object , nothing persisted
+# Exception during ensemble pipeline; modifications persisted , metadata failed
+# Exception during ensemble pipeline; modifications persisted , metadata persisted
+# Exception during ensemble pipeline; modifications persisted , metadata persisted
 
 
 class Unit(Enum):

{ipulse_shared_core_ftredge-2.39 → ipulse_shared_core_ftredge-2.51}/src/ipulse_shared_core_ftredge/enums/enums_data_eng.py

@@ -20,25 +20,24 @@ class SourcingPipelineType(Enum):
     CLOUD_GET_API_INMEMORY = "cloud_get_api_inmemory"
 
 class DWEventTriggerType(Enum):
-    GCS_BUCKET_UPLOAD = "gcs_bucket_upload"
+    GCS_UPLOAD_TRIGGER_CF = "gcs_upload_trigger_cf"
+    HTTP_TRIGGER_CF_FOR_GCS_FILE = "http_trigger_cf_for_gcs_file"
+    PUBSUB_TRIGGER_CF_FOR_GCS_FILE = "pubsub_trigger_cf_for_gcs_file"
+    LOCAL_SCRIPT_FOR_GCS_FILE = "local_script_for_gcs_file"
     INSIDE_SOURCING_FUNCTION = "inside_sourcing_function"
-    HTTP_FUNC_TO_GCS = "http_func_to_gcs"
-    LOCAL_FROM_GCS_FILE = "local_from_gcs_file"
-    MANUAL_FROM_LOCAL_FILE = "manual_from_local_file"
-    PUBSUBC_TOPIC = "pubsubc_topic"
 
 class DWEvent(Enum):
-    INSERT_NOREPLACE_1A_NT = "insert_noreplace_1a_nt"
-    MERGE_NOREPLACE_NA_1T = "merge_noreplace_na_1t"
-    MERGE_NOREPLACE_NA_NT = "merge_noreplace_na_nt"
-    INSERT_NOREPLACE_1A_1T = "insert_noreplace_1a_1t"
-    MERGE_NOREPLACE_1A_NT = "merge_noreplace_1a_nt"
-    INSERT_REPLACE_1A_1T = "insert_replace_1a_1t"
-    INSERT_REPLACE_1A_NT = "insert_replace_1a_nt"
-    MERGE_REPLACE_NA_NT = "merge_replace_na_nt"
-    MERGE_REPLACE_1A_NT = "merge_replace_1a_nt"
-    MERGE_REPLACE_NA_1T = "merge_replace_na_1t"
-    DELETE_1A_1T = "delete_1a_1t"
-    DELETE_1A_NT = "delete_1a_nt"
-    DELETE_NA_1T = "delete_na_1t"
-    DELETE_NA_NT = "delete_na_nt"
+    INSERT_NOREPLACE_1O_NT = "insert_noreplace_1o_nt"
+    MERGE_NOREPLACE_NO_1T = "merge_noreplace_no_1t"
+    MERGE_NOREPLACE_NO_NT = "merge_noreplace_no_nt"
+    INSERT_NOREPLACE_1O_1T = "insert_noreplace_1o_1t"
+    MERGE_NOREPLACE_1O_NT = "merge_noreplace_1o_nt"
+    INSERT_REPLACE_1O_1T = "insert_replace_1o_1t"
+    INSERT_REPLACE_1O_NT = "insert_replace_1o_nt"
+    MERGE_REPLACE_NO_NT = "merge_replace_no_nt"
+    MERGE_REPLACE_1O_NT = "merge_replace_1o_nt"
+    MERGE_REPLACE_NO_1T = "merge_replace_no_1t"
+    DELETE_1O_1T = "delete_1o_1t"
+    DELETE_1O_NT = "delete_1o_nt"
+    DELETE_NO_1T = "delete_no_1t"
+    DELETE_NO_NT = "delete_no_nt"

ipulse_shared_core_ftredge-2.51/src/ipulse_shared_core_ftredge/utils_common.py

@@ -0,0 +1,369 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+# pylint: disable=missing-class-docstring
+import traceback
+import json
+import uuid
+from datetime import datetime, timezone
+from contextlib import contextmanager
+from typing import List
+from google.cloud import logging as cloudlogging
+from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeLevel, NoticeManagerCategory, NoticeStatus
+from ipulse_shared_core_ftredge.utils_gcp import write_data_to_gcs
+
+
+# ["data_import","data_quality", "data_processing","data_general","data_persistance","metadata_quality", "metadata_processing", "metadata_persistance","metadata_general"]
+
+class Notice:
+    MAX_TRACEBACK_LINES = 14  # Define the maximum number of traceback lines to include
+    def __init__(self, level: NoticeLevel, start_context: str = None, notice_manager_id: str = None,
+                 e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
+                 subject: str = None, description: str = None, context: str = None,
+                 notice_status: NoticeStatus = NoticeStatus.OPEN):
+        if e is not None:
+            e_type = type(e).__name__ if e_type is None else e_type
+            e_message = str(e) if e_message is None else e_message
+            e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
+        elif e_traceback is None and (e_type or e_message):
+            e_traceback = traceback.format_exc()
+
+        self.level = level
+        self.subject = subject
+        self.description = description
+        self._start_context = start_context
+        self._context = context
+        self.notice_manager_id = notice_manager_id
+        self.exception_type = e_type
+        self.exception_message = e_message
+        self.exception_traceback = self._format_traceback(e_traceback,e_message)
+        self.notice_status = notice_status
+        self.timestamp = datetime.now(timezone.utc).isoformat()
+
+    def _format_traceback(self, e_traceback, e_message):
+        if not e_traceback or e_traceback == 'None\n':
+            return None
+
+        traceback_lines = e_traceback.splitlines()
+        
+        # Remove lines that are part of the exception message if they are present in traceback
+        message_lines = e_message.splitlines() if e_message else []
+        if message_lines:
+            for message_line in message_lines:
+                if message_line in traceback_lines:
+                    traceback_lines.remove(message_line)
+
+        # Filter out lines from third-party libraries (like site-packages)
+        filtered_lines = [line for line in traceback_lines if "site-packages" not in line]
+        
+        # If filtering results in too few lines, revert to original traceback
+        if len(filtered_lines) < 2:
+            filtered_lines = traceback_lines
+
+        # Combine standalone bracket lines with previous or next lines
+        combined_lines = []
+        for line in filtered_lines:
+            if line.strip() in {"(", ")", "{", "}", "[", "]"} and combined_lines:
+                combined_lines[-1] += " " + line.strip()
+            else:
+                combined_lines.append(line)
+
+        # Determine the number of lines to keep from the start and end
+        keep_lines_start = min(self.MAX_TRACEBACK_LINES // 2, len(combined_lines))
+        keep_lines_end = min(self.MAX_TRACEBACK_LINES // 2, len(combined_lines) - keep_lines_start)
+        
+        if len(combined_lines) > self.MAX_TRACEBACK_LINES:
+            # Include the first few and last few lines, and an indicator of truncation
+            formatted_traceback = '\n'.join(
+                combined_lines[:keep_lines_start] + 
+                ['... (truncated) ...'] + 
+                combined_lines[-keep_lines_end:]
+            )
+        else:
+            formatted_traceback = '\n'.join(combined_lines)
+        
+        return formatted_traceback  
+
+    @property
+    def start_context(self):
+        return self._start_context
+
+    @start_context.setter
+    def start_context(self, value):
+        self._start_context = value
+
+    @property
+    def context(self):
+        return self._context
+
+    @context.setter
+    def context(self, value):
+        self._context = value
+
+    def to_dict(self):
+        return {
+            "start_context": self.start_context,
+            "context": self.context,
+            "level_code": self.level.value,
+            "level_name": self.level.name,
+            "subject": self.subject,
+            "description": self.description,
+            "exception_type": self.exception_type,
+            "exception_message": self.exception_message,
+            "exception_traceback": self.exception_traceback,
+            "notice_status": self.notice_status.value,
+            "notice_manager_id": self.notice_manager_id,
+            "timestamp": self.timestamp
+        }
+
+class NoticesManager:
+    ERROR_CODE_START_VALUE = NoticeLevel.ERROR.value
+    WARNING_CODE_START_VALUE = NoticeLevel.WARNING.value
+    SUCCESS_CODE_START_VALUE = NoticeLevel.SUCCESS.value
+
+    def __init__(self, start_context: str, category: NoticeManagerCategory = NoticeManagerCategory.NOTICES, logger_name=None):
+        self._notice_manager_id = str(uuid.uuid4())
+        self._notices = []
+        self._early_stop = False
+        self._error_count = 0
+        self._warning_count = 0
+        self._success_count = 0
+        self._level_counts = {level.name: 0 for level in NoticeLevel}
+        self._start_context = start_context
+        self._context_stack = []
+        self._category = category.value
+        self._logger = self._initialize_logger(logger_name)
+
+    def _initialize_logger(self, logger_name):
+        if logger_name:
+            logging_client = cloudlogging.Client()
+            return logging_client.logger(logger_name)
+        return None
+
+
+    @contextmanager
+    def context(self, context):
+        self.push_context(context)
+        try:
+            yield
+        finally:
+            self.pop_context()
+
+    def push_context(self, context):
+        self._context_stack.append(context)
+
+    def pop_context(self):
+        if self._context_stack:
+            self._context_stack.pop()
+
+    def get_current_context(self):
+        return " >> ".join(self._context_stack)
+
+    def get_start_context(self):
+        return self._start_context
+    
+    def get_notice_manager_id(self):
+        return self._notice_manager_id
+    
+    def set_early_stop(self, max_errors_tolerance:int, create_error_notice=True,pop_context=False):
+        self._early_stop = True
+        if create_error_notice:
+            if pop_context:
+                self.pop_context()
+            self.add_notice(Notice(level=NoticeLevel.ERROR,
+                    subject="EARLY_STOP",
+                    description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
+    def reset_early_stop(self):
+        self._early_stop = False
+
+    def get_early_stop(self):
+        return self._early_stop
+
+    def add_notice(self, notice: Notice):
+        if (self._category == NoticeManagerCategory.SUCCESSES.value and notice.level != NoticeLevel.SUCCESS) or \
+           (self._category == NoticeManagerCategory.WARN_ERRS.value and notice.level.value < self.WARNING_CODE_START_VALUE):
+            raise ValueError(f"Invalid notice level {notice.level.name} for category {self._category}")
+        notice.start_context = self.get_start_context()
+        notice.context = self.get_current_context()
+        notice.notice_manager_id = self._notice_manager_id
+        notice_dict = notice.to_dict()
+        self._notices.append(notice_dict)
+        self._update_counts(notice_dict)
+
+        if self._logger:
+            if notice.level.value >= self.WARNING_CODE_START_VALUE:
+                self._logger.log_struct(notice_dict, severity="WARNING")
+            else:
+                self._logger.log_struct(notice_dict, severity="INFO")
+
+    def add_notices(self, notices: List[Notice]):
+        for notice in notices:
+            self.add_notice(notice)
+
+    def clear_notices_and_counts(self):
+        self._notices = []
+        self._error_count = 0
+        self._warning_count = 0
+        self._success_count = 0
+        self._level_counts = {level.name: 0 for level in NoticeLevel}
+
+    def clear_notices(self):
+        self._notices = []
+
+    def get_all_notices(self):
+        return self._notices
+
+    def get_notices_for_level(self, level: NoticeLevel):
+        return [notice for notice in self._notices if notice["level_code"] == level.value]
+
+    def get_notices_by_str_in_context(self, context_substring: str):
+        return [
+            notice for notice in self._notices
+            if context_substring in notice["context"]
+        ]
+
+    def contains_errors(self):
+        return self._error_count > 0
+
+    def count_errors(self):
+        return self._error_count
+
+    def contains_warnings_or_errors(self):
+        return self._warning_count > 0 or self._error_count > 0
+
+    def count_warnings_and_errors(self):
+        return self._warning_count + self._error_count
+
+    def count_warnings(self):
+        return self._warning_count
+
+    def count_successes(self):
+        return self._success_count
+
+    def count_all_notices(self):
+        return len(self._notices)
+
+    def count_notices_by_level(self, level: NoticeLevel):
+        return self._level_counts.get(level.name, 0)
+
+    def _count_notices(self, context_substring: str, exact_match=False, level_code_min=None, level_code_max=None):
+        return sum(
+            1 for notice in self._notices
+            if (notice["context"] == context_substring if exact_match else context_substring in notice["context"]) and
+               (level_code_min is None or notice["level_code"] >= level_code_min) and
+               (level_code_max is None or notice["level_code"] <= level_code_max)
+        )
+
+    def count_notices_for_current_context(self):
+        return self._count_notices(self.get_current_context(), exact_match=True)
+
+    def count_notices_for_current_and_nested_contexts(self):
+        return self._count_notices(self.get_current_context())
+
+    def count_notices_by_level_for_current_context(self, level: NoticeLevel):
+        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=level.value, level_code_max=level.value)
+
+    def count_notices_by_level_for_current_and_nested_contexts(self, level: NoticeLevel):
+        return self._count_notices(self.get_current_context(), level_code_min=level.value, level_code_max=level.value)
+
+    def count_errors_for_current_context(self):
+        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.ERROR_CODE_START_VALUE)
+
+    def count_errors_for_current_and_nested_contexts(self):
+        return self._count_notices(self.get_current_context(), level_code_min=self.ERROR_CODE_START_VALUE)
+
+    def count_warnings_and_errors_for_current_context(self):
+        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE)
+
+    def count_warnings_and_errors_for_current_and_nested_contexts(self):
+        return self._count_notices(self.get_current_context(), level_code_min=self.WARNING_CODE_START_VALUE)
+
+    def count_warnings_for_current_context(self):
+        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
+
+    def count_warnings_for_current_and_nested_contexts(self):
+        return self._count_notices(self.get_current_context(), level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
+
+    def count_successes_for_current_context(self):
+        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
+
+    def count_successes_for_current_and_nested_contexts(self):
+        return self._count_notices(self.get_current_context(), level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
+
+    def export_notices_to_gcs_file(self, bucket_name, storage_client, file_prefix=None, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+
+        def log_error(message, exc_info=False):
+            if logger:
+                logger.error(message, exc_info=exc_info)
+
+        if not file_prefix:
+            file_prefix = self._category
+        if not file_name:
+            timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+            if top_level_context:
+                file_name = f"{file_prefix}_{timestamp}_{top_level_context}_len{len(self._notices)}.json"
+            else:
+                file_name = f"{file_prefix}_{timestamp}_len{len(self._notices)}.json"
+
+        cloud_path = None
+        local_path = None
+        try:
+            cloud_path, local_path = write_data_to_gcs(
+                bucket_name=bucket_name,
+                storage_client=storage_client,
+                data=self._notices,
+                file_name=file_name,
+                save_locally=save_locally,
+                local_path=local_path,
+                logger=logger,
+                max_retries=max_retries
+            )
+            log_message(f"{file_prefix} successfully saved to GCS at {cloud_path} and locally at {local_path}.")
+        except Exception as e:
+            log_error(f"Failed at export_notices_to_gcs_file for {file_prefix} for file {file_name} to bucket {bucket_name}: {type(e).__name__} - {str(e)}")
+
+        return cloud_path, local_path
+
+    def import_notices_from_json(self, json_or_file, logger=None):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+
+        def log_warning(message, exc_info=False):
+            if logger:
+                logger.warning(message, exc_info=exc_info)
+
+        try:
+            if isinstance(json_or_file, str):  # Load from string
+                imported_notices = json.loads(json_or_file)
+            elif hasattr(json_or_file, 'read'):  # Load from file-like object
+                imported_notices = json.load(json_or_file)
+            self.add_notices(imported_notices)
+            log_message("Successfully imported notices from json.")
+        except Exception as e:
+            log_warning(f"Failed to import notices from json: {type(e).__name__} - {str(e)}", exc_info=True)
+
+    def _update_counts(self, notice, remove=False):
+        level_code = notice["level_code"]
+        level_name = notice["level_name"]
+
+        if remove:
+            if level_code >= self.ERROR_CODE_START_VALUE:
+                self._error_count -= 1
+            elif level_code >= self.WARNING_CODE_START_VALUE:
+                self._warning_count -= 1
+            elif level_code >= self.SUCCESS_CODE_START_VALUE:
+                self._success_count -= 1
+            self._level_counts[level_name] -= 1
+        else:
+            if level_code >= self.ERROR_CODE_START_VALUE:
+                self._error_count += 1
+            elif level_code >= self.WARNING_CODE_START_VALUE:
+                self._warning_count += 1
+            elif level_code == self.SUCCESS_CODE_START_VALUE:
+                self._success_count += 1
+            self._level_counts[level_name] += 1

{ipulse_shared_core_ftredge-2.39 → ipulse_shared_core_ftredge-2.51}/src/ipulse_shared_core_ftredge/utils_gcp.py

@@ -1,8 +1,12 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
 import json
 import csv
 from io import StringIO
 import logging
 import os
+import time
 import traceback
 from google.cloud import error_reporting, logging as cloud_logging
 from google.api_core.exceptions import NotFound
@@ -22,7 +26,7 @@ from google.api_core.exceptions import NotFound
 ## TODO Fix the issue with POST 0B Nan.... printed in Cloud Logging , which is referring to posting to Cloud Logging probably.
 ENV = os.getenv('ENV', 'LOCAL').strip("'")
 
-def setup_gcp_logger_and_error_report(logger_name):
+def setup_gcp_logger_and_error_report(logger_name,level=logging.INFO, use_cloud_logging=True):
     """Sets up a logger with Error Reporting and Cloud Logging handlers.
 
     Args:
@@ -52,26 +56,26 @@ def setup_gcp_logger_and_error_report(logger_name):
                 self.handleError(record)
 
     logger = logging.getLogger(logger_name)
-    logger.setLevel(logging.INFO)
-
-    # Create Error Reporting handler
-    error_reporting_handler = ErrorReportingHandler()
-
-    # Create Google Cloud Logging handler
-    cloud_logging_client = cloud_logging.Client()
-    cloud_logging_handler = cloud_logging_client.get_default_handler()
-
-    # Add handlers to the logger
-    logger.addHandler(error_reporting_handler)
-    logger.addHandler(cloud_logging_handler)
+    logger.setLevel(level)
 
     # Add a console handler for local development
-    if ENV == "LOCAL":
+    if ENV == "LOCAL" or not use_cloud_logging:
         formatter = logging.Formatter('%(levelname)s : %(name)s : %(asctime)s : %(message)s')
         console_handler = logging.StreamHandler()
         console_handler.setFormatter(formatter)
         logger.addHandler(console_handler)
 
+    if use_cloud_logging:
+        # Create Error Reporting handler
+        error_reporting_handler = ErrorReportingHandler()
+
+        # Create Google Cloud Logging handler
+        cloud_logging_client = cloud_logging.Client()
+        cloud_logging_handler = cloud_logging_client.get_default_handler()
+
+        # Add handlers to the logger
+        logger.addHandler(error_reporting_handler)
+        logger.addHandler(cloud_logging_handler)
     return logger
 ############################################################################
 
@@ -116,21 +120,93 @@ def read_csv_from_gcs(bucket_name, file_name, storage_client, logger):
         logger.error(f"An unexpected error occurred: {e}", exc_info=True)
         return None
 
-def write_json_to_gcs(bucket_name, file_name, data, stor_client, logger, log_info_verbose=True):
-    """ Helper function to write a JSON file to Google Cloud Storage """
-    try:
-        bucket = stor_client.bucket(bucket_name)
-        blob = bucket.blob(file_name)
-        # Check if data is already a JSON string
-        if isinstance(data, str):
-            data_string = data
-        else:
-            data_string = json.dumps(data)
-        blob.upload_from_string(data_string, content_type='application/json')
-        if log_info_verbose:
-            logger.info(f"Successfully wrote JSON to {file_name} in bucket {bucket_name}.")
-    except Exception as e:
-        logger.error(f"An unexpected error occurred while writing JSON to GCS: {e}", exc_info=True)
+
+
+def write_data_to_gcs(bucket_name, storage_client, data, file_name=None, 
+                      save_locally=False, local_path=None, logger=None, max_retries=3):
+    """Saves data to Google Cloud Storage and optionally locally.
+
+    This function attempts to upload data to GCS. If the upload fails after
+    retries and `save_locally` is True or `local_path` is provided, it attempts
+    to save the data locally.
+
+    Args:
+        bucket_name (str): Name of the GCS bucket.
+        storage_client (google.cloud.storage.Client): GCS client object.
+        data (list, dict, or str): Data to be saved.
+        file_name (str, optional): File name for GCS and local. Defaults to None.
+        save_locally (bool, optional): Save locally if GCS fails. Defaults to False.
+        local_path (str, optional): Local directory to save. Defaults to None.
+        logger (logging.Logger, optional): Logger for messages. Defaults to None.
+        max_retries (int, optional): Number of GCS upload retries. Defaults to 3.
+
+    Returns:
+        tuple: A tuple containing the GCS path (or None if upload failed) and
+               the local path (or None if not saved locally).
+
+    Raises:
+        ValueError: If data is not a list, dict, or str.
+        Exception: If GCS upload fails after retries and local saving fails or
+                   is not requested. If GCS upload fails after retries and
+                   local saving is requested but unsuccessful.
+    """
+
+    def log_message(message):
+        if logger:
+            logger.info(message)
+
+    def log_error(message, exc_info=False):
+        if logger:
+            logger.error(message, exc_info=exc_info)
+
+    attempts = 0
+    success = False
+    cloud_path = None
+    local_path_final = None
+    gcs_upload_exception = None  # Store potential GCS exception
+
+    if isinstance(data, (list, dict)):
+        data_str = json.dumps(data, indent=2)
+    elif isinstance(data, str):
+        data_str = data
+    else:
+        raise ValueError("Unsupported data type. It should be a list, dict, or str.")
+
+    while attempts < max_retries and not success:
+        try:
+            bucket = storage_client.bucket(bucket_name)
+            blob = bucket.blob(file_name)
+            blob.upload_from_string(data_str, content_type='application/json')
+            cloud_path = f"{bucket_name}/{file_name}"
+            log_message(f"Successfully saved file to GCS {cloud_path}.")
+            success = True
+        except Exception as e:
+            gcs_upload_exception = e
+            attempts += 1
+            if attempts < max_retries:
+                time.sleep(2 ** attempts)
+            else:
+                log_error(f"Failed to write {file_name} to GCS bucket {bucket_name} after {max_retries} attempts :{e}")
+
+    if not success and (save_locally or local_path):
+        try:
+            if not local_path:
+                local_path_final = os.path.join("/tmp", file_name)
+            else:
+                local_path_final = os.path.join(local_path, file_name)
+            with open(local_path_final, 'w', encoding='utf-8') as f:
+                f.write(data_str)
+            log_message(f"Saved {file_name} locally at {local_path_final}.")
+        except Exception as local_e:
+            log_error(f"Failed to write {file_name} locally: {local_e}",exc_info=True)
+
+        # If GCS upload failed, raise a single exception here
+
+    if gcs_upload_exception is not None:
+        raise gcs_upload_exception # Propagate without nesting
+
+    return cloud_path, local_path_final
+
 
 def write_csv_to_gcs(bucket_name, file_name, data, storage_client, logger,log_info_verbose=True):
     """ Helper function to write a CSV file to Google Cloud Storage """
@@ -151,4 +227,4 @@ def write_csv_to_gcs(bucket_name, file_name, data, storage_client, logger,log_in
     except ValueError as e:
         logger.error(f"ValueError: {e}")
     except Exception as e:
-        logger.error(f"An unexpected error occurred while writing CSV to GCS: {e}", exc_info=True)
+     logger.error(f"An unexpected error occurred while writing CSV to GCS: {e}", exc_info=True)

{ipulse_shared_core_ftredge-2.39 → ipulse_shared_core_ftredge-2.51}/src/ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -5,8 +5,8 @@
 
 import datetime
 from google.cloud import bigquery
-from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeSeverity
-from ipulse_shared_core_ftredge.utils_common import create_custom_notice
+from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeLevel
+from ipulse_shared_core_ftredge.utils_common import Notice
 
 
 def create_bigquery_schema_from_json(json_schema):
@@ -23,14 +23,14 @@ def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_
 
     """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
     valid_updates = {}
-    notices=[] ### THIS IS TO AVOID LOGGING A WARNING RANDOMLY , INSTEAD GROUPPING FOR A GIVEN RUN
+    notices=[] ### THIS IS TO AVOID LOGGING A WARNING RANDOMLY, INSTEAD GROUPPING FOR A GIVEN RUN
 
     # Process updates to conform to the schema
     for field in schema:
         field_name = field["name"]
         field_type = field["type"]
         mode = field["mode"]
-        
+
         # Initialize notice to None at the start of each field processing
         notice = None
 
@@ -38,7 +38,7 @@ def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_
             value = updates[field_name]
 
             # Handle date and timestamp formatting
-            
+
             # Validate and potentially convert date and timestamp fields
             if field_type == "DATE":
                 value, notice = handle_date_fields(field_name, value, dt_ts_to_str)
@@ -55,18 +55,18 @@ def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_
                 value,notice = check_and_truncate_length(field_name, value, field["max_length"])
                 if notice:
                     notices.append(notice)
-            
+
             # Only add to the dictionary if value is not None or the field is required
             if value is not None or mode == "REQUIRED":
                 valid_updates[field_name] = value
 
         elif mode == "REQUIRED":
-            notice=create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+            notice=Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                              subject=field_name,
-                             message=f"Required field '{field_name}' is missing in the updates.")
+                             description=f"Required field '{field_name}' is missing in the updates.")
 
             notices.append(notice)
-      
+
     return valid_updates, notices
 
 def handle_date_fields(field_name, value, dt_ts_to_str):
@@ -82,13 +82,13 @@ def handle_date_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_date, None
         except ValueError:
-            return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
-                                                   message=f"Expected a DATE in YYYY-MM-DD format but got {value}.")
+                                                   description=f"Expected a DATE in YYYY-MM-DD format but got {value}.")
     else:
-        return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+        return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                                               subject=field_name,
-                                              message= f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
+                                              description= f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
 
 
 def handle_timestamp_fields(field_name, value, dt_ts_to_str):
@@ -104,47 +104,49 @@ def handle_timestamp_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_datetime, None
         except ValueError:
-            return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
-                                                  message= f"Expected ISO format TIMESTAMP but got {value}.")
+                                                  description= f"Expected ISO format TIMESTAMP but got {value}.")
     else:
-        return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+        return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                                               subject=field_name,
-                                              message= f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
+                                              description= f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
 
 
 def check_and_truncate_length(field_name, value, max_length):
     """Checks and truncates the length of string fields if they exceed the max length."""
     if isinstance(value, str) and len(value) > max_length:
-        return value[:max_length], create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_RECOMMENDED, 
+        return value[:max_length], Notice(level=NoticeLevel.WARNING_FIX_RECOMMENDED,
                             subject= field_name,
-                             message= f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
-   
+                             description= f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
+
     return value, None
 
 
 
 def handle_type_conversion(field_type, field_name, value):
     if field_type == "STRING" and not isinstance(value, str):
-        return str(value), create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED, 
+        return str(value), Notice(level=NoticeLevel.WARNING_REVIEW_RECOMMENDED,
                              subject=field_name,
-                             message= f"Expected STRING but got {value} of type {type(value).__name__}.")
+                             description= f"Expected STRING but got {value} of type {type(value).__name__}.")
 
     if field_type == "INT64" and not isinstance(value, int):
         try:
             return int(value), None
         except ValueError:
-            return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED,
                                                 subject= field_name,
-                                                message=f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
+                                                description=f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
     if field_type == "FLOAT64" and not isinstance(value, float):
         try:
             return float(value), None
         except ValueError:
-            return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED, 
-                                                  subject=field_name,
-                                                   message=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
+            return None, Notice(level=NoticeLevel.WARNING_FIX_REQUIRED, 
+                                                subject=field_name,
+                                                description=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
     if field_type == "BOOL" and not isinstance(value, bool):
-        return bool(value), None
-    
-    return value, None
+        return bool(value), Notice(level=NoticeLevel.WARNING_REVIEW_RECOMMENDED, 
+                                                subject=field_name,
+                                                description=f"Expected BOOL, but got  {value}. Converting as {bool(value)}.")
+
+    return value, None

{ipulse_shared_core_ftredge-2.39 → ipulse_shared_core_ftredge-2.51/src/ipulse_shared_core_ftredge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.39
+Version: 2.51
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

ipulse_shared_core_ftredge-2.39/src/ipulse_shared_core_ftredge/utils_common.py

@@ -1,24 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=logging-fstring-interpolation
-# pylint: disable=line-too-long
-
-def create_custom_notice(severity, subject, message):
-    return {
-        "severity_code": severity.value,
-        "severity_name": severity.name,
-        "subject": subject,
-        "message": message
-    }
-
-
-def create_exception_notice(severity, exception_code,exception_message, exception_traceback=None, subject=None, message=None):
-    return {
-        "severity_code": severity.value,
-        "severity_name": severity.name,
-        "subject": subject,
-        "message": message,
-        "exception_code": exception_code,
-        "exception_message": exception_message,
-        "exception_traceback": exception_traceback
-    }