ipulse-shared-core-ftredge 2.38__tar.gz → 2.50__tar.gz

{ipulse_shared_core_ftredge-2.38/src/ipulse_shared_core_ftredge.egg-info → ipulse_shared_core_ftredge-2.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.38
+Version: 2.50
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.38 → ipulse_shared_core_ftredge-2.50}/setup.py

@@ -1,8 +1,9 @@
+# pylint: disable=import-error
 from setuptools import setup, find_packages
 
 setup(
     name='ipulse_shared_core_ftredge',
-    version='2.38',
+    version='2.50',
     package_dir={'': 'src'},  # Specify the source directory
     packages=find_packages(where='src'),  # Look for packages in 'src'
     install_requires=[

{ipulse_shared_core_ftredge-2.38 → ipulse_shared_core_ftredge-2.50}/src/ipulse_shared_core_ftredge/__init__.py

@@ -2,15 +2,14 @@ from .models import (Organisation, UserAuth, UserProfile,
                      UserStatus, UserProfileUpdate, pulse_enums)
 from .utils_gcp import (setup_gcp_logger_and_error_report, 
                         read_csv_from_gcs, read_json_from_gcs, 
-                        write_csv_to_gcs, write_json_to_gcs)
+                        write_csv_to_gcs, write_data_to_gcs)
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
-                                          update_check_with_schema_template,
-                                          create_data_check_notice)
+                                          update_check_with_schema_template)
+from .utils_common import (Notice, NoticeSeverity, NoticesManager,SuccessLog, SuccessLogManager)
+
 from .enums import (NoticeSeverity, Unit, Frequency,
                     Module, SubModule, BaseDataCategory,
                     FinCoreCategory, FincCoreSubCategory,
                     FinCoreRecordsCategory, ExchangeOrPublisher,
                     SourcingPipelineType, SourcingTriggerType,
                     DWEvent, DWEventTriggerType)
-
-                    

{ipulse_shared_core_ftredge-2.38 → ipulse_shared_core_ftredge-2.50}/src/ipulse_shared_core_ftredge/enums/enums_common_utils.py

@@ -17,20 +17,31 @@ class NoticeSeverity(Enum):
 
      # Warnings indicate potential issues that might require attention:
     WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
-    WARNING_ACTION_RECOMMENDED = 402 # Action recommended to prevent potential future issues
-    WARNING_ACTION_REQUIRED = 403  # Action required, pipeline can likely continue
+    WARNING_REVIEW_RECOMMENDED = 402 # Action recommended to prevent potential future issues
+    WARNING_FIX_RECOMMENDED = 403 # Action recommended to prevent potential future issues
+    WARNING_FIX_REQUIRED = 404  # Action required, pipeline can likely continue
 
     # Errors indicate a problem that disrupts normal pipeline execution:
-    ERROR_TRANSIENT_RETRY = 501 # Temporary error, automatic retry likely to succeed
-    ERROR_DATA_ISSUE_ISOLATED = 502 # Error likely caused by data issues, manual intervention likely needed
-    ERROR_DATA_ISSUE_WITH_DEPENDENCIES = 503 # Error likely in code/configuration, investigation required
-    ERROR_CONFIG_OR_CODE_ISSUE = 504 # Error likely in code/configuration, investigation required
-    ERROR_UNKNOWN_EXCEPTION = 505
+    ERROR_EXCEPTION_REDO = 502
+    ERROR_CUSTOM_REDO = 503 # Temporary error, automatic retry likely to succeed
+    
+
+    ERROR_EXCEPTION_INVESTIGATE = 601 # Exception occured after some data was likely persisted (e.g., to GCS or BQ)
+    ERROR_CUSTOM_INVESTIGATE= 602
+    ERROR_EXCEPTION_PERSTISTANCE = 603 # Exception occured after data was persisted (e.g., to GCS or BQ)
+    ERROR_CUSTOM_PERSTISTANCE = 604
 
     # Critical errors indicate severe failures requiring immediate attention:
-    CRITICAL_SYSTEM_FAILURE = 601 # System-level failure (e.g., infrastructure), requires immediate action
-    CRITICAL_PIPELINE_FAILURE = 602 # Complete pipeline failure, requires investigation and potential rollback
+    CRITICAL_SYSTEM_FAILURE = 701 # System-level failure (e.g., infrastructure), requires immediate action
+    CRITICAL_PIPELINE_FAILURE = 702 # Complete pipeline failure, requires investigation and potential rollback
+
+    UNKNOWN=1001 # Unknown error, should not be used in normal operation
 
+### Exception during full exection, partially saved
+# Exception during ensemble pipeline; modifications collected in local object , nothing persisted
+# Exception during ensemble pipeline; modifications persisted , metadata failed
+# Exception during ensemble pipeline; modifications persisted , metadata persisted
+# Exception during ensemble pipeline; modifications persisted , metadata persisted
 
 
 class Unit(Enum):

{ipulse_shared_core_ftredge-2.38 → ipulse_shared_core_ftredge-2.50}/src/ipulse_shared_core_ftredge/enums/enums_data_eng.py

@@ -20,25 +20,24 @@ class SourcingPipelineType(Enum):
     CLOUD_GET_API_INMEMORY = "cloud_get_api_inmemory"
 
 class DWEventTriggerType(Enum):
-    GCS_BUCKET_UPLOAD = "gcs_bucket_upload"
+    GCS_UPLOAD_TRIGGER_CF = "gcs_upload_trigger_cf"
+    HTTP_TRIGGER_CF_FOR_GCS_FILE = "http_trigger_cf_for_gcs_file"
+    PUBSUB_TRIGGER_CF_FOR_GCS_FILE = "pubsub_trigger_cf_for_gcs_file"
+    LOCAL_SCRIPT_FOR_GCS_FILE = "local_script_for_gcs_file"
     INSIDE_SOURCING_FUNCTION = "inside_sourcing_function"
-    HTTP_FUNC_TO_GCS = "http_func_to_gcs"
-    LOCAL_FROM_GCS_FILE = "local_from_gcs_file"
-    MANUAL_FROM_LOCAL_FILE = "manual_from_local_file"
-    PUBSUBC_TOPIC = "pubsubc_topic"
 
 class DWEvent(Enum):
-    INSERT_NOREPLACE_1A_NT = "insert_noreplace_1a_nt"
-    MERGE_NOREPLACE_NA_1T = "merge_noreplace_na_1t"
-    MERGE_NOREPLACE_NA_NT = "merge_noreplace_na_nt"
-    INSERT_NOREPLACE_1A_1T = "insert_noreplace_1a_1t"
-    MERGE_NOREPLACE_1A_NT = "merge_noreplace_1a_nt"
-    INSERT_REPLACE_1A_1T = "insert_replace_1a_1t"
-    INSERT_REPLACE_1A_NT = "insert_replace_1a_nt"
-    MERGE_REPLACE_NA_NT = "merge_replace_na_nt"
-    MERGE_REPLACE_1A_NT = "merge_replace_1a_nt"
-    MERGE_REPLACE_NA_1T = "merge_replace_na_1t"
-    DELETE_1A_1T = "delete_1a_1t"
-    DELETE_1A_NT = "delete_1a_nt"
-    DELETE_NA_1T = "delete_na_1t"
-    DELETE_NA_NT = "delete_na_nt"
+    INSERT_NOREPLACE_1O_NT = "insert_noreplace_1o_nt"
+    MERGE_NOREPLACE_NO_1T = "merge_noreplace_no_1t"
+    MERGE_NOREPLACE_NO_NT = "merge_noreplace_no_nt"
+    INSERT_NOREPLACE_1O_1T = "insert_noreplace_1o_1t"
+    MERGE_NOREPLACE_1O_NT = "merge_noreplace_1o_nt"
+    INSERT_REPLACE_1O_1T = "insert_replace_1o_1t"
+    INSERT_REPLACE_1O_NT = "insert_replace_1o_nt"
+    MERGE_REPLACE_NO_NT = "merge_replace_no_nt"
+    MERGE_REPLACE_1O_NT = "merge_replace_1o_nt"
+    MERGE_REPLACE_NO_1T = "merge_replace_no_1t"
+    DELETE_1O_1T = "delete_1o_1t"
+    DELETE_1O_NT = "delete_1o_nt"
+    DELETE_NO_1T = "delete_no_1t"
+    DELETE_NO_NT = "delete_no_nt"

ipulse_shared_core_ftredge-2.50/src/ipulse_shared_core_ftredge/utils_common.py

@@ -0,0 +1,415 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+import traceback
+import json
+import os
+import time
+from datetime import datetime, timezone
+from contextlib import contextmanager
+from typing import List
+from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeSeverity
+from ipulse_shared_core_ftredge.utils_gcp import write_data_to_gcs
+
+def create_notice(severity, e=None, e_type=None, e_message=None, e_traceback=None, subject=None, message=None,context=None):
+    # Validate input: ensure severity is provided, use a default if not
+    if severity is None:
+        severity = NoticeSeverity.UNKNOWN  # Assume Severity.UNKNOWN is a default fallback
+
+    # If an exception object is provided, use it to extract details
+    if e is not None:
+        e_type = type(e).__name__ if e_type is None else e_type
+        e_message = str(e) if e_message is None else e_message
+        e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
+    else:
+        # Calculate traceback if not provided and if exception details are partially present
+        if e_traceback is None and (e_type or e_message):
+            e_traceback = traceback.format_exc()
+
+    # Prepare the base notice dictionary with all fields
+    notice = {
+        "severity_code": severity.value,
+        "severity_name": severity.name,
+        "subject": subject,
+        "message": message,
+        "exception_code": e_type,
+        "exception_message": e_message,
+        "exception_traceback": e_traceback or None,  # Ensure field is present even if traceback isn't calculated
+        "context": context or ""
+    }
+    return notice
+
+
+
+
+def merge_notices_dicts(dict1, dict2):
+    """
+    Merge two dictionaries of lists, combining lists for overlapping keys.
+
+    Parameters:
+    dict1 (dict): The first dictionary of lists.
+    dict2 (dict): The second dictionary of lists.
+
+    Returns:
+    dict: A new dictionary with combined lists for overlapping keys.
+    """
+    merged_dict = {}
+
+    # Get all unique keys from both dictionaries
+    all_keys = set(dict1) | set(dict2)
+
+    for key in all_keys:
+        # Combine lists from both dictionaries for each key
+        merged_dict[key] = dict1.get(key, []) + dict2.get(key, [])
+
+    return merged_dict
+
+
+# ["data_import","data_quality", "data_processing","data_general","data_persistance","metadata_quality", "metadata_processing", "metadata_persistance","metadata_general"]
+
+class Notice:
+    def __init__(self, severity: NoticeSeverity, e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None, subject: str = None, message: str = None, context: str = None):
+   
+        # If an exception object is provided, use it to extract details
+        if e is not None:
+            e_type = type(e).__name__ if e_type is None else e_type
+            e_message = str(e) if e_message is None else e_message
+            e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
+         # If exception details are provided but not from an exception object
+        elif e_traceback is None and (e_type or e_message):
+            e_traceback = traceback.format_exc()
+
+        self.timestamp = datetime.now(timezone.utc).isoformat()
+        self.severity = severity
+        self.subject = subject
+        self.message = message
+        self.context = context
+        self.exception_type = e_type
+        self.exception_message = e_message
+        self.exception_traceback = e_traceback
+
+    def to_dict(self):
+        return {
+            "context": self.context,
+            "severity_code": self.severity.value,
+            "severity_name": self.severity.name,
+            "subject": self.subject,
+            "message": self.message,
+            "exception_type": self.exception_type,
+            "exception_message": self.exception_message,
+            "exception_traceback": self.exception_traceback,
+        }
+
+class NoticesManager:
+    ERROR_CODE_START_VALUE = 500
+
+    def __init__(self):
+        self.notices = []
+        self.error_count = 0
+        self.severity_counts = {severity.name: 0 for severity in NoticeSeverity}
+        self.context_stack = []
+
+    @contextmanager
+    def notice_context(self, context):
+        self.push_context(context)
+        try:
+            yield
+        finally:
+            self.pop_context()
+
+    def push_context(self, context):
+        self.context_stack.append(context)
+
+    def pop_context(self):
+        if self.context_stack:
+            self.context_stack.pop()
+
+    def get_notices_by_context(self, context_substring: str):
+        return [
+            notice for notice in self.notices
+            if context_substring in notice["context"]
+        ]
+
+    def get_current_context(self):
+        return " >> ".join(self.context_stack)
+
+    def get_all_notices(self):
+        return self.notices
+    def add_notice(self, notice: Notice):
+        notice.context = self.get_current_context()
+        notice_dict = notice.to_dict()
+        self.notices.append(notice_dict)
+        self._update_counts(notice_dict)
+
+    def add_notices(self, notices: List[Notice]):
+        for notice in notices:
+            notice.context = self.get_current_context()
+            notice_dict = notice.to_dict()
+            self.notices.append(notice_dict)
+            self._update_counts(notice_dict)
+   
+    def remove_notice(self, notice: Notice):
+        notice_dict = notice.to_dict()
+        if notice_dict in self.notices:
+            self.notices.remove(notice_dict)
+            self._update_counts(notice_dict, remove=True)
+
+    def clear_notices(self):
+        self.notices = []
+        self.error_count = 0
+        self.severity_counts = {severity.name: 0 for severity in NoticeSeverity}
+
+    def contains_errors(self):
+        return self.error_count > 0
+
+    def count_errors(self):
+        return self.error_count
+
+    def count_notices_by_severity(self, severity: NoticeSeverity):
+        return self.severity_counts.get(severity.name, 0)
+    
+    def count_errors_for_current_context(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if notice["context"] == current_context and notice["severity_code"] >= self.ERROR_CODE_START_VALUE
+        )
+    def count_all_notices(self):
+        return len(self.notices)
+
+    def count_notices_for_current_context(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if notice["context"] == current_context
+        )
+    
+    def count_notices_by_severity_for_current_context(self, severity: NoticeSeverity):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if notice["context"] == current_context and notice["severity_code"] == severity.value
+        )
+    def count_notices_for_current_and_nested_contexts(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if current_context in notice["context"]
+        )
+    def count_errors_for_current_and_nested_contexts(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if current_context in notice["context"] and notice["severity_code"] >= self.ERROR_CODE_START_VALUE
+        )
+    def count_notices_by_severity_for_current_and_nested_contexts(self, severity: NoticeSeverity):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if current_context in notice["context"] and notice["severity_code"] == severity.value
+        )
+    
+    def export_notices_to_gcs_file(self, bucket_name, storage_client, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+
+        def log_error(message, exc_info=False):
+            if logger:
+                logger.error(message, exc_info=exc_info)
+
+        if not file_name:
+            timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+            if top_level_context:
+                file_name = f"notices_{timestamp}_{top_level_context}_len{len(self.notices)}.json"
+            else:
+                file_name = f"notices_{timestamp}_len{len(self.notices)}.json"
+
+        cloud_path = None  # Initialize cloud_path here
+        local_path = None  # Initialize local_path here
+        try:
+            cloud_path, local_path = write_data_to_gcs(
+                bucket_name=bucket_name,
+                storage_client=storage_client,
+                data=self.notices,
+                file_name=file_name,
+                save_locally=save_locally,
+                local_path=local_path,
+                logger=logger,
+                max_retries=max_retries
+            )
+            log_message(f"Notices successfully saved to GCS at {cloud_path} and locally at {local_path}.")
+        except Exception as e:
+            log_error(f"Failed to export notices: {type(e).__name__} - {str(e)}", exc_info=True)
+
+        return cloud_path , local_path
+
+    def import_notices_from_json(self, json_or_file, logger=None):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+            else:
+                print(message)
+
+        def log_error(message, exc_info=False):
+            if logger:
+                logger.error(message, exc_info=exc_info)
+            else:
+                print(message)
+        try:
+            if isinstance(json_or_file, str):  # Load from string
+                imported_notices = json.loads(json_or_file)
+            elif hasattr(json_or_file, 'read'):  # Load from file-like object
+                imported_notices = json.load(json_or_file)
+            self.add_notice(imported_notices)
+            log_message("Successfully imported notices from json.")
+        except Exception as e:
+            log_error(f"Failed to import notices from json: {type(e).__name__} - {str(e)}", exc_info=True)
+
+    def _update_counts(self, notice, remove=False):
+        if remove:
+            if notice["severity_code"] >= self.ERROR_CODE_START_VALUE:
+                self.error_count -= 1
+            self.severity_counts[notice["severity_name"]] -= 1
+        else:
+            if notice["severity_code"] >= self.ERROR_CODE_START_VALUE:
+                self.error_count += 1
+            self.severity_counts[notice["severity_name"]] += 1
+
+
+class SuccessLog:
+    def __init__(self, subject:str, description:str=None, context:str=None):
+        self.context = context
+        self.subject = subject
+        self.timestamp = datetime.now(timezone.utc).isoformat()
+        self.description = description
+
+    def to_dict(self):
+        return {
+            "context": self.context or "",
+            "subject": self.subject,
+            "timestamp": self.timestamp,
+            "description": self.description or ""
+        }
+
+
+class SuccessLogManager:
+    def __init__(self):
+        self.successlogs = []
+        self.context_stack = []
+
+    @contextmanager
+    def successlog_context(self, context):
+        self.push_context(context)
+        try:
+            yield
+        finally:
+            self.pop_context()
+
+    def push_context(self, context):
+        self.context_stack.append(context)
+
+    def pop_context(self):
+        if self.context_stack:
+            self.context_stack.pop()
+
+    def get_current_context(self):
+        return " >> ".join(self.context_stack)
+
+    def get_all_successlogs(self):
+        return self.successlogs
+
+    def add_successlog(self, successlog: SuccessLog):
+        successlog.context = self.get_current_context()
+        successlog_dict = successlog.to_dict()
+        self.successlogs.append(successlog_dict)
+
+    def add_successlogs(self, successlogs: List[SuccessLog]):
+        for successlog in successlogs:
+            successlog.context = self.get_current_context()
+            successlog_dict = successlog.to_dict()
+            self.successlogs.append(successlog_dict)
+
+    def remove_successlog(self, successlog: SuccessLog):
+        successlog_dict = successlog.to_dict()
+        if successlog_dict in self.successlogs:
+            self.successlogs.remove(successlog_dict)
+
+    def clear_successlogs(self):
+        self.successlogs = []
+
+    def count_all_successlogs(self):
+        return len(self.successlogs)
+
+    def count_successlogs_for_current_context(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for successlog in self.successlogs
+            if successlog["context"] == current_context
+        )
+
+    def count_successlogs_for_current_and_nested_contexts(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for successlog in self.successlogs
+            if current_context in successlog["context"]
+        )
+
+
+    def export_successlogs_to_gcs_file(self, bucket_name, storage_client, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=3):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+
+        def log_error(message, exc_info=False):
+            if logger:
+                logger.error(message, exc_info=exc_info)
+
+        if not file_name:
+            timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+            if top_level_context:
+                file_name = f"successlogs_{timestamp}_{top_level_context}_len{len(self.successlogs)}.json"
+            else:
+                file_name = f"successlogs_{timestamp}_len{len(self.successlogs)}.json"
+
+        cloud_path=None
+        local_path=None
+        try:
+            cloud_path, local_path = write_data_to_gcs(
+                bucket_name=bucket_name,
+                storage_client=storage_client,
+                data=self.successlogs,
+                file_name=file_name,
+                save_locally=save_locally,
+                local_path=local_path,
+                logger=logger,
+                max_retries=max_retries
+            )
+            log_message(f"Success logs successfully saved to GCS at {cloud_path} and locally at {local_path}.")
+        except Exception as e:
+            log_error(f"Failed to export success logs: {type(e).__name__} - {str(e)}", exc_info=True)
+
+        return cloud_path, local_path
+    
+    def import_successlogs_from_json(self, json_or_file, logger=None):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+            else:
+                print(message)
+
+        def log_error(message, exc_info=False):
+            if logger:
+                logger.error(message, exc_info=exc_info)
+            else:
+                print(message)
+        try:
+            if isinstance(json_or_file, str):  # Load from string
+                imported_success_logs = json.loads(json_or_file)
+            elif hasattr(json_or_file, 'read'):  # Load from file-like object
+                imported_success_logs = json.load(json_or_file)
+            self.add_successlog(imported_success_logs)
+            log_message("Successfully imported success logs from json.")
+        except Exception as e:
+            log_error(f"Failed to import success logs from json: {type(e).__name__} - {str(e)}", exc_info=True)

{ipulse_shared_core_ftredge-2.38 → ipulse_shared_core_ftredge-2.50}/src/ipulse_shared_core_ftredge/utils_gcp.py

@@ -1,8 +1,13 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
 import json
 import csv
 from io import StringIO
 import logging
 import os
+import time
+from datetime import datetime, timezone
 import traceback
 from google.cloud import error_reporting, logging as cloud_logging
 from google.api_core.exceptions import NotFound
@@ -22,7 +27,7 @@ from google.api_core.exceptions import NotFound
 ## TODO Fix the issue with POST 0B Nan.... printed in Cloud Logging , which is referring to posting to Cloud Logging probably.
 ENV = os.getenv('ENV', 'LOCAL').strip("'")
 
-def setup_gcp_logger_and_error_report(logger_name):
+def setup_gcp_logger_and_error_report(logger_name,level=logging.INFO, use_cloud_logging=True):
     """Sets up a logger with Error Reporting and Cloud Logging handlers.
 
     Args:
@@ -52,26 +57,26 @@ def setup_gcp_logger_and_error_report(logger_name):
                 self.handleError(record)
 
     logger = logging.getLogger(logger_name)
-    logger.setLevel(logging.INFO)
-
-    # Create Error Reporting handler
-    error_reporting_handler = ErrorReportingHandler()
-
-    # Create Google Cloud Logging handler
-    cloud_logging_client = cloud_logging.Client()
-    cloud_logging_handler = cloud_logging_client.get_default_handler()
-
-    # Add handlers to the logger
-    logger.addHandler(error_reporting_handler)
-    logger.addHandler(cloud_logging_handler)
+    logger.setLevel(level)
 
     # Add a console handler for local development
-    if ENV == "LOCAL":
+    if ENV == "LOCAL" or not use_cloud_logging:
         formatter = logging.Formatter('%(levelname)s : %(name)s : %(asctime)s : %(message)s')
         console_handler = logging.StreamHandler()
         console_handler.setFormatter(formatter)
         logger.addHandler(console_handler)
 
+    if use_cloud_logging:
+        # Create Error Reporting handler
+        error_reporting_handler = ErrorReportingHandler()
+
+        # Create Google Cloud Logging handler
+        cloud_logging_client = cloud_logging.Client()
+        cloud_logging_handler = cloud_logging_client.get_default_handler()
+
+        # Add handlers to the logger
+        logger.addHandler(error_reporting_handler)
+        logger.addHandler(cloud_logging_handler)
     return logger
 ############################################################################
 
@@ -116,21 +121,93 @@ def read_csv_from_gcs(bucket_name, file_name, storage_client, logger):
         logger.error(f"An unexpected error occurred: {e}", exc_info=True)
         return None
 
-def write_json_to_gcs(bucket_name, file_name, data, stor_client, logger, log_info_verbose=True):
-    """ Helper function to write a JSON file to Google Cloud Storage """
-    try:
-        bucket = stor_client.bucket(bucket_name)
-        blob = bucket.blob(file_name)
-        # Check if data is already a JSON string
-        if isinstance(data, str):
-            data_string = data
-        else:
-            data_string = json.dumps(data)
-        blob.upload_from_string(data_string, content_type='application/json')
-        if log_info_verbose:
-            logger.info(f"Successfully wrote JSON to {file_name} in bucket {bucket_name}.")
-    except Exception as e:
-        logger.error(f"An unexpected error occurred while writing JSON to GCS: {e}", exc_info=True)
+
+
+def write_data_to_gcs(bucket_name, storage_client, data, file_name=None, 
+                      save_locally=False, local_path=None, logger=None, max_retries=3):
+    """Saves data to Google Cloud Storage and optionally locally.
+
+    This function attempts to upload data to GCS. If the upload fails after
+    retries and `save_locally` is True or `local_path` is provided, it attempts
+    to save the data locally.
+
+    Args:
+        bucket_name (str): Name of the GCS bucket.
+        storage_client (google.cloud.storage.Client): GCS client object.
+        data (list, dict, or str): Data to be saved.
+        file_name (str, optional): File name for GCS and local. Defaults to None.
+        save_locally (bool, optional): Save locally if GCS fails. Defaults to False.
+        local_path (str, optional): Local directory to save. Defaults to None.
+        logger (logging.Logger, optional): Logger for messages. Defaults to None.
+        max_retries (int, optional): Number of GCS upload retries. Defaults to 3.
+
+    Returns:
+        tuple: A tuple containing the GCS path (or None if upload failed) and
+               the local path (or None if not saved locally).
+
+    Raises:
+        ValueError: If data is not a list, dict, or str.
+        Exception: If GCS upload fails after retries and local saving fails or
+                   is not requested. If GCS upload fails after retries and
+                   local saving is requested but unsuccessful.
+    """
+
+    def log_message(message):
+        if logger:
+            logger.info(message)
+
+    def log_error(message, exc_info=False):
+        if logger:
+            logger.error(message, exc_info=exc_info)
+
+    attempts = 0
+    success = False
+    cloud_path = None
+    local_path_final = None
+    gcs_upload_exception = None  # Store potential GCS exception
+
+    if isinstance(data, (list, dict)):
+        data_str = json.dumps(data, indent=2)
+    elif isinstance(data, str):
+        data_str = data
+    else:
+        raise ValueError("Unsupported data type. It should be a list, dict, or str.")
+
+    while attempts < max_retries and not success:
+        try:
+            bucket = storage_client.bucket(bucket_name)
+            blob = bucket.blob(file_name)
+            blob.upload_from_string(data_str, content_type='application/json')
+            cloud_path = f"{bucket_name}/{file_name}"
+            log_message(f"Successfully saved file to GCS {cloud_path}.")
+            success = True
+        except Exception as e:
+            gcs_upload_exception = e 
+            attempts += 1
+            log_error(f"Attempt {attempts} - Failed to write {file_name} "
+                        f"to GCS bucket '{bucket_name}': {e}")  # Log with full traceback
+            if attempts < max_retries:
+                time.sleep(2 ** attempts)
+
+    if not success and (save_locally or local_path):
+        try:
+            if not local_path:
+                local_path_final = os.path.join("/tmp", file_name)
+            else:
+                local_path_final = os.path.join(local_path, file_name)
+            with open(local_path_final, 'w', encoding='utf-8') as f:
+                f.write(data_str)
+            log_message(f"Saved {file_name} locally at {local_path_final}.")
+        except Exception as local_e:
+            log_error(f"Failed to write {file_name} locally: {local_e}",exc_info=True)
+
+        # If GCS upload failed, raise a single exception here
+
+        if gcs_upload_exception:
+            raise gcs_upload_exception from None  # Propagate without nesting
+
+    return cloud_path, local_path_final
+
 
 def write_csv_to_gcs(bucket_name, file_name, data, storage_client, logger,log_info_verbose=True):
     """ Helper function to write a CSV file to Google Cloud Storage """
@@ -151,4 +228,4 @@ def write_csv_to_gcs(bucket_name, file_name, data, storage_client, logger,log_in
     except ValueError as e:
         logger.error(f"ValueError: {e}")
     except Exception as e:
-        logger.error(f"An unexpected error occurred while writing CSV to GCS: {e}", exc_info=True)
+     logger.error(f"An unexpected error occurred while writing CSV to GCS: {e}", exc_info=True)

{ipulse_shared_core_ftredge-2.38 → ipulse_shared_core_ftredge-2.50}/src/ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -6,14 +6,8 @@
 import datetime
 from google.cloud import bigquery
 from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeSeverity
+from ipulse_shared_core_ftredge.utils_common import Notice
 
-def create_data_check_notice(severity, field_name, message):
-    return {
-        "severity_code": severity.value,
-        "severity_name": severity.name,
-        "subject": field_name,
-        "message": message
-    }
 
 def create_bigquery_schema_from_json(json_schema):
     schema = []
@@ -29,14 +23,14 @@ def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_
 
     """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
     valid_updates = {}
-    notices=[] ### THIS IS TO AVOID LOGGING A WARNING RANDOMLY , INSTEAD GROUPPING FOR A GIVEN RUN
+    notices=[] ### THIS IS TO AVOID LOGGING A WARNING RANDOMLY, INSTEAD GROUPPING FOR A GIVEN RUN
 
     # Process updates to conform to the schema
     for field in schema:
         field_name = field["name"]
         field_type = field["type"]
         mode = field["mode"]
-        
+
         # Initialize notice to None at the start of each field processing
         notice = None
 
@@ -44,7 +38,7 @@ def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_
             value = updates[field_name]
 
             # Handle date and timestamp formatting
-            
+
             # Validate and potentially convert date and timestamp fields
             if field_type == "DATE":
                 value, notice = handle_date_fields(field_name, value, dt_ts_to_str)
@@ -61,18 +55,18 @@ def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_
                 value,notice = check_and_truncate_length(field_name, value, field["max_length"])
                 if notice:
                     notices.append(notice)
-            
+
             # Only add to the dictionary if value is not None or the field is required
             if value is not None or mode == "REQUIRED":
                 valid_updates[field_name] = value
 
         elif mode == "REQUIRED":
-            notice=create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED,
-                             field_name,
-                             f"Required field '{field_name}' is missing in the updates.")
+            notice=Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+                             subject=field_name,
+                             message=f"Required field '{field_name}' is missing in the updates.")
 
             notices.append(notice)
-      
+
     return valid_updates, notices
 
 def handle_date_fields(field_name, value, dt_ts_to_str):
@@ -88,13 +82,13 @@ def handle_date_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_date, None
         except ValueError:
-            return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED,
-                                                  field_name,
-                                                  f"Expected a DATE in YYYY-MM-DD format but got {value}.")
+            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+                                                  subject=field_name,
+                                                   message=f"Expected a DATE in YYYY-MM-DD format but got {value}.")
     else:
-        return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED,
-                                              field_name,
-                                              f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
+        return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+                                              subject=field_name,
+                                              message= f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
 
 
 def handle_timestamp_fields(field_name, value, dt_ts_to_str):
@@ -110,47 +104,49 @@ def handle_timestamp_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_datetime, None
         except ValueError:
-            return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED,
-                                                  field_name,
-                                                  f"Expected ISO format TIMESTAMP but got {value}.")
+            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+                                                  subject=field_name,
+                                                  message= f"Expected ISO format TIMESTAMP but got {value}.")
     else:
-        return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED,
-                                              field_name,
-                                              f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
+        return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+                                              subject=field_name,
+                                              message= f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
 
 
 def check_and_truncate_length(field_name, value, max_length):
     """Checks and truncates the length of string fields if they exceed the max length."""
     if isinstance(value, str) and len(value) > max_length:
-        return value[:max_length], create_data_check_notice(NoticeSeverity.WARNING_ACTION_RECOMMENDED, 
-                             field_name,
-                             f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
-   
+        return value[:max_length], Notice(severity=NoticeSeverity.WARNING_FIX_RECOMMENDED,
+                            subject= field_name,
+                             message= f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
+
     return value, None
 
 
 
 def handle_type_conversion(field_type, field_name, value):
     if field_type == "STRING" and not isinstance(value, str):
-        return str(value), create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED, 
-                             field_name,
-                             f"Expected STRING but got {value} of type {type(value).__name__}.")
+        return str(value), Notice(severity=NoticeSeverity.WARNING_REVIEW_RECOMMENDED,
+                             subject=field_name,
+                             message= f"Expected STRING but got {value} of type {type(value).__name__}.")
 
     if field_type == "INT64" and not isinstance(value, int):
         try:
             return int(value), None
         except ValueError:
-            return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED,
-                                                   field_name,
-                                                   f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
+            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
+                                                subject= field_name,
+                                                message=f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
     if field_type == "FLOAT64" and not isinstance(value, float):
         try:
             return float(value), None
         except ValueError:
-            return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED, 
-                                                  field_name,
-                                                  f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
+            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED, 
+                                                subject=field_name,
+                                                message=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
     if field_type == "BOOL" and not isinstance(value, bool):
-        return bool(value), None
-    
-    return value, None
+        return bool(value), Notice(severity=NoticeSeverity.WARNING_REVIEW_RECOMMENDED, 
+                                                subject=field_name,
+                                                message=f"Expected BOOL, but got  {value}. Converting as {bool(value)}.")
+
+    return value, None

{ipulse_shared_core_ftredge-2.38 → ipulse_shared_core_ftredge-2.50/src/ipulse_shared_core_ftredge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.38
+Version: 2.50
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.38 → ipulse_shared_core_ftredge-2.50}/src/ipulse_shared_core_ftredge.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ README.md
 pyproject.toml
 setup.py
 src/ipulse_shared_core_ftredge/__init__.py
+src/ipulse_shared_core_ftredge/utils_common.py
 src/ipulse_shared_core_ftredge/utils_gcp.py
 src/ipulse_shared_core_ftredge/utils_templates_and_schemas.py
 src/ipulse_shared_core_ftredge.egg-info/PKG-INFO