ipulse-shared-core-ftredge 2.39__py3-none-any.whl → 2.50__py3-none-any.whl

ipulse_shared_core_ftredge/__init__.py

@@ -2,10 +2,10 @@ from .models import (Organisation, UserAuth, UserProfile,
                      UserStatus, UserProfileUpdate, pulse_enums)
 from .utils_gcp import (setup_gcp_logger_and_error_report, 
                         read_csv_from_gcs, read_json_from_gcs, 
-                        write_csv_to_gcs, write_json_to_gcs)
+                        write_csv_to_gcs, write_data_to_gcs)
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
                                           update_check_with_schema_template)
-from .utils_common import (create_custom_notice, create_exception_notice)
+from .utils_common import (Notice, NoticeSeverity, NoticesManager,SuccessLog, SuccessLogManager)
 
 from .enums import (NoticeSeverity, Unit, Frequency,
                     Module, SubModule, BaseDataCategory,
@@ -13,5 +13,3 @@ from .enums import (NoticeSeverity, Unit, Frequency,
                     FinCoreRecordsCategory, ExchangeOrPublisher,
                     SourcingPipelineType, SourcingTriggerType,
                     DWEvent, DWEventTriggerType)
-
-                    

ipulse_shared_core_ftredge/enums/enums_common_utils.py

@@ -17,20 +17,31 @@ class NoticeSeverity(Enum):
 
      # Warnings indicate potential issues that might require attention:
     WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
-    WARNING_ACTION_RECOMMENDED = 402 # Action recommended to prevent potential future issues
-    WARNING_ACTION_REQUIRED = 403  # Action required, pipeline can likely continue
+    WARNING_REVIEW_RECOMMENDED = 402 # Action recommended to prevent potential future issues
+    WARNING_FIX_RECOMMENDED = 403 # Action recommended to prevent potential future issues
+    WARNING_FIX_REQUIRED = 404  # Action required, pipeline can likely continue
 
     # Errors indicate a problem that disrupts normal pipeline execution:
-    ERROR_TRANSIENT_RETRY = 501 # Temporary error, automatic retry likely to succeed
-    ERROR_DATA_ISSUE_ISOLATED = 502 # Error likely caused by data issues, manual intervention likely needed
-    ERROR_DATA_ISSUE_WITH_DEPENDENCIES = 503 # Error likely in code/configuration, investigation required
-    ERROR_CONFIG_OR_CODE_ISSUE = 504 # Error likely in code/configuration, investigation required
-    ERROR_UNKNOWN_EXCEPTION = 505
+    ERROR_EXCEPTION_REDO = 502
+    ERROR_CUSTOM_REDO = 503 # Temporary error, automatic retry likely to succeed
+    
+
+    ERROR_EXCEPTION_INVESTIGATE = 601 # Exception occured after some data was likely persisted (e.g., to GCS or BQ)
+    ERROR_CUSTOM_INVESTIGATE= 602
+    ERROR_EXCEPTION_PERSTISTANCE = 603 # Exception occured after data was persisted (e.g., to GCS or BQ)
+    ERROR_CUSTOM_PERSTISTANCE = 604
 
     # Critical errors indicate severe failures requiring immediate attention:
-    CRITICAL_SYSTEM_FAILURE = 601 # System-level failure (e.g., infrastructure), requires immediate action
-    CRITICAL_PIPELINE_FAILURE = 602 # Complete pipeline failure, requires investigation and potential rollback
+    CRITICAL_SYSTEM_FAILURE = 701 # System-level failure (e.g., infrastructure), requires immediate action
+    CRITICAL_PIPELINE_FAILURE = 702 # Complete pipeline failure, requires investigation and potential rollback
+
+    UNKNOWN=1001 # Unknown error, should not be used in normal operation
 
+### Exception during full exection, partially saved
+# Exception during ensemble pipeline; modifications collected in local object , nothing persisted
+# Exception during ensemble pipeline; modifications persisted , metadata failed
+# Exception during ensemble pipeline; modifications persisted , metadata persisted
+# Exception during ensemble pipeline; modifications persisted , metadata persisted
 
 
 class Unit(Enum):

ipulse_shared_core_ftredge/enums/enums_data_eng.py

@@ -20,25 +20,24 @@ class SourcingPipelineType(Enum):
     CLOUD_GET_API_INMEMORY = "cloud_get_api_inmemory"
 
 class DWEventTriggerType(Enum):
-    GCS_BUCKET_UPLOAD = "gcs_bucket_upload"
+    GCS_UPLOAD_TRIGGER_CF = "gcs_upload_trigger_cf"
+    HTTP_TRIGGER_CF_FOR_GCS_FILE = "http_trigger_cf_for_gcs_file"
+    PUBSUB_TRIGGER_CF_FOR_GCS_FILE = "pubsub_trigger_cf_for_gcs_file"
+    LOCAL_SCRIPT_FOR_GCS_FILE = "local_script_for_gcs_file"
     INSIDE_SOURCING_FUNCTION = "inside_sourcing_function"
-    HTTP_FUNC_TO_GCS = "http_func_to_gcs"
-    LOCAL_FROM_GCS_FILE = "local_from_gcs_file"
-    MANUAL_FROM_LOCAL_FILE = "manual_from_local_file"
-    PUBSUBC_TOPIC = "pubsubc_topic"
 
 class DWEvent(Enum):
-    INSERT_NOREPLACE_1A_NT = "insert_noreplace_1a_nt"
-    MERGE_NOREPLACE_NA_1T = "merge_noreplace_na_1t"
-    MERGE_NOREPLACE_NA_NT = "merge_noreplace_na_nt"
-    INSERT_NOREPLACE_1A_1T = "insert_noreplace_1a_1t"
-    MERGE_NOREPLACE_1A_NT = "merge_noreplace_1a_nt"
-    INSERT_REPLACE_1A_1T = "insert_replace_1a_1t"
-    INSERT_REPLACE_1A_NT = "insert_replace_1a_nt"
-    MERGE_REPLACE_NA_NT = "merge_replace_na_nt"
-    MERGE_REPLACE_1A_NT = "merge_replace_1a_nt"
-    MERGE_REPLACE_NA_1T = "merge_replace_na_1t"
-    DELETE_1A_1T = "delete_1a_1t"
-    DELETE_1A_NT = "delete_1a_nt"
-    DELETE_NA_1T = "delete_na_1t"
-    DELETE_NA_NT = "delete_na_nt"
+    INSERT_NOREPLACE_1O_NT = "insert_noreplace_1o_nt"
+    MERGE_NOREPLACE_NO_1T = "merge_noreplace_no_1t"
+    MERGE_NOREPLACE_NO_NT = "merge_noreplace_no_nt"
+    INSERT_NOREPLACE_1O_1T = "insert_noreplace_1o_1t"
+    MERGE_NOREPLACE_1O_NT = "merge_noreplace_1o_nt"
+    INSERT_REPLACE_1O_1T = "insert_replace_1o_1t"
+    INSERT_REPLACE_1O_NT = "insert_replace_1o_nt"
+    MERGE_REPLACE_NO_NT = "merge_replace_no_nt"
+    MERGE_REPLACE_1O_NT = "merge_replace_1o_nt"
+    MERGE_REPLACE_NO_1T = "merge_replace_no_1t"
+    DELETE_1O_1T = "delete_1o_1t"
+    DELETE_1O_NT = "delete_1o_nt"
+    DELETE_NO_1T = "delete_no_1t"
+    DELETE_NO_NT = "delete_no_nt"

ipulse_shared_core_ftredge/utils_common.py

@@ -2,23 +2,414 @@
 # pylint: disable=missing-function-docstring
 # pylint: disable=logging-fstring-interpolation
 # pylint: disable=line-too-long
+import traceback
+import json
+import os
+import time
+from datetime import datetime, timezone
+from contextlib import contextmanager
+from typing import List
+from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeSeverity
+from ipulse_shared_core_ftredge.utils_gcp import write_data_to_gcs
 
-def create_custom_notice(severity, subject, message):
-    return {
-        "severity_code": severity.value,
-        "severity_name": severity.name,
-        "subject": subject,
-        "message": message
-    }
+def create_notice(severity, e=None, e_type=None, e_message=None, e_traceback=None, subject=None, message=None,context=None):
+    # Validate input: ensure severity is provided, use a default if not
+    if severity is None:
+        severity = NoticeSeverity.UNKNOWN  # Assume Severity.UNKNOWN is a default fallback
 
+    # If an exception object is provided, use it to extract details
+    if e is not None:
+        e_type = type(e).__name__ if e_type is None else e_type
+        e_message = str(e) if e_message is None else e_message
+        e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
+    else:
+        # Calculate traceback if not provided and if exception details are partially present
+        if e_traceback is None and (e_type or e_message):
+            e_traceback = traceback.format_exc()
 
-def create_exception_notice(severity, exception_code,exception_message, exception_traceback=None, subject=None, message=None):
-    return {
+    # Prepare the base notice dictionary with all fields
+    notice = {
         "severity_code": severity.value,
         "severity_name": severity.name,
         "subject": subject,
         "message": message,
-        "exception_code": exception_code,
-        "exception_message": exception_message,
-        "exception_traceback": exception_traceback
-    }
+        "exception_code": e_type,
+        "exception_message": e_message,
+        "exception_traceback": e_traceback or None,  # Ensure field is present even if traceback isn't calculated
+        "context": context or ""
+    }
+    return notice
+
+
+
+
+def merge_notices_dicts(dict1, dict2):
+    """
+    Merge two dictionaries of lists, combining lists for overlapping keys.
+
+    Parameters:
+    dict1 (dict): The first dictionary of lists.
+    dict2 (dict): The second dictionary of lists.
+
+    Returns:
+    dict: A new dictionary with combined lists for overlapping keys.
+    """
+    merged_dict = {}
+
+    # Get all unique keys from both dictionaries
+    all_keys = set(dict1) | set(dict2)
+
+    for key in all_keys:
+        # Combine lists from both dictionaries for each key
+        merged_dict[key] = dict1.get(key, []) + dict2.get(key, [])
+
+    return merged_dict
+
+
+# ["data_import","data_quality", "data_processing","data_general","data_persistance","metadata_quality", "metadata_processing", "metadata_persistance","metadata_general"]
+
+class Notice:
+    def __init__(self, severity: NoticeSeverity, e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None, subject: str = None, message: str = None, context: str = None):
+   
+        # If an exception object is provided, use it to extract details
+        if e is not None:
+            e_type = type(e).__name__ if e_type is None else e_type
+            e_message = str(e) if e_message is None else e_message
+            e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
+         # If exception details are provided but not from an exception object
+        elif e_traceback is None and (e_type or e_message):
+            e_traceback = traceback.format_exc()
+
+        self.timestamp = datetime.now(timezone.utc).isoformat()
+        self.severity = severity
+        self.subject = subject
+        self.message = message
+        self.context = context
+        self.exception_type = e_type
+        self.exception_message = e_message
+        self.exception_traceback = e_traceback
+
+    def to_dict(self):
+        return {
+            "context": self.context,
+            "severity_code": self.severity.value,
+            "severity_name": self.severity.name,
+            "subject": self.subject,
+            "message": self.message,
+            "exception_type": self.exception_type,
+            "exception_message": self.exception_message,
+            "exception_traceback": self.exception_traceback,
+        }
+
+class NoticesManager:
+    ERROR_CODE_START_VALUE = 500
+
+    def __init__(self):
+        self.notices = []
+        self.error_count = 0
+        self.severity_counts = {severity.name: 0 for severity in NoticeSeverity}
+        self.context_stack = []
+
+    @contextmanager
+    def notice_context(self, context):
+        self.push_context(context)
+        try:
+            yield
+        finally:
+            self.pop_context()
+
+    def push_context(self, context):
+        self.context_stack.append(context)
+
+    def pop_context(self):
+        if self.context_stack:
+            self.context_stack.pop()
+
+    def get_notices_by_context(self, context_substring: str):
+        return [
+            notice for notice in self.notices
+            if context_substring in notice["context"]
+        ]
+
+    def get_current_context(self):
+        return " >> ".join(self.context_stack)
+
+    def get_all_notices(self):
+        return self.notices
+    def add_notice(self, notice: Notice):
+        notice.context = self.get_current_context()
+        notice_dict = notice.to_dict()
+        self.notices.append(notice_dict)
+        self._update_counts(notice_dict)
+
+    def add_notices(self, notices: List[Notice]):
+        for notice in notices:
+            notice.context = self.get_current_context()
+            notice_dict = notice.to_dict()
+            self.notices.append(notice_dict)
+            self._update_counts(notice_dict)
+   
+    def remove_notice(self, notice: Notice):
+        notice_dict = notice.to_dict()
+        if notice_dict in self.notices:
+            self.notices.remove(notice_dict)
+            self._update_counts(notice_dict, remove=True)
+
+    def clear_notices(self):
+        self.notices = []
+        self.error_count = 0
+        self.severity_counts = {severity.name: 0 for severity in NoticeSeverity}
+
+    def contains_errors(self):
+        return self.error_count > 0
+
+    def count_errors(self):
+        return self.error_count
+
+    def count_notices_by_severity(self, severity: NoticeSeverity):
+        return self.severity_counts.get(severity.name, 0)
+    
+    def count_errors_for_current_context(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if notice["context"] == current_context and notice["severity_code"] >= self.ERROR_CODE_START_VALUE
+        )
+    def count_all_notices(self):
+        return len(self.notices)
+
+    def count_notices_for_current_context(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if notice["context"] == current_context
+        )
+    
+    def count_notices_by_severity_for_current_context(self, severity: NoticeSeverity):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if notice["context"] == current_context and notice["severity_code"] == severity.value
+        )
+    def count_notices_for_current_and_nested_contexts(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if current_context in notice["context"]
+        )
+    def count_errors_for_current_and_nested_contexts(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if current_context in notice["context"] and notice["severity_code"] >= self.ERROR_CODE_START_VALUE
+        )
+    def count_notices_by_severity_for_current_and_nested_contexts(self, severity: NoticeSeverity):
+        current_context = self.get_current_context()
+        return sum(
+            1 for notice in self.notices
+            if current_context in notice["context"] and notice["severity_code"] == severity.value
+        )
+    
+    def export_notices_to_gcs_file(self, bucket_name, storage_client, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+
+        def log_error(message, exc_info=False):
+            if logger:
+                logger.error(message, exc_info=exc_info)
+
+        if not file_name:
+            timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+            if top_level_context:
+                file_name = f"notices_{timestamp}_{top_level_context}_len{len(self.notices)}.json"
+            else:
+                file_name = f"notices_{timestamp}_len{len(self.notices)}.json"
+
+        cloud_path = None  # Initialize cloud_path here
+        local_path = None  # Initialize local_path here
+        try:
+            cloud_path, local_path = write_data_to_gcs(
+                bucket_name=bucket_name,
+                storage_client=storage_client,
+                data=self.notices,
+                file_name=file_name,
+                save_locally=save_locally,
+                local_path=local_path,
+                logger=logger,
+                max_retries=max_retries
+            )
+            log_message(f"Notices successfully saved to GCS at {cloud_path} and locally at {local_path}.")
+        except Exception as e:
+            log_error(f"Failed to export notices: {type(e).__name__} - {str(e)}", exc_info=True)
+
+        return cloud_path , local_path
+
+    def import_notices_from_json(self, json_or_file, logger=None):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+            else:
+                print(message)
+
+        def log_error(message, exc_info=False):
+            if logger:
+                logger.error(message, exc_info=exc_info)
+            else:
+                print(message)
+        try:
+            if isinstance(json_or_file, str):  # Load from string
+                imported_notices = json.loads(json_or_file)
+            elif hasattr(json_or_file, 'read'):  # Load from file-like object
+                imported_notices = json.load(json_or_file)
+            self.add_notice(imported_notices)
+            log_message("Successfully imported notices from json.")
+        except Exception as e:
+            log_error(f"Failed to import notices from json: {type(e).__name__} - {str(e)}", exc_info=True)
+
+    def _update_counts(self, notice, remove=False):
+        if remove:
+            if notice["severity_code"] >= self.ERROR_CODE_START_VALUE:
+                self.error_count -= 1
+            self.severity_counts[notice["severity_name"]] -= 1
+        else:
+            if notice["severity_code"] >= self.ERROR_CODE_START_VALUE:
+                self.error_count += 1
+            self.severity_counts[notice["severity_name"]] += 1
+
+
+class SuccessLog:
+    def __init__(self, subject:str, description:str=None, context:str=None):
+        self.context = context
+        self.subject = subject
+        self.timestamp = datetime.now(timezone.utc).isoformat()
+        self.description = description
+
+    def to_dict(self):
+        return {
+            "context": self.context or "",
+            "subject": self.subject,
+            "timestamp": self.timestamp,
+            "description": self.description or ""
+        }
+
+
+class SuccessLogManager:
+    def __init__(self):
+        self.successlogs = []
+        self.context_stack = []
+
+    @contextmanager
+    def successlog_context(self, context):
+        self.push_context(context)
+        try:
+            yield
+        finally:
+            self.pop_context()
+
+    def push_context(self, context):
+        self.context_stack.append(context)
+
+    def pop_context(self):
+        if self.context_stack:
+            self.context_stack.pop()
+
+    def get_current_context(self):
+        return " >> ".join(self.context_stack)
+
+    def get_all_successlogs(self):
+        return self.successlogs
+
+    def add_successlog(self, successlog: SuccessLog):
+        successlog.context = self.get_current_context()
+        successlog_dict = successlog.to_dict()
+        self.successlogs.append(successlog_dict)
+
+    def add_successlogs(self, successlogs: List[SuccessLog]):
+        for successlog in successlogs:
+            successlog.context = self.get_current_context()
+            successlog_dict = successlog.to_dict()
+            self.successlogs.append(successlog_dict)
+
+    def remove_successlog(self, successlog: SuccessLog):
+        successlog_dict = successlog.to_dict()
+        if successlog_dict in self.successlogs:
+            self.successlogs.remove(successlog_dict)
+
+    def clear_successlogs(self):
+        self.successlogs = []
+
+    def count_all_successlogs(self):
+        return len(self.successlogs)
+
+    def count_successlogs_for_current_context(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for successlog in self.successlogs
+            if successlog["context"] == current_context
+        )
+
+    def count_successlogs_for_current_and_nested_contexts(self):
+        current_context = self.get_current_context()
+        return sum(
+            1 for successlog in self.successlogs
+            if current_context in successlog["context"]
+        )
+
+
+    def export_successlogs_to_gcs_file(self, bucket_name, storage_client, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=3):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+
+        def log_error(message, exc_info=False):
+            if logger:
+                logger.error(message, exc_info=exc_info)
+
+        if not file_name:
+            timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+            if top_level_context:
+                file_name = f"successlogs_{timestamp}_{top_level_context}_len{len(self.successlogs)}.json"
+            else:
+                file_name = f"successlogs_{timestamp}_len{len(self.successlogs)}.json"
+
+        cloud_path=None
+        local_path=None
+        try:
+            cloud_path, local_path = write_data_to_gcs(
+                bucket_name=bucket_name,
+                storage_client=storage_client,
+                data=self.successlogs,
+                file_name=file_name,
+                save_locally=save_locally,
+                local_path=local_path,
+                logger=logger,
+                max_retries=max_retries
+            )
+            log_message(f"Success logs successfully saved to GCS at {cloud_path} and locally at {local_path}.")
+        except Exception as e:
+            log_error(f"Failed to export success logs: {type(e).__name__} - {str(e)}", exc_info=True)
+
+        return cloud_path, local_path
+    
+    def import_successlogs_from_json(self, json_or_file, logger=None):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+            else:
+                print(message)
+
+        def log_error(message, exc_info=False):
+            if logger:
+                logger.error(message, exc_info=exc_info)
+            else:
+                print(message)
+        try:
+            if isinstance(json_or_file, str):  # Load from string
+                imported_success_logs = json.loads(json_or_file)
+            elif hasattr(json_or_file, 'read'):  # Load from file-like object
+                imported_success_logs = json.load(json_or_file)
+            self.add_successlog(imported_success_logs)
+            log_message("Successfully imported success logs from json.")
+        except Exception as e:
+            log_error(f"Failed to import success logs from json: {type(e).__name__} - {str(e)}", exc_info=True)

ipulse_shared_core_ftredge/utils_gcp.py

@@ -1,8 +1,13 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
 import json
 import csv
 from io import StringIO
 import logging
 import os
+import time
+from datetime import datetime, timezone
 import traceback
 from google.cloud import error_reporting, logging as cloud_logging
 from google.api_core.exceptions import NotFound
@@ -22,7 +27,7 @@ from google.api_core.exceptions import NotFound
 ## TODO Fix the issue with POST 0B Nan.... printed in Cloud Logging , which is referring to posting to Cloud Logging probably.
 ENV = os.getenv('ENV', 'LOCAL').strip("'")
 
-def setup_gcp_logger_and_error_report(logger_name):
+def setup_gcp_logger_and_error_report(logger_name,level=logging.INFO, use_cloud_logging=True):
     """Sets up a logger with Error Reporting and Cloud Logging handlers.
 
     Args:
@@ -52,26 +57,26 @@ def setup_gcp_logger_and_error_report(logger_name):
                 self.handleError(record)
 
     logger = logging.getLogger(logger_name)
-    logger.setLevel(logging.INFO)
-
-    # Create Error Reporting handler
-    error_reporting_handler = ErrorReportingHandler()
-
-    # Create Google Cloud Logging handler
-    cloud_logging_client = cloud_logging.Client()
-    cloud_logging_handler = cloud_logging_client.get_default_handler()
-
-    # Add handlers to the logger
-    logger.addHandler(error_reporting_handler)
-    logger.addHandler(cloud_logging_handler)
+    logger.setLevel(level)
 
     # Add a console handler for local development
-    if ENV == "LOCAL":
+    if ENV == "LOCAL" or not use_cloud_logging:
         formatter = logging.Formatter('%(levelname)s : %(name)s : %(asctime)s : %(message)s')
         console_handler = logging.StreamHandler()
         console_handler.setFormatter(formatter)
         logger.addHandler(console_handler)
 
+    if use_cloud_logging:
+        # Create Error Reporting handler
+        error_reporting_handler = ErrorReportingHandler()
+
+        # Create Google Cloud Logging handler
+        cloud_logging_client = cloud_logging.Client()
+        cloud_logging_handler = cloud_logging_client.get_default_handler()
+
+        # Add handlers to the logger
+        logger.addHandler(error_reporting_handler)
+        logger.addHandler(cloud_logging_handler)
     return logger
 ############################################################################
 
@@ -116,21 +121,93 @@ def read_csv_from_gcs(bucket_name, file_name, storage_client, logger):
         logger.error(f"An unexpected error occurred: {e}", exc_info=True)
         return None
 
-def write_json_to_gcs(bucket_name, file_name, data, stor_client, logger, log_info_verbose=True):
-    """ Helper function to write a JSON file to Google Cloud Storage """
-    try:
-        bucket = stor_client.bucket(bucket_name)
-        blob = bucket.blob(file_name)
-        # Check if data is already a JSON string
-        if isinstance(data, str):
-            data_string = data
-        else:
-            data_string = json.dumps(data)
-        blob.upload_from_string(data_string, content_type='application/json')
-        if log_info_verbose:
-            logger.info(f"Successfully wrote JSON to {file_name} in bucket {bucket_name}.")
-    except Exception as e:
-        logger.error(f"An unexpected error occurred while writing JSON to GCS: {e}", exc_info=True)
+
+
+def write_data_to_gcs(bucket_name, storage_client, data, file_name=None, 
+                      save_locally=False, local_path=None, logger=None, max_retries=3):
+    """Saves data to Google Cloud Storage and optionally locally.
+
+    This function attempts to upload data to GCS. If the upload fails after
+    retries and `save_locally` is True or `local_path` is provided, it attempts
+    to save the data locally.
+
+    Args:
+        bucket_name (str): Name of the GCS bucket.
+        storage_client (google.cloud.storage.Client): GCS client object.
+        data (list, dict, or str): Data to be saved.
+        file_name (str, optional): File name for GCS and local. Defaults to None.
+        save_locally (bool, optional): Save locally if GCS fails. Defaults to False.
+        local_path (str, optional): Local directory to save. Defaults to None.
+        logger (logging.Logger, optional): Logger for messages. Defaults to None.
+        max_retries (int, optional): Number of GCS upload retries. Defaults to 3.
+
+    Returns:
+        tuple: A tuple containing the GCS path (or None if upload failed) and
+               the local path (or None if not saved locally).
+
+    Raises:
+        ValueError: If data is not a list, dict, or str.
+        Exception: If GCS upload fails after retries and local saving fails or
+                   is not requested. If GCS upload fails after retries and
+                   local saving is requested but unsuccessful.
+    """
+
+    def log_message(message):
+        if logger:
+            logger.info(message)
+
+    def log_error(message, exc_info=False):
+        if logger:
+            logger.error(message, exc_info=exc_info)
+
+    attempts = 0
+    success = False
+    cloud_path = None
+    local_path_final = None
+    gcs_upload_exception = None  # Store potential GCS exception
+
+    if isinstance(data, (list, dict)):
+        data_str = json.dumps(data, indent=2)
+    elif isinstance(data, str):
+        data_str = data
+    else:
+        raise ValueError("Unsupported data type. It should be a list, dict, or str.")
+
+    while attempts < max_retries and not success:
+        try:
+            bucket = storage_client.bucket(bucket_name)
+            blob = bucket.blob(file_name)
+            blob.upload_from_string(data_str, content_type='application/json')
+            cloud_path = f"{bucket_name}/{file_name}"
+            log_message(f"Successfully saved file to GCS {cloud_path}.")
+            success = True
+        except Exception as e:
+            gcs_upload_exception = e 
+            attempts += 1
+            log_error(f"Attempt {attempts} - Failed to write {file_name} "
+                        f"to GCS bucket '{bucket_name}': {e}")  # Log with full traceback
+            if attempts < max_retries:
+                time.sleep(2 ** attempts)
+
+    if not success and (save_locally or local_path):
+        try:
+            if not local_path:
+                local_path_final = os.path.join("/tmp", file_name)
+            else:
+                local_path_final = os.path.join(local_path, file_name)
+            with open(local_path_final, 'w', encoding='utf-8') as f:
+                f.write(data_str)
+            log_message(f"Saved {file_name} locally at {local_path_final}.")
+        except Exception as local_e:
+            log_error(f"Failed to write {file_name} locally: {local_e}",exc_info=True)
+
+        # If GCS upload failed, raise a single exception here
+
+        if gcs_upload_exception:
+            raise gcs_upload_exception from None  # Propagate without nesting
+
+    return cloud_path, local_path_final
+
 
 def write_csv_to_gcs(bucket_name, file_name, data, storage_client, logger,log_info_verbose=True):
     """ Helper function to write a CSV file to Google Cloud Storage """
@@ -151,4 +228,4 @@ def write_csv_to_gcs(bucket_name, file_name, data, storage_client, logger,log_in
     except ValueError as e:
         logger.error(f"ValueError: {e}")
     except Exception as e:
-        logger.error(f"An unexpected error occurred while writing CSV to GCS: {e}", exc_info=True)
+     logger.error(f"An unexpected error occurred while writing CSV to GCS: {e}", exc_info=True)

ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -6,7 +6,7 @@
 import datetime
 from google.cloud import bigquery
 from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeSeverity
-from ipulse_shared_core_ftredge.utils_common import create_custom_notice
+from ipulse_shared_core_ftredge.utils_common import Notice
 
 
 def create_bigquery_schema_from_json(json_schema):
@@ -23,14 +23,14 @@ def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_
 
     """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
     valid_updates = {}
-    notices=[] ### THIS IS TO AVOID LOGGING A WARNING RANDOMLY , INSTEAD GROUPPING FOR A GIVEN RUN
+    notices=[] ### THIS IS TO AVOID LOGGING A WARNING RANDOMLY, INSTEAD GROUPPING FOR A GIVEN RUN
 
     # Process updates to conform to the schema
     for field in schema:
         field_name = field["name"]
         field_type = field["type"]
         mode = field["mode"]
-        
+
         # Initialize notice to None at the start of each field processing
         notice = None
 
@@ -38,7 +38,7 @@ def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_
             value = updates[field_name]
 
             # Handle date and timestamp formatting
-            
+
             # Validate and potentially convert date and timestamp fields
             if field_type == "DATE":
                 value, notice = handle_date_fields(field_name, value, dt_ts_to_str)
@@ -55,18 +55,18 @@ def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_
                 value,notice = check_and_truncate_length(field_name, value, field["max_length"])
                 if notice:
                     notices.append(notice)
-            
+
             # Only add to the dictionary if value is not None or the field is required
             if value is not None or mode == "REQUIRED":
                 valid_updates[field_name] = value
 
         elif mode == "REQUIRED":
-            notice=create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+            notice=Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
                              subject=field_name,
                              message=f"Required field '{field_name}' is missing in the updates.")
 
             notices.append(notice)
-      
+
     return valid_updates, notices
 
 def handle_date_fields(field_name, value, dt_ts_to_str):
@@ -82,11 +82,11 @@ def handle_date_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_date, None
         except ValueError:
-            return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
                                                    message=f"Expected a DATE in YYYY-MM-DD format but got {value}.")
     else:
-        return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+        return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
                                               subject=field_name,
                                               message= f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
 
@@ -104,11 +104,11 @@ def handle_timestamp_fields(field_name, value, dt_ts_to_str):
                 return value, None
             return parsed_datetime, None
         except ValueError:
-            return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
                                                   subject=field_name,
                                                   message= f"Expected ISO format TIMESTAMP but got {value}.")
     else:
-        return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+        return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
                                               subject=field_name,
                                               message= f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
 
@@ -116,17 +116,17 @@ def handle_timestamp_fields(field_name, value, dt_ts_to_str):
 def check_and_truncate_length(field_name, value, max_length):
     """Checks and truncates the length of string fields if they exceed the max length."""
     if isinstance(value, str) and len(value) > max_length:
-        return value[:max_length], create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_RECOMMENDED, 
+        return value[:max_length], Notice(severity=NoticeSeverity.WARNING_FIX_RECOMMENDED,
                             subject= field_name,
                              message= f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
-   
+
     return value, None
 
 
 
 def handle_type_conversion(field_type, field_name, value):
     if field_type == "STRING" and not isinstance(value, str):
-        return str(value), create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED, 
+        return str(value), Notice(severity=NoticeSeverity.WARNING_REVIEW_RECOMMENDED,
                              subject=field_name,
                              message= f"Expected STRING but got {value} of type {type(value).__name__}.")
 
@@ -134,17 +134,19 @@ def handle_type_conversion(field_type, field_name, value):
         try:
             return int(value), None
         except ValueError:
-            return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED,
+            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED,
                                                 subject= field_name,
                                                 message=f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
     if field_type == "FLOAT64" and not isinstance(value, float):
         try:
             return float(value), None
         except ValueError:
-            return None, create_custom_notice(severity=NoticeSeverity.WARNING_ACTION_REQUIRED, 
-                                                  subject=field_name,
-                                                   message=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
+            return None, Notice(severity=NoticeSeverity.WARNING_FIX_REQUIRED, 
+                                                subject=field_name,
+                                                message=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
     if field_type == "BOOL" and not isinstance(value, bool):
-        return bool(value), None
-    
-    return value, None
+        return bool(value), Notice(severity=NoticeSeverity.WARNING_REVIEW_RECOMMENDED, 
+                                                subject=field_name,
+                                                message=f"Expected BOOL, but got  {value}. Converting as {bool(value)}.")
+
+    return value, None

{ipulse_shared_core_ftredge-2.39.dist-info → ipulse_shared_core_ftredge-2.50.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.39
+Version: 2.50
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.39.dist-info → ipulse_shared_core_ftredge-2.50.dist-info}/RECORD

@@ -1,10 +1,10 @@
-ipulse_shared_core_ftredge/__init__.py,sha256=87fWkJqrx0FMqOlzxclR6vBRJPisz3SbwhCWbeeP4kk,902
-ipulse_shared_core_ftredge/utils_common.py,sha256=Fs8-guNuz_UJd4p7Vh0bFDzxyE2YCVaj4bxJQynOLpk,803
-ipulse_shared_core_ftredge/utils_gcp.py,sha256=E8TvZ05fTjNr-VQXxSZNCiqT9PwPhtqeKOifIGhb2sg,6289
-ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=l70p5NV6R4pNEkqdSz0WOy8RPdsR8BqrP-qlZ7Jjep8,7075
+ipulse_shared_core_ftredge/__init__.py,sha256=jAtj9nifLy5nOgpVuc5JeAQAkyF8a_5IzWHKexFZT2Q,904
+ipulse_shared_core_ftredge/utils_common.py,sha256=w038BKQZqqndM-9hYT-tZ0opw3QbVgBemJxug8d6HK0,15565
+ipulse_shared_core_ftredge/utils_gcp.py,sha256=OuR0tMGDSn69kGWhrDP1URuItrtGLu6T3vQlQS65IMk,9389
+ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=67WaefoBXFiRKoYKCo3XcThODJyW7YoSWhsZOj1nTlo,7107
 ipulse_shared_core_ftredge/enums/__init__.py,sha256=PjxJiUConI2TuaG_Ushe2BaFVjBDw1rbq1E9Vt9nXvE,801
-ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=ukChcm8R2QwlK4NAfM291mWryCAZvfg7bMpdScQQfok,4360
-ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=A8_uiGGacd-_AZP09Zft-DX3rF8aVqqFzFak8s4MsfY,1801
+ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=aUH5SJOInEvVA_jtKwaBYNc1_5E3ud5IWCVSEiK1cCI,4895
+ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=2i6Qo6Yi_j_O9xxnOD6QA-r0Cv7mWAUaKUx907XMRio,1825
 ipulse_shared_core_ftredge/enums/enums_module_fincore.py,sha256=MuqQg249clrWUOBb1S-iPsoOldN2_F3ohRQizbjhwG0,1374
 ipulse_shared_core_ftredge/enums/enums_modules.py,sha256=AyXUoNmR75DZLaEHi3snV6LngR25LeZRqzrLDaAupbY,1244
 ipulse_shared_core_ftredge/models/__init__.py,sha256=gE22Gzhil0RYQa7YLtdtT44_AsWqklcDfRtgLAQc1dI,200
@@ -18,8 +18,8 @@ ipulse_shared_core_ftredge/models/user_profile_update.py,sha256=oKK0XsQDKkgDvjFP
 ipulse_shared_core_ftredge/models/user_status.py,sha256=8TyRd8tBK9_xb0MPKbI5pn9-lX7ovKbeiuWYYPtIOiw,3202
 ipulse_shared_core_ftredge/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ipulse_shared_core_ftredge/tests/test.py,sha256=0lS8HP5Quo_BqNoscU40qOH9aJRaa1Pfam5VUBmdld8,682
-ipulse_shared_core_ftredge-2.39.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
-ipulse_shared_core_ftredge-2.39.dist-info/METADATA,sha256=I99qJdttOu8XaJhp1dCaYAuJw-17QlN57kJtSghhtpM,561
-ipulse_shared_core_ftredge-2.39.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-ipulse_shared_core_ftredge-2.39.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
-ipulse_shared_core_ftredge-2.39.dist-info/RECORD,,
+ipulse_shared_core_ftredge-2.50.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
+ipulse_shared_core_ftredge-2.50.dist-info/METADATA,sha256=Y00Y547YKmK5-I0zlA522urqyihif5GUVgCG5yAXZIg,561
+ipulse_shared_core_ftredge-2.50.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
+ipulse_shared_core_ftredge-2.50.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
+ipulse_shared_core_ftredge-2.50.dist-info/RECORD,,

{ipulse_shared_core_ftredge-2.39.dist-info → ipulse_shared_core_ftredge-2.50.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: setuptools (70.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any