ipulse-shared-core-ftredge 2.6__py3-none-any.whl → 2.7.1__py3-none-any.whl

ipulse_shared_core_ftredge/utils/utils_cloud_gcp.py

@@ -0,0 +1,442 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
+# pylint: disable=broad-exception-raised
+import json
+import csv
+from io import StringIO
+import os
+import time
+import logging
+import traceback
+from google.api_core.exceptions import NotFound
+from google.cloud import error_reporting
+from google.cloud import logging as cloud_logging
+from google.cloud.storage import Client as GCSClient
+from google.cloud import bigquery
+from ipulse_shared_core_ftredge.enums import DuplicationHandling, DuplicationHandlingStatus, MatchConditionType,DataSourceType, LogLevel
+from ipulse_shared_core_ftredge.utils.utils_common import log_error, log_warning, log_info
+from ipulse_shared_core_ftredge.utils.logs import ContextLog
+from ipulse_shared_core_ftredge.utils.utils_collector_pipelinemon import Pipelinemon
+
+############################################################################
+##################### GOOGLE CLOUD PLATFORM UTILS ##################################
+############################################################################
+
+class CustomGCPLoggingHandler(cloud_logging.handlers.CloudLoggingHandler):
+    """Custom handler for Google Cloud Logging with a dynamic logName."""
+    def __init__(self, client, name, resource=None, labels=None):
+        super().__init__(client=client, name=name, resource=resource, labels=labels)
+        self.client = client  # Ensure client is consistently used
+
+    def emit(self, record):
+        try:
+            # 1. Create the basic log entry dictionary
+            log_entry = {
+                'message': record.msg,
+                'severity': record.levelname,
+                'name': record.name,
+                'pathname': record.filename,
+                'lineno': record.lineno,
+            }
+            if record.exc_info:
+                log_entry['exception_traceback'] = ''.join(
+                    traceback.format_exception(*record.exc_info)
+                )
+
+            # 2. Apply the formatter to the 'message' field if it's a dictionary
+            if isinstance(record.msg, dict):
+                formatted_message = self.formatter.format(record)
+                try:
+                    log_entry['message'] = json.loads(formatted_message)
+                except json.JSONDecodeError:
+                    log_entry['message'] = formatted_message
+            else:
+                log_entry['message'] = record.msg
+
+            # 3. Set the custom logName
+            log_entry['logName'] = f"projects/{self.client.project}/logs/{record.name}"
+
+            # 4. Send to Google Cloud Logging
+            super().emit(record)
+        except Exception as e:
+            self.handleError(record)
+
+class CustomGCPErrorReportingHandler(logging.Handler):
+    def __init__(self, client=None, level=logging.ERROR):
+        super().__init__(level)
+        self.error_client = error_reporting.Client() if client is None else client
+        self.propagate = True
+
+    def emit(self, record):
+        try:
+            if record.levelno >= logging.ERROR:
+                log_struct = {
+                'message': self.format(record),
+                'severity': record.levelname,
+                'pathname': getattr(record, 'pathname', None),
+                'lineno': getattr(record, 'lineno', None)
+            }
+            if record.exc_info:
+                log_struct['exception'] = ''.join(
+                    traceback.format_exception(*record.exc_info)
+                )
+            self.error_client.report(str(log_struct))
+        except Exception as e:
+            self.handleError(record)
+
+
+def add_gcp_cloud_logging(logger, formatter, client=None):
+    """Sets up Google Cloud Logging for the logger."""
+    client = client or cloud_logging.Client()
+    handler = CustomGCPLoggingHandler(client, logger.name)
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+def add_gcp_error_reporting(logger, client=None):
+    """Sets up Google Cloud Error Reporting for the logger."""
+    client = client or error_reporting.Client()
+    handler = CustomGCPErrorReportingHandler(client=client)
+    logger.addHandler(handler)
+
+
+
+def create_bigquery_schema_from_json(json_schema: list) -> list:
+    schema = []
+    for field in json_schema:
+        if "max_length" in field:
+            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"], max_length=field["max_length"]))
+        else:
+            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"]))
+    return schema
+
+
+def read_json_from_gcs(storage_client:GCSClient, bucket_name:str, file_name:str, logger=None,print_out=False):
+    """ Helper function to read a JSON file from Google Cloud Storage """
+    try:
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(file_name)
+        data_string = blob.download_as_text()
+        data = json.loads(data_string)
+        return data
+    except NotFound:
+        log_warning(msg=f"Warning: The file {file_name} was not found in the bucket {bucket_name}.", logger=logger, print_out=print_out)
+        return None
+    except json.JSONDecodeError:
+        log_error(msg=f"Error: The file {file_name} could not be decoded as JSON.", logger=logger, print_out=print_out)
+        return None
+    except Exception as e:
+        log_error(msg=f"An unexpected error occurred: {e}", exc_info=True, logger=logger, print_out=print_out)
+        return None
+
+def read_csv_from_gcs(bucket_name:str, file_name:str, storage_client:GCSClient, logger=None, print_out=False):
+    """ Helper function to read a CSV file from Google Cloud Storage """
+
+    try:
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(file_name)
+        data_string = blob.download_as_text()
+        data_file = StringIO(data_string)
+        reader = csv.DictReader(data_file)
+        return list(reader)
+    except NotFound:
+        log_warning(msg=f"Error: The file {file_name} was not found in the bucket {bucket_name}.", logger=logger, print_out=print_out)
+        return None
+    except csv.Error:
+        log_error(msg=f"Error: The file {file_name} could not be read as CSV.", logger=logger, print_out=print_out)
+        return None
+    except Exception as e:
+        log_error(msg=f"An unexpected error occurred: {e}", logger=logger, print_out=print_out, exc_info=True)
+        return None
+
+def write_json_to_gcs_extended(storage_client: GCSClient, data: dict | list | str, bucket_name: str, file_name: str,
+                                                duplication_handling_enum: DuplicationHandling, duplication_match_condition_type_enum: MatchConditionType,
+                                                duplication_match_condition: str = "", max_retries: int = 2, max_deletable_files: int = 1,
+                                                logger=None, print_out=False, raise_e=False, pipelinemon: Pipelinemon = None):
+
+    """Saves data to Google Cloud Storage with optional Pipelinemon monitoring.
+
+    Handles duplication with strategies: OVERWRITE, INCREMENT, SKIP, or RAISE_ERROR.
+    """
+
+    max_deletable_files_allowed = 3
+    cloud_storage_ref=DataSourceType.GCS.value
+
+    # GCS-related metadata
+    saved_to_path = None
+    matched_duplicates_count = 0
+    matched_duplicates_deleted = []
+    duplication_handling_status = None
+    error_during_operation = None
+
+    response = {
+        "saved_to_path": saved_to_path,
+        "matched_duplicates_count": matched_duplicates_count,
+        "matched_duplicates_deleted": matched_duplicates_deleted,
+        "duplication_handling_status": duplication_handling_status,
+        "duplication_match_condition_type": duplication_match_condition_type_enum.value,
+        "duplication_match_condition": duplication_match_condition,
+        "error_during_operation": error_during_operation
+    }
+
+    supported_match_condition_types = [MatchConditionType.EXACT, MatchConditionType.PREFIX]
+    supported_duplication_handling = [DuplicationHandling.RAISE_ERROR, DuplicationHandling.OVERWRITE, DuplicationHandling.INCREMENT, DuplicationHandling.SKIP]
+
+    try:
+        if max_deletable_files > max_deletable_files_allowed:
+            raise ValueError(f"max_deletable_files should be less than or equal to {max_deletable_files_allowed} for safety.")
+        if duplication_handling_enum not in supported_duplication_handling:
+            msg = f"Error: Duplication handling not supported. Supported types: {[dh.value for dh in supported_duplication_handling]}"
+            raise ValueError(msg)
+        if duplication_match_condition_type_enum not in supported_match_condition_types:
+            msg = f"Error: Match condition type not supported. Supported types: {[mct.value for mct in supported_match_condition_types]}"
+            raise ValueError(msg)
+        elif duplication_match_condition_type_enum != MatchConditionType.EXACT and not duplication_match_condition:
+            msg = f"Error: Match condition is required for match condition type: {duplication_match_condition_type_enum.value}"
+            raise ValueError(msg)
+
+        # Prepare data
+        if isinstance(data, (list, dict)):
+            data_str = json.dumps(data, indent=2)
+        else:
+            data_str = data
+
+        increment = 0
+        attempts = 0
+        success = False
+
+        # Check for existing files based on duplication_match_condition_type
+        files_matched_on_condition = []
+        bucket = storage_client.bucket(bucket_name)
+        base_file_name, ext = os.path.splitext(file_name)
+        if duplication_match_condition_type_enum == MatchConditionType.PREFIX:
+            files_matched_on_condition = list(bucket.list_blobs(prefix=duplication_match_condition))
+        elif duplication_match_condition_type_enum == MatchConditionType.EXACT:
+            duplication_match_condition = file_name if not duplication_match_condition else duplication_match_condition
+            if bucket.blob(duplication_match_condition).exists():
+                files_matched_on_condition = [bucket.blob(file_name)]
+
+        matched_duplicates_count = len(files_matched_on_condition)
+        response["matched_duplicates_count"] = matched_duplicates_count
+
+        # Handle duplication based on duplication_handling
+        if matched_duplicates_count:
+            log_msg = f"Duplicate FOUND, matched_duplicates_count: {matched_duplicates_count}"
+            if pipelinemon:
+                    pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject="duplicate_found", description=log_msg))
+
+            if duplication_handling_enum == DuplicationHandling.RAISE_ERROR:
+                raise FileExistsError("File(s) matching the condition already exist.")
+
+            if duplication_handling_enum == DuplicationHandling.SKIP:
+                response["duplication_handling_status"] = DuplicationHandlingStatus.SKIPPED.value
+                log_msg = f"SKIPPING, response: {response}"
+                log_info(log_msg, logger=logger, print_out=print_out) ## only logsor prints if logger is provided and print_out is True
+                return response
+
+            if duplication_handling_enum == DuplicationHandling.OVERWRITE:
+                if matched_duplicates_count > max_deletable_files:
+                    raise ValueError(f"Error: Attempt to delete {matched_duplicates_count} matched files, but limit is {max_deletable_files}. Operation Cancelled.")
+
+                for blob in files_matched_on_condition:
+                    cloud_storage_path_to_delete = f"gs://{bucket_name}/{blob.name}"
+                    blob.delete()
+                    matched_duplicates_deleted.append(cloud_storage_path_to_delete)
+                    log_msg = f"File deleted as part of overwrite: {cloud_storage_path_to_delete}"
+                    if pipelinemon:
+                        pipelinemon.add_system_impacted(f"delete: {cloud_storage_ref}_bucket_file: {cloud_storage_path_to_delete}")
+                        pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_DELETE_COMPLETE, subject="delete_duplicate", description=log_msg))
+                    log_info(log_msg, logger=logger, print_out=print_out)
+
+                response["matched_duplicates_deleted"] = matched_duplicates_deleted
+                response["duplication_handling_status"] = DuplicationHandlingStatus.OVERWRITTEN.value
+
+            elif duplication_handling_enum == DuplicationHandling.INCREMENT:
+                while bucket.blob(file_name).exists():
+                    increment += 1
+                    file_name = f"{base_file_name}_v{increment}{ext}"
+                saved_to_path = f"gs://{bucket_name}/{file_name}"
+                response["duplication_handling_status"] = DuplicationHandlingStatus.INCREMENTED.value
+                log_msg = "INCREMENTING as Duplicate FOUND "
+                log_info(log_msg, logger=logger, print_out=print_out) ## only logsor prints if logger is provided and print_out is True
+
+        # GCS Upload
+        saved_to_path = f"gs://{bucket_name}/{file_name}"
+        while attempts < max_retries and not success:
+            try:
+                blob = bucket.blob(file_name)
+                blob.upload_from_string(data_str, content_type='application/json')
+                log_msg = f"File uploaded to GCS: {saved_to_path}"
+                if pipelinemon:
+                    pipelinemon.add_system_impacted(f"upload: {cloud_storage_ref}_bucket_file: {saved_to_path}")
+                    pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_PERSISTNACE_COMPLETE, subject="file_upload", description=log_msg))
+                log_info(log_msg, logger=logger, print_out=print_out)
+                success = True
+            except Exception as e:
+                attempts += 1
+                if attempts < max_retries:
+                    time.sleep(2 ** attempts)
+                else:
+                    raise e
+
+    except Exception as e:
+        error_during_operation = f"Error occurred while writing JSON to GCS path: {saved_to_path} ; Error details: {type(e).__name__} - {str(e)}"
+        response["error_during_operation"] = error_during_operation
+        if pipelinemon:
+            pipelinemon.add_log(ContextLog(LogLevel.ERROR_EXCEPTION, e=e, description="response: {response}"))
+        log_error(response, logger=logger, print_out=print_out)
+        if raise_e:
+            raise e
+
+    response["saved_to_path"] = saved_to_path if success else None
+    return response
+
+# def write_json_to_gcs_extended(storage_client: GCSClient, data: dict | list | str, bucket_name: str, file_name: str,
+#                                duplication_handling: DuplicationHandling, duplication_match_condition_type: MatchConditionType,
+#                                duplication_match_condition: str | List[str] = "", max_retries: int = 2, max_deletable_files: int = 1,
+#                                logger=None, print_out=False, raise_e=False):
+
+#     """Saves data to Google Cloud Storage.
+
+#     Handles duplication with strategies: OVERWRITE, INCREMENT, SKIP, or RAISE_ERROR.
+#     """
+
+#     max_deletable_files_allowed = 3
+
+#     # GCS-related metadata
+#     saved_to_path = None
+#     matched_duplicates_count = 0
+#     matched_duplicates_deleted = []
+#     duplication_handling_status = None
+#     error_during_operation = None
+
+#     response = {
+#         "saved_to_path": saved_to_path,
+#         "matched_duplicates_count": matched_duplicates_count,
+#         "matched_duplicates_deleted": matched_duplicates_deleted,
+#         "duplication_handling_status": duplication_handling_status,
+#         "duplication_match_condition_type": duplication_match_condition_type,
+#         "duplication_match_condition": duplication_match_condition,
+#         "error_during_operation": error_during_operation
+#     }
+
+#     supported_match_condition_types = [MatchConditionType.EXACT, MatchConditionType.PREFIX]
+#     supported_duplication_handling = [DuplicationHandling.RAISE_ERROR, DuplicationHandling.OVERWRITE, DuplicationHandling.INCREMENT, DuplicationHandling.SKIP]
+
+#     try:
+#         if max_deletable_files > max_deletable_files_allowed:
+#             raise ValueError(f"max_deletable_files should be less than or equal to {max_deletable_files_allowed} for safety.")
+#         if duplication_handling not in supported_duplication_handling:
+#             msg = f"Error: Duplication handling not supported. Supported types: {supported_duplication_handling}"
+#             raise ValueError(msg)
+#         if duplication_match_condition_type not in supported_match_condition_types:
+#             msg = f"Error: Match condition type not supported. Supported types: {supported_match_condition_types}"
+#             raise ValueError(msg)
+#         elif duplication_match_condition_type!=MatchConditionType.EXACT and not duplication_match_condition:
+#             msg = f"Error: Match condition is required for match condition type: {duplication_match_condition_type}"
+#             raise ValueError(msg)
+
+#         # Prepare data
+#         if isinstance(data, (list, dict)):
+#             data_str = json.dumps(data, indent=2)
+#         else:
+#             data_str = data
+
+#         increment = 0
+#         attempts = 0
+#         success = False
+
+#         # Check for existing files based on duplication_match_condition_type
+#         files_matched_on_condition = []
+#         bucket = storage_client.bucket(bucket_name)
+#         base_file_name, ext = os.path.splitext(file_name)
+#         if duplication_match_condition_type == MatchConditionType.PREFIX:
+#             files_matched_on_condition = list(bucket.list_blobs(prefix=duplication_match_condition))
+#         elif duplication_match_condition_type == MatchConditionType.EXACT:
+#             if bucket.blob(file_name).exists():
+#                 files_matched_on_condition = [bucket.blob(file_name)]
+
+#         matched_duplicates_count = len(files_matched_on_condition)
+#         response["matched_duplicates_count"] = matched_duplicates_count
+
+#         # Handle duplication based on duplication_handling
+#         if matched_duplicates_count:
+#             if duplication_handling == DuplicationHandling.RAISE_ERROR:
+#                 raise FileExistsError("File(s) matching the condition already exist.")
+
+#             if duplication_handling == DuplicationHandling.SKIP:
+#                 log_warning("Skipping saving to GCS: file(s) matching the condition already exist.", logger=logger, print_out=print_out)
+#                 response["duplication_handling_status"] = DuplicationHandlingStatus.SKIPPED.value
+#                 return response
+
+#             if duplication_handling == DuplicationHandling.OVERWRITE:
+#                 if matched_duplicates_count > max_deletable_files:
+#                     raise ValueError(f"Error: Attempt to delete {matched_duplicates_count} matched files, but limit is {max_deletable_files}. Operation Cancelled.")
+
+#                 for blob in files_matched_on_condition:
+#                     cloud_storage_path_to_delete = f"gs://{bucket_name}/{blob.name}"
+#                     blob.delete()
+#                     matched_duplicates_deleted.append(cloud_storage_path_to_delete)
+
+#                 response["matched_duplicates_deleted"] = matched_duplicates_deleted
+#                 response["duplication_handling_status"] = DuplicationHandlingStatus.OVERWRITTEN.value
+
+#             elif duplication_handling == DuplicationHandling.INCREMENT:
+#                 while bucket.blob(file_name).exists():
+#                     increment += 1
+#                     file_name = f"{base_file_name}_v{increment}{ext}"
+#                 saved_to_path = f"gs://{bucket_name}/{file_name}"
+#                 response["duplication_handling_status"] = DuplicationHandlingStatus.INCREMENTED.value
+
+#         # GCS Upload
+#         saved_to_path = f"gs://{bucket_name}/{file_name}"
+#         while attempts < max_retries and not success:
+#             try:
+#                 blob = bucket.blob(file_name)
+#                 blob.upload_from_string(data_str, content_type='application/json')
+#                 success = True
+#             except Exception as e:
+#                 attempts += 1
+#                 if attempts < max_retries:
+#                     time.sleep(2 ** attempts)
+#                 else:
+#                     if raise_e:
+#                         raise e
+
+#     except Exception as e:
+#         error_message = f"Error occurred while writing JSON to GCS path: {saved_to_path} : {type(e).__name__} - {str(e)}"
+#         log_error(error_message, logger=logger, print_out=print_out)
+#         response["error_during_operation"] = error_message
+#         if raise_e:
+#             raise e
+
+#     response["saved_to_path"] = saved_to_path if success else None
+#     return response
+
+
+def write_csv_to_gcs(bucket_name:str, file_name:str, data:dict | list | str, storage_client:GCSClient, logger=None, print_out=False, raise_e=False):
+    """ Helper function to write a CSV file to Google Cloud Storage """
+    try:
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(file_name)
+        data_file = StringIO()
+        if data and isinstance(data, list) and isinstance(data[0], dict):
+            fieldnames = data[0].keys()
+            writer = csv.DictWriter(data_file, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(data)
+        else:
+            raise ValueError("Data should be a list of dictionaries")
+        blob.upload_from_string(data_file.getvalue(), content_type='text/csv')
+        log_info(msg=f"Successfully wrote CSV to {file_name} in bucket {bucket_name}.", logger=logger, print_out=print_out)
+    except ValueError as e:
+        log_error(msg=f"ValueError: {e}",logger=logger, print_out=print_out)
+        if raise_e:
+            raise e
+    except Exception as e:
+        log_error(msg=f"An unexpected error occurred while writing CSV to GCS: {e}", logger=logger, print_out=print_out, exc_info=True)
+        if raise_e:
+            raise e

ipulse_shared_core_ftredge/utils/utils_cloud_gcp_with_collectors.py

@@ -0,0 +1,166 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
+# pylint: disable=broad-exception-raised
+
+# import json
+# import os
+# import time
+# from google.cloud.storage import Client as GCSClient
+# from ipulse_shared_core_ftredge.enums import LogLevel, DuplicationHandling, DuplicationHandlingStatus, MatchConditionType, DataSourceType
+# from ipulse_shared_core_ftredge.utils import log_error, log_info
+# from .utils_collector_pipelinemon import Pipelinemon
+# from .logs.context_log import ContextLog
+
+
+
+
+
+# def write_json_to_gcs_with_pipelinemon_extended( pipelinemon:Pipelinemon, storage_client:GCSClient, data:dict | list | str, bucket_name: str, file_name: str,
+#                       file_exists_if_starts_with_prefix:Optional[str] =None, overwrite_if_exists:bool=False, increment_if_exists:bool=False,
+#                       max_retries:int=2, max_deletable_files:int=1):
+#     """Saves data to Google Cloud Storage and optionally locally.
+    
+#     This function attempts to upload data to GCS. 
+#     - If the upload fails after retries and `save_locally` is True or `local_path` is provided, it attempts to save the data locally.
+#     - It handles file name conflicts based on these rules:
+#         - If `overwrite_if_exists` is True: 
+#             - If `file_exists_if_contains_substr` is provided, ANY existing file containing the substring is deleted, and the new file is saved with the provided `file_name`.
+#             - If `file_exists_if_contains_substr` is None, and a file with the exact `file_name` exists, it's overwritten.
+#         - If `increment_if_exists` is True:
+#             - If `file_exists_if_contains_substr` is provided, a new file with an incremented version is created ONLY if a file with the EXACT `file_name` exists.
+#             - If `file_exists_if_contains_substr` is None, a new file with an incremented version is created if a file with the exact `file_name` exists. 
+            
+#         -If both overwrite_if_exists and increment_if_exists are provided as Ture, an exception will be raised.
+#     """
+
+#     cloud_storage_ref="GCP_GCS"
+
+#     with pipelinemon.context(f"write_json_to_{cloud_storage_ref}_with_pipelinemon"):
+#         cloud_storage_upload_error = None
+#         # Input validation
+#         if overwrite_if_exists and increment_if_exists:
+#             err_msg="Both 'overwrite_if_exists' and 'increment_if_exists' cannot be True simultaneously."
+#             pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM, subject="Param validation", description=err_msg))
+#             return {"cloud_storage_upload_error": err_msg}
+#         if max_deletable_files > 10:
+#             err_msg="max_deletable_files should be less than 10 for safety. For more use another method."
+#             pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM,subject="max_deletable_files", description=err_msg))
+#             return {"cloud_storage_upload_error": err_msg}
+
+#         # Prepare data
+#         if isinstance(data, (list, dict)):
+#             data_str = json.dumps(data, indent=2)
+#         else:
+#             data_str = data
+
+#         bucket = storage_client.bucket(bucket_name)
+#         base_file_name, ext = os.path.splitext(file_name)
+#         increment = 0
+#         attempts = 0
+#         success = False
+
+#         # GCS-related metadata
+#         cloud_storage_path = None
+#         cloud_storage_file_overwritten = False
+#         cloud_storage_file_already_exists = False
+#         cloud_storage_file_saved_with_increment = False
+#         cloud_storage_file_exists_checked_on_name = file_name
+#         cloud_storage_deleted_files=[]
+
+#         try:
+#             upload_allowed = True
+#             # --- Overwrite Logic ---
+#             if overwrite_if_exists:
+#                 with pipelinemon.context("overwriting"):
+#                     if file_exists_if_starts_with_prefix:
+#                         cloud_storage_file_exists_checked_on_name = file_exists_if_starts_with_prefix
+#                         blobs_to_delete = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
+#                         if len(blobs_to_delete) > max_deletable_files:
+#                             err_msg=f"Error: Attempt to delete {len(blobs_to_delete)} matched files, but limit is {max_deletable_files}."
+#                             pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_to_delete)} files in bucket {bucket_name}"))
+#                             pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM, subject="Too many files", description=err_msg))
+#                             #### Ensuring to quit the operation if too many files are found, it will be catched below
+#                             return {"cloud_storage_upload_error": err_msg}
+#                         if blobs_to_delete:
+#                             cloud_storage_file_already_exists = True
+#                             pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_to_delete)} files in bucket {bucket_name}"))
+#                             for blob in blobs_to_delete:
+#                                 cloud_storage_path_del = f"gs://{bucket_name}/{blob.name}"
+#                                 pipelinemon.add_system_impacted(f"delete: {cloud_storage_ref}_bucket_file: {cloud_storage_path_del}")
+#                                 blob.delete()
+#                                 pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_DELETE_COMPLETE, subject= cloud_storage_path_del, description=f"file deleted from {cloud_storage_ref} as part of overwrite, matched with prefix"))
+#                                 cloud_storage_deleted_files.append(cloud_storage_path_del)
+#                             cloud_storage_file_overwritten = True
+#                     elif bucket.blob(file_name).exists():
+#                         cloud_storage_file_already_exists = True
+#                         pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"Exact name matched with existing file in bucket {bucket_name}"))
+#                         cloud_storage_path_del = f"gs://{bucket_name}/{file_name}"
+#                         pipelinemon.add_system_impacted(f"delete: {cloud_storage_ref}_bucket_file: {cloud_storage_path_del}")
+#                         blob.delete()  # Delete the existing blob
+#                         pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_DELETE_COMPLETE, subject= cloud_storage_path_del, description=f"file deleted from {cloud_storage_ref} as part of overwrite, matched with exact name"))
+#                         cloud_storage_deleted_files.append(cloud_storage_path_del)
+#                         cloud_storage_file_overwritten = True
+#             # --- Increment Logic ---
+#             elif increment_if_exists:
+#                 with pipelinemon.context("incrementing"):
+#                     cloud_storage_file_exists_checked_on_name = file_name  # We only increment if the exact name exists
+#                     while bucket.blob(file_name).exists():
+#                         cloud_storage_file_already_exists = True
+#                         increment += 1
+#                         file_name = f"{base_file_name}_v{increment}{ext}"
+#                         cloud_storage_file_saved_with_increment = True
+#                     if increment>0:
+#                         cloud_storage_path = f"gs://{bucket_name}/{file_name}"
+#                         pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"Attempting to save file with incremented version in {bucket_name}"))
+#              # --- Check for Conflicts (Including Prefix) ---
+#             else:
+#                 if file_exists_if_starts_with_prefix:
+#                     blobs_matched = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
+#                     cloud_storage_file_exists_checked_on_name = file_exists_if_starts_with_prefix
+#                     if blobs_matched:
+#                         upload_allowed = False
+#                         cloud_storage_file_already_exists = True
+#                         pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_matched)} existing files in bucket {bucket_name}."))
+#                 elif bucket.blob(file_name).exists():
+#                     pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"Exact name matched with existing file in bucket {bucket_name}."))
+#                     upload_allowed = False
+#                     cloud_storage_file_already_exists = True
+
+#             # --- GCS Upload ---
+#             cloud_storage_path = f"gs://{bucket_name}/{file_name}"
+#             if overwrite_if_exists or increment_if_exists or upload_allowed:
+#                 with pipelinemon.context("uploading"):
+#                     while attempts < max_retries and not success:
+#                         try:
+#                             blob = bucket.blob(file_name)  # Use the potentially updated file_name
+#                             pipelinemon.add_system_impacted(f"upload: {cloud_storage_ref}_bucket_file: {cloud_storage_path}")
+#                             blob.upload_from_string(data_str, content_type='application/json')
+#                             pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_PERSISTNACE_COMPLETE, subject= cloud_storage_path, description=f"file uploaded to {cloud_storage_ref}"))
+#                             success = True
+#                         except Exception as e:
+#                             attempts += 1
+#                             if attempts < max_retries:
+#                                 time.sleep(2 ** attempts)
+#                             else:
+#                                 err_msg=f"Error uploading file to {cloud_storage_ref} bucket {bucket_name} with name {file_name} : {type(e).__name__}-{str(e)}"
+#                                 pipelinemon.add_log(ContextLog(LogLevel.ERROR_EXCEPTION, e=e, description=err_msg))
+#                                 return {"cloud_storage_upload_error": err_msg}
+
+#         except Exception as e:
+#             pipelinemon.add_log(ContextLog(LogLevel.ERROR_EXCEPTION, e=e))
+#             return {"cloud_storage_upload_error": f"Exception in GCS upload {type(e).__name__}-{str(e)}"}
+#     # --- Return Metadata ---
+#     return {
+#         "cloud_storage_path": cloud_storage_path if ((success or not upload_allowed) and not cloud_storage_upload_error ) else None,
+#         "cloud_storage_file_already_exists": cloud_storage_file_already_exists,
+#         "cloud_storage_file_exists_checked_on_name":cloud_storage_file_exists_checked_on_name ,
+#         "cloud_storage_file_overwritten": cloud_storage_file_overwritten,
+#         "cloud_storage_deleted_file_names": ",,,".join(cloud_storage_deleted_files) if cloud_storage_deleted_files else None,
+#         "cloud_storage_file_saved_with_increment": cloud_storage_file_saved_with_increment,
+#         "cloud_storage_upload_error": cloud_storage_upload_error
+#     }
+

ipulse_shared_core_ftredge/utils/utils_cloud_with_collectors.py

@@ -0,0 +1,27 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
+# pylint: disable=broad-exception-raised
+# from typing import Optional
+# from ipulse_shared_core_ftredge.enums import DataSourceType
+# from .utils_collector_pipelinemon import Pipelinemon
+# from .utils_cloud_gcp import write_json_to_gcs_extended
+
+
+# def write_json_to_cloud_storage_with_pipelinemon_extended(cloud_storage_type:DataSourceType, cloud_storage_client, pipelinemon:Pipelinemon,
+#                                                            data:dict | list | str, bucket_name: str, file_name: str,
+                                                            
+                                                            
+#                                                             max_retries:int=2, max_deletable_files:int=1):
+
+#     supported_cloud_storage_types = [DataSourceType.GCS]
+#     if cloud_storage_type == DataSourceType.GCS:
+#         return write_json_to_gcs_extended(pipelinemon=pipelinemon, storage_client=cloud_storage_client, data=data, bucket_name=bucket_name, file_name=file_name,
+#                                                   ,
+#                                                   max_retries=max_retries,
+#                                                   max_deletable_files=max_deletable_files)
+
+#     raise ValueError(f"Unsupported cloud provider: {cloud_storage_type}. Supported cloud providers: {supported_cloud_storage_types}")