ipulse-shared-core-ftredge 2.7.1__py3-none-any.whl → 2.8.1__py3-none-any.whl

ipulse_shared_core_ftredge/utils/utils_cloud_gcp.py

@@ -1,442 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-# pylint: disable=broad-exception-caught
-# pylint: disable=line-too-long
-# pylint: disable=unused-variable
-# pylint: disable=broad-exception-raised
-import json
-import csv
-from io import StringIO
-import os
-import time
-import logging
-import traceback
-from google.api_core.exceptions import NotFound
-from google.cloud import error_reporting
-from google.cloud import logging as cloud_logging
-from google.cloud.storage import Client as GCSClient
-from google.cloud import bigquery
-from ipulse_shared_core_ftredge.enums import DuplicationHandling, DuplicationHandlingStatus, MatchConditionType,DataSourceType, LogLevel
-from ipulse_shared_core_ftredge.utils.utils_common import log_error, log_warning, log_info
-from ipulse_shared_core_ftredge.utils.logs import ContextLog
-from ipulse_shared_core_ftredge.utils.utils_collector_pipelinemon import Pipelinemon
-
-############################################################################
-##################### GOOGLE CLOUD PLATFORM UTILS ##################################
-############################################################################
-
-class CustomGCPLoggingHandler(cloud_logging.handlers.CloudLoggingHandler):
-    """Custom handler for Google Cloud Logging with a dynamic logName."""
-    def __init__(self, client, name, resource=None, labels=None):
-        super().__init__(client=client, name=name, resource=resource, labels=labels)
-        self.client = client  # Ensure client is consistently used
-
-    def emit(self, record):
-        try:
-            # 1. Create the basic log entry dictionary
-            log_entry = {
-                'message': record.msg,
-                'severity': record.levelname,
-                'name': record.name,
-                'pathname': record.filename,
-                'lineno': record.lineno,
-            }
-            if record.exc_info:
-                log_entry['exception_traceback'] = ''.join(
-                    traceback.format_exception(*record.exc_info)
-                )
-
-            # 2. Apply the formatter to the 'message' field if it's a dictionary
-            if isinstance(record.msg, dict):
-                formatted_message = self.formatter.format(record)
-                try:
-                    log_entry['message'] = json.loads(formatted_message)
-                except json.JSONDecodeError:
-                    log_entry['message'] = formatted_message
-            else:
-                log_entry['message'] = record.msg
-
-            # 3. Set the custom logName
-            log_entry['logName'] = f"projects/{self.client.project}/logs/{record.name}"
-
-            # 4. Send to Google Cloud Logging
-            super().emit(record)
-        except Exception as e:
-            self.handleError(record)
-
-class CustomGCPErrorReportingHandler(logging.Handler):
-    def __init__(self, client=None, level=logging.ERROR):
-        super().__init__(level)
-        self.error_client = error_reporting.Client() if client is None else client
-        self.propagate = True
-
-    def emit(self, record):
-        try:
-            if record.levelno >= logging.ERROR:
-                log_struct = {
-                'message': self.format(record),
-                'severity': record.levelname,
-                'pathname': getattr(record, 'pathname', None),
-                'lineno': getattr(record, 'lineno', None)
-            }
-            if record.exc_info:
-                log_struct['exception'] = ''.join(
-                    traceback.format_exception(*record.exc_info)
-                )
-            self.error_client.report(str(log_struct))
-        except Exception as e:
-            self.handleError(record)
-
-
-def add_gcp_cloud_logging(logger, formatter, client=None):
-    """Sets up Google Cloud Logging for the logger."""
-    client = client or cloud_logging.Client()
-    handler = CustomGCPLoggingHandler(client, logger.name)
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-
-def add_gcp_error_reporting(logger, client=None):
-    """Sets up Google Cloud Error Reporting for the logger."""
-    client = client or error_reporting.Client()
-    handler = CustomGCPErrorReportingHandler(client=client)
-    logger.addHandler(handler)
-
-
-
-def create_bigquery_schema_from_json(json_schema: list) -> list:
-    schema = []
-    for field in json_schema:
-        if "max_length" in field:
-            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"], max_length=field["max_length"]))
-        else:
-            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"]))
-    return schema
-
-
-def read_json_from_gcs(storage_client:GCSClient, bucket_name:str, file_name:str, logger=None,print_out=False):
-    """ Helper function to read a JSON file from Google Cloud Storage """
-    try:
-        bucket = storage_client.bucket(bucket_name)
-        blob = bucket.blob(file_name)
-        data_string = blob.download_as_text()
-        data = json.loads(data_string)
-        return data
-    except NotFound:
-        log_warning(msg=f"Warning: The file {file_name} was not found in the bucket {bucket_name}.", logger=logger, print_out=print_out)
-        return None
-    except json.JSONDecodeError:
-        log_error(msg=f"Error: The file {file_name} could not be decoded as JSON.", logger=logger, print_out=print_out)
-        return None
-    except Exception as e:
-        log_error(msg=f"An unexpected error occurred: {e}", exc_info=True, logger=logger, print_out=print_out)
-        return None
-
-def read_csv_from_gcs(bucket_name:str, file_name:str, storage_client:GCSClient, logger=None, print_out=False):
-    """ Helper function to read a CSV file from Google Cloud Storage """
-
-    try:
-        bucket = storage_client.bucket(bucket_name)
-        blob = bucket.blob(file_name)
-        data_string = blob.download_as_text()
-        data_file = StringIO(data_string)
-        reader = csv.DictReader(data_file)
-        return list(reader)
-    except NotFound:
-        log_warning(msg=f"Error: The file {file_name} was not found in the bucket {bucket_name}.", logger=logger, print_out=print_out)
-        return None
-    except csv.Error:
-        log_error(msg=f"Error: The file {file_name} could not be read as CSV.", logger=logger, print_out=print_out)
-        return None
-    except Exception as e:
-        log_error(msg=f"An unexpected error occurred: {e}", logger=logger, print_out=print_out, exc_info=True)
-        return None
-
-def write_json_to_gcs_extended(storage_client: GCSClient, data: dict | list | str, bucket_name: str, file_name: str,
-                                                duplication_handling_enum: DuplicationHandling, duplication_match_condition_type_enum: MatchConditionType,
-                                                duplication_match_condition: str = "", max_retries: int = 2, max_deletable_files: int = 1,
-                                                logger=None, print_out=False, raise_e=False, pipelinemon: Pipelinemon = None):
-
-    """Saves data to Google Cloud Storage with optional Pipelinemon monitoring.
-
-    Handles duplication with strategies: OVERWRITE, INCREMENT, SKIP, or RAISE_ERROR.
-    """
-
-    max_deletable_files_allowed = 3
-    cloud_storage_ref=DataSourceType.GCS.value
-
-    # GCS-related metadata
-    saved_to_path = None
-    matched_duplicates_count = 0
-    matched_duplicates_deleted = []
-    duplication_handling_status = None
-    error_during_operation = None
-
-    response = {
-        "saved_to_path": saved_to_path,
-        "matched_duplicates_count": matched_duplicates_count,
-        "matched_duplicates_deleted": matched_duplicates_deleted,
-        "duplication_handling_status": duplication_handling_status,
-        "duplication_match_condition_type": duplication_match_condition_type_enum.value,
-        "duplication_match_condition": duplication_match_condition,
-        "error_during_operation": error_during_operation
-    }
-
-    supported_match_condition_types = [MatchConditionType.EXACT, MatchConditionType.PREFIX]
-    supported_duplication_handling = [DuplicationHandling.RAISE_ERROR, DuplicationHandling.OVERWRITE, DuplicationHandling.INCREMENT, DuplicationHandling.SKIP]
-
-    try:
-        if max_deletable_files > max_deletable_files_allowed:
-            raise ValueError(f"max_deletable_files should be less than or equal to {max_deletable_files_allowed} for safety.")
-        if duplication_handling_enum not in supported_duplication_handling:
-            msg = f"Error: Duplication handling not supported. Supported types: {[dh.value for dh in supported_duplication_handling]}"
-            raise ValueError(msg)
-        if duplication_match_condition_type_enum not in supported_match_condition_types:
-            msg = f"Error: Match condition type not supported. Supported types: {[mct.value for mct in supported_match_condition_types]}"
-            raise ValueError(msg)
-        elif duplication_match_condition_type_enum != MatchConditionType.EXACT and not duplication_match_condition:
-            msg = f"Error: Match condition is required for match condition type: {duplication_match_condition_type_enum.value}"
-            raise ValueError(msg)
-
-        # Prepare data
-        if isinstance(data, (list, dict)):
-            data_str = json.dumps(data, indent=2)
-        else:
-            data_str = data
-
-        increment = 0
-        attempts = 0
-        success = False
-
-        # Check for existing files based on duplication_match_condition_type
-        files_matched_on_condition = []
-        bucket = storage_client.bucket(bucket_name)
-        base_file_name, ext = os.path.splitext(file_name)
-        if duplication_match_condition_type_enum == MatchConditionType.PREFIX:
-            files_matched_on_condition = list(bucket.list_blobs(prefix=duplication_match_condition))
-        elif duplication_match_condition_type_enum == MatchConditionType.EXACT:
-            duplication_match_condition = file_name if not duplication_match_condition else duplication_match_condition
-            if bucket.blob(duplication_match_condition).exists():
-                files_matched_on_condition = [bucket.blob(file_name)]
-
-        matched_duplicates_count = len(files_matched_on_condition)
-        response["matched_duplicates_count"] = matched_duplicates_count
-
-        # Handle duplication based on duplication_handling
-        if matched_duplicates_count:
-            log_msg = f"Duplicate FOUND, matched_duplicates_count: {matched_duplicates_count}"
-            if pipelinemon:
-                    pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject="duplicate_found", description=log_msg))
-
-            if duplication_handling_enum == DuplicationHandling.RAISE_ERROR:
-                raise FileExistsError("File(s) matching the condition already exist.")
-
-            if duplication_handling_enum == DuplicationHandling.SKIP:
-                response["duplication_handling_status"] = DuplicationHandlingStatus.SKIPPED.value
-                log_msg = f"SKIPPING, response: {response}"
-                log_info(log_msg, logger=logger, print_out=print_out) ## only logsor prints if logger is provided and print_out is True
-                return response
-
-            if duplication_handling_enum == DuplicationHandling.OVERWRITE:
-                if matched_duplicates_count > max_deletable_files:
-                    raise ValueError(f"Error: Attempt to delete {matched_duplicates_count} matched files, but limit is {max_deletable_files}. Operation Cancelled.")
-
-                for blob in files_matched_on_condition:
-                    cloud_storage_path_to_delete = f"gs://{bucket_name}/{blob.name}"
-                    blob.delete()
-                    matched_duplicates_deleted.append(cloud_storage_path_to_delete)
-                    log_msg = f"File deleted as part of overwrite: {cloud_storage_path_to_delete}"
-                    if pipelinemon:
-                        pipelinemon.add_system_impacted(f"delete: {cloud_storage_ref}_bucket_file: {cloud_storage_path_to_delete}")
-                        pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_DELETE_COMPLETE, subject="delete_duplicate", description=log_msg))
-                    log_info(log_msg, logger=logger, print_out=print_out)
-
-                response["matched_duplicates_deleted"] = matched_duplicates_deleted
-                response["duplication_handling_status"] = DuplicationHandlingStatus.OVERWRITTEN.value
-
-            elif duplication_handling_enum == DuplicationHandling.INCREMENT:
-                while bucket.blob(file_name).exists():
-                    increment += 1
-                    file_name = f"{base_file_name}_v{increment}{ext}"
-                saved_to_path = f"gs://{bucket_name}/{file_name}"
-                response["duplication_handling_status"] = DuplicationHandlingStatus.INCREMENTED.value
-                log_msg = "INCREMENTING as Duplicate FOUND "
-                log_info(log_msg, logger=logger, print_out=print_out) ## only logsor prints if logger is provided and print_out is True
-
-        # GCS Upload
-        saved_to_path = f"gs://{bucket_name}/{file_name}"
-        while attempts < max_retries and not success:
-            try:
-                blob = bucket.blob(file_name)
-                blob.upload_from_string(data_str, content_type='application/json')
-                log_msg = f"File uploaded to GCS: {saved_to_path}"
-                if pipelinemon:
-                    pipelinemon.add_system_impacted(f"upload: {cloud_storage_ref}_bucket_file: {saved_to_path}")
-                    pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_PERSISTNACE_COMPLETE, subject="file_upload", description=log_msg))
-                log_info(log_msg, logger=logger, print_out=print_out)
-                success = True
-            except Exception as e:
-                attempts += 1
-                if attempts < max_retries:
-                    time.sleep(2 ** attempts)
-                else:
-                    raise e
-
-    except Exception as e:
-        error_during_operation = f"Error occurred while writing JSON to GCS path: {saved_to_path} ; Error details: {type(e).__name__} - {str(e)}"
-        response["error_during_operation"] = error_during_operation
-        if pipelinemon:
-            pipelinemon.add_log(ContextLog(LogLevel.ERROR_EXCEPTION, e=e, description="response: {response}"))
-        log_error(response, logger=logger, print_out=print_out)
-        if raise_e:
-            raise e
-
-    response["saved_to_path"] = saved_to_path if success else None
-    return response
-
-# def write_json_to_gcs_extended(storage_client: GCSClient, data: dict | list | str, bucket_name: str, file_name: str,
-#                                duplication_handling: DuplicationHandling, duplication_match_condition_type: MatchConditionType,
-#                                duplication_match_condition: str | List[str] = "", max_retries: int = 2, max_deletable_files: int = 1,
-#                                logger=None, print_out=False, raise_e=False):
-
-#     """Saves data to Google Cloud Storage.
-
-#     Handles duplication with strategies: OVERWRITE, INCREMENT, SKIP, or RAISE_ERROR.
-#     """
-
-#     max_deletable_files_allowed = 3
-
-#     # GCS-related metadata
-#     saved_to_path = None
-#     matched_duplicates_count = 0
-#     matched_duplicates_deleted = []
-#     duplication_handling_status = None
-#     error_during_operation = None
-
-#     response = {
-#         "saved_to_path": saved_to_path,
-#         "matched_duplicates_count": matched_duplicates_count,
-#         "matched_duplicates_deleted": matched_duplicates_deleted,
-#         "duplication_handling_status": duplication_handling_status,
-#         "duplication_match_condition_type": duplication_match_condition_type,
-#         "duplication_match_condition": duplication_match_condition,
-#         "error_during_operation": error_during_operation
-#     }
-
-#     supported_match_condition_types = [MatchConditionType.EXACT, MatchConditionType.PREFIX]
-#     supported_duplication_handling = [DuplicationHandling.RAISE_ERROR, DuplicationHandling.OVERWRITE, DuplicationHandling.INCREMENT, DuplicationHandling.SKIP]
-
-#     try:
-#         if max_deletable_files > max_deletable_files_allowed:
-#             raise ValueError(f"max_deletable_files should be less than or equal to {max_deletable_files_allowed} for safety.")
-#         if duplication_handling not in supported_duplication_handling:
-#             msg = f"Error: Duplication handling not supported. Supported types: {supported_duplication_handling}"
-#             raise ValueError(msg)
-#         if duplication_match_condition_type not in supported_match_condition_types:
-#             msg = f"Error: Match condition type not supported. Supported types: {supported_match_condition_types}"
-#             raise ValueError(msg)
-#         elif duplication_match_condition_type!=MatchConditionType.EXACT and not duplication_match_condition:
-#             msg = f"Error: Match condition is required for match condition type: {duplication_match_condition_type}"
-#             raise ValueError(msg)
-
-#         # Prepare data
-#         if isinstance(data, (list, dict)):
-#             data_str = json.dumps(data, indent=2)
-#         else:
-#             data_str = data
-
-#         increment = 0
-#         attempts = 0
-#         success = False
-
-#         # Check for existing files based on duplication_match_condition_type
-#         files_matched_on_condition = []
-#         bucket = storage_client.bucket(bucket_name)
-#         base_file_name, ext = os.path.splitext(file_name)
-#         if duplication_match_condition_type == MatchConditionType.PREFIX:
-#             files_matched_on_condition = list(bucket.list_blobs(prefix=duplication_match_condition))
-#         elif duplication_match_condition_type == MatchConditionType.EXACT:
-#             if bucket.blob(file_name).exists():
-#                 files_matched_on_condition = [bucket.blob(file_name)]
-
-#         matched_duplicates_count = len(files_matched_on_condition)
-#         response["matched_duplicates_count"] = matched_duplicates_count
-
-#         # Handle duplication based on duplication_handling
-#         if matched_duplicates_count:
-#             if duplication_handling == DuplicationHandling.RAISE_ERROR:
-#                 raise FileExistsError("File(s) matching the condition already exist.")
-
-#             if duplication_handling == DuplicationHandling.SKIP:
-#                 log_warning("Skipping saving to GCS: file(s) matching the condition already exist.", logger=logger, print_out=print_out)
-#                 response["duplication_handling_status"] = DuplicationHandlingStatus.SKIPPED.value
-#                 return response
-
-#             if duplication_handling == DuplicationHandling.OVERWRITE:
-#                 if matched_duplicates_count > max_deletable_files:
-#                     raise ValueError(f"Error: Attempt to delete {matched_duplicates_count} matched files, but limit is {max_deletable_files}. Operation Cancelled.")
-
-#                 for blob in files_matched_on_condition:
-#                     cloud_storage_path_to_delete = f"gs://{bucket_name}/{blob.name}"
-#                     blob.delete()
-#                     matched_duplicates_deleted.append(cloud_storage_path_to_delete)
-
-#                 response["matched_duplicates_deleted"] = matched_duplicates_deleted
-#                 response["duplication_handling_status"] = DuplicationHandlingStatus.OVERWRITTEN.value
-
-#             elif duplication_handling == DuplicationHandling.INCREMENT:
-#                 while bucket.blob(file_name).exists():
-#                     increment += 1
-#                     file_name = f"{base_file_name}_v{increment}{ext}"
-#                 saved_to_path = f"gs://{bucket_name}/{file_name}"
-#                 response["duplication_handling_status"] = DuplicationHandlingStatus.INCREMENTED.value
-
-#         # GCS Upload
-#         saved_to_path = f"gs://{bucket_name}/{file_name}"
-#         while attempts < max_retries and not success:
-#             try:
-#                 blob = bucket.blob(file_name)
-#                 blob.upload_from_string(data_str, content_type='application/json')
-#                 success = True
-#             except Exception as e:
-#                 attempts += 1
-#                 if attempts < max_retries:
-#                     time.sleep(2 ** attempts)
-#                 else:
-#                     if raise_e:
-#                         raise e
-
-#     except Exception as e:
-#         error_message = f"Error occurred while writing JSON to GCS path: {saved_to_path} : {type(e).__name__} - {str(e)}"
-#         log_error(error_message, logger=logger, print_out=print_out)
-#         response["error_during_operation"] = error_message
-#         if raise_e:
-#             raise e
-
-#     response["saved_to_path"] = saved_to_path if success else None
-#     return response
-
-
-def write_csv_to_gcs(bucket_name:str, file_name:str, data:dict | list | str, storage_client:GCSClient, logger=None, print_out=False, raise_e=False):
-    """ Helper function to write a CSV file to Google Cloud Storage """
-    try:
-        bucket = storage_client.bucket(bucket_name)
-        blob = bucket.blob(file_name)
-        data_file = StringIO()
-        if data and isinstance(data, list) and isinstance(data[0], dict):
-            fieldnames = data[0].keys()
-            writer = csv.DictWriter(data_file, fieldnames=fieldnames)
-            writer.writeheader()
-            writer.writerows(data)
-        else:
-            raise ValueError("Data should be a list of dictionaries")
-        blob.upload_from_string(data_file.getvalue(), content_type='text/csv')
-        log_info(msg=f"Successfully wrote CSV to {file_name} in bucket {bucket_name}.", logger=logger, print_out=print_out)
-    except ValueError as e:
-        log_error(msg=f"ValueError: {e}",logger=logger, print_out=print_out)
-        if raise_e:
-            raise e
-    except Exception as e:
-        log_error(msg=f"An unexpected error occurred while writing CSV to GCS: {e}", logger=logger, print_out=print_out, exc_info=True)
-        if raise_e:
-            raise e

ipulse_shared_core_ftredge/utils/utils_cloud_gcp_with_collectors.py

@@ -1,166 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-# pylint: disable=broad-exception-caught
-# pylint: disable=line-too-long
-# pylint: disable=unused-variable
-# pylint: disable=broad-exception-raised
-
-# import json
-# import os
-# import time
-# from google.cloud.storage import Client as GCSClient
-# from ipulse_shared_core_ftredge.enums import LogLevel, DuplicationHandling, DuplicationHandlingStatus, MatchConditionType, DataSourceType
-# from ipulse_shared_core_ftredge.utils import log_error, log_info
-# from .utils_collector_pipelinemon import Pipelinemon
-# from .logs.context_log import ContextLog
-
-
-
-
-
-# def write_json_to_gcs_with_pipelinemon_extended( pipelinemon:Pipelinemon, storage_client:GCSClient, data:dict | list | str, bucket_name: str, file_name: str,
-#                       file_exists_if_starts_with_prefix:Optional[str] =None, overwrite_if_exists:bool=False, increment_if_exists:bool=False,
-#                       max_retries:int=2, max_deletable_files:int=1):
-#     """Saves data to Google Cloud Storage and optionally locally.
-    
-#     This function attempts to upload data to GCS. 
-#     - If the upload fails after retries and `save_locally` is True or `local_path` is provided, it attempts to save the data locally.
-#     - It handles file name conflicts based on these rules:
-#         - If `overwrite_if_exists` is True: 
-#             - If `file_exists_if_contains_substr` is provided, ANY existing file containing the substring is deleted, and the new file is saved with the provided `file_name`.
-#             - If `file_exists_if_contains_substr` is None, and a file with the exact `file_name` exists, it's overwritten.
-#         - If `increment_if_exists` is True:
-#             - If `file_exists_if_contains_substr` is provided, a new file with an incremented version is created ONLY if a file with the EXACT `file_name` exists.
-#             - If `file_exists_if_contains_substr` is None, a new file with an incremented version is created if a file with the exact `file_name` exists. 
-            
-#         -If both overwrite_if_exists and increment_if_exists are provided as Ture, an exception will be raised.
-#     """
-
-#     cloud_storage_ref="GCP_GCS"
-
-#     with pipelinemon.context(f"write_json_to_{cloud_storage_ref}_with_pipelinemon"):
-#         cloud_storage_upload_error = None
-#         # Input validation
-#         if overwrite_if_exists and increment_if_exists:
-#             err_msg="Both 'overwrite_if_exists' and 'increment_if_exists' cannot be True simultaneously."
-#             pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM, subject="Param validation", description=err_msg))
-#             return {"cloud_storage_upload_error": err_msg}
-#         if max_deletable_files > 10:
-#             err_msg="max_deletable_files should be less than 10 for safety. For more use another method."
-#             pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM,subject="max_deletable_files", description=err_msg))
-#             return {"cloud_storage_upload_error": err_msg}
-
-#         # Prepare data
-#         if isinstance(data, (list, dict)):
-#             data_str = json.dumps(data, indent=2)
-#         else:
-#             data_str = data
-
-#         bucket = storage_client.bucket(bucket_name)
-#         base_file_name, ext = os.path.splitext(file_name)
-#         increment = 0
-#         attempts = 0
-#         success = False
-
-#         # GCS-related metadata
-#         cloud_storage_path = None
-#         cloud_storage_file_overwritten = False
-#         cloud_storage_file_already_exists = False
-#         cloud_storage_file_saved_with_increment = False
-#         cloud_storage_file_exists_checked_on_name = file_name
-#         cloud_storage_deleted_files=[]
-
-#         try:
-#             upload_allowed = True
-#             # --- Overwrite Logic ---
-#             if overwrite_if_exists:
-#                 with pipelinemon.context("overwriting"):
-#                     if file_exists_if_starts_with_prefix:
-#                         cloud_storage_file_exists_checked_on_name = file_exists_if_starts_with_prefix
-#                         blobs_to_delete = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
-#                         if len(blobs_to_delete) > max_deletable_files:
-#                             err_msg=f"Error: Attempt to delete {len(blobs_to_delete)} matched files, but limit is {max_deletable_files}."
-#                             pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_to_delete)} files in bucket {bucket_name}"))
-#                             pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM, subject="Too many files", description=err_msg))
-#                             #### Ensuring to quit the operation if too many files are found, it will be catched below
-#                             return {"cloud_storage_upload_error": err_msg}
-#                         if blobs_to_delete:
-#                             cloud_storage_file_already_exists = True
-#                             pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_to_delete)} files in bucket {bucket_name}"))
-#                             for blob in blobs_to_delete:
-#                                 cloud_storage_path_del = f"gs://{bucket_name}/{blob.name}"
-#                                 pipelinemon.add_system_impacted(f"delete: {cloud_storage_ref}_bucket_file: {cloud_storage_path_del}")
-#                                 blob.delete()
-#                                 pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_DELETE_COMPLETE, subject= cloud_storage_path_del, description=f"file deleted from {cloud_storage_ref} as part of overwrite, matched with prefix"))
-#                                 cloud_storage_deleted_files.append(cloud_storage_path_del)
-#                             cloud_storage_file_overwritten = True
-#                     elif bucket.blob(file_name).exists():
-#                         cloud_storage_file_already_exists = True
-#                         pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"Exact name matched with existing file in bucket {bucket_name}"))
-#                         cloud_storage_path_del = f"gs://{bucket_name}/{file_name}"
-#                         pipelinemon.add_system_impacted(f"delete: {cloud_storage_ref}_bucket_file: {cloud_storage_path_del}")
-#                         blob.delete()  # Delete the existing blob
-#                         pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_DELETE_COMPLETE, subject= cloud_storage_path_del, description=f"file deleted from {cloud_storage_ref} as part of overwrite, matched with exact name"))
-#                         cloud_storage_deleted_files.append(cloud_storage_path_del)
-#                         cloud_storage_file_overwritten = True
-#             # --- Increment Logic ---
-#             elif increment_if_exists:
-#                 with pipelinemon.context("incrementing"):
-#                     cloud_storage_file_exists_checked_on_name = file_name  # We only increment if the exact name exists
-#                     while bucket.blob(file_name).exists():
-#                         cloud_storage_file_already_exists = True
-#                         increment += 1
-#                         file_name = f"{base_file_name}_v{increment}{ext}"
-#                         cloud_storage_file_saved_with_increment = True
-#                     if increment>0:
-#                         cloud_storage_path = f"gs://{bucket_name}/{file_name}"
-#                         pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"Attempting to save file with incremented version in {bucket_name}"))
-#              # --- Check for Conflicts (Including Prefix) ---
-#             else:
-#                 if file_exists_if_starts_with_prefix:
-#                     blobs_matched = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
-#                     cloud_storage_file_exists_checked_on_name = file_exists_if_starts_with_prefix
-#                     if blobs_matched:
-#                         upload_allowed = False
-#                         cloud_storage_file_already_exists = True
-#                         pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_matched)} existing files in bucket {bucket_name}."))
-#                 elif bucket.blob(file_name).exists():
-#                     pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"Exact name matched with existing file in bucket {bucket_name}."))
-#                     upload_allowed = False
-#                     cloud_storage_file_already_exists = True
-
-#             # --- GCS Upload ---
-#             cloud_storage_path = f"gs://{bucket_name}/{file_name}"
-#             if overwrite_if_exists or increment_if_exists or upload_allowed:
-#                 with pipelinemon.context("uploading"):
-#                     while attempts < max_retries and not success:
-#                         try:
-#                             blob = bucket.blob(file_name)  # Use the potentially updated file_name
-#                             pipelinemon.add_system_impacted(f"upload: {cloud_storage_ref}_bucket_file: {cloud_storage_path}")
-#                             blob.upload_from_string(data_str, content_type='application/json')
-#                             pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_PERSISTNACE_COMPLETE, subject= cloud_storage_path, description=f"file uploaded to {cloud_storage_ref}"))
-#                             success = True
-#                         except Exception as e:
-#                             attempts += 1
-#                             if attempts < max_retries:
-#                                 time.sleep(2 ** attempts)
-#                             else:
-#                                 err_msg=f"Error uploading file to {cloud_storage_ref} bucket {bucket_name} with name {file_name} : {type(e).__name__}-{str(e)}"
-#                                 pipelinemon.add_log(ContextLog(LogLevel.ERROR_EXCEPTION, e=e, description=err_msg))
-#                                 return {"cloud_storage_upload_error": err_msg}
-
-#         except Exception as e:
-#             pipelinemon.add_log(ContextLog(LogLevel.ERROR_EXCEPTION, e=e))
-#             return {"cloud_storage_upload_error": f"Exception in GCS upload {type(e).__name__}-{str(e)}"}
-#     # --- Return Metadata ---
-#     return {
-#         "cloud_storage_path": cloud_storage_path if ((success or not upload_allowed) and not cloud_storage_upload_error ) else None,
-#         "cloud_storage_file_already_exists": cloud_storage_file_already_exists,
-#         "cloud_storage_file_exists_checked_on_name":cloud_storage_file_exists_checked_on_name ,
-#         "cloud_storage_file_overwritten": cloud_storage_file_overwritten,
-#         "cloud_storage_deleted_file_names": ",,,".join(cloud_storage_deleted_files) if cloud_storage_deleted_files else None,
-#         "cloud_storage_file_saved_with_increment": cloud_storage_file_saved_with_increment,
-#         "cloud_storage_upload_error": cloud_storage_upload_error
-#     }
-

ipulse_shared_core_ftredge/utils/utils_cloud_with_collectors.py

@@ -1,27 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-# pylint: disable=broad-exception-caught
-# pylint: disable=line-too-long
-# pylint: disable=unused-variable
-# pylint: disable=broad-exception-raised
-# from typing import Optional
-# from ipulse_shared_core_ftredge.enums import DataSourceType
-# from .utils_collector_pipelinemon import Pipelinemon
-# from .utils_cloud_gcp import write_json_to_gcs_extended
-
-
-# def write_json_to_cloud_storage_with_pipelinemon_extended(cloud_storage_type:DataSourceType, cloud_storage_client, pipelinemon:Pipelinemon,
-#                                                            data:dict | list | str, bucket_name: str, file_name: str,
-                                                            
-                                                            
-#                                                             max_retries:int=2, max_deletable_files:int=1):
-
-#     supported_cloud_storage_types = [DataSourceType.GCS]
-#     if cloud_storage_type == DataSourceType.GCS:
-#         return write_json_to_gcs_extended(pipelinemon=pipelinemon, storage_client=cloud_storage_client, data=data, bucket_name=bucket_name, file_name=file_name,
-#                                                   ,
-#                                                   max_retries=max_retries,
-#                                                   max_deletable_files=max_deletable_files)
-
-#     raise ValueError(f"Unsupported cloud provider: {cloud_storage_type}. Supported cloud providers: {supported_cloud_storage_types}")