ipulse-shared-core-ftredge 2.6.1__py3-none-any.whl → 2.7.1__py3-none-any.whl

ipulse_shared_core_ftredge/utils/utils_cloud_gcp.py

@@ -11,76 +11,29 @@ from io import StringIO
 import os
 import time
 import logging
-from typing import Optional
 import traceback
 from google.api_core.exceptions import NotFound
 from google.cloud import error_reporting
 from google.cloud import logging as cloud_logging
 from google.cloud.storage import Client as GCSClient
 from google.cloud import bigquery
+from ipulse_shared_core_ftredge.enums import DuplicationHandling, DuplicationHandlingStatus, MatchConditionType,DataSourceType, LogLevel
+from ipulse_shared_core_ftredge.utils.utils_common import log_error, log_warning, log_info
+from ipulse_shared_core_ftredge.utils.logs import ContextLog
+from ipulse_shared_core_ftredge.utils.utils_collector_pipelinemon import Pipelinemon
 
 ############################################################################
-##################### GOOGLE CLOUD  UTILS ##################################
+##################### GOOGLE CLOUD PLATFORM UTILS ##################################
 ############################################################################
 
-def log_error(message, logger=None , print_out=False, exc_info=False):
-    if logger:
-        logger.error(message, exc_info=exc_info)
-    elif print_out:
-        print(message)
+class CustomGCPLoggingHandler(cloud_logging.handlers.CloudLoggingHandler):
+    """Custom handler for Google Cloud Logging with a dynamic logName."""
+    def __init__(self, client, name, resource=None, labels=None):
+        super().__init__(client=client, name=name, resource=resource, labels=labels)
+        self.client = client  # Ensure client is consistently used
 
-def log_warning(message, logger=None, print_out=False):
-    if logger:
-        logger.warning(message)
-    elif print_out:
-        print(message)
-
-def log_info(message, logger=None, print_out=False):
-    if logger:
-        logger.info(message)
-    elif print_out:
-        print(message)
-
-
-
-############################################################################
-##################### LOGGING and ERROR reporting ##########################
-####DEPCREACATED: THIS APPROACH WAS GOOD, BUT ERRORS WERE NOT REPORTED TO ERROR REPORTING
-# logging.basicConfig(level=logging.INFO)
-# logging_client = google.cloud.logging.Client()
-# logging_client.setup_logging()
-###################################
-def setup_gcp_logging(logger, formatter, enable_error_reporting=True):
-
-    class CustomGCPErrorReportingHandler(logging.Handler):
-        def __init__(self, level=logging.ERROR):
-            super().__init__(level)
-            self.error_client = error_reporting.Client()
-            self.propagate = True
-
-        def emit(self, record):
-            try:
-                if record.levelno >= logging.ERROR:
-                    log_struct = {
-                    'message': self.format(record),
-                    'severity': record.levelname,
-                    'pathname': getattr(record, 'pathname', None),
-                    'lineno': getattr(record, 'lineno', None)
-                }
-                if record.exc_info:
-                    log_struct['exception'] = ''.join(
-                        traceback.format_exception(*record.exc_info)
-                    )
-                self.error_client.report(str(log_struct))
-            except Exception as e:
-                self.handleError(record)
-        
-    class CustomGCPLoggingHandler(cloud_logging.handlers.CloudLoggingHandler):
-        """Custom handler for Google Cloud Logging with a dynamic logName."""
-        def __init__(self, client, name, resource=None, labels=None):
-            super().__init__(client=client, name=name, resource=resource, labels=labels)
-            
-        def emit(self, record):
+    def emit(self, record):
+        try:
             # 1. Create the basic log entry dictionary
             log_entry = {
                 'message': record.msg,
@@ -109,17 +62,45 @@ def setup_gcp_logging(logger, formatter, enable_error_reporting=True):
 
             # 4. Send to Google Cloud Logging
             super().emit(record)
+        except Exception as e:
+            self.handleError(record)
+
+class CustomGCPErrorReportingHandler(logging.Handler):
+    def __init__(self, client=None, level=logging.ERROR):
+        super().__init__(level)
+        self.error_client = error_reporting.Client() if client is None else client
+        self.propagate = True
+
+    def emit(self, record):
+        try:
+            if record.levelno >= logging.ERROR:
+                log_struct = {
+                'message': self.format(record),
+                'severity': record.levelname,
+                'pathname': getattr(record, 'pathname', None),
+                'lineno': getattr(record, 'lineno', None)
+            }
+            if record.exc_info:
+                log_struct['exception'] = ''.join(
+                    traceback.format_exception(*record.exc_info)
+                )
+            self.error_client.report(str(log_struct))
+        except Exception as e:
+            self.handleError(record)
+
 
-    # Create Google Cloud Logging handler
-    cloud_logging_client = cloud_logging.Client()
-    cloud_logging_handler = CustomGCPLoggingHandler(cloud_logging_client, logger.name)  # No prefix needed
-    cloud_logging_handler.setFormatter(formatter)
-    logger.addHandler(cloud_logging_handler)
+def add_gcp_cloud_logging(logger, formatter, client=None):
+    """Sets up Google Cloud Logging for the logger."""
+    client = client or cloud_logging.Client()
+    handler = CustomGCPLoggingHandler(client, logger.name)
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
 
-    if enable_error_reporting:
-        # Create and add Error Reporting handler
-        error_reporting_handler = CustomGCPErrorReportingHandler()
-        logger.addHandler(error_reporting_handler)
+def add_gcp_error_reporting(logger, client=None):
+    """Sets up Google Cloud Error Reporting for the logger."""
+    client = client or error_reporting.Client()
+    handler = CustomGCPErrorReportingHandler(client=client)
+    logger.addHandler(handler)
 
 
 
@@ -142,13 +123,13 @@ def read_json_from_gcs(storage_client:GCSClient, bucket_name:str, file_name:str,
         data = json.loads(data_string)
         return data
     except NotFound:
-        log_error(message=f"Error: The file {file_name} was not found in the bucket {bucket_name}.", logger=logger, print_out=print_out)
+        log_warning(msg=f"Warning: The file {file_name} was not found in the bucket {bucket_name}.", logger=logger, print_out=print_out)
         return None
     except json.JSONDecodeError:
-        log_error(message=f"Error: The file {file_name} could not be decoded as JSON.", logger=logger, print_out=print_out)
+        log_error(msg=f"Error: The file {file_name} could not be decoded as JSON.", logger=logger, print_out=print_out)
         return None
     except Exception as e:
-        log_error(message=f"An unexpected error occurred: {e}", exc_info=True, logger=logger, print_out=print_out)
+        log_error(msg=f"An unexpected error occurred: {e}", exc_info=True, logger=logger, print_out=print_out)
         return None
 
 def read_csv_from_gcs(bucket_name:str, file_name:str, storage_client:GCSClient, logger=None, print_out=False):
@@ -162,114 +143,137 @@ def read_csv_from_gcs(bucket_name:str, file_name:str, storage_client:GCSClient,
         reader = csv.DictReader(data_file)
         return list(reader)
     except NotFound:
-        log_error(message=f"Error: The file {file_name} was not found in the bucket {bucket_name}.", logger=logger, print_out=print_out)
+        log_warning(msg=f"Error: The file {file_name} was not found in the bucket {bucket_name}.", logger=logger, print_out=print_out)
         return None
     except csv.Error:
-        log_error(message=f"Error: The file {file_name} could not be read as CSV.", logger=logger, print_out=print_out)
+        log_error(msg=f"Error: The file {file_name} could not be read as CSV.", logger=logger, print_out=print_out)
         return None
     except Exception as e:
-        log_error(message=f"An unexpected error occurred: {e}", logger=logger, print_out=print_out, exc_info=True)
+        log_error(msg=f"An unexpected error occurred: {e}", logger=logger, print_out=print_out, exc_info=True)
         return None
 
+def write_json_to_gcs_extended(storage_client: GCSClient, data: dict | list | str, bucket_name: str, file_name: str,
+                                                duplication_handling_enum: DuplicationHandling, duplication_match_condition_type_enum: MatchConditionType,
+                                                duplication_match_condition: str = "", max_retries: int = 2, max_deletable_files: int = 1,
+                                                logger=None, print_out=False, raise_e=False, pipelinemon: Pipelinemon = None):
 
+    """Saves data to Google Cloud Storage with optional Pipelinemon monitoring.
 
-def write_json_to_gcs_extended( storage_client:GCSClient, data:dict | list | str, bucket_name: str, file_name: str,
-                      file_exists_if_starts_with_prefix:Optional[str] =None, overwrite_if_exists:bool=False, increment_if_exists:bool=False,
-                      max_retries:int=2, max_deletable_files:int=1, logger=None, print_out=False):
-    
-    """Saves data to Google Cloud Storage and optionally locally.
-    
-    This function attempts to upload data to GCS. 
-    - If the upload fails after retries and `save_locally` is True or `local_path` is provided, it attempts to save the data locally.
-    - It handles file name conflicts based on these rules:
-        - If `overwrite_if_exists` is True: 
-            - If `file_exists_if_contains_substr` is provided, ANY existing file containing the substring is deleted, and the new file is saved with the provided `file_name`.
-            - If `file_exists_if_contains_substr` is None, and a file with the exact `file_name` exists, it's overwritten.
-        - If `increment_if_exists` is True:
-            - If `file_exists_if_contains_substr` is provided, a new file with an incremented version is created ONLY if a file with the EXACT `file_name` exists.
-            - If `file_exists_if_contains_substr` is None, a new file with an incremented version is created if a file with the exact `file_name` exists. 
-            
-        -If both overwrite_if_exists and increment_if_exists are provided as Ture, an exception will be raised.
+    Handles duplication with strategies: OVERWRITE, INCREMENT, SKIP, or RAISE_ERROR.
     """
-    # GCS upload exception
-    # Input validation
-    if overwrite_if_exists and increment_if_exists:
-        raise ValueError("Both 'overwrite_if_exists' and 'increment_if_exists' cannot be True simultaneously.")
-    if not isinstance(data, (list, dict, str)):
-        raise ValueError("Data should be a list, dict, or string.")
-    if max_deletable_files > 10:
-        raise ValueError("max_deletable_files should be less than 10 for safety. For more use another method.")
-
-    # Prepare data
-    if isinstance(data, (list, dict)):
-        data_str = json.dumps(data, indent=2)
-    else:
-        data_str = data
-
-    bucket = storage_client.bucket(bucket_name)
-    base_file_name, ext = os.path.splitext(file_name)
-    increment = 0
-    attempts = 0
-    success = False
+
+    max_deletable_files_allowed = 3
+    cloud_storage_ref=DataSourceType.GCS.value
 
     # GCS-related metadata
-    cloud_storage_path = None
-    cloud_storage_file_overwritten = False
-    cloud_storage_file_already_exists = False
-    cloud_storage_file_saved_with_increment = False
-    cloud_storage_file_exists_checked_on_name = file_name
-    cloud_storage_deleted_files=[]
-
-    
-    upload_allowed = True
-    # --- Overwrite Logic ---
-    if overwrite_if_exists:
-        if file_exists_if_starts_with_prefix:
-            cloud_storage_file_exists_checked_on_name = file_exists_if_starts_with_prefix
-            blobs_to_delete = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
-            if len(blobs_to_delete) > max_deletable_files:
-                raise Exception(f"Error: Attempt to delete {len(blobs_to_delete)} matched files, but limit is {max_deletable_files}.")
-            if blobs_to_delete:
-                for blob in blobs_to_delete:
-                    cloud_storage_path_del = f"gs://{bucket_name}/{blob.name}"
+    saved_to_path = None
+    matched_duplicates_count = 0
+    matched_duplicates_deleted = []
+    duplication_handling_status = None
+    error_during_operation = None
+
+    response = {
+        "saved_to_path": saved_to_path,
+        "matched_duplicates_count": matched_duplicates_count,
+        "matched_duplicates_deleted": matched_duplicates_deleted,
+        "duplication_handling_status": duplication_handling_status,
+        "duplication_match_condition_type": duplication_match_condition_type_enum.value,
+        "duplication_match_condition": duplication_match_condition,
+        "error_during_operation": error_during_operation
+    }
+
+    supported_match_condition_types = [MatchConditionType.EXACT, MatchConditionType.PREFIX]
+    supported_duplication_handling = [DuplicationHandling.RAISE_ERROR, DuplicationHandling.OVERWRITE, DuplicationHandling.INCREMENT, DuplicationHandling.SKIP]
+
+    try:
+        if max_deletable_files > max_deletable_files_allowed:
+            raise ValueError(f"max_deletable_files should be less than or equal to {max_deletable_files_allowed} for safety.")
+        if duplication_handling_enum not in supported_duplication_handling:
+            msg = f"Error: Duplication handling not supported. Supported types: {[dh.value for dh in supported_duplication_handling]}"
+            raise ValueError(msg)
+        if duplication_match_condition_type_enum not in supported_match_condition_types:
+            msg = f"Error: Match condition type not supported. Supported types: {[mct.value for mct in supported_match_condition_types]}"
+            raise ValueError(msg)
+        elif duplication_match_condition_type_enum != MatchConditionType.EXACT and not duplication_match_condition:
+            msg = f"Error: Match condition is required for match condition type: {duplication_match_condition_type_enum.value}"
+            raise ValueError(msg)
+
+        # Prepare data
+        if isinstance(data, (list, dict)):
+            data_str = json.dumps(data, indent=2)
+        else:
+            data_str = data
+
+        increment = 0
+        attempts = 0
+        success = False
+
+        # Check for existing files based on duplication_match_condition_type
+        files_matched_on_condition = []
+        bucket = storage_client.bucket(bucket_name)
+        base_file_name, ext = os.path.splitext(file_name)
+        if duplication_match_condition_type_enum == MatchConditionType.PREFIX:
+            files_matched_on_condition = list(bucket.list_blobs(prefix=duplication_match_condition))
+        elif duplication_match_condition_type_enum == MatchConditionType.EXACT:
+            duplication_match_condition = file_name if not duplication_match_condition else duplication_match_condition
+            if bucket.blob(duplication_match_condition).exists():
+                files_matched_on_condition = [bucket.blob(file_name)]
+
+        matched_duplicates_count = len(files_matched_on_condition)
+        response["matched_duplicates_count"] = matched_duplicates_count
+
+        # Handle duplication based on duplication_handling
+        if matched_duplicates_count:
+            log_msg = f"Duplicate FOUND, matched_duplicates_count: {matched_duplicates_count}"
+            if pipelinemon:
+                    pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject="duplicate_found", description=log_msg))
+
+            if duplication_handling_enum == DuplicationHandling.RAISE_ERROR:
+                raise FileExistsError("File(s) matching the condition already exist.")
+
+            if duplication_handling_enum == DuplicationHandling.SKIP:
+                response["duplication_handling_status"] = DuplicationHandlingStatus.SKIPPED.value
+                log_msg = f"SKIPPING, response: {response}"
+                log_info(log_msg, logger=logger, print_out=print_out) ## only logsor prints if logger is provided and print_out is True
+                return response
+
+            if duplication_handling_enum == DuplicationHandling.OVERWRITE:
+                if matched_duplicates_count > max_deletable_files:
+                    raise ValueError(f"Error: Attempt to delete {matched_duplicates_count} matched files, but limit is {max_deletable_files}. Operation Cancelled.")
+
+                for blob in files_matched_on_condition:
+                    cloud_storage_path_to_delete = f"gs://{bucket_name}/{blob.name}"
                     blob.delete()
-                    cloud_storage_deleted_files.append(cloud_storage_path_del)
-                cloud_storage_file_overwritten = True
-        elif bucket.blob(file_name).exists():
-            cloud_storage_file_already_exists = True
-            cloud_storage_path_del = f"gs://{bucket_name}/{file_name}"
-            blob.delete()  # Delete the existing blob
-            cloud_storage_deleted_files.append(cloud_storage_path_del)
-            cloud_storage_file_overwritten = True
-    # --- Increment Logic ---
-    elif increment_if_exists:
-        cloud_storage_file_exists_checked_on_name = file_name  # We only increment if the exact name exists
-        while bucket.blob(file_name).exists():
-            cloud_storage_file_already_exists = True
-            increment += 1
-            file_name = f"{base_file_name}_v{increment}{ext}"
-            cloud_storage_file_saved_with_increment = True
-        if increment>0:
-            cloud_storage_path = f"gs://{bucket_name}/{file_name}"
-        # --- Check for Conflicts (Including Prefix) ---
-    else:
-        if file_exists_if_starts_with_prefix:
-            blobs_matched = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
-            cloud_storage_file_exists_checked_on_name = file_exists_if_starts_with_prefix
-            if blobs_matched:
-                upload_allowed = False
-                cloud_storage_file_already_exists = True
-        elif bucket.blob(file_name).exists():
-            upload_allowed = False
-            cloud_storage_file_already_exists = True
-
-    # --- GCS Upload ---
-    cloud_storage_path = f"gs://{bucket_name}/{file_name}"
-    if overwrite_if_exists or increment_if_exists or upload_allowed:
+                    matched_duplicates_deleted.append(cloud_storage_path_to_delete)
+                    log_msg = f"File deleted as part of overwrite: {cloud_storage_path_to_delete}"
+                    if pipelinemon:
+                        pipelinemon.add_system_impacted(f"delete: {cloud_storage_ref}_bucket_file: {cloud_storage_path_to_delete}")
+                        pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_DELETE_COMPLETE, subject="delete_duplicate", description=log_msg))
+                    log_info(log_msg, logger=logger, print_out=print_out)
+
+                response["matched_duplicates_deleted"] = matched_duplicates_deleted
+                response["duplication_handling_status"] = DuplicationHandlingStatus.OVERWRITTEN.value
+
+            elif duplication_handling_enum == DuplicationHandling.INCREMENT:
+                while bucket.blob(file_name).exists():
+                    increment += 1
+                    file_name = f"{base_file_name}_v{increment}{ext}"
+                saved_to_path = f"gs://{bucket_name}/{file_name}"
+                response["duplication_handling_status"] = DuplicationHandlingStatus.INCREMENTED.value
+                log_msg = "INCREMENTING as Duplicate FOUND "
+                log_info(log_msg, logger=logger, print_out=print_out) ## only logsor prints if logger is provided and print_out is True
+
+        # GCS Upload
+        saved_to_path = f"gs://{bucket_name}/{file_name}"
         while attempts < max_retries and not success:
             try:
-                blob = bucket.blob(file_name)  # Use the potentially updated file_name
+                blob = bucket.blob(file_name)
                 blob.upload_from_string(data_str, content_type='application/json')
+                log_msg = f"File uploaded to GCS: {saved_to_path}"
+                if pipelinemon:
+                    pipelinemon.add_system_impacted(f"upload: {cloud_storage_ref}_bucket_file: {saved_to_path}")
+                    pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_PERSISTNACE_COMPLETE, subject="file_upload", description=log_msg))
+                log_info(log_msg, logger=logger, print_out=print_out)
                 success = True
             except Exception as e:
                 attempts += 1
@@ -278,18 +282,142 @@ def write_json_to_gcs_extended( storage_client:GCSClient, data:dict | list | str
                 else:
                     raise e
 
-    # --- Return Metadata ---
-    return {
-        "cloud_storage_path": cloud_storage_path if (success or not upload_allowed ) else None,
-        "cloud_storage_file_already_exists": cloud_storage_file_already_exists,
-        "cloud_storage_file_exists_checked_on_name":cloud_storage_file_exists_checked_on_name ,
-        "cloud_storage_file_overwritten": cloud_storage_file_overwritten,
-        "cloud_storage_deleted_file_names": ",,,".join(cloud_storage_deleted_files) if cloud_storage_deleted_files else None,
-        "cloud_storage_file_saved_with_increment": cloud_storage_file_saved_with_increment
-    }
-
-
-def write_csv_to_gcs(bucket_name:str, file_name:str, data:dict | list | str, storage_client:GCSClient, logger,log_info_verbose=True):
+    except Exception as e:
+        error_during_operation = f"Error occurred while writing JSON to GCS path: {saved_to_path} ; Error details: {type(e).__name__} - {str(e)}"
+        response["error_during_operation"] = error_during_operation
+        if pipelinemon:
+            pipelinemon.add_log(ContextLog(LogLevel.ERROR_EXCEPTION, e=e, description="response: {response}"))
+        log_error(response, logger=logger, print_out=print_out)
+        if raise_e:
+            raise e
+
+    response["saved_to_path"] = saved_to_path if success else None
+    return response
+
+# def write_json_to_gcs_extended(storage_client: GCSClient, data: dict | list | str, bucket_name: str, file_name: str,
+#                                duplication_handling: DuplicationHandling, duplication_match_condition_type: MatchConditionType,
+#                                duplication_match_condition: str | List[str] = "", max_retries: int = 2, max_deletable_files: int = 1,
+#                                logger=None, print_out=False, raise_e=False):
+
+#     """Saves data to Google Cloud Storage.
+
+#     Handles duplication with strategies: OVERWRITE, INCREMENT, SKIP, or RAISE_ERROR.
+#     """
+
+#     max_deletable_files_allowed = 3
+
+#     # GCS-related metadata
+#     saved_to_path = None
+#     matched_duplicates_count = 0
+#     matched_duplicates_deleted = []
+#     duplication_handling_status = None
+#     error_during_operation = None
+
+#     response = {
+#         "saved_to_path": saved_to_path,
+#         "matched_duplicates_count": matched_duplicates_count,
+#         "matched_duplicates_deleted": matched_duplicates_deleted,
+#         "duplication_handling_status": duplication_handling_status,
+#         "duplication_match_condition_type": duplication_match_condition_type,
+#         "duplication_match_condition": duplication_match_condition,
+#         "error_during_operation": error_during_operation
+#     }
+
+#     supported_match_condition_types = [MatchConditionType.EXACT, MatchConditionType.PREFIX]
+#     supported_duplication_handling = [DuplicationHandling.RAISE_ERROR, DuplicationHandling.OVERWRITE, DuplicationHandling.INCREMENT, DuplicationHandling.SKIP]
+
+#     try:
+#         if max_deletable_files > max_deletable_files_allowed:
+#             raise ValueError(f"max_deletable_files should be less than or equal to {max_deletable_files_allowed} for safety.")
+#         if duplication_handling not in supported_duplication_handling:
+#             msg = f"Error: Duplication handling not supported. Supported types: {supported_duplication_handling}"
+#             raise ValueError(msg)
+#         if duplication_match_condition_type not in supported_match_condition_types:
+#             msg = f"Error: Match condition type not supported. Supported types: {supported_match_condition_types}"
+#             raise ValueError(msg)
+#         elif duplication_match_condition_type!=MatchConditionType.EXACT and not duplication_match_condition:
+#             msg = f"Error: Match condition is required for match condition type: {duplication_match_condition_type}"
+#             raise ValueError(msg)
+
+#         # Prepare data
+#         if isinstance(data, (list, dict)):
+#             data_str = json.dumps(data, indent=2)
+#         else:
+#             data_str = data
+
+#         increment = 0
+#         attempts = 0
+#         success = False
+
+#         # Check for existing files based on duplication_match_condition_type
+#         files_matched_on_condition = []
+#         bucket = storage_client.bucket(bucket_name)
+#         base_file_name, ext = os.path.splitext(file_name)
+#         if duplication_match_condition_type == MatchConditionType.PREFIX:
+#             files_matched_on_condition = list(bucket.list_blobs(prefix=duplication_match_condition))
+#         elif duplication_match_condition_type == MatchConditionType.EXACT:
+#             if bucket.blob(file_name).exists():
+#                 files_matched_on_condition = [bucket.blob(file_name)]
+
+#         matched_duplicates_count = len(files_matched_on_condition)
+#         response["matched_duplicates_count"] = matched_duplicates_count
+
+#         # Handle duplication based on duplication_handling
+#         if matched_duplicates_count:
+#             if duplication_handling == DuplicationHandling.RAISE_ERROR:
+#                 raise FileExistsError("File(s) matching the condition already exist.")
+
+#             if duplication_handling == DuplicationHandling.SKIP:
+#                 log_warning("Skipping saving to GCS: file(s) matching the condition already exist.", logger=logger, print_out=print_out)
+#                 response["duplication_handling_status"] = DuplicationHandlingStatus.SKIPPED.value
+#                 return response
+
+#             if duplication_handling == DuplicationHandling.OVERWRITE:
+#                 if matched_duplicates_count > max_deletable_files:
+#                     raise ValueError(f"Error: Attempt to delete {matched_duplicates_count} matched files, but limit is {max_deletable_files}. Operation Cancelled.")
+
+#                 for blob in files_matched_on_condition:
+#                     cloud_storage_path_to_delete = f"gs://{bucket_name}/{blob.name}"
+#                     blob.delete()
+#                     matched_duplicates_deleted.append(cloud_storage_path_to_delete)
+
+#                 response["matched_duplicates_deleted"] = matched_duplicates_deleted
+#                 response["duplication_handling_status"] = DuplicationHandlingStatus.OVERWRITTEN.value
+
+#             elif duplication_handling == DuplicationHandling.INCREMENT:
+#                 while bucket.blob(file_name).exists():
+#                     increment += 1
+#                     file_name = f"{base_file_name}_v{increment}{ext}"
+#                 saved_to_path = f"gs://{bucket_name}/{file_name}"
+#                 response["duplication_handling_status"] = DuplicationHandlingStatus.INCREMENTED.value
+
+#         # GCS Upload
+#         saved_to_path = f"gs://{bucket_name}/{file_name}"
+#         while attempts < max_retries and not success:
+#             try:
+#                 blob = bucket.blob(file_name)
+#                 blob.upload_from_string(data_str, content_type='application/json')
+#                 success = True
+#             except Exception as e:
+#                 attempts += 1
+#                 if attempts < max_retries:
+#                     time.sleep(2 ** attempts)
+#                 else:
+#                     if raise_e:
+#                         raise e
+
+#     except Exception as e:
+#         error_message = f"Error occurred while writing JSON to GCS path: {saved_to_path} : {type(e).__name__} - {str(e)}"
+#         log_error(error_message, logger=logger, print_out=print_out)
+#         response["error_during_operation"] = error_message
+#         if raise_e:
+#             raise e
+
+#     response["saved_to_path"] = saved_to_path if success else None
+#     return response
+
+
+def write_csv_to_gcs(bucket_name:str, file_name:str, data:dict | list | str, storage_client:GCSClient, logger=None, print_out=False, raise_e=False):
     """ Helper function to write a CSV file to Google Cloud Storage """
     try:
         bucket = storage_client.bucket(bucket_name)
@@ -303,9 +431,12 @@ def write_csv_to_gcs(bucket_name:str, file_name:str, data:dict | list | str, sto
         else:
             raise ValueError("Data should be a list of dictionaries")
         blob.upload_from_string(data_file.getvalue(), content_type='text/csv')
-        if log_info_verbose:
-            logger.info(f"Successfully wrote CSV to {file_name} in bucket {bucket_name}.")
+        log_info(msg=f"Successfully wrote CSV to {file_name} in bucket {bucket_name}.", logger=logger, print_out=print_out)
     except ValueError as e:
-        logger.error(f"ValueError: {e}")
+        log_error(msg=f"ValueError: {e}",logger=logger, print_out=print_out)
+        if raise_e:
+            raise e
     except Exception as e:
-        logger.error(f"An unexpected error occurred while writing CSV to GCS: {e}", exc_info=True)
+        log_error(msg=f"An unexpected error occurred while writing CSV to GCS: {e}", logger=logger, print_out=print_out, exc_info=True)
+        if raise_e:
+            raise e