ipulse-shared-core-ftredge 2.6__py3-none-any.whl → 2.6.1__py3-none-any.whl

ipulse_shared_core_ftredge/utils/logs/context_log.py

@@ -0,0 +1,211 @@
+
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=unused-variable
+import traceback
+import json
+from datetime import datetime, timezone
+from typing import List
+from ipulse_shared_core_ftredge.enums.enums_common_utils import Status
+from ipulse_shared_core_ftredge.enums.enums_logs import LogLevel
+
+############################################################################
+##################### SETTING UP custom LOGGING format= DICT ##########################
+### Cloud  Agnostic, can be used with any cloud provider , jsut use to_dict() method to get the log in dict format
+class ContextLog:
+
+    def __init__(self, level: LogLevel, base_context: str = None, collector_id: str = None,
+                context: str = None, description: str = None,
+                e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
+                log_status: Status = Status.OPEN, subject: str = None, systems_impacted: List[str] = None,
+                ):
+
+        if e is not None:
+            e_type = type(e).__name__ if e_type is None else e_type
+            e_message = str(e) if e_message is None else e_message
+            e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
+        elif e_traceback is None and (e_type or e_message):
+            e_traceback = traceback.format_exc()
+
+        self.level = level
+        self.subject = subject
+        self.description = description
+        self._base_context = base_context
+        self._context = context
+        self._systems_impacted = systems_impacted if systems_impacted else []
+        self.collector_id = collector_id
+        self.exception_type = e_type
+        self.exception_message = e_message
+        self.exception_traceback = e_traceback
+        self.log_status = log_status
+        self.timestamp = datetime.now(timezone.utc).isoformat()
+
+    @property
+    def base_context(self):
+        return self._base_context
+
+    @base_context.setter
+    def base_context(self, value):
+        self._base_context = value
+
+    @property
+    def context(self):
+        return self._context
+
+    @context.setter
+    def context(self, value):
+        self._context = value
+
+    @property
+    def systems_impacted(self):
+        return self._systems_impacted
+
+    @systems_impacted.setter
+    def systems_impacted(self, list_of_si: List[str]):
+        self._systems_impacted = list_of_si
+
+    def add_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is None:
+            self._systems_impacted = []
+        self._systems_impacted.append(system_impacted)
+
+    def remove_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is not None:
+            self._systems_impacted.remove(system_impacted)
+
+    def clear_systems_impacted(self):
+        self._systems_impacted = []
+
+    def _format_traceback(self, e_traceback, e_message, max_field_len:int, max_traceback_lines:int):
+        if not e_traceback or e_traceback == 'None\n':
+            return None
+
+        traceback_lines = e_traceback.splitlines()
+
+            # Check if the traceback is within the limits
+        if len(traceback_lines) <= max_traceback_lines and len(e_traceback) <= max_field_len:
+            return e_traceback
+
+        # Remove lines that are part of the exception message if they are present in traceback
+        message_lines = e_message.splitlines() if e_message else []
+        if message_lines:
+            for message_line in message_lines:
+                if message_line in traceback_lines:
+                    traceback_lines.remove(message_line)
+
+        # Filter out lines from third-party libraries (like site-packages)
+        filtered_lines = [line for line in traceback_lines if "site-packages" not in line]
+
+        # If filtering results in too few lines, revert to original traceback
+        if len(filtered_lines) < 2:
+            filtered_lines = traceback_lines
+
+        # Combine standalone bracket lines with previous or next lines
+        combined_lines = []
+        for line in filtered_lines:
+            if line.strip() in {"(", ")", "{", "}", "[", "]"} and combined_lines:
+                combined_lines[-1] += " " + line.strip()
+            else:
+                combined_lines.append(line)
+
+            # Ensure the number of lines doesn't exceed MAX_TRACEBACK_LINES
+        if len(combined_lines) > max_traceback_lines:
+            keep_lines_start = min(max_traceback_lines // 2, len(combined_lines))
+            keep_lines_end = min(max_traceback_lines // 2, len(combined_lines) - keep_lines_start)
+            combined_lines = (
+                combined_lines[:keep_lines_start] +
+                ['... (truncated) ...'] +
+                combined_lines[-keep_lines_end:]
+            )
+
+        formatted_traceback = '\n'.join(combined_lines)
+
+        # Ensure the total length doesn't exceed MAX_TRACEBACK_LENGTH
+        if len(formatted_traceback) > max_field_len:
+            truncated_length = max_field_len - len('... (truncated) ...')
+            half_truncated_length = truncated_length // 2
+            formatted_traceback = (
+                formatted_traceback[:half_truncated_length] +
+                '\n... (truncated) ...\n' +
+                formatted_traceback[-half_truncated_length:]
+            )
+        return formatted_traceback
+
+    def to_dict(self, max_field_len:int =10000, size_limit:float=256 * 1024 * 0.80,max_traceback_lines:int = 30):
+        size_limit = int(size_limit)  # Ensure size_limit is an integer
+
+        # Unified list of all fields
+        systems_impacted_str = f"{len(self.systems_impacted)} system(s): " + " ,,, ".join(self.systems_impacted) if self.systems_impacted else None
+        fields = [
+            ("log_status", str(self.log_status.name)),
+            ("level_code", self.level.value),
+            ("level_name", str(self.level.name)),
+            ("base_context", str(self.base_context)),
+            ("timestamp", str(self.timestamp)),
+            ("collector_id", str(self.collector_id)),
+            ("systems_impacted", systems_impacted_str),
+            ("context", str(self.context)),  # special sizing rules apply to it
+            ("subject", str(self.subject)),
+            ("description", str(self.description)),
+            ("exception_type", str(self.exception_type)),
+            ("exception_message", str(self.exception_message)),
+            ("exception_traceback", str(self._format_traceback(self.exception_traceback,self.exception_message, max_field_len, max_traceback_lines)))
+        ]
+
+        # Function to calculate the byte size of a JSON-encoded field
+        def field_size(key, value):
+            return len(json.dumps({key: value}).encode('utf-8'))
+
+        # Function to truncate a value based on its type
+        # Function to truncate a value based on its type
+        def truncate_value(value, max_size):
+            if isinstance(value, str):
+                half_size = max_size // 2
+                return value[:half_size] + '...' + value[-(max_size - half_size - 3):]
+            return value
+
+         # Ensure no field exceeds max_field_len
+        for i, (key, value) in enumerate(fields):
+            if isinstance(value, str) and len(value) > max_field_len:
+                fields[i] = (key, truncate_value(value, max_field_len))
+
+        # Ensure total size of the dict doesn't exceed size_limit
+        total_size = sum(field_size(key, value) for key, value in fields)
+        log_dict = {}
+        truncated = False
+
+        if total_size > size_limit:
+            truncated = True
+            remaining_size = size_limit
+            remaining_fields = len(fields)
+
+            for key, value in fields:
+                if remaining_fields > 0:
+                    max_size_per_field = remaining_size // remaining_fields
+                else:
+                    max_size_per_field = 0
+
+                field_sz = field_size(key, value)
+                if field_sz > max_size_per_field:
+                    value = truncate_value(value, max_size_per_field)
+                    field_sz = field_size(key, value)
+
+                log_dict[key] = value
+                remaining_size -= field_sz
+                remaining_fields -= 1
+        else:
+            log_dict = dict(fields)
+
+        log_dict['trunc'] = truncated
+
+        return log_dict
+
+    def __str__(self):
+        return json.dumps(self.to_dict(), indent=4)
+
+    def __repr__(self):
+        return self.__str__()

ipulse_shared_core_ftredge/utils/logs/get_logger.py

@@ -0,0 +1,76 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
+# pylint: disable=broad-exception-raised
+import logging
+import os
+import json
+import traceback
+from ipulse_shared_core_ftredge.enums.enums_cloud import CloudProvider
+from ipulse_shared_core_ftredge.utils.utils_cloud_gcp import setup_gcp_logging
+
+###################################################################################################
+##################################################################################################
+##################################### SETTING UP LOGGER ##########################################
+
+class CloudLogFormatter(logging.Formatter):
+    """Formats log records as structured JSON."""
+
+    def format(self, record):
+        log_entry = {
+            'timestamp': self.formatTime(record, self.datefmt),
+            'name': record.name,
+            'severity': record.levelname,
+            'message': record.msg,
+            'pathname': record.pathname,
+            'lineno': record.lineno,
+        }
+        if record.exc_info:
+            log_entry['exception_traceback'] = ''.join(traceback.format_exception(*record.exc_info))
+        if isinstance(record.msg, dict):
+            log_entry.update(record.msg)
+        return json.dumps(log_entry)
+
+
+class LocalLogFormatter(logging.Formatter):
+    """Formats log records for local output to the console."""
+
+    def format(self, record):  # Make sure you have the 'record' argument here!
+        path_parts = record.pathname.split(os.sep)
+        
+        # Get the last two parts of the path if they exist
+        if len(path_parts) >= 2:
+            short_path = os.path.join(path_parts[-2], path_parts[-1])
+        else:
+            short_path = record.pathname
+
+        log_message = f"{record.levelname} ::: {record.name} ::: {short_path} ::: lineno: {record.lineno} ::: {self.formatTime(record, self.datefmt)} ::: message: {record.msg}"
+        if record.exc_info:
+            log_message += "\n" + ''.join(
+                traceback.format_exception(*record.exc_info)
+            )
+        return log_message
+
+
+def get_logger( logger_name:str ,level=logging.INFO,  enable_local_streamer=False, cloud_provider:CloudProvider=CloudProvider.NO_CLOUD,  enable_error_reporting=True ):
+
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(level)
+    cloud_formatter = CloudLogFormatter()
+
+    without_cloud_logging_handler = [CloudProvider.NO_CLOUD, CloudProvider.CLOUD_AGNOSTIC, CloudProvider.UNKNWON, CloudProvider.OTHER]
+
+    if cloud_provider in without_cloud_logging_handler or enable_local_streamer:
+        handler = logging.StreamHandler()
+        handler.setFormatter(LocalLogFormatter())
+        logger.addHandler(handler)
+
+    if cloud_provider == CloudProvider.GCP:
+        setup_gcp_logging(logger=logger, formatter=cloud_formatter, enable_error_reporting=enable_error_reporting)
+    elif  cloud_provider not in without_cloud_logging_handler:
+        raise ValueError(f"Unsupported cloud provider: {cloud_provider}. Supported cloud providers: {CloudProvider.GCP.value}")
+
+    return logger

ipulse_shared_core_ftredge/utils/utils_cloud.py

@@ -0,0 +1,44 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
+# pylint: disable=broad-exception-caught
+from typing import Optional
+from ipulse_shared_core_ftredge.enums.enums_cloud import CloudProvider
+from .utils_cloud_gcp import (write_json_to_gcs_extended, read_json_from_gcs)
+
+#######################################################################################################################
+#######################################################################################################################
+#################################################     cloud IO functions      ########################################
+
+# Define the central function that routes to the relevant cloud-specific function
+def write_json_to_cloud_storage_extended(cloud_provider:CloudProvider, storage_client, data:dict | list | str, bucket_name: str, file_name: str,
+                      file_exists_if_starts_with_prefix:Optional[str] =None, overwrite_if_exists:bool=False, increment_if_exists:bool=False,
+                      max_retries:int=2, max_deletable_files:int=1, logger=None, print_out=False):
+
+
+    if cloud_provider == CloudProvider.GCP:
+       return write_json_to_gcs_extended(
+            storage_client=storage_client,
+            data=data,
+            bucket_name=bucket_name,
+            file_name=file_name,
+            file_exists_if_starts_with_prefix=file_exists_if_starts_with_prefix,
+            overwrite_if_exists=overwrite_if_exists,
+            increment_if_exists=increment_if_exists,
+            max_retries=max_retries,
+            max_deletable_files=max_deletable_files,
+            logger=logger,
+            print_out=print_out
+        )
+
+    raise ValueError(f"Unsupported cloud provider: {cloud_provider}. Supported cloud providers: gcp")
+
+
+def read_json_from_cloud_storage(cloud_provider:CloudProvider, storage_client, bucket_name, file_name, logger=None, print_out=False):
+    if cloud_provider == CloudProvider.GCP:
+        return read_json_from_gcs(storage_client=storage_client, bucket_name=bucket_name, file_name=file_name, logger=logger, print_out=print_out)
+
+    raise ValueError(f"Unsupported cloud provider: {cloud_provider}. Supported cloud providers: gcp")

ipulse_shared_core_ftredge/utils/utils_cloud_gcp.py

@@ -0,0 +1,311 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
+# pylint: disable=broad-exception-raised
+import json
+import csv
+from io import StringIO
+import os
+import time
+import logging
+from typing import Optional
+import traceback
+from google.api_core.exceptions import NotFound
+from google.cloud import error_reporting
+from google.cloud import logging as cloud_logging
+from google.cloud.storage import Client as GCSClient
+from google.cloud import bigquery
+
+############################################################################
+##################### GOOGLE CLOUD  UTILS ##################################
+############################################################################
+
+def log_error(message, logger=None , print_out=False, exc_info=False):
+    if logger:
+        logger.error(message, exc_info=exc_info)
+    elif print_out:
+        print(message)
+
+def log_warning(message, logger=None, print_out=False):
+    if logger:
+        logger.warning(message)
+    elif print_out:
+        print(message)
+
+def log_info(message, logger=None, print_out=False):
+    if logger:
+        logger.info(message)
+    elif print_out:
+        print(message)
+
+
+
+############################################################################
+##################### LOGGING and ERROR reporting ##########################
+####DEPCREACATED: THIS APPROACH WAS GOOD, BUT ERRORS WERE NOT REPORTED TO ERROR REPORTING
+# logging.basicConfig(level=logging.INFO)
+# logging_client = google.cloud.logging.Client()
+# logging_client.setup_logging()
+###################################
+def setup_gcp_logging(logger, formatter, enable_error_reporting=True):
+
+    class CustomGCPErrorReportingHandler(logging.Handler):
+        def __init__(self, level=logging.ERROR):
+            super().__init__(level)
+            self.error_client = error_reporting.Client()
+            self.propagate = True
+
+        def emit(self, record):
+            try:
+                if record.levelno >= logging.ERROR:
+                    log_struct = {
+                    'message': self.format(record),
+                    'severity': record.levelname,
+                    'pathname': getattr(record, 'pathname', None),
+                    'lineno': getattr(record, 'lineno', None)
+                }
+                if record.exc_info:
+                    log_struct['exception'] = ''.join(
+                        traceback.format_exception(*record.exc_info)
+                    )
+                self.error_client.report(str(log_struct))
+            except Exception as e:
+                self.handleError(record)
+        
+    class CustomGCPLoggingHandler(cloud_logging.handlers.CloudLoggingHandler):
+        """Custom handler for Google Cloud Logging with a dynamic logName."""
+        def __init__(self, client, name, resource=None, labels=None):
+            super().__init__(client=client, name=name, resource=resource, labels=labels)
+            
+        def emit(self, record):
+            # 1. Create the basic log entry dictionary
+            log_entry = {
+                'message': record.msg,
+                'severity': record.levelname,
+                'name': record.name,
+                'pathname': record.filename,
+                'lineno': record.lineno,
+            }
+            if record.exc_info:
+                log_entry['exception_traceback'] = ''.join(
+                    traceback.format_exception(*record.exc_info)
+                )
+
+            # 2. Apply the formatter to the 'message' field if it's a dictionary
+            if isinstance(record.msg, dict):
+                formatted_message = self.formatter.format(record)
+                try:
+                    log_entry['message'] = json.loads(formatted_message)
+                except json.JSONDecodeError:
+                    log_entry['message'] = formatted_message
+            else:
+                log_entry['message'] = record.msg
+
+            # 3. Set the custom logName
+            log_entry['logName'] = f"projects/{self.client.project}/logs/{record.name}"
+
+            # 4. Send to Google Cloud Logging
+            super().emit(record)
+
+    # Create Google Cloud Logging handler
+    cloud_logging_client = cloud_logging.Client()
+    cloud_logging_handler = CustomGCPLoggingHandler(cloud_logging_client, logger.name)  # No prefix needed
+    cloud_logging_handler.setFormatter(formatter)
+    logger.addHandler(cloud_logging_handler)
+
+    if enable_error_reporting:
+        # Create and add Error Reporting handler
+        error_reporting_handler = CustomGCPErrorReportingHandler()
+        logger.addHandler(error_reporting_handler)
+
+
+
+def create_bigquery_schema_from_json(json_schema: list) -> list:
+    schema = []
+    for field in json_schema:
+        if "max_length" in field:
+            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"], max_length=field["max_length"]))
+        else:
+            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"]))
+    return schema
+
+
+def read_json_from_gcs(storage_client:GCSClient, bucket_name:str, file_name:str, logger=None,print_out=False):
+    """ Helper function to read a JSON file from Google Cloud Storage """
+    try:
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(file_name)
+        data_string = blob.download_as_text()
+        data = json.loads(data_string)
+        return data
+    except NotFound:
+        log_error(message=f"Error: The file {file_name} was not found in the bucket {bucket_name}.", logger=logger, print_out=print_out)
+        return None
+    except json.JSONDecodeError:
+        log_error(message=f"Error: The file {file_name} could not be decoded as JSON.", logger=logger, print_out=print_out)
+        return None
+    except Exception as e:
+        log_error(message=f"An unexpected error occurred: {e}", exc_info=True, logger=logger, print_out=print_out)
+        return None
+
+def read_csv_from_gcs(bucket_name:str, file_name:str, storage_client:GCSClient, logger=None, print_out=False):
+    """ Helper function to read a CSV file from Google Cloud Storage """
+
+    try:
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(file_name)
+        data_string = blob.download_as_text()
+        data_file = StringIO(data_string)
+        reader = csv.DictReader(data_file)
+        return list(reader)
+    except NotFound:
+        log_error(message=f"Error: The file {file_name} was not found in the bucket {bucket_name}.", logger=logger, print_out=print_out)
+        return None
+    except csv.Error:
+        log_error(message=f"Error: The file {file_name} could not be read as CSV.", logger=logger, print_out=print_out)
+        return None
+    except Exception as e:
+        log_error(message=f"An unexpected error occurred: {e}", logger=logger, print_out=print_out, exc_info=True)
+        return None
+
+
+
+def write_json_to_gcs_extended( storage_client:GCSClient, data:dict | list | str, bucket_name: str, file_name: str,
+                      file_exists_if_starts_with_prefix:Optional[str] =None, overwrite_if_exists:bool=False, increment_if_exists:bool=False,
+                      max_retries:int=2, max_deletable_files:int=1, logger=None, print_out=False):
+    
+    """Saves data to Google Cloud Storage and optionally locally.
+    
+    This function attempts to upload data to GCS. 
+    - If the upload fails after retries and `save_locally` is True or `local_path` is provided, it attempts to save the data locally.
+    - It handles file name conflicts based on these rules:
+        - If `overwrite_if_exists` is True: 
+            - If `file_exists_if_contains_substr` is provided, ANY existing file containing the substring is deleted, and the new file is saved with the provided `file_name`.
+            - If `file_exists_if_contains_substr` is None, and a file with the exact `file_name` exists, it's overwritten.
+        - If `increment_if_exists` is True:
+            - If `file_exists_if_contains_substr` is provided, a new file with an incremented version is created ONLY if a file with the EXACT `file_name` exists.
+            - If `file_exists_if_contains_substr` is None, a new file with an incremented version is created if a file with the exact `file_name` exists. 
+            
+        -If both overwrite_if_exists and increment_if_exists are provided as Ture, an exception will be raised.
+    """
+    # GCS upload exception
+    # Input validation
+    if overwrite_if_exists and increment_if_exists:
+        raise ValueError("Both 'overwrite_if_exists' and 'increment_if_exists' cannot be True simultaneously.")
+    if not isinstance(data, (list, dict, str)):
+        raise ValueError("Data should be a list, dict, or string.")
+    if max_deletable_files > 10:
+        raise ValueError("max_deletable_files should be less than 10 for safety. For more use another method.")
+
+    # Prepare data
+    if isinstance(data, (list, dict)):
+        data_str = json.dumps(data, indent=2)
+    else:
+        data_str = data
+
+    bucket = storage_client.bucket(bucket_name)
+    base_file_name, ext = os.path.splitext(file_name)
+    increment = 0
+    attempts = 0
+    success = False
+
+    # GCS-related metadata
+    cloud_storage_path = None
+    cloud_storage_file_overwritten = False
+    cloud_storage_file_already_exists = False
+    cloud_storage_file_saved_with_increment = False
+    cloud_storage_file_exists_checked_on_name = file_name
+    cloud_storage_deleted_files=[]
+
+    
+    upload_allowed = True
+    # --- Overwrite Logic ---
+    if overwrite_if_exists:
+        if file_exists_if_starts_with_prefix:
+            cloud_storage_file_exists_checked_on_name = file_exists_if_starts_with_prefix
+            blobs_to_delete = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
+            if len(blobs_to_delete) > max_deletable_files:
+                raise Exception(f"Error: Attempt to delete {len(blobs_to_delete)} matched files, but limit is {max_deletable_files}.")
+            if blobs_to_delete:
+                for blob in blobs_to_delete:
+                    cloud_storage_path_del = f"gs://{bucket_name}/{blob.name}"
+                    blob.delete()
+                    cloud_storage_deleted_files.append(cloud_storage_path_del)
+                cloud_storage_file_overwritten = True
+        elif bucket.blob(file_name).exists():
+            cloud_storage_file_already_exists = True
+            cloud_storage_path_del = f"gs://{bucket_name}/{file_name}"
+            blob.delete()  # Delete the existing blob
+            cloud_storage_deleted_files.append(cloud_storage_path_del)
+            cloud_storage_file_overwritten = True
+    # --- Increment Logic ---
+    elif increment_if_exists:
+        cloud_storage_file_exists_checked_on_name = file_name  # We only increment if the exact name exists
+        while bucket.blob(file_name).exists():
+            cloud_storage_file_already_exists = True
+            increment += 1
+            file_name = f"{base_file_name}_v{increment}{ext}"
+            cloud_storage_file_saved_with_increment = True
+        if increment>0:
+            cloud_storage_path = f"gs://{bucket_name}/{file_name}"
+        # --- Check for Conflicts (Including Prefix) ---
+    else:
+        if file_exists_if_starts_with_prefix:
+            blobs_matched = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
+            cloud_storage_file_exists_checked_on_name = file_exists_if_starts_with_prefix
+            if blobs_matched:
+                upload_allowed = False
+                cloud_storage_file_already_exists = True
+        elif bucket.blob(file_name).exists():
+            upload_allowed = False
+            cloud_storage_file_already_exists = True
+
+    # --- GCS Upload ---
+    cloud_storage_path = f"gs://{bucket_name}/{file_name}"
+    if overwrite_if_exists or increment_if_exists or upload_allowed:
+        while attempts < max_retries and not success:
+            try:
+                blob = bucket.blob(file_name)  # Use the potentially updated file_name
+                blob.upload_from_string(data_str, content_type='application/json')
+                success = True
+            except Exception as e:
+                attempts += 1
+                if attempts < max_retries:
+                    time.sleep(2 ** attempts)
+                else:
+                    raise e
+
+    # --- Return Metadata ---
+    return {
+        "cloud_storage_path": cloud_storage_path if (success or not upload_allowed ) else None,
+        "cloud_storage_file_already_exists": cloud_storage_file_already_exists,
+        "cloud_storage_file_exists_checked_on_name":cloud_storage_file_exists_checked_on_name ,
+        "cloud_storage_file_overwritten": cloud_storage_file_overwritten,
+        "cloud_storage_deleted_file_names": ",,,".join(cloud_storage_deleted_files) if cloud_storage_deleted_files else None,
+        "cloud_storage_file_saved_with_increment": cloud_storage_file_saved_with_increment
+    }
+
+
+def write_csv_to_gcs(bucket_name:str, file_name:str, data:dict | list | str, storage_client:GCSClient, logger,log_info_verbose=True):
+    """ Helper function to write a CSV file to Google Cloud Storage """
+    try:
+        bucket = storage_client.bucket(bucket_name)
+        blob = bucket.blob(file_name)
+        data_file = StringIO()
+        if data and isinstance(data, list) and isinstance(data[0], dict):
+            fieldnames = data[0].keys()
+            writer = csv.DictWriter(data_file, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(data)
+        else:
+            raise ValueError("Data should be a list of dictionaries")
+        blob.upload_from_string(data_file.getvalue(), content_type='text/csv')
+        if log_info_verbose:
+            logger.info(f"Successfully wrote CSV to {file_name} in bucket {bucket_name}.")
+    except ValueError as e:
+        logger.error(f"ValueError: {e}")
+    except Exception as e:
+        logger.error(f"An unexpected error occurred while writing CSV to GCS: {e}", exc_info=True)