ipulse-shared-core-ftredge 2.6.1__py3-none-any.whl → 2.7.1__py3-none-any.whl

ipulse_shared_core_ftredge/utils/logs/get_logger.py

@@ -9,8 +9,9 @@ import logging
 import os
 import json
 import traceback
-from ipulse_shared_core_ftredge.enums.enums_cloud import CloudProvider
-from ipulse_shared_core_ftredge.utils.utils_cloud_gcp import setup_gcp_logging
+from typing import List, Union
+from ipulse_shared_core_ftredge.enums import LoggingHandlers
+from ipulse_shared_core_ftredge.utils.utils_cloud_gcp import add_gcp_cloud_logging, add_gcp_error_reporting
 
 ###################################################################################################
 ##################################################################################################
@@ -21,10 +22,10 @@ class CloudLogFormatter(logging.Formatter):
 
     def format(self, record):
         log_entry = {
+            'message': record.msg,
             'timestamp': self.formatTime(record, self.datefmt),
             'name': record.name,
             'severity': record.levelname,
-            'message': record.msg,
             'pathname': record.pathname,
             'lineno': record.lineno,
         }
@@ -40,37 +41,63 @@ class LocalLogFormatter(logging.Formatter):
 
     def format(self, record):  # Make sure you have the 'record' argument here!
         path_parts = record.pathname.split(os.sep)
-        
+
         # Get the last two parts of the path if they exist
         if len(path_parts) >= 2:
             short_path = os.path.join(path_parts[-2], path_parts[-1])
         else:
             short_path = record.pathname
+        
+        # Format log messages differently based on the log level
+        if record.levelno == logging.INFO:
+            log_message = f"[INFO] {self.formatTime(record, self.datefmt)} :: {record.msg}"
+        elif record.levelno == logging.DEBUG:
+            log_message = f"[DEBUG] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
+        elif record.levelno == logging.ERROR:
+            log_message = f"[ERROR] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
+            if record.exc_info:
+                log_message += "\n" + ''.join(traceback.format_exception(*record.exc_info))
+        else:
+            log_message = f"[{record.levelname}] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
+
 
-        log_message = f"{record.levelname} ::: {record.name} ::: {short_path} ::: lineno: {record.lineno} ::: {self.formatTime(record, self.datefmt)} ::: message: {record.msg}"
-        if record.exc_info:
-            log_message += "\n" + ''.join(
-                traceback.format_exception(*record.exc_info)
-            )
         return log_message
 
 
-def get_logger( logger_name:str ,level=logging.INFO,  enable_local_streamer=False, cloud_provider:CloudProvider=CloudProvider.NO_CLOUD,  enable_error_reporting=True ):
+def get_logger( logger_name:str ,level=logging.INFO, logging_handler_providers: Union[LoggingHandlers, List[LoggingHandlers]] = LoggingHandlers.NONE):
+
+    """Creates and configures a logger with the specified handlers."""
 
     logger = logging.getLogger(logger_name)
     logger.setLevel(level)
     cloud_formatter = CloudLogFormatter()
 
-    without_cloud_logging_handler = [CloudProvider.NO_CLOUD, CloudProvider.CLOUD_AGNOSTIC, CloudProvider.UNKNWON, CloudProvider.OTHER]
+    # Ensure logging_handler_providers is a list for consistent processing
+    if not isinstance(logging_handler_providers, list):
+        logging_handler_providers = [logging_handler_providers]
 
-    if cloud_provider in without_cloud_logging_handler or enable_local_streamer:
-        handler = logging.StreamHandler()
-        handler.setFormatter(LocalLogFormatter())
-        logger.addHandler(handler)
+    supported_remote_handlers = [
+        LoggingHandlers.GCP_CLOUD_LOGGING,
+        LoggingHandlers.GCP_ERROR_REPORTING,
+        LoggingHandlers.LOCAL_STREAM,
+        LoggingHandlers.NONE, # If NONE is considered a remote handler
+    ]
 
-    if cloud_provider == CloudProvider.GCP:
-        setup_gcp_logging(logger=logger, formatter=cloud_formatter, enable_error_reporting=enable_error_reporting)
-    elif  cloud_provider not in without_cloud_logging_handler:
-        raise ValueError(f"Unsupported cloud provider: {cloud_provider}. Supported cloud providers: {CloudProvider.GCP.value}")
+    # Remote handlers
 
-    return logger
+    for handler_provider in logging_handler_providers:
+        if handler_provider in supported_remote_handlers:
+            if handler_provider == LoggingHandlers.GCP_CLOUD_LOGGING:
+                add_gcp_cloud_logging(logger, cloud_formatter)
+            elif handler_provider == LoggingHandlers.GCP_ERROR_REPORTING:
+                add_gcp_error_reporting(logger)
+            elif handler_provider == LoggingHandlers.LOCAL_STREAM:  # Handle local stream
+                local_handler = logging.StreamHandler()
+                local_handler.setFormatter(LocalLogFormatter())
+                logger.addHandler(local_handler)
+        else:
+            raise ValueError(
+            f"Unsupported logging provider: {handler_provider}. "
+            f"Supported providers: {[h.value for h in supported_remote_handlers]}"
+            )
+    return logger

ipulse_shared_core_ftredge/utils/utils_cloud.py

@@ -5,40 +5,49 @@
 # pylint: disable=line-too-long
 # pylint: disable=unused-variable
 # pylint: disable=broad-exception-caught
-from typing import Optional
-from ipulse_shared_core_ftredge.enums.enums_cloud import CloudProvider
-from .utils_cloud_gcp import (write_json_to_gcs_extended, read_json_from_gcs)
+from ipulse_shared_core_ftredge.enums import CloudProvider, DataSourceType, DuplicationHandling, MatchConditionType
+from .utils_collector_pipelinemon import Pipelinemon
+from .utils_cloud_gcp import (write_json_to_gcs_extended,
+                              read_json_from_gcs)
 
 #######################################################################################################################
 #######################################################################################################################
 #################################################     cloud IO functions      ########################################
 
 # Define the central function that routes to the relevant cloud-specific function
-def write_json_to_cloud_storage_extended(cloud_provider:CloudProvider, storage_client, data:dict | list | str, bucket_name: str, file_name: str,
-                      file_exists_if_starts_with_prefix:Optional[str] =None, overwrite_if_exists:bool=False, increment_if_exists:bool=False,
-                      max_retries:int=2, max_deletable_files:int=1, logger=None, print_out=False):
+def write_json_to_cloud_storage_extended(cloud_storage:CloudProvider | DataSourceType, storage_client, data:dict | list | str, bucket_name: str, file_name: str,
+                      duplication_handling:DuplicationHandling,  duplication_match_condition_type: MatchConditionType, duplication_match_condition: str = "",
+                       max_retries:int=2, max_matched_deletable_files:int=1,
+                      pipelinemon: Pipelinemon = None, logger=None, print_out=False, raise_e=False):
 
 
-    if cloud_provider == CloudProvider.GCP:
-       return write_json_to_gcs_extended(
+    supported_cloud_storage_values = [CloudProvider.GCP, DataSourceType.GCS]
+
+    if cloud_storage in [CloudProvider.GCP, DataSourceType.GCS]:
+        return write_json_to_gcs_extended(
+            pipelinemon=pipelinemon,
             storage_client=storage_client,
             data=data,
             bucket_name=bucket_name,
             file_name=file_name,
-            file_exists_if_starts_with_prefix=file_exists_if_starts_with_prefix,
-            overwrite_if_exists=overwrite_if_exists,
-            increment_if_exists=increment_if_exists,
+            duplication_handling_enum=duplication_handling,
+            duplication_match_condition_type_enum=duplication_match_condition_type,
+            duplication_match_condition=duplication_match_condition,
             max_retries=max_retries,
-            max_deletable_files=max_deletable_files,
+            max_deletable_files=max_matched_deletable_files,
             logger=logger,
-            print_out=print_out
+            print_out=print_out,
+            raise_e=raise_e
         )
 
-    raise ValueError(f"Unsupported cloud provider: {cloud_provider}. Supported cloud providers: gcp")
+    raise ValueError(f"Unsupported cloud storage : {cloud_storage}. Supported cloud storage values: {supported_cloud_storage_values}")
+
+
+def read_json_from_cloud_storage(cloud_storage:CloudProvider | DataSourceType , storage_client, bucket_name:str, file_name:str, logger=None, print_out:bool=False):
 
+    supported_cloud_storage_values = [CloudProvider.GCP, DataSourceType.GCS]
 
-def read_json_from_cloud_storage(cloud_provider:CloudProvider, storage_client, bucket_name, file_name, logger=None, print_out=False):
-    if cloud_provider == CloudProvider.GCP:
+    if cloud_storage in [CloudProvider.GCP, DataSourceType.GCS]:
         return read_json_from_gcs(storage_client=storage_client, bucket_name=bucket_name, file_name=file_name, logger=logger, print_out=print_out)
 
-    raise ValueError(f"Unsupported cloud provider: {cloud_provider}. Supported cloud providers: gcp")
+    raise ValueError(f"Unsupported cloud storage: {cloud_storage}. Supported cloud storage values: {supported_cloud_storage_values}")