cloe-logging 0.3.7__py3-none-any.whl

cloe_logging/__init__.py

@@ -0,0 +1,3 @@
+from .logger_factory import LoggerFactory
+
+__all__ = ["LoggerFactory"]

cloe_logging/decorators/__init__.py

@@ -0,0 +1,3 @@
+from .devops_decorator import build_logger
+
+__all__ = ["build_logger"]

cloe_logging/decorators/devops_decorator.py

@@ -0,0 +1,63 @@
+import functools
+import logging
+import sys
+import time
+from typing import Any
+from collections.abc import Callable
+
+from cloe_logging.formatters import DevOpsFormatter
+
+
+def filter_arg_logger(arg: Any) -> bool:
+    """
+    Filter out all arguments that shall NOT be printed, i.e.,
+    - strings with more than 25 characters
+    - dictionaries
+    - logging.Logger instances
+    """
+    match arg:
+        case str() if len(arg) >= 25:
+            result = False
+        case dict():
+            result = False
+        case logging.Logger():
+            result = False
+        case _:
+            result = True
+    return result
+
+
+def init_logging() -> logging.Logger:
+    logger = logging.getLogger("azure-pipeline-logger")
+    logger.setLevel(logging.INFO)
+    section_formatter = DevOpsFormatter(section_info=True)
+    section_handler = logging.StreamHandler()
+    section_handler.setFormatter(section_formatter)
+    logger.addHandler(section_handler)
+    return logger
+
+
+def build_logger():
+    def log_decorator_info(func: Callable):
+        @functools.wraps(func)
+        def log_decorator_wrapper(*args, **kwargs):
+            logger = init_logging()
+            args_passed_in_function = [repr(a) for a in args if filter_arg_logger(a)]
+            kwargs_passed_in_function = [f"{k}={v!r}" for k, v in kwargs.items()]
+            formatted_arguments = ", ".join(args_passed_in_function + kwargs_passed_in_function)
+
+            logger.info(f"##### START {func.__name__} WITH args [ {formatted_arguments} ] #####\n")
+
+            try:
+                start = time.time()
+                value = func(*args, **kwargs)
+                end = time.time()
+                logger.info(f"\n##### END {func.__name__} DURATION [ '{round(end-start)}'s ] #####")
+            except:
+                logger.error(f"ERROR: {str(sys.exc_info()[1])}")
+                raise
+            return value
+
+        return log_decorator_wrapper
+
+    return log_decorator_info

cloe_logging/formatters/__init__.py

@@ -0,0 +1,4 @@
+from .dict_formatter import DictFormatter
+from .devops_formatter import DevOpsFormatter
+
+__all__ = ["DictFormatter", "DevOpsFormatter"]

cloe_logging/formatters/devops_formatter.py

@@ -0,0 +1,84 @@
+import logging
+import re
+
+
+class DevOpsFormatter(logging.Formatter):
+    error_format = "##vso[task.logissue type=error]%(levelname)s -- %(name)s -- %(message)s"
+    warning_format = "##vso[task.logissue type=warning]%(levelname)s -- %(name)s -- %(message)s"
+    dbg_fmt = "DBG: %(module)s: %(lineno)d: %(msg)s"
+    info_format = "%(message)s"  # "%(name)s -- %(message)s"
+    section_format = "%(name)s -- %(message)s"
+
+    def __init__(self, fmt="%(levelno)s: %(msg)s", section_info=False):
+        super().__init__(fmt=fmt)
+        self._section_info = section_info
+
+    def parse_progress(self, message: str) -> str:
+        """
+        Parses Method for progress information
+        """
+        progress_pattern = re.compile(r".*PROGRESS\s\[\s*'(?P<x>\d+)\/(?P<y>\d+)'\s*\].*", re.IGNORECASE | re.MULTILINE)
+
+        if progress_pattern.match(message):
+            progress_match = progress_pattern.search(message)
+            if progress_match:
+                x = int(progress_match.group("x"))
+                y = int(progress_match.group("y"))
+                progress_value = round((x / y) * 100)
+                return f"##vso[task.setprogress value={progress_value};]script progress\n"
+
+        return ""
+
+    def parse_group_start(self, message: str) -> str:
+        """
+        Parses if its a group start and prepends a command string to the message
+        """
+        start_pattern = re.compile(
+            r".*#####\s*START\s*(?P<gname>.+?)\sWITH.*", re.IGNORECASE | re.MULTILINE | re.DOTALL
+        )
+
+        if start_pattern.match(message):
+            start_match = start_pattern.search(message)
+            if start_match:
+                return f"##[group]{start_match.group('gname')}\n"
+        return ""
+
+    def parse_group_end(self, message: str) -> str:
+        """
+        Parses if its a group end and appends a command string to the message
+        """
+        end_pattern = re.compile(r".*#####\sEND.*", re.IGNORECASE | re.MULTILINE | re.DOTALL)
+
+        if end_pattern.match(message):
+            return "\n##[endgroup]"
+        return ""
+
+    def format(self, record):
+        # Save the original format configured by the user
+        # when the logger formatter was instantiated
+        format_orig = self._style._fmt
+
+        if record.levelno == logging.INFO:
+            record_message = record.msg  # f"{record.name} -- {record.msg}"
+
+            return f"{self.parse_progress(record.msg)}{self.parse_group_start(record.msg)}{record_message}{self.parse_group_end(record.msg)}"
+
+        # Replace the original format with one customized by logging level
+        if record.levelno == logging.DEBUG:
+            self._fmt = DevOpsFormatter.dbg_fmt
+        elif record.levelno == logging.INFO and not self._section_info:
+            self._style._fmt = DevOpsFormatter.info_format
+
+        elif record.levelno == logging.INFO and self._section_info:
+            self._style._fmt = DevOpsFormatter.section_format
+
+        elif record.levelno == logging.ERROR:
+            self._style._fmt = DevOpsFormatter.error_format
+
+        # Call the original formatter class to do the grunt work
+        result = logging.Formatter.format(self, record)
+
+        # Restore the original format configured by the user
+        self._style._fmt = format_orig
+
+        return result

cloe_logging/formatters/dict_formatter.py

@@ -0,0 +1,44 @@
+import logging
+import json
+
+
+class DictFormatter(logging.Formatter):
+    def __init__(
+        self,
+        column_split_char: str = "|",
+        key_value_split_char: str = ":",
+        fmt=None,
+        datefmt=None,
+        style="%",
+        skip_missing_key_value_split_char: bool = False,
+    ):
+        super().__init__(fmt, datefmt, style)
+        self.column_split_char: str = column_split_char
+        self.key_value_split_char: str = key_value_split_char
+        self.skip_missing_key_value_split_char: bool = skip_missing_key_value_split_char
+
+    def format(self, record):
+        """
+        Converts a formatted string to a dictionary.
+
+        Parameters:
+            record: The log record to be converted to a dictionary.
+
+        Returns:
+            str: The converted dictionary as a JSON string.
+        """
+        log_record = super().format(record)
+        parts = [part.strip() for part in log_record.split(self.column_split_char)]
+        result_dict = {
+            "timestamp": self.formatTime(record),
+            "level": record.levelname,
+        }
+        for part in parts:
+            if self.key_value_split_char in part:
+                key, value = part.split(self.key_value_split_char, maxsplit=1)
+                result_dict[key.strip()] = value.strip()
+            else:
+                if self.skip_missing_key_value_split_char:
+                    continue
+                raise ValueError(f"Each part of the record must contain the key_value_split_char. Part: {part}")
+        return json.dumps(result_dict)

cloe_logging/handlers/__init__.py

@@ -0,0 +1,5 @@
+from .unity_catalog_handler import UnityCatalogHandler
+from .log_analytics_handler import LogAnalyticsHandler
+from .snowflake_handler import SnowflakeHandler
+
+__all__ = ["UnityCatalogHandler", "LogAnalyticsHandler", "SnowflakeHandler"]

cloe_logging/handlers/log_analytics_handler.py

@@ -0,0 +1,217 @@
+import base64
+import hashlib
+import hmac
+import logging
+import os
+from datetime import datetime
+
+try:
+    import requests
+except ImportError:
+    requests = None  # type: ignore
+    print("Optional dependency 'log_analytics' is not installed. Some functionalities may not be available.")
+
+from cloe_logging.utility.serializer import create_logserializer
+
+
+class LogAnalyticsHandler(logging.Handler):
+    """A custom logging handler for Azure Log Analytics.
+
+    The handler will by default always send the timestamp and loglevel of the log message.
+
+    Attributes:
+        METHOD (str): The HTTP method for the requests.
+        RESOURCE (str): The resource path for the requests.
+        CONTENT_TYPE (str): The content type for the requests.
+    """
+
+    METHOD = "POST"
+    RESOURCE = "/api/logs"
+    CONTENT_TYPE = "application/json; charset=utf-8"
+
+    def __init__(
+        self,
+        workspace_id: str | None = None,
+        shared_key: str | None = None,
+        log_type: str | None = None,
+        test_connectivity: bool = True,
+        column_split_char: str = "|",
+        key_value_split_char: str = ":",
+        **kwargs,  # required to work with the Factory
+    ):
+        """Initializes a new instance of the LogAnalyticsHandler class.
+
+        Args:
+            workspace_id (str): The workspace ID for Azure Log Analytics.
+            shared_key (str): The shared key for Azure Log Analytics.
+            log_type (str): The log type for Azure Log Analytics.
+            column_split_char (str, optional): The character used to split columns in the log message. Defaults to "|".
+            key_value_split_char (str, optional): The character used to split keys and values in the log message.
+                                                  Defaults to ":".
+            test_connectivity (bool, optional): Whether to test connectivity to Azure Log Analytics when initializing
+                                                the handler. Defaults to True.
+        """
+        self.column_split_char: str = column_split_char
+        self.key_value_split_char: str = key_value_split_char
+        self.workspace_id: str | None = workspace_id or os.environ.get("LOG_ANALYTICS_WORKSPACE_ID")
+        self.shared_key: str | None = shared_key or os.environ.get("LOG_ANALYTICS_WORKSPACE_SHARED_KEY")
+        self.log_type: str | None = log_type or os.environ.get("LOG_TYPE")
+        if not self.workspace_id or not self.shared_key or not self.log_type:
+            raise ValueError(
+                "The workspace_id, shared_key, and log_type must be provided or set as environment variables."
+            )
+        logging.Handler.__init__(self)
+        self.session = requests.Session()
+        formatter = logging.Formatter("timestamp:%(asctime)s | level: %(levelname)-8s | %(message)s")
+        self.setFormatter(formatter)
+        self.serializer = create_logserializer()
+        self.serializer.column_split_char = self.column_split_char
+        self.serializer.key_value_split_char = self.key_value_split_char
+        if test_connectivity:
+            self.test_connectivity()
+
+    def test_connectivity(self):
+        """Checks the connectivity to the Log Analytics workspace without sending a log.
+
+        Raises:
+            ValueError: If the connection to Azure Log Analytics fails.
+        """
+
+        class FakeRecord(logging.LogRecord):
+            """Mock Record to use in the emit method."""
+
+            def __init__(self, msg, level=logging.INFO):
+                name = "test"
+                pathname = "test_path"
+                lineno = 1
+                args = ()
+                exc_info = None
+                super().__init__(
+                    name,
+                    level,
+                    pathname,
+                    lineno,
+                    msg,
+                    args,
+                    exc_info,
+                    func=None,
+                    sinfo=None,
+                )
+                self.levelname = "INFO"
+
+            def getMessage(self):
+                return self.msg
+
+        try:
+            self.emit(FakeRecord(msg=f"''{self.key_value_split_char}''"))
+        except ValueError as err:
+            raise ValueError(f"Failed to connect to Azure Log Analytics: {str(err)}") from err
+
+    def __eq__(self, other):
+        """Checks if two LogAnalyticsHandler instances are equal.
+
+        Instances are considered equal if they have the same workspace_id, shared_key, and log_type.
+        This will prevent the same handler from being added multiple times to a single logger.
+
+        Args:
+            other (LogAnalyticsHandler): The other LogAnalyticsHandler instance to compare with.
+
+        Returns:
+            bool: True if instances are equal, False otherwise.
+        """
+        if isinstance(other, LogAnalyticsHandler):
+            return (
+                self.workspace_id == other.workspace_id
+                and self.shared_key == other.shared_key
+                and self.log_type == other.log_type
+            )
+        return False
+
+    def __hash__(self):
+        """Generates a unique hash value for the object.
+
+        This method overrides the built-in `__hash__` method to generate a unique hash value for the object,
+        which is particularly useful for using the object in sets or as keys in dictionaries.
+
+        The hash value is computed based on the 'workspace_id', 'shared_key', and 'log_type' attributes of the object.
+        """
+        return hash((self.workspace_id, self.shared_key, self.log_type))
+
+    def _build_signature(self, date, content_length):
+        """Builds the signature for the request.
+
+        Args:
+            date (str): The date of the request.
+            content_length (int): The length of the content in the request.
+
+        Returns:
+            str: The authorization signature for the request.
+        """
+        x_headers = "x-ms-date:" + date
+        string_to_hash = f"{self.METHOD}\n{content_length}\n{self.CONTENT_TYPE}\n{x_headers}\n{self.RESOURCE}"
+        bytes_to_hash = bytes(string_to_hash, encoding="utf-8")
+        decoded_key = base64.b64decode(self.shared_key)
+        encoded_hash = base64.b64encode(
+            hmac.new(decoded_key, bytes_to_hash, digestmod=hashlib.sha256).digest(),
+        ).decode()
+        authorization = f"SharedKey {self.workspace_id}:{encoded_hash}"
+        return authorization
+
+    def _make_message_compliant(self, input_string):
+        """Encodes the input string as UTF-8 to make it compliant.
+
+        Args:
+            input_string (str): The string to be encoded.
+
+        Returns:
+            str: The encoded string.
+        """
+        return str(input_string).encode("utf-8")
+
+    def _get_url(self):
+        """Generates the URL for the Azure Log Analytics workspace.
+
+        Returns:
+            str: The URL of the Azure Log Analytics workspace.
+        """
+        uri = f"https://{self.workspace_id}.ods.opinsights.azure.com{self.RESOURCE}?api-version=2016-04-01"
+        return uri
+
+    def emit(self, record: logging.LogRecord):
+        """Sends the log message to Azure Log Analytics.
+
+        Args:
+            record (logging.LogRecord): The record instance with the log message.
+
+        Raises:
+            ValueError: If record.msg is not a string, or if failed to send log to Azure Log Analytics.
+
+        Note:
+            This method uses the following methods:
+            - _parse_string_to_dict to convert the log message to a dictionary.
+            - _make_message_compliant to make the log message compliant.
+            - _build_signature to build the signature for the request.
+            - _get_url to get the URL of the Azure Log Analytics workspace.
+        """
+        try:
+            log_message = self.format(record)
+            log_message_dict = self.serializer.serialize(log_message)
+            compliant_log_message = self._make_message_compliant(str(log_message_dict))
+            content_length = len(compliant_log_message)
+            rfc1123date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT")
+            signature = self._build_signature(rfc1123date, content_length)
+
+            headers = {
+                "content-type": self.CONTENT_TYPE,
+                "Authorization": signature,
+                "Log-Type": self.log_type,
+                "x-ms-date": rfc1123date,
+            }
+            response = self.session.post(self._get_url(), data=compliant_log_message, headers=headers, timeout=30)
+            response.raise_for_status()
+        except AttributeError as exc:
+            raise ValueError(exc) from exc
+        except requests.exceptions.RequestException as exc:
+            raise ValueError(f"Failed to send log to Azure Log Analytics: {exc}") from exc
+        except Exception as exc:
+            raise ValueError(exc) from exc

cloe_logging/handlers/snowflake_handler.py

@@ -0,0 +1,120 @@
+import logging
+import os
+
+try:
+    from cloe_util_snowflake_connector.connection_parameters import ConnectionParameters
+    from cloe_util_snowflake_connector.snowflake_interface import SnowflakeInterface
+except ImportError:
+    ConnectionParameters = None  # type: ignore
+    SnowflakeInterface = None  # type: ignore
+    print("Optional dependency 'snowflake' is not installed. Some functionalities may not be available.")
+from cloe_logging.utility.serializer import create_logserializer
+
+
+class SnowflakeHandler(logging.Handler):
+    """A custom logging handler for Snowflake.
+
+    The handler will by default always send the timestamp and loglevel of the log message.
+    """
+
+    def __init__(
+        self,
+        target_db: str = "",
+        target_schema: str = "",
+        target_table: str = "",
+        column_split_char: str = "|",
+        key_value_split_char: str = ":",
+        **kwargs,  # required to work with the Factory
+    ):
+        """Initializes a new instance of the SnowflakeHandler class.
+
+        Args:
+            target_db: The name of the Database to send logs to.
+            target_schema: the name of the table to send logs to.
+            target_table: the name of the schema to send logs to.
+            column_split_char (str, optional): The character used to split columns in the log message. Defaults to "|".
+            key_value_split_char (str, optional): The character used to split keys and values in the log message.
+                                                  Defaults to ":".
+        """
+        self.column_split_char: str = column_split_char
+        self.key_value_split_char: str = key_value_split_char
+        self.target_db: str = os.environ.get("CLOE_SNOWFLAKE_DATABASE", target_db)
+        self.target_schema: str = os.environ.get("CLOE_SNOWFLAKE_SCHEMA", target_schema)
+        self.target_table: str = os.environ.get("CLOE_SNOWFLAKE_TABLE", target_table)
+        logging.Handler.__init__(self)
+        self.connection = self._get_snowflake_connection()
+        formatter = logging.Formatter("timestamp:%(asctime)s | level: %(levelname)-8s | %(message)s")
+        self.setFormatter(formatter)
+        self.serializer = create_logserializer()
+
+    def __eq__(self, other):
+        """Checks if two SnowflakeHandler instances are equal.
+
+        Instances are considered equal if they have the same workspace_id, shared_key, and log_type.
+        This will prevent the same handler from being added multiple times to a single logger.
+
+        Args:
+            other (SnowflakeHandler): The other SnowflakeHandler instance to compare with.
+
+        Returns:
+            bool: True if instances are equal, False otherwise.
+        """
+        if isinstance(other, SnowflakeHandler):
+            return (
+                self.target_db == other.target_db
+                and self.target_schema == other.target_schema
+                and self.target_table == other.target_table
+            )
+        return False
+
+    def __hash__(self):
+        """Generates a unique hash value for the object.
+
+        This method overrides the built-in `__hash__` method to generate a unique hash value for the object,
+        which is particularly useful for using the object in sets or as keys in dictionaries.
+
+        The hash value is computed based on the target_db, target_schema, and target_table attributes.
+        """
+        return hash((self.target_db, self.target_schema, self.target_table))
+
+    def _get_snowflake_connection(self) -> SnowflakeInterface:
+        conn_params = ConnectionParameters.init_from_env_variables()
+        snowflake_conn = SnowflakeInterface(conn_params)
+        return snowflake_conn
+
+    def _parse_dict_to_sql_insert(self, input_dict: dict) -> str:
+        """
+        Generate a SQL INSERT statement from a dictionary.
+
+        Parameters:
+            table_name (str): The name of the table to insert into.
+            data (dict): A dictionary where keys are column names and values are the data to insert.
+
+        Returns:
+            str: A SQL INSERT statement as a string.
+        """
+        columns = ", ".join(input_dict.keys())
+        values = ", ".join(f"'{str(v)}'" for v in input_dict.values())
+        sql_statement = (
+            f"INSERT INTO {self.target_db}.{self.target_schema}.{self.target_table} ({columns}) VALUES ({values})"
+        )
+
+        return sql_statement
+
+    def emit(self, record: logging.LogRecord):
+        """Sends the log message to Snowflake.
+
+        Args:
+            record (logging.LogRecord): The record instance with the log message.
+
+        Raises:
+            ValueError: If record.msg is not a string, or if failed to send log to Snowflake.
+
+        """
+        try:
+            log_message = self.format(record)
+            log_message_dict = self.serializer.serialize(log_message)
+            log_insert_statement = self._parse_dict_to_sql_insert(log_message_dict)
+            self.connection.run_one_with_return(log_insert_statement)
+        except Exception as exc:
+            raise ValueError(exc) from exc

cloe_logging/handlers/unity_catalog_handler.py

@@ -0,0 +1,154 @@
+import json
+import logging
+from typing import cast
+
+try:
+    from databricks.sdk import WorkspaceClient
+    from databricks.sdk.service.sql import ExecuteStatementRequestOnWaitTimeout
+except ImportError:
+    WorkspaceClient = None  # type: ignore
+    ExecuteStatementRequestOnWaitTimeout = None  # type: ignore
+    print("Optional dependency 'databricks' is not installed. Some functionalities may not be available.")
+
+from ..formatters import DictFormatter
+
+
+class UnityCatalogHandler(logging.Handler):
+    """A custom logging handler for Databricks Unity Catalog.
+
+    The handler will by default always send the timestamp and loglevel of the log message.
+    """
+
+    def __init__(
+        self,
+        catalog: str | None = None,
+        schema: str | None = None,
+        table: str | None = None,
+        columns: dict[str, str] | None = None,
+        workspace_url: str | None = None,
+        warehouse_id: str | None = None,
+        column_split_char: str = "|",
+        key_value_split_char: str = ":",
+        workspace_client: WorkspaceClient | None = None,
+        formatter: DictFormatter | None = None,
+        **kwargs,  # required to work with the Factory
+    ):
+        """Initializes a new instance of the DatabricksHandler class.
+
+        Note:
+            The handler will reuse the existing authentication from the Azure and Databricks CLI or any
+            other spark connection that is already established.
+
+        Args:
+            catalog: The name of the catalog to send logs to.
+            schema: The name of the schema to send logs to.
+            table: The name of the table to send logs to.
+            columns: A dictionary of column names and their corresponding data types.
+            workspace_url: The URL of the Azure Databricks workspace.
+            warehouse_id: The ID of the Databricks warehouse.
+            column_split_char: The character used to split columns in the log message. Defaults to "|".
+            key_value_split_char: The character used to split keys and values in the log message. Defaults to ":".
+            workspace_client: An instance of WorkspaceClient for dependency injection.
+            formatter: An instance of DictFormatter for dependency injection.
+        """
+        self.workspace_url = workspace_url
+        self.column_split_char = column_split_char
+        self.key_value_split_char = key_value_split_char
+        self.catalog = catalog
+        self.schema = schema
+        self.table = table
+        self.warehouse_id = cast(str, warehouse_id)
+        if not all([self.catalog, self.schema, self.table, self.warehouse_id, self.workspace_url]):
+            raise ValueError(
+                "You must provide a workspace_url, warehouse_id, catalog, schema, and table to create a DatabricksHandler."
+            )
+        self.table_identifier = f"{self.catalog}.{self.schema}.{self.table}"
+        self.workspace_client = workspace_client or WorkspaceClient(host=self.workspace_url)
+        super().__init__(**kwargs)
+        self.setFormatter(
+            formatter or DictFormatter(column_split_char=column_split_char, key_value_split_char=key_value_split_char)
+        )
+        self.ensure_table_exists(columns)
+
+    def ensure_table_exists(self, columns: dict[str, str] | None) -> None:
+        """Ensure that the table exists in the catalog.
+
+        This method will create the table in the catalog if it does not already exist.
+
+        Args:
+            columns: A dictionary of column names and their corresponding data types
+
+        Raises:
+            ValueError: If the columns dictionary is empty.
+        """
+        if not columns:
+            raise ValueError("You must provide a dictionary of columns to create the logging table.")
+        columns = {**columns, "timestamp": "timestamp", "level": "string"}
+        table_exists = self.workspace_client.tables.exists(self.table_identifier).table_exists is True
+        if table_exists is False:
+            columns["timestamp"] = "TIMESTAMP"
+            columns["level"] = "STRING"
+            columns_definition = ", ".join([f"{col_name} {col_type}" for col_name, col_type in columns.items()])
+
+            self.workspace_client.statement_execution.execute_statement(
+                statement=f"CREATE TABLE IF NOT EXISTS {self.table_identifier} ({columns_definition})",
+                warehouse_id=self.warehouse_id,
+                wait_timeout="30s",
+            )
+
+    def __eq__(self, other: object) -> bool:
+        """Checks if two DatabricksHandler instances are equal.
+
+        Instances are considered equal if they have the same catalog, schema, and table.
+        This will prevent the same handler from being added multiple times to a single logger.
+
+        Args:
+            other: The other DatabricksHandler instance to compare with.
+
+        Returns:
+            True if instances are equal, False otherwise.
+        """
+        return (
+            isinstance(other, UnityCatalogHandler)
+            and self.catalog == other.catalog
+            and self.schema == other.schema
+            and self.table == other.table
+        )
+
+    def __hash__(self):
+        """Generates a unique hash value for the object.
+
+        This method overrides the built-in `__hash__` method to generate a unique hash value for the object,
+        which is particularly useful for using the object in sets or as keys in dictionaries.
+        The hash value is computed based on the catalog, schema, and table attributes.
+        """
+        return hash((self.catalog, self.schema, self.table))
+
+    def _parse_dict_to_sql_insert(self, input_dict: dict) -> str:
+        """Generate a SQL INSERT statement from a dictionary.
+
+        Parameters:
+            input_dict (dict): A dictionary where keys are column names and values are the data to insert.
+        """
+        columns = ", ".join(input_dict.keys())
+        values = ", ".join(f"'{str(v)}'" for v in input_dict.values())
+        split_values = values.split(", ")
+        timestamp = split_values[0]
+        casted_timestamp = f"to_timestamp({timestamp}, 'yyyy-MM-dd HH:mm:ss,SSS')"
+        joined_values = ", ".join([casted_timestamp] + split_values[1:])
+        sql_statement = f"INSERT INTO {self.table_identifier} ({columns}) VALUES ({joined_values})"
+        return sql_statement
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Put a log record into the Queue.
+
+        Args:
+            record (logging.LogRecord): The log record to put into the Queue.
+        """
+        log_message = self.format(record)
+        log_insert_statement = self._parse_dict_to_sql_insert(json.loads(log_message))
+        self.workspace_client.statement_execution.execute_statement(
+            statement=log_insert_statement,
+            warehouse_id=self.warehouse_id,
+            on_wait_timeout=ExecuteStatementRequestOnWaitTimeout.CONTINUE,
+        )

cloe_logging/logger_factory.py

@@ -0,0 +1,159 @@
+import logging
+from collections.abc import Callable
+
+from cloe_logging.handlers import UnityCatalogHandler, SnowflakeHandler, LogAnalyticsHandler
+
+
+class LoggerFactory:
+    DEFAULT_COLUMN_SPLIT_CHAR = "|"
+    DEFAULT_KEY_VALUE_SPLIT_CHAR = ":"
+
+    @staticmethod
+    def get_logger(
+        handler_types: str | list[str],
+        logger_name: str = __name__,
+        logging_level: int = logging.INFO,
+        log_format: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        **kwargs,
+    ) -> logging.Logger:
+        """Creates a logger with the specified handler types.
+
+        Args:
+            handler_types: The type of handler to use for the logger.
+            logger_name: The name of the logger.
+            logging_level: The logging level for the logger.
+            log_format: The format of the log messages.
+            kwargs: Additional arguments to pass to the handler.
+
+        Note:
+            Supported handler types are "console", "file", "unity_catalog", "snowflake", and "log_analytics".
+
+        Returns:
+            The logger with the specified handler types.
+        """
+        logger = logging.getLogger(logger_name)
+        logger.setLevel(logging_level)
+        if isinstance(handler_types, str):
+            handler_types = [handler_types]
+
+        for handler_type in handler_types:
+            handler = LoggerFactory.get_handler(handler_type, log_format, **kwargs)
+            LoggerFactory.add_handler_if_not_exists(logger, handler)
+        return logger
+
+    @staticmethod
+    def get_handler(
+        handler_type: str,
+        log_format: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        **kwargs,
+    ) -> logging.Handler:
+        HANDLER_FUNCTIONS: dict[str, Callable] = {
+            "console": LoggerFactory.get_console_handler,
+            "file": LoggerFactory.get_file_handler,
+            "unity_catalog": LoggerFactory.get_unity_catalog_handler,
+            "snowflake": LoggerFactory.get_snowflake_handler,
+            "log_analytics": LoggerFactory.get_log_analytics_handler,
+        }
+        handler = HANDLER_FUNCTIONS[handler_type](**kwargs, log_format=log_format)
+        return handler
+
+    @staticmethod
+    def get_console_handler(log_format: str, **kwargs) -> logging.Handler:
+        handler = logging.StreamHandler()
+        handler.setFormatter(logging.Formatter(log_format))
+        return handler
+
+    @staticmethod
+    def get_file_handler(
+        log_format: str,
+        filename: str | None = None,
+        mode: str = "a",
+        encoding: str | None = None,
+        delay: bool = False,
+        **kwargs,
+    ) -> logging.Handler:
+        if filename is None:
+            raise ValueError("filename is required for file logger")
+        handler = logging.FileHandler(filename, mode, encoding, delay)
+        handler.setFormatter(logging.Formatter(log_format))
+        return handler
+
+    @classmethod
+    def get_unity_catalog_handler(
+        cls,
+        uc_table_name: str,
+        uc_catalog_name: str,
+        uc_schema_name: str,
+        uc_table_columns: dict[str, str],
+        workspace_url: str,
+        warehouse_id: str,
+        column_split_char: str = DEFAULT_COLUMN_SPLIT_CHAR,
+        key_value_split_char: str = DEFAULT_KEY_VALUE_SPLIT_CHAR,
+        **kwargs,
+    ) -> logging.Handler:
+        return UnityCatalogHandler(
+            catalog=uc_catalog_name,
+            schema=uc_schema_name,
+            table=uc_table_name,
+            columns=uc_table_columns,
+            workspace_url=workspace_url,
+            warehouse_id=warehouse_id,
+            column_split_char=column_split_char,
+            key_value_split_char=key_value_split_char,
+        )
+
+    @classmethod
+    def get_snowflake_handler(
+        cls,
+        target_db: str,
+        target_schema: str,
+        target_table: str,
+        column_split_char: str = DEFAULT_COLUMN_SPLIT_CHAR,
+        key_value_split_char: str = DEFAULT_KEY_VALUE_SPLIT_CHAR,
+        **kwargs,
+    ) -> logging.Handler:
+        return SnowflakeHandler(
+            target_db=target_db,
+            target_schema=target_schema,
+            target_table=target_table,
+            column_split_char=column_split_char,
+            key_value_split_char=key_value_split_char,
+        )
+
+    @classmethod
+    def get_log_analytics_handler(
+        cls,
+        workspace_id: str,
+        shared_key: str,
+        log_type: str,
+        test_connectivity: bool,
+        column_split_char: str = DEFAULT_COLUMN_SPLIT_CHAR,
+        key_value_split_char: str = DEFAULT_KEY_VALUE_SPLIT_CHAR,
+        **kwargs,
+    ) -> logging.Handler:
+        return LogAnalyticsHandler(
+            workspace_id=workspace_id,
+            shared_key=shared_key,
+            log_type=log_type,
+            test_connectivity=test_connectivity,
+            column_split_char=column_split_char,
+            key_value_split_char=key_value_split_char,
+        )
+
+    @staticmethod
+    def add_handler_if_not_exists(logger: logging.Logger, handler: logging.Handler) -> logging.Logger:
+        """Adds a handler to the logger if it does not already exist.
+
+        Args:
+            logger: The logger to add the handler to.
+            handler: The handler to add to the logger.
+
+        Returns:
+            The logger with the handler added.
+        """
+        if len(logger.handlers) > 0:
+            if not any([isinstance(h, handler.__class__) for h in logger.handlers]):
+                logger.addHandler(handler)
+        else:
+            logger.addHandler(handler)
+        return logger

cloe_logging/utility/__init__.py

@@ -0,0 +1,3 @@
+from .serializer import LogSerializer
+
+__all__ = ["LogSerializer"]

cloe_logging/utility/serializer.py

@@ -0,0 +1,80 @@
+from abc import ABC, abstractmethod
+
+
+class LogSerializer(ABC):
+    """
+    Used to serialize log records into various formats.
+
+    Currently supported formats:
+    -   'dict'
+    """
+
+    @abstractmethod
+    def serialize(self, log_record):
+        """
+        Serializes a log record into the specified format.
+
+        Args:
+            log_record: The log record to be serialized.
+            format: The format to serialize the log record into. Currently supports "dict".
+
+        Returns:
+            dict: The serialized log record in dictionary format.
+
+        Raises:
+            NotImplementedError: If the specified format is not supported.
+        """
+        pass
+
+
+class DictSerializer(LogSerializer):
+    def __init__(
+        self,
+        column_split_char: str = "|",
+        key_value_split_char: str = ":",
+    ):
+        self.column_split_char: str = column_split_char
+        self.key_value_split_char: str = key_value_split_char
+
+    def serialize(self, log_record):
+        """
+        Converts a formatted string to a dictionary.
+
+        Parameters:
+            log_record: The string to be converted to a dictionary.
+            column_split_char: The character that separates different key-value pairs in the string (default is "|").
+            key_value_split_char: The character that separates keys from values in the string (default is ":").
+
+        Returns:
+            dict: The converted dictionary.
+
+        Raises:
+            ValueError: If the log_record is not a string, or if any part of the log_record does not contain the key_value_split_char.
+        """
+        if not isinstance(log_record, str):
+            raise ValueError("record must be a string.")
+        parts = [part.strip() for part in log_record.split(self.column_split_char)]
+        result_dict = {}
+
+        for part in parts:
+            try:
+                key, value = part.split(self.key_value_split_char, maxsplit=1)
+            except ValueError as exc:
+                raise ValueError(
+                    f"Each part of the record must contain the key_value_split_char. Error: {str(exc)}",
+                ) from exc
+            result_dict[key.strip()] = value.strip()
+        return result_dict
+
+
+def create_logserializer(format="dict"):
+    logserializers = {
+        "dict": DictSerializer,
+    }
+    try:
+        serializer = logserializers[format]()
+    except KeyError as exc:
+        raise NotImplementedError(
+            f"The selected format is not supported yet. Error: {str(exc)}",
+        ) from exc
+    return serializer

cloe_logging-0.3.7.dist-info/METADATA

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: cloe-logging
+Version: 0.3.7
+Summary: A Standardized Solution for logging to various targets.
+Home-page: https://initions.com/
+Author: initions
+Author-email: ICSMC_EXT_PYPIORG@accenture.com
+License: MIT
+Requires-Python: <3.12,>=3.11
+Requires-Dist: pydantic<3.0.0,>=2.7.0
+Requires-Dist: typing-extensions<5.0.0,>=4.12.2
+Requires-Dist: databricks-sdk>=0.39.0; extra == 'databricks'
+Requires-Dist: requests<3.0.0,>=2.31.0; extra == 'log-analytics'
+Requires-Dist: cloe-util-snowflake-connector<2.0.0,>=1.0.2; extra == 'snowflake'

cloe_logging-0.3.7.dist-info/RECORD

@@ -0,0 +1,17 @@
+cloe_logging/__init__.py,sha256=3rsW-XtJbAaENMtN9ah2qRZyQnZOtqTnkCE_OqXBLNw,74
+cloe_logging/logger_factory.py,sha256=az_XcRSrUCge4yG3HXNqFPSPRJ0V8EMKmjPPl54UIms,5541
+cloe_logging/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cloe_logging/decorators/__init__.py,sha256=ZFfADCScnpZ3ofxNgle1UybF322cdH9vuIGCx3S9wUU,71
+cloe_logging/decorators/devops_decorator.py,sha256=1jCfNa22lU87u-QPNUt6fVLJOCINhZncYz76UjUh2XE,1959
+cloe_logging/formatters/__init__.py,sha256=0FbiRRJbo4zYgNb2QSxY5aGthu8evt9Gex5c_-jxCIs,136
+cloe_logging/formatters/devops_formatter.py,sha256=KwarCm_EPpRV-cZk6nGM3GZVNKN39FNOofXfC0KfkQg,3301
+cloe_logging/formatters/dict_formatter.py,sha256=PsbNXxdCZxtEC9g-nIukhdQwCcsCfz42EumHPHFt9-4,1553
+cloe_logging/handlers/__init__.py,sha256=xgYTK6qXJEgNFPZE6yHXwS1o9bmGuQQZ7so_7BncW7A,236
+cloe_logging/handlers/log_analytics_handler.py,sha256=2Z-Hg2mWgvqAbO3nvego9AjDy3PGgzPQOxVWL7x5GAY,9038
+cloe_logging/handlers/snowflake_handler.py,sha256=bZR5UIJIlivAJT3IeniVoFXlsQhq5-YDuSUH4qrXeL0,5054
+cloe_logging/handlers/unity_catalog_handler.py,sha256=aXIiwDq9tHHheAFMusHhEI9uvYpI39KCscuVntFvNLA,6879
+cloe_logging/utility/__init__.py,sha256=wp758l5P1M20bNUbbGojSkBUscFUgRd_TLLdnHgQ_l8,70
+cloe_logging/utility/serializer.py,sha256=uPdmpawqM3WXZQVxz63deCalvKJYxRNecxjBQeDzTIY,2639
+cloe_logging-0.3.7.dist-info/METADATA,sha256=TCZNiRj3PJJpq8PcKGqkiYRpYWnesboKsv5TDGKdAM8,1033
+cloe_logging-0.3.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cloe_logging-0.3.7.dist-info/RECORD,,

cloe_logging-0.3.7.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any