cloe-nessy 0.3.3__py3-none-any.whl → 0.3.8__py3-none-any.whl

cloe_nessy/object_manager/table_manager.py

@@ -1,22 +1,119 @@
+import functools
+import logging
+from dataclasses import dataclass, field
+
+from delta import DeltaTable  # type: ignore
+
 from ..logging import LoggerMixin
+from ..models import Table
 from ..session import SessionManager
 
 
+@dataclass
+class TableManagerLogs:
+    """Dataclass defining the table manager logs table."""
+
+    logger_name = "Tabular:TableManager"
+    log_type: str = "nessy_simple_logs"
+    uc_table_name: str = "nessy_simple_logs"
+    uc_table_columns: dict[str, str] = field(
+        default_factory=lambda: {
+            "message": "STRING",
+        }
+    )
+
+
+def table_log_decorator(operation: str):
+    """Creates a decorator that logs the start, failure (if any), and completion of a table operation.
+
+    The created decorator wraps a function that performs an operation on a table. The decorator logs
+    the start of the operation, calls the original function, logs if there was an exception, and logs
+    the completion of the operation. Functions that are wrapped must support the self._table_logger
+    attribute.
+
+    Args:
+        operation: The name of the operation to be logged. This will be included in the log messages.
+
+    Returns:
+        inner_decorator: A decorator that can be used to wrap a function that performs an operation on a table.
+
+    Example:
+        ```python
+        @table_log_decorator(operation='delete_physical_data_for_table')
+        def _delete_physical_data(self, table_identifier: str):
+            self._dbutils.fs.rm(table_location, recurse=True)
+        ```
+    """
+
+    def inner_decorator(func):
+        @functools.wraps(func)
+        def wrapper(self, *args, **kwargs):
+            table_identifier = kwargs.get("table_identifier") or kwargs.get("table").identifier or args[0]
+            if not isinstance(table_identifier, str):
+                # assume its a Table object
+                table_identifier = table_identifier.identifier
+            self._tabular_logger.info(
+                "operation:%s | identifier:%s | status:start | error:''",
+                operation,
+                table_identifier,
+            )
+            try:
+                func(self, *args, **kwargs)
+            except Exception as e:
+                self._tabular_logger.error(
+                    "operation:%s | identifier:%s | status:failed | error:%s",
+                    operation,
+                    table_identifier,
+                    e,
+                )
+                raise e
+            else:
+                self._tabular_logger.info(
+                    "operation:%s | identifier:%s | status:completed | error:''",
+                    operation,
+                    table_identifier,
+                )
+
+        return wrapper
+
+    return inner_decorator
+
+
 class TableManager(LoggerMixin):
-    """TableManager class for managing tables in the catalog."""
+    """TableManager class for managing tables."""
 
-    def __init__(self):
+    def __init__(self, tabular_logger: logging.Logger | None = None):
         self._spark = SessionManager.get_spark_session()
         self._utils = SessionManager.get_utils()
         self._console_logger = self.get_console_logger()
         self._console_logger.debug("TableManager initialized...")
-        self._tabular_logger = self.get_tabular_logger(uc_table_name="TableManager")
+        self._tabular_logger = tabular_logger or self.get_tabular_logger(**TableManagerLogs().__dict__)
         self._tabular_logger.debug("message:TableManager initialized.")
 
-    @staticmethod
-    def create_table():
-        """Create a table in the catalog."""
-        raise NotImplementedError
+    @table_log_decorator(operation="create")
+    def create_table(
+        self,
+        table: Table,
+        ignore_if_exists: bool = False,
+        replace: bool = False,
+    ) -> None:
+        """Creates a Table in the catalog.
+
+        Args:
+            table: A Table object representing the Delta table.
+            ignore_if_exists: If set to True, the function will return early
+                without doing anything if the table already exists.
+            replace: If set to True, the function will replace the table if it
+                already exists.
+        """
+        if ignore_if_exists and self.table_exists(table):
+            return
+        self._console_logger.info(f"Creating table: {table.identifier}")
+        self._spark.sql(f"USE CATALOG {table.catalog};")
+        self._spark.sql(f"USE SCHEMA {table.schema};")
+        for statement in table.get_create_statement(replace=replace).split(";"):
+            if statement and statement != "\n":
+                self._spark.sql(statement)
 
     def drop_table(self, table_identifier: str, delete_physical_data: bool = False):
         """Deletes a Table. For security reasons you are forced to pass the table_name.
@@ -56,3 +153,82 @@ class TableManager(LoggerMixin):
         """
         self._console_logger.info("... deleting physical data for table [ '' ] from Catalog.")
         raise NotImplementedError("This can be implemented, once a Table object is available.")
+
+    def get_delta_table(self, table: Table | None = None, location: str | None = None) -> DeltaTable:
+        """Get the DeltaTable object from the Table objects location or a location string.
+
+        Args:
+            table: A Table object representing the Delta table.
+            location: A string representing the table location.
+
+        Returns:
+            The DeltaTable object corresponding to the given Table object or location string.
+
+        Raises:
+            ValueError: If neither table nor location is provided, or if both are provided.
+        """
+        if (table is None and location is None) or (table is not None and location is not None):
+            raise ValueError("Either table or location must be provided, but not both.")
+
+        if table is not None:
+            location = str(table.storage_path)
+        self._console_logger.info(f"Getting DeltaTable object for location: {location}")
+        return DeltaTable.forPath(self._spark, str(location))
+
+    def table_exists(self, table: Table | None = None, table_identifier: str | None = None) -> bool:
+        """Checks if a table exists in the catalog.
+
+        Args:
+            table: A Table object representing the Delta table.
+            table_identifier: A string representing the table identifier in the format 'catalog.schema.table'.
+
+        Returns:
+            True if the table exists, else False.
+
+        Raises:
+            ValueError: If neither table nor table_identifier is provided, or if both are provided.
+            ValueError: If the table_identifier is not in the format 'catalog.schema.table'.
+        """
+        if (table is None and table_identifier is None) or (table is not None and table_identifier is not None):
+            raise ValueError("Either table or table_identifier must be provided, but not both.")
+
+        if table is not None:
+            catalog = table.catalog
+            schema = table.schema
+            table_name = table.name
+        else:
+            assert table_identifier is not None, "table_identifier must be provided."
+            catalog, schema, table_name = table_identifier.split(".")
+            if not all([catalog, schema, table_name]):
+                raise ValueError("Invalid table identifier format. Expected 'catalog.schema.table'.")
+
+        query_result = self._spark.sql(
+            f"""
+                SELECT 1 FROM {catalog}.information_schema.tables
+                WHERE table_name = '{table_name}'
+                AND table_schema = '{schema}'
+                LIMIT 1""",
+        )
+        result = query_result.count() > 0
+        self._console_logger.info(f"Table [ '{catalog}.{schema}.{table_name}' ] exists: {result}")
+        return result is True
+
+    @table_log_decorator(operation="refresh")
+    def refresh_table(self, table: Table | None = None, table_identifier: str | None = None):
+        """Refreshes the metadata of a Delta table.
+
+        Args:
+            table: A Table object representing the Delta table.
+            table_identifier: The identifier of the Delta table in the format 'catalog.schema.table'.
+
+        Raises:
+            ValueError: If neither table nor table_identifier is provided, or if both are provided.
+        """
+        if (table is None and table_identifier is None) or (table is not None and table_identifier is not None):
+            raise ValueError("Either table or table_identifier must be provided, but not both.")
+
+        if table is not None:
+            table_identifier = f"{table.catalog}.{table.schema}.{table.name}"
+
+        self._console_logger.info(f"Refreshing table: {table_identifier}")
+        self._spark.sql(f"REFRESH TABLE {table_identifier};")

cloe_nessy/object_manager/volume_manager.py

@@ -0,0 +1,70 @@
+import logging
+
+from ..logging import LoggerMixin
+from ..models import Volume
+from ..session import SessionManager
+
+
+class VolumeManager(LoggerMixin):
+    """VolumeManager class for managing volumes."""
+
+    def __init__(self, console_logger: logging.Logger | None = None):
+        self._spark = SessionManager.get_spark_session()
+        self._console_logger = console_logger or self.get_console_logger()
+
+    def create_volume(self, volume: Volume):
+        """Creates a Volume in the catalog.
+
+        Args:
+            volume: A Volume object representing the UC object.
+        """
+        self._console_logger.info(f"Creating volume: {volume.identifier}")
+        self._spark.sql(f"USE CATALOG {volume.catalog};")
+        self._spark.sql(f"USE SCHEMA {volume.schema_name};")
+        for statement in volume.get_create_statement().split(";"):
+            if statement and statement != "\n":
+                self._spark.sql(statement)
+
+    def drop_volume(self, volume: Volume, if_exists: bool = True):
+        """Delete the volume.
+
+        Args:
+            volume: The volume to be deleted.
+            if_exists: If False, an error will be raised if the volume does not exist.
+        """
+        self._console_logger.info(f"Deleting volume: [' {volume.identifier}' ]")
+        self._spark.sql(f"DROP VOLUME {'IF EXISTS' if if_exists else ''} {volume.escaped_identifier};")
+        self._console_logger.info(f"Volume [' {volume.identifier}' ] has been deleted.")
+
+    def volume_exists(self, volume: Volume | None = None, volume_identifier: str | None = None) -> bool:
+        """Check if the volume exists.
+
+        Args:
+            volume: The volume to check.
+            volume_identifier: The identifier of the volume to check.
+
+        Raises:
+            ValueError: If both volume and volume_identifier are provided.
+
+        Returns:
+            True if the volume exists, False otherwise.
+        """
+        if volume and volume_identifier:
+            raise ValueError("Only one of volume or volume_identifier should be provided.")
+        if volume:
+            volume_identifier = volume.identifier
+
+        assert volume_identifier is not None
+
+        if volume_identifier.count(".") != 2:
+            raise ValueError("The identifier must be in the format 'catalog.schema.volume_name'.")
+        catalog, volume_schema, table_name = volume_identifier.split(".")
+        query_result = self._spark.sql(
+            f"""
+                SELECT 1 FROM {catalog}.information_schema.volumes
+                WHERE volume_name = '{table_name}'
+                AND volume_schema = '{volume_schema}'
+                LIMIT 1""",
+        )
+        result = query_result.count() > 0
+        return result is True

cloe_nessy/pipeline/actions/__init__.py

@@ -14,6 +14,7 @@ from .transform_distinct import TransformDistinctAction
 from .transform_filter import TransformFilterAction
 from .transform_generic_sql import TransformSqlAction
 from .transform_group_aggregate import TransformGroupAggregate
+from .transform_hash_columns import TransformHashColumnsAction
 from .transform_join import TransformJoinAction
 from .transform_json_normalize import TransformJsonNormalize
 from .transform_rename_columns import TransformRenameColumnsAction
@@ -51,4 +52,5 @@ __all__ = [
     "TransformRenameColumnsAction",
     "TransformReplaceValuesAction",
     "TransformSelectColumnsAction",
+    "TransformHashColumnsAction",
 ]

cloe_nessy/pipeline/actions/read_api.py

@@ -55,51 +55,75 @@ class ReadAPIAction(PipelineAction):
     DataFrame containing the response data.
 
     Example:
-    ```yaml
-    Read API:
-        action: READ_API
-        options:
-            base_url: https://some_url.com/api/
-            endpoint: my/endpoint/
-            method: GET
-            timeout: 90
-            auth:
-                - type: basic
-                  username: my_username
-                  password: my_password
-                - type: secret_scope
-                  secret_scope: my_secret_scope
-                  header_template:
-                    "header_key_1": "<ENVIRONMENT_VARIABLE_NAME>"
-                - type: secret_scope
-                  secret_scope: my_secret_scope
-                  header_template:
-                    "header_key_2": "<SECRET_NAME>"
-                - type: secret_scope
-                  secret_scope: my_other_secret_scope
-                  header_template:
-                    "header_key_3": "<SECRET_NAME>"
-                - type: azure_oauth
-                  client_id: my_client_id
-                  client_secret: my_client_secret
-                  tenant_id: my_tenant_id
-                  scope: <entra-id-client-id>
-    ```
-
-    The above example will combine the headers from the different auth types. The resulting header will look like this:
-
-    ```json
-    {
-        "header_key_1": "value_from_environment_variable",
-        "header_key_2": "value_from_secret",
-        "header_key_3": "value_from_secret",
-        "Authorization": "Bearer <access_token> (from azure_oauth)",
-        "Authorization": "Basic am9obkBleGFtcGxlLmNvbTphYmMxMjM= (from basic)"
-    }
-    ```
-
-    !!! warning
-
+        === "Basic Usage"
+            ```yaml
+            Read API:
+                action: READ_API
+                options:
+                    base_url: https://some_url.com/api/
+                    endpoint: my/endpoint/
+            ```
+        === "Usage with Parameters and Headers"
+            ```yaml
+            Read API:
+                action: READ_API
+                options:
+                    base_url: https://some_url.com/api/
+                    endpoint: my/endpoint/
+                    method: GET
+                    timeout: 90
+                    headers:
+                        key1: value1
+                        key2: value2
+                    params:
+                        key1: value1
+                        key2: value2
+            ```
+        === "Usage with Authentication"
+            ```yaml
+            Read API:
+                action: READ_API
+                options:
+                    base_url: https://some_url.com/api/
+                    endpoint: my/endpoint/
+                    method: GET
+                    timeout: 90
+                    auth:
+                        - type: basic
+                          username: my_username
+                          password: my_password
+                        - type: secret_scope
+                          secret_scope: my_secret_scope
+                          header_template:
+                            "header_key_1": "<ENVIRONMENT_VARIABLE_NAME>"
+                        - type: secret_scope
+                          secret_scope: my_secret_scope
+                          header_template:
+                            "header_key_2": "<SECRET_NAME>"
+                        - type: secret_scope
+                          secret_scope: my_other_secret_scope
+                          header_template:
+                            "header_key_3": "<SECRET_NAME>"
+                        - type: azure_oauth
+                          client_id: my_client_id
+                          client_secret: my_client_secret
+                          tenant_id: my_tenant_id
+                          scope: <entra-id-client-id>
+            ```
+
+            The above example will combine the headers from the different auth types. The resulting header will look like this:
+
+            ```json
+            {
+                "header_key_1": "value_from_environment_variable",
+                "header_key_2": "value_from_secret",
+                "header_key_3": "value_from_secret",
+                "Authorization": "Bearer <access_token> (from azure_oauth)",
+                "Authorization": "Basic am9obkBleGFtcGxlLmNvbTphYmMxMjM= (from basic)"
+            }
+            ```
+
+    !!! warning "Secret information"
         Don't write sensitive information like passwords or tokens directly in the pipeline configuration.
         Use secret scopes or environment variables instead.
     """

cloe_nessy/pipeline/actions/read_catalog_table.py

@@ -15,13 +15,13 @@ class ReadCatalogTableAction(PipelineAction):
     into a DataFrame and returned as part of an updated `PipelineContext`.
 
     Example:
-    ```yaml
-    Read Sales Table:
-        action: READ_CATALOG_TABLE
-        options:
-            table_identifier: my_catalog.business_schema.sales_table
-            options: <options for the reader>
-    ```
+        ```yaml
+        Read Sales Table:
+            action: READ_CATALOG_TABLE
+            options:
+                table_identifier: my_catalog.business_schema.sales_table
+                options: <options for the CatalogReader read method>
+        ```
     """
 
     name: str = "READ_CATALOG_TABLE"
@@ -43,8 +43,8 @@ class ReadCatalogTableAction(PipelineAction):
                 read. If not provided, the function will attempt to use the table
                 identifier from the `table_metadata` in the `context`.
             options: A dictionary of options for customizing
-                the catalog reader's behavior, such as filters or reading modes. Defaults
-                to None.
+                the [`CatalogReader`][cloe_nessy.integration.reader.catalog_reader]
+                behavior, such as filters or reading modes. Defaults to None.
 
         Raises:
             ValueError: If neither `table_identifier` nor `table_metadata.identifier` in the `context` is provided.

cloe_nessy/pipeline/actions/read_excel.py

@@ -21,16 +21,20 @@ class ReadExcelAction(PipelineAction):
     the read files can be included in the context.
 
     Example:
-    ```yaml
-    Read Excel Table:
-        action: READ_EXCEL
-        options:
-            file: excel_file_folder/excel_files_june/interesting_excel_file.xlsx
-            usecols:
-                - key_column
-                - interesting_column
-            options: <more options for the reader>
-    ```
+        ```yaml
+        Read Excel Table:
+            action: READ_EXCEL
+            options:
+                file: excel_file_folder/excel_files_june/interesting_excel_file.xlsx
+                usecols:
+                    - key_column
+                    - interesting_column
+                options: <options for the ExcelDataFrameReader read method>
+        ```
+
+    !!! note "More Options"
+        The `READ_EXCEL` action supports additional options that can be passed to the
+        run method. For more information, refer to the method documentation.
     """
 
     name: str = "READ_EXCEL"

cloe_nessy/pipeline/actions/read_files.py

@@ -14,14 +14,47 @@ class ReadFilesAction(PipelineAction):
     location will be read using a DataFrameReader with the specified format.
 
     Example:
-    ```yaml
-    Read Excel Table:
-        action: READ_FILES
-        options:
-            location: excel_file_folder/excel_files_june/
-            search_subdirs: True
-            spark_format: AVRO
-    ```
+        === "Read files specified by spark_format"
+            ```yaml
+            Read Files:
+                action: READ_FILES
+                options:
+                    location: json_file_folder/
+                    search_subdirs: True
+                    spark_format: JSON
+            ```
+            !!! note "Define Spark Format"
+                Use the `spark_format` option to specify the format with which
+                to read the files. Supported formats are e.g., `CSV`, `JSON`,
+                `PARQUET`, `TEXT`, and `XML`.
+
+        === "Read files specified by extension"
+            ```yaml
+            Read Files:
+                action: READ_FILES
+                options:
+                    location: csv_file_folder/
+                    search_subdirs: True
+                    extension: csv
+            ```
+            !!! note "Define Extension"
+                Use the `extension` option to specify the extension of the files
+                to read. If not specified, the `spark_format` will be derived from
+                the extension.
+
+        === "Read files with a specified spark_format AND extension"
+            ```yaml
+            Read Files:
+                action: READ_FILES
+                options:
+                    location: file_folder/
+                    extension: abc_custom_extension  # specifies the files to read
+                    spark_format: CSV  # specifies the format to read the files with
+            ```
+            !!! note "Define both Extension & Spark Format"
+                Use the `extension` option to specify the extension of the files
+                to read. Additionally, use the `spark_format` option to specify
+                the format with which to read the files.
     """
 
     name: str = "READ_FILES"
@@ -47,7 +80,8 @@ class ReadFilesAction(PipelineAction):
             search_subdirs: Recursively search subdirectories for files
                 if an extension is provided.
             extension: The file extension to filter files by.
-            spark_format: The format to use for reading the files.
+            spark_format: The format to use for reading the files. If not provided,
+                it will be deferred from the file extension.
             schema: The schema of the data. If None, schema is obtained from
                 the context metadata.
             add_metadata_column: Whether to include the `__metadata` column with
@@ -65,30 +99,22 @@ class ReadFilesAction(PipelineAction):
             raise ValueError("No location provided. Please specify location to read files from.")
         if not options:
             options = dict()
+        if not spark_format and not extension:
+            raise ValueError("Either spark_format or extension must be provided.")
 
         if (metadata := context.table_metadata) and schema is None:
             schema = metadata.schema
 
         file_reader = FileReader()
-        if extension:
-            df = file_reader.read(
-                location=location,
-                schema=schema,
-                extension=extension,
-                search_subdirs=search_subdirs,
-                options=options,
-                add_metadata_column=add_metadata_column,
-            )
-        elif spark_format:
-            df = file_reader.read(
-                location=location,
-                schema=schema,
-                spark_format=spark_format,
-                options=options,
-                add_metadata_column=add_metadata_column,
-            )
-        else:
-            raise ValueError("Please provide either the 'extension' or 'spark_format'")
+        df = file_reader.read(
+            location=location,
+            schema=schema,
+            extension=extension,
+            spark_format=spark_format,
+            search_subdirs=search_subdirs,
+            options=options,
+            add_metadata_column=add_metadata_column,
+        )
 
         runtime_info = context.runtime_info
 

cloe_nessy/pipeline/actions/read_metadata_yaml.py

@@ -10,14 +10,14 @@ class ReadMetadataYAMLAction(PipelineAction):
     """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
 
     Example:
-    ```yaml
-    Read Schema Metadata:
-        action: READ_METADATA_YAML_ACTION
-        options:
-            path: excel_file_folder/excel_files_june/
-            file_name: sales_schema.yml
-            table_name: sales
-    ```
+        ```yaml
+        Read Schema Metadata:
+            action: READ_METADATA_YAML_ACTION
+            options:
+                path: excel_file_folder/excel_files_june/
+                file_name: sales_schema.yml
+                table_name: sales
+        ```
     """
 
     name: str = "READ_METADATA_YAML_ACTION"
@@ -31,7 +31,7 @@ class ReadMetadataYAMLAction(PipelineAction):
         table_name: str | None = None,
         **_: Any,
     ) -> PipelineContext:
-        """Reads schema metadata from a yaml file using the `Schema` model.
+        """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
 
         Args:
             context: The context in which this Action is executed.

cloe_nessy/pipeline/actions/transform_change_datatype.py

@@ -9,15 +9,20 @@ from ..pipeline_context import PipelineContext
 class TransformChangeDatatypeAction(PipelineAction):
     """Changes the datatypes of specified columns in the given DataFrame.
 
+    !!! note "Data Types"
+        We make use of the PySpark `cast` function to change the data types of
+        the columns. Valid data types can be found in the [PySpark
+        documentation](https://spark.apache.org/docs/3.5.3/sql-ref-datatypes.html).
+
     Example:
-    ```yaml
-    Transform Columns:
-        action: TRANSFORM_CHANGE_DATATYPE
-        options:
-            columns:
-                id: string
-                revenue: long
-    ```
+        ```yaml
+        Cast Columns:
+            action: TRANSFORM_CHANGE_DATATYPE
+            options:
+                columns:
+                    id: string
+                    revenue: long
+        ```
     """
 
     name: str = "TRANSFORM_CHANGE_DATATYPE"

cloe_nessy/pipeline/actions/transform_clean_column_names.py

@@ -15,6 +15,10 @@ class TransformCleanColumnNamesAction(PipelineAction):
     Removes invalid characters from the column names, including the fields of a struct and
     replaces a single leading underscore by a double underscore.
 
+    Invalid characters include:
+        - Any non-word character (anything other than letters, digits, and underscores).
+        - A single leading underscore.
+
     Example:
         ```yaml
         Clean Column Names: