cloe-nessy 0.2.9__py3-none-any.whl

cloe_nessy/models/schema.py

@@ -0,0 +1,76 @@
+from pathlib import Path
+from typing import Any, Self
+
+from pydantic import Field
+
+from ..utils.file_and_directory_handler import process_path
+from .mixins.read_instance_mixin import ReadInstancesMixin
+from .table import Table
+from .types import ValidationErrorType
+
+
+class Schema(ReadInstancesMixin):
+    """A Class to represent a Schema in Unity Catalog."""
+
+    catalog: str
+    name: str
+    storage_path: str | None = None
+    tables: list[Table] = Field(default_factory=list)
+    properties: dict[str, Any] = Field(default_factory=dict)
+
+    @classmethod
+    def read_instance_from_file(
+        cls,
+        instance_path: str | Path,
+        fail_on_missing_subfolder: bool = True,
+        table_dir_name: str = "tables",
+        **_: Any,
+    ) -> tuple[Self | None, list[ValidationErrorType]]:
+        """Read a schema from file.
+
+        Adds the table objects from a path relative to the schema definition.
+
+        Args:
+            instance_path: The path to the Schema definition YAML file.
+            fail_on_missing_subfolder: If False return a tuple with 2 empty
+                    lists. Otherwise raise a FileNotFoundError.
+            table_dir_name: The name of the directory containing the Table
+                    definitions related to this schema. Can be a relative path.
+        """
+        processed_instance_path = process_path(instance_path)
+        if not processed_instance_path:
+            raise FileNotFoundError("Schema file not found.")
+
+        schema, schema_errors = super().read_instance_from_file(processed_instance_path)
+        if schema:
+            schema.storage_path = "" if not schema.storage_path else schema.storage_path
+            tables, table_errors = Table.read_instances_from_directory(
+                instance_path=processed_instance_path.parents[0] / table_dir_name,
+                catalog_name=schema.catalog,
+                schema_name=schema.name,
+                schema_storage_path=Path(schema.storage_path),
+                fail_on_missing_subfolder=fail_on_missing_subfolder,
+            )
+            schema.tables = tables
+        return schema, schema_errors + table_errors
+
+    def get_table_by_name(self, table_name: str) -> Table:
+        """Return table in schema.
+
+        Filters tables in schema for table_name and returns Table object.
+
+        Args:
+            table_name: Name of table to return from schema.
+
+        Raises:
+            ValueError: If table not found in schema metadata.
+
+        Returns:
+            The table.
+        """
+        table = next((table for table in self.tables if table.name == table_name), None)
+
+        if not table:
+            raise ValueError(f"Table {table_name} not found in {self.catalog}.{self.name} metadata.")
+
+        return table

cloe_nessy/models/table.py

@@ -0,0 +1,236 @@
+from pathlib import Path
+from typing import Any, Self
+
+import yaml
+import yaml.scanner
+from jinja2 import TemplateNotFound
+from pydantic import Field, ValidationError, ValidationInfo, field_validator, model_validator
+
+from ..logging import LoggerMixin
+from ..utils.file_and_directory_handler import process_path
+from .column import Column
+from .constraint import Constraint
+from .foreign_key import ForeignKey
+from .mixins.read_instance_mixin import ReadInstancesMixin
+from .mixins.template_loader_mixin import TemplateLoaderMixin
+from .types import ValidationErrorType
+
+
+class Table(TemplateLoaderMixin, ReadInstancesMixin, LoggerMixin):
+    """A Class to represent a Table in Unity Catalog."""
+
+    identifier: str
+    columns: list[Column]
+    is_external: bool | None = None
+    partition_by: list[str] = Field(default_factory=list)
+    liquid_clustering: bool | None = None
+    properties: dict[str, str] = Field(default_factory=dict)
+    constraints: list[Constraint] = Field(default_factory=list)
+    foreign_keys: list[ForeignKey] = Field(default_factory=list)
+    storage_path: Path | None = None
+    comment: str | None = None
+
+    def model_post_init(self, __context: Any) -> None:
+        """Post init method for the Table model."""
+        self._console_logger = self.get_console_logger()
+        self._tabular_logger = self.get_tabular_logger(uc_table_name="nessy_table_logs", log_type="nessy_table_logs")
+        self._console_logger.debug(f"Model for table [ '{self.identifier}' ] has been initialized.")
+        self._tabular_logger.debug(f"Message : Model for table [ '{self.identifier}' ] has been initialized.")
+
+    @property
+    def catalog(self):
+        """The name of the Catalog of the Table."""
+        return self.identifier.split(".")[0]
+
+    @property
+    def schema(self):
+        """The name of the Schema of the Table."""
+        return self.identifier.split(".")[1]
+
+    @property
+    def name(self):
+        """The name of the Table."""
+        return self.identifier.split(".")[2]
+
+    @property
+    def escaped_identifier(self):
+        """The escaped identifier of the Table."""
+        return f"`{self.catalog}`.`{self.schema}`.`{self.name}`"
+
+    @field_validator("constraints", mode="before")
+    def _validate_constraints(cls, raw: dict[str, dict[str, str]]) -> list[Constraint]:
+        """The constraints are defined with the name as key or as a list and must therefore be transformed."""
+        if isinstance(raw, dict):
+            constraints = [Constraint(name=constraint, **raw[constraint]) for constraint in raw]
+        elif isinstance(raw, list):
+            constraints = []
+            for item in raw:
+                if isinstance(item, Constraint):
+                    constraints.append(item)
+                elif isinstance(item, dict):
+                    constraints.append(Constraint(**item))
+                else:
+                    raise ValueError("Invalid constraint format")
+        else:
+            raise ValueError("Constraints must be either a list or a dictionary")
+        return constraints
+
+    @field_validator("foreign_keys", mode="after")
+    def _validate_fk_columns(cls, v: list[ForeignKey], values: ValidationInfo):
+        """Foreign keys need to be columns in the table as well."""
+        column_names = [c.name for c in values.data.get("columns", [])]
+        for fk in v:
+            for column in fk.foreign_key_columns:
+                if column not in column_names:
+                    raise ValueError(f"Foreign key column '{column}' does not match any column in 'columns'")
+        return v
+
+    @model_validator(mode="after")
+    def _validate_is_external(cls, table: Self):
+        """If is_external is set to False, storage_path has to be None."""
+        if not table.is_external and table.storage_path is not None:
+            raise ValueError("is_external cannot be false while storage_path is set.")
+        elif table.is_external and table.storage_path is None:
+            raise ValueError("is_external cannot be true while storage_path is None.")
+
+    @classmethod
+    def read_instances_from_directory(
+        cls,
+        instance_path: str | Path,
+        fail_on_missing_subfolder: bool = True,
+        catalog_name: str | None = None,
+        schema_name: str | None = None,
+        schema_storage_path: str | Path | None = None,
+        **_: Any,
+    ) -> tuple[list[Self], list[ValidationErrorType]]:
+        """Reads instances from a directory containing YAML files.
+
+        This method scans a specified directory for YAML files (.yaml or .yml),
+        attempts to read and parse each file as an instance of the class, and
+        collects any errors encountered during the process. If the directory
+        does not exist or is not a directory, it either raises a
+        FileNotFoundError (if fail_on_missing_subfolder is True) or returns
+        empty lists.
+
+        Args:
+            instance_path: The path to the directory containing instance files.
+            catalog_name: Name of the catalog to which these instances belong.
+            schema_name: Name of the schema used for validating the instances.
+            fail_on_missing_subfolder: Determines behavior when the specified
+                    directory does not exist or is not a directory. Defaults to True,
+                    which will raise a FileNotFoundError.
+            schema_storage_path: Path to the storage location of the schema
+                    these tables instances belong to.
+
+        Returns:
+            - The first list contains instances of the class that were
+                successfully read and validated from the files.
+            - The second list contains errors encountered during the
+                process, which could be validation errors or YAML
+                parsing/scanning errors.
+
+        Raises:
+            FileNotFoundError: If the specified directory does not exist or is
+                               not a directory and fail_on_missing_subfolder is True.
+            ValueError: If catalog_name or schema_name are not provided.
+        """
+        processed_instance_path = process_path(instance_path)
+        schema_storage_path = process_path(schema_storage_path)
+        errors: list[ValidationErrorType] = []
+
+        if not catalog_name or not schema_name:
+            errors.append(ValueError("catalog_name and schema_name must be provided."))
+            return [], errors
+        instances: list[Self] = []
+
+        if not processed_instance_path or not processed_instance_path.exists() or not processed_instance_path.is_dir():
+            if fail_on_missing_subfolder:
+                raise FileNotFoundError(f"Directory not found: {processed_instance_path}")
+            else:
+                return instances, errors
+
+        for instance_file in processed_instance_path.iterdir():
+            sub_errors: list[ValidationErrorType] = []
+            if instance_file.is_file() and instance_file.suffix in (".yaml", ".yml"):
+                instance, sub_errors = cls.read_instance_from_file(
+                    instance_file, catalog_name, schema_name, schema_storage_path
+                )
+                instances += [] if instance is None else [instance]
+            errors += sub_errors
+
+        return instances, errors
+
+    @classmethod
+    def read_instance_from_file(
+        cls,
+        instance_path: str | Path,
+        catalog_name: str | None = None,
+        schema_name: str | None = None,
+        schema_storage_path: str | Path | None = None,
+        **_: Any,
+    ) -> tuple[Self | None, list[ValidationErrorType]]:
+        """Read a table instance from file.
+
+        Args:
+            instance_path: The path to the Schema definition YAML file.
+            catalog_name: The name of the Catalog of the Table.
+            schema_name: The name of the Schema of the Table.
+            schema_storage_path: The storage path location of the Schema of the Table.
+
+        Returns:
+            - Instance of the class that was successfully instantiated and
+                  validated
+            - The second list contains errors encountered during the process,
+                  which could be validation errors or YAML parsing/scanning errors.
+
+        Raises:
+            FileNotFoundError: If the specified directory does not exist or is
+                not a directory and fail_on_missing_subfolder is
+                True.
+            ValueError: If catalog_name or schema_name are not provided.
+        """
+        processed_instance_path = process_path(instance_path)
+        if not processed_instance_path:
+            raise FileNotFoundError("Table file not found.")
+        schema_storage_path = process_path(schema_storage_path)
+        errors: list[ValidationErrorType] = []
+
+        if not catalog_name or not schema_name:
+            errors.append(ValueError("catalog_name and schema_name must be provided."))
+            return None, errors
+
+        try:
+            with processed_instance_path.open("r") as file:
+                data = yaml.safe_load(file)
+                data["identifier"] = f"{catalog_name}.{schema_name}.{data['name']}"
+                if data.get("is_external"):
+                    if storage_path := data.get("storage_path"):
+                        data["storage_path"] = Path(storage_path)
+                    elif schema_storage_path:
+                        data["storage_path"] = schema_storage_path / data["name"]
+                    else:
+                        raise ValueError(
+                            f"Neither storage path nor schema storage path of table {data['name']} has been provided."
+                        )
+
+                instance, sub_errors = cls.metadata_to_instance(data)
+                errors += sub_errors
+        except (ValidationError, yaml.parser.ParserError, yaml.scanner.ScannerError) as e:
+            instance = None
+            errors.append(e)
+        return instance, errors
+
+    def get_create_statement(
+        self,
+        templates: Path = Path("./templates"),
+        template_name: str = "create_table.sql.j2",
+        replace: bool = True,
+    ):
+        """Get the create statement for the Table."""
+        try:
+            template = self.get_template(templates, template_name)
+        except TemplateNotFound as err:
+            self._console_logger.error(f"Template [ {template_name} ] not found.")
+            raise err
+        render = template.render(table=self, replace=replace)
+        return render

cloe_nessy/models/types.py

@@ -0,0 +1,7 @@
+from typing import TypeAlias
+
+import yaml.parser
+import yaml.scanner
+from pydantic import ValidationError
+
+ValidationErrorType: TypeAlias = ValidationError | yaml.parser.ParserError | yaml.scanner.ScannerError | ValueError

cloe_nessy/object_manager/__init__.py

@@ -0,0 +1,3 @@
+from .table_manager import TableManager
+
+__all__ = ["TableManager"]

cloe_nessy/object_manager/table_manager.py

@@ -0,0 +1,58 @@
+from ..logging import LoggerMixin
+from ..session import SessionManager
+
+
+class TableManager(LoggerMixin):
+    """TableManager class for managing tables in the catalog."""
+
+    def __init__(self):
+        self._spark = SessionManager.get_spark_session()
+        self._utils = SessionManager.get_utils()
+        self._console_logger = self.get_console_logger()
+        self._console_logger.debug("TableManager initialized...")
+        self._tabular_logger = self.get_tabular_logger(uc_table_name="TableManager")
+        self._tabular_logger.debug("message:TableManager initialized.")
+
+    @staticmethod
+    def create_table():
+        """Create a table in the catalog."""
+        raise NotImplementedError
+
+    def drop_table(self, table_identifier: str, delete_physical_data: bool = False):
+        """Deletes a Table. For security reasons you are forced to pass the table_name.
+
+        If delete_physical_data is True the actual physical data on the ADLS will be deleted.
+        Use with caution!
+
+        Args:
+            table_identifier: The table identifier in the catalog. Must be in the format 'catalog.schema.table'.
+            delete_physical_data: If set to True, deletes not only the metadata
+                                  within the Catalog but also the physical data.
+        """
+        self._console_logger.info(f"Deleting table [ '{table_identifier}' ] ...")
+        if not isinstance(table_identifier, str):
+            raise NotImplementedError("table_identifier must be a string, can be a Table object in the future.")
+
+        if delete_physical_data:
+            self._delete_physical_data()
+        self.drop_table_from_catalog(table_identifier)
+
+    def drop_table_from_catalog(self, table_identifier: str) -> None:
+        """Removes a table from the catalog. Physical data is retained.
+
+        Args:
+            table_identifier: The table identifier in the catalog. Must be in the format 'catalog.schema.table'.
+        """
+        self._console_logger.info(f"... deleting table [ '{table_identifier}' ] from Catalog.")
+        if not isinstance(table_identifier, str):
+            raise NotImplementedError("table_identifier must be a string, can be a Table object in the future.")
+        self._spark.sql(f"DROP TABLE IF EXISTS {table_identifier};")
+
+    def _delete_physical_data(self):
+        """Removes the physical data on the ADLS for the location of this table.
+
+        Raises:
+            NotImplementedError: This can be implemented, once a Table object is available.
+        """
+        self._console_logger.info("... deleting physical data for table [ '' ] from Catalog.")
+        raise NotImplementedError("This can be implemented, once a Table object is available.")

cloe_nessy/pipeline/__init__.py

@@ -0,0 +1,7 @@
+from .pipeline import Pipeline
+from .pipeline_action import PipelineAction
+from .pipeline_context import PipelineContext
+from .pipeline_parsing_service import PipelineParsingService
+from .pipeline_step import PipelineStep
+
+__all__ = ["Pipeline", "PipelineParsingService", "PipelineContext", "PipelineAction", "PipelineStep"]

cloe_nessy/pipeline/actions/__init__.py

@@ -0,0 +1,50 @@
+from enum import Enum
+
+from ..pipeline_action import PipelineAction
+from .read_api import ReadAPIAction
+from .read_catalog_table import ReadCatalogTableAction
+from .read_excel import ReadExcelAction
+from .read_files import ReadFilesAction
+from .read_metadata_yaml import ReadMetadataYAMLAction
+from .transform_change_datatype import TransformChangeDatatypeAction
+from .transform_concat_columns import TransformConcatColumnsAction
+from .transform_decode import TransformDecodeAction
+from .transform_distinct import TransformDistinctAction
+from .transform_filter import TransformFilterAction
+from .transform_generic_sql import TransformSqlAction
+from .transform_join import TransformJoinAction
+from .transform_json_normalize import TransformJsonNormalize
+from .transform_rename_columns import TransformRenameColumnsAction
+from .transform_replace_values import TransformReplaceValuesAction
+from .transform_select_columns import TransformSelectColumnsAction
+from .transform_union import TransformUnionAction
+from .write_catalog_table import WriteCatalogTableAction
+
+# Get all subclasses of PipelineAction defined in this submodule
+pipeline_actions = {cls.name: cls for cls in PipelineAction.__subclasses__()}
+# Register all subclasses dynamically as enum using their "name" attribute as
+# key. We need to do this here, because otherwise we don't get all subclasses
+# from a relative import of PipelineAction
+PipelineActionType = Enum("PipelineActionType", pipeline_actions)  # type: ignore
+
+__all__ = [
+    "ReadAPIAction",
+    "ReadCatalogTableAction",
+    "ReadExcelAction",
+    "ReadFilesAction",
+    "ReadMetadataYAMLAction",
+    "WriteCatalogTableAction",
+    "PipelineActionType",
+    "TransformFilterAction",
+    "TransformUnionAction",
+    "TransformChangeDatatypeAction",
+    "TransformConcatColumnsAction",
+    "TransformDecodeAction",
+    "TransformDistinctAction",
+    "TransformSqlAction",
+    "TransformJoinAction",
+    "TransformJsonNormalize",
+    "TransformRenameColumnsAction",
+    "TransformReplaceValuesAction",
+    "TransformSelectColumnsAction",
+]

cloe_nessy/pipeline/actions/read_api.py

@@ -0,0 +1,178 @@
+from collections.abc import Mapping
+from typing import Any, cast
+
+from requests.auth import AuthBase, HTTPBasicAuth
+
+from ...clients.api_client.auth import AzureCredentialAuth, ChainedAuth, EnvVariableAuth, SecretScopeAuth
+from ...integration.reader import APIReader
+from ..pipeline_action import PipelineAction
+from ..pipeline_context import PipelineContext
+
+
+def process_auth(
+    auth: Mapping[str, str | Mapping[str, str] | list[Mapping[str, str]]] | AuthBase | None,
+) -> AuthBase | None:
+    """Processes the auth parameter to create an AuthBase object.
+
+    Args:
+        auth: The auth parameter to be processed.
+    """
+    result: AuthBase | None = None
+
+    if isinstance(auth, list):
+        auths = [process_auth(sub_auth) for sub_auth in auth]
+        result = ChainedAuth(*auths)
+    elif isinstance(auth, dict):
+        match auth.get("type"):
+            case "basic":
+                result = HTTPBasicAuth(auth["username"], auth["password"])
+            case "secret_scope":
+                secret_scope_header_template: dict[str, str] = auth["header_template"]
+                result = SecretScopeAuth(secret_scope_header_template, auth["secret_scope"])
+            case "env":
+                env_header_template: dict[str, str] = auth["header_template"]
+                result = EnvVariableAuth(env_header_template)
+            case "azure_oauth":
+                result = AzureCredentialAuth(
+                    scope=auth["scope"],
+                    client_id=auth["client_id"],
+                    client_secret=auth["client_secret"],
+                    tenant_id=auth["tenant_id"],
+                )
+            case _:
+                raise ValueError("Invalid auth type specified. Supported types are: basic, secret_scope, env")
+    else:
+        result = cast(AuthBase, auth)
+
+    return result
+
+
+class ReadAPIAction(PipelineAction):
+    """Reads data from an API and loads it into a Spark DataFrame.
+
+    This method uses the provided API parameters to make a request using the
+    [`APIReader`][cloe_nessy.integration.reader.api_reader] and return a
+    DataFrame containing the response data.
+
+    Example:
+    ```yaml
+    Read API:
+        action: READ_API
+        options:
+            base_url: https://some_url.com/api/
+            endpoint: my/endpoint/
+            method: GET
+            timeout: 90
+            auth:
+                - type: basic
+                  username: my_username
+                  password: my_password
+                - type: secret_scope
+                  secret_scope: my_secret_scope
+                  header_template:
+                    "header_key_1": "<ENVIRONMENT_VARIABLE_NAME>"
+                - type: secret_scope
+                  secret_scope: my_secret_scope
+                  header_template:
+                    "header_key_2": "<SECRET_NAME>"
+                - type: secret_scope
+                  secret_scope: my_other_secret_scope
+                  header_template:
+                    "header_key_3": "<SECRET_NAME>"
+                - type: azure_oauth
+                  client_id: my_client_id
+                  client_secret: my_client_secret
+                  tenant_id: my_tenant_id
+                  scope: <entra-id-client-id>
+    ```
+
+    The above example will combine the headers from the different auth types. The resulting header will look like this:
+
+    ```json
+    {
+        "header_key_1": "value_from_environment_variable",
+        "header_key_2": "value_from_secret",
+        "header_key_3": "value_from_secret",
+        "Authorization": "Bearer <access_token> (from azure_oauth)",
+        "Authorization": "Basic am9obkBleGFtcGxlLmNvbTphYmMxMjM= (from basic)"
+    }
+    ```
+
+    !!! warning
+
+        Don't write sensitive information like passwords or tokens directly in the pipeline configuration.
+        Use secret scopes or environment variables instead.
+    """
+
+    name: str = "READ_API"
+
+    @staticmethod
+    def run(
+        context: PipelineContext,
+        *,
+        base_url: str | None = None,
+        auth: AuthBase | dict[str, str] | None = None,
+        default_headers: dict[str, str] | None = None,
+        endpoint: str = "",  # www.neo4j.de/api/table/2020/01/01
+        method: str = "GET",
+        key: str | None = None,
+        timeout: int = 30,
+        params: dict[str, str] | None = None,
+        headers: dict[str, str] | None = None,
+        data: dict[str, str] | None = None,
+        json: dict[str, str] | None = None,
+        max_retries: int = 0,
+        options: dict[str, str] | None = None,
+        **_: Any,
+    ) -> PipelineContext:
+        """Utility class for reading an API into a DataFrame.
+
+        This class uses an APIClient to fetch data from an API and load it into a Spark DataFrame.
+
+
+        Args:
+            context: The pipeline context containing information about the pipeline.
+            base_url: The base URL for the API to be called.
+            auth: The authentication credentials for the API.
+            default_headers: Default headers to include in the API request.
+            endpoint: The specific API endpoint to call.
+            method: The HTTP method to use for the request (default is "GET").
+            key: Key for accessing specific data in the response.
+            timeout: Timeout for the API request in seconds (default is 30).
+            params: URL parameters to include in the API request.
+            headers: Additional headers to include in the request.
+            data: Data to send with the request for POST methods.
+            json: JSON data to send with the request for POST methods.
+            max_retries: Maximum number of retries for the API request (default is 0).
+            options: Additional options for the API request.
+
+        Returns:
+            The updated pipeline context containing the DataFrame with the API response data.
+
+        Raises:
+            ValueError: If the base_url is not specified.
+        """
+        if not options:
+            options = dict()
+
+        if base_url is None:
+            raise ValueError("base_url must be specified to fetch data from API.")
+
+        deserialized_auth = process_auth(auth)
+
+        api_reader = APIReader(base_url=base_url, auth=deserialized_auth, default_headers=default_headers)
+
+        df = api_reader.read(
+            method=method,
+            endpoint=endpoint,
+            timeout=timeout,
+            params=params,
+            key=key,
+            headers=headers,
+            data=data,
+            json=json,
+            max_retries=max_retries,
+            options=options,
+        )
+
+        return context.from_existing(data=df)