tgedr-dataops-ext 0.0.1__py3-none-any.whl

tgedr_dataops_ext/commons/dataset.py

@@ -0,0 +1,27 @@
+"""Dataset module for wrapping dataframes with metadata.
+
+This module provides:
+- Dataset: an immutable dataclass that combines a DataFrame with Metadata.
+"""
+
+from dataclasses import dataclass
+import json
+from pyspark.sql import DataFrame
+from tgedr_dataops_ext.commons.metadata import Metadata
+
+
+@dataclass(frozen=True)
+class Dataset:
+    """Utility immutable class to wrap up a dataframe along with metadata."""
+
+    __slots__ = ["data", "metadata"]
+    metadata: Metadata
+    data: DataFrame
+
+    def as_dict(self) -> dict:
+        """Serialize the dataset as a dictionary."""
+        return {"metadata": self.metadata.as_dict(), "data": str(self.data.__repr__)}
+
+    def __str__(self) -> str:
+        """Serialize the dataset as a json string."""
+        return json.dumps(self.as_dict())

tgedr_dataops_ext/commons/metadata.py

@@ -0,0 +1,236 @@
+"""Module for dataset metadata classes."""
+
+from dataclasses import dataclass
+import json
+
+
+@dataclass(frozen=True)
+class FieldFrame:
+    """Class depicting a field values range, to be used in metadata.
+
+    Parameters
+    ----------
+    field : str
+        The name of the field.
+    lower : Union[int, str, float]
+        Field lower bound.
+    upper : Union[int, str, float]
+        Field upper bound.
+
+    """
+
+    __slots__ = ["field", "lower", "upper"]
+    field: str
+    lower: int | str | float
+    upper: int | str | float
+
+    def as_dict(self) -> dict[str, int | str | float]:
+        """Convert the FieldFrame to a dictionary representation.
+
+        Returns
+        -------
+        dict[str, int | str | float]
+            A dictionary containing the field name, lower bound, and upper bound.
+        """
+        return {"field": self.field, "lower": self.lower, "upper": self.upper}
+
+    @staticmethod
+    def from_str(src: str) -> "FieldFrame":
+        """Create a FieldFrame instance from a JSON string.
+
+        Parameters
+        ----------
+        src : str
+            A JSON string representation of a FieldFrame.
+
+        Returns
+        -------
+        FieldFrame
+            A new FieldFrame instance created from the JSON string.
+        """
+        r = json.loads(src)
+        field = r["field"]
+        lower = r["lower"]
+        upper = r["upper"]
+        return FieldFrame(field=field, lower=lower, upper=upper)
+
+    def __str__(self) -> str:
+        """Return a JSON string representation of the FieldFrame."""
+        return json.dumps(self.as_dict())
+
+    def __eq__(self, other: "FieldFrame") -> bool:
+        """Check equality between two FieldFrame instances."""
+        return self.field == other.field and self.lower == other.lower and self.upper == other.upper
+
+    def __gt__(self, other: "FieldFrame") -> bool:
+        """Check if this FieldFrame is greater than another FieldFrame."""
+        return self.field > other.field or (
+            self.field == other.field
+            and (self.lower > other.lower or (self.lower == other.lower and self.upper > other.upper))
+        )
+
+    def __ne__(self, other: "FieldFrame") -> bool:
+        """Check inequality between two FieldFrame instances."""
+        return not other == self
+
+    def __ge__(self, other: "FieldFrame") -> bool:
+        """Check if this FieldFrame is greater than or equal to another FieldFrame."""
+        return other == self or self > other
+
+    def __le__(self, other: "FieldFrame") -> bool:
+        """Check if this FieldFrame is less than or equal to another FieldFrame."""
+        return other == self or self < other
+
+    def __lt__(self, other: "FieldFrame") -> bool:
+        """Check if this FieldFrame is less than another FieldFrame."""
+        return other > self
+
+
+@dataclass(frozen=True)
+class Metadata:
+    """Immutable class depicting dataset metadata.
+
+    Parameters
+    ----------
+    name : str
+        The name of the dataset.
+    version : Optional[str]
+        Version of this dataset, if available.
+    framing : Optional[List[FieldFrame]]
+        Multiple field frames.
+    sources : Optional[List["Metadata"]]
+        Metadatas related to the datasets sourcing this one.
+
+    """
+
+    __slots__ = ["framing", "name", "sources", "version"]
+    name: str
+    version: str | None
+    framing: list[FieldFrame] | None
+    sources: list["Metadata"] | None
+
+    def as_dict(self) -> dict:
+        """Convert the Metadata to a dictionary representation.
+
+        Returns
+        -------
+        dict
+            A dictionary containing the metadata fields including name, version,
+            framing, and sources if they are not None.
+        """
+        result = {"name": self.name}
+        if self.version is not None:
+            result["version"] = self.version
+        if self.framing is not None:
+            result["framing"] = []
+            for f in self.framing:
+                (result["framing"]).append(f.as_dict())
+        if self.sources is not None:
+            result["sources"] = []
+            for source in self.sources:
+                (result["sources"]).append(source.as_dict())
+
+        return result
+
+    def __str__(self) -> str:
+        """Return a JSON string representation of the Metadata."""
+        return json.dumps(self.as_dict())
+
+    def __eq__(self, other: object) -> bool:
+        """Check equality between two Metadata instances."""
+        return (
+            self.name == other.name
+            and (
+                (self.version is None and other.version is None)
+                or ((self.version is not None and other.version is not None) and self.version == other.version)
+            )
+            and (
+                (self.framing is None and other.framing is None)
+                or (
+                    (self.framing is not None and other.framing is not None)
+                    and sorted(self.framing) == sorted(other.framing)
+                )
+            )
+            and (
+                (self.sources is None and other.sources is None)
+                or (
+                    (self.sources is not None and other.sources is not None)
+                    and sorted(self.sources) == sorted(other.sources)
+                )
+            )
+        )
+
+    def __gt__(self, other: "Metadata") -> bool:
+        """Check if this Metadata is greater than another Metadata."""
+        return self.name > other.name or (
+            self.name == other.name
+            and (
+                ((self.version is not None and other.version is not None) and (self.version > other.version))
+                or (self.version is not None and other.version is None)
+                or (
+                    (
+                        (self.framing is not None and other.framing is not None)
+                        and (sorted(self.framing) > sorted(other.framing))
+                    )
+                    or (self.framing is not None and other.framing is None)
+                    or (
+                        (
+                            (self.sources is not None and other.sources is not None)
+                            and (sorted(self.sources) > sorted(other.sources))
+                        )
+                        or (self.sources is not None and other.sources is None)
+                    )
+                )
+            )
+        )
+
+    def __ne__(self, other: "Metadata") -> bool:
+        """Check inequality between two Metadata instances."""
+        return not other == self
+
+    def __ge__(self, other: "Metadata") -> bool:
+        """Check if this Metadata is greater than or equal to another Metadata."""
+        return other == self or self > other
+
+    def __le__(self, other: "Metadata") -> bool:
+        """Check if this Metadata is less than or equal to another Metadata."""
+        return other == self or self < other
+
+    def __lt__(self, other: "Metadata") -> bool:
+        """Check if this Metadata is less than another Metadata."""
+        return other > self
+
+    @staticmethod
+    def from_str(src: str) -> "Metadata":
+        """Create a Metadata instance from a JSON string.
+
+        Parameters
+        ----------
+        src : str
+            A JSON string representation of a Metadata object.
+
+        Returns
+        -------
+        Metadata
+            A new Metadata instance created from the JSON string.
+        """
+        r = json.loads(src)
+        name = r["name"]
+        version = r.get("version", None)
+
+        framing = None
+        framing_entries = r.get("framing", None)
+        if framing_entries is not None:
+            framing = []
+            for framing_entry in framing_entries:
+                framing.append(FieldFrame.from_str(json.dumps(framing_entry)))
+
+        sources = None
+        sources_entries = r.get("sources", None)
+        if sources_entries is not None:
+            sources = []
+            for source_entry in sources_entries:
+                source_entry_as_str = json.dumps(source_entry)
+                sources.append(Metadata.from_str(source_entry_as_str))
+
+        return Metadata(name=name, version=version, framing=framing, sources=sources)

tgedr_dataops_ext/commons/utils_spark.py

@@ -0,0 +1,133 @@
+"""Spark utility functions and classes.
+
+This module provides:
+- UtilsSpark: A utility class with handy functions to work with Spark sessions,
+  including creating local and remote sessions with Delta Lake support,
+  and building PySpark schemas from data type dictionaries.
+"""
+
+import logging
+import os
+from typing import ClassVar
+from pyspark.sql import SparkSession
+from pyspark import SparkConf
+from pyspark.sql import types as T  # noqa: N812
+from pyspark.context import SparkContext
+
+
+logger = logging.getLogger(__name__)
+
+
+class UtilsSpark:
+    """class with handy functions to work with spark."""
+
+    __ENV_KEY_PYSPARK_IS_LOCAL = "PYSPARK_IS_LOCAL"
+    __ENV_KEY_NOT_AWS_CLOUD = "NOT_AWS_CLOUD"
+    __DTYPES_MAP: ClassVar[dict[str, type]] = {
+        "bigint": T.LongType,
+        "string": T.StringType,
+        "double": T.DoubleType,
+        "int": T.IntegerType,
+        "boolean": T.BooleanType,
+        "timestamp": T.TimestampType,
+        "date": T.DateType,
+    }
+
+    @staticmethod
+    def get_local_spark_session(config: dict | None = None) -> SparkSession:
+        """Get a local Spark session configured for Delta Lake.
+
+        Parameters
+        ----------
+        config : dict | None, optional
+            Additional Spark configuration parameters, by default None.
+
+        Returns
+        -------
+        SparkSession
+            A configured local Spark session instance with Delta Lake support.
+        """
+        logger.debug(f"[get_local_spark_session|in] ({config})")
+        # PySpark 3.4 uses Scala 2.12, so we need delta-core_2.12
+        builder = (
+            SparkSession.builder.config("spark.jars.packages", "io.delta:delta-core_2.12:2.4.0")
+            .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+            .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+            .config("spark.driver.host", "localhost")
+        )
+
+        if config is not None:
+            for k, v in config.items():
+                builder.config(k, v)
+
+        spark = builder.getOrCreate()
+
+        logger.debug(f"[get_local_spark_session|out] => {spark}")
+        return spark
+
+    @staticmethod
+    def get_spark_session(config: dict | None = None) -> SparkSession:
+        """Get a Spark session based on the environment configuration.
+
+        Parameters
+        ----------
+        config : dict | None, optional
+            Additional Spark configuration parameters, by default None.
+
+        Returns
+        -------
+        SparkSession
+            A configured Spark session instance.
+        """
+        logger.debug(f"[get_spark_session|in] ({config})")
+
+        if "1" == os.getenv(UtilsSpark.__ENV_KEY_PYSPARK_IS_LOCAL):
+            spark: SparkSession = UtilsSpark.get_local_spark_session(config)
+        else:
+            if "1" == os.getenv(UtilsSpark.__ENV_KEY_NOT_AWS_CLOUD):
+                active_session = SparkSession.getActiveSession()
+            else:
+                from awsglue.context import GlueContext  # type: ignore #   pragma: no cover  # noqa: PGH003
+
+                glueContext = GlueContext(SparkContext.getOrCreate())  #   pragma: no cover  # noqa: N806
+                active_session = glueContext.spark_session  #   pragma: no cover
+
+            spark_config = SparkConf()
+
+            if active_session is not None:
+                former_config = active_session.sparkContext.getConf().getAll()
+                for entry in former_config:
+                    spark_config.set(entry[0], entry[1])
+            if config is not None:
+                for k, v in config.items():
+                    spark_config.set(k, v)
+                spark: SparkSession = SparkSession.builder.config(conf=spark_config).getOrCreate()
+            else:
+                spark: SparkSession = SparkSession.builder.getOrCreate()
+
+        logger.debug(f"[get_spark_session|out] => {spark}")
+        return spark
+
+    @staticmethod
+    def build_schema_from_dtypes(dtypes_schema: dict[str, str]) -> T.StructType:
+        """Build a PySpark StructType schema from a dictionary of data types.
+
+        Parameters
+        ----------
+        dtypes_schema : dict[str, str]
+            A dictionary mapping field names to their corresponding data type strings.
+            Supported types: 'bigint', 'string', 'double', 'int', 'boolean', 'timestamp', 'date'.
+
+        Returns
+        -------
+        T.StructType
+            A PySpark StructType schema with fields defined by the input dictionary.
+        """
+        logger.info(f"[build_schema_from_dtypes|in] ({dtypes_schema})")
+        result = T.StructType()
+        for field, dtype in dtypes_schema.items():
+            new_type = UtilsSpark.__DTYPES_MAP[dtype]
+            result.add(field, new_type(), True)  # noqa: FBT003
+
+        logger.info(f"[build_schema_from_dtypes|out] => {result}")
+        return result

tgedr_dataops_ext/quality/pyspark_validation.py

@@ -0,0 +1,22 @@
+"""Pyspark DataFrame validation implementation module.
+
+This module provides the Pyspark-specific implementation of Great Expectations validation.
+"""
+
+from great_expectations.execution_engine import ExecutionEngine
+from great_expectations.execution_engine import SparkDFExecutionEngine
+from tgedr_dataops_abs.great_expectations_validation import GreatExpectationsValidation
+
+
+class PysparkValidation(GreatExpectationsValidation):
+    """Pyspark DataFrame validation implementation."""
+
+    def _get_execution_engine(self, batch_data_dict: dict) -> ExecutionEngine:
+        """Get the execution engine used by the validation implementation.
+
+        Returns
+        -------
+        ExecutionEngine
+            The execution engine instance.
+        """
+        return SparkDFExecutionEngine(batch_data_dict=batch_data_dict)

tgedr_dataops_ext/source/delta_table_source.py

@@ -0,0 +1,57 @@
+"""Delta Lake table source implementation.
+
+This module provides:
+- DeltaTableSource: abstract base class for reading delta lake format datasets
+  and returning pandas DataFrames with configurable storage options.
+"""
+
+from abc import ABC, abstractmethod
+import logging
+from typing import Any
+from pandas import DataFrame
+from deltalake import DeltaTable
+from deltalake.exceptions import TableNotFoundError
+
+from tgedr_dataops_abs.source import Source, SourceException, NoSourceException
+
+
+logger = logging.getLogger()
+
+
+class DeltaTableSource(Source, ABC):
+    """abstract class used to read delta lake format datasets returning a pandas dataframe."""
+
+    CONTEXT_KEY_URL: str = "url"
+    CONTEXT_KEY_COLUMNS: str = "columns"
+
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize the DeltaTableSource with optional configuration."""
+        super().__init__(config=config)
+
+    @property
+    @abstractmethod
+    def _storage_options(self) -> Any:
+        return None  # pragma: no cover
+
+    def get(self, context: dict[str, Any] | None = None) -> DataFrame:
+        """Retrieves a delta lake table."""
+        logger.info(f"[get|in] ({context})")
+        result: DataFrame = None
+
+        if self.CONTEXT_KEY_URL not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
+
+        columns: list[str] = None
+        if self.CONTEXT_KEY_COLUMNS in context:
+            columns = context[self.CONTEXT_KEY_COLUMNS]
+
+        try:
+            delta_table = DeltaTable(
+                table_uri=context[self.CONTEXT_KEY_URL], storage_options=self._storage_options, without_files=True
+            )
+            result = delta_table.to_pandas(columns=columns)
+        except TableNotFoundError as exc:
+            raise NoSourceException(f"could not find delta table: {context[self.CONTEXT_KEY_URL]}") from exc
+
+        logger.info(f"[get|out] => {result}")
+        return result

tgedr_dataops_ext/source/local_delta_table.py

@@ -0,0 +1,58 @@
+"""Local Delta Table source implementation.
+
+This module provides the LocalDeltaTable class for reading Delta Lake format datasets
+from the local filesystem using Python only (PySpark not required), returning pandas DataFrames.
+"""
+
+import logging
+import re
+from typing import Any
+import glob
+from pathlib import Path
+
+from tgedr_dataops_ext.source.delta_table_source import DeltaTableSource
+from tgedr_dataops_abs.source import SourceException
+
+
+logger = logging.getLogger()
+
+
+class LocalDeltaTable(DeltaTableSource):
+    """class used to read delta lake format datasets from local fs with python only, pyspark not needed, returning a pandas dataframe."""
+
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize LocalDeltaTable with optional configuration.
+
+        Args:
+            config: Optional configuration dictionary for the delta table source.
+        """
+        super().__init__(config=config)
+
+    @property
+    def _storage_options(self) -> Any:
+        return None
+
+    def list(self, context: dict[str, Any] | None = None) -> list[str]:
+        """Lists the available delta lake datasets in the url provided."""
+        logger.info(f"[list|in] ({context})")
+
+        result: list[str] = []
+        if self.CONTEXT_KEY_URL not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
+
+        url = context[self.CONTEXT_KEY_URL]
+        if not Path(url).is_dir():
+            raise SourceException(f"not a delta lake url: {url}")
+
+        matches: set[str] = set()
+        pattern: str = f".*{url}/(.*)/_delta_log/.*"
+        for entry in glob.iglob(url + "**/**", recursive=True):  # noqa: PTH207
+            match = re.search(pattern, entry)
+            if match:
+                matches.add(match.group(1))
+
+        result = list(matches)
+
+        logger.info(f"[list] result: {result}")
+        logger.info(f"[list|out] => result len: {len(result)}")
+        return result

tgedr_dataops_ext/source/s3_delta_table.py

@@ -0,0 +1,83 @@
+"""S3 Delta Table source module for reading Delta Lake format datasets from S3.
+
+This module provides the S3DeltaTable class for reading Delta Lake format datasets
+from S3 buckets using Python only (no PySpark required), returning pandas DataFrames.
+"""
+
+import logging
+import re
+from typing import Any
+
+from tgedr_dataops.commons.s3_connector import S3Connector
+from tgedr_dataops.commons.utils_fs import remove_s3_protocol
+from tgedr_dataops_ext.source.delta_table_source import DeltaTableSource
+from tgedr_dataops_abs.source import SourceException
+
+
+logger = logging.getLogger()
+
+
+class S3DeltaTable(DeltaTableSource, S3Connector):
+    """class used to read delta lake format datasets from s3 bucket with python only, pyspark not needed, returning a pandas dataframe."""
+
+    CONFIG_KEY_AWS_ACCESS_KEY_ID: str = "AWS_ACCESS_KEY_ID"
+    CONFIG_KEY_AWS_SECRET_ACCESS_KEY: str = "AWS_SECRET_ACCESS_KEY"  # noqa: S105
+    CONFIG_KEY_AWS_SESSION_TOKEN: str = "AWS_SESSION_TOKEN"  # noqa: S105
+    CONFIG_KEY_AWS_REGION: str = "AWS_REGION"
+
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize the S3DeltaTable with optional configuration.
+
+        Args:
+            config: Optional dictionary containing AWS credentials and configuration.
+        """
+        DeltaTableSource.__init__(self, config=config)
+        S3Connector.__init__(self)
+
+    @property
+    def _storage_options(self) -> Any:
+        result = None
+        if (self._config is not None) and all(
+            element in list(self._config.keys())
+            for element in [
+                self.CONFIG_KEY_AWS_ACCESS_KEY_ID,
+                self.CONFIG_KEY_AWS_SECRET_ACCESS_KEY,
+                self.CONFIG_KEY_AWS_SESSION_TOKEN,
+                self.CONFIG_KEY_AWS_REGION,
+            ]
+        ):
+            result = {
+                "AWS_ACCESS_KEY_ID": self._config[self.CONFIG_KEY_AWS_ACCESS_KEY_ID],
+                "AWS_SECRET_ACCESS_KEY": self._config[self.CONFIG_KEY_AWS_SECRET_ACCESS_KEY],
+                "AWS_SESSION_TOKEN": self._config[self.CONFIG_KEY_AWS_SESSION_TOKEN],
+                "AWS_REGION": self._config[self.CONFIG_KEY_AWS_REGION],
+            }
+
+        return result
+
+    def list(self, context: dict[str, Any] | None = None) -> list[str]:
+        """Lists the available delta lake datasets in the url provided."""
+        logger.info(f"[list|in] ({context})")
+
+        result: list[str] = []
+        if self.CONTEXT_KEY_URL not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
+
+        path = remove_s3_protocol(context[self.CONTEXT_KEY_URL])
+        path_elements = path.split("/")
+        bucket = path_elements[0]
+        key = "/".join(path_elements[1:])
+
+        matches: set[str] = set()
+        pattern: str = f".*{key}/(.*)/_delta_log/.*"
+        for entry in self._client.list_objects_v2(Bucket=bucket, Prefix=key)["Contents"]:
+            output_key: str = entry["Key"]
+            match = re.search(pattern, output_key)
+            if match:
+                matches.add(f"{key}/{match.group(1)}")
+
+        result = list(matches)
+
+        logger.info(f"[list] result: {result}")
+        logger.info(f"[list|out] => result len: {len(result)}")
+        return result

tgedr_dataops_ext/store/spark_delta.py

@@ -0,0 +1,515 @@
+"""Spark Delta Lake store implementation module.
+
+This module provides a Store implementation for working with Delta Lake format
+using Apache Spark. It includes the SparkDeltaStore class which supports:
+- Reading and writing data in Delta Lake format
+- Versioning and time travel queries
+- Update/merge operations (upserts)
+- Partitioning and schema evolution
+- Retention policies for log and deleted files
+- Metadata management
+"""
+
+from abc import ABC
+import dataclasses
+import logging
+from typing import Any
+from datetime import datetime
+from pyspark.sql import DataFrame
+from delta.tables import DeltaTable
+from pyspark.sql import functions as f
+from pyspark.sql import types as T  # noqa: N812
+from pyspark.sql.utils import AnalysisException
+from pyspark.sql.functions import monotonically_increasing_id
+from tgedr_dataops_abs.store import NoStoreException, Store, StoreException
+from tgedr_dataops_ext.commons.metadata import Metadata
+from tgedr_dataops_ext.commons.utils_spark import UtilsSpark
+
+logger = logging.getLogger(__name__)
+
+
+class SparkDeltaStore(Store, ABC):
+    """A store implementation using Spark Delta Lake format.
+
+    This class provides methods for reading, writing, updating, and deleting
+    data in Delta Lake format using Apache Spark. It supports features like
+    versioning, partitioning, schema evolution, and retention policies.
+
+    Attributes
+    ----------
+    config : dict[str, Any] | None
+        Optional configuration dictionary for the store.
+
+    Methods
+    -------
+    get(key: str, version: Optional[str] = None, **kwargs) -> DataFrame
+        Retrieve data from the specified key/path.
+    save(df: DataFrame, key: str, append: bool = False, ...) -> None
+        Save a DataFrame to the specified key/path.
+    update(df: Any, key: str, match_fields: list[str], ...) -> None
+        Update or insert data using merge operation.
+    delete(key: str, condition: Union[f.Column, str, None] = None, **kwargs) -> None
+        Delete data from the specified key/path.
+    """
+
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize SparkDeltaStore with optional configuration.
+
+        Args:
+            config: Optional configuration dictionary for the store.
+        """
+        Store.__init__(self, config)
+
+    def get(self, key: str, version: str | None = None, **kwargs) -> DataFrame:  # noqa: ANN003, ARG002
+        """Retrieve data from the specified key/path.
+
+        Parameters
+        ----------
+        key : str
+            The path to the Delta table to read from.
+        version : str | None, optional
+            The version of the Delta table to read, is None by default.
+        **kwargs
+            Additional keyword arguments (currently unused).
+
+        Returns
+        -------
+        DataFrame
+            The Spark DataFrame containing the data from the Delta table.
+
+        Raises
+        ------
+        NoStoreException
+            If no data is found at the specified key/path.
+        """
+        logger.info(f"[get|in] ({key}, {version})")
+
+        table = self._get_table(path=key)
+        if table is None:
+            raise NoStoreException(f"[get] couldn't find data in key: {key}")
+
+        reader = UtilsSpark.get_spark_session().read.format("delta")
+        if version is not None:
+            reader = reader.option("versionAsOf", version)
+
+        result = reader.load(key)
+
+        logger.info("[get_df|out]")
+        return result
+
+    def __get_deletion_criteria(self, df: DataFrame) -> Any:
+        logger.debug("[__get_deletion_criteria|in])")
+        fields = df.dtypes
+        numerics = [
+            x
+            for x in fields
+            if x[1] in ["bigint", "int", "double", "float", "long", "decimal.Decimal"] or (x[1][:7]) == "decimal"
+        ]
+        dates = [x for x in fields if (x[1]) in ["datetime", "datetime.datetime"]]
+        textuals = [x for x in fields if x[1] in ["string"]]
+        if 0 < len(numerics):
+            column = numerics[0][0]
+            result = (f.col(column) > 0) | (f.col(column) <= 0)
+        elif 0 < len(dates):
+            column = dates[0][0]
+            now = datetime.now(tz=datetime.UTC)
+            result = (f.col(column) > now) | (f.col(column) <= now)
+        elif 0 < len(textuals):
+            column = textuals[0][0]
+            result = (f.col(column) > "a") | (f.col(column) <= "a")
+        else:
+            raise StoreException(
+                "[__get_deletion_criteria] failed to figure out column types handy to create a full deletion criteria"
+            )
+
+        logger.debug(f"[__get_deletion_criteria|out] = {result}")
+        return result
+
+    def delete(self, key: str, condition: f.Column | str | None = None, **kwargs) -> None:  # noqa: ANN003, ARG002
+        """Delete data from the specified key/path.
+
+        Parameters
+        ----------
+        key : str
+            The path to the Delta table to delete from.
+        condition : f.Column | str | None, optional
+            The condition to filter rows for deletion. If None, deletes all rows.
+        **kwargs
+            Additional keyword arguments (currently unused).
+
+        Raises
+        ------
+        StoreException
+            If deletion fails or column types cannot be determined for full deletion.
+        """
+        logger.info(f"[delete|in] ({key}, {condition})")
+
+        spark = UtilsSpark.get_spark_session()
+        """
+        is_s3_operation = True if key.startswith("s3") else False
+        if is_s3_operation:
+        """
+        delta_table = DeltaTable.forPath(spark, key)
+        if condition is None:
+            condition = self.__get_deletion_criteria(delta_table.toDF())
+        delta_table.delete(condition=condition)
+        """
+        else:   # local development mostly for temporary or test purposes
+            spark_fs = spark._jvm.org.apache.hadoop.fs.FileSystem.get(spark._jsc.hadoopConfiguration())
+            # get spark context path
+            spark_path = spark._jvm.org.apache.hadoop.fs.Path(key)
+            logger.info(f"[delete] spark path is {spark_path}")
+            try:
+                if spark_fs.exists(spark_path):
+                    spark_fs.delete(spark_path, True)
+            except AnalysisException as x:
+                raise StoreException(f"[delete] couldn't do it on key {key}: {x}")
+        """
+        logger.info("[delete|out]")
+
+    def save(
+        self,
+        df: DataFrame,
+        key: str,
+        append: bool = False,
+        partition_fields: list[str] | None = None,
+        metadata: Metadata | None = None,
+        retention_days: int = 7,
+        deleted_retention_days: int = 7,
+        column_descriptions: dict[str, str] | None = None,
+        table_name: str | None = None,
+        **kwargs,  # noqa: ANN003
+    ) -> None:
+        """Save a DataFrame to the specified key/path in Delta format.
+
+        Parameters
+        ----------
+        df : DataFrame
+            The Spark DataFrame to save.
+        key : str
+            The path where the Delta table will be saved.
+        append : bool, optional
+            Whether to append to existing data, is False by default.
+        partition_fields : list[str] | None, optional
+            List of column names to partition the table by, is None by default.
+        metadata : Metadata | None, optional
+            Optional metadata to attach to the table, is None by default.
+        retention_days : int, optional
+            Number of days to retain log files, is 7 by default.
+        deleted_retention_days : int, optional
+            Number of days to retain deleted files, is 7 by default.
+        column_descriptions : dict[str, str] | None, optional
+            Dictionary mapping column names to their descriptions, is None by default.
+        table_name : str | None, optional
+            Optional table name in format 'db.table', is None by default.
+        **kwargs
+            Additional keyword arguments.
+        """
+        logger.info(
+            f"[save|in] ({df}, {key}, {append}, {partition_fields}, {metadata}, {retention_days}, {deleted_retention_days}, {column_descriptions}, {table_name}, {kwargs})"
+        )
+
+        if column_descriptions is not None:
+            df = self._set_column_descriptions(df, column_descriptions)
+
+        writer = df.write.format("delta").mode("append") if append else df.write.format("delta").mode("overwrite")
+
+        if partition_fields is not None:
+            table = self._get_table(path=key)
+            if table is not None:
+                self._set_table_partitions(path=key, partition_fields=partition_fields)
+            writer = writer.partitionBy(*partition_fields)
+
+        if self._has_schema_changed(path=key, df=df):
+            writer = writer.option("overwriteSchema", "true")
+
+        if metadata:
+            writer = writer.option("userMetadata", metadata)
+
+        if table_name is not None:
+            # assume we have db.table
+            db = table_name.split(".")[0]
+            UtilsSpark.get_spark_session().sql(f"CREATE DATABASE IF NOT EXISTS {db}")
+            writer = writer.option("path", key).saveAsTable(table_name)
+        else:
+            writer.save(key)
+
+        logger.info("[save] optimizing...")
+        table = self._get_table(path=key)
+
+        if retention_days is not None and deleted_retention_days is not None:
+            self.enforce_retention_policy(
+                path=key, retention_days=retention_days, deleted_retention_days=deleted_retention_days
+            )
+        elif retention_days is not None:
+            self.enforce_retention_policy(path=key, retention_days=retention_days)
+
+        table.optimize().executeCompaction()
+
+        logger.info("[save|out]")
+
+    def update(
+        self,
+        df: Any,
+        key: str,
+        match_fields: list[str],
+        partition_fields: list[str] | None = None,
+        metadata: Metadata | None = None,
+        retention_days: int = 7,
+        deleted_retention_days: int = 7,
+        **kwargs,  # noqa: ANN003
+    ) -> None:
+        """Update or insert data using merge operation.
+
+        Parameters
+        ----------
+        df : Any
+            The DataFrame containing updates to merge.
+        key : str
+            The path to the Delta table to update.
+        match_fields : list[str]
+            List of column names to use for matching rows during merge.
+        partition_fields : list[str] | None, optional
+            List of column names to partition the table by, is None by default.
+        metadata : Metadata | None, optional
+            Optional metadata to attach to the table, is None by default.
+        retention_days : int, optional
+            Number of days to retain log files, is 7 by default.
+        deleted_retention_days : int, optional
+            Number of days to retain deleted files, is 7 by default.
+        **kwargs
+            Additional keyword arguments.
+        """
+        logger.info(
+            f"[update|in] ({df}, {key}, {match_fields}, {partition_fields}, {metadata}, {retention_days}, {deleted_retention_days}, {kwargs})"
+        )
+
+        table = self._get_table(path=key)
+        if table is None:
+            self.save(
+                df=df,
+                key=key,
+                partition_fields=partition_fields,
+                metadata=metadata,
+                retention_days=retention_days,
+                deleted_retention_days=deleted_retention_days,
+                **kwargs,
+            )
+        else:
+            if partition_fields is not None:
+                self._set_table_partitions(path=key, partition_fields=partition_fields)
+
+            match_clause = None
+            for field in match_fields:
+                match_clause = (
+                    f"current.{field} = updates.{field}"
+                    if match_clause is None
+                    else f"{match_clause} and current.{field} = updates.{field}"
+                )
+            logger.info(f"[update] match clause: {match_clause}")
+
+            # check if the df has all the required columns
+            # as we are upserting the updated columns coming in must at least match or exceed the current columns
+            for column in table.toDF().columns:
+                # we'll assume missing columns are nullable, typically metrics
+                if column not in df.columns:
+                    df = df.withColumn(column, f.lit(None).cast(T.StringType()))
+
+            table.alias("current").merge(
+                df.alias("updates"), match_clause
+            ).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute()
+
+            if retention_days is not None and deleted_retention_days is not None:
+                self.enforce_retention_policy(
+                    path=key, retention_days=retention_days, deleted_retention_days=deleted_retention_days
+                )
+            elif retention_days is not None:
+                self.enforce_retention_policy(path=key, retention_days=retention_days)
+
+            table.optimize().executeCompaction()
+
+        logger.info("[UtilsDeltaTable.upsert|out]")
+
+    def enforce_retention_policy(self, path: str, retention_days: int = 7, deleted_retention_days: int = 7) -> None:
+        """Enforce retention policy on Delta table log and deleted files.
+
+        Parameters
+        ----------
+        path : str
+            The path to the Delta table.
+        retention_days : int, optional
+            Number of days to retain log files, is 7 by default.
+        deleted_retention_days : int, optional
+            Number of days to retain deleted files, is 7 by default.
+        """
+        logger.info(f"[enforce_retention_policy|in] ({path}, {retention_days}, {deleted_retention_days})")
+
+        retention = f"interval {retention_days} days"
+        deleted_retention = f"interval {deleted_retention_days} days"
+
+        UtilsSpark.get_spark_session().sql(
+            f"ALTER TABLE delta.`{path}` SET TBLPROPERTIES('delta.logRetentionDuration' = '{retention}', 'delta.deletedFileRetentionDuration' = '{deleted_retention}')"
+        )
+        logger.info("[enforce_retention_policy|out]")
+
+    def get_latest_table_versions(self, path: str, how_many: int = 1) -> list[str]:
+        """Checks the delta table history and retrieves the latest n versions.
+
+        Sorted from the newest to the oldest.
+        """
+        logger.info(f"[get_latest_table_versions|in] ({path}, {how_many})")
+        result: list[str] = []
+
+        table = self._get_table(path=path)
+        if table is not None:
+            history_rows = table.history().orderBy(f.desc("timestamp")).limit(how_many)
+            result = [str(x.version) for x in history_rows.collect()]
+
+        logger.info(f"[get_latest_table_versions|out] => {result}")
+        return result
+
+    def get_metadata(self, path: str, version: str | None = None) -> Metadata | None:
+        """Retrieve metadata from the Delta table at the specified path.
+
+        Raises
+        ------
+        NoStoreException
+        """
+        logger.info(f"[get_metadata|in] ({path}, {version})")
+        table = self._get_table(path)
+        if table is None:
+            raise NoStoreException(f"[get_metadata] no data in path: {path}")
+
+        result = None
+
+        df_history = table.history().filter(f.col("userMetadata").isNotNull())
+        if version is not None:
+            df_history = df_history.filter(f.col("version") <= int(version))
+
+        df_history = df_history.orderBy(f.col("version").desc())
+        if not df_history.isEmpty():
+            user_metadata = df_history.take(1)[0].userMetadata
+            result = Metadata.from_str(user_metadata)
+            if version is not None:
+                result = dataclasses.replace(result, version=version)
+
+        logger.info(f"[get_metadata|out] => ({result})")
+        return result
+
+    def _get_delta_log(self, path: str) -> DataFrame:
+        logger.info(f"[_get_delta_log|in] ({path})")
+
+        spark = UtilsSpark.get_spark_session()
+        jdf = (
+            spark._jvm.org.apache.spark.sql.delta.DeltaLog.forTable(spark._jsparkSession, path)  # noqa: SLF001
+            .snapshot()
+            .allFiles()
+            .toDF()
+        )
+        result = DataFrame(jdf, spark)
+
+        logger.info(f"[_get_delta_log|out] => {result}")
+        return result
+
+    def _get_table_partitions(self, path: str) -> list[str]:
+        logger.info(f"[_get_table_partitions|in] ({path})")
+        result: list[str] = []
+
+        delta_log: DataFrame = self._get_delta_log(path=path)
+        partition_keys = [
+            x.keys
+            for x in delta_log.select(f.map_keys(f.col("partitionValues")).alias("keys")).distinct().collect()
+            if 0 < len(x)
+        ]
+        if 0 < len(partition_keys):
+            result: list[str] = list({y for y in partition_keys for y in y})
+
+        logger.info(f"[_get_table_partitions|out] => {result}")
+        return result
+
+    def _vacuum_now(self, path: str) -> None:
+        logger.info("[_vacuum_now|in]")
+
+        spark = UtilsSpark.get_spark_session()
+        old_conf_value = spark.conf.get("spark.databricks.delta.retentionDurationCheck.enabled")
+        spark.conf.set("spark.databricks.delta.retentionDurationCheck.enabled", "false")
+        DeltaTable.forPath(spark, path).vacuum(0)
+        spark.conf.set("spark.databricks.delta.retentionDurationCheck.enabled", old_conf_value)
+
+        logger.info("[_vacuum_now|out]")
+
+    def _has_schema_changed(self, path: str, df: DataFrame) -> bool:
+        logger.info(f"[_has_schema_changed|in] ({path},{df})")
+        result: bool = False
+        table = self._get_table(path=path)
+        if table is not None:
+            result = table.toDF().schema != df.schema
+        logger.info(f"[_has_schema_changed|out] => {result}")
+        return result
+
+    def _set_table_partitions(self, path: str, partition_fields: list[str]) -> None:
+        logger.info(f"[_set_table_partitions|in] ({path},{partition_fields})")
+
+        spark = UtilsSpark.get_spark_session()
+        # let's check partition_cols
+        current_partition_fields = self._get_table_partitions(path=path)
+        shall_we_repartition = sorted(partition_fields) != sorted(current_partition_fields)
+
+        if shall_we_repartition:
+            logger.info("[_set_table_partitions] going to repartition")
+            new_df = spark.read.format("delta").load(path)
+            new_df.write.format("delta").mode("overwrite").partitionBy(*partition_fields).option(
+                "overwriteSchema", "true"
+            ).save(path)
+            self._vacuum_now(path)
+            logger.info(
+                f"[_set_table_partitions] changed partition cols from {current_partition_fields} to {partition_fields}"
+            )
+        logger.info("[_set_table_partitions|out]")
+
+    def _get_table(self, path: str) -> DeltaTable | None:
+        logger.debug(f"[_get_table|in] ({path})")
+        result: DeltaTable = None
+        try:
+            result: DeltaTable = DeltaTable.forPath(UtilsSpark.get_spark_session(), path)
+        except AnalysisException as ax:
+            logger.warning(f"[_get_table] couldn't load from {path}: {ax}")
+
+        logger.debug(f"[_get_table|out] => {result}")
+        return result
+
+    def set_column_comments(self, db: str, table: str, col_comments: dict[str, str]) -> None:
+        """Set comments for columns in a Delta table.
+
+        Parameters
+        ----------
+        db : str
+            The database name where the table is located.
+        table : str
+            The table name to set column comments for.
+        col_comments : dict[str, str]
+            Dictionary mapping column names to their comments.
+        """
+        logger.info(f"[set_column_comments|in] ({db}, {table}, {col_comments})")
+        spark = UtilsSpark.get_spark_session()
+
+        table_description: DataFrame = spark.sql(f"describe {db}.{table}").withColumn(
+            "set_column_comments_id", monotonically_increasing_id()
+        )
+        partition_info_id = (
+            table_description.filter(f.col("col_name") == "# Partition Information").collect()[0].set_column_comments_id
+        )
+
+        table_description = table_description.filter(
+            (f.col("set_column_comments_id") < f.lit(partition_info_id)) & (f.col("col_name") != "")
+        ).drop("set_column_comments_id")
+        rows = [r.asDict() for r in table_description.collect()]
+        for row in rows:
+            col = row["col_name"]
+            data_type = row["data_type"]
+            if col in col_comments:
+                new_comment = col_comments[col]
+                logger.info(f"[set_column_comments] setting new comment ({new_comment}) to column {col}")
+                spark.sql(f"ALTER TABLE {db}.{table} CHANGE COLUMN {col} {col} {data_type} COMMENT '{new_comment}'")
+
+        logger.info("[set_column_comments|out]")

tgedr_dataops_ext-0.0.1.dist-info/METADATA

@@ -0,0 +1,60 @@
+Metadata-Version: 2.4
+Name: tgedr-dataops-ext
+Version: 0.0.1
+Summary: this is a template for a python package
+Author-email: developer <developer@email.com>
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=2.3.0
+Requires-Dist: deltalake~=0.16.4
+Requires-Dist: delta-spark~=2.4.0
+Requires-Dist: tgedr-dataops>=1.0.1
+Requires-Dist: pyspark~=3.4.0
+
+# tgedr-dataops-ext
+
+![Coverage](./coverage.svg)
+[![PyPI](https://img.shields.io/pypi/v/tgedr-dataops-ext)](https://pypi.org/project/tgedr-dataops-ext/)
+
+
+data operations related code - extended
+
+## motivation
+*dataops-ext* is a library with tested and used code aligning on some standards regarding code structure and quality and to avoid reinventing the wheel. It builds on top of *dataops-abs* and *dataops* providing distributed processing features based on pyspark.
+
+## installation
+        `pip install tgedr-dataops-ext`
+
+## package namespaces and its contents
+
+#### commons
+- __Dataset__: immutable class to wrap up a dataframe along with metadata ([example](tests/tgedr_dataops_ext/commons/test_dataset.py))
+- __Metadata__: immutable class depicting dataset metadata ([example](tests/tgedr_dataops_ext/commons/test_metadata.py))
+- __UtilsSpark__: utility class to work with spark, mostly helping on creating a session ([example](tests/tgedr_dataops_ext/commons/test_utils_spark.py))
+
+#### quality
+- __PysparkValidation__ : __GreatExpectationsValidation__ implementation to validate pyspark dataframes with Great Expectations library ([example](tests/tgedr_dataops_ext/quality/test_pyspark_validation.py))
+
+#### source
+
+- __DeltaTableSource__: abstract __Source__ class used to read delta lake format datasets returning a pandas dataframe" ([example](tests/tgedr_dataops_ext/source/test_delta_table_source.py))
+- __LocalDeltaTable__: __Source__ class used to read delta lake format datasets from local fs with python only, pyspark not needed, returning a pandas dataframe ([example](tests/tgedr_dataops_ext/source/test_local_delta_table.py))
+- __S3DeltaTable__: __Source__ class used to read delta lake format datasets from s3 bucket with python only, pyspark not needed, returning a pandas dataframe ([example](tests/tgedr_dataops_ext/source/test_s3_delta_table.py))
+
+
+#### store
+- __SparkDeltaStore__ : __Store__ implementation for pyspark distributed processing with delta table format ([example](tests/tgedr_dataops_ext/store/test_spark_delta.py))
+
+
+
+## development
+- main requirements:
+  - _uv_  
+  - _bash_
+- Clone the repository like this:
+
+  ``` bash
+  git clone git@github.com:jtviegas/dataops-ext
+  ```
+- cd into the folder: `cd dataops-ext`
+- install requirements: `./helper.sh reqs`

tgedr_dataops_ext-0.0.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+tgedr_dataops_ext/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tgedr_dataops_ext/commons/dataset.py,sha256=cYJkqm-w4VxwprUAgB8QyCLiJ-bnK7PLGZWdmkahbhM,794
+tgedr_dataops_ext/commons/metadata.py,sha256=UClsNoo9BUbz6Defp9Jd01k00h7GV9FhET6wXbScSLw,8106
+tgedr_dataops_ext/commons/utils_spark.py,sha256=NcJRlcGky0abc28hq6Lrn0-OY3DjTo8pCOGcWb4qaco,4886
+tgedr_dataops_ext/quality/pyspark_validation.py,sha256=ppnLWBDz2n2rchhyPnfUuNbZVw63dTvkdM56bYWeQYY,824
+tgedr_dataops_ext/source/delta_table_source.py,sha256=lcGgAKpNj8FNF8DTnMK_KAQ3GgEUxHU8sKcwjcCTR84,1961
+tgedr_dataops_ext/source/local_delta_table.py,sha256=3ffk3kWwLjaNLbSFU50R_fTd9WE9ItAaHR7Dv5pATLg,1975
+tgedr_dataops_ext/source/s3_delta_table.py,sha256=z4e5LTAAeqt7GvtNsaoc8z_sZhOBOjQikp35thodZwo,3266
+tgedr_dataops_ext/store/spark_delta.py,sha256=33zKhJYmlGq5mFvy5Yzr9z_zX5gzjZSFW8RuYWHcdjA,20805
+tgedr_dataops_ext-0.0.1.dist-info/METADATA,sha256=kBiDkP5rdWMJRJ_Xb8G6hjJYkAq89uf4BMFE5KCvM5Q,2686
+tgedr_dataops_ext-0.0.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+tgedr_dataops_ext-0.0.1.dist-info/top_level.txt,sha256=st7VbEQz5kyNJ8ww2zjv-uWKyKww1npcI-qr-XQClaY,18
+tgedr_dataops_ext-0.0.1.dist-info/RECORD,,

tgedr_dataops_ext-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tgedr_dataops_ext-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+tgedr_dataops_ext