tgedr-dataops-ext 0.0.1__tar.gz

tgedr_dataops_ext-0.0.1/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.4
+Name: tgedr-dataops-ext
+Version: 0.0.1
+Summary: this is a template for a python package
+Author-email: developer <developer@email.com>
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=2.3.0
+Requires-Dist: deltalake~=0.16.4
+Requires-Dist: delta-spark~=2.4.0
+Requires-Dist: tgedr-dataops>=1.0.1
+Requires-Dist: pyspark~=3.4.0
+
+# tgedr-dataops-ext
+
+![Coverage](./coverage.svg)
+[![PyPI](https://img.shields.io/pypi/v/tgedr-dataops-ext)](https://pypi.org/project/tgedr-dataops-ext/)
+
+
+data operations related code - extended
+
+## motivation
+*dataops-ext* is a library with tested and used code aligning on some standards regarding code structure and quality and to avoid reinventing the wheel. It builds on top of *dataops-abs* and *dataops* providing distributed processing features based on pyspark.
+
+## installation
+        `pip install tgedr-dataops-ext`
+
+## package namespaces and its contents
+
+#### commons
+- __Dataset__: immutable class to wrap up a dataframe along with metadata ([example](tests/tgedr_dataops_ext/commons/test_dataset.py))
+- __Metadata__: immutable class depicting dataset metadata ([example](tests/tgedr_dataops_ext/commons/test_metadata.py))
+- __UtilsSpark__: utility class to work with spark, mostly helping on creating a session ([example](tests/tgedr_dataops_ext/commons/test_utils_spark.py))
+
+#### quality
+- __PysparkValidation__ : __GreatExpectationsValidation__ implementation to validate pyspark dataframes with Great Expectations library ([example](tests/tgedr_dataops_ext/quality/test_pyspark_validation.py))
+
+#### source
+
+- __DeltaTableSource__: abstract __Source__ class used to read delta lake format datasets returning a pandas dataframe" ([example](tests/tgedr_dataops_ext/source/test_delta_table_source.py))
+- __LocalDeltaTable__: __Source__ class used to read delta lake format datasets from local fs with python only, pyspark not needed, returning a pandas dataframe ([example](tests/tgedr_dataops_ext/source/test_local_delta_table.py))
+- __S3DeltaTable__: __Source__ class used to read delta lake format datasets from s3 bucket with python only, pyspark not needed, returning a pandas dataframe ([example](tests/tgedr_dataops_ext/source/test_s3_delta_table.py))
+
+
+#### store
+- __SparkDeltaStore__ : __Store__ implementation for pyspark distributed processing with delta table format ([example](tests/tgedr_dataops_ext/store/test_spark_delta.py))
+
+
+
+## development
+- main requirements:
+  - _uv_  
+  - _bash_
+- Clone the repository like this:
+
+  ``` bash
+  git clone git@github.com:jtviegas/dataops-ext
+  ```
+- cd into the folder: `cd dataops-ext`
+- install requirements: `./helper.sh reqs`

tgedr_dataops_ext-0.0.1/README.md

@@ -0,0 +1,47 @@
+# tgedr-dataops-ext
+
+![Coverage](./coverage.svg)
+[![PyPI](https://img.shields.io/pypi/v/tgedr-dataops-ext)](https://pypi.org/project/tgedr-dataops-ext/)
+
+
+data operations related code - extended
+
+## motivation
+*dataops-ext* is a library with tested and used code aligning on some standards regarding code structure and quality and to avoid reinventing the wheel. It builds on top of *dataops-abs* and *dataops* providing distributed processing features based on pyspark.
+
+## installation
+        `pip install tgedr-dataops-ext`
+
+## package namespaces and its contents
+
+#### commons
+- __Dataset__: immutable class to wrap up a dataframe along with metadata ([example](tests/tgedr_dataops_ext/commons/test_dataset.py))
+- __Metadata__: immutable class depicting dataset metadata ([example](tests/tgedr_dataops_ext/commons/test_metadata.py))
+- __UtilsSpark__: utility class to work with spark, mostly helping on creating a session ([example](tests/tgedr_dataops_ext/commons/test_utils_spark.py))
+
+#### quality
+- __PysparkValidation__ : __GreatExpectationsValidation__ implementation to validate pyspark dataframes with Great Expectations library ([example](tests/tgedr_dataops_ext/quality/test_pyspark_validation.py))
+
+#### source
+
+- __DeltaTableSource__: abstract __Source__ class used to read delta lake format datasets returning a pandas dataframe" ([example](tests/tgedr_dataops_ext/source/test_delta_table_source.py))
+- __LocalDeltaTable__: __Source__ class used to read delta lake format datasets from local fs with python only, pyspark not needed, returning a pandas dataframe ([example](tests/tgedr_dataops_ext/source/test_local_delta_table.py))
+- __S3DeltaTable__: __Source__ class used to read delta lake format datasets from s3 bucket with python only, pyspark not needed, returning a pandas dataframe ([example](tests/tgedr_dataops_ext/source/test_s3_delta_table.py))
+
+
+#### store
+- __SparkDeltaStore__ : __Store__ implementation for pyspark distributed processing with delta table format ([example](tests/tgedr_dataops_ext/store/test_spark_delta.py))
+
+
+
+## development
+- main requirements:
+  - _uv_  
+  - _bash_
+- Clone the repository like this:
+
+  ``` bash
+  git clone git@github.com:jtviegas/dataops-ext
+  ```
+- cd into the folder: `cd dataops-ext`
+- install requirements: `./helper.sh reqs`

tgedr_dataops_ext-0.0.1/pyproject.toml

@@ -0,0 +1,134 @@
+[project]
+name = "tgedr-dataops-ext"
+version = "0.0.1"
+description = "this is a template for a python package"
+authors = [
+    {name = "developer",email = "developer@email.com"}
+]
+readme = "README.md"
+requires-python = ">=3.11"
+
+dependencies = [
+    "pandas>=2.3.0",
+    "deltalake~=0.16.4",
+    "delta-spark~=2.4.0",
+    "tgedr-dataops>=1.0.1",
+    "pyspark~=3.4.0"
+]
+[dependency-groups]
+dev = [
+    "pre-commit~=4.2.0",
+    "pytest~=8.3.5",
+    "pytest-bdd~=8.1.0",
+    "pytest-cov~=4.1.0",
+    "pytest-mock~=3.15.0",
+    "ruff==0.9.10",
+    "bandit==1.8.3",
+    "safety==3.5.1",
+    "typer<0.17.0",
+    "genbadge[coverage]>=1.1.3",
+]
+
+# [project.scripts]
+# run = "tgedr.pycommons.entrypoint:entrypoint" 
+
+[build-system]
+requires = ["setuptools>=78.1.0", "wheel>=0.45.1"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.package-data]
+"*" = ["CHANGELOG"]
+
+[tool.coverage.paths]
+source = ["src/"]
+
+[tool.coverage.run]
+source = ["src/"]
+include = ["src/*"]
+omit = [
+    "*/tests/*",
+    "*/test_*",
+    "*/__pycache__/*",
+    "*/migrations/*",
+    "*/venv/*",
+    "*/.venv/*"
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+]
+show_missing = true
+skip_covered = false
+skip_empty = false
+
+[tool.pytest.ini_options]
+# bdd_features_base_dir = "documentation/features"
+pythonpath = "."
+
+[tool.ruff]
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+    "tests/",
+    "typings/"
+]
+
+line-length = 120
+indent-width = 4
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = ["D203", "S101", "D104", "INP001", "D213", "COM812", "I001", 
+    "D401", "D407", "RET504", "PLR2004", "FA102", "E501", "EXE002", "PLR0913", 
+    "PLR0912", "C901", "PLR0911", "D413", "N818", "B024", "ANN401", "SIM300",
+    "FBT001", "FBT002", "G004", "TRY003", "EM102", "EM101", "PD015", "PD901"]
+fixable = ["ALL"]
+unfixable = []
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[tool.ruff.format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"

tgedr_dataops_ext-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tgedr_dataops_ext-0.0.1/src/tgedr_dataops_ext/commons/dataset.py

@@ -0,0 +1,27 @@
+"""Dataset module for wrapping dataframes with metadata.
+
+This module provides:
+- Dataset: an immutable dataclass that combines a DataFrame with Metadata.
+"""
+
+from dataclasses import dataclass
+import json
+from pyspark.sql import DataFrame
+from tgedr_dataops_ext.commons.metadata import Metadata
+
+
+@dataclass(frozen=True)
+class Dataset:
+    """Utility immutable class to wrap up a dataframe along with metadata."""
+
+    __slots__ = ["data", "metadata"]
+    metadata: Metadata
+    data: DataFrame
+
+    def as_dict(self) -> dict:
+        """Serialize the dataset as a dictionary."""
+        return {"metadata": self.metadata.as_dict(), "data": str(self.data.__repr__)}
+
+    def __str__(self) -> str:
+        """Serialize the dataset as a json string."""
+        return json.dumps(self.as_dict())

tgedr_dataops_ext-0.0.1/src/tgedr_dataops_ext/commons/metadata.py

@@ -0,0 +1,236 @@
+"""Module for dataset metadata classes."""
+
+from dataclasses import dataclass
+import json
+
+
+@dataclass(frozen=True)
+class FieldFrame:
+    """Class depicting a field values range, to be used in metadata.
+
+    Parameters
+    ----------
+    field : str
+        The name of the field.
+    lower : Union[int, str, float]
+        Field lower bound.
+    upper : Union[int, str, float]
+        Field upper bound.
+
+    """
+
+    __slots__ = ["field", "lower", "upper"]
+    field: str
+    lower: int | str | float
+    upper: int | str | float
+
+    def as_dict(self) -> dict[str, int | str | float]:
+        """Convert the FieldFrame to a dictionary representation.
+
+        Returns
+        -------
+        dict[str, int | str | float]
+            A dictionary containing the field name, lower bound, and upper bound.
+        """
+        return {"field": self.field, "lower": self.lower, "upper": self.upper}
+
+    @staticmethod
+    def from_str(src: str) -> "FieldFrame":
+        """Create a FieldFrame instance from a JSON string.
+
+        Parameters
+        ----------
+        src : str
+            A JSON string representation of a FieldFrame.
+
+        Returns
+        -------
+        FieldFrame
+            A new FieldFrame instance created from the JSON string.
+        """
+        r = json.loads(src)
+        field = r["field"]
+        lower = r["lower"]
+        upper = r["upper"]
+        return FieldFrame(field=field, lower=lower, upper=upper)
+
+    def __str__(self) -> str:
+        """Return a JSON string representation of the FieldFrame."""
+        return json.dumps(self.as_dict())
+
+    def __eq__(self, other: "FieldFrame") -> bool:
+        """Check equality between two FieldFrame instances."""
+        return self.field == other.field and self.lower == other.lower and self.upper == other.upper
+
+    def __gt__(self, other: "FieldFrame") -> bool:
+        """Check if this FieldFrame is greater than another FieldFrame."""
+        return self.field > other.field or (
+            self.field == other.field
+            and (self.lower > other.lower or (self.lower == other.lower and self.upper > other.upper))
+        )
+
+    def __ne__(self, other: "FieldFrame") -> bool:
+        """Check inequality between two FieldFrame instances."""
+        return not other == self
+
+    def __ge__(self, other: "FieldFrame") -> bool:
+        """Check if this FieldFrame is greater than or equal to another FieldFrame."""
+        return other == self or self > other
+
+    def __le__(self, other: "FieldFrame") -> bool:
+        """Check if this FieldFrame is less than or equal to another FieldFrame."""
+        return other == self or self < other
+
+    def __lt__(self, other: "FieldFrame") -> bool:
+        """Check if this FieldFrame is less than another FieldFrame."""
+        return other > self
+
+
+@dataclass(frozen=True)
+class Metadata:
+    """Immutable class depicting dataset metadata.
+
+    Parameters
+    ----------
+    name : str
+        The name of the dataset.
+    version : Optional[str]
+        Version of this dataset, if available.
+    framing : Optional[List[FieldFrame]]
+        Multiple field frames.
+    sources : Optional[List["Metadata"]]
+        Metadatas related to the datasets sourcing this one.
+
+    """
+
+    __slots__ = ["framing", "name", "sources", "version"]
+    name: str
+    version: str | None
+    framing: list[FieldFrame] | None
+    sources: list["Metadata"] | None
+
+    def as_dict(self) -> dict:
+        """Convert the Metadata to a dictionary representation.
+
+        Returns
+        -------
+        dict
+            A dictionary containing the metadata fields including name, version,
+            framing, and sources if they are not None.
+        """
+        result = {"name": self.name}
+        if self.version is not None:
+            result["version"] = self.version
+        if self.framing is not None:
+            result["framing"] = []
+            for f in self.framing:
+                (result["framing"]).append(f.as_dict())
+        if self.sources is not None:
+            result["sources"] = []
+            for source in self.sources:
+                (result["sources"]).append(source.as_dict())
+
+        return result
+
+    def __str__(self) -> str:
+        """Return a JSON string representation of the Metadata."""
+        return json.dumps(self.as_dict())
+
+    def __eq__(self, other: object) -> bool:
+        """Check equality between two Metadata instances."""
+        return (
+            self.name == other.name
+            and (
+                (self.version is None and other.version is None)
+                or ((self.version is not None and other.version is not None) and self.version == other.version)
+            )
+            and (
+                (self.framing is None and other.framing is None)
+                or (
+                    (self.framing is not None and other.framing is not None)
+                    and sorted(self.framing) == sorted(other.framing)
+                )
+            )
+            and (
+                (self.sources is None and other.sources is None)
+                or (
+                    (self.sources is not None and other.sources is not None)
+                    and sorted(self.sources) == sorted(other.sources)
+                )
+            )
+        )
+
+    def __gt__(self, other: "Metadata") -> bool:
+        """Check if this Metadata is greater than another Metadata."""
+        return self.name > other.name or (
+            self.name == other.name
+            and (
+                ((self.version is not None and other.version is not None) and (self.version > other.version))
+                or (self.version is not None and other.version is None)
+                or (
+                    (
+                        (self.framing is not None and other.framing is not None)
+                        and (sorted(self.framing) > sorted(other.framing))
+                    )
+                    or (self.framing is not None and other.framing is None)
+                    or (
+                        (
+                            (self.sources is not None and other.sources is not None)
+                            and (sorted(self.sources) > sorted(other.sources))
+                        )
+                        or (self.sources is not None and other.sources is None)
+                    )
+                )
+            )
+        )
+
+    def __ne__(self, other: "Metadata") -> bool:
+        """Check inequality between two Metadata instances."""
+        return not other == self
+
+    def __ge__(self, other: "Metadata") -> bool:
+        """Check if this Metadata is greater than or equal to another Metadata."""
+        return other == self or self > other
+
+    def __le__(self, other: "Metadata") -> bool:
+        """Check if this Metadata is less than or equal to another Metadata."""
+        return other == self or self < other
+
+    def __lt__(self, other: "Metadata") -> bool:
+        """Check if this Metadata is less than another Metadata."""
+        return other > self
+
+    @staticmethod
+    def from_str(src: str) -> "Metadata":
+        """Create a Metadata instance from a JSON string.
+
+        Parameters
+        ----------
+        src : str
+            A JSON string representation of a Metadata object.
+
+        Returns
+        -------
+        Metadata
+            A new Metadata instance created from the JSON string.
+        """
+        r = json.loads(src)
+        name = r["name"]
+        version = r.get("version", None)
+
+        framing = None
+        framing_entries = r.get("framing", None)
+        if framing_entries is not None:
+            framing = []
+            for framing_entry in framing_entries:
+                framing.append(FieldFrame.from_str(json.dumps(framing_entry)))
+
+        sources = None
+        sources_entries = r.get("sources", None)
+        if sources_entries is not None:
+            sources = []
+            for source_entry in sources_entries:
+                source_entry_as_str = json.dumps(source_entry)
+                sources.append(Metadata.from_str(source_entry_as_str))
+
+        return Metadata(name=name, version=version, framing=framing, sources=sources)

tgedr_dataops_ext-0.0.1/src/tgedr_dataops_ext/commons/utils_spark.py

@@ -0,0 +1,133 @@
+"""Spark utility functions and classes.
+
+This module provides:
+- UtilsSpark: A utility class with handy functions to work with Spark sessions,
+  including creating local and remote sessions with Delta Lake support,
+  and building PySpark schemas from data type dictionaries.
+"""
+
+import logging
+import os
+from typing import ClassVar
+from pyspark.sql import SparkSession
+from pyspark import SparkConf
+from pyspark.sql import types as T  # noqa: N812
+from pyspark.context import SparkContext
+
+
+logger = logging.getLogger(__name__)
+
+
+class UtilsSpark:
+    """class with handy functions to work with spark."""
+
+    __ENV_KEY_PYSPARK_IS_LOCAL = "PYSPARK_IS_LOCAL"
+    __ENV_KEY_NOT_AWS_CLOUD = "NOT_AWS_CLOUD"
+    __DTYPES_MAP: ClassVar[dict[str, type]] = {
+        "bigint": T.LongType,
+        "string": T.StringType,
+        "double": T.DoubleType,
+        "int": T.IntegerType,
+        "boolean": T.BooleanType,
+        "timestamp": T.TimestampType,
+        "date": T.DateType,
+    }
+
+    @staticmethod
+    def get_local_spark_session(config: dict | None = None) -> SparkSession:
+        """Get a local Spark session configured for Delta Lake.
+
+        Parameters
+        ----------
+        config : dict | None, optional
+            Additional Spark configuration parameters, by default None.
+
+        Returns
+        -------
+        SparkSession
+            A configured local Spark session instance with Delta Lake support.
+        """
+        logger.debug(f"[get_local_spark_session|in] ({config})")
+        # PySpark 3.4 uses Scala 2.12, so we need delta-core_2.12
+        builder = (
+            SparkSession.builder.config("spark.jars.packages", "io.delta:delta-core_2.12:2.4.0")
+            .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+            .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+            .config("spark.driver.host", "localhost")
+        )
+
+        if config is not None:
+            for k, v in config.items():
+                builder.config(k, v)
+
+        spark = builder.getOrCreate()
+
+        logger.debug(f"[get_local_spark_session|out] => {spark}")
+        return spark
+
+    @staticmethod
+    def get_spark_session(config: dict | None = None) -> SparkSession:
+        """Get a Spark session based on the environment configuration.
+
+        Parameters
+        ----------
+        config : dict | None, optional
+            Additional Spark configuration parameters, by default None.
+
+        Returns
+        -------
+        SparkSession
+            A configured Spark session instance.
+        """
+        logger.debug(f"[get_spark_session|in] ({config})")
+
+        if "1" == os.getenv(UtilsSpark.__ENV_KEY_PYSPARK_IS_LOCAL):
+            spark: SparkSession = UtilsSpark.get_local_spark_session(config)
+        else:
+            if "1" == os.getenv(UtilsSpark.__ENV_KEY_NOT_AWS_CLOUD):
+                active_session = SparkSession.getActiveSession()
+            else:
+                from awsglue.context import GlueContext  # type: ignore #   pragma: no cover  # noqa: PGH003
+
+                glueContext = GlueContext(SparkContext.getOrCreate())  #   pragma: no cover  # noqa: N806
+                active_session = glueContext.spark_session  #   pragma: no cover
+
+            spark_config = SparkConf()
+
+            if active_session is not None:
+                former_config = active_session.sparkContext.getConf().getAll()
+                for entry in former_config:
+                    spark_config.set(entry[0], entry[1])
+            if config is not None:
+                for k, v in config.items():
+                    spark_config.set(k, v)
+                spark: SparkSession = SparkSession.builder.config(conf=spark_config).getOrCreate()
+            else:
+                spark: SparkSession = SparkSession.builder.getOrCreate()
+
+        logger.debug(f"[get_spark_session|out] => {spark}")
+        return spark
+
+    @staticmethod
+    def build_schema_from_dtypes(dtypes_schema: dict[str, str]) -> T.StructType:
+        """Build a PySpark StructType schema from a dictionary of data types.
+
+        Parameters
+        ----------
+        dtypes_schema : dict[str, str]
+            A dictionary mapping field names to their corresponding data type strings.
+            Supported types: 'bigint', 'string', 'double', 'int', 'boolean', 'timestamp', 'date'.
+
+        Returns
+        -------
+        T.StructType
+            A PySpark StructType schema with fields defined by the input dictionary.
+        """
+        logger.info(f"[build_schema_from_dtypes|in] ({dtypes_schema})")
+        result = T.StructType()
+        for field, dtype in dtypes_schema.items():
+            new_type = UtilsSpark.__DTYPES_MAP[dtype]
+            result.add(field, new_type(), True)  # noqa: FBT003
+
+        logger.info(f"[build_schema_from_dtypes|out] => {result}")
+        return result

tgedr_dataops_ext-0.0.1/src/tgedr_dataops_ext/quality/pyspark_validation.py

@@ -0,0 +1,22 @@
+"""Pyspark DataFrame validation implementation module.
+
+This module provides the Pyspark-specific implementation of Great Expectations validation.
+"""
+
+from great_expectations.execution_engine import ExecutionEngine
+from great_expectations.execution_engine import SparkDFExecutionEngine
+from tgedr_dataops_abs.great_expectations_validation import GreatExpectationsValidation
+
+
+class PysparkValidation(GreatExpectationsValidation):
+    """Pyspark DataFrame validation implementation."""
+
+    def _get_execution_engine(self, batch_data_dict: dict) -> ExecutionEngine:
+        """Get the execution engine used by the validation implementation.
+
+        Returns
+        -------
+        ExecutionEngine
+            The execution engine instance.
+        """
+        return SparkDFExecutionEngine(batch_data_dict=batch_data_dict)