pyspark-testframework 0.1.0__py3-none-any.whl

pyspark_testframework-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Woonstad Rotterdam
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyspark_testframework-0.1.0.dist-info/METADATA

@@ -0,0 +1,104 @@
+Metadata-Version: 2.1
+Name: pyspark-testframework
+Version: 0.1.0
+Summary: Testframework for PySpark DataFrames
+Author-email: Woonstad Rotterdam <info@woonstadrotterdam.nl>, Tomer Gabay <tomer.gabay@woonstadrotterdam.nl>, Vincent van der Meij <vincent.van.der.meij@woonstadrotterdam.nl>, Tiddo Loos <tiddo.loos@woonstadrotterdam.nl>, Ben Verhees <ben.verhees@woonstadrotterdam.nl>
+License: MIT License
+        
+        Copyright (c) 2024 Woonstad Rotterdam
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/woonstadrotterdam/pyspark-testframework
+Project-URL: Issues, https://github.com/woonstadrotterdam/pyspark-testframework/issues
+Keywords: pyspark,dataframe,test,testframework
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9.5
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyspark >=3.3.4
+Provides-Extra: dev
+Requires-Dist: pyspark-testframework[test] ; extra == 'dev'
+Requires-Dist: ruff ==0.4.8 ; extra == 'dev'
+Provides-Extra: test
+Requires-Dist: pre-commit ==3.7.1 ; extra == 'test'
+Requires-Dist: pytest ==8.2.2 ; extra == 'test'
+
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+![MIT License](https://img.shields.io/badge/License-MIT-yellow.svg)
+![Build Status](https://github.com/woonstadrotterdam/pyspark-testframework/actions/workflows/cicd.yml/badge.svg)
+
+# pyspark-testframework
+
+⏳ **Work in progress**
+
+![](https://progress-bar.dev/100/?title=RegexTest&width=120)  
+![](https://progress-bar.dev/50/?title=IsInteger&width=120)  
+![](https://progress-bar.dev/50/?title=ValidEmail&width=113)  
+![](https://progress-bar.dev/0/?title=ContainsValue&width=95)  
+![](https://progress-bar.dev/0/?title=ValidValueRange&width=83)  
+![](https://progress-bar.dev/0/?title=ValidCategory&width=95)  
+![](https://progress-bar.dev/0/?title=CorrectValue&width=102)  
+![](<https://progress-bar.dev/0/?title=(...)&width=145>)
+
+The goal of the `pyspark-testframework` is to provide a simple way to create tests for PySpark DataFrames. The test results are returned in DataFrame format as well.
+
+## Example
+
+Input DataFrame:
+
+| primary_key | email                     |
+| ----------- | ------------------------- |
+| 1           | info@woonstadrotterdam.nl |
+| 2           | infowoonstadrotterdam.nl  |
+| 3           | @woonstadrotterdam.nl     |
+| 4           | dev@woonstadrotterdam.nl  |
+| 5           | Null                      |
+
+```python
+from testframework.tests import RegexTest
+
+email_regex = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"
+
+mail_tester = RegexTest(
+    name="ValidEmail",
+    pattern=email_regex
+)
+
+test_result = mail_tester.test(
+    df=df,
+    col="email",
+    nullable=False
+)
+
+test_result.show()
+```
+
+| primary_key | email\_\_ValidEmail |
+| ----------- | ------------------- |
+| 1           | True                |
+| 2           | False               |
+| 3           | False               |
+| 4           | True                |
+| 5           | False               |

pyspark_testframework-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+testframework/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+testframework/_version.py,sha256=IMl2Pr_Sy4LVRKy_Sm4CdwUl1Gryous6ncL96EMYsnM,411
+testframework/base.py,sha256=4L6UD6TgRiQMvCyJykvJKzXxhR8NXay7N2r7UTJmT-M,2854
+testframework/tests.py,sha256=IcI1O-A6HCMFPPkHKB4Q4B8d1hTx8PNU1aO3JDJ95ww,691
+testframework/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+testframework/utils/decorators.py,sha256=IuW0DXnYqFFGZqdiZQmr01MqTUQLni71MIHhC7MAVQ0,2427
+pyspark_testframework-0.1.0.dist-info/LICENSE,sha256=XHXrWnYTM8HAM7azt1zAhsP6aD-sM-2DPoUAfoYx57Y,1075
+pyspark_testframework-0.1.0.dist-info/METADATA,sha256=bW7590ROTcc824BtSR4sF141nnlpd2RjzZLOHhNzoOQ,4482
+pyspark_testframework-0.1.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+pyspark_testframework-0.1.0.dist-info/top_level.txt,sha256=_4NVWFQs5_tlaT7Uw1qDG1-F4Q8L9cQxmK9wvaHOtss,14
+pyspark_testframework-0.1.0.dist-info/RECORD,,

pyspark_testframework-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.43.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pyspark_testframework-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+testframework

testframework/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)

testframework/base.py

@@ -0,0 +1,88 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+
+from pyspark.sql import Column, DataFrame
+
+
+class DataQualityTest(ABC):
+    def __init__(self, name: str, description: Optional[str] = None):
+        self.name = name
+        self.description = description
+
+    @abstractmethod
+    def test(
+        self,
+        df: DataFrame,
+        col: str,
+        primary_key: Union[str, list[str]],
+        nullable: bool,
+        result_col: Optional[str] = None,
+    ) -> DataFrame:
+        pass
+
+    @abstractmethod
+    def _test_impl(self, df: DataFrame, col: str, nullable: bool) -> Column:
+        pass
+
+
+class Test(DataQualityTest):
+    """
+    Abstract base class for implementing various tests on DataFrame columns.
+    """
+
+    @abstractmethod
+    def _test_impl(self, df: DataFrame, col: str, nullable: bool) -> Column:
+        """
+        Abstract method for implementing the test logic on a specified column.
+
+        Args:
+            df (DataFrame): The DataFrame containing the column to test.
+            col (str): The name of the column to test.
+            nullable (bool): Flag indicating whether the column can contain null values.
+
+        Returns:
+            Column: The result of the test as a Spark SQL Column.
+        """
+        pass
+
+    def generate_result_col_name(self, col: str) -> str:
+        """
+        Generates a standardized name for the test result column.
+
+        Args:
+            col (str): The name of the column being tested.
+
+        Returns:
+            str: The name of the test result column.
+        """
+        return f"{col}__{self.name}"
+
+    def test(
+        self,
+        df: DataFrame,
+        col: str,
+        primary_key: Union[str, list[str]],
+        nullable: bool,
+        result_col: Optional[str] = None,
+    ) -> DataFrame:
+        """
+        Applies the test to the specified column of the DataFrame.
+
+        Args:
+            df (DataFrame): The DataFrame to test.
+            col (str): The name of the column to test.
+            primary_key (Union[str, list[str]]): The column name(s) of the primary key(s).
+            nullable (bool): Flag indicating whether the column is allowed to have Null values.
+            result_col (Optional[str]): The name of the column to store the test result. By default None. If None, a default name will be generated.
+
+        Returns:
+            DataFrame: A DataFrame with the test results for the specified column.
+        """
+        primary_key = [primary_key] if isinstance(primary_key, str) else primary_key
+        test_function = self._test_impl(df, col, nullable)
+        result_col = result_col if result_col else self.generate_result_col_name(col)
+
+        # Apply the test result to the DataFrame
+        return df.withColumn(result_col, test_function).select(
+            primary_key + [col, result_col]
+        )

testframework/tests.py

@@ -0,0 +1,21 @@
+from pyspark.sql import Column, DataFrame
+from pyspark.sql import functions as F
+from pyspark.sql.types import StringType
+
+from testframework.base import Test
+from testframework.utils.decorators import account_for_nullable, allowed_col_types
+
+
+class RegexTest(Test):
+    """
+    Test to check if a column's values match a regular expression pattern.
+    """
+
+    def __init__(self, name: str, pattern: str) -> None:
+        super().__init__(name=name)
+        self.pattern = pattern
+
+    @account_for_nullable
+    @allowed_col_types([StringType])
+    def _test_impl(self, df: DataFrame, col: str, nullable: bool) -> Column:
+        return F.regexp_extract(F.col(col), self.pattern, 0) != ""

testframework/utils/decorators.py

@@ -0,0 +1,71 @@
+from functools import wraps
+from typing import Callable
+
+import pyspark.sql.functions as F
+from pyspark.sql import Column, DataFrame
+from pyspark.sql.types import DataType
+
+from testframework.base import Test
+
+
+def account_for_nullable(
+    func: Callable[[Test, DataFrame, str, bool], Column],
+) -> Column:
+    """
+    A decorator for test methods that adjusts the return value based on the column's nullability
+
+    This decorator wraps a test method to handle nullable columns appropriately. If a column
+    contains null values, those values will be converted to True or False, depending on the `nullable` parameter
+    passed to the test method. The decorator ensures that the final result is a boolean column,
+    with True for values matching the test criteria, False for non-matching values.
+    """
+
+    @wraps(func)
+    def wrapper(test: Test, df: DataFrame, column: str, nullable: bool) -> Column:
+        # Call the original test implementation
+        test_result = func(test, df, column, nullable)
+
+        # Adjust the result based on the nullable flag and null values
+        if nullable:
+            return (
+                F.when(F.col(column).isNull(), F.lit(True))
+                .otherwise(test_result)
+                .cast("boolean")
+            )
+        else:
+            return (
+                F.when(F.col(column).isNull(), F.lit(False))
+                .otherwise(test_result)
+                .cast("boolean")
+            )
+
+    return wrapper
+
+
+def allowed_col_types(
+    expected_types: list[DataType],
+) -> Callable[
+    [Callable[[Test, DataFrame, str, bool], DataFrame]],
+    Callable[[Test, DataFrame, str, bool], DataFrame],
+]:
+    expected_types_tuple = tuple(
+        expected_types
+    )  # Convert list to tuple once outside the wrapper
+
+    def decorator(
+        func: Callable[[Test, DataFrame, str, bool], DataFrame],
+    ) -> Callable[[Test, DataFrame, str, bool], DataFrame]:
+        @wraps(func)
+        def wrapper(
+            test: Test, df: DataFrame, column: str, nullable: bool
+        ) -> DataFrame:
+            col_type = df.schema[column].dataType
+            if not isinstance(col_type, expected_types_tuple):
+                raise TypeError(
+                    f"Column '{column}' is not of expected types {expected_types}. Found type: {col_type}"
+                )
+            return func(test, df, column, nullable)
+
+        return wrapper
+
+    return decorator