pyspark-testframework 2.2.1__tar.gz → 2.2.3__tar.gz

{pyspark_testframework-2.2.1 → pyspark_testframework-2.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyspark-testframework
-Version: 2.2.1
+Version: 2.2.3
 Summary: Testframework for PySpark DataFrames
 Author-email: Woonstad Rotterdam <info@woonstadrotterdam.nl>, Tomer Gabay <tomer.gabay@woonstadrotterdam.nl>, Vincent van der Meij <vincent.van.der.meij@woonstadrotterdam.nl>, Tiddo Loos <tiddo.loos@woonstadrotterdam.nl>, Ben Verhees <ben.verhees@woonstadrotterdam.nl>
 License: MIT License
@@ -169,6 +169,8 @@ df_tester.test(
 
 **Run the `IntegerString` test on the _number_ column**
 
+By setting the `return_failed_rows` parameter to `True`, we can get only the rows that failed the test.
+
 ```python
 df_tester.test(
     col="house_number",
@@ -177,17 +179,14 @@ df_tester.test(
     ),
     nullable=True,  # nullable, hence null values are converted to True
     # description is optional, let's not define it for illustration purposes
+    return_failed_rows=True,  # only return the failed rows
 ).show()
 ```
 
     +-----------+------------+-------------------------------+
     |primary_key|house_number|house_number__ValidNumericRange|
     +-----------+------------+-------------------------------+
-    |          1|          27|                           true|
-    |          2|          31|                           true|
-    |          3|          27|                           true|
     |          4|          -3|                          false|
-    |          5|          13|                           true|
     +-----------+------------+-------------------------------+
 
 **Let's take a look at the test results of the DataFrame using the `.results` attribute.**

{pyspark_testframework-2.2.1 → pyspark_testframework-2.2.3}/README.md

@@ -121,6 +121,8 @@ df_tester.test(
 
 **Run the `IntegerString` test on the _number_ column**
 
+By setting the `return_failed_rows` parameter to `True`, we can get only the rows that failed the test.
+
 ```python
 df_tester.test(
     col="house_number",
@@ -129,17 +131,14 @@ df_tester.test(
     ),
     nullable=True,  # nullable, hence null values are converted to True
     # description is optional, let's not define it for illustration purposes
+    return_failed_rows=True,  # only return the failed rows
 ).show()
 ```
 
     +-----------+------------+-------------------------------+
     |primary_key|house_number|house_number__ValidNumericRange|
     +-----------+------------+-------------------------------+
-    |          1|          27|                           true|
-    |          2|          31|                           true|
-    |          3|          27|                           true|
     |          4|          -3|                          false|
-    |          5|          13|                           true|
     +-----------+------------+-------------------------------+
 
 **Let's take a look at the test results of the DataFrame using the `.results` attribute.**

{pyspark_testframework-2.2.1 → pyspark_testframework-2.2.3}/src/pyspark_testframework.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyspark-testframework
-Version: 2.2.1
+Version: 2.2.3
 Summary: Testframework for PySpark DataFrames
 Author-email: Woonstad Rotterdam <info@woonstadrotterdam.nl>, Tomer Gabay <tomer.gabay@woonstadrotterdam.nl>, Vincent van der Meij <vincent.van.der.meij@woonstadrotterdam.nl>, Tiddo Loos <tiddo.loos@woonstadrotterdam.nl>, Ben Verhees <ben.verhees@woonstadrotterdam.nl>
 License: MIT License
@@ -169,6 +169,8 @@ df_tester.test(
 
 **Run the `IntegerString` test on the _number_ column**
 
+By setting the `return_failed_rows` parameter to `True`, we can get only the rows that failed the test.
+
 ```python
 df_tester.test(
     col="house_number",
@@ -177,17 +179,14 @@ df_tester.test(
     ),
     nullable=True,  # nullable, hence null values are converted to True
     # description is optional, let's not define it for illustration purposes
+    return_failed_rows=True,  # only return the failed rows
 ).show()
 ```
 
     +-----------+------------+-------------------------------+
     |primary_key|house_number|house_number__ValidNumericRange|
     +-----------+------------+-------------------------------+
-    |          1|          27|                           true|
-    |          2|          31|                           true|
-    |          3|          27|                           true|
     |          4|          -3|                          false|
-    |          5|          13|                           true|
     +-----------+------------+-------------------------------+
 
 **Let's take a look at the test results of the DataFrame using the `.results` attribute.**

{pyspark_testframework-2.2.1 → pyspark_testframework-2.2.3}/src/testframework/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.2.1'
-__version_tuple__ = version_tuple = (2, 2, 1)
+__version__ = version = '2.2.3'
+__version_tuple__ = version_tuple = (2, 2, 3)

{pyspark_testframework-2.2.1 → pyspark_testframework-2.2.3}/src/testframework/dataquality/dataframe/dataframe_tester.py

@@ -138,6 +138,7 @@ class DataFrameTester:
         filter_rows: Optional[Column] = None,
         return_extra_cols: Optional[list[str]] = None,
         dummy_run: bool = False,
+        return_failed_rows: bool = False,
     ) -> DataFrame:
         """
         Executes a specific test on a given column of the DataFrame.
@@ -150,6 +151,7 @@ class DataFrameTester:
             filter_rows (Optional[Column]): Uses df.filter(filter_rows) to filter rows for which the test doesn't apply.
             return_extra_cols (Optional[list[str]]): Return extra columns from the original dataframe. Defaults to None.
             dummy_run (bool): If True, perform a dummy run without saving results. Defaults to False.
+            return_failed_rows (bool): If True, return only the rows where the test has failed. Defaults to False.
 
         Returns:
             DataFrame: The test results as a DataFrame.
@@ -194,6 +196,9 @@ class DataFrameTester:
             )
             self.results = new_test_results
 
+        if return_failed_rows:
+            test_result = test_result.filter(F.col(test_name) == F.lit(False))
+
         if return_extra_cols:
             return test_result.join(
                 self.df.select(*self.primary_key, *return_extra_cols),
@@ -222,6 +227,7 @@ class DataFrameTester:
         description: Optional[str] = None,
         fillna_value: Optional[Any] = None,
         return_extra_cols: Optional[list[str]] = None,
+        return_failed_rows: bool = False,
     ) -> DataFrame:
         """
         Adds custom test results to the test DataFrame.
@@ -232,6 +238,7 @@ class DataFrameTester:
             description (Optional[str]): Description of the test for reporting purposes.
             fillna_value (Optional[Any]): The value to fill nulls in the test result column after left joining on the primary_key. Defaults to None.
             return_extra_cols (Optional[list[str]]): Return extra columns from the original dataframe. Defaults to None.
+            return_failed_rows (bool): If True, return only the rows where the test has failed. Defaults to False.
 
         Returns:
             DataFrame: The updated test DataFrame with the custom test results.
@@ -278,14 +285,17 @@ class DataFrameTester:
 
         self.results = new_test_results
 
+        if return_failed_rows:
+            new_test_results = new_test_results.filter(F.col(name) == F.lit(False))
+
         if return_extra_cols:
-            return self.results.select(*self.primary_key, name).join(
+            return new_test_results.select(*self.primary_key, name).join(
                 self.df.select(*self.primary_key, *return_extra_cols),
                 on=self.primary_key,
                 how="left",
             )
 
-        return self.results.select(*self.primary_key, name)
+        return new_test_results.select(*self.primary_key, name)
 
     @property
     def summary(self) -> DataFrame:
@@ -408,7 +418,7 @@ class DataFrameTester:
         if not conditions:
             return self.results
 
-        combined_condition = reduce(lambda x, y: x | y, conditions)
+        combined_condition = reduce(lambda x, y: x & y, conditions)
 
         return self.results.filter(combined_condition)
 

{pyspark_testframework-2.2.1 → pyspark_testframework-2.2.3}/tests/dataquality/dataframe/test_DataFrameTester.py

@@ -260,3 +260,31 @@ def test_failed_tests(sample_df, spark):
     tester.test(col="value", test=ValidNumericRange(min_value=11), nullable=True)
 
     assert tester.failed_tests.count() == 1
+
+
+def test_return_failed_rows(sample_df, spark):
+    tester = DataFrameTester(df=sample_df, primary_key="id", spark=spark)
+    test_result = tester.test(
+        col="value",
+        test=ValidNumericRange(min_value=11),
+        nullable=True,
+        return_failed_rows=True,
+    )
+    assert test_result.count() == 1
+
+
+def test_return_failed_rows_custom_test(sample_df, spark):
+    tester = DataFrameTester(df=sample_df, primary_key="id", spark=spark)
+    custom_test_data = [(1, True), (2, False), (3, True), (4, False)]
+    custom_test_columns = ["id", "custom_test"]
+    custom_test_result = spark.createDataFrame(custom_test_data, custom_test_columns)
+
+    updated_results = tester.add_custom_test_result(
+        result=custom_test_result,
+        name="custom_test",
+        description="This is a custom test",
+        fillna_value=False,
+        return_failed_rows=True,
+    )
+
+    assert updated_results.count() == 2

{pyspark_testframework-2.2.1 → pyspark_testframework-2.2.3}/tutorial.ipynb

@@ -44,7 +44,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "24/08/12 14:43:33 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
+      "24/08/13 14:00:40 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
      ]
     },
     {
@@ -204,7 +204,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "**Run the `IntegerString` test on the _number_ column**"
+    "**Run the `IntegerString` test on the _number_ column**\n",
+    "\n",
+    "By setting the `return_failed_rows` parameter to `True`, we can get only the rows that failed the test."
    ]
   },
   {
@@ -219,11 +221,7 @@
       "+-----------+------------+-------------------------------+\n",
       "|primary_key|house_number|house_number__ValidNumericRange|\n",
       "+-----------+------------+-------------------------------+\n",
-      "|          1|          27|                           true|\n",
-      "|          2|          31|                           true|\n",
-      "|          3|          27|                           true|\n",
       "|          4|          -3|                          false|\n",
-      "|          5|          13|                           true|\n",
       "+-----------+------------+-------------------------------+\n",
       "\n"
      ]
@@ -237,6 +235,7 @@
     "    ),\n",
     "    nullable=True,  # nullable, hence null values are converted to True\n",
     "    # description is optional, let's not define it for illustration purposes\n",
+    "    return_failed_rows=True,  # only return the failed rows\n",
     ").show()"
    ]
   },
@@ -532,22 +531,10 @@
    "execution_count": 17,
    "metadata": {},
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Java HotSpot(TM) 64-Bit Server VM warning: CodeCache is full. Compiler has been disabled.\n",
-      "Java HotSpot(TM) 64-Bit Server VM warning: Try increasing the code cache size using -XX:ReservedCodeCacheSize=\n"
-     ]
-    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CodeCache: size=131072Kb used=36992Kb max_used=37010Kb free=94079Kb\n",
-      " bounds [0x000000010b1e8000, 0x000000010d658000, 0x00000001131e8000]\n",
-      " total_blobs=13403 nmethods=12493 adapters=821\n",
-      " compilation: disabled (not enough contiguous free space left)\n",
       "+-------------------------------+-------------------------------------------------------------+-------+--------+-----------------+--------+-----------------+\n",
       "|test                           |description                                                  |n_tests|n_passed|percentage_passed|n_failed|percentage_failed|\n",
       "+-------------------------------+-------------------------------------------------------------+-------+--------+-----------------+--------+-----------------+\n",
@@ -607,10 +594,22 @@
    "execution_count": 19,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Java HotSpot(TM) 64-Bit Server VM warning: CodeCache is full. Compiler has been disabled.\n",
+      "Java HotSpot(TM) 64-Bit Server VM warning: Try increasing the code cache size using -XX:ReservedCodeCacheSize=\n"
+     ]
+    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "CodeCache: size=131072Kb used=42866Kb max_used=43080Kb free=88205Kb\n",
+      " bounds [0x00000001069e8000, 0x0000000109458000, 0x000000010e9e8000]\n",
+      " total_blobs=14494 nmethods=13584 adapters=821\n",
+      " compilation: disabled (not enough contiguous free space left)\n",
       "+-----------+-----------------------+-------------------------------+-------------+\n",
       "|primary_key|street__ValidStreetName|house_number__ValidNumericRange|has_bath_room|\n",
       "+-----------+-----------------------+-------------------------------+-------------+\n",