prophecy-libs 2.1.6.dev2__tar.gz → 2.1.7.dev1__tar.gz

{prophecy_libs-2.1.6.dev2 → prophecy_libs-2.1.7.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: prophecy_libs
-Version: 2.1.6.dev2
+Version: 2.1.7.dev1
 Summary: Helper library for prophecy generated code
 Home-page: https://github.com/SimpleDataLabsInc/prophecy-python-libs
 Author: UNKNOWN

{prophecy_libs-2.1.6.dev2 → prophecy_libs-2.1.7.dev1}/prophecy/config/config_base.py

@@ -9,7 +9,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 is_serverless = bool(int(os.environ.get("DATABRICKS_SERVERLESS_MODE_ENABLED", "0")))
-logger.info(f'is_serverless is {is_serverless}')
+logger.debug(f'is_serverless is {is_serverless}')
 
 
 class ProjectConfig:
@@ -551,4 +551,4 @@ class ConfigBase:
                     field_name = field.name
                     if hasattr(source_config, field_name):
                         attr_value = getattr(source_config, field_name)
-                        setattr(self, field_name, attr_value)
+                        setattr(self, field_name, attr_value)

{prophecy_libs-2.1.6.dev2 → prophecy_libs-2.1.7.dev1}/prophecy/utils/diff.py

@@ -22,6 +22,7 @@ class DiffKeys(enum.Enum):
     JOINED = "joined"
     SUMMARY = "summary"
     CLEANED = "cleaned"
+    MISMATCHED = "mismatched"
     EXPECTED = "expected"
     GENERATED = "generated"
     KEY_COLUMNS = "keyCols"
@@ -840,11 +841,15 @@ class DataFrameDiff:
             right_df=processed_expected_df,
         )
 
+        # Create mismatched dataframe by filtering for non-matching rows only
+        mismatched_df = clean_joined_df.filter(F.col("row_matches") == False)
+
         # Register the diff results with the full DataFrames.
         cls.COMPUTED_DIFFS[diff_key] = {
             DiffKeys.JOINED.value: joined_df,
             DiffKeys.SUMMARY.value: summary_df,
             DiffKeys.CLEANED.value: clean_joined_df,
+            DiffKeys.MISMATCHED.value: mismatched_df,  # Only non-matching rows
             DiffKeys.EXPECTED.value: expected_df,  # full expected DataFrame registered.
             DiffKeys.GENERATED.value: generated_df,  # full generated DataFrame registered.
             DiffKeys.KEY_COLUMNS.value: key_columns,

{prophecy_libs-2.1.6.dev2 → prophecy_libs-2.1.7.dev1}/prophecy/utils/utils.py

@@ -41,6 +41,12 @@ except:
     pass
 from pyspark.sql import *
 
+try:
+    from pyspark.sql.connect.session import SparkSession as ConnectSparkSession
+    SPARK_SESSION_TYPES = (SparkSession, ConnectSparkSession)
+except ImportError:
+    SPARK_SESSION_TYPES = (SparkSession,)
+
 from prophecy.utils.monitoring_utils import (
     capture_streams,
     monkey_patch_print,
@@ -86,7 +92,7 @@ if not logger.hasHandlers():
 
 
 is_serverless = is_serverless_env()
-logger.info(f"is_serverless is {is_serverless}")
+logger.debug(f"is_serverless is {is_serverless}")
 
 
 class TaskState:
@@ -2225,7 +2231,7 @@ def instrument(function):
             else:
                 spark = args[1]
         else:
-            if isinstance(args[0], SparkSession):
+            if isinstance(args[0], SPARK_SESSION_TYPES):
                 spark = args[0]
             else:
                 spark = args[1]

{prophecy_libs-2.1.6.dev2 → prophecy_libs-2.1.7.dev1}/prophecy_libs.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: prophecy-libs
-Version: 2.1.6.dev2
+Version: 2.1.7.dev1
 Summary: Helper library for prophecy generated code
 Home-page: https://github.com/SimpleDataLabsInc/prophecy-python-libs
 Author: UNKNOWN

{prophecy_libs-2.1.6.dev2 → prophecy_libs-2.1.7.dev1}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="prophecy_libs",
-    version="2.1.6.dev2",
+    version="2.1.7.dev1",
     url="https://github.com/SimpleDataLabsInc/prophecy-python-libs",
     packages=find_packages(exclude=["test.*", "test"]),
     package_data={