snowpark-checkpoints-collectors 0.3.2__py3-none-any.whl → 0.3.3__py3-none-any.whl

snowflake/snowpark_checkpoints_collector/__version__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.3.2"
+__version__ = "0.3.3"

snowflake/snowpark_checkpoints_collector/summary_stats_collector.py

@@ -23,9 +23,9 @@ import pandera as pa
 
 from pyspark.sql import DataFrame as SparkDataFrame
 from pyspark.sql.functions import col
+from pyspark.sql.types import BooleanType, FloatType, IntegerType, StructField
 from pyspark.sql.types import DoubleType as SparkDoubleType
 from pyspark.sql.types import StringType as SparkStringType
-from pyspark.sql.types import StructField
 
 from snowflake.snowpark_checkpoints_collector.collection_common import (
     CHECKPOINT_JSON_OUTPUT_FILE_NAME_FORMAT,
@@ -72,6 +72,14 @@ from snowflake.snowpark_checkpoints_collector.utils.telemetry import report_tele
 
 LOGGER = logging.getLogger(__name__)
 
+default_null_types = {
+    IntegerType(): 0,
+    FloatType(): 0.0,
+    SparkDoubleType(): 0.0,
+    SparkStringType(): "",
+    BooleanType(): False,
+}
+
 
 @log
 def collect_dataframe_checkpoint(
@@ -253,6 +261,7 @@ def _collect_dataframe_checkpoint_mode_schema(
     column_type_dict: dict[str, any],
     output_path: Optional[str] = None,
 ) -> None:
+    df = normalize_missing_values(df)
     sampled_df = df.sample(sample)
     if sampled_df.isEmpty():
         LOGGER.warning("Sampled DataFrame is empty. Collecting full DataFrame.")
@@ -327,6 +336,15 @@ def _collect_dataframe_checkpoint_mode_schema(
     )
 
 
+def normalize_missing_values(df: SparkDataFrame) -> SparkDataFrame:
+    """Normalize missing values in a PySpark DataFrame to ensure consistent handling of NA values."""
+    for field in df.schema.fields:
+        default_value = default_null_types.get(field.dataType, None)
+        if default_value is not None:
+            df = df.fillna({field.name: default_value})
+    return df
+
+
 def _get_spark_column_types(df: SparkDataFrame) -> dict[str, StructField]:
     schema = df.schema
     column_type_collection = {}

{snowpark_checkpoints_collectors-0.3.2.dist-info → snowpark_checkpoints_collectors-0.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-checkpoints-collectors
-Version: 0.3.2
+Version: 0.3.3
 Summary: Snowpark column and table statistics collection
 Project-URL: Bug Tracker, https://github.com/snowflakedb/snowpark-checkpoints/issues
 Project-URL: Source code, https://github.com/snowflakedb/snowpark-checkpoints/

{snowpark_checkpoints_collectors-0.3.2.dist-info → snowpark_checkpoints_collectors-0.3.3.dist-info}/RECORD

@@ -1,8 +1,8 @@
 snowflake/snowpark_checkpoints_collector/__init__.py,sha256=g4NemuA6Mj4O2jkK0yLQ8sEV3owHiiJnBEz_OWvlW1I,1179
-snowflake/snowpark_checkpoints_collector/__version__.py,sha256=1W0aBeLTL5Svy-qrNkZc6gAKtQLDbncpMyN2SlnJhoU,632
+snowflake/snowpark_checkpoints_collector/__version__.py,sha256=Ui7rlwZptAHMmUJyQDko698T0mUizyxibZ43mcDDaqk,632
 snowflake/snowpark_checkpoints_collector/collection_common.py,sha256=ff5vYffrTRjoJXZQvVQBaOlegAUj_vXBbl1IZidz8Qo,4510
 snowflake/snowpark_checkpoints_collector/singleton.py,sha256=7AgIHQBXVRvPBBCkmBplzkdrrm-xVWf_N8svzA2vF8E,836
-snowflake/snowpark_checkpoints_collector/summary_stats_collector.py,sha256=kRJpVRE9Iy_uqeIPT-__Aan-YLWxQbgSjkJ3w4LpvCc,17214
+snowflake/snowpark_checkpoints_collector/summary_stats_collector.py,sha256=eh--VdfPbpdrD0fyyB8bHMtKAwxDEQgRPOf5IaR6iL0,17824
 snowflake/snowpark_checkpoints_collector/collection_result/model/__init__.py,sha256=jZzx29WzrjH7C_6ZsBGoe4PxbW_oM4uIjySS1axIM34,1000
 snowflake/snowpark_checkpoints_collector/collection_result/model/collection_point_result.py,sha256=XelL7LughZpKl1B_6bJoKOc_PqQg3UleX6zdgVXqTus,2926
 snowflake/snowpark_checkpoints_collector/collection_result/model/collection_point_result_manager.py,sha256=EY6WIIXRbvkTYC4bQn7jFALHh7D2PirVoiLZ5Kq8dNs,2659
@@ -37,7 +37,7 @@ snowflake/snowpark_checkpoints_collector/utils/extra_config.py,sha256=3kVf6WVA-E
 snowflake/snowpark_checkpoints_collector/utils/file_utils.py,sha256=5ztlNCv9GdSktUvtdfydv86cCFcmSXCdD4axZXJrOQQ,5125
 snowflake/snowpark_checkpoints_collector/utils/logging_utils.py,sha256=yyi6X5DqKeTg0HRhvsH6ymYp2P0wbnyKIzI2RzrQS7k,2278
 snowflake/snowpark_checkpoints_collector/utils/telemetry.py,sha256=ueN9vM8j5YNax7jMcnEj_UrgGkoeMv_hJHVKjN7hiJE,32161
-snowpark_checkpoints_collectors-0.3.2.dist-info/METADATA,sha256=ueYk6-aMlhiKfvH0CZbqjiEjlxUP1VQwKDejX28ju30,6613
-snowpark_checkpoints_collectors-0.3.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-snowpark_checkpoints_collectors-0.3.2.dist-info/licenses/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-snowpark_checkpoints_collectors-0.3.2.dist-info/RECORD,,
+snowpark_checkpoints_collectors-0.3.3.dist-info/METADATA,sha256=3YCACSMQwOCF5e1lf8sX765aYYDBw38ukiEPbxb-7iA,6613
+snowpark_checkpoints_collectors-0.3.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+snowpark_checkpoints_collectors-0.3.3.dist-info/licenses/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+snowpark_checkpoints_collectors-0.3.3.dist-info/RECORD,,