snowpark-checkpoints-collectors 0.3.2__tar.gz → 0.3.3__tar.gz

{snowpark_checkpoints_collectors-0.3.2 → snowpark_checkpoints_collectors-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-checkpoints-collectors
-Version: 0.3.2
+Version: 0.3.3
 Summary: Snowpark column and table statistics collection
 Project-URL: Bug Tracker, https://github.com/snowflakedb/snowpark-checkpoints/issues
 Project-URL: Source code, https://github.com/snowflakedb/snowpark-checkpoints/

{snowpark_checkpoints_collectors-0.3.2 → snowpark_checkpoints_collectors-0.3.3}/src/snowflake/snowpark_checkpoints_collector/__version__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.3.2"
+__version__ = "0.3.3"

{snowpark_checkpoints_collectors-0.3.2 → snowpark_checkpoints_collectors-0.3.3}/src/snowflake/snowpark_checkpoints_collector/summary_stats_collector.py

@@ -23,9 +23,9 @@ import pandera as pa
 
 from pyspark.sql import DataFrame as SparkDataFrame
 from pyspark.sql.functions import col
+from pyspark.sql.types import BooleanType, FloatType, IntegerType, StructField
 from pyspark.sql.types import DoubleType as SparkDoubleType
 from pyspark.sql.types import StringType as SparkStringType
-from pyspark.sql.types import StructField
 
 from snowflake.snowpark_checkpoints_collector.collection_common import (
     CHECKPOINT_JSON_OUTPUT_FILE_NAME_FORMAT,
@@ -72,6 +72,14 @@ from snowflake.snowpark_checkpoints_collector.utils.telemetry import report_tele
 
 LOGGER = logging.getLogger(__name__)
 
+default_null_types = {
+    IntegerType(): 0,
+    FloatType(): 0.0,
+    SparkDoubleType(): 0.0,
+    SparkStringType(): "",
+    BooleanType(): False,
+}
+
 
 @log
 def collect_dataframe_checkpoint(
@@ -253,6 +261,7 @@ def _collect_dataframe_checkpoint_mode_schema(
     column_type_dict: dict[str, any],
     output_path: Optional[str] = None,
 ) -> None:
+    df = normalize_missing_values(df)
     sampled_df = df.sample(sample)
     if sampled_df.isEmpty():
         LOGGER.warning("Sampled DataFrame is empty. Collecting full DataFrame.")
@@ -327,6 +336,15 @@ def _collect_dataframe_checkpoint_mode_schema(
     )
 
 
+def normalize_missing_values(df: SparkDataFrame) -> SparkDataFrame:
+    """Normalize missing values in a PySpark DataFrame to ensure consistent handling of NA values."""
+    for field in df.schema.fields:
+        default_value = default_null_types.get(field.dataType, None)
+        if default_value is not None:
+            df = df.fillna({field.name: default_value})
+    return df
+
+
 def _get_spark_column_types(df: SparkDataFrame) -> dict[str, StructField]:
     schema = df.schema
     column_type_collection = {}

{snowpark_checkpoints_collectors-0.3.2 → snowpark_checkpoints_collectors-0.3.3}/test/integ/test_collect_df_mode_1.py

@@ -378,7 +378,7 @@ def test_collect_dataframe_with_null_values(
         )
 
     validate_checkpoint_file_output(output_path, checkpoint_name)
-    assert "Converting column 'age' to 'Int64' type" in caplog.messages
+    assert "Collecting column 'age' of type 'integer'" in caplog.messages
 
 
 def test_collect_sampled_dataframe(spark_session, output_path):

snowpark_checkpoints_collectors-0.3.3/test/integ/test_collect_df_mode_1_expected/test_dataframe_all_column_types_with_null_values.json

@@ -0,0 +1 @@
+{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"a": {"title": null, "description": null, "dtype": "bool", "nullable": false, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}, "b": {"title": null, "description": null, "dtype": "Int64", "nullable": true, "checks": {"in_range": {"min_value": 1, "max_value": 1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "c": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "d": {"title": null, "description": null, "dtype": "timedelta64[ns]", "nullable": true, "checks": {"in_range": {"min_value": 1123200000000000, "max_value": 1123200000000000, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "e": {"title": null, "description": null, "dtype": "float64", "nullable": false, "checks": {"in_range": {"min_value": 0.0, "max_value": 2.1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "f": {"title": null, "description": null, "dtype": "float32", "nullable": false, "checks": {"in_range": {"min_value": 0.0, "max_value": 3.109999895095825, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "g": {"title": null, "description": null, "dtype": "int32", "nullable": false, "checks": {"in_range": {"min_value": 0, "max_value": 4, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "h": {"title": null, "description": null, "dtype": "Int64", "nullable": true, "checks": {"in_range": {"min_value": 5, "max_value": 5, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "i": {"title": null, "description": null, "dtype": "Int64", "nullable": true, "checks": {"in_range": {"min_value": 6, "max_value": 6, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "j": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": {"str_length": {"min_value": 0, "max_value": 7}}, "unique": false, "coerce": false, "required": true, "regex": false}, "m": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": true, "checks": {"in_range": {"min_value": "2000-01-01 12:53:00", "max_value": "2000-01-01 12:53:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "n": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": true, "checks": {"in_range": {"min_value": "2000-01-01 12:00:00", "max_value": "2000-01-01 12:00:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "p": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "q": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "r": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "s": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "t": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 2.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "a", "type": "boolean", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "true_count": 1, "false_count": 2}, {"name": "b", "type": "byte", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 1, "max": 1, "mean": 1.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "c", "type": "date", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01", "max": "2000-01-01", "format": "%Y-%m-%d"}, {"name": "d", "type": "daytimeinterval", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "13 days, 0:00:00", "max": "13 days, 0:00:00"}, {"name": "e", "type": "double", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 0.0, "max": 2.1, "mean": 1.4000000000000001, "decimal_precision": 1, "margin_error": 1.2124355652982142}, {"name": "f", "type": "float", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 0.0, "max": 3.109999895095825, "mean": 2.073333263397217, "decimal_precision": 15, "margin_error": 1.7955592766132826}, {"name": "g", "type": "integer", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 0, "max": 4, "mean": 2.6666666666666665, "decimal_precision": 0, "margin_error": 2.309401076758503}, {"name": "h", "type": "long", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 5, "max": 5, "mean": 5.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "i", "type": "short", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 6, "max": 6, "mean": 6.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "j", "type": "string", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min_length": 0, "max_length": 7}, {"name": "m", "type": "timestamp", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01 12:53:00", "max": "2000-01-01 12:53:00", "format": "%Y-%m-%dT%H:%M:%S%z"}, {"name": "n", "type": "timestamp_ntz", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01 12:00:00", "max": "2000-01-01 12:00:00", "format": "%Y-%m-%dH:%M:%S"}, {"name": "o", "type": "decimal", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "3.1415161718190", "max": "3.1415161718190", "mean": "3.14151617181900000", "decimal_precision": 13}, {"name": "p", "type": "array", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "value_type": "string", "allow_null": true, "null_value_proportion": 10.0, "max_size": 5, "min_size": 0, "mean_size": 3.3333333333333335, "is_unique_size": false}, {"name": "q", "type": "binary", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "max_size": 6, "min_size": 0, "mean_size": 2.6666666666666665, "is_unique_size": false}, {"name": "r", "type": "map", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "key_type": "string", "value_type": "string", "allow_null": true, "null_value_proportion": 0.0, "max_size": 5, "min_size": 0, "mean_size": 3, "is_unique_size": false}, {"name": "s", "type": "void", "nullable": true, "rows_count": 3, "rows_not_null_count": 0, "rows_null_count": 3}, {"name": "t", "type": "struct", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "metadata": [{"name": "inner1", "type": "string", "nullable": false, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1}, {"name": "inner2", "type": "long", "nullable": true, "rows_count": 3, "rows_not_null_count": 1, "rows_null_count": 2}]}]}}

snowpark_checkpoints_collectors-0.3.3/test/integ/test_collect_df_mode_1_expected/test_df_with_null_values.json

@@ -0,0 +1 @@
+{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"name": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": {"str_length": {"min_value": 0, "max_value": 6}}, "unique": false, "coerce": false, "required": true, "regex": false}, "age": {"title": null, "description": null, "dtype": "int32", "nullable": false, "checks": {"in_range": {"min_value": 0, "max_value": 51, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "active": {"title": null, "description": null, "dtype": "bool", "nullable": false, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 4.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "name", "type": "string", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "min_length": 0, "max_length": 6}, {"name": "age", "type": "integer", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "min": 0, "max": 51, "mean": 21.8, "decimal_precision": 0, "margin_error": 22.241852440837743}, {"name": "active", "type": "boolean", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "true_count": 2, "false_count": 3}]}}

snowpark_checkpoints_collectors-0.3.3/test/integ/test_collect_df_mode_1_expected/test_df_with_only_null_values.json

@@ -0,0 +1 @@
+{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"Description": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": {"str_length": {"min_value": 0, "max_value": 0}}, "unique": false, "coerce": false, "required": true, "regex": false}, "Price": {"title": null, "description": null, "dtype": "float64", "nullable": false, "checks": {"in_range": {"min_value": 0.0, "max_value": 0.0, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "Active": {"title": null, "description": null, "dtype": "bool", "nullable": false, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 0.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "Description", "type": "string", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "min_length": 0, "max_length": 0}, {"name": "Price", "type": "double", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "min": 0.0, "max": 0.0, "mean": 0.0, "decimal_precision": 1, "margin_error": null}, {"name": "Active", "type": "boolean", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "true_count": 0, "false_count": 1}]}}

{snowpark_checkpoints_collectors-0.3.2 → snowpark_checkpoints_collectors-0.3.3}/test/unit/test_summary_stats_collector.py

@@ -22,10 +22,22 @@ from typing import get_type_hints
 from unittest.mock import MagicMock, patch
 
 import pytest
+from pyspark.sql import SparkSession
+from pyspark.sql.types import (
+    StructField,
+    StructType,
+    IntegerType,
+    FloatType,
+    DoubleType,
+    StringType,
+    BooleanType,
+    TimestampType,
+)
 
 from snowflake.snowpark_checkpoints_collector.summary_stats_collector import (
     collect_dataframe_checkpoint,
     generate_parquet_for_spark_df,
+    normalize_missing_values,
     xcollect_dataframe_checkpoint,
 )
 
@@ -98,3 +110,70 @@ def test_skip_collector_return_type_commutability():
     assert (
         collect_return == x_collect_return
     ), "The return type of collect_dataframe_checkpoint and xcollect_dataframe_checkpoint must be the same."
+
+
+@pytest.fixture(scope="module")
+def spark():
+    spark = SparkSession.builder.master("local[1]").appName("pytest").getOrCreate()
+    yield spark
+    spark.stop()
+
+
+def test_normalize_missing_values_integer_and_float(spark):
+    schema = StructType(
+        [
+            StructField("a", IntegerType()),
+            StructField("b", FloatType()),
+            StructField("c", DoubleType()),
+        ]
+    )
+    data = [
+        (None, None, None),
+        (1, 2.0, 3.0),
+    ]
+    df = spark.createDataFrame(data, schema)
+    result_df = normalize_missing_values(df)
+    result = result_df.collect()
+    assert result[0]["a"] == 0
+    assert result[0]["b"] == 0.0
+    assert result[0]["c"] == 0.0
+    assert result[1]["a"] == 1
+    assert result[1]["b"] == 2.0
+    assert result[1]["c"] == 3.0
+
+
+def test_normalize_missing_values_string_and_bool(spark):
+    schema = StructType(
+        [
+            StructField("s", StringType(), True),
+            StructField("b", BooleanType(), True),
+        ]
+    )
+    data = [
+        (None, None),
+        ("foo", True),
+    ]
+    df = spark.createDataFrame(data, schema)
+    result_df = normalize_missing_values(df)
+    result = result_df.collect()
+    assert result[0]["s"] == ""
+    assert result[0]["b"] is False
+    assert result[1]["s"] == "foo"
+    assert result[1]["b"] is True
+
+
+def test_normalize_missing_values_unhandled_type(spark):
+    schema = StructType(
+        [
+            StructField("t", TimestampType(), True),
+            StructField("i", IntegerType(), True),
+        ]
+    )
+    data = [
+        (None, None),
+    ]
+    df = spark.createDataFrame(data, schema)
+    result_df = normalize_missing_values(df)
+    result = result_df.collect()
+    assert result[0]["t"] is None
+    assert result[0]["i"] == 0

snowpark_checkpoints_collectors-0.3.2/test/integ/test_collect_df_mode_1_expected/test_dataframe_all_column_types_with_null_values.json

@@ -1 +0,0 @@
-{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"a": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}, "b": {"title": null, "description": null, "dtype": "Int64", "nullable": true, "checks": {"in_range": {"min_value": 1, "max_value": 1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "c": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "d": {"title": null, "description": null, "dtype": "timedelta64[ns]", "nullable": true, "checks": {"in_range": {"min_value": 1123200000000000, "max_value": 1123200000000000, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "e": {"title": null, "description": null, "dtype": "float64", "nullable": true, "checks": {"in_range": {"min_value": 2.1, "max_value": 2.1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "f": {"title": null, "description": null, "dtype": "float32", "nullable": true, "checks": {"in_range": {"min_value": 3.109999895095825, "max_value": 3.109999895095825, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "g": {"title": null, "description": null, "dtype": "Int64", "nullable": true, "checks": {"in_range": {"min_value": 4, "max_value": 4, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "h": {"title": null, "description": null, "dtype": "Int64", "nullable": true, "checks": {"in_range": {"min_value": 5, "max_value": 5, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "i": {"title": null, "description": null, "dtype": "Int64", "nullable": true, "checks": {"in_range": {"min_value": 6, "max_value": 6, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "j": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": {"str_length": {"min_value": 7, "max_value": 7}}, "unique": false, "coerce": false, "required": true, "regex": false}, "m": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": true, "checks": {"in_range": {"min_value": "2000-01-01 12:53:00", "max_value": "2000-01-01 12:53:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "n": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": true, "checks": {"in_range": {"min_value": "2000-01-01 12:00:00", "max_value": "2000-01-01 12:00:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "p": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "q": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "r": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "s": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "t": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 2.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "a", "type": "boolean", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "true_count": 1, "false_count": 1}, {"name": "b", "type": "byte", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 1, "max": 1, "mean": 1, "decimal_precision": 0, "margin_error": 0.0}, {"name": "c", "type": "date", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01", "max": "2000-01-01", "format": "%Y-%m-%d"}, {"name": "d", "type": "daytimeinterval", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "13 days, 0:00:00", "max": "13 days, 0:00:00"}, {"name": "e", "type": "double", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 2.1, "max": 2.1, "mean": 2.1, "decimal_precision": 1, "margin_error": 0.0}, {"name": "f", "type": "float", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 3.109999895095825, "max": 3.109999895095825, "mean": 3.109999895095825, "decimal_precision": 15, "margin_error": 0.0}, {"name": "g", "type": "integer", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 4, "max": 4, "mean": 4, "decimal_precision": 0, "margin_error": 0.0}, {"name": "h", "type": "long", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 5, "max": 5, "mean": 5, "decimal_precision": 0, "margin_error": 0.0}, {"name": "i", "type": "short", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 6, "max": 6, "mean": 6, "decimal_precision": 0, "margin_error": 0.0}, {"name": "j", "type": "string", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min_length": 7, "max_length": 7}, {"name": "m", "type": "timestamp", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01 12:53:00", "max": "2000-01-01 12:53:00", "format": "%Y-%m-%dT%H:%M:%S%z"}, {"name": "n", "type": "timestamp_ntz", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01 12:00:00", "max": "2000-01-01 12:00:00", "format": "%Y-%m-%dH:%M:%S"}, {"name": "o", "type": "decimal", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "3.1415161718190", "max": "3.1415161718190", "mean": "3.14151617181900000", "decimal_precision": 13}, {"name": "p", "type": "array", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "value_type": "string", "allow_null": true, "null_value_proportion": 10.0, "max_size": 5, "min_size": 0, "mean_size": 3.3333333333333335, "is_unique_size": false}, {"name": "q", "type": "binary", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "max_size": 6, "min_size": 0, "mean_size": 2.6666666666666665, "is_unique_size": false}, {"name": "r", "type": "map", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "key_type": "string", "value_type": "string", "allow_null": true, "null_value_proportion": 0.0, "max_size": 5, "min_size": 0, "mean_size": 3, "is_unique_size": false}, {"name": "s", "type": "void", "nullable": true, "rows_count": 3, "rows_not_null_count": 0, "rows_null_count": 3}, {"name": "t", "type": "struct", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "metadata": [{"name": "inner1", "type": "string", "nullable": false, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1}, {"name": "inner2", "type": "long", "nullable": true, "rows_count": 3, "rows_not_null_count": 1, "rows_null_count": 2}]}]}}

snowpark_checkpoints_collectors-0.3.2/test/integ/test_collect_df_mode_1_expected/test_df_with_null_values.json

@@ -1 +0,0 @@
-{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"name": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": {"str_length": {"min_value": 4, "max_value": 6}}, "unique": false, "coerce": false, "required": true, "regex": false}, "age": {"title": null, "description": null, "dtype": "Int64", "nullable": true, "checks": {"in_range": {"min_value": 23, "max_value": 51, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "active": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 4.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "name", "type": "string", "nullable": true, "rows_count": 5, "rows_not_null_count": 4, "rows_null_count": 1, "min_length": 4, "max_length": 6}, {"name": "age", "type": "integer", "nullable": true, "rows_count": 5, "rows_not_null_count": 3, "rows_null_count": 2, "min": 23, "max": 51, "mean": 36.333333333333336, "decimal_precision": 0, "margin_error": 14.047538337136986}, {"name": "active", "type": "boolean", "nullable": true, "rows_count": 5, "rows_not_null_count": 4, "rows_null_count": 1, "true_count": 2, "false_count": 2}]}}

snowpark_checkpoints_collectors-0.3.2/test/integ/test_collect_df_mode_1_expected/test_df_with_only_null_values.json

@@ -1 +0,0 @@
-{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"Description": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "Price": {"title": null, "description": null, "dtype": "float64", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "Active": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 0.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "Description", "type": "string", "nullable": true, "rows_count": 1, "rows_not_null_count": 0, "rows_null_count": 1}, {"name": "Price", "type": "double", "nullable": true, "rows_count": 1, "rows_not_null_count": 0, "rows_null_count": 1}, {"name": "Active", "type": "boolean", "nullable": true, "rows_count": 1, "rows_not_null_count": 0, "rows_null_count": 1}]}}