snowpark-checkpoints-collectors 0.3.3__tar.gz → 0.4.0__tar.gz

{snowpark_checkpoints_collectors-0.3.3 → snowpark_checkpoints_collectors-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-checkpoints-collectors
-Version: 0.3.3
+Version: 0.4.0
 Summary: Snowpark column and table statistics collection
 Project-URL: Bug Tracker, https://github.com/snowflakedb/snowpark-checkpoints/issues
 Project-URL: Source code, https://github.com/snowflakedb/snowpark-checkpoints/

{snowpark_checkpoints_collectors-0.3.3 → snowpark_checkpoints_collectors-0.4.0}/pyproject.toml

@@ -129,7 +129,7 @@ exclude_lines = [
 
 [tool.hatch.envs.linter.scripts]
 check = [
-  'ruff check --fix .',
+  "echo 'Running linting checks...' && ruff check --config=../ruff.toml --statistics --verbose . || (echo '❌ LINTING FAILED: Please fix the above linting issues before proceeding. Use \"ruff check --config=../ruff.toml --fix .\" to auto-fix some issues, or fix them manually.' && exit 1)",
 ]
 
 [tool.hatch.envs.test.scripts]

{snowpark_checkpoints_collectors-0.3.3 → snowpark_checkpoints_collectors-0.4.0}/src/snowflake/snowpark_checkpoints_collector/__version__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.3.3"
+__version__ = "0.4.0"

{snowpark_checkpoints_collectors-0.3.3 → snowpark_checkpoints_collectors-0.4.0}/src/snowflake/snowpark_checkpoints_collector/collection_common.py

@@ -48,11 +48,12 @@ STRUCT_COLUMN_TYPE = "struct"
 TIMESTAMP_COLUMN_TYPE = "timestamp"
 TIMESTAMP_NTZ_COLUMN_TYPE = "timestamp_ntz"
 
-PANDAS_BOOLEAN_DTYPE = "bool"
+PANDAS_BOOLEAN_DTYPE = "boolean"
 PANDAS_DATETIME_DTYPE = "datetime64[ns]"
 PANDAS_FLOAT_DTYPE = "float64"
-PANDAS_INTEGER_DTYPE = "int64"
+PANDAS_INTEGER_DTYPE = "Int64"
 PANDAS_OBJECT_DTYPE = "object"
+PANDAS_STRING_DTYPE = "string[python]"
 PANDAS_TIMEDELTA_DTYPE = "timedelta64[ns]"
 
 NUMERIC_TYPE_COLLECTION = [
@@ -142,6 +143,8 @@ BACKSLASH_TOKEN = "\\"
 SLASH_TOKEN = "/"
 PYSPARK_NONE_SIZE_VALUE = -1
 PANDAS_LONG_TYPE = "Int64"
+PANDAS_STRING_TYPE = "string"
+PANDAS_FLOAT_TYPE = "float64"
 
 # ENVIRONMENT VARIABLES
 SNOWFLAKE_CHECKPOINT_CONTRACT_FILE_PATH_ENV_VAR = (

{snowpark_checkpoints_collectors-0.3.3 → snowpark_checkpoints_collectors-0.4.0}/src/snowflake/snowpark_checkpoints_collector/snow_connection_model/snow_connection.py

@@ -22,6 +22,8 @@ from pathlib import Path
 from typing import Callable, Optional
 
 from snowflake.snowpark import Session
+from snowflake.snowpark.functions import col, expr
+from snowflake.snowpark.types import TimestampType
 from snowflake.snowpark_checkpoints_collector.collection_common import (
     DOT_PARQUET_EXTENSION,
 )
@@ -195,9 +197,28 @@ class SnowConnection:
             stage_directory_path,
         )
         dataframe = self.session.read.parquet(path=stage_directory_path)
+        dataframe = convert_timestamps_to_utc_date(dataframe)
         LOGGER.info("Creating table '%s' from parquet files", table_name)
         dataframe.write.save_as_table(table_name=table_name, mode="overwrite")
 
     def _create_snowpark_session(self) -> Session:
         LOGGER.info("Creating a Snowpark session using the default connection")
         return Session.builder.getOrCreate()
+
+
+def convert_timestamps_to_utc_date(df):
+    """Convert all timestamp columns to UTC normalized timestamps.
+
+    Reading a parquet written by spark from a snowpark session modifies the original timestamps,
+    so this function normalizes timestamps for comparison.
+    """
+    new_cols = []
+    for field in df.schema.fields:
+        if isinstance(field.datatype, TimestampType):
+            utc_normalized_ts = expr(
+                f"convert_timezone('UTC', cast(to_date({field.name}) as timestamp_tz))"
+            ).alias(field.name)
+            new_cols.append(utc_normalized_ts)
+        else:
+            new_cols.append(col(field.name))
+    return df.select(new_cols)

{snowpark_checkpoints_collectors-0.3.3 → snowpark_checkpoints_collectors-0.4.0}/src/snowflake/snowpark_checkpoints_collector/summary_stats_collector.py

@@ -23,9 +23,15 @@ import pandera as pa
 
 from pyspark.sql import DataFrame as SparkDataFrame
 from pyspark.sql.functions import col
-from pyspark.sql.types import BooleanType, FloatType, IntegerType, StructField
+from pyspark.sql.types import BinaryType as SparkBinaryType
+from pyspark.sql.types import BooleanType as SparkBooleanType
+from pyspark.sql.types import DateType as SparkDateType
 from pyspark.sql.types import DoubleType as SparkDoubleType
+from pyspark.sql.types import FloatType as SparkFloatType
+from pyspark.sql.types import IntegerType as SparkIntegerType
 from pyspark.sql.types import StringType as SparkStringType
+from pyspark.sql.types import StructField as SparkStructField
+from pyspark.sql.types import TimestampType as SparkTimestampType
 
 from snowflake.snowpark_checkpoints_collector.collection_common import (
     CHECKPOINT_JSON_OUTPUT_FILE_NAME_FORMAT,
@@ -36,8 +42,10 @@ from snowflake.snowpark_checkpoints_collector.collection_common import (
     DOT_PARQUET_EXTENSION,
     INTEGER_TYPE_COLLECTION,
     NULL_COLUMN_TYPE,
+    PANDAS_FLOAT_TYPE,
     PANDAS_LONG_TYPE,
     PANDAS_OBJECT_TYPE_COLLECTION,
+    PANDAS_STRING_TYPE,
     CheckpointMode,
 )
 from snowflake.snowpark_checkpoints_collector.collection_result.model import (
@@ -73,11 +81,13 @@ from snowflake.snowpark_checkpoints_collector.utils.telemetry import report_tele
 LOGGER = logging.getLogger(__name__)
 
 default_null_types = {
-    IntegerType(): 0,
-    FloatType(): 0.0,
+    SparkIntegerType(): 0,
+    SparkFloatType(): 0.0,
     SparkDoubleType(): 0.0,
     SparkStringType(): "",
-    BooleanType(): False,
+    SparkBooleanType(): False,
+    SparkTimestampType(): None,
+    SparkDateType(): None,
 }
 
 
@@ -345,7 +355,7 @@ def normalize_missing_values(df: SparkDataFrame) -> SparkDataFrame:
     return df
 
 
-def _get_spark_column_types(df: SparkDataFrame) -> dict[str, StructField]:
+def _get_spark_column_types(df: SparkDataFrame) -> dict[str, SparkStructField]:
     schema = df.schema
     column_type_collection = {}
     for field in schema.fields:
@@ -475,14 +485,83 @@ def _to_pandas(sampled_df: SparkDataFrame) -> pandas.DataFrame:
     LOGGER.debug("Converting Spark DataFrame to Pandas DataFrame")
     pandas_df = sampled_df.toPandas()
     for field in sampled_df.schema.fields:
-        has_nan = pandas_df[field.name].isna().any()
         is_integer = field.dataType.typeName() in INTEGER_TYPE_COLLECTION
-        if has_nan and is_integer:
+        is_spark_string = isinstance(field.dataType, SparkStringType)
+        is_spark_binary = isinstance(field.dataType, SparkBinaryType)
+        is_spark_timestamp = isinstance(field.dataType, SparkTimestampType)
+        is_spark_float = isinstance(field.dataType, SparkFloatType)
+        is_spark_boolean = isinstance(field.dataType, SparkBooleanType)
+        is_spark_date = isinstance(field.dataType, SparkDateType)
+        if is_integer:
             LOGGER.debug(
-                "Converting column '%s' to '%s' type",
+                "Converting Spark integer column '%s' to Pandas nullable '%s' type",
                 field.name,
                 PANDAS_LONG_TYPE,
             )
-            pandas_df[field.name] = pandas_df[field.name].astype(PANDAS_LONG_TYPE)
+            pandas_df[field.name] = (
+                pandas_df[field.name].astype(PANDAS_LONG_TYPE).fillna(0)
+            )
+        elif is_spark_string or is_spark_binary:
+            LOGGER.debug(
+                "Converting Spark string column '%s' to Pandas nullable '%s' type",
+                field.name,
+                PANDAS_STRING_TYPE,
+            )
+            pandas_df[field.name] = (
+                pandas_df[field.name].astype(PANDAS_STRING_TYPE).fillna("")
+            )
+        elif is_spark_timestamp:
+            LOGGER.debug(
+                "Converting Spark timestamp column '%s' to UTC naive Pandas datetime",
+                field.name,
+            )
+            pandas_df[field.name] = convert_all_to_utc_naive(
+                pandas_df[field.name]
+            ).fillna(pandas.NaT)
+        elif is_spark_float:
+            LOGGER.debug(
+                "Converting Spark float column '%s' to Pandas nullable float",
+                field.name,
+            )
+            pandas_df[field.name] = (
+                pandas_df[field.name].astype(PANDAS_FLOAT_TYPE).fillna(0.0)
+            )
+        elif is_spark_boolean:
+            LOGGER.debug(
+                "Converting Spark boolean column '%s' to Pandas nullable boolean",
+                field.name,
+            )
+            pandas_df[field.name] = (
+                pandas_df[field.name].astype("boolean").fillna(False)
+            )
+        elif is_spark_date:
+            LOGGER.debug(
+                "Converting Spark date column '%s' to Pandas nullable datetime",
+                field.name,
+            )
+            pandas_df[field.name] = pandas_df[field.name].fillna(pandas.NaT)
 
     return pandas_df
+
+
+def convert_all_to_utc_naive(series: pandas.Series) -> pandas.Series:
+    """Convert all timezone-aware or naive timestamps in a series to UTC naive.
+
+    Naive timestamps are assumed to be in UTC and localized accordingly.
+    Timezone-aware timestamps are converted to UTC and then made naive.
+
+    Args:
+        series (pandas.Series): A Pandas Series of `pd.Timestamp` objects,
+            either naive or timezone-aware.
+
+    Returns:
+        pandas.Series: A Series of UTC-normalized naive timestamps (`tzinfo=None`).
+
+    """
+
+    def convert(ts):
+        if ts.tz is None:
+            ts = ts.tz_localize("UTC")
+        return ts.tz_convert("UTC").tz_localize(None)
+
+    return series.apply(convert)

{snowpark_checkpoints_collectors-0.3.3 → snowpark_checkpoints_collectors-0.4.0}/test/.coveragerc

@@ -1,4 +1,5 @@
 [run]
+source = src/
 omit =
     */__init__.py
     **/__init__.py
@@ -8,6 +9,10 @@ omit =
     */test/*
     */.venv/*
     */.env/*
+    */.hatch/*
+    **/.hatch/*
+    */site-packages/*
+    **/site-packages/*
     **/telemetry.py
 
 [report]

{snowpark_checkpoints_collectors-0.3.3 → snowpark_checkpoints_collectors-0.4.0}/test/integ/test_collect_df_mode_1.py

@@ -66,7 +66,7 @@ from snowflake.snowpark_checkpoints_collector.collection_common import (
     PANDAS_DATETIME_DTYPE,
     PANDAS_FLOAT_DTYPE,
     PANDAS_INTEGER_DTYPE,
-    PANDAS_OBJECT_DTYPE,
+    PANDAS_STRING_DTYPE,
     PANDAS_TIMEDELTA_DTYPE,
     PANDERA_COLUMN_TYPE_KEY,
     SNOWPARK_CHECKPOINTS_OUTPUT_DIRECTORY_NAME,
@@ -428,7 +428,7 @@ def test_collect_sampled_dataframe(spark_session, output_path):
     schema_contract_output_json = json.loads(schema_contract_output)
 
     pandera_column_type_collection_expected = [
-        PANDAS_OBJECT_DTYPE,
+        PANDAS_STRING_DTYPE,
         PANDAS_INTEGER_DTYPE,
         PANDAS_FLOAT_DTYPE,
         PANDAS_DATETIME_DTYPE,

snowpark_checkpoints_collectors-0.4.0/test/integ/test_collect_df_mode_1_expected/test_dataframe_all_column_types_with_null_values.json

@@ -0,0 +1 @@
+{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"a": {"title": null, "description": null, "dtype": "boolean", "nullable": false, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}, "b": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": {"in_range": {"min_value": 1, "max_value": 1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "c": {"title": null, "description": null, "dtype": "date", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "d": {"title": null, "description": null, "dtype": "timedelta64[ns]", "nullable": true, "checks": {"in_range": {"min_value": 1123200000000000, "max_value": 1123200000000000, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "e": {"title": null, "description": null, "dtype": "float64", "nullable": false, "checks": {"in_range": {"min_value": 0.0, "max_value": 2.1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "f": {"title": null, "description": null, "dtype": "float64", "nullable": false, "checks": {"in_range": {"min_value": 0.0, "max_value": 3.109999895095825, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "g": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": {"in_range": {"min_value": 0, "max_value": 4, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "h": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": {"in_range": {"min_value": 5, "max_value": 5, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "i": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": {"in_range": {"min_value": 6, "max_value": 6, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "j": {"title": null, "description": null, "dtype": "string[python]", "nullable": false, "checks": {"str_length": {"min_value": 0, "max_value": 7}}, "unique": false, "coerce": false, "required": true, "regex": false}, "m": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": true, "checks": {"in_range": {"min_value": "2000-01-01 12:53:00", "max_value": "2000-01-01 12:53:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "n": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": true, "checks": {"in_range": {"min_value": "2000-01-01 12:00:00", "max_value": "2000-01-01 12:00:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "p": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "q": {"title": null, "description": null, "dtype": "string[python]", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "r": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "s": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "t": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 2.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "a", "type": "boolean", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "true_count": 1, "false_count": 2}, {"name": "b", "type": "byte", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 1, "max": 1, "mean": 1.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "c", "type": "date", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01", "max": "2000-01-01", "format": "%Y-%m-%d"}, {"name": "d", "type": "daytimeinterval", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "13 days, 0:00:00", "max": "13 days, 0:00:00"}, {"name": "e", "type": "double", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 0.0, "max": 2.1, "mean": 1.4000000000000001, "decimal_precision": 1, "margin_error": 1.2124355652982142}, {"name": "f", "type": "float", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 0.0, "max": 3.109999895095825, "mean": 2.073333263397217, "decimal_precision": 15, "margin_error": 1.7955592766132826}, {"name": "g", "type": "integer", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 0, "max": 4, "mean": 2.6666666666666665, "decimal_precision": 0, "margin_error": 2.309401076758503}, {"name": "h", "type": "long", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 5, "max": 5, "mean": 5.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "i", "type": "short", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 6, "max": 6, "mean": 6.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "j", "type": "string", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min_length": 0, "max_length": 7}, {"name": "m", "type": "timestamp", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01 12:53:00", "max": "2000-01-01 12:53:00", "format": "%Y-%m-%dT%H:%M:%S%z"}, {"name": "n", "type": "timestamp_ntz", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01 12:00:00", "max": "2000-01-01 12:00:00", "format": "%Y-%m-%dH:%M:%S"}, {"name": "o", "type": "decimal", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "3.1415161718190", "max": "3.1415161718190", "mean": "3.14151617181900000", "decimal_precision": 13}, {"name": "p", "type": "array", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "value_type": "string", "allow_null": true, "null_value_proportion": 10.0, "max_size": 5, "min_size": 0, "mean_size": 3.3333333333333335, "is_unique_size": false}, {"name": "q", "type": "binary", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "max_size": 6, "min_size": 0, "mean_size": 2.6666666666666665, "is_unique_size": false}, {"name": "r", "type": "map", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "key_type": "string", "value_type": "string", "allow_null": true, "null_value_proportion": 0.0, "max_size": 5, "min_size": 0, "mean_size": 3, "is_unique_size": false}, {"name": "s", "type": "void", "nullable": true, "rows_count": 3, "rows_not_null_count": 0, "rows_null_count": 3}, {"name": "t", "type": "struct", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "metadata": [{"name": "inner1", "type": "string", "nullable": false, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1}, {"name": "inner2", "type": "long", "nullable": true, "rows_count": 3, "rows_not_null_count": 1, "rows_null_count": 2}]}]}}

snowpark_checkpoints_collectors-0.4.0/test/integ/test_collect_df_mode_1_expected/test_dataframe_with_unsupported_pandera_column_type.json

@@ -0,0 +1 @@
+{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"Name": {"title": null, "description": null, "dtype": "string[python]", "nullable": false, "checks": {"str_length": {"min_value": 2, "max_value": 2}}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 4.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "Name", "type": "string", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "min_length": 2, "max_length": 2}, {"name": "Value", "type": "decimal", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "min": "1.123456789", "max": "5.123450000", "mean": "3.1234552538000", "decimal_precision": 9}]}}

snowpark_checkpoints_collectors-0.4.0/test/integ/test_collect_df_mode_1_expected/test_df_with_null_values.json

@@ -0,0 +1 @@
+{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"name": {"title": null, "description": null, "dtype": "string[python]", "nullable": false, "checks": {"str_length": {"min_value": 0, "max_value": 6}}, "unique": false, "coerce": false, "required": true, "regex": false}, "age": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": {"in_range": {"min_value": 0, "max_value": 51, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "active": {"title": null, "description": null, "dtype": "boolean", "nullable": false, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 4.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "name", "type": "string", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "min_length": 0, "max_length": 6}, {"name": "age", "type": "integer", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "min": 0, "max": 51, "mean": 21.8, "decimal_precision": 0, "margin_error": 22.241852440837743}, {"name": "active", "type": "boolean", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "true_count": 2, "false_count": 3}]}}

snowpark_checkpoints_collectors-0.4.0/test/integ/test_collect_df_mode_1_expected/test_df_with_only_null_values.json

@@ -0,0 +1 @@
+{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"Description": {"title": null, "description": null, "dtype": "string[python]", "nullable": false, "checks": {"str_length": {"min_value": 0, "max_value": 0}}, "unique": false, "coerce": false, "required": true, "regex": false}, "Price": {"title": null, "description": null, "dtype": "float64", "nullable": false, "checks": {"in_range": {"min_value": 0.0, "max_value": 0.0, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "Active": {"title": null, "description": null, "dtype": "boolean", "nullable": false, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 0.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "Description", "type": "string", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "min_length": 0, "max_length": 0}, {"name": "Price", "type": "double", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "min": 0.0, "max": 0.0, "mean": 0.0, "decimal_precision": 1, "margin_error": null}, {"name": "Active", "type": "boolean", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "true_count": 0, "false_count": 1}]}}

snowpark_checkpoints_collectors-0.4.0/test/integ/test_collect_df_mode_1_expected/test_empty_df_with_schema.json

@@ -0,0 +1 @@
+{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"Code": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "Active": {"title": null, "description": null, "dtype": "boolean", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": null, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "Code", "type": "long", "nullable": true, "rows_count": 0, "rows_not_null_count": 0, "rows_null_count": 0}, {"name": "Active", "type": "boolean", "nullable": true, "rows_count": 0, "rows_not_null_count": 0, "rows_null_count": 0}]}}

snowpark_checkpoints_collectors-0.4.0/test/integ/test_collect_df_mode_1_expected/test_full_df.json

@@ -0,0 +1 @@
+{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"name": {"title": null, "description": null, "dtype": "string[python]", "nullable": false, "checks": {"str_length": {"min_value": 4, "max_value": 4}}, "unique": false, "coerce": false, "required": true, "regex": false}, "age": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": {"in_range": {"min_value": 21, "max_value": 50, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 2.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "name", "type": "string", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min_length": 4, "max_length": 4}, {"name": "age", "type": "integer", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 21, "max": 50, "mean": 35.0, "decimal_precision": 0, "margin_error": 14.52583904633395}]}}

snowpark_checkpoints_collectors-0.4.0/test/integ/test_collect_df_mode_1_expected/test_full_df_all_column_type.json

@@ -0,0 +1 @@
+{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"a": {"title": null, "description": null, "dtype": "boolean", "nullable": false, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}, "b": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": {"in_range": {"min_value": 1, "max_value": 1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "c": {"title": null, "description": null, "dtype": "date", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "d": {"title": null, "description": null, "dtype": "timedelta64[ns]", "nullable": false, "checks": {"in_range": {"min_value": 1123200000000000, "max_value": 1123200000000000, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "e": {"title": null, "description": null, "dtype": "float64", "nullable": false, "checks": {"in_range": {"min_value": 2.1, "max_value": 2.1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "f": {"title": null, "description": null, "dtype": "float64", "nullable": false, "checks": {"in_range": {"min_value": 3.109999895095825, "max_value": 3.109999895095825, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "g": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": {"in_range": {"min_value": 4, "max_value": 4, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "h": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": {"in_range": {"min_value": 5, "max_value": 5, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "i": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": {"in_range": {"min_value": 6, "max_value": 6, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "j": {"title": null, "description": null, "dtype": "string[python]", "nullable": false, "checks": {"str_length": {"min_value": 7, "max_value": 7}}, "unique": false, "coerce": false, "required": true, "regex": false}, "m": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": false, "checks": {"in_range": {"min_value": "2000-01-01 12:53:00", "max_value": "2000-01-01 12:53:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "n": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": false, "checks": {"in_range": {"min_value": "2000-01-01 12:00:00", "max_value": "2000-01-01 12:00:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "p": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "q": {"title": null, "description": null, "dtype": "string[python]", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "r": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "s": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "t": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 2.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "a", "type": "boolean", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "true_count": 2, "false_count": 1}, {"name": "b", "type": "byte", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 1, "max": 1, "mean": 1.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "c", "type": "date", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": "2000-01-01", "max": "2000-01-01", "format": "%Y-%m-%d"}, {"name": "d", "type": "daytimeinterval", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": "13 days, 0:00:00", "max": "13 days, 0:00:00"}, {"name": "e", "type": "double", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 2.1, "max": 2.1, "mean": 2.1, "decimal_precision": 1, "margin_error": 0.0}, {"name": "f", "type": "float", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 3.109999895095825, "max": 3.109999895095825, "mean": 3.109999895095825, "decimal_precision": 15, "margin_error": 0.0}, {"name": "g", "type": "integer", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 4, "max": 4, "mean": 4.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "h", "type": "long", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 5, "max": 5, "mean": 5.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "i", "type": "short", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 6, "max": 6, "mean": 6.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "j", "type": "string", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min_length": 7, "max_length": 7}, {"name": "m", "type": "timestamp", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": "2000-01-01 12:53:00", "max": "2000-01-01 12:53:00", "format": "%Y-%m-%dT%H:%M:%S%z"}, {"name": "n", "type": "timestamp_ntz", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": "2000-01-01 12:00:00", "max": "2000-01-01 12:00:00", "format": "%Y-%m-%dH:%M:%S"}, {"name": "o", "type": "decimal", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": "3.1415161718190", "max": "3.1415161718190", "mean": "3.14151617181900000", "decimal_precision": 13}, {"name": "p", "type": "array", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "value_type": "string", "allow_null": true, "null_value_proportion": 13.333333333333334, "max_size": 5, "min_size": 5, "mean_size": 5, "is_unique_size": true}, {"name": "q", "type": "binary", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "max_size": 6, "min_size": 2, "mean_size": 3.6666666666666665, "is_unique_size": false}, {"name": "r", "type": "map", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "key_type": "string", "value_type": "string", "allow_null": true, "null_value_proportion": 20.0, "max_size": 5, "min_size": 1, "mean_size": 3.3333333333333335, "is_unique_size": false}, {"name": "s", "type": "void", "nullable": true, "rows_count": 3, "rows_not_null_count": 0, "rows_null_count": 3}, {"name": "t", "type": "struct", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "metadata": [{"name": "inner1", "type": "string", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0}, {"name": "inner2", "type": "long", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0}]}]}}

snowpark_checkpoints_collectors-0.4.0/test/integ/test_collect_df_mode_1_expected/test_io_strategy.json

@@ -0,0 +1 @@
+{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"name": {"title": null, "description": null, "dtype": "string[python]", "nullable": false, "checks": {"str_length": {"min_value": 7, "max_value": 7}}, "unique": false, "coerce": false, "required": true, "regex": false}, "age": {"title": null, "description": null, "dtype": "Int64", "nullable": false, "checks": {"in_range": {"min_value": 21, "max_value": 21, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 0.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "name", "type": "string", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "min_length": 7, "max_length": 7}, {"name": "age", "type": "integer", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "min": 21, "max": 21, "mean": 21.0, "decimal_precision": 0, "margin_error": null}]}}

{snowpark_checkpoints_collectors-0.3.3 → snowpark_checkpoints_collectors-0.4.0}/test/unit/test_snow_connection.py

@@ -56,13 +56,17 @@ def test_create_snowflake_table_from_parquet(input_path):
             with mock.patch(
                 "snowflake.snowpark_checkpoints_collector.io_utils.io_default_strategy.IODefaultStrategy.read_bytes"
             ) as read_bytes_mock:
-                read_bytes_mock.return_value = b"test"
-                isfile_mock.return_value = True
-                glob_mock.return_value = [parquet_file_path]
-                snow_connection = SnowConnection(mocked_session)
-                snow_connection.create_snowflake_table_from_local_parquet(
-                    checkpoint_name, input_path, stage_path=checkpoint_name
-                )
+                with mock.patch(
+                    "snowflake.snowpark_checkpoints_collector.snow_connection_model.snow_connection.convert_timestamps_to_utc_date"
+                ) as mock_convert:
+                    mock_convert.return_value = mock_df
+                    read_bytes_mock.return_value = b"test"
+                    isfile_mock.return_value = True
+                    glob_mock.return_value = [parquet_file_path]
+                    snow_connection = SnowConnection(mocked_session)
+                    snow_connection.create_snowflake_table_from_local_parquet(
+                        checkpoint_name, input_path, stage_path=checkpoint_name
+                    )
 
     stage_name = stage_name.format(snow_connection.stage_id)
 

snowpark_checkpoints_collectors-0.3.3/test/integ/test_collect_df_mode_1_expected/test_dataframe_all_column_types_with_null_values.json

@@ -1 +0,0 @@
-{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"a": {"title": null, "description": null, "dtype": "bool", "nullable": false, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}, "b": {"title": null, "description": null, "dtype": "Int64", "nullable": true, "checks": {"in_range": {"min_value": 1, "max_value": 1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "c": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "d": {"title": null, "description": null, "dtype": "timedelta64[ns]", "nullable": true, "checks": {"in_range": {"min_value": 1123200000000000, "max_value": 1123200000000000, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "e": {"title": null, "description": null, "dtype": "float64", "nullable": false, "checks": {"in_range": {"min_value": 0.0, "max_value": 2.1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "f": {"title": null, "description": null, "dtype": "float32", "nullable": false, "checks": {"in_range": {"min_value": 0.0, "max_value": 3.109999895095825, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "g": {"title": null, "description": null, "dtype": "int32", "nullable": false, "checks": {"in_range": {"min_value": 0, "max_value": 4, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "h": {"title": null, "description": null, "dtype": "Int64", "nullable": true, "checks": {"in_range": {"min_value": 5, "max_value": 5, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "i": {"title": null, "description": null, "dtype": "Int64", "nullable": true, "checks": {"in_range": {"min_value": 6, "max_value": 6, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "j": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": {"str_length": {"min_value": 0, "max_value": 7}}, "unique": false, "coerce": false, "required": true, "regex": false}, "m": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": true, "checks": {"in_range": {"min_value": "2000-01-01 12:53:00", "max_value": "2000-01-01 12:53:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "n": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": true, "checks": {"in_range": {"min_value": "2000-01-01 12:00:00", "max_value": "2000-01-01 12:00:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "p": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "q": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "r": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "s": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "t": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 2.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "a", "type": "boolean", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "true_count": 1, "false_count": 2}, {"name": "b", "type": "byte", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 1, "max": 1, "mean": 1.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "c", "type": "date", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01", "max": "2000-01-01", "format": "%Y-%m-%d"}, {"name": "d", "type": "daytimeinterval", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "13 days, 0:00:00", "max": "13 days, 0:00:00"}, {"name": "e", "type": "double", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 0.0, "max": 2.1, "mean": 1.4000000000000001, "decimal_precision": 1, "margin_error": 1.2124355652982142}, {"name": "f", "type": "float", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 0.0, "max": 3.109999895095825, "mean": 2.073333263397217, "decimal_precision": 15, "margin_error": 1.7955592766132826}, {"name": "g", "type": "integer", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 0, "max": 4, "mean": 2.6666666666666665, "decimal_precision": 0, "margin_error": 2.309401076758503}, {"name": "h", "type": "long", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 5, "max": 5, "mean": 5.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "i", "type": "short", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": 6, "max": 6, "mean": 6.0, "decimal_precision": 0, "margin_error": 0.0}, {"name": "j", "type": "string", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min_length": 0, "max_length": 7}, {"name": "m", "type": "timestamp", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01 12:53:00", "max": "2000-01-01 12:53:00", "format": "%Y-%m-%dT%H:%M:%S%z"}, {"name": "n", "type": "timestamp_ntz", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "2000-01-01 12:00:00", "max": "2000-01-01 12:00:00", "format": "%Y-%m-%dH:%M:%S"}, {"name": "o", "type": "decimal", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "min": "3.1415161718190", "max": "3.1415161718190", "mean": "3.14151617181900000", "decimal_precision": 13}, {"name": "p", "type": "array", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "value_type": "string", "allow_null": true, "null_value_proportion": 10.0, "max_size": 5, "min_size": 0, "mean_size": 3.3333333333333335, "is_unique_size": false}, {"name": "q", "type": "binary", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "max_size": 6, "min_size": 0, "mean_size": 2.6666666666666665, "is_unique_size": false}, {"name": "r", "type": "map", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "key_type": "string", "value_type": "string", "allow_null": true, "null_value_proportion": 0.0, "max_size": 5, "min_size": 0, "mean_size": 3, "is_unique_size": false}, {"name": "s", "type": "void", "nullable": true, "rows_count": 3, "rows_not_null_count": 0, "rows_null_count": 3}, {"name": "t", "type": "struct", "nullable": true, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1, "metadata": [{"name": "inner1", "type": "string", "nullable": false, "rows_count": 3, "rows_not_null_count": 2, "rows_null_count": 1}, {"name": "inner2", "type": "long", "nullable": true, "rows_count": 3, "rows_not_null_count": 1, "rows_null_count": 2}]}]}}

snowpark_checkpoints_collectors-0.3.3/test/integ/test_collect_df_mode_1_expected/test_dataframe_with_unsupported_pandera_column_type.json

@@ -1 +0,0 @@
-{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"Name": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": {"str_length": {"min_value": 2, "max_value": 2}}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 4.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "Name", "type": "string", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "min_length": 2, "max_length": 2}, {"name": "Value", "type": "decimal", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "min": "1.123456789", "max": "5.123450000", "mean": "3.1234552538000", "decimal_precision": 9}]}}

snowpark_checkpoints_collectors-0.3.3/test/integ/test_collect_df_mode_1_expected/test_df_with_null_values.json

@@ -1 +0,0 @@
-{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"name": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": {"str_length": {"min_value": 0, "max_value": 6}}, "unique": false, "coerce": false, "required": true, "regex": false}, "age": {"title": null, "description": null, "dtype": "int32", "nullable": false, "checks": {"in_range": {"min_value": 0, "max_value": 51, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "active": {"title": null, "description": null, "dtype": "bool", "nullable": false, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 4.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "name", "type": "string", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "min_length": 0, "max_length": 6}, {"name": "age", "type": "integer", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "min": 0, "max": 51, "mean": 21.8, "decimal_precision": 0, "margin_error": 22.241852440837743}, {"name": "active", "type": "boolean", "nullable": true, "rows_count": 5, "rows_not_null_count": 5, "rows_null_count": 0, "true_count": 2, "false_count": 3}]}}

snowpark_checkpoints_collectors-0.3.3/test/integ/test_collect_df_mode_1_expected/test_df_with_only_null_values.json

@@ -1 +0,0 @@
-{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"Description": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": {"str_length": {"min_value": 0, "max_value": 0}}, "unique": false, "coerce": false, "required": true, "regex": false}, "Price": {"title": null, "description": null, "dtype": "float64", "nullable": false, "checks": {"in_range": {"min_value": 0.0, "max_value": 0.0, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "Active": {"title": null, "description": null, "dtype": "bool", "nullable": false, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 0.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "Description", "type": "string", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "min_length": 0, "max_length": 0}, {"name": "Price", "type": "double", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "min": 0.0, "max": 0.0, "mean": 0.0, "decimal_precision": 1, "margin_error": null}, {"name": "Active", "type": "boolean", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "true_count": 0, "false_count": 1}]}}

snowpark_checkpoints_collectors-0.3.3/test/integ/test_collect_df_mode_1_expected/test_empty_df_with_schema.json

@@ -1 +0,0 @@
-{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"Code": {"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "Active": {"title": null, "description": null, "dtype": "bool", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": null, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "Code", "type": "long", "nullable": true, "rows_count": 0, "rows_not_null_count": 0, "rows_null_count": 0}, {"name": "Active", "type": "boolean", "nullable": true, "rows_count": 0, "rows_not_null_count": 0, "rows_null_count": 0}]}}

snowpark_checkpoints_collectors-0.3.3/test/integ/test_collect_df_mode_1_expected/test_full_df.json

@@ -1 +0,0 @@
-{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"name": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": {"str_length": {"min_value": 4, "max_value": 4}}, "unique": false, "coerce": false, "required": true, "regex": false}, "age": {"title": null, "description": null, "dtype": "int32", "nullable": false, "checks": {"in_range": {"min_value": 21, "max_value": 50, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 2.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "name", "type": "string", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min_length": 4, "max_length": 4}, {"name": "age", "type": "integer", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 21, "max": 50, "mean": 35, "decimal_precision": 0, "margin_error": 14.52583904633395}]}}

snowpark_checkpoints_collectors-0.3.3/test/integ/test_collect_df_mode_1_expected/test_full_df_all_column_type.json

@@ -1 +0,0 @@
-{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"a": {"title": null, "description": null, "dtype": "bool", "nullable": false, "checks": {"isin": [true, false]}, "unique": false, "coerce": false, "required": true, "regex": false}, "b": {"title": null, "description": null, "dtype": "int8", "nullable": false, "checks": {"in_range": {"min_value": 1, "max_value": 1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "c": {"title": null, "description": null, "dtype": "date", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "d": {"title": null, "description": null, "dtype": "timedelta64[ns]", "nullable": false, "checks": {"in_range": {"min_value": 1123200000000000, "max_value": 1123200000000000, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "e": {"title": null, "description": null, "dtype": "float64", "nullable": false, "checks": {"in_range": {"min_value": 2.1, "max_value": 2.1, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "f": {"title": null, "description": null, "dtype": "float32", "nullable": false, "checks": {"in_range": {"min_value": 3.109999895095825, "max_value": 3.109999895095825, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "g": {"title": null, "description": null, "dtype": "int32", "nullable": false, "checks": {"in_range": {"min_value": 4, "max_value": 4, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "h": {"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"in_range": {"min_value": 5, "max_value": 5, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "i": {"title": null, "description": null, "dtype": "int16", "nullable": false, "checks": {"in_range": {"min_value": 6, "max_value": 6, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "j": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": {"str_length": {"min_value": 7, "max_value": 7}}, "unique": false, "coerce": false, "required": true, "regex": false}, "m": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": false, "checks": {"in_range": {"min_value": "2000-01-01 12:53:00", "max_value": "2000-01-01 12:53:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "n": {"title": null, "description": null, "dtype": "datetime64[ns]", "nullable": false, "checks": {"in_range": {"min_value": "2000-01-01 12:00:00", "max_value": "2000-01-01 12:00:00", "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}, "p": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "q": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "r": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "s": {"title": null, "description": null, "dtype": "object", "nullable": true, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}, "t": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": null, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 2.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "a", "type": "boolean", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "true_count": 2, "false_count": 1}, {"name": "b", "type": "byte", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 1, "max": 1, "mean": 1, "decimal_precision": 0, "margin_error": 0.0}, {"name": "c", "type": "date", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": "2000-01-01", "max": "2000-01-01", "format": "%Y-%m-%d"}, {"name": "d", "type": "daytimeinterval", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": "13 days, 0:00:00", "max": "13 days, 0:00:00"}, {"name": "e", "type": "double", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 2.1, "max": 2.1, "mean": 2.1, "decimal_precision": 1, "margin_error": 0.0}, {"name": "f", "type": "float", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 3.109999895095825, "max": 3.109999895095825, "mean": 3.109999895095825, "decimal_precision": 15, "margin_error": 0.0}, {"name": "g", "type": "integer", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 4, "max": 4, "mean": 4, "decimal_precision": 0, "margin_error": 0.0}, {"name": "h", "type": "long", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 5, "max": 5, "mean": 5, "decimal_precision": 0, "margin_error": 0.0}, {"name": "i", "type": "short", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": 6, "max": 6, "mean": 6, "decimal_precision": 0, "margin_error": 0.0}, {"name": "j", "type": "string", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min_length": 7, "max_length": 7}, {"name": "m", "type": "timestamp", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": "2000-01-01 12:53:00", "max": "2000-01-01 12:53:00", "format": "%Y-%m-%dT%H:%M:%S%z"}, {"name": "n", "type": "timestamp_ntz", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": "2000-01-01 12:00:00", "max": "2000-01-01 12:00:00", "format": "%Y-%m-%dH:%M:%S"}, {"name": "o", "type": "decimal", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "min": "3.1415161718190", "max": "3.1415161718190", "mean": "3.14151617181900000", "decimal_precision": 13}, {"name": "p", "type": "array", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "value_type": "string", "allow_null": true, "null_value_proportion": 13.333333333333334, "max_size": 5, "min_size": 5, "mean_size": 5, "is_unique_size": true}, {"name": "q", "type": "binary", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "max_size": 6, "min_size": 2, "mean_size": 3.6666666666666665, "is_unique_size": false}, {"name": "r", "type": "map", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "key_type": "string", "value_type": "string", "allow_null": true, "null_value_proportion": 20.0, "max_size": 5, "min_size": 1, "mean_size": 3.3333333333333335, "is_unique_size": false}, {"name": "s", "type": "void", "nullable": true, "rows_count": 3, "rows_not_null_count": 0, "rows_null_count": 3}, {"name": "t", "type": "struct", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0, "metadata": [{"name": "inner1", "type": "string", "nullable": false, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0}, {"name": "inner2", "type": "long", "nullable": true, "rows_count": 3, "rows_not_null_count": 3, "rows_null_count": 0}]}]}}

snowpark_checkpoints_collectors-0.3.3/test/integ/test_collect_df_mode_1_expected/test_io_strategy.json

@@ -1 +0,0 @@
-{"pandera_schema": {"schema_type": "dataframe", "version": "0.20.4", "columns": {"name": {"title": null, "description": null, "dtype": "object", "nullable": false, "checks": {"str_length": {"min_value": 7, "max_value": 7}}, "unique": false, "coerce": false, "required": true, "regex": false}, "age": {"title": null, "description": null, "dtype": "int32", "nullable": false, "checks": {"in_range": {"min_value": 21, "max_value": 21, "include_min": true, "include_max": true}}, "unique": false, "coerce": false, "required": true, "regex": false}}, "checks": null, "index": [{"title": null, "description": null, "dtype": "int64", "nullable": false, "checks": {"greater_than_or_equal_to": 0.0, "less_than_or_equal_to": 0.0}, "name": null, "unique": false, "coerce": false}], "dtype": null, "coerce": true, "strict": false, "name": null, "ordered": false, "unique": null, "report_duplicates": "all", "unique_column_names": false, "add_missing_columns": false, "title": null, "description": null}, "custom_data": {"columns": [{"name": "name", "type": "string", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "min_length": 7, "max_length": 7}, {"name": "age", "type": "integer", "nullable": true, "rows_count": 1, "rows_not_null_count": 1, "rows_null_count": 0, "min": 21, "max": 21, "mean": 21.0, "decimal_precision": 0, "margin_error": null}]}}