snowpark-checkpoints-collectors 0.1.2__tar.gz → 0.1.4__tar.gz

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-checkpoints-collectors
-Version: 0.1.2
+Version: 0.1.4
 Summary: Snowpark column and table statistics collection
 Project-URL: Bug Tracker, https://github.com/snowflakedb/snowpark-checkpoints/issues
 Project-URL: Source code, https://github.com/snowflakedb/snowpark-checkpoints/
@@ -27,19 +27,21 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: <3.12,>=3.9
 Requires-Dist: pandera[io]==0.20.4
-Requires-Dist: pyspark
 Requires-Dist: snowflake-connector-python
-Requires-Dist: snowflake-snowpark-python==1.26.0
+Requires-Dist: snowflake-snowpark-python>=1.23.0
 Provides-Extra: development
 Requires-Dist: coverage>=7.6.7; extra == 'development'
 Requires-Dist: deepdiff>=8.0.0; extra == 'development'
 Requires-Dist: hatchling==1.25.0; extra == 'development'
 Requires-Dist: pre-commit>=4.0.1; extra == 'development'
 Requires-Dist: pyarrow>=18.0.0; extra == 'development'
+Requires-Dist: pyspark>=3.5.0; extra == 'development'
 Requires-Dist: pytest-cov>=6.0.0; extra == 'development'
 Requires-Dist: pytest>=8.3.3; extra == 'development'
 Requires-Dist: setuptools>=70.0.0; extra == 'development'
 Requires-Dist: twine==5.1.1; extra == 'development'
+Provides-Extra: pyspark
+Requires-Dist: pyspark>=3.5.0; extra == 'pyspark'
 Description-Content-Type: text/markdown
 
 # snowpark-checkpoints-collectors
@@ -50,6 +52,18 @@ Description-Content-Type: text/markdown
 ---
 
 **snowpark-checkpoints-collector** package offers a function for extracting information from PySpark dataframes. We can then use that data to validate against the converted Snowpark dataframes to ensure that behavioral equivalence has been achieved.
+
+---
+## Install the library
+```bash
+pip install snowpark-checkpoints-collectors
+```
+This package requires PySpark to be installed in the same environment. If you do not have it, you can install PySpark alongside Snowpark Checkpoints by running the following command:
+```bash
+pip install "snowpark-checkpoints-collectors[pyspark]"
+```
+---
+
 ## Features
 
 - Schema inference collected data mode (Schema): This is the default mode, which leverages Pandera schema inference to obtain the metadata and checks that will be evaluated for the specified dataframe. This mode also collects custom data from columns of the DataFrame based on the PySpark type.

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/README.md

@@ -6,6 +6,18 @@
 ---
 
 **snowpark-checkpoints-collector** package offers a function for extracting information from PySpark dataframes. We can then use that data to validate against the converted Snowpark dataframes to ensure that behavioral equivalence has been achieved.
+
+---
+## Install the library
+```bash
+pip install snowpark-checkpoints-collectors
+```
+This package requires PySpark to be installed in the same environment. If you do not have it, you can install PySpark alongside Snowpark Checkpoints by running the following command:
+```bash
+pip install "snowpark-checkpoints-collectors[pyspark]"
+```
+---
+
 ## Features
 
 - Schema inference collected data mode (Schema): This is the default mode, which leverages Pandera schema inference to obtain the metadata and checks that will be evaluated for the specified dataframe. This mode also collects custom data from columns of the DataFrame based on the PySpark type.

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/pyproject.toml

@@ -26,9 +26,8 @@ classifiers = [
   "Topic :: Scientific/Engineering :: Information Analysis",
 ]
 dependencies = [
-  "snowflake-snowpark-python==1.26.0",
+  "snowflake-snowpark-python>=1.23.0",
   "snowflake-connector-python",
-  "pyspark",
   "pandera[io]==0.20.4",
 ]
 description = "Snowpark column and table statistics collection"
@@ -47,6 +46,9 @@ readme = "README.md"
 requires-python = '>=3.9,<3.12'
 
 [project.optional-dependencies]
+pyspark = [
+  "pyspark>=3.5.0",
+]
 development = [
   "pytest>=8.3.3",
   "pytest-cov>=6.0.0",
@@ -57,6 +59,7 @@ development = [
   "setuptools>=70.0.0",
   "pyarrow>=18.0.0",
   "deepdiff>=8.0.0",
+  "pyspark>=3.5.0",
 ]
 
 [project.urls]
@@ -130,8 +133,10 @@ check = [
 
 [tool.hatch.envs.test.scripts]
 check = [
-  "pip install -e ../snowpark-checkpoints-configuration",
-  'pytest -v --junitxml=test/outcome/test-results.xml --cov=. --cov-config=test/.coveragerc --cov-report=xml:test/outcome/coverage-{matrix:python:{env:PYTHON_VERSION:unset}}.xml {args:test} --cov-report=term --cov-report=json:test/outcome/coverage-{matrix:python:{env:PYTHON_VERSION:unset}}.json',
+  'python -m pip install --upgrade pip -q',
+  'pip install -q -e ../snowpark-checkpoints-configuration',
+  'pip list',
+  'pytest -vvv --junitxml=test/outcome/test-results.xml --cov=. --cov-branch --cov-config=test/.coveragerc --cov-report=xml:test/outcome/coverage-{matrix:python:{env:PYTHON_VERSION:unset}}.xml {args:test} --cov-report=term --cov-report=html:test/outcome/coverage-{matrix:python:{env:PYTHON_VERSION:unset}}.html --cov-report=json:test/outcome/coverage-{matrix:python:{env:PYTHON_VERSION:unset}}.json',
 ]
 
 coverage = [

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/src/snowflake/snowpark_checkpoints_collector/__version__.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.1.2"
+__version__ = "0.1.4"

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/src/snowflake/snowpark_checkpoints_collector/utils/telemetry.py

@@ -319,11 +319,11 @@ def _generate_event(
     if sc_version is not None:
         metadata["snowpark_checkpoints_version"] = sc_version
     message = {
-        "type": event_type,
+        "event_type": event_type,
+        "type": "snowpark-checkpoints",
         "event_name": event_name,
         "driver_type": "PythonConnector",
         "driver_version": SNOWFLAKE_CONNECTOR_VERSION,
-        "source": "snowpark-checkpoints",
         "metadata": metadata,
         "data": json.dumps(parameters_info or {}),
     }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_1_expected/test_dataframe_all_column_types_with_null_values_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_Schema",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc2",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1737734308350"
+    "timestamp": "1738950259123"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_1_expected/test_dataframe_with_unsupported_pandera_column_type_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_Schema",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc2",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1737734290963"
+    "timestamp": "1738950240209"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_1_expected/test_df_with_null_values_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_Schema",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc2",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1737734292871"
+    "timestamp": "1738950242185"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_1_expected/test_df_with_only_null_values_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_Schema",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc2",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1737734298407"
+    "timestamp": "1738950248289"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_1_expected/test_empty_df_with_object_column_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_Schema",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc2",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1737734289535"
+    "timestamp": "1738950238654"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_1_expected/test_empty_df_with_schema_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_Schema",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc2",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1737734288293"
+    "timestamp": "1738950237212"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_1_expected/test_full_df_all_column_type_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_Schema",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc2",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1737734286969"
+    "timestamp": "1738950235933"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_1_expected/test_full_df_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_Schema",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc2",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1737734275410"
+    "timestamp": "1738950224176"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_2_expected/test_collect_checkpoint_mode_2_parquet_directory _telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_DF",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc3",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1738085687445"
+    "timestamp": "1738950053193"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_2_expected/test_collect_checkpoint_mode_2_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_DF",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc3",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1738085687445"
+    "timestamp": "1738950053193"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_2_expected/test_collect_empty_dataframe_with_schema_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_DF",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc3",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1738085687445"
+    "timestamp": "1738950053193"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_2_expected/test_collect_invalid_mode_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_DF",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc3",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1738085687445"
+    "timestamp": "1738950053193"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_2_expected/test_generate_parquet_for_spark_df_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_DF",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc3",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1738085687445"
+    "timestamp": "1738950053193"
 }

{snowpark_checkpoints_collectors-0.1.2 → snowpark_checkpoints_collectors-0.1.4}/test/integ/test_collect_df_mode_2_expected/test_spark_df_mode_dataframe_telemetry.json

@@ -4,15 +4,15 @@
         "driver_type": "PythonConnector",
         "driver_version": "3.12.4",
         "event_name": "DataFrame_Collection_DF",
+        "event_type": "info",
         "metadata": {
-            "device_id": "0471186eb1a6ba58c82e97d19006a08b163bc0056c05e6f770090eade2c3a809",
+            "device_id": "a16562762b83c9f0b6ffa4649c0b6c20348ccfcf49d1144fc4c6557dd1c683ec",
             "os_version": "darwin",
             "python_version": "3.11.11",
-            "snowpark_checkpoints_version": "0.1.0rc3",
+            "snowpark_checkpoints_version": "0.1.2",
             "snowpark_version": "1.26.0"
         },
-        "source": "snowpark-checkpoints",
-        "type": "info"
+        "type": "snowpark-checkpoints"
     },
-    "timestamp": "1738085687445"
+    "timestamp": "1738950053193"
 }