snowpark-checkpoints-validators 0.1.0rc3__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/.gitignore +1 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/PKG-INFO +4 -7
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/README.md +1 -4
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/pyproject.toml +13 -5
- snowpark_checkpoints_validators-0.1.2/src/snowflake/snowpark_checkpoints/__init__.py +34 -0
- snowpark_checkpoints_validators-0.1.2/src/snowflake/snowpark_checkpoints/__version__.py +16 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/checkpoint.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/errors.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/job_context.py +14 -3
- snowpark_checkpoints_validators-0.1.2/src/snowflake/snowpark_checkpoints/singleton.py +23 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/snowpark_sampler.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/spark_migration.py +14 -3
- snowpark_checkpoints_validators-0.1.2/src/snowflake/snowpark_checkpoints/utils/__init__.py +14 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/utils/checkpoint_logger.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/utils/constants.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/utils/extra_config.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/utils/supported_types.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/utils/telemetry.py +36 -25
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/utils/utils_checks.py +16 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/validation_result_metadata.py +23 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/validation_results.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/e2eexample.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_compare_utils.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/test_pandera.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/test_spark_checkpoint.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/unit/test_extra_config.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/unit/test_spark_migration.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/unit/test_telemetry.py +156 -71
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/unit/test_utils_checks.py +14 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/unit/test_validation_result_metadata.py +41 -0
- snowpark_checkpoints_validators-0.1.0rc3/src/snowflake/snowpark_checkpoints/__init__.py +0 -23
- snowpark_checkpoints_validators-0.1.0rc3/src/snowflake/snowpark_checkpoints/singleton.py +0 -12
- snowpark_checkpoints_validators-0.1.0rc3/src/snowflake/snowpark_checkpoints/utils/__init__.py +0 -3
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/CHANGELOG.md +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/LICENSE +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/src/snowflake/snowpark_checkpoints/utils/pandera_check_manager.py +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/.coveragerc +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/df_mode_dataframe_mismatch_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/df_mode_dataframe_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/spark_checkpoint_df_fail_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/spark_checkpoint_df_pass_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/spark_checkpoint_limit_sample_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/spark_checkpoint_random_sample_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/spark_checkpoint_scalar_fail_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/spark_checkpoint_scalar_passing_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/test_df_check_custom_check_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/test_df_check_fail_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/test_df_check_from_file_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/test_df_check_skip_check_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/test_df_check_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/test_input_fail_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/test_input_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/test_output_fail_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/telemetry_expected/test_output_telemetry.json +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/integ/test_parquet.py +0 -0
- {snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/test/unit/test_pandera_check_manager.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: snowpark-checkpoints-validators
|
3
|
-
Version: 0.1.
|
3
|
+
Version: 0.1.2
|
4
4
|
Summary: Migration tools for Snowpark
|
5
5
|
Project-URL: Bug Tracker, https://github.com/snowflakedb/snowpark-checkpoints/issues
|
6
6
|
Project-URL: Source code, https://github.com/snowflakedb/snowpark-checkpoints/
|
@@ -29,8 +29,8 @@ Requires-Python: <3.12,>=3.9
|
|
29
29
|
Requires-Dist: pandera-report==0.1.2
|
30
30
|
Requires-Dist: pandera[io]==0.20.4
|
31
31
|
Requires-Dist: pyspark
|
32
|
-
Requires-Dist: snowflake-connector-python
|
33
|
-
Requires-Dist: snowflake-snowpark-python
|
32
|
+
Requires-Dist: snowflake-connector-python[pandas]
|
33
|
+
Requires-Dist: snowflake-snowpark-python==1.26.0
|
34
34
|
Provides-Extra: development
|
35
35
|
Requires-Dist: coverage>=7.6.7; extra == 'development'
|
36
36
|
Requires-Dist: deepdiff>=8.0.0; extra == 'development'
|
@@ -46,10 +46,7 @@ Description-Content-Type: text/markdown
|
|
46
46
|
# snowpark-checkpoints-validators
|
47
47
|
|
48
48
|
---
|
49
|
-
|
50
|
-
|
51
|
-
This package is on Private Preview.
|
52
|
-
|
49
|
+
##### This package is on Public Preview.
|
53
50
|
---
|
54
51
|
|
55
52
|
**snowpark-checkpoints-validators** is a package designed to validate Snowpark DataFrames against predefined schemas and checkpoints. This package ensures data integrity and consistency by performing schema and data validation checks at various stages of a Snowpark pipeline.
|
{snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/README.md
RENAMED
@@ -1,10 +1,7 @@
|
|
1
1
|
# snowpark-checkpoints-validators
|
2
2
|
|
3
3
|
---
|
4
|
-
|
5
|
-
|
6
|
-
This package is on Private Preview.
|
7
|
-
|
4
|
+
##### This package is on Public Preview.
|
8
5
|
---
|
9
6
|
|
10
7
|
**snowpark-checkpoints-validators** is a package designed to validate Snowpark DataFrames against predefined schemas and checkpoints. This package ensures data integrity and consistency by performing schema and data validation checks at various stages of a Snowpark pipeline.
|
{snowpark_checkpoints_validators-0.1.0rc3 → snowpark_checkpoints_validators-0.1.2}/pyproject.toml
RENAMED
@@ -26,8 +26,8 @@ classifiers = [
|
|
26
26
|
"Topic :: Scientific/Engineering :: Information Analysis",
|
27
27
|
]
|
28
28
|
dependencies = [
|
29
|
-
"snowflake-snowpark-python",
|
30
|
-
"snowflake-connector-python",
|
29
|
+
"snowflake-snowpark-python==1.26.0",
|
30
|
+
"snowflake-connector-python[pandas]",
|
31
31
|
"pyspark",
|
32
32
|
"pandera[io]==0.20.4",
|
33
33
|
"pandera-report==0.1.2",
|
@@ -64,8 +64,9 @@ development = [
|
|
64
64
|
"Bug Tracker" = "https://github.com/snowflakedb/snowpark-checkpoints/issues"
|
65
65
|
"Source code" = "https://github.com/snowflakedb/snowpark-checkpoints/"
|
66
66
|
|
67
|
+
|
67
68
|
[tool.hatch.version]
|
68
|
-
path = "__version__.py"
|
69
|
+
path = "src/snowflake/snowpark_checkpoints/__version__.py"
|
69
70
|
pattern = '^__version__ = "(?P<version>.*)"'
|
70
71
|
source = "regex"
|
71
72
|
|
@@ -77,15 +78,22 @@ where = ["src/"]
|
|
77
78
|
dev-mode-dirs = ['src']
|
78
79
|
directory = 'snowpark-checkpoints-validators'
|
79
80
|
|
81
|
+
[[tool.hatch.sources]]
|
82
|
+
dir = "src/snowflake/snowpark_checkpoints"
|
83
|
+
name = "snowpark-checkpoints-validators"
|
84
|
+
type = "package"
|
85
|
+
|
80
86
|
[tool.hatch.build.targets.wheel]
|
81
87
|
directory = "dist"
|
82
|
-
packages = [
|
88
|
+
packages = [
|
89
|
+
"src/snowflake",
|
90
|
+
]
|
83
91
|
|
84
92
|
[tool.hatch.build.targets.sdist]
|
85
93
|
directory = "dist"
|
86
94
|
exclude = ["/.github", "/.idea"]
|
87
95
|
include = [
|
88
|
-
'src
|
96
|
+
'src/**',
|
89
97
|
'README.md',
|
90
98
|
'LICENSE',
|
91
99
|
'test/',
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
|
16
|
+
from snowflake.snowpark_checkpoints.checkpoint import (
|
17
|
+
check_dataframe_schema,
|
18
|
+
check_output_schema,
|
19
|
+
check_input_schema,
|
20
|
+
validate_dataframe_checkpoint,
|
21
|
+
)
|
22
|
+
from snowflake.snowpark_checkpoints.job_context import SnowparkJobContext
|
23
|
+
from snowflake.snowpark_checkpoints.spark_migration import check_with_spark
|
24
|
+
from snowflake.snowpark_checkpoints.utils.constants import CheckpointMode
|
25
|
+
|
26
|
+
__all__ = [
|
27
|
+
"check_with_spark",
|
28
|
+
"SnowparkJobContext",
|
29
|
+
"check_dataframe_schema",
|
30
|
+
"check_output_schema",
|
31
|
+
"check_input_schema",
|
32
|
+
"validate_dataframe_checkpoint",
|
33
|
+
"CheckpointMode",
|
34
|
+
]
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
|
16
|
+
__version__ = "0.1.2"
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
# Wrapper around pandera which logs to snowflake
|
6
17
|
from typing import Any, Optional, Union
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
from typing import Optional
|
6
17
|
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
from datetime import datetime
|
6
17
|
from typing import Optional
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
|
16
|
+
|
17
|
+
class Singleton(type):
|
18
|
+
_instances = {}
|
19
|
+
|
20
|
+
def __call__(cls, *args, **kwargs):
|
21
|
+
if cls not in cls._instances:
|
22
|
+
cls._instances[cls] = super().__call__(*args, **kwargs)
|
23
|
+
return cls._instances[cls]
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
from typing import Optional
|
6
17
|
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
from typing import Callable, Optional, TypeVar
|
5
16
|
|
6
17
|
import pandas as pd
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
import logging
|
6
17
|
import threading
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
# Skip type
|
6
17
|
from enum import IntEnum
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
import os
|
6
17
|
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
from snowflake.snowpark_checkpoints.utils.constants import (
|
6
17
|
BINARY_TYPE,
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
import atexit
|
6
17
|
import datetime
|
@@ -54,7 +65,9 @@ VERSION_FILE_NAME = "__version__.py"
|
|
54
65
|
|
55
66
|
|
56
67
|
class TelemetryManager(TelemetryClient):
|
57
|
-
def __init__(
|
68
|
+
def __init__(
|
69
|
+
self, rest: Optional[SnowflakeRestful] = None, is_telemetry_enabled: bool = True
|
70
|
+
):
|
58
71
|
"""TelemetryManager class to log telemetry events."""
|
59
72
|
super().__init__(rest)
|
60
73
|
self.sc_folder_path = (
|
@@ -62,14 +75,15 @@ class TelemetryManager(TelemetryClient):
|
|
62
75
|
)
|
63
76
|
self.sc_sf_path_telemetry = "/telemetry/send"
|
64
77
|
self.sc_flush_size = 25
|
65
|
-
self.sc_is_enabled =
|
78
|
+
self.sc_is_enabled = is_telemetry_enabled
|
66
79
|
self.sc_is_testing = self._sc_is_telemetry_testing()
|
67
80
|
self.sc_memory_limit = 5 * 1024 * 1024
|
68
81
|
self._sc_upload_local_telemetry()
|
69
82
|
self.sc_log_batch = []
|
70
83
|
self.sc_hypothesis_input_events = []
|
71
84
|
self.sc_version = _get_version()
|
72
|
-
|
85
|
+
if rest:
|
86
|
+
atexit.register(self._sc_close_at_exit)
|
73
87
|
|
74
88
|
def set_sc_output_path(self, path: Path) -> None:
|
75
89
|
"""Set the output path for testing.
|
@@ -221,7 +235,7 @@ class TelemetryManager(TelemetryClient):
|
|
221
235
|
|
222
236
|
def _sc_upload_local_telemetry(self) -> None:
|
223
237
|
"""Send a request to the API to upload the local telemetry events."""
|
224
|
-
if not self.
|
238
|
+
if not self.sc_is_enabled or self.sc_is_testing or not self._rest:
|
225
239
|
return
|
226
240
|
batch = []
|
227
241
|
for file in self.sc_folder_path.glob("*.json"):
|
@@ -242,19 +256,6 @@ class TelemetryManager(TelemetryClient):
|
|
242
256
|
for file in self.sc_folder_path.glob("*.json"):
|
243
257
|
file.unlink()
|
244
258
|
|
245
|
-
def _sc_is_telemetry_enabled(self) -> bool:
|
246
|
-
"""Check if telemetry is enabled.
|
247
|
-
|
248
|
-
Returns:
|
249
|
-
bool: True if telemetry is enabled, False otherwise.
|
250
|
-
|
251
|
-
"""
|
252
|
-
if self._sc_is_telemetry_testing():
|
253
|
-
return True
|
254
|
-
if os.getenv("SNOWPARK_CHECKPOINTS_TELEMETRY_ENABLED") == "false":
|
255
|
-
return False
|
256
|
-
return self._rest is not None
|
257
|
-
|
258
259
|
def _sc_is_telemetry_testing(self) -> bool:
|
259
260
|
is_testing = os.getenv("SNOWPARK_CHECKPOINTS_TELEMETRY_TESTING") == "true"
|
260
261
|
if is_testing:
|
@@ -262,6 +263,7 @@ class TelemetryManager(TelemetryClient):
|
|
262
263
|
Path(os.getcwd()) / "snowpark-checkpoints-output" / "telemetry"
|
263
264
|
)
|
264
265
|
self.set_sc_output_path(local_telemetry_path)
|
266
|
+
self.sc_is_enabled = True
|
265
267
|
return is_testing
|
266
268
|
|
267
269
|
def _sc_is_telemetry_manager(self) -> bool:
|
@@ -365,7 +367,7 @@ def _get_version() -> str:
|
|
365
367
|
|
366
368
|
"""
|
367
369
|
try:
|
368
|
-
directory_levels_up =
|
370
|
+
directory_levels_up = 1
|
369
371
|
project_root = Path(__file__).resolve().parents[directory_levels_up]
|
370
372
|
version_file_path = project_root / VERSION_FILE_NAME
|
371
373
|
with open(version_file_path) as file:
|
@@ -427,10 +429,17 @@ def get_telemetry_manager() -> TelemetryManager:
|
|
427
429
|
TelemetryManager: The telemetry manager.
|
428
430
|
|
429
431
|
"""
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
432
|
+
try:
|
433
|
+
connection = Session.builder.getOrCreate().connection
|
434
|
+
if not hasattr(connection._telemetry, "_sc_is_telemetry_manager"):
|
435
|
+
connection._telemetry = TelemetryManager(
|
436
|
+
connection._rest, connection.telemetry_enabled
|
437
|
+
)
|
438
|
+
return connection._telemetry
|
439
|
+
except Exception:
|
440
|
+
telemetry_manager = TelemetryManager(None, is_telemetry_enabled=True)
|
441
|
+
telemetry_manager.sc_flush_size = 1
|
442
|
+
return telemetry_manager
|
434
443
|
|
435
444
|
|
436
445
|
def get_snowflake_schema_types(df: snowpark_dataframe.DataFrame) -> list[str]:
|
@@ -826,6 +835,8 @@ def report_telemetry(
|
|
826
835
|
except Exception as err:
|
827
836
|
func_exception = err
|
828
837
|
|
838
|
+
if os.getenv("SNOWPARK_CHECKPOINTS_TELEMETRY_ENABLED") == "false":
|
839
|
+
return result
|
829
840
|
telemetry_event = None
|
830
841
|
data = None
|
831
842
|
telemetry_m = None
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
import inspect
|
6
17
|
import json
|
@@ -348,6 +359,8 @@ def _update_validation_result(
|
|
348
359
|
|
349
360
|
pipeline_result_metadata = ValidationResultsMetadata(output_path)
|
350
361
|
|
362
|
+
pipeline_result_metadata.clean()
|
363
|
+
|
351
364
|
pipeline_result_metadata.add_validation_result(
|
352
365
|
ValidationResult(
|
353
366
|
timestamp=datetime.now().isoformat(),
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
import os
|
6
17
|
|
@@ -77,6 +88,15 @@ class ValidationResultsMetadata(metaclass=Singleton):
|
|
77
88
|
f"Error reading validation results file: {self.validation_results_file} \n {e}"
|
78
89
|
) from None
|
79
90
|
|
91
|
+
def clean(self):
|
92
|
+
"""Clean the validation results list.
|
93
|
+
|
94
|
+
This method empties the validation results list.
|
95
|
+
|
96
|
+
"""
|
97
|
+
if not os.path.exists(self.validation_results_file):
|
98
|
+
self.validation_results.results = []
|
99
|
+
|
80
100
|
def add_validation_result(self, validation_result: ValidationResult):
|
81
101
|
"""Add a validation result to the pipeline result list.
|
82
102
|
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
|
6
17
|
from pydantic import BaseModel
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
from pandera import DataFrameSchema, Column, Check
|
6
17
|
from snowflake.snowpark import DataFrame as SnowparkDataFrame
|
@@ -1,6 +1,17 @@
|
|
1
|
-
#
|
2
|
-
#
|
3
|
-
|
1
|
+
# Copyright 2025 Snowflake Inc.
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
3
|
+
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
4
15
|
|
5
16
|
import os
|
6
17
|
import json
|