snowpark-checkpoints-collectors 0.2.0rc1__py3-none-any.whl → 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. snowflake/snowpark_checkpoints_collector/__init__.py +30 -0
  2. snowflake/snowpark_checkpoints_collector/__version__.py +16 -0
  3. snowflake/snowpark_checkpoints_collector/collection_common.py +160 -0
  4. snowflake/snowpark_checkpoints_collector/collection_result/model/__init__.py +24 -0
  5. snowflake/snowpark_checkpoints_collector/collection_result/model/collection_point_result.py +91 -0
  6. snowflake/snowpark_checkpoints_collector/collection_result/model/collection_point_result_manager.py +74 -0
  7. snowflake/snowpark_checkpoints_collector/column_collection/__init__.py +22 -0
  8. snowflake/snowpark_checkpoints_collector/column_collection/column_collector_manager.py +276 -0
  9. snowflake/snowpark_checkpoints_collector/column_collection/model/__init__.py +75 -0
  10. snowflake/snowpark_checkpoints_collector/column_collection/model/array_column_collector.py +113 -0
  11. snowflake/snowpark_checkpoints_collector/column_collection/model/binary_column_collector.py +87 -0
  12. snowflake/snowpark_checkpoints_collector/column_collection/model/boolean_column_collector.py +71 -0
  13. snowflake/snowpark_checkpoints_collector/column_collection/model/column_collector_base.py +95 -0
  14. snowflake/snowpark_checkpoints_collector/column_collection/model/date_column_collector.py +74 -0
  15. snowflake/snowpark_checkpoints_collector/column_collection/model/day_time_interval_column_collector.py +67 -0
  16. snowflake/snowpark_checkpoints_collector/column_collection/model/decimal_column_collector.py +92 -0
  17. snowflake/snowpark_checkpoints_collector/column_collection/model/empty_column_collector.py +88 -0
  18. snowflake/snowpark_checkpoints_collector/column_collection/model/map_column_collector.py +120 -0
  19. snowflake/snowpark_checkpoints_collector/column_collection/model/null_column_collector.py +49 -0
  20. snowflake/snowpark_checkpoints_collector/column_collection/model/numeric_column_collector.py +108 -0
  21. snowflake/snowpark_checkpoints_collector/column_collection/model/string_column_collector.py +70 -0
  22. snowflake/snowpark_checkpoints_collector/column_collection/model/struct_column_collector.py +102 -0
  23. snowflake/snowpark_checkpoints_collector/column_collection/model/timestamp_column_collector.py +75 -0
  24. snowflake/snowpark_checkpoints_collector/column_collection/model/timestamp_ntz_column_collector.py +75 -0
  25. snowflake/snowpark_checkpoints_collector/column_pandera_checks/__init__.py +20 -0
  26. snowflake/snowpark_checkpoints_collector/column_pandera_checks/pandera_column_checks_manager.py +241 -0
  27. snowflake/snowpark_checkpoints_collector/singleton.py +23 -0
  28. snowflake/snowpark_checkpoints_collector/snow_connection_model/__init__.py +20 -0
  29. snowflake/snowpark_checkpoints_collector/snow_connection_model/snow_connection.py +201 -0
  30. snowflake/snowpark_checkpoints_collector/summary_stats_collector.py +410 -0
  31. snowflake/snowpark_checkpoints_collector/utils/checkpoint_name_utils.py +53 -0
  32. snowflake/snowpark_checkpoints_collector/utils/extra_config.py +119 -0
  33. snowflake/snowpark_checkpoints_collector/utils/file_utils.py +132 -0
  34. snowflake/snowpark_checkpoints_collector/utils/logging_utils.py +67 -0
  35. snowflake/snowpark_checkpoints_collector/utils/telemetry.py +889 -0
  36. snowpark_checkpoints_collectors-0.2.1.dist-info/METADATA +158 -0
  37. snowpark_checkpoints_collectors-0.2.1.dist-info/RECORD +39 -0
  38. {snowpark_checkpoints_collectors-0.2.0rc1.dist-info → snowpark_checkpoints_collectors-0.2.1.dist-info}/licenses/LICENSE +0 -25
  39. snowpark_checkpoints_collectors-0.2.0rc1.dist-info/METADATA +0 -347
  40. snowpark_checkpoints_collectors-0.2.0rc1.dist-info/RECORD +0 -4
  41. {snowpark_checkpoints_collectors-0.2.0rc1.dist-info → snowpark_checkpoints_collectors-0.2.1.dist-info}/WHEEL +0 -0
@@ -0,0 +1,20 @@
1
+ # Copyright 2025 Snowflake Inc.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ __all__ = ["PanderaColumnChecksManager"]
17
+
18
+ from snowflake.snowpark_checkpoints_collector.column_pandera_checks.pandera_column_checks_manager import (
19
+ PanderaColumnChecksManager,
20
+ )
@@ -0,0 +1,241 @@
1
+ # Copyright 2025 Snowflake Inc.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ import logging
17
+
18
+ import pandas as pd
19
+
20
+ from pandera import Check, Column
21
+ from pyspark.sql import DataFrame as SparkDataFrame
22
+ from pyspark.sql.functions import col as spark_col
23
+ from pyspark.sql.functions import length as spark_length
24
+ from pyspark.sql.functions import max as spark_max
25
+ from pyspark.sql.functions import min as spark_min
26
+
27
+ from snowflake.snowpark_checkpoints_collector.collection_common import (
28
+ BETWEEN_CHECK_ERROR_MESSAGE_FORMAT,
29
+ BOOLEAN_COLUMN_TYPE,
30
+ BYTE_COLUMN_TYPE,
31
+ COLUMN_MAX_KEY,
32
+ COLUMN_MIN_KEY,
33
+ DAYTIMEINTERVAL_COLUMN_TYPE,
34
+ DOUBLE_COLUMN_TYPE,
35
+ FLOAT_COLUMN_TYPE,
36
+ INTEGER_COLUMN_TYPE,
37
+ LONG_COLUMN_TYPE,
38
+ SHORT_COLUMN_TYPE,
39
+ STRING_COLUMN_TYPE,
40
+ TIMESTAMP_COLUMN_TYPE,
41
+ TIMESTAMP_NTZ_COLUMN_TYPE,
42
+ )
43
+
44
+
45
+ LOGGER = logging.getLogger(__name__)
46
+
47
+
48
+ def collector_register(cls):
49
+ """Decorate a class with the checks mechanism.
50
+
51
+ Args:
52
+ cls: The class to decorate.
53
+
54
+ Returns:
55
+ The class to decorate.
56
+
57
+ """
58
+ LOGGER.debug("Starting to register checks from class %s", cls.__name__)
59
+ cls._collectors = {}
60
+ for method_name in dir(cls):
61
+ method = getattr(cls, method_name)
62
+ if hasattr(method, "_column_type"):
63
+ col_type_collection = method._column_type
64
+ for col_type in col_type_collection:
65
+ cls._collectors[col_type] = method_name
66
+ LOGGER.debug(
67
+ "Registered check '%s' for column type '%s'", method_name, col_type
68
+ )
69
+ return cls
70
+
71
+
72
+ def column_register(*args):
73
+ """Decorate a method to register it in the checks mechanism based on column type.
74
+
75
+ Args:
76
+ args: the column type to register.
77
+
78
+ Returns:
79
+ The wrapper.
80
+
81
+ """
82
+
83
+ def wrapper(func):
84
+ has_arguments = len(args) > 0
85
+ if has_arguments:
86
+ func._column_type = args
87
+ return func
88
+
89
+ return wrapper
90
+
91
+
92
+ @collector_register
93
+ class PanderaColumnChecksManager:
94
+
95
+ """Manage class for Pandera column checks based on type."""
96
+
97
+ def add_checks_column(
98
+ self,
99
+ clm_name: str,
100
+ clm_type: str,
101
+ pyspark_df: SparkDataFrame,
102
+ pandera_column: Column,
103
+ ) -> None:
104
+ """Add checks to Pandera column based on the column type.
105
+
106
+ Args:
107
+ clm_name (str): the name of the column.
108
+ clm_type (str): the type of the column.
109
+ pyspark_df (pyspark.sql.DataFrame): the DataFrame.
110
+ pandera_column (pandera.Column): the Pandera column.
111
+
112
+ """
113
+ if clm_type not in self._collectors:
114
+ LOGGER.debug(
115
+ "No Pandera checks found for column '%s' of type '%s'. Skipping checks for this column.",
116
+ clm_name,
117
+ clm_type,
118
+ )
119
+ return
120
+
121
+ func_name = self._collectors[clm_type]
122
+ func = getattr(self, func_name)
123
+ LOGGER.debug(
124
+ "Adding Pandera checks to column '%s' of type '%s'", clm_name, clm_type
125
+ )
126
+ func(clm_name, pyspark_df, pandera_column)
127
+
128
+ @column_register(BOOLEAN_COLUMN_TYPE)
129
+ def _add_boolean_type_checks(
130
+ self, clm_name: str, pyspark_df: SparkDataFrame, pandera_column: Column
131
+ ) -> None:
132
+ pandera_column.checks.extend([Check.isin([True, False])])
133
+
134
+ @column_register(DAYTIMEINTERVAL_COLUMN_TYPE)
135
+ def _add_daytimeinterval_type_checks(
136
+ self, clm_name: str, pyspark_df: SparkDataFrame, pandera_column: Column
137
+ ) -> None:
138
+ select_result = pyspark_df.select(
139
+ spark_min(spark_col(clm_name)).alias(COLUMN_MIN_KEY),
140
+ spark_max(spark_col(clm_name)).alias(COLUMN_MAX_KEY),
141
+ ).collect()[0]
142
+
143
+ min_value = pd.to_timedelta(select_result[COLUMN_MIN_KEY])
144
+ max_value = pd.to_timedelta(select_result[COLUMN_MAX_KEY])
145
+
146
+ pandera_column.checks.append(
147
+ Check.between(
148
+ min_value=min_value,
149
+ max_value=max_value,
150
+ include_max=True,
151
+ include_min=True,
152
+ title=BETWEEN_CHECK_ERROR_MESSAGE_FORMAT.format(min_value, max_value),
153
+ )
154
+ )
155
+
156
+ @column_register(
157
+ BYTE_COLUMN_TYPE,
158
+ SHORT_COLUMN_TYPE,
159
+ INTEGER_COLUMN_TYPE,
160
+ LONG_COLUMN_TYPE,
161
+ FLOAT_COLUMN_TYPE,
162
+ DOUBLE_COLUMN_TYPE,
163
+ )
164
+ def _add_numeric_type_checks(
165
+ self, clm_name: str, pyspark_df: SparkDataFrame, pandera_column: Column
166
+ ) -> None:
167
+ select_result = pyspark_df.select(
168
+ spark_min(spark_col(clm_name)).alias(COLUMN_MIN_KEY),
169
+ spark_max(spark_col(clm_name)).alias(COLUMN_MAX_KEY),
170
+ ).collect()[0]
171
+
172
+ min_value = select_result[COLUMN_MIN_KEY]
173
+ max_value = select_result[COLUMN_MAX_KEY]
174
+
175
+ pandera_column.checks.append(
176
+ Check.between(
177
+ min_value=min_value,
178
+ max_value=max_value,
179
+ include_max=True,
180
+ include_min=True,
181
+ title=BETWEEN_CHECK_ERROR_MESSAGE_FORMAT.format(min_value, max_value),
182
+ )
183
+ )
184
+
185
+ @column_register(STRING_COLUMN_TYPE)
186
+ def _add_string_type_checks(
187
+ self, clm_name: str, pyspark_df: SparkDataFrame, pandera_column: Column
188
+ ) -> None:
189
+ select_result = pyspark_df.select(
190
+ spark_min(spark_length(spark_col(clm_name))).alias(COLUMN_MIN_KEY),
191
+ spark_max(spark_length(spark_col(clm_name))).alias(COLUMN_MAX_KEY),
192
+ ).collect()[0]
193
+
194
+ min_length = select_result[COLUMN_MIN_KEY]
195
+ max_length = select_result[COLUMN_MAX_KEY]
196
+
197
+ pandera_column.checks.append(Check.str_length(min_length, max_length))
198
+
199
+ @column_register(TIMESTAMP_COLUMN_TYPE)
200
+ def _add_timestamp_type_checks(
201
+ self, clm_name: str, pyspark_df: SparkDataFrame, pandera_column: Column
202
+ ) -> None:
203
+ select_result = pyspark_df.select(
204
+ spark_min(spark_col(clm_name)).alias(COLUMN_MIN_KEY),
205
+ spark_max(spark_col(clm_name)).alias(COLUMN_MAX_KEY),
206
+ ).collect()[0]
207
+
208
+ min_value = pd.Timestamp(select_result[COLUMN_MIN_KEY])
209
+ max_value = pd.Timestamp(select_result[COLUMN_MAX_KEY])
210
+
211
+ pandera_column.checks.append(
212
+ Check.between(
213
+ min_value=min_value,
214
+ max_value=max_value,
215
+ include_max=True,
216
+ include_min=True,
217
+ title=BETWEEN_CHECK_ERROR_MESSAGE_FORMAT.format(min_value, max_value),
218
+ )
219
+ )
220
+
221
+ @column_register(TIMESTAMP_NTZ_COLUMN_TYPE)
222
+ def _add_timestamp_ntz_type_checks(
223
+ self, clm_name: str, pyspark_df: SparkDataFrame, pandera_column: Column
224
+ ) -> None:
225
+ select_result = pyspark_df.select(
226
+ spark_min(spark_col(clm_name)).alias(COLUMN_MIN_KEY),
227
+ spark_max(spark_col(clm_name)).alias(COLUMN_MAX_KEY),
228
+ ).collect()[0]
229
+
230
+ min_value = pd.Timestamp(select_result[COLUMN_MIN_KEY])
231
+ max_value = pd.Timestamp(select_result[COLUMN_MAX_KEY])
232
+
233
+ pandera_column.checks.append(
234
+ Check.between(
235
+ min_value=min_value,
236
+ max_value=max_value,
237
+ include_max=True,
238
+ include_min=True,
239
+ title=BETWEEN_CHECK_ERROR_MESSAGE_FORMAT.format(min_value, max_value),
240
+ )
241
+ )
@@ -0,0 +1,23 @@
1
+ # Copyright 2025 Snowflake Inc.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+
17
+ class Singleton(type):
18
+ _instances = {}
19
+
20
+ def __call__(cls, *args, **kwargs):
21
+ if cls not in cls._instances:
22
+ cls._instances[cls] = super().__call__(*args, **kwargs)
23
+ return cls._instances[cls]
@@ -0,0 +1,20 @@
1
+ # Copyright 2025 Snowflake Inc.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ __all__ = ["SnowConnection"]
17
+
18
+ from snowflake.snowpark_checkpoints_collector.snow_connection_model.snow_connection import (
19
+ SnowConnection,
20
+ )
@@ -0,0 +1,201 @@
1
+ # Copyright 2025 Snowflake Inc.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ import glob
17
+ import logging
18
+ import os.path
19
+ import time
20
+
21
+ from pathlib import Path
22
+ from typing import Callable, Optional
23
+
24
+ from snowflake.snowpark import Session
25
+ from snowflake.snowpark_checkpoints_collector.collection_common import (
26
+ DOT_PARQUET_EXTENSION,
27
+ )
28
+
29
+
30
+ STAGE_NAME = "CHECKPOINT_STAGE"
31
+ CREATE_STAGE_STATEMENT_FORMAT = "CREATE TEMP STAGE IF NOT EXISTS {}"
32
+ REMOVE_STAGE_FOLDER_STATEMENT_FORMAT = "REMOVE {}"
33
+ STAGE_PATH_FORMAT = "'@{}/{}'"
34
+ PUT_FILE_IN_STAGE_STATEMENT_FORMAT = "PUT '{}' {} AUTO_COMPRESS=FALSE"
35
+ LOGGER = logging.getLogger(__name__)
36
+
37
+
38
+ class SnowConnection:
39
+
40
+ """Class for manage the Snowpark Connection.
41
+
42
+ Attributes:
43
+ session (Snowpark.Session): the Snowpark session.
44
+
45
+ """
46
+
47
+ def __init__(self, session: Optional[Session] = None) -> None:
48
+ """Init SnowConnection.
49
+
50
+ Args:
51
+ session (Snowpark.Session): the Snowpark session.
52
+
53
+ """
54
+ self.session = (
55
+ session if session is not None else self._create_snowpark_session()
56
+ )
57
+ self.stage_id = int(time.time())
58
+
59
+ def create_snowflake_table_from_local_parquet(
60
+ self,
61
+ table_name: str,
62
+ input_path: str,
63
+ stage_path: Optional[str] = None,
64
+ ) -> None:
65
+ """Upload to parquet files from the input path and create a table.
66
+
67
+ Args:
68
+ table_name (str): the name of the table to be created.
69
+ input_path (str): the input directory path.
70
+ stage_path: (str, optional): the stage path.
71
+
72
+ """
73
+ input_path = (
74
+ os.path.abspath(input_path)
75
+ if not os.path.isabs(input_path)
76
+ else str(Path(input_path).resolve())
77
+ )
78
+ folder = f"table_files_{int(time.time())}"
79
+ stage_path = stage_path if stage_path else folder
80
+ stage_name = f"{STAGE_NAME}_{self.stage_id}"
81
+ stage_directory_path = STAGE_PATH_FORMAT.format(stage_name, stage_path)
82
+
83
+ def is_parquet_file(file: str):
84
+ return file.endswith(DOT_PARQUET_EXTENSION)
85
+
86
+ try:
87
+ self.create_tmp_stage(stage_name)
88
+ self.load_files_to_stage(
89
+ stage_name, stage_path, input_path, is_parquet_file
90
+ )
91
+ self.create_table_from_parquet(table_name, stage_directory_path)
92
+ finally:
93
+ LOGGER.info("Removing stage folder %s", stage_directory_path)
94
+ self.session.sql(
95
+ REMOVE_STAGE_FOLDER_STATEMENT_FORMAT.format(stage_directory_path)
96
+ ).collect()
97
+
98
+ def create_tmp_stage(self, stage_name: str) -> None:
99
+ """Create a temp stage in Snowflake.
100
+
101
+ Args:
102
+ stage_name (str): the name of the stage.
103
+
104
+ """
105
+ create_stage_statement = CREATE_STAGE_STATEMENT_FORMAT.format(stage_name)
106
+ LOGGER.info("Creating temporal stage '%s'", stage_name)
107
+ self.session.sql(create_stage_statement).collect()
108
+
109
+ def load_files_to_stage(
110
+ self,
111
+ stage_name: str,
112
+ folder_name: str,
113
+ input_path: str,
114
+ filter_func: Optional[Callable] = None,
115
+ ) -> None:
116
+ """Load files to a stage in Snowflake.
117
+
118
+ Args:
119
+ stage_name (str): the name of the stage.
120
+ folder_name (str): the folder name.
121
+ input_path (str): the input directory path.
122
+ filter_func (Callable): the filter function to apply to the files.
123
+
124
+ """
125
+ LOGGER.info("Starting to load files to '%s'", stage_name)
126
+ input_path = (
127
+ os.path.abspath(input_path)
128
+ if not os.path.isabs(input_path)
129
+ else str(Path(input_path).resolve())
130
+ )
131
+
132
+ def filter_files(name: str):
133
+ return os.path.isfile(name) and (filter_func(name) if filter_func else True)
134
+
135
+ target_dir = os.path.join(input_path, "**", "*")
136
+ LOGGER.debug("Searching for files in '%s'", input_path)
137
+ files_collection = glob.glob(target_dir, recursive=True)
138
+
139
+ files = [file for file in files_collection if filter_files(file)]
140
+ files_count = len(files)
141
+
142
+ if files_count == 0:
143
+ raise Exception(f"No files were found in the input directory: {input_path}")
144
+
145
+ LOGGER.debug("Found %s files in '%s'", files_count, input_path)
146
+
147
+ for file in files:
148
+ # if file is relative path, convert to absolute path
149
+ # if absolute path, then try to resolve as some Win32 paths are not in LPN.
150
+ file_full_path = (
151
+ str(os.path.abspath(file))
152
+ if not os.path.isabs(file)
153
+ else str(Path(file).resolve())
154
+ )
155
+ # Snowflake required URI format for input in the put.
156
+ normalize_file_path = Path(file_full_path).as_uri()
157
+ new_file_path = file_full_path.replace(input_path, folder_name)
158
+ # as Posix to convert Windows dir to posix
159
+ new_file_path = Path(new_file_path).as_posix()
160
+ stage_file_path = STAGE_PATH_FORMAT.format(stage_name, new_file_path)
161
+ put_statement = PUT_FILE_IN_STAGE_STATEMENT_FORMAT.format(
162
+ normalize_file_path, stage_file_path
163
+ )
164
+ LOGGER.info("Loading file '%s' to %s", file_full_path, stage_file_path)
165
+ self.session.sql(put_statement).collect()
166
+
167
+ def create_table_from_parquet(
168
+ self, table_name: str, stage_directory_path: str
169
+ ) -> None:
170
+ """Create a table from a parquet file in Snowflake.
171
+
172
+ Args:
173
+ table_name (str): the name of the table.
174
+ stage_directory_path (str): the stage directory path.
175
+
176
+ Raise:
177
+ Exception: No parquet files were found in the stage
178
+
179
+ """
180
+ LOGGER.info("Starting to create table '%s' from parquet files", table_name)
181
+ parquet_files = self.session.sql(
182
+ f"LIST {stage_directory_path} PATTERN='.*{DOT_PARQUET_EXTENSION}'"
183
+ ).collect()
184
+ parquet_files_count = len(parquet_files)
185
+ if parquet_files_count == 0:
186
+ raise Exception(
187
+ f"No parquet files were found in the stage: {stage_directory_path}"
188
+ )
189
+
190
+ LOGGER.info(
191
+ "Reading %s parquet files from %s",
192
+ parquet_files_count,
193
+ stage_directory_path,
194
+ )
195
+ dataframe = self.session.read.parquet(path=stage_directory_path)
196
+ LOGGER.info("Creating table '%s' from parquet files", table_name)
197
+ dataframe.write.save_as_table(table_name=table_name, mode="overwrite")
198
+
199
+ def _create_snowpark_session(self) -> Session:
200
+ LOGGER.info("Creating a Snowpark session using the default connection")
201
+ return Session.builder.getOrCreate()