snowpark-connect 0.28.1__tar.gz → 0.30.0__tar.gz

{snowpark_connect-0.28.1/src/snowpark_connect.egg-info → snowpark_connect-0.30.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-connect
-Version: 0.28.1
+Version: 0.30.0
 Summary: Snowpark Connect for Spark
 Author: Snowflake, Inc
 License: Apache License, Version 2.0
@@ -16,7 +16,7 @@ Requires-Dist: jpype1
 Requires-Dist: protobuf<5.0,>=4.25.3
 Requires-Dist: s3fs>=2025.3.0
 Requires-Dist: snowflake.core<2,>=1.0.5
-Requires-Dist: snowflake-snowpark-python[pandas]<1.40.0,==1.39.0
+Requires-Dist: snowflake-snowpark-python[pandas]<1.40.0,==1.39.1
 Requires-Dist: sqlglot>=26.3.8
 Requires-Dist: jaydebeapi
 Requires-Dist: aiobotocore~=2.23.0
@@ -27,6 +27,7 @@ Requires-Dist: grpcio<1.63,>=1.56.0
 Requires-Dist: grpcio-status<1.63,>=1.56.0
 Requires-Dist: googleapis-common-protos>=1.56.4
 Requires-Dist: numpy<2,>=1.15
+Requires-Dist: gcsfs>=2025.9.0
 Dynamic: author
 Dynamic: description
 Dynamic: description-content-type

{snowpark_connect-0.28.1 → snowpark_connect-0.30.0}/setup.py

@@ -46,7 +46,7 @@ setup(
         "protobuf>=4.25.3,<5.0",
         "s3fs>=2025.3.0",  # prod-297255-inc0132291
         "snowflake.core>=1.0.5,<2",
-        "snowflake-snowpark-python[pandas]==1.39.0,<1.40.0",
+        "snowflake-snowpark-python[pandas]==1.39.1,<1.40.0",
         "sqlglot>=26.3.8",
         "jaydebeapi",
         "aiobotocore~=2.23.0",
@@ -58,5 +58,6 @@ setup(
         "grpcio-status>=1.56.0,<1.63",
         "googleapis-common-protos>=1.56.4",
         "numpy>=1.15,<2",
+        "gcsfs>=2025.9.0",
     ],
 )

{snowpark_connect-0.28.1 → snowpark_connect-0.30.0}/src/snowflake/snowpark_connect/analyze_plan/map_tree_string.py

@@ -5,6 +5,9 @@
 import pyspark.sql.connect.proto.base_pb2 as proto_base
 
 from snowflake.snowpark_connect.relation.map_relation import map_relation
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    filter_metadata_columns,
+)
 from snowflake.snowpark_connect.type_mapping import (
     SNOWPARK_TYPE_NAME_TO_PYSPARK_TYPE_NAME,
 )
@@ -16,12 +19,13 @@ def map_tree_string(
     # TODO: tracking the difference with pyspark in SNOW-1853347
     tree_string = request.tree_string
     snowpark_df_container = map_relation(tree_string.plan.root)
-    snowpark_df = snowpark_df_container.dataframe
-    column_map = snowpark_df_container.column_map
+    filtered_container = filter_metadata_columns(snowpark_df_container)
+    display_df = filtered_container.dataframe
+    filtered_column_mapping = filtered_container.column_map.snowpark_to_spark_map()
 
-    snowpark_tree_string = snowpark_df._format_schema(
+    snowpark_tree_string = display_df._format_schema(
         level=tree_string.level if tree_string.HasField("level") else None,
-        translate_columns=column_map.snowpark_to_spark_map(),
+        translate_columns=filtered_column_mapping,
         translate_types=SNOWPARK_TYPE_NAME_TO_PYSPARK_TYPE_NAME,
     )
     # workaround for the capitalization of nullable boolean value.

snowpark_connect-0.30.0/src/snowflake/snowpark_connect/client.py

@@ -0,0 +1,65 @@
+#
+# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
+#
+
+import zlib
+
+from pyspark.sql import DataFrame, SparkSession
+
+SQL_PASS_THROUGH_MARKER = "PRIVATE-SNOWFLAKE-SQL"
+
+
+def calculate_checksum(data: str) -> str:
+    checksum = zlib.crc32(data.encode("utf-8"))
+    return format(checksum, "08X")
+
+
+class SnowflakeSession:
+    """
+    Provides a wrapper around SparkSession to enable Snowflake SQL pass-through functionality.
+    Also provides helper methods to switch to different database, schema, role, warehouse, etc.
+    """
+
+    def __init__(self, spark_session: SparkSession) -> None:
+        self.spark_session = spark_session
+
+    def sql(self, sql_stmt: str) -> DataFrame:
+        """
+        Execute Snowflake specific SQL directly against Snowflake.
+        """
+        checksum = calculate_checksum(sql_stmt)
+        return self.spark_session.sql(
+            f"{SQL_PASS_THROUGH_MARKER} {checksum} {sql_stmt}"
+        )
+
+    def use_database(self, database: str, preserve_case: bool = False) -> DataFrame:
+        """
+        Switch to the database specified by `database`.
+        """
+        if preserve_case:
+            database = f'"{database}"'
+        return self.sql(f"USE DATABASE {database}")
+
+    def use_schema(self, schema: str, preserve_case: bool = False) -> DataFrame:
+        """
+        Switch to the schema specified by `schema`.
+        """
+        if preserve_case:
+            schema = f'"{schema}"'
+        return self.sql(f"USE SCHEMA {schema}")
+
+    def use_role(self, role: str, preserve_case: bool = False) -> DataFrame:
+        """
+        Switch to the role specified by `role`.
+        """
+        if preserve_case:
+            role = f'"{role}"'
+        return self.sql(f"USE ROLE {role}")
+
+    def use_warehouse(self, warehouse: str, preserve_case: bool = False) -> DataFrame:
+        """
+        Switch to the warehouse specified by `warehouse`.
+        """
+        if preserve_case:
+            warehouse = f'"{warehouse}"'
+        return self.sql(f"USE WAREHOUSE {warehouse}")

{snowpark_connect-0.28.1 → snowpark_connect-0.30.0}/src/snowflake/snowpark_connect/column_name_handler.py

@@ -75,7 +75,13 @@ def make_column_names_snowpark_compatible(
         In this case the function call should be `make_column_names_snowpark_compatible(['a', 'b'], 5, 2)`,
         to avoid naming conflicts between the new columns and the old columns.
     """
+    from snowflake.snowpark_connect.relation.read.metadata_utils import (
+        METADATA_FILENAME_COLUMN,
+    )
+
     return [
+        # Skip METADATA$FILENAME - preserve original name without quoting
+        name if name == METADATA_FILENAME_COLUMN else
         # Use `-` in the name to force df.column to return double-quoted names
         quote_name(f"{unquote_if_quoted(name)}-{plan_id:08x}-{i + offset}")
         for i, name in enumerate(names)

{snowpark_connect-0.28.1 → snowpark_connect-0.30.0}/src/snowflake/snowpark_connect/config.py

@@ -263,17 +263,24 @@ SESSION_CONFIG_KEY_WHITELIST = {
     "snowpark.connect.sql.identifiers.auto-uppercase",
     "snowpark.connect.udtf.compatibility_mode",
     "snowpark.connect.views.duplicate_column_names_handling_mode",
-    "enable_snowflake_extension_behavior",
+    "snowpark.connect.enable_snowflake_extension_behavior",
+    "spark.hadoop.fs.s3a.server-side-encryption.key",
+    "spark.hadoop.fs.s3a.assumed.role.arn",
+    "snowpark.connect.describe_cache_ttl_seconds",
 }
-AZURE_SAS_KEY = re.compile(
+AZURE_ACCOUNT_KEY = re.compile(
     r"^fs\.azure\.sas\.[^\.]+\.[^\.]+\.blob\.core\.windows\.net$"
 )
+AZURE_SAS_KEY = re.compile(
+    r"^fs\.azure\.sas\.fixed\.token\.[^\.]+\.dfs\.core\.windows\.net$"
+)
 
 
 def valid_session_config_key(key: str):
     return (
         key in SESSION_CONFIG_KEY_WHITELIST  # AWS session keys
         or AZURE_SAS_KEY.match(key)  # Azure session keys
+        or AZURE_ACCOUNT_KEY.match(key)  # Azure account keys
     )
 
 
@@ -288,7 +295,8 @@ class SessionConfig:
         "snowpark.connect.views.duplicate_column_names_handling_mode": "rename",
         "spark.sql.execution.pythonUDTF.arrow.enabled": "false",
         "spark.sql.tvf.allowMultipleTableArguments.enabled": "true",
-        "enable_snowflake_extension_behavior": "false",
+        "snowpark.connect.enable_snowflake_extension_behavior": "false",
+        "snowpark.connect.describe_cache_ttl_seconds": "300",
     }
 
     def __init__(self) -> None:
@@ -578,7 +586,10 @@ def set_snowflake_parameters(
             cte_enabled = str_to_bool(value)
             snowpark_session.cte_optimization_enabled = cte_enabled
             logger.info(f"Updated snowpark session CTE optimization: {cte_enabled}")
-
+        case "snowpark.connect.structured_types.fix":
+            # TODO: SNOW-2367714 Remove this once the fix is automatically enabled in Snowpark
+            snowpark.context._enable_fix_2360274 = str_to_bool(value)
+            logger.info(f"Updated snowpark session structured types fix: {value}")
         case _:
             pass
 
@@ -598,6 +609,21 @@ def get_cte_optimization_enabled() -> bool:
     return get_boolean_session_config_param("snowpark.connect.cte.optimization_enabled")
 
 
+def get_describe_cache_ttl_seconds() -> int:
+    """Get the describe query cache TTL from session config, with a default fallback."""
+    session_config: SessionConfig = sessions_config[get_session_id()]
+    default_ttl: str = SessionConfig.default_session_config[
+        "snowpark.connect.describe_cache_ttl_seconds"
+    ]
+    try:
+        ttl_str = session_config.get(
+            "snowpark.connect.describe_cache_ttl_seconds", default_ttl
+        )
+        return int(ttl_str)
+    except ValueError:  # fallback to default ttl
+        return int(default_ttl)
+
+
 def auto_uppercase_column_identifiers() -> bool:
     session_config = sessions_config[get_session_id()]
     return session_config[
@@ -701,7 +727,9 @@ def check_table_supports_operation(table_identifier: str, operation: str) -> boo
     session_id = get_session_id()
     session_config = sessions_config[session_id]
     enable_extensions = str_to_bool(
-        session_config.get("enable_snowflake_extension_behavior", "false")
+        session_config.get(
+            "snowpark.connect.enable_snowflake_extension_behavior", "false"
+        )
     )
 
     if enable_extensions:

{snowpark_connect-0.28.1 → snowpark_connect-0.30.0}/src/snowflake/snowpark_connect/execute_plan/map_execution_root.py

@@ -26,6 +26,9 @@ from snowflake.snowpark_connect.execute_plan.utils import (
     pandas_to_arrow_batches_bytes,
 )
 from snowflake.snowpark_connect.relation.map_relation import map_relation
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    filter_metadata_columns,
+)
 from snowflake.snowpark_connect.type_mapping import (
     map_snowpark_types_to_pyarrow_types,
     snowpark_to_proto_type,
@@ -92,14 +95,19 @@ def map_execution_root(
 ) -> Iterator[proto_base.ExecutePlanResponse | QueryResult]:
     result: DataFrameContainer | pandas.DataFrame = map_relation(request.plan.root)
     if isinstance(result, pandas.DataFrame):
-        result_df = result
+        pandas_df = result
+        data_bytes = pandas_to_arrow_batches_bytes(pandas_df)
+        row_count = len(pandas_df)
+        schema = None
+        yield _build_execute_plan_response(row_count, data_bytes, schema, request)
     else:
-        result_df = result.dataframe
-
-    if isinstance(result_df, snowpark.DataFrame):
-        snowpark_schema = result_df.schema
-        schema = snowpark_to_proto_type(snowpark_schema, result.column_map, result_df)
-        spark_columns = result.column_map.get_spark_columns()
+        filtered_result = filter_metadata_columns(result)
+        filtered_result_df = filtered_result.dataframe
+        snowpark_schema = filtered_result_df.schema
+        schema = snowpark_to_proto_type(
+            snowpark_schema, filtered_result.column_map, filtered_result_df
+        )
+        spark_columns = filtered_result.column_map.get_spark_columns()
         if tcm.TCM_MODE:
             # TCM result handling:
             # - small result (only one batch): just return the executePlanResponse
@@ -108,22 +116,22 @@ def map_execution_root(
             is_large_result = False
             second_batch = False
             first_arrow_table = None
-            with result_df.session.query_history() as qh:
-                for arrow_table in to_arrow_batch_iter(result_df):
+            with filtered_result_df.session.query_history() as qh:
+                for arrow_table in to_arrow_batch_iter(filtered_result_df):
                     if second_batch:
                         is_large_result = True
                         break
                     first_arrow_table = arrow_table
                     second_batch = True
                 queries_cnt = len(
-                    result_df._plan.execution_queries[PlanQueryType.QUERIES]
+                    filtered_result_df._plan.execution_queries[PlanQueryType.QUERIES]
                 )
                 # get query uuid from the last query; this may not be the last queries in query history because snowpark
                 # may run some post action queries, e.g., drop temp table.
                 query_id = qh.queries[queries_cnt - 1].query_id
             if first_arrow_table is None:
                 # empty arrow batch iterator
-                pandas_df = result_df.to_pandas()
+                pandas_df = filtered_result_df.to_pandas()
                 data_bytes = pandas_to_arrow_batches_bytes(pandas_df)
                 yield _build_execute_plan_response(0, data_bytes, schema, request)
             elif not tcm.TCM_RETURN_QUERY_ID_FOR_SMALL_RESULT and not is_large_result:
@@ -150,7 +158,7 @@ def map_execution_root(
                     spark_schema.SerializeToString(),
                 )
         else:
-            arrow_table_iter = to_arrow_batch_iter(result_df)
+            arrow_table_iter = to_arrow_batch_iter(filtered_result_df)
             batch_count = 0
             for arrow_table in arrow_table_iter:
                 if arrow_table.num_rows > 0:
@@ -166,12 +174,6 @@ def map_execution_root(
 
             # Empty result needs special processing
             if batch_count == 0:
-                pandas_df = result_df.to_pandas()
+                pandas_df = filtered_result_df.to_pandas()
                 data_bytes = pandas_to_arrow_batches_bytes(pandas_df)
                 yield _build_execute_plan_response(0, data_bytes, schema, request)
-    else:
-        pandas_df = result_df
-        data_bytes = pandas_to_arrow_batches_bytes(pandas_df)
-        row_count = len(pandas_df)
-        schema = None
-        yield _build_execute_plan_response(row_count, data_bytes, schema, request)