snowpark-connect 0.31.0__py3-none-any.whl → 0.33.0__py3-none-any.whl

snowflake/snowpark_connect/resources_initializer.py

@@ -1,7 +1,6 @@
 #
 # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #
-import pathlib
 import threading
 import time
 
@@ -51,11 +50,9 @@ def initialize_resources() -> None:
         """Upload Spark jar files required for creating Scala UDFs."""
         stage = session.get_session_stage()
         resource_path = stage + RESOURCE_PATH
-        import snowflake
+        import snowpark_connect_deps_1
+        import snowpark_connect_deps_2
 
-        pyspark_jars = (
-            pathlib.Path(snowflake.snowpark_connect.__file__).parent / "includes/jars"
-        )
         jar_files = [
             f"spark-sql_2.12-{SPARK_VERSION}.jar",
             f"spark-connect-client-jvm_2.12-{SPARK_VERSION}.jar",
@@ -64,14 +61,29 @@ def initialize_resources() -> None:
             "json4s-ast_2.12-3.7.0-M11.jar",
         ]
 
-        for jar in jar_files:
-            session.file.put(
-                str(pyspark_jars) + "/" + jar,
-                resource_path,
-                auto_compress=False,
-                overwrite=False,
-                source_compression="NONE",
-            )
+        for jar_name in jar_files:
+            # Try to find the JAR in package 1 first, then package 2
+            jar_path = None
+            try:
+                jar_path = snowpark_connect_deps_1.get_jar_path(jar_name)
+            except FileNotFoundError:
+                try:
+                    jar_path = snowpark_connect_deps_2.get_jar_path(jar_name)
+                except FileNotFoundError:
+                    raise FileNotFoundError(
+                        f"JAR {jar_name} not found in either package"
+                    )
+
+            try:
+                session.file.put(
+                    str(jar_path),
+                    resource_path,
+                    auto_compress=False,
+                    overwrite=False,
+                    source_compression="NONE",
+                )
+            except Exception as e:
+                raise RuntimeError(f"Failed to upload JAR {jar_name}: {e}")
 
     start_time = time.time()
 

snowflake/snowpark_connect/server.py

@@ -24,12 +24,10 @@
 import atexit
 import logging
 import os
-import pathlib
 import socket
 import tempfile
 import threading
 import urllib.parse
-import zipfile
 from concurrent import futures
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
@@ -48,7 +46,6 @@ from pyspark.errors import PySparkValueError
 from pyspark.sql.connect.client.core import ChannelBuilder
 from pyspark.sql.connect.session import SparkSession
 
-import snowflake.snowpark_connect
 import snowflake.snowpark_connect.proto.control_pb2_grpc as control_grpc
 import snowflake.snowpark_connect.tcm as tcm
 from snowflake import snowpark
@@ -158,9 +155,8 @@ def _handle_exception(context, e: Exception):
         logger.error("Error: %s - %s", type(e).__name__, str(e))
 
     telemetry.report_request_failure(e)
-
     if tcm.TCM_MODE:
-        # TODO: SNOW-2009834 gracefully return error back in TCM
+        # spark decoder will catch the error and return it to GS gracefully
         attach_custom_error_code(e, ErrorCodes.INTERNAL_ERROR)
         raise e
 
@@ -1033,28 +1029,16 @@ def start_jvm():
         attach_custom_error_code(exception, ErrorCodes.INTERNAL_ERROR)
         raise exception
 
-    pyspark_jars = (
-        pathlib.Path(snowflake.snowpark_connect.__file__).parent / "includes/jars"
-    )
-
-    if "dataframe_processor.zip" in str(pyspark_jars):
-        # importlib.resource doesn't work when local stage package is used in TCM
-        zip_path = pathlib.Path(
-            snowflake.snowpark_connect.__file__
-        ).parent.parent.parent
-        temp_dir = tempfile.gettempdir()
-
-        extract_folder = "snowflake/snowpark_connect/includes/jars/"  # Folder to extract (must end with '/')
+    # Import both JAR dependency packages
+    import snowpark_connect_deps_1
+    import snowpark_connect_deps_2
 
-        with zipfile.ZipFile(zip_path, "r") as zip_ref:
-            for member in zip_ref.namelist():
-                if member.startswith(extract_folder):
-                    zip_ref.extract(member, path=temp_dir)
-
-        pyspark_jars = pathlib.Path(temp_dir) / extract_folder
-
-    for path in pyspark_jars.glob("**/*.jar"):
-        jpype.addClassPath(path)
+    # Load all the jar files from both packages
+    jar_path_list = (
+        snowpark_connect_deps_1.list_jars() + snowpark_connect_deps_2.list_jars()
+    )
+    for jar_path in jar_path_list:
+        jpype.addClassPath(jar_path)
 
     # TODO: Should remove convertStrings, but it breaks the JDBC code.
     jvm_settings: list[str] = list(

snowflake/snowpark_connect/type_mapping.py

@@ -373,6 +373,8 @@ def cast_to_match_snowpark_type(
             return str(content)
         case snowpark.types.DayTimeIntervalType:
             return str(content)
+        case snowpark.types.MapType:
+            return content
         case _:
             exception = SnowparkConnectNotImplementedError(
                 f"Unsupported snowpark data type in casting: {data_type}"
@@ -582,11 +584,14 @@ def map_snowpark_types_to_pyarrow_types(
                 attach_custom_error_code(exception, ErrorCodes.UNSUPPORTED_TYPE)
                 raise exception
         case snowpark.types.TimestampType:
-            unit = pa_type.unit
-            tz = pa_type.tz
+            # Check if pa_type has unit attribute (it should be a timestamp type)
+            unit = pa_type.unit if hasattr(pa_type, "unit") else "us"
+            tz = pa_type.tz if hasattr(pa_type, "tz") else None
+
+            # Spark truncates nanosecond precision to microseconds
             if unit == "ns":
-                # Spark truncates nanosecond precision to microseconds
                 unit = "us"
+
             return pa.timestamp(unit, tz=tz)
         case snowpark.types.VariantType:
             return pa.string()
@@ -670,6 +675,9 @@ def map_pyarrow_to_snowpark_types(pa_type: pa.DataType) -> snowpark.types.DataTy
         return snowpark.types.TimestampType()
     elif pa.types.is_null(pa_type):
         return snowpark.types.NullType()
+    elif pa.types.is_duration(pa_type):
+        # Map PyArrow duration[us] to DayTimeIntervalType
+        return snowpark.types.DayTimeIntervalType()
     else:
         exception = SnowparkConnectNotImplementedError(
             f"Unsupported PyArrow data type: {pa_type}"
@@ -892,6 +900,33 @@ def map_simple_types(simple_type: str) -> snowpark.types.DataType:
             return snowpark.types.YearMonthIntervalType()
         case type_name if _INTERVAL_DAYTIME_PATTERN_RE.match(type_name):
             return snowpark.types.DayTimeIntervalType()
+        # Year-Month interval cases
+        case "interval year":
+            return snowpark.types.YearMonthIntervalType(0)
+        case "interval month":
+            return snowpark.types.YearMonthIntervalType(1)
+        case "interval year to month":
+            return snowpark.types.YearMonthIntervalType(0, 1)
+        case "interval day":
+            return snowpark.types.DayTimeIntervalType(0)
+        case "interval hour":
+            return snowpark.types.DayTimeIntervalType(1)
+        case "interval minute":
+            return snowpark.types.DayTimeIntervalType(2)
+        case "interval second":
+            return snowpark.types.DayTimeIntervalType(3)
+        case "interval day to hour":
+            return snowpark.types.DayTimeIntervalType(0, 1)
+        case "interval day to minute":
+            return snowpark.types.DayTimeIntervalType(0, 2)
+        case "interval day to second":
+            return snowpark.types.DayTimeIntervalType(0, 3)
+        case "interval hour to minute":
+            return snowpark.types.DayTimeIntervalType(1, 2)
+        case "interval hour to second":
+            return snowpark.types.DayTimeIntervalType(1, 3)
+        case "interval minute to second":
+            return snowpark.types.DayTimeIntervalType(2, 3)
         case _:
             if simple_type.startswith("decimal"):
                 precision = int(simple_type.split("(")[1].split(",")[0])

snowflake/snowpark_connect/typed_column.py

@@ -8,6 +8,7 @@ from functools import cached_property
 import snowflake.snowpark.functions as snowpark_fn
 from snowflake import snowpark
 from snowflake.snowpark.column import Column
+from snowflake.snowpark_connect.column_qualifier import ColumnQualifier
 
 _EMPTY_COLUMN = Column("")
 
@@ -44,11 +45,11 @@ class TypedColumn:
     def alias(self, alias_name: str):
         return TypedColumn(self.col.alias(alias_name), self._type_resolver)
 
-    def set_qualifiers(self, qualifiers: list[str]):
+    def set_qualifiers(self, qualifiers: set[ColumnQualifier]) -> None:
         self.qualifiers = qualifiers
 
-    def get_qualifiers(self) -> list[str]:
-        return getattr(self, "qualifiers", [])
+    def get_qualifiers(self) -> set[ColumnQualifier]:
+        return getattr(self, "qualifiers", set())
 
     def set_catalog_database_info(self, catalog_database_info: dict[str, str]) -> None:
         self._catalog_database_info = catalog_database_info
@@ -63,12 +64,13 @@ class TypedColumn:
     def get_database(self) -> str | None:
         return self._catalog_database_info.get("database")
 
-    def set_multi_col_qualifiers(self, qualifiers: list[list[str]]):
+    def set_multi_col_qualifiers(self, qualifiers: list[set[ColumnQualifier]]) -> None:
         self.multi_col_qualifiers = qualifiers
 
-    def get_multi_col_qualifiers(self, num_columns) -> list[list[str]]:
+    def get_multi_col_qualifiers(self, num_columns) -> list[set[ColumnQualifier]]:
         if not hasattr(self, "multi_col_qualifiers"):
-            return [[] for i in range(num_columns)]
+
+            return [set() for i in range(num_columns)]
         assert (
             len(self.multi_col_qualifiers) == num_columns
         ), f"Expected {num_columns} multi-column qualifiers, got {len(self.multi_col_qualifiers)}"

snowflake/snowpark_connect/utils/sequence.py

@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
+#
+
+import threading
+from collections import defaultdict
+
+from snowflake.snowpark_connect.utils.context import get_session_id
+
+# per session number sequences to generate unique snowpark columns
+_session_sequences = defaultdict(int)
+
+_lock = threading.Lock()
+
+
+def next_unique_num():
+    session_id = get_session_id()
+    with _lock:
+        next_num = _session_sequences[session_id]
+        _session_sequences[session_id] = next_num + 1
+    return next_num

snowflake/snowpark_connect/utils/session.py

@@ -23,6 +23,13 @@ from snowflake.snowpark_connect.utils.snowpark_connect_logging import logger
 from snowflake.snowpark_connect.utils.telemetry import telemetry
 from snowflake.snowpark_connect.utils.udf_cache import init_builtin_udf_cache
 
+SKIP_SESSION_CONFIGURATION = False
+
+
+def skip_session_configuration(skip: bool):
+    global SKIP_SESSION_CONFIGURATION
+    SKIP_SESSION_CONFIGURATION = skip
+
 
 # Suppress experimental warnings from snowflake.snowpark logger
 def _filter_experimental_warnings(record):
@@ -57,6 +64,8 @@ def configure_snowpark_session(session: snowpark.Session):
         global_config,
     )
 
+    global SKIP_SESSION_CONFIGURATION
+
     logger.info(f"Configuring session {session}")
 
     telemetry.initialize(session)
@@ -85,6 +94,12 @@ def configure_snowpark_session(session: snowpark.Session):
     session.connection.arrow_number_to_decimal_setter = True
     session.custom_package_usage_config["enabled"] = True
 
+    # Scoped temp objects may not be accessible in stored procedure and cause "object does not exist" error. So disable
+    # _use_scoped_temp_objects here and use temp table instead.
+    # Note that we also set PYTHON_SNOWPARK_USE_SCOPED_TEMP_OBJECTS to false below to enforce the current Snowpark
+    # session does not use scoped temp objects too.
+    session._use_scoped_temp_objects = False
+
     # Configure CTE optimization based on session configuration
     cte_optimization_enabled = get_cte_optimization_enabled()
     session.cte_optimization_enabled = cte_optimization_enabled
@@ -119,14 +134,22 @@ def configure_snowpark_session(session: snowpark.Session):
         "TIMEZONE": f"'{global_config.spark_sql_session_timeZone}'",
         "QUOTED_IDENTIFIERS_IGNORE_CASE": "false",
         "PYTHON_SNOWPARK_ENABLE_THREAD_SAFE_SESSION": "true",
-        "PYTHON_SNOWPARK_USE_SCOPED_TEMP_OBJECTS": "false",  # this is required for creating udfs from sproc
+        # this is required for creating udfs from sproc and avoid "object does not exist" error
+        "PYTHON_SNOWPARK_USE_SCOPED_TEMP_OBJECTS": "false",
         "ENABLE_STRUCTURED_TYPES_IN_SNOWPARK_CONNECT_RESPONSE": "true",
         "QUERY_TAG": f"'{query_tag}'",
     }
 
-    session.sql(
-        f"ALTER SESSION SET {', '.join([f'{k} = {v}' for k, v in session_params.items()])}"
-    ).collect()
+    # SNOW-2245971: Stored procedures inside Native Apps run as Execute As Owner and hence cannot set session params.
+    if not SKIP_SESSION_CONFIGURATION:
+        session.sql(
+            f"ALTER SESSION SET {', '.join([f'{k} = {v}' for k, v in session_params.items()])}"
+        ).collect()
+    else:
+        session_param_names = ", ".join(session_params.keys())
+        logger.info(
+            f"Skipping Snowpark Connect session configuration as requested. Please make sure following session parameters are set correctly: {session_param_names}"
+        )
 
     # Instrument the snowpark session to use a cache for describe queries.
     instrument_session_for_describe_cache(session)

snowflake/snowpark_connect/version.py

@@ -2,4 +2,4 @@
 #
 # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #
-VERSION = (0,31,0)
+VERSION = (0,33,0)

snowflake/snowpark_decoder/dp_session.py

@@ -107,5 +107,5 @@ class DataframeProcessorSession:
             )
             return proto2str(dp_res_proto)
         except Exception:
-            # TODO: SNOW-1857056 error handling
+            # raise the error to GS
             raise

{snowpark_connect-0.31.0.dist-info → snowpark_connect-0.33.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-connect
-Version: 0.31.0
+Version: 0.33.0
 Summary: Snowpark Connect for Spark
 Author: Snowflake, Inc
 License: Apache License, Version 2.0
@@ -9,11 +9,13 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 License-File: LICENSE-binary
 License-File: NOTICE-binary
+Requires-Dist: snowpark-connect-deps-1==3.56.2
+Requires-Dist: snowpark-connect-deps-2==3.56.2
 Requires-Dist: certifi>=2025.1.31
 Requires-Dist: cloudpickle
 Requires-Dist: fsspec[http]
 Requires-Dist: jpype1
-Requires-Dist: protobuf<5.0,>=4.25.3
+Requires-Dist: protobuf<6.32.0,>=4.25.3
 Requires-Dist: s3fs>=2025.3.0
 Requires-Dist: snowflake.core<2,>=1.0.5
 Requires-Dist: snowflake-snowpark-python[pandas]<1.41.0,==1.40.0
@@ -29,11 +31,14 @@ Requires-Dist: grpcio-status<1.63,>=1.56.0
 Requires-Dist: googleapis-common-protos>=1.56.4
 Requires-Dist: numpy<2,>=1.15
 Requires-Dist: gcsfs>=2025.9.0
+Provides-Extra: jdk
+Requires-Dist: jdk4py==17.0.9.2; extra == "jdk"
 Dynamic: author
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: license
 Dynamic: license-file
+Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary