snowpark-connect 0.27.0__py3-none-any.whl → 1.6.0__py3-none-any.whl

snowflake/snowpark_connect/utils/udtf_utils.py

@@ -32,14 +32,16 @@ def create_udtf(
     udtf = udtf_proto.python_udtf
     callable_func = CloudPickleSerializer().loads(udtf.command)
 
-    func_signature = inspect.signature(callable_func.eval)
+    original_func = callable_func.eval
+    func_signature = inspect.signature(original_func)
     # Set all input types to VariantType regardless of type hints so that we can pass all arguments as VariantType.
     # Otherwise, we will run into issues with type mismatches. This only applies for UDTF registration.
     # We subtract one here since UDTF functions are class methods and always have "self" as the first parameter.
     input_types = [VariantType()] * (len(func_signature.parameters) - 1)
 
-    # Wrapp callable to allow reading imported files
-    callable_func = artifacts_reader_wrapper(callable_func)
+    if imports:
+        # Wrapp callable to allow reading imported files
+        callable_func = artifacts_reader_wrapper(callable_func)
 
     if is_arrow_enabled:
         callable_func = spark_compatible_udtf_wrapper_with_arrow(
@@ -48,7 +50,7 @@ def create_udtf(
     elif is_spark_compatible_udtf_mode_enabled:
         callable_func = spark_compatible_udtf_wrapper(callable_func, expected_types)
     else:
-        callable_func.process = callable_func.eval
+        callable_func.process = original_func
         if hasattr(callable_func, "terminate"):
             callable_func.end_partition = callable_func.terminate
 
@@ -107,7 +109,9 @@ def create_udtf(
                 imports=imports,
             )
         case _:
-            raise NotImplementedError(f"{called_from}")
+            raise NotImplementedError(
+                f"[snowpark_connect::unsupported_operation] {called_from}"
+            )
 
 
 def artifacts_reader_wrapper(user_udtf_cls: type) -> type:
@@ -127,7 +131,12 @@ def artifacts_reader_wrapper(user_udtf_cls: type) -> type:
 
                 # Change directory to the one containing the UDF imported files
                 import_path = sys._xoptions["snowflake_import_directory"]
-                tmp_path = f"/tmp/sas-{os.getpid()}"
+                if os.name == "nt":
+                    import tempfile
+
+                    tmp_path = os.path.join(tempfile.gettempdir(), f"sas-{os.getpid()}")
+                else:
+                    tmp_path = f"/tmp/sas-{os.getpid()}"
                 os.makedirs(tmp_path, exist_ok=True)
                 os.chdir(tmp_path)
                 shutil.copytree(import_path, tmp_path, dirs_exist_ok=True)
@@ -195,17 +204,19 @@ def _create_convert_table_argument_to_row():
                 # Named access: row["col1"], row["col2"]
                 if key in self._field_to_index:
                     return self._values[self._field_to_index[key]]
-                raise KeyError(key)
+                raise KeyError(f"[snowpark_connect::invalid_operation] {key}")
             else:
-                raise TypeError(f"Invalid key type: {type(key)}")
+                raise TypeError(
+                    f"[snowpark_connect::type_mismatch] Invalid key type: {type(key)}"
+                )
 
         def __getattr__(self, name):
             # Attribute access: row.col1, row.col2
             if name.startswith("_"):
-                raise AttributeError(name)
+                raise AttributeError(f"[snowpark_connect::invalid_operation] {name}")
             if name in self._field_to_index:
                 return self._values[self._field_to_index[name]]
-            raise AttributeError(name)
+            raise AttributeError(f"[snowpark_connect::invalid_operation] {name}")
 
         def __len__(self):
             return len(self._values)
@@ -279,7 +290,9 @@ def spark_compatible_udtf_wrapper(
             return val
         if isinstance(val, datetime.datetime):
             return val.date()
-        raise AttributeError(f"Invalid date value {val}")
+        raise AttributeError(
+            f"[snowpark_connect::invalid_input] Invalid date value {val}"
+        )
 
     def _coerce_to_binary(val: object, target_type_name: str = "byte") -> bytes | None:
         if target_type_name == "binary":
@@ -343,7 +356,9 @@ def spark_compatible_udtf_wrapper(
     def _coerce_to_timestamp(val: object) -> datetime.datetime | None:
         if isinstance(val, datetime.datetime):
             return val
-        raise AttributeError(f"Invalid time stamp value {val}")
+        raise AttributeError(
+            f"[snowpark_connect::invalid_input] Invalid time stamp value {val}"
+        )
 
     SCALAR_COERCERS = {
         "bool": _coerce_to_bool,
@@ -447,7 +462,7 @@ def spark_compatible_udtf_wrapper(
 
                 if not isinstance(raw_row_tuple, (tuple, list)):
                     raise TypeError(
-                        f"[UDTF_INVALID_OUTPUT_ROW_TYPE] return value should be an iterable object containing tuples, but got {type(raw_row_tuple)}"
+                        f"[snowpark_connect::type_mismatch] [UDTF_INVALID_OUTPUT_ROW_TYPE] return value should be an iterable object containing tuples, but got {type(raw_row_tuple)}"
                     )
 
                 if len(raw_row_tuple) != len(expected_types):
@@ -467,7 +482,7 @@ def spark_compatible_udtf_wrapper(
                         and val is not None
                     ):
                         raise RuntimeError(
-                            f"[UNEXPECTED_TUPLE_WITH_STRUCT] Expected a struct for column at position {i}, but got a primitive value of type {type(val)}"
+                            f"[snowpark_connect::type_mismatch] [UNEXPECTED_TUPLE_WITH_STRUCT] Expected a struct for column at position {i}, but got a primitive value of type {type(val)}"
                         )
 
                 coerced_row_list = [None] * len(expected_types)
@@ -533,7 +548,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
                     return pa.map_(key_type, value_type)
                 case _, _:
                     raise TypeError(
-                        f"[UDTF_ARROW_TYPE_CAST_ERROR] Unsupported Python scalar type for Arrow conversion: {target_py_type}"
+                        f"[snowpark_connect::unsupported_type] [UDTF_ARROW_TYPE_CAST_ERROR] Unsupported Python scalar type for Arrow conversion: {target_py_type}"
                     )
         elif kind == "array":
             element_type_info = type_marker
@@ -543,7 +558,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
             struct_fields_info = type_marker
             if not isinstance(struct_fields_info, dict):
                 raise TypeError(
-                    f"[UDTF_ARROW_TYPE_CAST_ERROR] Invalid struct definition for Arrow: expected dict, got {type(struct_fields_info)}"
+                    f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Invalid struct definition for Arrow: expected dict, got {type(struct_fields_info)}"
                 )
             fields = []
             for field_name, field_type_info in struct_fields_info.items():
@@ -552,7 +567,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
             return pa.struct(fields)
         else:
             raise TypeError(
-                f"[UDTF_ARROW_TYPE_CAST_ERROR] Unsupported data kind for Arrow conversion: {kind}"
+                f"[snowpark_connect::unsupported_type] [UDTF_ARROW_TYPE_CAST_ERROR] Unsupported data kind for Arrow conversion: {kind}"
             )
 
     def _convert_to_arrow_value(
@@ -576,7 +591,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
                 ]
             if not isinstance(obj, (list, tuple)):
                 raise TypeError(
-                    f"[UDTF_ARROW_TYPE_CAST_ERROR] Expected list or tuple for Arrow array type, got {type(obj).__name__}"
+                    f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Expected list or tuple for Arrow array type, got {type(obj).__name__}"
                 )
             element_type = arrow_type.value_type
             return [_convert_to_arrow_value(e, element_type, "array") for e in obj]
@@ -584,7 +599,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
         if pa.types.is_map(arrow_type):
             if not isinstance(obj, dict):
                 raise TypeError(
-                    f"[UDTF_ARROW_TYPE_CAST_ERROR] Expected dict for Arrow map type, got {type(obj).__name__}"
+                    f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Expected dict for Arrow map type, got {type(obj).__name__}"
                 )
             key_type = arrow_type.key_type
             value_type = arrow_type.item_type
@@ -610,7 +625,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
             else:
                 # If the UDTF yields a list/tuple (or anything not a dict) for a struct column, it's an error.
                 raise TypeError(
-                    f"[UDTF_ARROW_TYPE_CAST_ERROR] Expected a dictionary for Arrow struct type column, but got {type(obj).__name__}"
+                    f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Expected a dictionary for Arrow struct type column, but got {type(obj).__name__}"
                 )
 
         # Check if a scalar type is expected and if obj is a collection; if so, error out.
@@ -622,7 +637,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
         ):
             if isinstance(obj, (list, tuple, dict)):
                 raise TypeError(
-                    f"[UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert Python collection type {type(obj).__name__} to scalar Arrow type {arrow_type}"
+                    f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert Python collection type {type(obj).__name__} to scalar Arrow type {arrow_type}"
                 )
 
         if pa.types.is_boolean(arrow_type):
@@ -638,7 +653,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
                 elif obj == 1:
                     return True
                 raise TypeError(
-                    f"[UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {obj} to Arrow boolean"
+                    f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {obj} to Arrow boolean"
                 )
             if isinstance(obj, str):
                 v_str = obj.strip().lower()
@@ -647,7 +662,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
                 if v_str == "false":
                     return False
             raise TypeError(
-                f"[UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow boolean"
+                f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow boolean"
             )
 
         if pa.types.is_integer(arrow_type):
@@ -663,7 +678,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
                 except ValueError:
                     pass
             raise TypeError(
-                f"[UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow integer"
+                f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow integer"
             )
 
         if pa.types.is_floating(arrow_type):
@@ -675,7 +690,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
                 except ValueError:
                     pass
             raise TypeError(
-                f"[UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow float"
+                f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow float"
             )
 
         if pa.types.is_string(arrow_type):
@@ -687,7 +702,7 @@ def spark_compatible_udtf_wrapper_with_arrow(
             if isinstance(obj, str):
                 return obj
             raise TypeError(
-                f"[UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow string"
+                f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow string"
             )
 
         if pa.types.is_binary(arrow_type) or pa.types.is_fixed_size_binary(arrow_type):
@@ -698,21 +713,21 @@ def spark_compatible_udtf_wrapper_with_arrow(
             if isinstance(obj, int):
                 return bytearray([obj])
             raise TypeError(
-                f"[UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow binary"
+                f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow binary"
             )
 
         if pa.types.is_date(arrow_type):
             if isinstance(obj, datetime.date):
                 return obj
             raise TypeError(
-                f"[UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow date. Expected datetime.date."
+                f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow date. Expected datetime.date."
             )
 
         if pa.types.is_timestamp(arrow_type):
             if isinstance(obj, datetime.datetime):
                 return obj
             raise TypeError(
-                f"[UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow timestamp. Expected datetime.datetime."
+                f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow timestamp. Expected datetime.datetime."
             )
 
         if pa.types.is_decimal(arrow_type):
@@ -727,11 +742,11 @@ def spark_compatible_udtf_wrapper_with_arrow(
                     pass
 
             raise TypeError(
-                f"[UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow decimal. Expected decimal.Decimal or compatible int/str."
+                f"[snowpark_connect::type_mismatch] [UDTF_ARROW_TYPE_CAST_ERROR] Cannot convert {type(obj).__name__} to Arrow decimal. Expected decimal.Decimal or compatible int/str."
             )
 
         raise TypeError(
-            f"[UDTF_ARROW_TYPE_CAST_ERROR] Unsupported type conversion for {type(obj).__name__} to Arrow type {arrow_type}"
+            f"[snowpark_connect::unsupported_operation] [UDTF_ARROW_TYPE_CAST_ERROR] Unsupported type conversion for {type(obj).__name__} to Arrow type {arrow_type}"
         )
 
     class WrappedUDTF:

snowflake/snowpark_connect/utils/upload_java_jar.py

@@ -0,0 +1,57 @@
+#
+# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
+#
+
+import importlib
+import tempfile
+import threading
+import zipfile
+from pathlib import Path
+
+from snowflake.snowpark import Session
+
+_java_initialized_ = threading.Event()
+_java_initialized_lock = threading.Lock()
+JAVA_UDFS_JAR_NAME = "java_udfs-1.0-SNAPSHOT.jar"
+
+
+def upload_java_udf_jar(session: Session) -> None:
+    global _java_initialized_
+    if _java_initialized_.is_set():
+        return
+
+    with _java_initialized_lock:
+        if not _java_initialized_.is_set():
+            stage = session.get_session_stage()
+            try:
+                jar_path = importlib.resources.files(
+                    "snowflake.snowpark_connect.resources"
+                ).joinpath(JAVA_UDFS_JAR_NAME)
+            except NotADirectoryError:
+                # importlib.resource doesn't work in Stage Package method
+                zip_path = Path(__file__).parent.parent.parent.parent
+                jar_path_in_zip = (
+                    f"snowflake/snowpark_connect/resources/{JAVA_UDFS_JAR_NAME}"
+                )
+                temp_dir = tempfile.gettempdir()
+
+                with zipfile.ZipFile(zip_path, "r") as zip_ref:
+                    if jar_path_in_zip not in zip_ref.namelist():
+                        raise FileNotFoundError(
+                            f"[snowpark_connect::invalid_input] {jar_path_in_zip} not found"
+                        )
+                    zip_ref.extract(jar_path_in_zip, temp_dir)
+
+                jar_path = f"{temp_dir}/{jar_path_in_zip}"
+
+            from snowflake.snowpark_connect.resources_initializer import RESOURCE_PATH
+
+            upload_result = session.file.put(
+                str(jar_path), f"{stage}/{RESOURCE_PATH}", overwrite=True
+            )
+
+            if upload_result[0].status != "UPLOADED":
+                raise RuntimeError(
+                    f"[snowpark_connect::internal_error] Failed to upload JAR with UDF definitions to stage: {upload_result[0].message}"
+                )
+            _java_initialized_.set()

snowflake/snowpark_connect/version.py

@@ -2,4 +2,4 @@
 #
 # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #
-VERSION = (0,27,0)
+VERSION = (1,6,0)

snowflake/snowpark_decoder/dp_session.py

@@ -85,8 +85,12 @@ class DataframeProcessorSession:
                 request = spark_proto.ExecutePlanRequest()
             elif any_msg.Is(spark_proto.AnalyzePlanRequest.DESCRIPTOR):
                 request = spark_proto.AnalyzePlanRequest()
+            elif any_msg.Is(spark_proto.AddArtifactsRequest.DESCRIPTOR):
+                request = spark_proto.AddArtifactsRequest()
+            elif any_msg.Is(spark_proto.ArtifactStatusesRequest.DESCRIPTOR):
+                request = spark_proto.ArtifactStatusesRequest()
             else:
-                raise NotImplementedError("Unknown request type")
+                raise NotImplementedError(f"Unknown request type: {any_msg.TypeName()}")
             dp_req_proto.payload.Unpack(request)
             result = self._decoder.request(request)
 
@@ -107,5 +111,5 @@ class DataframeProcessorSession:
             )
             return proto2str(dp_res_proto)
         except Exception:
-            # TODO: SNOW-1857056 error handling
+            # raise the error to GS
             raise

snowflake/snowpark_decoder/spark_decoder.py

@@ -60,6 +60,18 @@ class SparkDecoder:
                     return ResponseEnvelope(
                         config_response=self.servicer.Config(request, ctx)
                     )
+                case proto.AddArtifactsRequest():
+                    return ResponseEnvelope(
+                        add_artifacts_response=self.servicer.AddArtifacts(
+                            iter([request]), ctx
+                        )
+                    )
+                case proto.ArtifactStatusesRequest():
+                    return ResponseEnvelope(
+                        artifact_status_response=self.servicer.ArtifactStatus(
+                            request, ctx
+                        )
+                    )
                 case _:
                     raise NotImplementedError(
                         "Unknown request type: %s" % type(request)

{snowpark_connect-0.27.0.data → snowpark_connect-1.6.0.data}/scripts/snowpark-submit

@@ -312,9 +312,9 @@ def generate_spark_submit_cmd(
             args_for_spark.append(v)
     if args.verbose:
         args_for_spark.append("--verbose")
-        setup_logging(logging.INFO)
+        setup_logging(logging.DEBUG)
     else:
-        setup_logging(logging.ERROR)
+        setup_logging(logging.INFO)
     args_for_spark.append(args.filename)
     args_for_spark.extend(args.app_arguments)
     return args_for_spark

{snowpark_connect-0.27.0.dist-info → snowpark_connect-1.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-connect
-Version: 0.27.0
+Version: 1.6.0
 Summary: Snowpark Connect for Spark
 Author: Snowflake, Inc
 License: Apache License, Version 2.0
@@ -9,29 +9,36 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 License-File: LICENSE-binary
 License-File: NOTICE-binary
+Requires-Dist: snowpark-connect-deps-1==3.56.3
+Requires-Dist: snowpark-connect-deps-2==3.56.3
 Requires-Dist: certifi>=2025.1.31
 Requires-Dist: cloudpickle
-Requires-Dist: fsspec[http]
+Requires-Dist: fsspec
 Requires-Dist: jpype1
-Requires-Dist: protobuf<5.0,>=4.25.3
+Requires-Dist: protobuf<6.32.0,>=4.25.3
 Requires-Dist: s3fs>=2025.3.0
 Requires-Dist: snowflake.core<2,>=1.0.5
-Requires-Dist: snowflake-snowpark-python[pandas]<1.39.0,==1.38.0
+Requires-Dist: snowflake-snowpark-python[pandas]<1.43.0,>=1.42.0
+Requires-Dist: snowflake-connector-python<4.2.0,>=3.18.0
 Requires-Dist: sqlglot>=26.3.8
 Requires-Dist: jaydebeapi
-Requires-Dist: aiobotocore~=2.23.0
+Requires-Dist: aiobotocore<=2.25.0,>=2.23.0
 Requires-Dist: py4j==0.10.9.7
 Requires-Dist: pandas>=1.0.5
 Requires-Dist: pyarrow>=4.0.0
-Requires-Dist: grpcio<1.63,>=1.56.0
-Requires-Dist: grpcio-status<1.63,>=1.56.0
+Requires-Dist: grpcio<=1.71.0,>=1.56.0
+Requires-Dist: grpcio-status<=1.71.0,>=1.56.0
 Requires-Dist: googleapis-common-protos>=1.56.4
 Requires-Dist: numpy<2,>=1.15
+Requires-Dist: gcsfs>=2025.9.0
+Provides-Extra: jdk
+Requires-Dist: jdk4py==17.0.9.2; extra == "jdk"
 Dynamic: author
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: license
 Dynamic: license-file
+Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary