snowpark-connect 0.26.0__py3-none-any.whl → 0.28.0__py3-none-any.whl

snowflake/snowpark_connect/utils/session.py

@@ -8,7 +8,7 @@ from collections.abc import Sequence
 from typing import Any
 
 from snowflake import snowpark
-from snowflake.snowpark.exceptions import SnowparkClientException, SnowparkSQLException
+from snowflake.snowpark.exceptions import SnowparkClientException
 from snowflake.snowpark.session import _get_active_session
 from snowflake.snowpark_connect.constants import DEFAULT_CONNECTION_NAME
 from snowflake.snowpark_connect.utils.describe_query_cache import (
@@ -50,7 +50,10 @@ def _get_current_snowpark_session() -> snowpark.Session | None:
 
 def configure_snowpark_session(session: snowpark.Session):
     """Configure a snowpark session with required parameters and settings."""
-    from snowflake.snowpark_connect.config import global_config
+    from snowflake.snowpark_connect.config import (
+        get_cte_optimization_enabled,
+        global_config,
+    )
 
     logger.info(f"Configuring session {session}")
 
@@ -77,6 +80,14 @@ def configure_snowpark_session(session: snowpark.Session):
     session.connection.arrow_number_to_decimal_setter = True
     session.custom_package_usage_config["enabled"] = True
 
+    # Configure CTE optimization based on session configuration
+    cte_optimization_enabled = get_cte_optimization_enabled()
+    session.cte_optimization_enabled = cte_optimization_enabled
+    logger.info(f"CTE optimization enabled: {cte_optimization_enabled}")
+
+    # Default query tag to be used unless overridden by user using AppName or spark.addTag()
+    query_tag = "SNOWPARK_CONNECT_QUERY"
+
     default_fallback_timezone = "UTC"
     if global_config.spark_sql_session_timeZone is None:
         try:
@@ -104,35 +115,14 @@ def configure_snowpark_session(session: snowpark.Session):
         "QUOTED_IDENTIFIERS_IGNORE_CASE": "false",
         "PYTHON_SNOWPARK_ENABLE_THREAD_SAFE_SESSION": "true",
         "PYTHON_SNOWPARK_USE_SCOPED_TEMP_OBJECTS": "false",  # this is required for creating udfs from sproc
+        "ENABLE_STRUCTURED_TYPES_IN_SNOWPARK_CONNECT_RESPONSE": "true",
+        "QUERY_TAG": f"'{query_tag}'",
     }
 
     session.sql(
         f"ALTER SESSION SET {', '.join([f'{k} = {v}' for k, v in session_params.items()])}"
     ).collect()
 
-    # Rolling ahead in preparation of GS release 9.22 (ETA 8/5/2025). Once 9.22 is past rollback risk, merge this
-    # parameter with other in the session_params dictionary above
-    try:
-        session.sql(
-            "ALTER SESSION SET ENABLE_STRUCTURED_TYPES_IN_SNOWPARK_CONNECT_RESPONSE=true"
-        ).collect()
-    except SnowparkSQLException:
-        logger.debug(
-            "ENABLE_STRUCTURED_TYPES_IN_SNOWPARK_CONNECT_RESPONSE is not defined"
-        )
-    try:
-        session.sql(
-            "ALTER SESSION SET ENABLE_STRUCTURED_TYPES_NATIVE_ARROW_FORMAT=true"
-        ).collect()
-    except SnowparkSQLException:
-        logger.debug("ENABLE_STRUCTURED_TYPES_NATIVE_ARROW_FORMAT is not defined")
-    try:
-        session.sql(
-            "ALTER SESSION SET ENABLE_STRUCTURED_TYPES_IN_CLIENT_RESPONSE=true"
-        ).collect()
-    except SnowparkSQLException:
-        logger.debug("ENABLE_STRUCTURED_TYPES_IN_CLIENT_RESPONSE is not defined")
-
     # Instrument the snowpark session to use a cache for describe queries.
     instrument_session_for_describe_cache(session)
 
@@ -204,5 +194,5 @@ def set_query_tags(spark_tags: Sequence[str]) -> None:
     snowpark_session = get_or_create_snowpark_session()
     spark_tags_str = ",".join(sorted(spark_tags)) if spark_tags else None
 
-    if spark_tags_str != snowpark_session.query_tag:
+    if spark_tags_str and spark_tags_str != snowpark_session.query_tag:
         snowpark_session.query_tag = spark_tags_str

snowflake/snowpark_connect/utils/telemetry.py

@@ -88,6 +88,7 @@ RECORDED_CONFIG_KEYS = {
     "spark.sql.session.localRelationCacheThreshold",
     "spark.sql.mapKeyDedupPolicy",
     "snowpark.connect.sql.passthrough",
+    "snowpark.connect.cte.optimization_enabled",
     "snowpark.connect.iceberg.external_volume",
     "snowpark.connect.sql.identifiers.auto-uppercase",
     "snowpark.connect.udtf.compatibility_mode",
@@ -426,6 +427,58 @@ class Telemetry:
 
         summary["internal_queries"] += 1
 
+    @safe
+    def report_describe_query_cache_lookup(self):
+        """Report a describe query cache lookup."""
+        if self._not_in_request():
+            return
+
+        summary = self._request_summary.get()
+
+        if "describe_cache_lookups" not in summary:
+            summary["describe_cache_lookups"] = 0
+
+        summary["describe_cache_lookups"] += 1
+
+    @safe
+    def report_describe_query_cache_hit(self):
+        """Report a describe query cache hit."""
+        if self._not_in_request():
+            return
+
+        summary = self._request_summary.get()
+
+        if "describe_cache_hits" not in summary:
+            summary["describe_cache_hits"] = 0
+
+        summary["describe_cache_hits"] += 1
+
+    @safe
+    def report_describe_query_cache_expired(self):
+        """Report a describe query cache hit."""
+        if self._not_in_request():
+            return
+
+        summary = self._request_summary.get()
+
+        if "describe_cache_expired" not in summary:
+            summary["describe_cache_expired"] = 0
+
+        summary["describe_cache_expired"] += 1
+
+    @safe
+    def report_describe_query_cache_clear(self, query_prefix: str):
+        """Report a describe query cache clear."""
+        if self._not_in_request():
+            return
+
+        summary = self._request_summary.get()
+
+        if "describe_cache_clears" not in summary:
+            summary["describe_cache_clears"] = []
+
+        summary["describe_cache_clears"].append(query_prefix)
+
     @safe
     def report_udf_usage(self, udf_name: str):
         if self._not_in_request():

snowflake/snowpark_connect/utils/udf_utils.py

@@ -6,7 +6,6 @@
 # So its dependencies are restricted to pandas, snowpark, and, pyspark
 import functools
 import inspect
-from typing import Iterator
 
 import pandas
 import pyspark.sql.connect.proto.expressions_pb2 as expressions_proto
@@ -198,111 +197,75 @@ class ProcessCommonInlineUserDefinedFunction:
 
         needs_struct_conversion = isinstance(self._original_return_type, StructType)
 
-        match self._called_from:
-            case "map_map_partitions":
-                column_names = self._input_column_names
-                eval_type = self._eval_type
+        if not needs_struct_conversion:
+            return snowpark_fn.udf(
+                create_null_safe_wrapper(callable_func),
+                return_type=self._return_type,
+                input_types=self._input_types,
+                name=self._udf_name,
+                replace=self._replace,
+                packages=packages,
+                imports=imports,
+                immutable=self._is_deterministic,
+            )
 
-                def wrapped_function(*args):
-                    result = callable_func(
-                        pandas.DataFrame(iter([list(args)]), columns=column_names)
-                    )
-                    if eval_type == MAP_IN_ARROW_EVAL_TYPE:
-                        if not isinstance(result, Iterator) and not hasattr(
-                            result, "__iter__"
-                        ):
-                            raise RuntimeError(
-                                f"snowpark_connect::UDF_RETURN_TYPE Return type of the user-defined function should be "
-                                f"iterator of pyarrow.RecordBatch, but is {type(result).__name__}"
-                            )
-                        import pyarrow as pa
-
-                        for elem in result:
-                            if not isinstance(elem, pa.RecordBatch):
-                                raise RuntimeError(
-                                    f"snowpark_connect::UDF_RETURN_TYPE Return type of the user-defined function should "
-                                    f"be iterator of pyarrow.RecordBatch, but is iterator of {type(elem).__name__}"
-                                )
-                    return result
-
-                udf_function = create_null_safe_wrapper(wrapped_function)
-                packages += ["pyarrow", "pandas"]
-            case _:
-                if not needs_struct_conversion:
-                    return snowpark_fn.udf(
-                        create_null_safe_wrapper(callable_func),
-                        return_type=self._return_type,
-                        input_types=self._input_types,
-                        name=self._udf_name,
-                        replace=self._replace,
-                        packages=packages,
-                        imports=imports,
-                        immutable=self._is_deterministic,
+        is_pandas_udf, _, return_types, _ = extract_return_input_types(
+            callable_func,
+            self._original_return_type,
+            self._input_types,
+            TempObjectType.FUNCTION,
+        )
+        if is_pandas_udf and isinstance(return_types, PandasDataFrameType):
+            # Snowpark Python UDFs only support returning a Pandas Series.
+            # We change the return type to make the input callable compatible with Snowpark Python UDFs,
+            # and then in the wrapper function we convert the pandas DataFrame of the
+            # original callable to a Pandas Series.
+            original_callable.__annotations__["return"] = pandas.Series
+
+        field_names = [field.name for field in self._original_return_type.fields]
+
+        def struct_wrapper(*args):
+            result = callable_func(*args)
+            if isinstance(result, (tuple, list)):
+                # Convert tuple/list to dict using struct field names
+                if len(result) == len(field_names):
+                    return dict(zip(field_names, result))
+            return result
+
+        def pandas_struct_wrapper(*args):
+            # inspired by the following snowpark modin code to handle Pandas int/bool/null data in Snowflake VariantType
+            # https://github.com/snowflakedb/snowpark-python/blob/e095d5a54f3a697416c3f1df87d239def47a5495/src/snowflake/snowpark/modin/plugin/_internal/apply_utils.py#L1309-L1366
+            def convert_to_snowflake_compatible_type(value):
+                import numpy as np
+                from pandas.api.types import is_scalar
+
+                if is_scalar(value) and pandas.isna(value):
+                    return None
+
+                return (
+                    int(value)
+                    if np.issubdtype(type(value), np.integer)
+                    else (
+                        bool(value) if np.issubdtype(type(value), np.bool_) else value
                     )
-
-                is_pandas_udf, _, return_types, _ = extract_return_input_types(
-                    callable_func,
-                    self._original_return_type,
-                    self._input_types,
-                    TempObjectType.FUNCTION,
                 )
-                if is_pandas_udf and isinstance(return_types, PandasDataFrameType):
-                    # Snowpark Python UDFs only support returning a Pandas Series.
-                    # We change the return type to make the input callable compatible with Snowpark Python UDFs,
-                    # and then in the wrapper function we convert the pandas DataFrame of the
-                    # original callable to a Pandas Series.
-                    original_callable.__annotations__["return"] = pandas.Series
-
-                field_names = [
-                    field.name for field in self._original_return_type.fields
-                ]
-
-                def struct_wrapper(*args):
-                    result = callable_func(*args)
-                    if isinstance(result, (tuple, list)):
-                        # Convert tuple/list to dict using struct field names
-                        if len(result) == len(field_names):
-                            return dict(zip(field_names, result))
-                    return result
-
-                def pandas_struct_wrapper(*args):
-                    # inspired by the following snowpark modin code to handle Pandas int/bool/null data in Snowflake VariantType
-                    # https://github.com/snowflakedb/snowpark-python/blob/e095d5a54f3a697416c3f1df87d239def47a5495/src/snowflake/snowpark/modin/plugin/_internal/apply_utils.py#L1309-L1366
-                    def convert_to_snowflake_compatible_type(value):
-                        import numpy as np
-                        from pandas.api.types import is_scalar
-
-                        if is_scalar(value) and pandas.isna(value):
-                            return None
-
-                        return (
-                            int(value)
-                            if np.issubdtype(type(value), np.integer)
-                            else (
-                                bool(value)
-                                if np.issubdtype(type(value), np.bool_)
-                                else value
-                            )
-                        )
-
-                    result = callable_func(*args)
-                    assert (
-                        len(result) == 1
-                    ), "Expected result to be a single row DataFrame"
-                    # df.applymap doesn't help here, the original type was preserved, hence we convert each value
-                    row_data = [
-                        convert_to_snowflake_compatible_type(value)
-                        for value in result.iloc[0].tolist()
-                    ]
-                    result = pandas.Series([dict(zip(field_names, row_data))])
-                    return result
-
-                if is_pandas_udf:
-                    udf_function = pandas_struct_wrapper
-                    if isinstance(return_types, PandasDataFrameType):
-                        udf_function.__annotations__ = original_callable.__annotations__
-                else:
-                    udf_function = create_null_safe_wrapper(struct_wrapper)
+
+            result = callable_func(*args)
+            assert len(result) == 1, "Expected result to be a single row DataFrame"
+            # df.applymap doesn't help here, the original type was preserved, hence we convert each value
+            row_data = [
+                convert_to_snowflake_compatible_type(value)
+                for value in result.iloc[0].tolist()
+            ]
+            result = pandas.Series([dict(zip(field_names, row_data))])
+            return result
+
+        if is_pandas_udf:
+            udf_function = pandas_struct_wrapper
+            if isinstance(return_types, PandasDataFrameType):
+                udf_function.__annotations__ = original_callable.__annotations__
+        else:
+            udf_function = create_null_safe_wrapper(struct_wrapper)
 
         return snowpark_fn.udf(
             udf_function,

snowflake/snowpark_connect/utils/udtf_helper.py

@@ -261,15 +261,25 @@ def create(session, b64_str, spark_column_names_json_str, input_schema_json_str,
     udf_proto = CommonInlineUserDefinedFunction()
     udf_proto.ParseFromString(restored_bytes)
 
+    if not input_schema_json_str:
+        raise ValueError("Input schema is required for pandas UDTF.")
+    if not return_schema_json_str:
+        raise ValueError("Return schema is required for pandas UDTF.")
+
     spark_column_names = json.loads(spark_column_names_json_str)
-    input_schema = StructType.fromJson(json.loads(input_schema_json_str)) if input_schema_json_str else None
-    return_schema = StructType.fromJson(json.loads(return_schema_json_str)) if return_schema_json_str else None
+    input_schema = StructType.fromJson(json.loads(input_schema_json_str))
+    return_schema = StructType.fromJson(json.loads(return_schema_json_str))
 
-    map_in_arrow_udtf = create_pandas_udtf(
-        udf_proto, spark_column_names,
-        input_schema, return_schema
-    )
-    return map_in_arrow_udtf.name
+    map_in_arrow = udf_proto.WhichOneof("function") == "python_udf" and udf_proto.python_udf.eval_type == 207
+    if map_in_arrow:
+        map_udtf = create_pandas_udtf_with_arrow(
+            udf_proto, spark_column_names, input_schema, return_schema
+        )
+    else:
+        map_udtf = create_pandas_udtf(
+            udf_proto, spark_column_names, input_schema, return_schema
+        )
+    return map_udtf.name
 $$;
 """
     session.sql(create_udtf_sproc_sql).collect()

snowflake/snowpark_connect/version.py

@@ -1,6 +1,5 @@
-#!/usr/bin/env python3
+# !/usr/bin/env python3
 #
 # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #
-
-VERSION = (0, 26, 0)
+VERSION = (0,28,0)

{snowpark_connect-0.26.0.dist-info → snowpark_connect-0.28.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-connect
-Version: 0.26.0
+Version: 0.28.0
 Summary: Snowpark Connect for Spark
 Author: Snowflake, Inc
 License: Apache License, Version 2.0
@@ -16,7 +16,7 @@ Requires-Dist: jpype1
 Requires-Dist: protobuf<5.0,>=4.25.3
 Requires-Dist: s3fs>=2025.3.0
 Requires-Dist: snowflake.core<2,>=1.0.5
-Requires-Dist: snowflake-snowpark-python[pandas]<1.38.0,>=1.37.0
+Requires-Dist: snowflake-snowpark-python[pandas]<1.40.0,==1.39.0
 Requires-Dist: sqlglot>=26.3.8
 Requires-Dist: jaydebeapi
 Requires-Dist: aiobotocore~=2.23.0

{snowpark_connect-0.26.0.dist-info → snowpark_connect-0.28.0.dist-info}/RECORD

@@ -1,20 +1,19 @@
 snowflake/snowpark_connect/__init__.py,sha256=Sml4x1LTNnxZyw6nnDeJrZWUi3eUAR46Rsw6N-wHUSA,605
-snowflake/snowpark_connect/column_name_handler.py,sha256=_bTrguwdiF_cqFvoihFU7f42lNqtJ3Af-9fPvSLbf_A,31919
-snowflake/snowpark_connect/config.py,sha256=KQlRIZsclZfzWe3Co4XBvT_WBeuC6xb9GNAnGEeOWPY,25094
+snowflake/snowpark_connect/column_name_handler.py,sha256=_Z2j4teR_3nsCLjMxhyChGmds_1v6tP51OfkEpmWXWk,27164
+snowflake/snowpark_connect/config.py,sha256=vfuM2TBuuoBe_y9Y_utPelU-9hLoVE-lJszmN8vT5qw,28145
 snowflake/snowpark_connect/constants.py,sha256=FBDxNUxdqWxnf6d5-eanHlYdFFyQqCqvNyZG-uOiO6Q,598
 snowflake/snowpark_connect/control_server.py,sha256=mz3huYo84hgqUB6maZxu3LYyGq7vVL1nv7-7-MjuSYY,1956
-snowflake/snowpark_connect/dataframe_container.py,sha256=sCzMxzb6UIJz1mVpwzSYeVBiqtSzZv5n_pn7FVhbRv0,9114
+snowflake/snowpark_connect/dataframe_container.py,sha256=0ozyUXrWErzM7Gltnb-i2o5ZyXVVeT_HCqpuYliQXwc,8798
 snowflake/snowpark_connect/dataframe_name_handler.py,sha256=aR-CpdGsN2d6tNW0H_F9P-FLe5hDU68zJwKjsrgeA2g,1682
 snowflake/snowpark_connect/date_time_format_mapping.py,sha256=qtQ-JTGR1VRWM2oxM1aYggE_g-BNouffeHxmCk89xkk,16809
 snowflake/snowpark_connect/empty_dataframe.py,sha256=aKO6JkYnArWCpLGcn9BzvTspw2k_c6eAM0mQImAY0J0,428
-snowflake/snowpark_connect/hidden_column.py,sha256=k56-e97vqQmoOFtOtIPy6rSu8mZAz-toKHoIBXYW3CU,1322
-snowflake/snowpark_connect/resources_initializer.py,sha256=njE4_L82evaCwFYPGBUX6OEVaHqww42oGXXmzUylt0I,4424
-snowflake/snowpark_connect/server.py,sha256=JRgj7PXJAPZ1LyoRs6heMh2ISXjhOMIxw3sipXlhQd8,50978
+snowflake/snowpark_connect/resources_initializer.py,sha256=GqBaiFrK9dWec3NLuNZuuxx5Ebw_ZCiMyFrS2KjK2fw,5084
+snowflake/snowpark_connect/server.py,sha256=Hi9FYtIHJXURgMS3On50b5BXJQjMv220_h_jttkHUkI,51015
 snowflake/snowpark_connect/start_server.py,sha256=udegO0rk2FeSnXsIcCIYQW3VRlGDjB_cU8lJ8xSzuM8,942
 snowflake/snowpark_connect/tcm.py,sha256=ftncZFbVO-uyWMhF1_HYKQykB7KobHEYoyQsYbQj1EM,203
 snowflake/snowpark_connect/type_mapping.py,sha256=6Hg-h1iVzVB_FnwG3Sjl-UGr2Itrs4LxVb2Pz5Ue-YA,41566
 snowflake/snowpark_connect/typed_column.py,sha256=Tavii8b4zMj5IWOvN6tlOVmC80W6eQct0pC_tF2nlhU,3867
-snowflake/snowpark_connect/version.py,sha256=n9kt8bTa93V973HaRfyqLqeVkvUhOJqglXJqOAdmsWw,121
+snowflake/snowpark_connect/version.py,sha256=h25IrA984uyrAk0fG8Iy3TJC7b-VG1YTDkoW-sfpxDU,118
 snowflake/snowpark_connect/analyze_plan/__init__.py,sha256=xsIE96jDASko3F-MeNf4T4Gg5ufthS8CejeiJDfri0M,76
 snowflake/snowpark_connect/analyze_plan/map_tree_string.py,sha256=Q3ZD-Z7uForrF7W3mSAjwaiEcIv2KDXr5jPfVbromVg,1470
 snowflake/snowpark_connect/error/__init__.py,sha256=oQo6k4zztLmNF1c5IvJLcS99J6RWY9KBTN3RJ2pKimg,249
@@ -30,14 +29,14 @@ snowflake/snowpark_connect/expression/function_defaults.py,sha256=WEnzc_uzZZltcf
 snowflake/snowpark_connect/expression/hybrid_column_map.py,sha256=2jItaXnFnqcaOIiHKxscFLj7hi2zQE7F3xcher8Zo2U,7614
 snowflake/snowpark_connect/expression/literal.py,sha256=wk5NnLR85SLHe7GoAvSzMTW0B-3yhAiRz4F5SfU2REs,4459
 snowflake/snowpark_connect/expression/map_cast.py,sha256=uxNukg9op0i_sKLhY43fJJJ2-SKZ-8wkRMRBiPikJ0c,14799
-snowflake/snowpark_connect/expression/map_expression.py,sha256=cotjHDdKiHWbQZx3buBcISCnty8TYyUbbUat4W9qhIk,14119
+snowflake/snowpark_connect/expression/map_expression.py,sha256=04EZNHV7bb0KAIhbnmhIgssvafn3XTSE9Ax_uMishP8,15711
 snowflake/snowpark_connect/expression/map_extension.py,sha256=Qm1Jn3JunswD_hO7ornvKpR6WcM3UKO_JpZE6ovH3VM,4939
-snowflake/snowpark_connect/expression/map_sql_expression.py,sha256=bHl7_YrKnWy0k1gMz02HYF8lTZKoamQdv5nFCHEfbzI,25824
+snowflake/snowpark_connect/expression/map_sql_expression.py,sha256=ultWVVcCeXHUlNSkq4PEdeFuezLDELcemFJVSIJczbE,25923
 snowflake/snowpark_connect/expression/map_udf.py,sha256=_om_Kqxm_sm3RhwP4DZbGPUpArX90MoJQm-KwEWrTiQ,8034
-snowflake/snowpark_connect/expression/map_unresolved_attribute.py,sha256=xIJYxs1EJZNs1skX_xgshvcddsqO8jHgR9QUEvwXm9E,18520
+snowflake/snowpark_connect/expression/map_unresolved_attribute.py,sha256=cyyEtAsPqcCpCYulAobSbtPDV3xkCIM0_tq4wdMxuJM,18843
 snowflake/snowpark_connect/expression/map_unresolved_extract_value.py,sha256=A-m-RczZW6xHMjgYR5RV_vzMTpNBRoH3Tk_A1V8z_pk,5382
-snowflake/snowpark_connect/expression/map_unresolved_function.py,sha256=x8LRI_BXgnclTEfCkJp6O8CPkxsYTc4fYGK_zPGpGdM,477028
-snowflake/snowpark_connect/expression/map_unresolved_star.py,sha256=XNJurGS5RLAaIAcalGGBbPJujqg8YZru_RsoB61m0WQ,8865
+snowflake/snowpark_connect/expression/map_unresolved_function.py,sha256=5Ne94Mip18iXmiqpBNO9wVkiDULGhjxD2wU4ueEsSXs,478168
+snowflake/snowpark_connect/expression/map_unresolved_star.py,sha256=gecSxJ9flsn9chTXmYHP8nLzs_209xnVlyNwANDXRFg,8820
 snowflake/snowpark_connect/expression/map_update_fields.py,sha256=bMuZjCOE5wgrZv3ApsoiwEUv-lPgnMWUk-DFX11QuX0,6710
 snowflake/snowpark_connect/expression/map_window_function.py,sha256=apJxtWSnLogMG0F_En67JoGrGRGYQ0I8O0xPRk4TR_o,12152
 snowflake/snowpark_connect/expression/typer.py,sha256=aV2dmsqsJWhZzeJtu-_xLqVZ8REsU0hTYAEpCrbO5y8,4582
@@ -78,7 +77,7 @@ snowflake/snowpark_connect/includes/jars/log4j-core-2.20.0.jar,sha256=YTffhIza7Z
 snowflake/snowpark_connect/includes/jars/log4j-slf4j2-impl-2.20.0.jar,sha256=uN0-TqnP-hjbXzAc2MU5FYZi5pHv1HAaqHtNCZYb2LA,26430
 snowflake/snowpark_connect/includes/jars/paranamer-2.8.3.jar,sha256=qd8Tby6SazeoOKW04iJzQ8OnVdFXJLOoNQtK6ksViUU,37899
 snowflake/snowpark_connect/includes/jars/paranamer-2.8.jar,sha256=aIyxGKYCHYGROOhVIIyVYDFoi-S0eiS7YVvsxjrO3wc,34654
-snowflake/snowpark_connect/includes/jars/sas-scala-udf_2.12-0.1.0.jar,sha256=cl-jS8dYqWuqawLz8zybq2QTbS9Uuf46k8WEU_Cexdo,3242
+snowflake/snowpark_connect/includes/jars/sas-scala-udf_2.12-0.1.0.jar,sha256=2flk7VxMgrNtLKNE6owH8QDS4pQMmWNvSajkjkdtdQA,6613712
 snowflake/snowpark_connect/includes/jars/scala-collection-compat_2.12-2.7.0.jar,sha256=-2iGSgJIrwl51Fkhhp6xIwtZJqph5cCoc3w_c65YLpI,247466
 snowflake/snowpark_connect/includes/jars/scala-library-2.12.18.jar,sha256=5R5mNsADNZ4Qa-pK2Z3vcOYTwpAZDIyE8Q-VYN1bAK4,5433857
 snowflake/snowpark_connect/includes/jars/scala-parser-combinators_2.12-2.3.0.jar,sha256=Fb8T9c4On_QiRgPEQoEmXQUJrDHqgjxGqimlpS4JpXQ,186777
@@ -401,15 +400,15 @@ snowflake/snowpark_connect/relation/map_aggregate.py,sha256=KElVYO62V3jkU8ldTCfT
 snowflake/snowpark_connect/relation/map_catalog.py,sha256=mcx6An4fqHAxy2OhOC66Xe_0aRtYPDGkBEgMK5CfaXU,5822
 snowflake/snowpark_connect/relation/map_column_ops.py,sha256=grleA0S6jyia6T5IGbPOEdz0zI0TkkV7jENzlfzoQlQ,49017
 snowflake/snowpark_connect/relation/map_crosstab.py,sha256=H_J8-IARK6zMEUFrOjKif1St6M20gvBAnP0EuArFHGg,2422
-snowflake/snowpark_connect/relation/map_extension.py,sha256=OeaCtrmvRJBfn6TKJ-tb9ynoBs6QybGOsDJqaJMSkjw,22172
-snowflake/snowpark_connect/relation/map_join.py,sha256=oINd7AMa_O0bLPqsFFeKnZfE0anKa2RAR9QYh3qdbbY,25042
+snowflake/snowpark_connect/relation/map_extension.py,sha256=ItzB9AMG0OAwQhmO8bvIJxVToYhAyQu19pprone7feE,22473
+snowflake/snowpark_connect/relation/map_join.py,sha256=YGOy6nmpb0mqw8D3TzsV4bDlvaq6QFYPwfF5S6YgydU,15201
 snowflake/snowpark_connect/relation/map_local_relation.py,sha256=VBfwBT75GQUe01UOZptwcYsI7zZxaIMQyTOG6kmVuJ0,15219
-snowflake/snowpark_connect/relation/map_map_partitions.py,sha256=2vN7wfe0mR0qzMYeHOmWrgsdCq573j2qQSLnzp7pSFg,5881
-snowflake/snowpark_connect/relation/map_relation.py,sha256=gCko2zjUEBYkiw2GmJerGl-YcRAeONGTIF1GhoGhQl0,12468
+snowflake/snowpark_connect/relation/map_map_partitions.py,sha256=JGiPnqmFFNdHAx54dHyOUwo9JZzyggDd1yLBmu-hasQ,4307
+snowflake/snowpark_connect/relation/map_relation.py,sha256=etQ_SGXAAXNZxqfG6ouh6XVMSbvVLo2NS2Hhj3vOAas,12806
 snowflake/snowpark_connect/relation/map_row_ops.py,sha256=x1Jqircy4I0iiSljx3zbq0YxwGvGzPcXIY8_nhtl2PM,30528
 snowflake/snowpark_connect/relation/map_sample_by.py,sha256=8ALQbeUsB89sI3uiUFqG3w1A4TtOzOAL4umdKp6-c38,1530
-snowflake/snowpark_connect/relation/map_show_string.py,sha256=fuYCuThp7V3VPb5NETJvW0JDf9Xv2qCxdvBmlbHgF7c,3254
-snowflake/snowpark_connect/relation/map_sql.py,sha256=UphF9mWLV8csDLNlFTTC0Xq2-OfoHI1ztcyAE5X6d-M,83484
+snowflake/snowpark_connect/relation/map_show_string.py,sha256=GgKg0qp1pGqSC7TuFedTU4IYaIm-Fx23OJ1LfkcGOHw,3382
+snowflake/snowpark_connect/relation/map_sql.py,sha256=BvGVUG2fH_oyym8DVoMXnJy4Zzi_hSy-wa-x1xcapTE,101175
 snowflake/snowpark_connect/relation/map_stats.py,sha256=kqRYvix8RfluTKx1cAy9JhBUv6arYQHgfxpP1R4QwBM,13985
 snowflake/snowpark_connect/relation/map_subquery_alias.py,sha256=rHgE9XUzuWWkjNPtJz3Sxzz2aFo690paHKZh9frqPXk,1456
 snowflake/snowpark_connect/relation/map_udtf.py,sha256=cfDnbZ3TRJ6eb0EVResu6GL-OwQpaEabWLbrhgWnkRw,13316
@@ -433,30 +432,30 @@ snowflake/snowpark_connect/relation/read/reader_config.py,sha256=PMh1R5IjqqTwiAA
 snowflake/snowpark_connect/relation/read/utils.py,sha256=rIIM6d2WXHh7MLGyHNiRc9tS8b0dmyFQr7rHepIYJOU,4111
 snowflake/snowpark_connect/relation/write/__init__.py,sha256=xsIE96jDASko3F-MeNf4T4Gg5ufthS8CejeiJDfri0M,76
 snowflake/snowpark_connect/relation/write/jdbc_write_dbapi.py,sha256=GI9FyGZuQQNV-6Q8Ob-Xr0im3iAPdH-Jkyx8bjwbOuE,11931
-snowflake/snowpark_connect/relation/write/map_write.py,sha256=UE5A-m1NWgbziGbUiZh48YCZHwogwTegAXEPEcFuepg,39361
+snowflake/snowpark_connect/relation/write/map_write.py,sha256=zHWjjzYxj4rNPzybFYPnCK3mUSJMjmXv2FMiMAG0krY,43014
 snowflake/snowpark_connect/relation/write/map_write_jdbc.py,sha256=1nOWRgjtZzfRwnSRGFP9V6mqBVlGhSBr2KHGHbe4JMU,1404
 snowflake/snowpark_connect/resources/java_udfs-1.0-SNAPSHOT.jar,sha256=tVyOp6tXxu9nm6SDufwQiGzfH3pnuh_7PowsMZxOolY,9773
 snowflake/snowpark_connect/utils/__init__.py,sha256=xsIE96jDASko3F-MeNf4T4Gg5ufthS8CejeiJDfri0M,76
-snowflake/snowpark_connect/utils/artifacts.py,sha256=CDSO9ve_yHvT1CKjoLnwiQNarsM5cMUdypGgpRELtK0,2407
+snowflake/snowpark_connect/utils/artifacts.py,sha256=TkHZ2uNfZiphgtG91V1_c_h9yP9dP677BXUMymboCss,2498
 snowflake/snowpark_connect/utils/cache.py,sha256=bAyoNBW6Z1ui9BuppDywbQeG6fdju4L-owFHzySOTnk,3382
-snowflake/snowpark_connect/utils/concurrent.py,sha256=2UrQfJPWFrtpx131514t-9aXFDphbGI3N2pumLrltNk,3543
-snowflake/snowpark_connect/utils/context.py,sha256=nZVj1EMH-aP32cKFCcHqDut3lPtdt0A8DniBzMxCS4k,12132
-snowflake/snowpark_connect/utils/describe_query_cache.py,sha256=2VcPgGP9bUpdIhnN2s_MOG8oGHKX0hS0rT7Y26MJb3A,9001
+snowflake/snowpark_connect/utils/concurrent.py,sha256=BTbUmvupLzUSRd6L7kKk9yIXFdqlDOkXebVMaECRD-A,3653
+snowflake/snowpark_connect/utils/context.py,sha256=W9j9eC-lbGp7tfXWhnvI88CVOcLGspYEhEgxGPYVbYE,13288
+snowflake/snowpark_connect/utils/describe_query_cache.py,sha256=HByBsP-XJ1PAk414OQtwIqcFnqpC2zq-ZK_YxR4O5gg,9100
 snowflake/snowpark_connect/utils/env_utils.py,sha256=g__Uio5ae20Tm1evahIHdJUXQYPmjNUT_kYPSIy5JDU,1488
 snowflake/snowpark_connect/utils/external_udxf_cache.py,sha256=eSZHMbjTxnkg78IlbG5P1Vno6j5ag_FSI0c4Xi2UyPs,1044
-snowflake/snowpark_connect/utils/identifiers.py,sha256=YgtVIQGuUnnTiNdtRficdBwUICWaWkJltjOPnTnfrak,3881
+snowflake/snowpark_connect/utils/identifiers.py,sha256=A65-aQpuhTjOjGv0cmlE8fTNCI9_ONn-AYatKb_VfGM,7635
 snowflake/snowpark_connect/utils/interrupt.py,sha256=_awhdrzF1KQO-EQThneEcfMg3Zxed4p3HtMpkcAb6ek,2790
-snowflake/snowpark_connect/utils/io_utils.py,sha256=noBlKpJvzEA6iwLjFgBVGlCLlzjZ16-w0fsGimTyBAQ,1039
-snowflake/snowpark_connect/utils/pandas_udtf_utils.py,sha256=QwdLGLg5bX0JJTrWKfL4Ou4MIp443cryEbrZNBrulNE,4207
+snowflake/snowpark_connect/utils/io_utils.py,sha256=xu0AMrHy-qsY7TfdIxzWChf0hU_7bnvm3Ruk0XScRns,1781
+snowflake/snowpark_connect/utils/pandas_udtf_utils.py,sha256=3WA_9IVRZL8fnwIHo048LTg62-bPGfCDUZzYd-zjzQQ,7564
 snowflake/snowpark_connect/utils/profiling.py,sha256=ttdHzQUYarvTqJASLNuKFIax7ejO39Tv1mHKl0QjRkg,1519
-snowflake/snowpark_connect/utils/scala_udf_utils.py,sha256=YDHmhqsjgBeuKdv1TaInEUUOvrnbrU3_toQY_MfEygk,22895
-snowflake/snowpark_connect/utils/session.py,sha256=dxt5VoqsQqBZd3OaEpoPfA0U-uNFWDP-HWtrcoey7_w,7770
+snowflake/snowpark_connect/utils/scala_udf_utils.py,sha256=RFDDMmgQ_xBWk98kdfWaw4Hla3ZqYf3UAijU4uAUNdA,23011
+snowflake/snowpark_connect/utils/session.py,sha256=BWwpIbhdplQOhvY4GpbLgZ-BrJLidrWR6A_cJmGFCq8,7372
 snowflake/snowpark_connect/utils/snowpark_connect_logging.py,sha256=23bvbALGqixJ3Ap9QWM3OpcKNK-sog2mr9liSmvwqYU,1123
-snowflake/snowpark_connect/utils/telemetry.py,sha256=BRDX1eRXcww2SAiBlVr2fQVR2I6gH50qSCFkOop_x4M,20736
+snowflake/snowpark_connect/utils/telemetry.py,sha256=4TImrE4XY5rfiesHyPQQ0xtU63qnN1wUpDXUP12PAvA,22259
 snowflake/snowpark_connect/utils/udf_cache.py,sha256=8K7kASEhvpnp-l1hjzovjyboUzKctDq7PiGXRcNv6Lg,12125
 snowflake/snowpark_connect/utils/udf_helper.py,sha256=g-TxTs4ARyJWYgADrosfQQG-ykBBQdm1g5opslxJq_E,12563
-snowflake/snowpark_connect/utils/udf_utils.py,sha256=Ey_clI2d-1m4zc51JWNsl8m0i93cS2L9NboLGGVwII8,14213
-snowflake/snowpark_connect/utils/udtf_helper.py,sha256=nTWbrFMc4hHLkalT4nJIsC_c00T6GFkB55JHOw1D-wg,14571
+snowflake/snowpark_connect/utils/udf_utils.py,sha256=pxERcJKum2M5jHxPqsl1NgHFAqZV4RxoEnSLxJV5ups,12009
+snowflake/snowpark_connect/utils/udtf_helper.py,sha256=9B_1iOckfFXQfVv-UHerIJ32fDd4qucKaHGqxtBEi4w,14969
 snowflake/snowpark_connect/utils/udtf_utils.py,sha256=wHO5V0BXRQOLqAYos1vGt8bbdH7jBvD2gwspWywjTtY,33110
 snowflake/snowpark_connect/utils/udxf_import_utils.py,sha256=pPtcaGsyh0tUdy0aAvNqTj04jqPKlEcGmvaZDP9O8Gc,536
 snowflake/snowpark_connect/utils/xxhash64.py,sha256=ysJRxhBPf25LeNhM1RK_H36MWl6q6C6vBRHa-jIna_A,7477
@@ -465,17 +464,17 @@ snowflake/snowpark_decoder/dp_session.py,sha256=HIr3TfKgYl5zqaGR5xpFU9ZVkcaTB9I8
 snowflake/snowpark_decoder/spark_decoder.py,sha256=EQiCvBiqB736Bc17o3gnYGtcYVcyfGxroO5e1kbe1Co,2885
 snowflake/snowpark_decoder/_internal/proto/generated/DataframeProcessorMsg_pb2.py,sha256=2eSDqeyfMvmIJ6_rF663DrEe1dg_anrP4OpVJNTJHaQ,2598
 snowflake/snowpark_decoder/_internal/proto/generated/DataframeProcessorMsg_pb2.pyi,sha256=aIH23k52bXdw5vO3RtM5UcOjDPaWsJFx1SRUSk3qOK8,6142
-snowpark_connect-0.26.0.data/scripts/snowpark-connect,sha256=yZ94KqbWACxnwV8mpg8NjILvvRNjnF8B3cs3ZFNuIM4,1546
-snowpark_connect-0.26.0.data/scripts/snowpark-session,sha256=NMAHSonTo-nmOZSkQNlszUC0jLJ8QWEDUsUmMe2UAOw,190
-snowpark_connect-0.26.0.data/scripts/snowpark-submit,sha256=Zd98H9W_d0dIqMSkQLdHyW5G3myxF0t4c3vNBt2nD6A,12056
-snowpark_connect-0.26.0.dist-info/licenses/LICENSE-binary,sha256=fmBlX39HwTlBUyiKEznaLZGuxQy-7ndLLG_rTXjF02Y,22916
-snowpark_connect-0.26.0.dist-info/licenses/LICENSE.txt,sha256=Ff9cPv4xu0z7bnMTHzo4vDncOShsy33w4oJMA2xjn6c,11365
-snowpark_connect-0.26.0.dist-info/licenses/NOTICE-binary,sha256=elMF8brgGNJwOz8YdorzBF6-U8ZhR8F-77FfGkZng7U,57843
+snowpark_connect-0.28.0.data/scripts/snowpark-connect,sha256=yZ94KqbWACxnwV8mpg8NjILvvRNjnF8B3cs3ZFNuIM4,1546
+snowpark_connect-0.28.0.data/scripts/snowpark-session,sha256=NMAHSonTo-nmOZSkQNlszUC0jLJ8QWEDUsUmMe2UAOw,190
+snowpark_connect-0.28.0.data/scripts/snowpark-submit,sha256=Zd98H9W_d0dIqMSkQLdHyW5G3myxF0t4c3vNBt2nD6A,12056
+snowpark_connect-0.28.0.dist-info/licenses/LICENSE-binary,sha256=fmBlX39HwTlBUyiKEznaLZGuxQy-7ndLLG_rTXjF02Y,22916
+snowpark_connect-0.28.0.dist-info/licenses/LICENSE.txt,sha256=Ff9cPv4xu0z7bnMTHzo4vDncOShsy33w4oJMA2xjn6c,11365
+snowpark_connect-0.28.0.dist-info/licenses/NOTICE-binary,sha256=elMF8brgGNJwOz8YdorzBF6-U8ZhR8F-77FfGkZng7U,57843
 spark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spark/connect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spark/connect/envelope_pb2.py,sha256=7Gc6OUA3vaCuTCIKamb_Iiw7W9jPTcWNEv1im20eWHM,2726
 spark/connect/envelope_pb2.pyi,sha256=VXTJSPpcxzB_dWqVdvPY4KkPhJfh0WmkX7SNHWoLhx0,3358
-snowpark_connect-0.26.0.dist-info/METADATA,sha256=IQ7OoHx9dFxYtBVJaEW2lsjfTj8-K-uyLr3Po1q2SBM,1594
-snowpark_connect-0.26.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-snowpark_connect-0.26.0.dist-info/top_level.txt,sha256=ExnWqVpoTHRG99fu_AxXZVOz8c-De7nNu0yFCGylM8I,16
-snowpark_connect-0.26.0.dist-info/RECORD,,
+snowpark_connect-0.28.0.dist-info/METADATA,sha256=ZYd6i8wRgv9zajEKwzZ_v8jMcLYvPwN4yOGeLF6pj7g,1594
+snowpark_connect-0.28.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+snowpark_connect-0.28.0.dist-info/top_level.txt,sha256=ExnWqVpoTHRG99fu_AxXZVOz8c-De7nNu0yFCGylM8I,16
+snowpark_connect-0.28.0.dist-info/RECORD,,

snowflake/snowpark_connect/hidden_column.py

@@ -1,39 +0,0 @@
-#
-# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
-#
-
-
-class HiddenColumn:
-    """
-    Represents a hidden column in a Snowflake table.
-
-    Hidden columns are not visible in standard queries but can be accessed
-    directly if needed. This class provides a way to reference such columns
-    in Snowpark operations
-    """
-
-    def __init__(
-        self,
-        hidden_snowpark_name: str,
-        spark_name: str,
-        visible_snowpark_name: str,
-        qualifiers: list[str] | None = None,
-        original_position: int | None = None,
-    ) -> None:
-        """
-        Initializes a HiddenColumn instance.
-
-        Args:
-            name (str): The name of the hidden column.
-        """
-
-        # The Snowpark internal name for the hidden column
-        self.hidden_snowpark_name = hidden_snowpark_name
-        # The Spark name for the hidden column
-        self.spark_name = spark_name
-        # The left side visible Snowpark name for the dropped right side column
-        self.visible_snowpark_name = visible_snowpark_name
-        # Qualifiers for the hidden column (e.g., table or schema names)
-        self.qualifiers = qualifiers if qualifiers is not None else []
-        # The position of the hidden column in the original schema
-        self.original_position = original_position