snowpark-connect 0.28.0__tar.gz → 0.28.1__tar.gz

{snowpark_connect-0.28.0/src/snowpark_connect.egg-info → snowpark_connect-0.28.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-connect
-Version: 0.28.0
+Version: 0.28.1
 Summary: Snowpark Connect for Spark
 Author: Snowflake, Inc
 License: Apache License, Version 2.0

{snowpark_connect-0.28.0 → snowpark_connect-0.28.1}/src/snowflake/snowpark_connect/config.py

@@ -283,7 +283,7 @@ class SessionConfig:
     default_session_config = {
         "snowpark.connect.sql.identifiers.auto-uppercase": "all_except_columns",
         "snowpark.connect.sql.passthrough": "false",
-        "snowpark.connect.cte.optimization_enabled": "true",
+        "snowpark.connect.cte.optimization_enabled": "false",
         "snowpark.connect.udtf.compatibility_mode": "false",
         "snowpark.connect.views.duplicate_column_names_handling_mode": "rename",
         "spark.sql.execution.pythonUDTF.arrow.enabled": "false",

{snowpark_connect-0.28.0 → snowpark_connect-0.28.1}/src/snowflake/snowpark_connect/execute_plan/map_execution_command.py

@@ -1,18 +1,16 @@
 #
 # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #
-import re
-import uuid
 from collections import Counter
 
 import pyspark.sql.connect.proto.base_pb2 as proto_base
 import pyspark.sql.connect.proto.relations_pb2 as relation_proto
 
-from snowflake.snowpark import DataFrame, Session
-from snowflake.snowpark.exceptions import SnowparkSQLException
+from snowflake.snowpark.types import StructField, StructType
 from snowflake.snowpark_connect.column_name_handler import ColumnNames
 from snowflake.snowpark_connect.config import global_config, sessions_config
 from snowflake.snowpark_connect.constants import SERVER_SIDE_SESSION_ID
+from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.execute_plan.utils import pandas_to_arrow_batches_bytes
 from snowflake.snowpark_connect.expression import map_udf
 from snowflake.snowpark_connect.relation import map_udtf
@@ -28,10 +26,7 @@ from snowflake.snowpark_connect.utils.snowpark_connect_logging import logger
 from snowflake.snowpark_connect.utils.telemetry import (
     SnowparkConnectNotImplementedError,
 )
-
-_INTERNAL_VIEW_PREFIX = "__SC_RENAMED_V_"
-
-_CREATE_VIEW_PATTERN = re.compile(r"create\s+or\s+replace\s+view", re.IGNORECASE)
+from snowflake.snowpark_connect.utils.temporary_view_cache import register_temp_view
 
 
 def _create_column_rename_map(
@@ -98,32 +93,35 @@ def map_execution_command(
             input_df = input_df_container.dataframe
             column_map = input_df_container.column_map
 
+            # TODO: Remove code handling deduplication. When view are not materialized we don't have to care about it.
             session_config = sessions_config[get_session_id()]
             duplicate_column_names_handling_mode = session_config[
                 "snowpark.connect.views.duplicate_column_names_handling_mode"
             ]
 
+            spark_columns = input_df_container.column_map.get_spark_columns()
             # rename columns to match spark names
             if duplicate_column_names_handling_mode == "rename":
                 # deduplicate column names by appending _DEDUP_1, _DEDUP_2, etc.
-                input_df = input_df.rename(
-                    _create_column_rename_map(column_map.columns, True)
-                )
+                rename_map = _create_column_rename_map(column_map.columns, True)
+                snowpark_columns = list(rename_map.values())
+                input_df = input_df.rename(rename_map)
             elif duplicate_column_names_handling_mode == "drop":
                 # Drop duplicate column names by removing all but the first occurrence.
                 duplicated_columns, remaining_columns = _find_duplicated_columns(
                     column_map.columns
                 )
+                rename_map = _create_column_rename_map(remaining_columns, False)
+                snowpark_columns = list(rename_map.values())
+                spark_columns = list(dict.fromkeys(spark_columns))
                 if len(duplicated_columns) > 0:
                     input_df = input_df.drop(*duplicated_columns)
-                input_df = input_df.rename(
-                    _create_column_rename_map(remaining_columns, False)
-                )
+                input_df = input_df.rename(rename_map)
             else:
                 # rename columns without deduplication
-                input_df = input_df.rename(
-                    _create_column_rename_map(column_map.columns, False)
-                )
+                rename_map = _create_column_rename_map(column_map.columns, True)
+                snowpark_columns = list(rename_map.values())
+                input_df = input_df.rename(rename_map)
 
             if req.is_global:
                 view_name = [global_config.spark_sql_globalTempDatabase, req.name]
@@ -132,18 +130,23 @@ def map_execution_command(
             view_name = [
                 spark_to_sf_single_id_with_unquoting(part) for part in view_name
             ]
+            joined_view_name = ".".join(view_name)
 
-            if req.replace:
-                try:
-                    input_df.create_or_replace_temp_view(view_name)
-                except SnowparkSQLException as exc:
-                    if _is_error_caused_by_view_referencing_itself(exc):
-                        # This error is caused by statement with self reference like `CREATE VIEW A AS SELECT X FROM A`.
-                        _create_chained_view(input_df, view_name)
-                    else:
-                        raise
-            else:
-                input_df.create_temp_view(view_name)
+            schema = StructType(
+                [
+                    StructField(field.name, field.datatype)
+                    for field in input_df.schema.fields
+                ]
+            )
+            input_df_container = DataFrameContainer.create_with_column_mapping(
+                dataframe=input_df,
+                spark_column_names=spark_columns,
+                snowpark_column_names=snowpark_columns,
+                parent_column_name_map=input_df_container.column_map,
+                cached_schema_getter=lambda: schema,
+            )
+
+            register_temp_view(joined_view_name, input_df_container, req.replace)
         case "write_stream_operation_start":
             match request.plan.command.write_stream_operation_start.format:
                 case "console":
@@ -207,43 +210,3 @@ def map_execution_command(
             raise SnowparkConnectNotImplementedError(
                 f"Command type {other} not implemented"
             )
-
-
-def _generate_random_builtin_view_name() -> str:
-    return _INTERNAL_VIEW_PREFIX + str(uuid.uuid4()).replace("-", "")
-
-
-def _is_error_caused_by_view_referencing_itself(exc: Exception) -> bool:
-    return "view definition refers to view being defined" in str(exc).lower()
-
-
-def _create_chained_view(input_df: DataFrame, view_name: str) -> None:
-    """
-    In order to create a view, which references itself, Spark would here take the previous
-    definition of A and paste it in place of `FROM A`. Snowflake would fail in such case, so
-    as a workaround, we create a chain of internal views instead. This function:
-    1. Renames previous definition of A to some internal name (instead of deleting).
-    2. Adjusts the DDL of a new statement to reference the name of a renmaed internal view, instead of itself.
-    """
-
-    session = Session.get_active_session()
-
-    view_name = ".".join(view_name)
-
-    tmp_name = _generate_random_builtin_view_name()
-    old_name_replacement = _generate_random_builtin_view_name()
-
-    input_df.create_or_replace_temp_view(tmp_name)
-
-    session.sql(f"ALTER VIEW {view_name} RENAME TO {old_name_replacement}").collect()
-
-    ddl: str = session.sql(f"SELECT GET_DDL('VIEW', '{tmp_name}')").collect()[0][0]
-
-    ddl = ddl.replace(view_name, old_name_replacement)
-
-    # GET_DDL result doesn't contain `TEMPORARY`, it's likely a bug.
-    ddl = _CREATE_VIEW_PATTERN.sub("create or replace temp view", ddl)
-
-    session.sql(ddl).collect()
-
-    session.sql(f"ALTER VIEW {tmp_name} RENAME TO {view_name}").collect()

{snowpark_connect-0.28.0 → snowpark_connect-0.28.1}/src/snowflake/snowpark_connect/relation/catalogs/snowflake_catalog.py

@@ -13,7 +13,6 @@ from snowflake.core.exceptions import APIError, NotFoundError
 from snowflake.core.schema import Schema
 from snowflake.core.table import Table, TableColumn
 
-from snowflake.snowpark import functions
 from snowflake.snowpark._internal.analyzer.analyzer_utils import (
     quote_name_without_upper_casing,
     unquote_if_quoted,
@@ -34,12 +33,19 @@ from snowflake.snowpark_connect.relation.catalogs.abstract_spark_catalog import
 )
 from snowflake.snowpark_connect.type_mapping import proto_to_snowpark_type
 from snowflake.snowpark_connect.utils.identifiers import (
+    FQN,
+    spark_to_sf_single_id_with_unquoting,
     split_fully_qualified_spark_name,
 )
 from snowflake.snowpark_connect.utils.session import get_or_create_snowpark_session
 from snowflake.snowpark_connect.utils.telemetry import (
     SnowparkConnectNotImplementedError,
 )
+from snowflake.snowpark_connect.utils.temporary_view_cache import (
+    get_temp_view,
+    get_temp_view_normalized_names,
+    unregister_temp_view,
+)
 from snowflake.snowpark_connect.utils.udf_cache import cached_udf
 
 
@@ -203,6 +209,93 @@ class SnowflakeCatalog(AbstractSparkCatalog):
             exists = False
         return pandas.DataFrame({"exists": [exists]})
 
+    def _get_temp_view_prefixes(self, spark_dbName: str | None) -> list[str]:
+        if spark_dbName is None:
+            return []
+        return [
+            quote_name_without_upper_casing(part)
+            for part in split_fully_qualified_spark_name(spark_dbName)
+        ]
+
+    def _list_temp_views(
+        self,
+        spark_dbName: str | None = None,
+        pattern: str | None = None,
+    ) -> typing.Tuple[
+        list[str | None],
+        list[list[str | None]],
+        list[str],
+        list[str | None],
+        list[str | None],
+        list[bool],
+    ]:
+        catalogs: list[str | None] = list()
+        namespaces: list[list[str | None]] = list()
+        names: list[str] = list()
+        descriptions: list[str | None] = list()
+        table_types: list[str | None] = list()
+        is_temporaries: list[bool] = list()
+
+        temp_views_prefix = ".".join(self._get_temp_view_prefixes(spark_dbName))
+        normalized_spark_dbName = (
+            temp_views_prefix.lower()
+            if global_config.spark_sql_caseSensitive
+            else temp_views_prefix
+        )
+        normalized_global_temp_database_name = (
+            quote_name_without_upper_casing(
+                global_config.spark_sql_globalTempDatabase.lower()
+            )
+            if global_config.spark_sql_caseSensitive
+            else quote_name_without_upper_casing(
+                global_config.spark_sql_globalTempDatabase
+            )
+        )
+
+        temp_views = get_temp_view_normalized_names()
+        null_safe_pattern = pattern if pattern is not None else ""
+
+        for temp_view in temp_views:
+            normalized_temp_view = (
+                temp_view.lower()
+                if global_config.spark_sql_caseSensitive
+                else temp_view
+            )
+            fqn = FQN.from_string(temp_view)
+            normalized_schema = (
+                fqn.schema.lower()
+                if fqn.schema is not None and global_config.spark_sql_caseSensitive
+                else fqn.schema
+            )
+
+            is_global_view = normalized_global_temp_database_name == normalized_schema
+            is_local_temp_view = fqn.schema is None
+            # Temporary views are always shown if they match the pattern
+            matches_prefix = (
+                normalized_spark_dbName == normalized_schema or is_local_temp_view
+            )
+            if matches_prefix and bool(
+                re.match(null_safe_pattern, normalized_temp_view)
+            ):
+                names.append(unquote_if_quoted(fqn.name))
+                catalogs.append(None)
+                namespaces.append(
+                    [global_config.spark_sql_globalTempDatabase]
+                    if is_global_view
+                    else []
+                )
+                descriptions.append(None)
+                table_types.append("TEMPORARY")
+                is_temporaries.append(True)
+        return (
+            catalogs,
+            namespaces,
+            names,
+            descriptions,
+            table_types,
+            is_temporaries,
+        )
+
     def listTables(
         self,
         spark_dbName: str | None = None,
@@ -232,8 +325,7 @@ class SnowflakeCatalog(AbstractSparkCatalog):
             schema=sf_quote(sf_schema),
             pattern=_normalize_identifier(pattern),
         )
-        names: list[str] = list()
-        catalogs: list[str] = list()
+        catalogs: list[str | None] = list()
         namespaces: list[list[str | None]] = list()
         names: list[str] = list()
         descriptions: list[str | None] = list()
@@ -253,6 +345,22 @@ class SnowflakeCatalog(AbstractSparkCatalog):
             descriptions.append(o[6] if o[6] else None)
             table_types.append("PERMANENT")
             is_temporaries.append(False)
+
+        (
+            non_materialized_catalogs,
+            non_materialized_namespaces,
+            non_materialized_names,
+            non_materialized_descriptions,
+            non_materialized_table_types,
+            non_materialized_is_temporaries,
+        ) = self._list_temp_views(spark_dbName, pattern)
+        catalogs.extend(non_materialized_catalogs)
+        namespaces.extend(non_materialized_namespaces)
+        names.extend(non_materialized_names)
+        descriptions.extend(non_materialized_descriptions)
+        table_types.extend(non_materialized_table_types)
+        is_temporaries.extend(non_materialized_is_temporaries)
+
         return pandas.DataFrame(
             {
                 "name": names,
@@ -297,6 +405,36 @@ class SnowflakeCatalog(AbstractSparkCatalog):
         spark_tableName: str,
     ) -> pandas.DataFrame:
         """Listing a single table/view with provided name that's accessible in Snowflake."""
+
+        def _get_temp_view():
+            spark_table_name_parts = [
+                quote_name_without_upper_casing(part)
+                for part in split_fully_qualified_spark_name(spark_tableName)
+            ]
+            spark_view_name = ".".join(spark_table_name_parts)
+            temp_view = get_temp_view(spark_view_name)
+            if temp_view:
+                return pandas.DataFrame(
+                    {
+                        "name": [unquote_if_quoted(spark_table_name_parts[-1])],
+                        "catalog": [None],
+                        "namespace": [
+                            [unquote_if_quoted(spark_table_name_parts[-2])]
+                            if len(spark_table_name_parts) > 1
+                            else []
+                        ],
+                        "description": [None],
+                        "tableType": ["TEMPORARY"],
+                        "isTemporary": [True],
+                    }
+                )
+            return None
+
+        # Attempt to get the view from the non materialized views first
+        temp_view = _get_temp_view()
+        if temp_view is not None:
+            return temp_view
+
         sp_catalog = get_or_create_snowpark_session().catalog
         catalog, sf_database, sf_schema, table_name = _process_multi_layer_identifier(
             spark_tableName
@@ -360,12 +498,64 @@ class SnowflakeCatalog(AbstractSparkCatalog):
             exists = False
         return pandas.DataFrame({"exists": [exists]})
 
+    def _list_temp_view_columns(
+        self,
+        spark_tableName: str,
+        spark_dbName: typing.Optional[str] = None,
+    ):
+        spark_view_name_parts = [
+            quote_name_without_upper_casing(part)
+            for part in split_fully_qualified_spark_name(spark_tableName)
+        ]
+        spark_view_name_parts = (
+            self._get_temp_view_prefixes(spark_dbName) + spark_view_name_parts
+        )
+        spark_view_name = ".".join(spark_view_name_parts)
+        temp_view = get_temp_view(spark_view_name)
+
+        if not temp_view:
+            return None
+
+        names: list[str] = list()
+        descriptions: list[str | None] = list()
+        data_types: list[str] = list()
+        nullables: list[bool] = list()
+        is_partitions: list[bool] = list()
+        is_buckets: list[bool] = list()
+
+        for field, spark_column in zip(
+            temp_view.dataframe.schema.fields,
+            temp_view.column_map.get_spark_columns(),
+        ):
+            names.append(spark_column)
+            descriptions.append(None)
+            data_types.append(field.datatype.simpleString())
+            nullables.append(field.nullable)
+            is_partitions.append(False)
+            is_buckets.append(False)
+
+        return pandas.DataFrame(
+            {
+                "name": names,
+                "description": descriptions,
+                "dataType": data_types,
+                "nullable": nullables,
+                "isPartition": is_partitions,
+                "isBucket": is_buckets,
+            }
+        )
+
     def listColumns(
         self,
         spark_tableName: str,
         spark_dbName: typing.Optional[str] = None,
     ) -> pandas.DataFrame:
         """List all columns in a table/view, optionally database name filter can be provided."""
+
+        temp_view_columns = self._list_temp_view_columns(spark_tableName, spark_dbName)
+        if temp_view_columns is not None:
+            return temp_view_columns
+
         sp_catalog = get_or_create_snowpark_session().catalog
         columns: list[TableColumn] | None = None
         if spark_dbName is None:
@@ -455,17 +645,15 @@ class SnowflakeCatalog(AbstractSparkCatalog):
         spark_view_name: str,
     ) -> DataFrameContainer:
         session = get_or_create_snowpark_session()
-        schema = global_config.spark_sql_globalTempDatabase
-        result_df = session.sql(
-            "drop view if exists identifier(?)",
-            params=[f"{sf_quote(schema)}.{sf_quote(spark_view_name)}"],
-        )
-        result_df = result_df.select(
-            functions.contains('"status"', functions.lit("successfully dropped")).alias(
-                "value"
+        if not spark_view_name == "":
+            schema = global_config.spark_sql_globalTempDatabase
+            result = unregister_temp_view(
+                f"{spark_to_sf_single_id_with_unquoting(schema)}.{spark_to_sf_single_id_with_unquoting(spark_view_name)}"
             )
-        )
+        else:
+            result = False
         columns = ["value"]
+        result_df = session.createDataFrame([result], schema=columns)
         return DataFrameContainer.create_with_column_mapping(
             dataframe=result_df,
             spark_column_names=columns,
@@ -479,15 +667,14 @@ class SnowflakeCatalog(AbstractSparkCatalog):
     ) -> DataFrameContainer:
         """Drop the current temporary view."""
         session = get_or_create_snowpark_session()
-        result = session.sql(
-            "drop view if exists identifier(?)",
-            params=[sf_quote(spark_view_name)],
-        ).collect()
-        view_was_dropped = (
-            len(result) == 1 and "successfully dropped" in result[0]["status"]
-        )
-        result_df = session.createDataFrame([(view_was_dropped,)], schema=["value"])
         columns = ["value"]
+        if spark_view_name:
+            result = unregister_temp_view(
+                spark_to_sf_single_id_with_unquoting(spark_view_name)
+            )
+        else:
+            result = False
+        result_df = session.createDataFrame([result], schema=columns)
         return DataFrameContainer.create_with_column_mapping(
             dataframe=result_df,
             spark_column_names=columns,