snowpark-connect 0.27.0__py3-none-any.whl → 1.6.0__py3-none-any.whl

snowflake/snowpark_connect/column_qualifier.py

@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
+#
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from snowflake.snowpark._internal.analyzer.analyzer_utils import (
+    quote_name_without_upper_casing,
+)
+
+
+@dataclass(frozen=True)
+class ColumnQualifier:
+    parts: tuple[str, ...]
+
+    def __post_init__(self) -> None:
+        if not all(isinstance(x, str) for x in self.parts):
+            raise TypeError("ColumnQualifier.parts must be strings")
+
+    @property
+    def is_empty(self) -> bool:
+        return len(self.parts) == 0
+
+    def all_qualified_names(self, name: str) -> list[str]:
+        qualifier_parts = self.parts
+        qualifier_prefixes = [
+            ".".join(quote_name_without_upper_casing(x) for x in qualifier_parts[i:])
+            for i in range(len(qualifier_parts))
+        ]
+        return [f"{prefix}.{name}" for prefix in qualifier_prefixes]
+
+    def to_upper(self):
+        return ColumnQualifier(tuple(part.upper() for part in self.parts))
+
+    def matches(self, target: ColumnQualifier) -> bool:
+        if self.is_empty or target.is_empty:
+            return False
+        # If the column has fewer qualifiers than the target, it cannot match
+        if len(self.parts) < len(target.parts):
+            return False
+        return self.parts[-len(target.parts) :] == target.parts

snowflake/snowpark_connect/config.py

@@ -8,7 +8,7 @@ import re
 import sys
 from collections import defaultdict
 from copy import copy, deepcopy
-from typing import Any
+from typing import Any, Dict, Optional
 
 import jpype
 import pyspark.sql.connect.proto.base_pb2 as proto_base
@@ -17,11 +17,17 @@ from tzlocal import get_localzone_name
 from snowflake import snowpark
 from snowflake.snowpark._internal.analyzer.analyzer_utils import (
     quote_name_without_upper_casing,
+    unquote_if_quoted,
 )
 from snowflake.snowpark.exceptions import SnowparkSQLException
 from snowflake.snowpark.types import TimestampTimeZone, TimestampType
+from snowflake.snowpark_connect.error.error_codes import ErrorCodes
+from snowflake.snowpark_connect.error.error_utils import attach_custom_error_code
 from snowflake.snowpark_connect.utils.concurrent import SynchronizedDict
-from snowflake.snowpark_connect.utils.context import get_session_id
+from snowflake.snowpark_connect.utils.context import (
+    get_jpype_jclass_lock,
+    get_spark_session_id,
+)
 from snowflake.snowpark_connect.utils.external_udxf_cache import (
     clear_external_udxf_cache,
 )
@@ -139,9 +145,21 @@ class GlobalConfig:
         "spark.sql.parser.quotedRegexColumnNames": "false",
         # custom configs
         "snowpark.connect.version": ".".join(map(str, sas_version)),
+        "snowpark.connect.temporary.views.create_in_snowflake": "false",
         # Control whether repartition(n) on a DataFrame forces splitting into n files during writes
         # This matches spark behavior more closely, but introduces overhead.
         "snowflake.repartition.for.writes": "false",
+        "snowpark.connect.structured_types.fix": "true",
+        # Local relation optimization: Use List[Row] for small data, PyArrow for large data
+        # Enabled in production by default to improve performance for createDataFrame on small local relations.
+        # Disabled in tests by default unless explicitly enabled to stabilize flaky tests that are not applying row ordering.
+        # SNOW-2719980: Remove this flag after test fragility issues are resolved
+        "snowpark.connect.localRelation.optimizeSmallData": "true",
+        "spark.sql.execution.arrow.maxRecordsPerBatch": "10000",  # TODO: no-op
+        # USE_VECTORIZED_SCANNER will become the default in a future BCR; Snowflake recommends setting it to TRUE for new workloads.
+        # This significantly reduces latency for loading Parquet files by downloading only relevant columnar sections into memory.
+        "snowpark.connect.parquet.useVectorizedScanner": "true",
+        "spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue": "false",
     }
 
     boolean_config_list = [
@@ -150,11 +168,14 @@ class GlobalConfig:
         "spark.sql.repl.eagerEval.enabled",
         "spark.sql.crossJoin.enabled",
         "spark.sql.caseSensitive",
+        "snowpark.connect.localRelation.optimizeSmallData",
+        "snowpark.connect.parquet.useVectorizedScanner",
         "spark.sql.ansi.enabled",
         "spark.sql.legacy.allowHashOnMapType",
         "spark.Catalog.databaseFilterInformationSchema",
         "spark.sql.parser.quotedRegexColumnNames",
         "snowflake.repartition.for.writes",
+        "spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue",
     ]
 
     int_config_list = [
@@ -257,21 +278,34 @@ SESSION_CONFIG_KEY_WHITELIST = {
     "spark.sql.execution.pythonUDTF.arrow.enabled",
     "spark.sql.tvf.allowMultipleTableArguments.enabled",
     "snowpark.connect.sql.passthrough",
+    "snowpark.connect.cte.optimization_enabled",
     "snowpark.connect.iceberg.external_volume",
     "snowpark.connect.sql.identifiers.auto-uppercase",
+    "snowpark.connect.sql.partition.external_table_location",
     "snowpark.connect.udtf.compatibility_mode",
     "snowpark.connect.views.duplicate_column_names_handling_mode",
-    "enable_snowflake_extension_behavior",
+    "snowpark.connect.temporary.views.create_in_snowflake",
+    "snowpark.connect.enable_snowflake_extension_behavior",
+    "spark.hadoop.fs.s3a.server-side-encryption.key",
+    "spark.hadoop.fs.s3a.assumed.role.arn",
+    "snowpark.connect.describe_cache_ttl_seconds",
+    "mapreduce.fileoutputcommitter.marksuccessfuljobs",
+    "spark.sql.parquet.enable.summary-metadata",
+    "parquet.enable.summary-metadata",
 }
-AZURE_SAS_KEY = re.compile(
+AZURE_ACCOUNT_KEY = re.compile(
     r"^fs\.azure\.sas\.[^\.]+\.[^\.]+\.blob\.core\.windows\.net$"
 )
+AZURE_SAS_KEY = re.compile(
+    r"^fs\.azure\.sas\.fixed\.token\.[^\.]+\.dfs\.core\.windows\.net$"
+)
 
 
 def valid_session_config_key(key: str):
     return (
         key in SESSION_CONFIG_KEY_WHITELIST  # AWS session keys
         or AZURE_SAS_KEY.match(key)  # Azure session keys
+        or AZURE_ACCOUNT_KEY.match(key)  # Azure account keys
     )
 
 
@@ -279,17 +313,23 @@ class SessionConfig:
     """This class contains the session configuration for the Spark Server."""
 
     default_session_config = {
-        "snowpark.connect.sql.identifiers.auto-uppercase": "all_except_columns",
         "snowpark.connect.sql.passthrough": "false",
+        "snowpark.connect.cte.optimization_enabled": "false",
         "snowpark.connect.udtf.compatibility_mode": "false",
         "snowpark.connect.views.duplicate_column_names_handling_mode": "rename",
         "spark.sql.execution.pythonUDTF.arrow.enabled": "false",
         "spark.sql.tvf.allowMultipleTableArguments.enabled": "true",
-        "enable_snowflake_extension_behavior": "false",
+        "snowpark.connect.enable_snowflake_extension_behavior": "false",
+        "snowpark.connect.describe_cache_ttl_seconds": "300",
+        "snowpark.connect.sql.partition.external_table_location": None,
+        "mapreduce.fileoutputcommitter.marksuccessfuljobs": "false",
+        "spark.sql.parquet.enable.summary-metadata": "false",
+        "parquet.enable.summary-metadata": "false",
     }
 
     def __init__(self) -> None:
         self.config = deepcopy(self.default_session_config)
+        self.table_metadata: Dict[str, Dict[str, Any]] = {}
 
     def __getitem__(self, item: str) -> str:
         return self.get(item)
@@ -344,9 +384,11 @@ def route_config_proto(
                 if not pair.HasField("value"):
                     from pyspark.errors import IllegalArgumentException
 
-                    raise IllegalArgumentException(
+                    exception = IllegalArgumentException(
                         f"Cannot set config '{pair.key}' to None"
                     )
+                    attach_custom_error_code(exception, ErrorCodes.INVALID_CONFIG_VALUE)
+                    raise exception
 
                 set_config_param(
                     config.session_id, pair.key, pair.value, snowpark_session
@@ -429,7 +471,11 @@ def route_config_proto(
                 pair.value = str(global_config.is_modifiable(key)).lower()
             return res
         case _:
-            raise SnowparkConnectNotImplementedError(f"Unexpected request {config}")
+            exception = SnowparkConnectNotImplementedError(
+                f"Unexpected request {config}"
+            )
+            attach_custom_error_code(exception, ErrorCodes.UNSUPPORTED_OPERATION)
+            raise exception
 
 
 def set_config_param(
@@ -469,19 +515,27 @@ def _verify_static_config_not_modified(key: str) -> None:
     # https://github.com/apache/spark/blob/v3.5.3/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala#L161
     # Spark does not allow to modify static configurations at runtime.
     if global_config.is_static_config(key) and global_config.is_set(key):
-        raise ValueError(f"Cannot modify the value of a static config: {key}")
+        exception = ValueError(f"Cannot modify the value of a static config: {key}")
+        attach_custom_error_code(exception, ErrorCodes.CONFIG_CHANGE_NOT_ALLOWED)
+        raise exception
 
 
 def _verify_is_valid_config_value(key: str, value: Any) -> None:
     if key in CONFIG_ALLOWED_VALUES and value not in CONFIG_ALLOWED_VALUES[key]:
-        raise ValueError(
+        exception = ValueError(
             f"Invalid value '{value}' for key '{key}'. Allowed values: {', '.join(CONFIG_ALLOWED_VALUES[key])}."
         )
+        attach_custom_error_code(exception, ErrorCodes.INVALID_CONFIG_VALUE)
+        raise exception
 
 
 def _verify_is_not_readonly_config(key):
     if key in global_config.readonly_config_list:
-        raise ValueError(f"Config with key {key} is read-only and cannot be modified.")
+        exception = ValueError(
+            f"Config with key {key} is read-only and cannot be modified."
+        )
+        attach_custom_error_code(exception, ErrorCodes.CONFIG_CHANGE_NOT_ALLOWED)
+        raise exception
 
 
 def set_jvm_timezone(timezone_id: str):
@@ -498,10 +552,13 @@ def set_jvm_timezone(timezone_id: str):
         RuntimeError: If JVM is not started
     """
     if not jpype.isJVMStarted():
-        raise RuntimeError("JVM must be started before setting timezone")
+        exception = RuntimeError("JVM must be started before setting timezone")
+        attach_custom_error_code(exception, ErrorCodes.INTERNAL_ERROR)
+        raise exception
 
     try:
-        TimeZone = jpype.JClass("java.util.TimeZone")
+        with get_jpype_jclass_lock():
+            TimeZone = jpype.JClass("java.util.TimeZone")
         new_timezone = TimeZone.getTimeZone(timezone_id)
         TimeZone.setDefault(new_timezone)
 
@@ -513,7 +570,9 @@ def set_jvm_timezone(timezone_id: str):
 def reset_jvm_timezone_to_system_default():
     """Reset JVM timezone to the system's default timezone"""
     if not jpype.isJVMStarted():
-        raise RuntimeError("JVM must be started first")
+        exception = RuntimeError("JVM must be started first")
+        attach_custom_error_code(exception, ErrorCodes.INTERNAL_ERROR)
+        raise exception
 
     try:
         TimeZone = jpype.JClass("java.util.TimeZone")
@@ -522,9 +581,13 @@ def reset_jvm_timezone_to_system_default():
             f"Reset JVM timezone to system default: {TimeZone.getDefault().getID()}"
         )
     except jpype.JException as e:
-        raise RuntimeError(f"Java exception while resetting timezone: {e}")
+        exception = RuntimeError(f"Java exception while resetting timezone: {e}")
+        attach_custom_error_code(exception, ErrorCodes.INTERNAL_ERROR)
+        raise exception
     except Exception as e:
-        raise RuntimeError(f"Unexpected error resetting JVM timezone: {e}")
+        exception = RuntimeError(f"Unexpected error resetting JVM timezone: {e}")
+        attach_custom_error_code(exception, ErrorCodes.INTERNAL_ERROR)
+        raise exception
 
 
 def set_snowflake_parameters(
@@ -569,27 +632,98 @@ def set_snowflake_parameters(
                         snowpark_session.use_database(db)
                     case (prev, curr) if prev != curr:
                         snowpark_session.use_schema(prev)
+        case "snowpark.connect.cte.optimization_enabled":
+            # Set CTE optimization on the snowpark session
+            cte_enabled = str_to_bool(value)
+            snowpark_session.cte_optimization_enabled = cte_enabled
+            logger.info(f"Updated snowpark session CTE optimization: {cte_enabled}")
+        case "snowpark.connect.structured_types.fix":
+            # TODO: SNOW-2367714 Remove this once the fix is automatically enabled in Snowpark
+            snowpark.context._enable_fix_2360274 = str_to_bool(value)
+            logger.info(f"Updated snowpark session structured types fix: {value}")
         case _:
             pass
 
 
 def get_boolean_session_config_param(name: str) -> bool:
-    session_config = sessions_config[get_session_id()]
+    session_config = sessions_config[get_spark_session_id()]
     return str_to_bool(session_config[name])
 
 
+def get_string_session_config_param(name: str) -> str:
+    session_config = sessions_config[get_spark_session_id()]
+    return str(session_config[name])
+
+
+def get_cte_optimization_enabled() -> bool:
+    """Get the CTE optimization configuration setting."""
+    return get_boolean_session_config_param("snowpark.connect.cte.optimization_enabled")
+
+
+def get_success_file_generation_enabled() -> bool:
+    """Get the _SUCCESS file generation configuration setting."""
+    return get_boolean_session_config_param(
+        "mapreduce.fileoutputcommitter.marksuccessfuljobs"
+    )
+
+
+def get_parquet_metadata_generation_enabled() -> bool:
+    """
+    Get the Parquet metadata file generation configuration setting.
+    """
+    return get_boolean_session_config_param(
+        "spark.sql.parquet.enable.summary-metadata"
+    ) or get_boolean_session_config_param("parquet.enable.summary-metadata")
+
+
+def get_describe_cache_ttl_seconds() -> int:
+    """Get the describe query cache TTL from session config, with a default fallback."""
+    session_config: SessionConfig = sessions_config[get_spark_session_id()]
+    default_ttl: str = SessionConfig.default_session_config[
+        "snowpark.connect.describe_cache_ttl_seconds"
+    ]
+    try:
+        ttl_str = session_config.get(
+            "snowpark.connect.describe_cache_ttl_seconds", default_ttl
+        )
+        return int(ttl_str)
+    except ValueError:  # fallback to default ttl
+        return int(default_ttl)
+
+
+def should_create_temporary_view_in_snowflake() -> bool:
+    return str_to_bool(
+        global_config["snowpark.connect.temporary.views.create_in_snowflake"]
+    )
+
+
 def auto_uppercase_column_identifiers() -> bool:
-    session_config = sessions_config[get_session_id()]
-    return session_config[
+    session_config = sessions_config[get_spark_session_id()]
+    auto_upper_case_config = session_config[
         "snowpark.connect.sql.identifiers.auto-uppercase"
-    ].lower() in ("all", "only_columns")
+    ]
+    if auto_upper_case_config:
+        return auto_upper_case_config.lower() in ("all", "only_columns")
+
+    return not global_config.spark_sql_caseSensitive
 
 
 def auto_uppercase_non_column_identifiers() -> bool:
-    session_config = sessions_config[get_session_id()]
-    return session_config[
+    session_config = sessions_config[get_spark_session_id()]
+    auto_upper_case_config = session_config[
         "snowpark.connect.sql.identifiers.auto-uppercase"
-    ].lower() in ("all", "all_except_columns")
+    ]
+    if auto_upper_case_config:
+        return auto_upper_case_config.lower() in ("all", "all_except_columns")
+
+    return not global_config.spark_sql_caseSensitive
+
+
+def external_table_location() -> Optional[str]:
+    session_config = sessions_config[get_spark_session_id()]
+    return session_config.get(
+        "snowpark.connect.sql.partition.external_table_location", None
+    )
 
 
 def parse_imports(session: snowpark.Session, imports: str | None) -> None:
@@ -613,3 +747,83 @@ def get_timestamp_type():
             # shouldn't happen since `spark.sql.timestampType` is always defined, and `spark.conf.unset` sets it to default (TIMESTAMP_LTZ)
             timestamp_type = TimestampType(TimestampTimeZone.LTZ)
     return timestamp_type
+
+
+def record_table_metadata(
+    table_identifier: str,
+    table_type: str,
+    data_source: str,
+    supports_column_rename: bool = True,
+) -> None:
+    """
+    Record metadata about a table for Spark compatibility checks.
+
+    Args:
+        table_identifier: Full table identifier (catalog.database.table)
+        table_type: "v1" or "v2"
+        data_source: Source format (parquet, csv, iceberg, etc.)
+        supports_column_rename: Whether the table supports RENAME COLUMN
+    """
+    session_id = get_spark_session_id()
+    session_config = sessions_config[session_id]
+
+    # Normalize table identifier for consistent lookup
+    # Use the full catalog.database.table identifier to avoid conflicts
+    normalized_identifier = table_identifier.upper().strip('"')
+
+    session_config.table_metadata[normalized_identifier] = {
+        "table_type": table_type,
+        "data_source": data_source,
+        "supports_column_rename": supports_column_rename,
+    }
+
+
+def get_table_metadata(table_identifier: str) -> Dict[str, Any] | None:
+    """
+    Get stored metadata for a table.
+
+    Args:
+        table_identifier: Full table identifier (catalog.database.table)
+
+    Returns:
+        Table metadata dict or None if not found
+    """
+    session_id = get_spark_session_id()
+    session_config = sessions_config[session_id]
+
+    normalized_identifier = unquote_if_quoted(table_identifier).upper()
+
+    return session_config.table_metadata.get(normalized_identifier)
+
+
+def check_table_supports_operation(table_identifier: str, operation: str) -> bool:
+    """
+    Check if a table supports a given operation based on metadata and config.
+
+    Args:
+        table_identifier: Full table identifier (catalog.database.table)
+        operation: Operation to check (e.g., "rename_column")
+
+    Returns:
+        True if operation is supported, False if should be blocked
+    """
+    table_metadata = get_table_metadata(table_identifier)
+
+    if not table_metadata:
+        return True
+
+    session_id = get_spark_session_id()
+    session_config = sessions_config[session_id]
+    enable_extensions = str_to_bool(
+        session_config.get(
+            "snowpark.connect.enable_snowflake_extension_behavior", "false"
+        )
+    )
+
+    if enable_extensions:
+        return True
+
+    if operation == "rename_column":
+        return table_metadata.get("supports_column_rename", True)
+
+    return True

snowflake/snowpark_connect/constants.py

@@ -16,3 +16,5 @@ MAP_IN_ARROW_EVAL_TYPE = 207  # eval_type for mapInArrow operations
 COLUMN_METADATA_COLLISION_KEY = "{expr_id}_{key}"
 
 DUPLICATE_KEY_FOUND_ERROR_TEMPLATE = "Duplicate key found: {key}. You can set spark.sql.mapKeyDedupPolicy to LAST_WIN to deduplicate map keys with last wins policy."
+
+SPARK_VERSION = "3.5.3"

snowflake/snowpark_connect/dataframe_container.py

@@ -4,14 +4,40 @@
 
 from __future__ import annotations
 
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Callable
 
 from snowflake import snowpark
 from snowflake.snowpark.types import StructField, StructType
-from snowflake.snowpark_connect.hidden_column import HiddenColumn
+from snowflake.snowpark_connect.column_qualifier import ColumnQualifier
 
 if TYPE_CHECKING:
+    import pyspark.sql.connect.proto.expressions_pb2 as expressions_proto
+
     from snowflake.snowpark_connect.column_name_handler import ColumnNameMap
+    from snowflake.snowpark_connect.typed_column import TypedColumn
+
+
+@dataclass
+class AggregateMetadata:
+    """
+    Metadata about aggregation for resolving expressions in ORDER BY.
+
+    When a Sort operation follows an Aggregate operation, ORDER BY expressions
+    may reference:
+    1. Grouping columns from the GROUP BY clause
+    2. Aggregate result columns (aliases)
+    3. Expressions on pre-aggregation columns (e.g., year(date) where date existed before GROUP BY)
+
+    This metadata enables hybrid resolution similar to HAVING clause.
+    """
+
+    input_column_map: ColumnNameMap
+    input_dataframe: snowpark.DataFrame
+    grouping_expressions: list[expressions_proto.Expression]
+    aggregate_expressions: list[expressions_proto.Expression]
+    spark_columns: list[str]
+    raw_aggregations: list[tuple[str, TypedColumn]]
 
 
 class DataFrameContainer:
@@ -30,6 +56,9 @@ class DataFrameContainer:
         alias: str | None = None,
         cached_schema_getter: Callable[[], StructType] | None = None,
         partition_hint: int | None = None,
+        can_be_cached: bool = True,
+        can_be_materialized: bool = True,
+        aggregate_metadata: AggregateMetadata | None = None,
     ) -> None:
         """
         Initialize a new DataFrameContainer.
@@ -41,12 +70,16 @@ class DataFrameContainer:
             alias: Optional alias for the DataFrame
             cached_schema_getter: Optional function to get cached schema
             partition_hint: Optional partition count from repartition() operations
+            aggregate_metadata: Optional metadata about aggregation for ORDER BY resolution
         """
         self._dataframe = dataframe
         self._column_map = self._create_default_column_map(column_map)
         self._table_name = table_name
         self._alias = alias
         self._partition_hint = partition_hint
+        self._can_be_cached = can_be_cached
+        self._can_be_materialized = can_be_materialized
+        self._aggregate_metadata = aggregate_metadata
 
         if cached_schema_getter is not None:
             self._apply_cached_schema_getter(cached_schema_getter)
@@ -59,13 +92,16 @@ class DataFrameContainer:
         snowpark_column_names: list[str],
         snowpark_column_types: list | None = None,
         column_metadata: dict | None = None,
-        column_qualifiers: list[list[str]] | None = None,
+        column_qualifiers: list[set[ColumnQualifier]] | None = None,
         parent_column_name_map: ColumnNameMap | None = None,
-        hidden_columns: set[HiddenColumn] | None = None,
         table_name: str | None = None,
         alias: str | None = None,
         cached_schema_getter: Callable[[], StructType] | None = None,
         partition_hint: int | None = None,
+        equivalent_snowpark_names: list[set[str]] | None = None,
+        column_is_hidden: list[bool] | None = None,
+        can_be_cached: bool = True,
+        aggregate_metadata: AggregateMetadata | None = None,
     ) -> DataFrameContainer:
         """
         Create a new container with complete column mapping configuration.
@@ -78,11 +114,14 @@ class DataFrameContainer:
             column_metadata: Optional metadata dictionary
             column_qualifiers: Optional column qualifiers
             parent_column_name_map: Optional parent column name map
-            hidden_columns: Optional list of hidden column names
             table_name: Optional table name
             alias: Optional alias
             cached_schema_getter: Optional function to get cached schema
             partition_hint: Optional partition count from repartition() operations
+            equivalent_snowpark_names: list of sets with old snowpark names that can be resolved with an existing column
+            column_is_hidden: Optional list of booleans indicating whether each column is hidden
+            can_be_cached: Optional boolean indicating if the dataframe can be cached
+            aggregate_metadata: Optional metadata about aggregation for ORDER BY resolution
 
         Returns:
             A new DataFrameContainer instance
@@ -101,7 +140,8 @@ class DataFrameContainer:
             column_metadata,
             column_qualifiers,
             parent_column_name_map,
-            hidden_columns,
+            equivalent_snowpark_names,
+            column_is_hidden,
         )
 
         # Determine the schema getter to use
@@ -129,8 +169,25 @@ class DataFrameContainer:
             alias=alias,
             cached_schema_getter=final_schema_getter,
             partition_hint=partition_hint,
+            can_be_cached=can_be_cached,
+            aggregate_metadata=aggregate_metadata,
         )
 
+    @property
+    def can_be_cached(self) -> bool:
+        """Indicate if the DataFrame can be cached in df_cache"""
+        return self._can_be_cached
+
+    @property
+    def can_be_materialized(self) -> bool:
+        """Indicate if the DataFrame can be materialized in df_cache"""
+        return self._can_be_materialized
+
+    def without_materialization(self):
+        """Prevent the DataFrame from being materialized in df_cache"""
+        self._can_be_materialized = False
+        return self
+
     @property
     def dataframe(self) -> snowpark.DataFrame:
         """Get the underlying Snowpark DataFrame."""
@@ -224,9 +281,10 @@ class DataFrameContainer:
         spark_column_names: list[str],
         snowpark_column_names: list[str],
         column_metadata: dict | None = None,
-        column_qualifiers: list[list[str]] | None = None,
+        column_qualifiers: list[set[ColumnQualifier]] | None = None,
         parent_column_name_map: ColumnNameMap | None = None,
-        hidden_columns: set[HiddenColumn] | None = None,
+        equivalent_snowpark_names: list[set[str]] | None = None,
+        column_is_hidden: list[bool] | None = None,
     ) -> ColumnNameMap:
         """Create a ColumnNameMap with the provided configuration."""
         from snowflake.snowpark_connect.column_name_handler import ColumnNameMap
@@ -237,7 +295,8 @@ class DataFrameContainer:
             column_metadata=column_metadata,
             column_qualifiers=column_qualifiers,
             parent_column_name_map=parent_column_name_map,
-            hidden_columns=hidden_columns,
+            equivalent_snowpark_names=equivalent_snowpark_names,
+            column_is_hidden=column_is_hidden,
         )
 
     @staticmethod
@@ -262,3 +321,38 @@ class DataFrameContainer:
                 )
             ]
         )
+
+    def without_hidden_columns(self) -> DataFrameContainer:
+        from snowflake.snowpark_connect.column_name_handler import ColumnNameMap
+
+        if not any(c.is_hidden for c in self._column_map.columns):
+            return self
+
+        hidden_column_names = [
+            c.snowpark_name for c in self._column_map.columns if c.is_hidden
+        ]
+        visible_columns = [c for c in self._column_map.columns if not c.is_hidden]
+
+        filtered_df = self._dataframe.drop(hidden_column_names)
+        filtered_column_map = ColumnNameMap(
+            spark_column_names=[c.spark_name for c in visible_columns],
+            snowpark_column_names=[c.snowpark_name for c in visible_columns],
+            column_metadata=self._column_map.column_metadata,
+            column_qualifiers=[c.qualifiers for c in visible_columns],
+            parent_column_name_map=self._column_map._parent_column_name_map,
+        )
+
+        return DataFrameContainer(
+            dataframe=filtered_df,
+            column_map=filtered_column_map,
+            table_name=self._table_name,
+            alias=self._alias,
+            cached_schema_getter=lambda: StructType(
+                [
+                    field
+                    for field in self._dataframe.schema.fields
+                    if field.name not in hidden_column_names
+                ]
+            ),
+            partition_hint=self._partition_hint,
+        )