snowpark-connect 0.29.0__py3-none-any.whl → 0.30.1__py3-none-any.whl

snowflake/snowpark_connect/proto/snowflake_expression_ext_pb2.py

@@ -16,7 +16,7 @@ from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_e
 from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1esnowflake_expression_ext.proto\x12\rsnowflake.ext\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\"\x98\x01\n\x0c\x45xpExtension\x12@\n\x0enamed_argument\x18\x01 \x01(\x0b\x32&.snowflake.ext.NamedArgumentExpressionH\x00\x12@\n\x13subquery_expression\x18\x02 \x01(\x0b\x32!.snowflake.ext.SubqueryExpressionH\x00\x42\x04\n\x02op\"P\n\x17NamedArgumentExpression\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.Expression\"\xf4\x04\n\x12SubqueryExpression\x12&\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.Relation\x12\x45\n\rsubquery_type\x18\x02 \x01(\x0e\x32..snowflake.ext.SubqueryExpression.SubqueryType\x12Q\n\x11table_arg_options\x18\x03 \x01(\x0b\x32\x31.snowflake.ext.SubqueryExpression.TableArgOptionsH\x00\x88\x01\x01\x12\x35\n\x12in_subquery_values\x18\x04 \x03(\x0b\x32\x19.spark.connect.Expression\x1a\xbb\x01\n\x0fTableArgOptions\x12\x31\n\x0epartition_spec\x18\x01 \x03(\x0b\x32\x19.spark.connect.Expression\x12\x37\n\norder_spec\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrder\x12\"\n\x15with_single_partition\x18\x03 \x01(\x08H\x00\x88\x01\x01\x42\x18\n\x16_with_single_partition\"\x90\x01\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x12\x1b\n\x17SUBQUERY_TYPE_TABLE_ARG\x10\x03\x12\x14\n\x10SUBQUERY_TYPE_IN\x10\x04\x42\x14\n\x12_table_arg_optionsb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1esnowflake_expression_ext.proto\x12\rsnowflake.ext\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\"\xde\x01\n\x0c\x45xpExtension\x12@\n\x0enamed_argument\x18\x01 \x01(\x0b\x32&.snowflake.ext.NamedArgumentExpressionH\x00\x12@\n\x13subquery_expression\x18\x02 \x01(\x0b\x32!.snowflake.ext.SubqueryExpressionH\x00\x12\x44\n\x10interval_literal\x18\x03 \x01(\x0b\x32(.snowflake.ext.IntervalLiteralExpressionH\x00\x42\x04\n\x02op\"P\n\x17NamedArgumentExpression\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.Expression\"\xf4\x04\n\x12SubqueryExpression\x12&\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.Relation\x12\x45\n\rsubquery_type\x18\x02 \x01(\x0e\x32..snowflake.ext.SubqueryExpression.SubqueryType\x12Q\n\x11table_arg_options\x18\x03 \x01(\x0b\x32\x31.snowflake.ext.SubqueryExpression.TableArgOptionsH\x00\x88\x01\x01\x12\x35\n\x12in_subquery_values\x18\x04 \x03(\x0b\x32\x19.spark.connect.Expression\x1a\xbb\x01\n\x0fTableArgOptions\x12\x31\n\x0epartition_spec\x18\x01 \x03(\x0b\x32\x19.spark.connect.Expression\x12\x37\n\norder_spec\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrder\x12\"\n\x15with_single_partition\x18\x03 \x01(\x08H\x00\x88\x01\x01\x42\x18\n\x16_with_single_partition\"\x90\x01\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x12\x1b\n\x17SUBQUERY_TYPE_TABLE_ARG\x10\x03\x12\x14\n\x10SUBQUERY_TYPE_IN\x10\x04\x42\x14\n\x12_table_arg_options\"\x9f\x01\n\x19IntervalLiteralExpression\x12\x32\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.Literal\x12\x18\n\x0bstart_field\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x16\n\tend_field\x18\x03 \x01(\x05H\x01\x88\x01\x01\x42\x0e\n\x0c_start_fieldB\x0c\n\n_end_fieldb\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -24,13 +24,15 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'snowflake_expression_ext_pb
 if _descriptor._USE_C_DESCRIPTORS == False:
   DESCRIPTOR._options = None
   _globals['_EXPEXTENSION']._serialized_start=114
-  _globals['_EXPEXTENSION']._serialized_end=266
-  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_start=268
-  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_end=348
-  _globals['_SUBQUERYEXPRESSION']._serialized_start=351
-  _globals['_SUBQUERYEXPRESSION']._serialized_end=979
-  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_start=623
-  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_end=810
-  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_start=813
-  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_end=957
+  _globals['_EXPEXTENSION']._serialized_end=336
+  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_start=338
+  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_end=418
+  _globals['_SUBQUERYEXPRESSION']._serialized_start=421
+  _globals['_SUBQUERYEXPRESSION']._serialized_end=1049
+  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_start=693
+  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_end=880
+  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_start=883
+  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_end=1027
+  _globals['_INTERVALLITERALEXPRESSION']._serialized_start=1052
+  _globals['_INTERVALLITERALEXPRESSION']._serialized_end=1211
 # @@protoc_insertion_point(module_scope)

snowflake/snowpark_connect/proto/snowflake_expression_ext_pb2.pyi

@@ -9,12 +9,14 @@ from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Map
 DESCRIPTOR: _descriptor.FileDescriptor
 
 class ExpExtension(_message.Message):
-    __slots__ = ("named_argument", "subquery_expression")
+    __slots__ = ("named_argument", "subquery_expression", "interval_literal")
     NAMED_ARGUMENT_FIELD_NUMBER: _ClassVar[int]
     SUBQUERY_EXPRESSION_FIELD_NUMBER: _ClassVar[int]
+    INTERVAL_LITERAL_FIELD_NUMBER: _ClassVar[int]
     named_argument: NamedArgumentExpression
     subquery_expression: SubqueryExpression
-    def __init__(self, named_argument: _Optional[_Union[NamedArgumentExpression, _Mapping]] = ..., subquery_expression: _Optional[_Union[SubqueryExpression, _Mapping]] = ...) -> None: ...
+    interval_literal: IntervalLiteralExpression
+    def __init__(self, named_argument: _Optional[_Union[NamedArgumentExpression, _Mapping]] = ..., subquery_expression: _Optional[_Union[SubqueryExpression, _Mapping]] = ..., interval_literal: _Optional[_Union[IntervalLiteralExpression, _Mapping]] = ...) -> None: ...
 
 class NamedArgumentExpression(_message.Message):
     __slots__ = ("key", "value")
@@ -56,3 +58,13 @@ class SubqueryExpression(_message.Message):
     table_arg_options: SubqueryExpression.TableArgOptions
     in_subquery_values: _containers.RepeatedCompositeFieldContainer[_expressions_pb2.Expression]
     def __init__(self, input: _Optional[_Union[_relations_pb2.Relation, _Mapping]] = ..., subquery_type: _Optional[_Union[SubqueryExpression.SubqueryType, str]] = ..., table_arg_options: _Optional[_Union[SubqueryExpression.TableArgOptions, _Mapping]] = ..., in_subquery_values: _Optional[_Iterable[_Union[_expressions_pb2.Expression, _Mapping]]] = ...) -> None: ...
+
+class IntervalLiteralExpression(_message.Message):
+    __slots__ = ("literal", "start_field", "end_field")
+    LITERAL_FIELD_NUMBER: _ClassVar[int]
+    START_FIELD_FIELD_NUMBER: _ClassVar[int]
+    END_FIELD_FIELD_NUMBER: _ClassVar[int]
+    literal: _expressions_pb2.Expression.Literal
+    start_field: int
+    end_field: int
+    def __init__(self, literal: _Optional[_Union[_expressions_pb2.Expression.Literal, _Mapping]] = ..., start_field: _Optional[int] = ..., end_field: _Optional[int] = ...) -> None: ...

snowflake/snowpark_connect/proto/snowflake_expression_ext_pb2_grpc.py

@@ -0,0 +1,4 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+

snowflake/snowpark_connect/proto/snowflake_relation_ext_pb2_grpc.py

@@ -0,0 +1,4 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+

snowflake/snowpark_connect/relation/io_utils.py

@@ -4,6 +4,8 @@
 
 from urllib.parse import urlparse
 
+from pyspark.errors.exceptions.base import AnalysisException
+
 CLOUD_PREFIX_TO_CLOUD = {
     "abfss": "azure",
     "wasbs": "azure",
@@ -12,10 +14,28 @@ CLOUD_PREFIX_TO_CLOUD = {
 }
 
 SUPPORTED_COMPRESSION_PER_FORMAT = {
-    "csv": {"AUTO", "GZIP", "BZ2", "BROTLI", "ZSTD", "DEFLATE", "RAW_DEFLATE", "NONE"},
-    "json": {"AUTO", "GZIP", "BZ2", "BROTLI", "ZSTD", "DEFLATE", "RAW_DEFLATE", "NONE"},
-    "parquet": {"AUTO", "LZO", "SNAPPY", "NONE"},
-    "text": {"NONE"},
+    "csv": {
+        "GZIP",
+        "BZ2",
+        "BROTLI",
+        "ZSTD",
+        "DEFLATE",
+        "RAW_DEFLATE",
+        "NONE",
+        "UNCOMPRESSED",
+    },
+    "json": {
+        "GZIP",
+        "BZ2",
+        "BROTLI",
+        "ZSTD",
+        "DEFLATE",
+        "RAW_DEFLATE",
+        "NONE",
+        "UNCOMPRESSED",
+    },
+    "parquet": {"LZO", "SNAPPY", "NONE", "UNCOMPRESSED"},
+    "text": {"NONE", "UNCOMPRESSED"},
 }
 
 
@@ -29,6 +49,43 @@ def is_supported_compression(format: str, compression: str | None) -> bool:
     return compression in supported_compressions_for_format(format)
 
 
+def get_compression_for_source_and_options(
+    source: str, options: dict[str, str], from_read: bool = False
+) -> str | None:
+    """
+    Determines the compression type to use for a given data source and options.
+    Args:
+        source (str): The data source format (e.g., "csv", "json", "parquet", "text").
+        options (dict[str, str]): A dictionary of options that may include a "compression" key.
+    Returns:
+        str: The compression type to use (e.g., "GZIP", "SNAPPY", "NONE").
+    Raises:
+        AnalysisException: If the specified compression is not supported for the given source format.
+    """
+    # From read, we don't have a default compression
+    if from_read and "compression" not in options:
+        return None
+
+    # Get compression from options for proper filename generation
+    default_compression = "NONE" if source != "parquet" else "snappy"
+    compression = options.get("compression", default_compression).upper()
+    if compression == "UNCOMPRESSED":
+        compression = "NONE"
+
+    if not is_supported_compression(source, compression):
+        supported_compressions = supported_compressions_for_format(source)
+        raise AnalysisException(
+            f"Compression {compression} is not supported for {source} format. "
+            + (
+                f"Supported compressions: {sorted(supported_compressions)}"
+                if supported_compressions
+                else "None compression supported for this format."
+            )
+        )
+
+    return compression
+
+
 def get_cloud_from_url(
     url: str,
 ):

snowflake/snowpark_connect/relation/map_column_ops.py

@@ -1124,10 +1124,15 @@ def map_group_map(
         group_by_df = input_df.group_by(*snowpark_grouping_expressions)
         inner_df = group_by_df._dataframe
 
-        result = inner_df.select(
-            snowpark_fn.call_table_function(
-                apply_udtf_temp_name, *inner_df.columns
-            ).over(partition_by=snowpark_grouping_expressions)
+        renamed_columns = [f"snowflake_jtf_{column}" for column in input_df.columns]
+        tfc = snowpark_fn.call_table_function(
+            apply_udtf_temp_name, *renamed_columns
+        ).over(partition_by=snowpark_grouping_expressions)
+
+        result = (
+            inner_df.to_df(renamed_columns)
+            .join_table_function(tfc)
+            .drop(*renamed_columns)
         )
     else:
         (

snowflake/snowpark_connect/relation/map_join.py

@@ -21,6 +21,9 @@ from snowflake.snowpark_connect.relation.map_relation import (
     NATURAL_JOIN_TYPE_BASE,
     map_relation,
 )
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    filter_metadata_columns,
+)
 from snowflake.snowpark_connect.utils.context import (
     push_evaluating_join_condition,
     push_sql_scope,
@@ -38,6 +41,11 @@ def map_join(rel: relation_proto.Relation) -> DataFrameContainer:
     left_container: DataFrameContainer = map_relation(rel.join.left)
     right_container: DataFrameContainer = map_relation(rel.join.right)
 
+    # Remove any metadata columns(like metada$filename) present in the dataframes.
+    # We cannot support inputfilename for multisources as each dataframe has it's own source.
+    left_container = filter_metadata_columns(left_container)
+    right_container = filter_metadata_columns(right_container)
+
     left_input: snowpark.DataFrame = left_container.dataframe
     right_input: snowpark.DataFrame = right_container.dataframe
     is_natural_join = rel.join.join_type >= NATURAL_JOIN_TYPE_BASE

snowflake/snowpark_connect/relation/map_row_ops.py

@@ -1,15 +1,14 @@
 #
 # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #
-
-
 import pyspark.sql.connect.proto.expressions_pb2 as expressions_proto
 import pyspark.sql.connect.proto.relations_pb2 as relation_proto
 from pyspark.errors.exceptions.base import AnalysisException, IllegalArgumentException
 
 import snowflake.snowpark_connect.relation.utils as utils
 from snowflake import snowpark
-from snowflake.snowpark.functions import col, expr as snowpark_expr
+from snowflake.snowpark._internal.error_message import SnowparkClientExceptionMessages
+from snowflake.snowpark.functions import col, expr as snowpark_expr, lit
 from snowflake.snowpark.types import (
     BooleanType,
     ByteType,
@@ -20,8 +19,14 @@ from snowflake.snowpark.types import (
     LongType,
     NullType,
     ShortType,
+    StructField,
+    StructType,
+)
+from snowflake.snowpark_connect.column_name_handler import (
+    ColumnNameMap,
+    schema_getter,
+    set_schema_getter,
 )
-from snowflake.snowpark_connect.column_name_handler import ColumnNameMap, schema_getter
 from snowflake.snowpark_connect.config import global_config
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.expression.literal import get_literal_field_and_name
@@ -30,6 +35,9 @@ from snowflake.snowpark_connect.expression.map_expression import (
 )
 from snowflake.snowpark_connect.expression.typer import ExpressionTyper
 from snowflake.snowpark_connect.relation.map_relation import map_relation
+from snowflake.snowpark_connect.utils.identifiers import (
+    split_fully_qualified_spark_name,
+)
 from snowflake.snowpark_connect.utils.telemetry import (
     SnowparkConnectNotImplementedError,
 )
@@ -126,11 +134,19 @@ def map_fillna(
     input_df = input_container.dataframe
 
     if len(rel.fill_na.cols) > 0:
+        if rel.fill_na.cols == ["*"]:
+            # Expand "*" to all columns
+            spark_col_names = input_container.column_map.get_spark_columns()
+        else:
+            spark_col_names = list(rel.fill_na.cols)
+
+        # We don't validate the fully qualified spark name here as fillNa is no-op for structured type colums.
+        # It only works for scalar type columns like float, int, string or bool.
         columns: list[str] = [
             input_container.column_map.get_snowpark_column_name_from_spark_column_name(
-                c
+                split_fully_qualified_spark_name(c)[0]
             )
-            for c in rel.fill_na.cols
+            for c in spark_col_names
         ]
         values = [get_literal_field_and_name(v)[0] for v in rel.fill_na.values]
         if len(values) == 1:
@@ -318,23 +334,37 @@ def map_union(
         right_column_map = right_result.column_map
         columns_to_restore: dict[str, tuple[str, str]] = {}
 
-        for column in right_df.columns:
+        original_right_schema = right_df.schema
+        right_renamed_fields = []
+        for field in original_right_schema.fields:
             spark_name = (
-                right_column_map.get_spark_column_name_from_snowpark_column_name(column)
+                right_column_map.get_spark_column_name_from_snowpark_column_name(
+                    field.name
+                )
+            )
+            right_df = right_df.withColumnRenamed(field.name, spark_name)
+            columns_to_restore[spark_name.upper()] = (spark_name, field.name)
+            right_renamed_fields.append(
+                StructField(spark_name, field.datatype, field.nullable)
             )
-            right_df = right_df.withColumnRenamed(column, spark_name)
-            columns_to_restore[spark_name.upper()] = (spark_name, column)
+        set_schema_getter(right_df, lambda: StructType(right_renamed_fields))
 
-        for column in left_df.columns:
+        original_left_schema = left_df.schema
+        left_renamed_fields = []
+        for field in original_left_schema.fields:
             spark_name = (
-                left_column_map.get_spark_column_name_from_snowpark_column_name(column)
+                left_column_map.get_spark_column_name_from_snowpark_column_name(
+                    field.name
+                )
+            )
+            left_df = left_df.withColumnRenamed(field.name, spark_name)
+            columns_to_restore[spark_name.upper()] = (spark_name, field.name)
+            left_renamed_fields.append(
+                StructField(spark_name, field.datatype, field.nullable)
             )
-            left_df = left_df.withColumnRenamed(column, spark_name)
-            columns_to_restore[spark_name.upper()] = (spark_name, column)
+        set_schema_getter(left_df, lambda: StructType(left_renamed_fields))
 
-        result = left_df.unionAllByName(
-            right_df, allow_missing_columns=allow_missing_columns
-        )
+        result = _union_by_name_optimized(left_df, right_df, allow_missing_columns)
 
         if allow_missing_columns:
             spark_columns = []
@@ -809,3 +839,85 @@ def map_tail(
         alias=input_container.alias,
         cached_schema_getter=lambda: input_df.schema,
     )
+
+
+def _union_by_name_optimized(
+    left_df: snowpark.DataFrame,
+    right_df: snowpark.DataFrame,
+    allow_missing_columns: bool = False,
+) -> snowpark.DataFrame:
+    """
+    This implementation is an optimized version of Snowpark's Dataframe::_union_by_name_internal.
+    The only change is, that it avoids redundant schema queries that occur in the standard Snowpark,
+    by reusing already-fetched/calculated schemas.
+    """
+
+    left_schema = left_df.schema
+    right_schema = right_df.schema
+
+    left_cols = {field.name for field in left_schema.fields}
+    right_cols = {field.name for field in right_schema.fields}
+    right_field_map = {field.name: field for field in right_schema.fields}
+
+    missing_left = right_cols - left_cols
+    missing_right = left_cols - right_cols
+
+    def add_nulls(
+        missing_cols: set[str], to_df: snowpark.DataFrame, from_df: snowpark.DataFrame
+    ) -> snowpark.DataFrame:
+        dt_map = {field.name: field.datatype for field in from_df.schema.fields}
+        result = to_df.select(
+            "*",
+            *[lit(None).cast(dt_map[col]).alias(col) for col in missing_cols],
+        )
+
+        result_fields = []
+        for field in to_df.schema.fields:
+            result_fields.append(
+                StructField(field.name, field.datatype, field.nullable)
+            )
+        for col_name in missing_cols:
+            from_field = next(
+                field for field in from_df.schema.fields if field.name == col_name
+            )
+            result_fields.append(
+                StructField(col_name, from_field.datatype, from_field.nullable)
+            )
+
+        set_schema_getter(result, lambda: StructType(result_fields))
+
+        return result
+
+    if missing_left or missing_right:
+        if allow_missing_columns:
+            left = left_df
+            right = right_df
+            if missing_left:
+                left = add_nulls(missing_left, left, right)
+            if missing_right:
+                right = add_nulls(missing_right, right, left)
+            result = left._union_by_name_internal(right, is_all=True)
+
+            result_fields = []
+            for field in left_schema.fields:
+                result_fields.append(
+                    StructField(field.name, field.datatype, field.nullable)
+                )
+            for col_name in missing_left:
+                right_field = right_field_map[col_name]
+                result_fields.append(
+                    StructField(col_name, right_field.datatype, right_field.nullable)
+                )
+
+            set_schema_getter(result, lambda: StructType(result_fields))
+            return result
+        else:
+            raise SnowparkClientExceptionMessages.DF_CANNOT_RESOLVE_COLUMN_NAME_AMONG(
+                missing_left, missing_right
+            )
+
+    result = left_df.unionAllByName(
+        right_df, allow_missing_columns=allow_missing_columns
+    )
+    set_schema_getter(result, lambda: left_df.schema)
+    return result

snowflake/snowpark_connect/relation/map_show_string.py

@@ -15,6 +15,9 @@ from snowflake.snowpark_connect.column_name_handler import set_schema_getter
 from snowflake.snowpark_connect.config import global_config
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.relation.map_relation import map_relation
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    filter_metadata_columns,
+)
 
 
 def map_show_string(rel: relation_proto.Relation) -> pandas.DataFrame:
@@ -26,14 +29,17 @@ def map_show_string(rel: relation_proto.Relation) -> pandas.DataFrame:
     Buffer object as a single cell.
     """
     input_df_container: DataFrameContainer = map_relation(rel.show_string.input)
-    raw_input_df = input_df_container.dataframe
-    input_df = _handle_datetype_columns(raw_input_df)
+    filtered_container = filter_metadata_columns(input_df_container)
+    display_df = filtered_container.dataframe
+    display_spark_columns = filtered_container.column_map.get_spark_columns()
+
+    input_df = _handle_datetype_columns(display_df)
 
     show_string = input_df._show_string_spark(
         num_rows=rel.show_string.num_rows,
         truncate=rel.show_string.truncate,
         vertical=rel.show_string.vertical,
-        _spark_column_names=input_df_container.column_map.get_spark_columns(),
+        _spark_column_names=display_spark_columns,
         _spark_session_tz=global_config.spark_sql_session_timeZone,
     )
     return pandas.DataFrame({"show_string": [show_string]})
@@ -44,14 +50,16 @@ def map_repr_html(rel: relation_proto.Relation) -> pandas.DataFrame:
     Generate the html string representation of the input dataframe.
     """
     input_df_container: DataFrameContainer = map_relation(rel.html_string.input)
-    input_df = input_df_container.dataframe
+
+    filtered_container = filter_metadata_columns(input_df_container)
+    input_df = filtered_container.dataframe
 
     input_panda = input_df.toPandas()
     input_panda.rename(
         columns={
             analyzer_utils.unquote_if_quoted(
-                input_df_container.column_map.get_snowpark_columns()[i]
-            ): input_df_container.column_map.get_spark_columns()[i]
+                filtered_container.column_map.get_snowpark_columns()[i]
+            ): filtered_container.column_map.get_spark_columns()[i]
             for i in range(len(input_panda.columns))
         },
         inplace=True,

snowflake/snowpark_connect/relation/map_sql.py

@@ -7,6 +7,7 @@ from collections.abc import MutableMapping, MutableSequence
 from contextlib import contextmanager, suppress
 from contextvars import ContextVar
 from functools import reduce
+from typing import Tuple
 
 import jpype
 import pandas
@@ -31,6 +32,10 @@ from snowflake.snowpark._internal.analyzer.analyzer_utils import (
 from snowflake.snowpark._internal.type_utils import convert_sp_to_sf_type
 from snowflake.snowpark._internal.utils import is_sql_select_statement, quote_name
 from snowflake.snowpark.functions import when_matched, when_not_matched
+from snowflake.snowpark_connect.client import (
+    SQL_PASS_THROUGH_MARKER,
+    calculate_checksum,
+)
 from snowflake.snowpark_connect.config import (
     auto_uppercase_non_column_identifiers,
     check_table_supports_operation,
@@ -397,7 +402,7 @@ def map_sql_to_pandas_df(
     returns a tuple of None for SELECT queries to enable lazy evaluation
     """
 
-    snowpark_connect_sql_passthrough = get_sql_passthrough()
+    snowpark_connect_sql_passthrough, sql_string = is_valid_passthrough_sql(sql_string)
 
     if not snowpark_connect_sql_passthrough:
         logical_plan = sql_parser().parsePlan(sql_string)
@@ -1047,7 +1052,7 @@ def map_sql_to_pandas_df(
                     raise AnalysisException(
                         f"ALTER TABLE RENAME COLUMN is not supported for table '{full_table_identifier}'. "
                         f"This table was created as a v1 table with a data source that doesn't support column renaming. "
-                        f"To enable this operation, set 'enable_snowflake_extension_behavior' to 'true'."
+                        f"To enable this operation, set 'snowpark.connect.enable_snowflake_extension_behavior' to 'true'."
                     )
 
                 column_obj = logical_plan.column()
@@ -1282,6 +1287,14 @@ def map_sql_to_pandas_df(
                         return pandas.DataFrame({"": [""]}), ""
 
                 rows = session.sql(snowflake_sql).collect()
+            case "RefreshTable":
+                table_name_unquoted = ".".join(
+                    str(part)
+                    for part in as_java_list(logical_plan.child().multipartIdentifier())
+                )
+                SNOWFLAKE_CATALOG.refreshTable(table_name_unquoted)
+
+                return pandas.DataFrame({"": [""]}), ""
             case _:
                 execute_logical_plan(logical_plan)
                 return None, None
@@ -1302,6 +1315,27 @@ def get_sql_passthrough() -> bool:
     return get_boolean_session_config_param("snowpark.connect.sql.passthrough")
 
 
+def is_valid_passthrough_sql(sql_stmt: str) -> Tuple[bool, str]:
+    """
+    Checks if :param sql_stmt: should be executed as SQL pass-through. SQL pass-through can be detected in 1 of 2 ways:
+    1) Either Spark config parameter "snowpark.connect.sql.passthrough" is set (legacy mode, to be deprecated)
+    2) If :param sql_stmt: is created through SnowflakeSession and has correct marker + checksum
+    """
+    if get_sql_passthrough():
+        # legacy style pass-through, sql_stmt should be a whole, valid SQL statement
+        return True, sql_stmt
+
+    # check for new style, SnowflakeSession based SQL pass-through
+    sql_parts = sql_stmt.split(" ", 2)
+    if len(sql_parts) == 3:
+        marker, checksum, sql = sql_parts
+        if marker == SQL_PASS_THROUGH_MARKER and checksum == calculate_checksum(sql):
+            return True, sql
+
+    # Not a SQL pass-through
+    return False, sql_stmt
+
+
 def change_default_to_public(name: str) -> str:
     """
     Change the namespace to PUBLIC when given name is DEFAULT
@@ -1397,10 +1431,10 @@ def map_sql(
     In passthough mode as True, SAS calls session.sql() and not calling Spark Parser.
     This is to mitigate any issue not covered by spark logical plan to protobuf conversion.
     """
-    snowpark_connect_sql_passthrough = get_sql_passthrough()
+    snowpark_connect_sql_passthrough, sql_stmt = is_valid_passthrough_sql(rel.sql.query)
 
     if not snowpark_connect_sql_passthrough:
-        logical_plan = sql_parser().parseQuery(rel.sql.query)
+        logical_plan = sql_parser().parseQuery(sql_stmt)
 
         parsed_pos_args = parse_pos_args(logical_plan, rel.sql.pos_args)
         set_sql_args(rel.sql.args, parsed_pos_args)
@@ -1408,7 +1442,7 @@ def map_sql(
         return execute_logical_plan(logical_plan)
     else:
         session = snowpark.Session.get_active_session()
-        sql_df = session.sql(rel.sql.query)
+        sql_df = session.sql(sql_stmt)
         columns = sql_df.columns
         return DataFrameContainer.create_with_column_mapping(
             dataframe=sql_df,

snowflake/snowpark_connect/relation/map_stats.py

@@ -81,7 +81,7 @@ def map_approx_quantile(
     input_df = input_container.dataframe
 
     snowflake_compatible = get_boolean_session_config_param(
-        "enable_snowflake_extension_behavior"
+        "snowpark.connect.enable_snowflake_extension_behavior"
     )
 
     if not snowflake_compatible:
@@ -309,9 +309,28 @@ def map_freq_items(rel: relation_proto.Relation) -> DataFrameContainer:
     cols = input_container.column_map.get_snowpark_column_names_from_spark_column_names(
         list(rel.freq_items.cols)
     )
+
+    # handle empty DataFrame case
+    row_count = input_df.count()
+
+    for sp_col_name in cols:
+        spark_col_names.append(
+            f"{input_container.column_map.get_spark_column_name_from_snowpark_column_name(sp_col_name)}_freqItems"
+        )
+
+    if row_count == 0:
+        # If DataFrame is empty, return empty arrays for each column
+        empty_values = [[] for _ in cols]
+        approx_top_k_df = session.createDataFrame([empty_values], spark_col_names)
+        return DataFrameContainer.create_with_column_mapping(
+            dataframe=approx_top_k_df,
+            spark_column_names=spark_col_names,
+            snowpark_column_names=spark_col_names,
+        )
+
     approx_top_k_df = input_df.select(
         *[
-            fn.function("approx_top_k")(fn.col(col), round(input_df.count() / support))
+            fn.function("approx_top_k")(fn.col(col), round(row_count / support))
             for col in cols
         ]
     )
@@ -330,10 +349,6 @@ def map_freq_items(rel: relation_proto.Relation) -> DataFrameContainer:
         for value in approx_top_k_values
     ]
 
-    for sp_col_name in cols:
-        spark_col_names.append(
-            f"{input_container.column_map.get_spark_column_name_from_snowpark_column_name(sp_col_name)}_freqItems"
-        )
     approx_top_k_df = session.createDataFrame([filtered_values], spark_col_names)
 
     return DataFrameContainer.create_with_column_mapping(

snowflake/snowpark_connect/relation/read/map_read.py

@@ -17,6 +17,7 @@ from snowflake.snowpark_connect.config import global_config
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.relation.io_utils import (
     convert_file_prefix_path,
+    get_compression_for_source_and_options,
     is_cloud_path,
 )
 from snowflake.snowpark_connect.relation.read.map_read_table import map_read_table
@@ -237,6 +238,14 @@ def _read_file(
     )
     upload_files_if_needed(paths, clean_source_paths, session, read_format)
     paths = [_quote_stage_path(path) for path in paths]
+
+    if read_format in ("csv", "text", "json", "parquet"):
+        compression = get_compression_for_source_and_options(
+            read_format, options, from_read=True
+        )
+        if compression is not None:
+            options["compression"] = compression
+
     match read_format:
         case "csv":
             from snowflake.snowpark_connect.relation.read.map_read_csv import (