snowpark-connect 0.28.1__py3-none-any.whl → 0.30.0__py3-none-any.whl

snowflake/snowpark_connect/proto/snowflake_expression_ext_pb2.py

@@ -16,7 +16,7 @@ from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_e
 from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1esnowflake_expression_ext.proto\x12\rsnowflake.ext\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\"\x98\x01\n\x0c\x45xpExtension\x12@\n\x0enamed_argument\x18\x01 \x01(\x0b\x32&.snowflake.ext.NamedArgumentExpressionH\x00\x12@\n\x13subquery_expression\x18\x02 \x01(\x0b\x32!.snowflake.ext.SubqueryExpressionH\x00\x42\x04\n\x02op\"P\n\x17NamedArgumentExpression\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.Expression\"\xf4\x04\n\x12SubqueryExpression\x12&\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.Relation\x12\x45\n\rsubquery_type\x18\x02 \x01(\x0e\x32..snowflake.ext.SubqueryExpression.SubqueryType\x12Q\n\x11table_arg_options\x18\x03 \x01(\x0b\x32\x31.snowflake.ext.SubqueryExpression.TableArgOptionsH\x00\x88\x01\x01\x12\x35\n\x12in_subquery_values\x18\x04 \x03(\x0b\x32\x19.spark.connect.Expression\x1a\xbb\x01\n\x0fTableArgOptions\x12\x31\n\x0epartition_spec\x18\x01 \x03(\x0b\x32\x19.spark.connect.Expression\x12\x37\n\norder_spec\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrder\x12\"\n\x15with_single_partition\x18\x03 \x01(\x08H\x00\x88\x01\x01\x42\x18\n\x16_with_single_partition\"\x90\x01\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x12\x1b\n\x17SUBQUERY_TYPE_TABLE_ARG\x10\x03\x12\x14\n\x10SUBQUERY_TYPE_IN\x10\x04\x42\x14\n\x12_table_arg_optionsb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1esnowflake_expression_ext.proto\x12\rsnowflake.ext\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\"\xde\x01\n\x0c\x45xpExtension\x12@\n\x0enamed_argument\x18\x01 \x01(\x0b\x32&.snowflake.ext.NamedArgumentExpressionH\x00\x12@\n\x13subquery_expression\x18\x02 \x01(\x0b\x32!.snowflake.ext.SubqueryExpressionH\x00\x12\x44\n\x10interval_literal\x18\x03 \x01(\x0b\x32(.snowflake.ext.IntervalLiteralExpressionH\x00\x42\x04\n\x02op\"P\n\x17NamedArgumentExpression\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.Expression\"\xf4\x04\n\x12SubqueryExpression\x12&\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.Relation\x12\x45\n\rsubquery_type\x18\x02 \x01(\x0e\x32..snowflake.ext.SubqueryExpression.SubqueryType\x12Q\n\x11table_arg_options\x18\x03 \x01(\x0b\x32\x31.snowflake.ext.SubqueryExpression.TableArgOptionsH\x00\x88\x01\x01\x12\x35\n\x12in_subquery_values\x18\x04 \x03(\x0b\x32\x19.spark.connect.Expression\x1a\xbb\x01\n\x0fTableArgOptions\x12\x31\n\x0epartition_spec\x18\x01 \x03(\x0b\x32\x19.spark.connect.Expression\x12\x37\n\norder_spec\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrder\x12\"\n\x15with_single_partition\x18\x03 \x01(\x08H\x00\x88\x01\x01\x42\x18\n\x16_with_single_partition\"\x90\x01\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x12\x1b\n\x17SUBQUERY_TYPE_TABLE_ARG\x10\x03\x12\x14\n\x10SUBQUERY_TYPE_IN\x10\x04\x42\x14\n\x12_table_arg_options\"\x9f\x01\n\x19IntervalLiteralExpression\x12\x32\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.Literal\x12\x18\n\x0bstart_field\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x16\n\tend_field\x18\x03 \x01(\x05H\x01\x88\x01\x01\x42\x0e\n\x0c_start_fieldB\x0c\n\n_end_fieldb\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -24,13 +24,15 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'snowflake_expression_ext_pb
 if _descriptor._USE_C_DESCRIPTORS == False:
   DESCRIPTOR._options = None
   _globals['_EXPEXTENSION']._serialized_start=114
-  _globals['_EXPEXTENSION']._serialized_end=266
-  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_start=268
-  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_end=348
-  _globals['_SUBQUERYEXPRESSION']._serialized_start=351
-  _globals['_SUBQUERYEXPRESSION']._serialized_end=979
-  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_start=623
-  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_end=810
-  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_start=813
-  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_end=957
+  _globals['_EXPEXTENSION']._serialized_end=336
+  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_start=338
+  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_end=418
+  _globals['_SUBQUERYEXPRESSION']._serialized_start=421
+  _globals['_SUBQUERYEXPRESSION']._serialized_end=1049
+  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_start=693
+  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_end=880
+  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_start=883
+  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_end=1027
+  _globals['_INTERVALLITERALEXPRESSION']._serialized_start=1052
+  _globals['_INTERVALLITERALEXPRESSION']._serialized_end=1211
 # @@protoc_insertion_point(module_scope)

snowflake/snowpark_connect/proto/snowflake_expression_ext_pb2.pyi

@@ -9,12 +9,14 @@ from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Map
 DESCRIPTOR: _descriptor.FileDescriptor
 
 class ExpExtension(_message.Message):
-    __slots__ = ("named_argument", "subquery_expression")
+    __slots__ = ("named_argument", "subquery_expression", "interval_literal")
     NAMED_ARGUMENT_FIELD_NUMBER: _ClassVar[int]
     SUBQUERY_EXPRESSION_FIELD_NUMBER: _ClassVar[int]
+    INTERVAL_LITERAL_FIELD_NUMBER: _ClassVar[int]
     named_argument: NamedArgumentExpression
     subquery_expression: SubqueryExpression
-    def __init__(self, named_argument: _Optional[_Union[NamedArgumentExpression, _Mapping]] = ..., subquery_expression: _Optional[_Union[SubqueryExpression, _Mapping]] = ...) -> None: ...
+    interval_literal: IntervalLiteralExpression
+    def __init__(self, named_argument: _Optional[_Union[NamedArgumentExpression, _Mapping]] = ..., subquery_expression: _Optional[_Union[SubqueryExpression, _Mapping]] = ..., interval_literal: _Optional[_Union[IntervalLiteralExpression, _Mapping]] = ...) -> None: ...
 
 class NamedArgumentExpression(_message.Message):
     __slots__ = ("key", "value")
@@ -56,3 +58,13 @@ class SubqueryExpression(_message.Message):
     table_arg_options: SubqueryExpression.TableArgOptions
     in_subquery_values: _containers.RepeatedCompositeFieldContainer[_expressions_pb2.Expression]
     def __init__(self, input: _Optional[_Union[_relations_pb2.Relation, _Mapping]] = ..., subquery_type: _Optional[_Union[SubqueryExpression.SubqueryType, str]] = ..., table_arg_options: _Optional[_Union[SubqueryExpression.TableArgOptions, _Mapping]] = ..., in_subquery_values: _Optional[_Iterable[_Union[_expressions_pb2.Expression, _Mapping]]] = ...) -> None: ...
+
+class IntervalLiteralExpression(_message.Message):
+    __slots__ = ("literal", "start_field", "end_field")
+    LITERAL_FIELD_NUMBER: _ClassVar[int]
+    START_FIELD_FIELD_NUMBER: _ClassVar[int]
+    END_FIELD_FIELD_NUMBER: _ClassVar[int]
+    literal: _expressions_pb2.Expression.Literal
+    start_field: int
+    end_field: int
+    def __init__(self, literal: _Optional[_Union[_expressions_pb2.Expression.Literal, _Mapping]] = ..., start_field: _Optional[int] = ..., end_field: _Optional[int] = ...) -> None: ...

snowflake/snowpark_connect/relation/io_utils.py

@@ -7,8 +7,27 @@ from urllib.parse import urlparse
 CLOUD_PREFIX_TO_CLOUD = {
     "abfss": "azure",
     "wasbs": "azure",
+    "gcs": "gcp",
+    "gs": "gcp",
 }
 
+SUPPORTED_COMPRESSION_PER_FORMAT = {
+    "csv": {"AUTO", "GZIP", "BZ2", "BROTLI", "ZSTD", "DEFLATE", "RAW_DEFLATE", "NONE"},
+    "json": {"AUTO", "GZIP", "BZ2", "BROTLI", "ZSTD", "DEFLATE", "RAW_DEFLATE", "NONE"},
+    "parquet": {"AUTO", "LZO", "SNAPPY", "NONE"},
+    "text": {"NONE"},
+}
+
+
+def supported_compressions_for_format(format: str) -> set[str]:
+    return SUPPORTED_COMPRESSION_PER_FORMAT.get(format, set())
+
+
+def is_supported_compression(format: str, compression: str | None) -> bool:
+    if compression is None:
+        return True
+    return compression in supported_compressions_for_format(format)
+
 
 def get_cloud_from_url(
     url: str,
@@ -66,7 +85,8 @@ def is_cloud_path(path: str) -> bool:
         or path.startswith("azure://")
         or path.startswith("abfss://")
         or path.startswith("wasbs://")  # Azure
-        or path.startswith("gcs://")  # GCP
+        or path.startswith("gcs://")
+        or path.startswith("gs://")  # GCP
     )
 
 

snowflake/snowpark_connect/relation/map_column_ops.py

@@ -1124,10 +1124,15 @@ def map_group_map(
         group_by_df = input_df.group_by(*snowpark_grouping_expressions)
         inner_df = group_by_df._dataframe
 
-        result = inner_df.select(
-            snowpark_fn.call_table_function(
-                apply_udtf_temp_name, *inner_df.columns
-            ).over(partition_by=snowpark_grouping_expressions)
+        renamed_columns = [f"snowflake_jtf_{column}" for column in input_df.columns]
+        tfc = snowpark_fn.call_table_function(
+            apply_udtf_temp_name, *renamed_columns
+        ).over(partition_by=snowpark_grouping_expressions)
+
+        result = (
+            inner_df.to_df(renamed_columns)
+            .join_table_function(tfc)
+            .drop(*renamed_columns)
         )
     else:
         (

snowflake/snowpark_connect/relation/map_extension.py

@@ -345,7 +345,7 @@ def map_aggregate(
         return new_names[0], snowpark_column
 
     raw_groupings: list[tuple[str, TypedColumn]] = []
-    raw_aggregations: list[tuple[str, TypedColumn]] = []
+    raw_aggregations: list[tuple[str, TypedColumn, list[str]]] = []
 
     if not is_group_by_all:
         raw_groupings = [_map_column(exp) for exp in aggregate.grouping_expressions]
@@ -375,10 +375,21 @@ def map_aggregate(
     # Note: We don't clear the map here to preserve any parent context aliases
     from snowflake.snowpark_connect.utils.context import register_lca_alias
 
+    # If it's an unresolved attribute when its in aggregate.aggregate_expressions, we know it came from the parent map straight away
+    # in this case, we should see if the parent map has a qualifier for it and propagate that here, in case the order by references it in
+    # a qualified way later.
     agg_count = get_sql_aggregate_function_count()
     for exp in aggregate.aggregate_expressions:
         col = _map_column(exp)
-        raw_aggregations.append(col)
+        if exp.WhichOneof("expr_type") == "unresolved_attribute":
+            spark_name = col[0]
+            qualifiers = input_container.column_map.get_qualifier_for_spark_column(
+                spark_name
+            )
+        else:
+            qualifiers = []
+
+        raw_aggregations.append((col[0], col[1], qualifiers))
 
         # If this is an alias, register it in the LCA map for subsequent expressions
         if (
@@ -409,18 +420,20 @@ def map_aggregate(
     spark_columns: list[str] = []
     snowpark_columns: list[str] = []
     snowpark_column_types: list[snowpark_types.DataType] = []
+    all_qualifiers: list[list[str]] = []
 
     # Use grouping columns directly without aliases
     groupings = [col.col for _, col in raw_groupings]
 
     # Create aliases only for aggregation columns
     aggregations = []
-    for i, (spark_name, snowpark_column) in enumerate(raw_aggregations):
+    for i, (spark_name, snowpark_column, qualifiers) in enumerate(raw_aggregations):
         alias = make_column_names_snowpark_compatible([spark_name], plan_id, i)[0]
 
         spark_columns.append(spark_name)
         snowpark_columns.append(alias)
         snowpark_column_types.append(snowpark_column.typ)
+        all_qualifiers.append(qualifiers)
 
         aggregations.append(snowpark_column.col.alias(alias))
 
@@ -483,6 +496,7 @@ def map_aggregate(
             spark_column_names=spark_columns,
             snowpark_column_names=snowpark_columns,
             snowpark_column_types=snowpark_column_types,
+            column_qualifiers=all_qualifiers,
         ).column_map
 
         # Create hybrid column map that can resolve both input and aggregate contexts
@@ -494,7 +508,9 @@ def map_aggregate(
             aggregate_expressions=list(aggregate.aggregate_expressions),
             grouping_expressions=list(aggregate.grouping_expressions),
             spark_columns=spark_columns,
-            raw_aggregations=raw_aggregations,
+            raw_aggregations=[
+                (spark_name, col) for spark_name, col, _ in raw_aggregations
+            ],
         )
 
         # Map the HAVING condition using hybrid resolution
@@ -515,4 +531,5 @@ def map_aggregate(
         snowpark_column_names=snowpark_columns,
         snowpark_column_types=snowpark_column_types,
         parent_column_name_map=input_df._column_map,
+        column_qualifiers=all_qualifiers,
     )

snowflake/snowpark_connect/relation/map_join.py

@@ -21,6 +21,9 @@ from snowflake.snowpark_connect.relation.map_relation import (
     NATURAL_JOIN_TYPE_BASE,
     map_relation,
 )
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    filter_metadata_columns,
+)
 from snowflake.snowpark_connect.utils.context import (
     push_evaluating_join_condition,
     push_sql_scope,
@@ -38,6 +41,11 @@ def map_join(rel: relation_proto.Relation) -> DataFrameContainer:
     left_container: DataFrameContainer = map_relation(rel.join.left)
     right_container: DataFrameContainer = map_relation(rel.join.right)
 
+    # Remove any metadata columns(like metada$filename) present in the dataframes.
+    # We cannot support inputfilename for multisources as each dataframe has it's own source.
+    left_container = filter_metadata_columns(left_container)
+    right_container = filter_metadata_columns(right_container)
+
     left_input: snowpark.DataFrame = left_container.dataframe
     right_input: snowpark.DataFrame = right_container.dataframe
     is_natural_join = rel.join.join_type >= NATURAL_JOIN_TYPE_BASE

snowflake/snowpark_connect/relation/map_map_partitions.py

@@ -12,7 +12,6 @@ from snowflake.snowpark_connect.constants import MAP_IN_ARROW_EVAL_TYPE
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.relation.map_relation import map_relation
 from snowflake.snowpark_connect.type_mapping import proto_to_snowpark_type
-from snowflake.snowpark_connect.utils.context import map_partitions_depth
 from snowflake.snowpark_connect.utils.pandas_udtf_utils import (
     create_pandas_udtf,
     create_pandas_udtf_with_arrow,
@@ -53,18 +52,18 @@ def _call_udtf(
         ).cast("int"),
     )
 
-    udtf_columns = input_df.columns + [snowpark_fn.col("_DUMMY_PARTITION_KEY")]
+    udtf_columns = [f"snowflake_jtf_{column}" for column in input_df.columns] + [
+        "_DUMMY_PARTITION_KEY"
+    ]
 
     tfc = snowpark_fn.call_table_function(udtf_name, *udtf_columns).over(
         partition_by=[snowpark_fn.col("_DUMMY_PARTITION_KEY")]
     )
 
-    # Use map_partitions_depth only when mapping non nested map_partitions
-    # When mapping chained functions additional column casting is necessary
-    if map_partitions_depth() == 1:
-        result_df_with_dummy = input_df_with_dummy.join_table_function(tfc)
-    else:
-        result_df_with_dummy = input_df_with_dummy.select(tfc)
+    # Overwrite the input_df columns to prevent name conflicts with UDTF output columns
+    result_df_with_dummy = input_df_with_dummy.to_df(udtf_columns).join_table_function(
+        tfc
+    )
 
     output_cols = [field.name for field in return_type.fields]
 

snowflake/snowpark_connect/relation/map_relation.py

@@ -16,7 +16,6 @@ from snowflake.snowpark_connect.utils.context import (
     get_plan_id_map,
     get_session_id,
     not_resolving_fun_args,
-    push_map_partitions,
     push_operation_scope,
     set_is_aggregate_function,
     set_plan_id_map,
@@ -185,8 +184,7 @@ def map_relation(
                     )
                 return cached_df
             case "map_partitions":
-                with push_map_partitions():
-                    result = map_map_partitions.map_map_partitions(rel)
+                result = map_map_partitions.map_map_partitions(rel)
             case "offset":
                 result = map_row_ops.map_offset(rel)
             case "project":

snowflake/snowpark_connect/relation/map_row_ops.py

@@ -1,15 +1,14 @@
 #
 # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #
-
-
 import pyspark.sql.connect.proto.expressions_pb2 as expressions_proto
 import pyspark.sql.connect.proto.relations_pb2 as relation_proto
 from pyspark.errors.exceptions.base import AnalysisException, IllegalArgumentException
 
 import snowflake.snowpark_connect.relation.utils as utils
 from snowflake import snowpark
-from snowflake.snowpark.functions import col, expr as snowpark_expr
+from snowflake.snowpark._internal.error_message import SnowparkClientExceptionMessages
+from snowflake.snowpark.functions import col, expr as snowpark_expr, lit
 from snowflake.snowpark.types import (
     BooleanType,
     ByteType,
@@ -20,8 +19,14 @@ from snowflake.snowpark.types import (
     LongType,
     NullType,
     ShortType,
+    StructField,
+    StructType,
+)
+from snowflake.snowpark_connect.column_name_handler import (
+    ColumnNameMap,
+    schema_getter,
+    set_schema_getter,
 )
-from snowflake.snowpark_connect.column_name_handler import ColumnNameMap, schema_getter
 from snowflake.snowpark_connect.config import global_config
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.expression.literal import get_literal_field_and_name
@@ -318,23 +323,37 @@ def map_union(
         right_column_map = right_result.column_map
         columns_to_restore: dict[str, tuple[str, str]] = {}
 
-        for column in right_df.columns:
+        original_right_schema = right_df.schema
+        right_renamed_fields = []
+        for field in original_right_schema.fields:
             spark_name = (
-                right_column_map.get_spark_column_name_from_snowpark_column_name(column)
+                right_column_map.get_spark_column_name_from_snowpark_column_name(
+                    field.name
+                )
             )
-            right_df = right_df.withColumnRenamed(column, spark_name)
-            columns_to_restore[spark_name.upper()] = (spark_name, column)
+            right_df = right_df.withColumnRenamed(field.name, spark_name)
+            columns_to_restore[spark_name.upper()] = (spark_name, field.name)
+            right_renamed_fields.append(
+                StructField(spark_name, field.datatype, field.nullable)
+            )
+        set_schema_getter(right_df, lambda: StructType(right_renamed_fields))
 
-        for column in left_df.columns:
+        original_left_schema = left_df.schema
+        left_renamed_fields = []
+        for field in original_left_schema.fields:
             spark_name = (
-                left_column_map.get_spark_column_name_from_snowpark_column_name(column)
+                left_column_map.get_spark_column_name_from_snowpark_column_name(
+                    field.name
+                )
             )
-            left_df = left_df.withColumnRenamed(column, spark_name)
-            columns_to_restore[spark_name.upper()] = (spark_name, column)
+            left_df = left_df.withColumnRenamed(field.name, spark_name)
+            columns_to_restore[spark_name.upper()] = (spark_name, field.name)
+            left_renamed_fields.append(
+                StructField(spark_name, field.datatype, field.nullable)
+            )
+        set_schema_getter(left_df, lambda: StructType(left_renamed_fields))
 
-        result = left_df.unionAllByName(
-            right_df, allow_missing_columns=allow_missing_columns
-        )
+        result = _union_by_name_optimized(left_df, right_df, allow_missing_columns)
 
         if allow_missing_columns:
             spark_columns = []
@@ -809,3 +828,85 @@ def map_tail(
         alias=input_container.alias,
         cached_schema_getter=lambda: input_df.schema,
     )
+
+
+def _union_by_name_optimized(
+    left_df: snowpark.DataFrame,
+    right_df: snowpark.DataFrame,
+    allow_missing_columns: bool = False,
+) -> snowpark.DataFrame:
+    """
+    This implementation is an optimized version of Snowpark's Dataframe::_union_by_name_internal.
+    The only change is, that it avoids redundant schema queries that occur in the standard Snowpark,
+    by reusing already-fetched/calculated schemas.
+    """
+
+    left_schema = left_df.schema
+    right_schema = right_df.schema
+
+    left_cols = {field.name for field in left_schema.fields}
+    right_cols = {field.name for field in right_schema.fields}
+    right_field_map = {field.name: field for field in right_schema.fields}
+
+    missing_left = right_cols - left_cols
+    missing_right = left_cols - right_cols
+
+    def add_nulls(
+        missing_cols: set[str], to_df: snowpark.DataFrame, from_df: snowpark.DataFrame
+    ) -> snowpark.DataFrame:
+        dt_map = {field.name: field.datatype for field in from_df.schema.fields}
+        result = to_df.select(
+            "*",
+            *[lit(None).cast(dt_map[col]).alias(col) for col in missing_cols],
+        )
+
+        result_fields = []
+        for field in to_df.schema.fields:
+            result_fields.append(
+                StructField(field.name, field.datatype, field.nullable)
+            )
+        for col_name in missing_cols:
+            from_field = next(
+                field for field in from_df.schema.fields if field.name == col_name
+            )
+            result_fields.append(
+                StructField(col_name, from_field.datatype, from_field.nullable)
+            )
+
+        set_schema_getter(result, lambda: StructType(result_fields))
+
+        return result
+
+    if missing_left or missing_right:
+        if allow_missing_columns:
+            left = left_df
+            right = right_df
+            if missing_left:
+                left = add_nulls(missing_left, left, right)
+            if missing_right:
+                right = add_nulls(missing_right, right, left)
+            result = left._union_by_name_internal(right, is_all=True)
+
+            result_fields = []
+            for field in left_schema.fields:
+                result_fields.append(
+                    StructField(field.name, field.datatype, field.nullable)
+                )
+            for col_name in missing_left:
+                right_field = right_field_map[col_name]
+                result_fields.append(
+                    StructField(col_name, right_field.datatype, right_field.nullable)
+                )
+
+            set_schema_getter(result, lambda: StructType(result_fields))
+            return result
+        else:
+            raise SnowparkClientExceptionMessages.DF_CANNOT_RESOLVE_COLUMN_NAME_AMONG(
+                missing_left, missing_right
+            )
+
+    result = left_df.unionAllByName(
+        right_df, allow_missing_columns=allow_missing_columns
+    )
+    set_schema_getter(result, lambda: left_df.schema)
+    return result

snowflake/snowpark_connect/relation/map_show_string.py

@@ -15,6 +15,9 @@ from snowflake.snowpark_connect.column_name_handler import set_schema_getter
 from snowflake.snowpark_connect.config import global_config
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.relation.map_relation import map_relation
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    filter_metadata_columns,
+)
 
 
 def map_show_string(rel: relation_proto.Relation) -> pandas.DataFrame:
@@ -26,14 +29,17 @@ def map_show_string(rel: relation_proto.Relation) -> pandas.DataFrame:
     Buffer object as a single cell.
     """
     input_df_container: DataFrameContainer = map_relation(rel.show_string.input)
-    raw_input_df = input_df_container.dataframe
-    input_df = _handle_datetype_columns(raw_input_df)
+    filtered_container = filter_metadata_columns(input_df_container)
+    display_df = filtered_container.dataframe
+    display_spark_columns = filtered_container.column_map.get_spark_columns()
+
+    input_df = _handle_datetype_columns(display_df)
 
     show_string = input_df._show_string_spark(
         num_rows=rel.show_string.num_rows,
         truncate=rel.show_string.truncate,
         vertical=rel.show_string.vertical,
-        _spark_column_names=input_df_container.column_map.get_spark_columns(),
+        _spark_column_names=display_spark_columns,
         _spark_session_tz=global_config.spark_sql_session_timeZone,
     )
     return pandas.DataFrame({"show_string": [show_string]})
@@ -44,14 +50,16 @@ def map_repr_html(rel: relation_proto.Relation) -> pandas.DataFrame:
     Generate the html string representation of the input dataframe.
     """
     input_df_container: DataFrameContainer = map_relation(rel.html_string.input)
-    input_df = input_df_container.dataframe
+
+    filtered_container = filter_metadata_columns(input_df_container)
+    input_df = filtered_container.dataframe
 
     input_panda = input_df.toPandas()
     input_panda.rename(
         columns={
             analyzer_utils.unquote_if_quoted(
-                input_df_container.column_map.get_snowpark_columns()[i]
-            ): input_df_container.column_map.get_spark_columns()[i]
+                filtered_container.column_map.get_snowpark_columns()[i]
+            ): filtered_container.column_map.get_spark_columns()[i]
             for i in range(len(input_panda.columns))
         },
         inplace=True,

snowflake/snowpark_connect/relation/map_sql.py

@@ -7,6 +7,7 @@ from collections.abc import MutableMapping, MutableSequence
 from contextlib import contextmanager, suppress
 from contextvars import ContextVar
 from functools import reduce
+from typing import Tuple
 
 import jpype
 import pandas
@@ -31,6 +32,10 @@ from snowflake.snowpark._internal.analyzer.analyzer_utils import (
 from snowflake.snowpark._internal.type_utils import convert_sp_to_sf_type
 from snowflake.snowpark._internal.utils import is_sql_select_statement, quote_name
 from snowflake.snowpark.functions import when_matched, when_not_matched
+from snowflake.snowpark_connect.client import (
+    SQL_PASS_THROUGH_MARKER,
+    calculate_checksum,
+)
 from snowflake.snowpark_connect.config import (
     auto_uppercase_non_column_identifiers,
     check_table_supports_operation,
@@ -397,7 +402,7 @@ def map_sql_to_pandas_df(
     returns a tuple of None for SELECT queries to enable lazy evaluation
     """
 
-    snowpark_connect_sql_passthrough = get_sql_passthrough()
+    snowpark_connect_sql_passthrough, sql_string = is_valid_passthrough_sql(sql_string)
 
     if not snowpark_connect_sql_passthrough:
         logical_plan = sql_parser().parsePlan(sql_string)
@@ -1047,7 +1052,7 @@ def map_sql_to_pandas_df(
                     raise AnalysisException(
                         f"ALTER TABLE RENAME COLUMN is not supported for table '{full_table_identifier}'. "
                         f"This table was created as a v1 table with a data source that doesn't support column renaming. "
-                        f"To enable this operation, set 'enable_snowflake_extension_behavior' to 'true'."
+                        f"To enable this operation, set 'snowpark.connect.enable_snowflake_extension_behavior' to 'true'."
                     )
 
                 column_obj = logical_plan.column()
@@ -1282,6 +1287,14 @@ def map_sql_to_pandas_df(
                         return pandas.DataFrame({"": [""]}), ""
 
                 rows = session.sql(snowflake_sql).collect()
+            case "RefreshTable":
+                table_name_unquoted = ".".join(
+                    str(part)
+                    for part in as_java_list(logical_plan.child().multipartIdentifier())
+                )
+                SNOWFLAKE_CATALOG.refreshTable(table_name_unquoted)
+
+                return pandas.DataFrame({"": [""]}), ""
             case _:
                 execute_logical_plan(logical_plan)
                 return None, None
@@ -1302,6 +1315,27 @@ def get_sql_passthrough() -> bool:
     return get_boolean_session_config_param("snowpark.connect.sql.passthrough")
 
 
+def is_valid_passthrough_sql(sql_stmt: str) -> Tuple[bool, str]:
+    """
+    Checks if :param sql_stmt: should be executed as SQL pass-through. SQL pass-through can be detected in 1 of 2 ways:
+    1) Either Spark config parameter "snowpark.connect.sql.passthrough" is set (legacy mode, to be deprecated)
+    2) If :param sql_stmt: is created through SnowflakeSession and has correct marker + checksum
+    """
+    if get_sql_passthrough():
+        # legacy style pass-through, sql_stmt should be a whole, valid SQL statement
+        return True, sql_stmt
+
+    # check for new style, SnowflakeSession based SQL pass-through
+    sql_parts = sql_stmt.split(" ", 2)
+    if len(sql_parts) == 3:
+        marker, checksum, sql = sql_parts
+        if marker == SQL_PASS_THROUGH_MARKER and checksum == calculate_checksum(sql):
+            return True, sql
+
+    # Not a SQL pass-through
+    return False, sql_stmt
+
+
 def change_default_to_public(name: str) -> str:
     """
     Change the namespace to PUBLIC when given name is DEFAULT
@@ -1397,10 +1431,10 @@ def map_sql(
     In passthough mode as True, SAS calls session.sql() and not calling Spark Parser.
     This is to mitigate any issue not covered by spark logical plan to protobuf conversion.
     """
-    snowpark_connect_sql_passthrough = get_sql_passthrough()
+    snowpark_connect_sql_passthrough, sql_stmt = is_valid_passthrough_sql(rel.sql.query)
 
     if not snowpark_connect_sql_passthrough:
-        logical_plan = sql_parser().parseQuery(rel.sql.query)
+        logical_plan = sql_parser().parseQuery(sql_stmt)
 
         parsed_pos_args = parse_pos_args(logical_plan, rel.sql.pos_args)
         set_sql_args(rel.sql.args, parsed_pos_args)
@@ -1408,7 +1442,7 @@ def map_sql(
         return execute_logical_plan(logical_plan)
     else:
         session = snowpark.Session.get_active_session()
-        sql_df = session.sql(rel.sql.query)
+        sql_df = session.sql(sql_stmt)
         columns = sql_df.columns
         return DataFrameContainer.create_with_column_mapping(
             dataframe=sql_df,

snowflake/snowpark_connect/relation/map_stats.py

@@ -81,7 +81,7 @@ def map_approx_quantile(
     input_df = input_container.dataframe
 
     snowflake_compatible = get_boolean_session_config_param(
-        "enable_snowflake_extension_behavior"
+        "snowpark.connect.enable_snowflake_extension_behavior"
     )
 
     if not snowflake_compatible: