snowpark-connect 0.29.0__py3-none-any.whl → 0.30.0__py3-none-any.whl

snowflake/snowpark_connect/proto/snowflake_expression_ext_pb2.py

@@ -16,7 +16,7 @@ from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_e
 from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1esnowflake_expression_ext.proto\x12\rsnowflake.ext\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\"\x98\x01\n\x0c\x45xpExtension\x12@\n\x0enamed_argument\x18\x01 \x01(\x0b\x32&.snowflake.ext.NamedArgumentExpressionH\x00\x12@\n\x13subquery_expression\x18\x02 \x01(\x0b\x32!.snowflake.ext.SubqueryExpressionH\x00\x42\x04\n\x02op\"P\n\x17NamedArgumentExpression\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.Expression\"\xf4\x04\n\x12SubqueryExpression\x12&\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.Relation\x12\x45\n\rsubquery_type\x18\x02 \x01(\x0e\x32..snowflake.ext.SubqueryExpression.SubqueryType\x12Q\n\x11table_arg_options\x18\x03 \x01(\x0b\x32\x31.snowflake.ext.SubqueryExpression.TableArgOptionsH\x00\x88\x01\x01\x12\x35\n\x12in_subquery_values\x18\x04 \x03(\x0b\x32\x19.spark.connect.Expression\x1a\xbb\x01\n\x0fTableArgOptions\x12\x31\n\x0epartition_spec\x18\x01 \x03(\x0b\x32\x19.spark.connect.Expression\x12\x37\n\norder_spec\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrder\x12\"\n\x15with_single_partition\x18\x03 \x01(\x08H\x00\x88\x01\x01\x42\x18\n\x16_with_single_partition\"\x90\x01\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x12\x1b\n\x17SUBQUERY_TYPE_TABLE_ARG\x10\x03\x12\x14\n\x10SUBQUERY_TYPE_IN\x10\x04\x42\x14\n\x12_table_arg_optionsb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1esnowflake_expression_ext.proto\x12\rsnowflake.ext\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\"\xde\x01\n\x0c\x45xpExtension\x12@\n\x0enamed_argument\x18\x01 \x01(\x0b\x32&.snowflake.ext.NamedArgumentExpressionH\x00\x12@\n\x13subquery_expression\x18\x02 \x01(\x0b\x32!.snowflake.ext.SubqueryExpressionH\x00\x12\x44\n\x10interval_literal\x18\x03 \x01(\x0b\x32(.snowflake.ext.IntervalLiteralExpressionH\x00\x42\x04\n\x02op\"P\n\x17NamedArgumentExpression\x12\x0b\n\x03key\x18\x01 \x01(\t\x12(\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.Expression\"\xf4\x04\n\x12SubqueryExpression\x12&\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.Relation\x12\x45\n\rsubquery_type\x18\x02 \x01(\x0e\x32..snowflake.ext.SubqueryExpression.SubqueryType\x12Q\n\x11table_arg_options\x18\x03 \x01(\x0b\x32\x31.snowflake.ext.SubqueryExpression.TableArgOptionsH\x00\x88\x01\x01\x12\x35\n\x12in_subquery_values\x18\x04 \x03(\x0b\x32\x19.spark.connect.Expression\x1a\xbb\x01\n\x0fTableArgOptions\x12\x31\n\x0epartition_spec\x18\x01 \x03(\x0b\x32\x19.spark.connect.Expression\x12\x37\n\norder_spec\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrder\x12\"\n\x15with_single_partition\x18\x03 \x01(\x08H\x00\x88\x01\x01\x42\x18\n\x16_with_single_partition\"\x90\x01\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x12\x1b\n\x17SUBQUERY_TYPE_TABLE_ARG\x10\x03\x12\x14\n\x10SUBQUERY_TYPE_IN\x10\x04\x42\x14\n\x12_table_arg_options\"\x9f\x01\n\x19IntervalLiteralExpression\x12\x32\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.Literal\x12\x18\n\x0bstart_field\x18\x02 \x01(\x05H\x00\x88\x01\x01\x12\x16\n\tend_field\x18\x03 \x01(\x05H\x01\x88\x01\x01\x42\x0e\n\x0c_start_fieldB\x0c\n\n_end_fieldb\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -24,13 +24,15 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'snowflake_expression_ext_pb
 if _descriptor._USE_C_DESCRIPTORS == False:
   DESCRIPTOR._options = None
   _globals['_EXPEXTENSION']._serialized_start=114
-  _globals['_EXPEXTENSION']._serialized_end=266
-  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_start=268
-  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_end=348
-  _globals['_SUBQUERYEXPRESSION']._serialized_start=351
-  _globals['_SUBQUERYEXPRESSION']._serialized_end=979
-  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_start=623
-  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_end=810
-  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_start=813
-  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_end=957
+  _globals['_EXPEXTENSION']._serialized_end=336
+  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_start=338
+  _globals['_NAMEDARGUMENTEXPRESSION']._serialized_end=418
+  _globals['_SUBQUERYEXPRESSION']._serialized_start=421
+  _globals['_SUBQUERYEXPRESSION']._serialized_end=1049
+  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_start=693
+  _globals['_SUBQUERYEXPRESSION_TABLEARGOPTIONS']._serialized_end=880
+  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_start=883
+  _globals['_SUBQUERYEXPRESSION_SUBQUERYTYPE']._serialized_end=1027
+  _globals['_INTERVALLITERALEXPRESSION']._serialized_start=1052
+  _globals['_INTERVALLITERALEXPRESSION']._serialized_end=1211
 # @@protoc_insertion_point(module_scope)

snowflake/snowpark_connect/proto/snowflake_expression_ext_pb2.pyi

@@ -9,12 +9,14 @@ from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Map
 DESCRIPTOR: _descriptor.FileDescriptor
 
 class ExpExtension(_message.Message):
-    __slots__ = ("named_argument", "subquery_expression")
+    __slots__ = ("named_argument", "subquery_expression", "interval_literal")
     NAMED_ARGUMENT_FIELD_NUMBER: _ClassVar[int]
     SUBQUERY_EXPRESSION_FIELD_NUMBER: _ClassVar[int]
+    INTERVAL_LITERAL_FIELD_NUMBER: _ClassVar[int]
     named_argument: NamedArgumentExpression
     subquery_expression: SubqueryExpression
-    def __init__(self, named_argument: _Optional[_Union[NamedArgumentExpression, _Mapping]] = ..., subquery_expression: _Optional[_Union[SubqueryExpression, _Mapping]] = ...) -> None: ...
+    interval_literal: IntervalLiteralExpression
+    def __init__(self, named_argument: _Optional[_Union[NamedArgumentExpression, _Mapping]] = ..., subquery_expression: _Optional[_Union[SubqueryExpression, _Mapping]] = ..., interval_literal: _Optional[_Union[IntervalLiteralExpression, _Mapping]] = ...) -> None: ...
 
 class NamedArgumentExpression(_message.Message):
     __slots__ = ("key", "value")
@@ -56,3 +58,13 @@ class SubqueryExpression(_message.Message):
     table_arg_options: SubqueryExpression.TableArgOptions
     in_subquery_values: _containers.RepeatedCompositeFieldContainer[_expressions_pb2.Expression]
     def __init__(self, input: _Optional[_Union[_relations_pb2.Relation, _Mapping]] = ..., subquery_type: _Optional[_Union[SubqueryExpression.SubqueryType, str]] = ..., table_arg_options: _Optional[_Union[SubqueryExpression.TableArgOptions, _Mapping]] = ..., in_subquery_values: _Optional[_Iterable[_Union[_expressions_pb2.Expression, _Mapping]]] = ...) -> None: ...
+
+class IntervalLiteralExpression(_message.Message):
+    __slots__ = ("literal", "start_field", "end_field")
+    LITERAL_FIELD_NUMBER: _ClassVar[int]
+    START_FIELD_FIELD_NUMBER: _ClassVar[int]
+    END_FIELD_FIELD_NUMBER: _ClassVar[int]
+    literal: _expressions_pb2.Expression.Literal
+    start_field: int
+    end_field: int
+    def __init__(self, literal: _Optional[_Union[_expressions_pb2.Expression.Literal, _Mapping]] = ..., start_field: _Optional[int] = ..., end_field: _Optional[int] = ...) -> None: ...

snowflake/snowpark_connect/relation/map_column_ops.py

@@ -1124,10 +1124,15 @@ def map_group_map(
         group_by_df = input_df.group_by(*snowpark_grouping_expressions)
         inner_df = group_by_df._dataframe
 
-        result = inner_df.select(
-            snowpark_fn.call_table_function(
-                apply_udtf_temp_name, *inner_df.columns
-            ).over(partition_by=snowpark_grouping_expressions)
+        renamed_columns = [f"snowflake_jtf_{column}" for column in input_df.columns]
+        tfc = snowpark_fn.call_table_function(
+            apply_udtf_temp_name, *renamed_columns
+        ).over(partition_by=snowpark_grouping_expressions)
+
+        result = (
+            inner_df.to_df(renamed_columns)
+            .join_table_function(tfc)
+            .drop(*renamed_columns)
         )
     else:
         (

snowflake/snowpark_connect/relation/map_join.py

@@ -21,6 +21,9 @@ from snowflake.snowpark_connect.relation.map_relation import (
     NATURAL_JOIN_TYPE_BASE,
     map_relation,
 )
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    filter_metadata_columns,
+)
 from snowflake.snowpark_connect.utils.context import (
     push_evaluating_join_condition,
     push_sql_scope,
@@ -38,6 +41,11 @@ def map_join(rel: relation_proto.Relation) -> DataFrameContainer:
     left_container: DataFrameContainer = map_relation(rel.join.left)
     right_container: DataFrameContainer = map_relation(rel.join.right)
 
+    # Remove any metadata columns(like metada$filename) present in the dataframes.
+    # We cannot support inputfilename for multisources as each dataframe has it's own source.
+    left_container = filter_metadata_columns(left_container)
+    right_container = filter_metadata_columns(right_container)
+
     left_input: snowpark.DataFrame = left_container.dataframe
     right_input: snowpark.DataFrame = right_container.dataframe
     is_natural_join = rel.join.join_type >= NATURAL_JOIN_TYPE_BASE

snowflake/snowpark_connect/relation/map_row_ops.py

@@ -1,15 +1,14 @@
 #
 # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #
-
-
 import pyspark.sql.connect.proto.expressions_pb2 as expressions_proto
 import pyspark.sql.connect.proto.relations_pb2 as relation_proto
 from pyspark.errors.exceptions.base import AnalysisException, IllegalArgumentException
 
 import snowflake.snowpark_connect.relation.utils as utils
 from snowflake import snowpark
-from snowflake.snowpark.functions import col, expr as snowpark_expr
+from snowflake.snowpark._internal.error_message import SnowparkClientExceptionMessages
+from snowflake.snowpark.functions import col, expr as snowpark_expr, lit
 from snowflake.snowpark.types import (
     BooleanType,
     ByteType,
@@ -20,8 +19,14 @@ from snowflake.snowpark.types import (
     LongType,
     NullType,
     ShortType,
+    StructField,
+    StructType,
+)
+from snowflake.snowpark_connect.column_name_handler import (
+    ColumnNameMap,
+    schema_getter,
+    set_schema_getter,
 )
-from snowflake.snowpark_connect.column_name_handler import ColumnNameMap, schema_getter
 from snowflake.snowpark_connect.config import global_config
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.expression.literal import get_literal_field_and_name
@@ -318,23 +323,37 @@ def map_union(
         right_column_map = right_result.column_map
         columns_to_restore: dict[str, tuple[str, str]] = {}
 
-        for column in right_df.columns:
+        original_right_schema = right_df.schema
+        right_renamed_fields = []
+        for field in original_right_schema.fields:
             spark_name = (
-                right_column_map.get_spark_column_name_from_snowpark_column_name(column)
+                right_column_map.get_spark_column_name_from_snowpark_column_name(
+                    field.name
+                )
             )
-            right_df = right_df.withColumnRenamed(column, spark_name)
-            columns_to_restore[spark_name.upper()] = (spark_name, column)
+            right_df = right_df.withColumnRenamed(field.name, spark_name)
+            columns_to_restore[spark_name.upper()] = (spark_name, field.name)
+            right_renamed_fields.append(
+                StructField(spark_name, field.datatype, field.nullable)
+            )
+        set_schema_getter(right_df, lambda: StructType(right_renamed_fields))
 
-        for column in left_df.columns:
+        original_left_schema = left_df.schema
+        left_renamed_fields = []
+        for field in original_left_schema.fields:
             spark_name = (
-                left_column_map.get_spark_column_name_from_snowpark_column_name(column)
+                left_column_map.get_spark_column_name_from_snowpark_column_name(
+                    field.name
+                )
             )
-            left_df = left_df.withColumnRenamed(column, spark_name)
-            columns_to_restore[spark_name.upper()] = (spark_name, column)
+            left_df = left_df.withColumnRenamed(field.name, spark_name)
+            columns_to_restore[spark_name.upper()] = (spark_name, field.name)
+            left_renamed_fields.append(
+                StructField(spark_name, field.datatype, field.nullable)
+            )
+        set_schema_getter(left_df, lambda: StructType(left_renamed_fields))
 
-        result = left_df.unionAllByName(
-            right_df, allow_missing_columns=allow_missing_columns
-        )
+        result = _union_by_name_optimized(left_df, right_df, allow_missing_columns)
 
         if allow_missing_columns:
             spark_columns = []
@@ -809,3 +828,85 @@ def map_tail(
         alias=input_container.alias,
         cached_schema_getter=lambda: input_df.schema,
     )
+
+
+def _union_by_name_optimized(
+    left_df: snowpark.DataFrame,
+    right_df: snowpark.DataFrame,
+    allow_missing_columns: bool = False,
+) -> snowpark.DataFrame:
+    """
+    This implementation is an optimized version of Snowpark's Dataframe::_union_by_name_internal.
+    The only change is, that it avoids redundant schema queries that occur in the standard Snowpark,
+    by reusing already-fetched/calculated schemas.
+    """
+
+    left_schema = left_df.schema
+    right_schema = right_df.schema
+
+    left_cols = {field.name for field in left_schema.fields}
+    right_cols = {field.name for field in right_schema.fields}
+    right_field_map = {field.name: field for field in right_schema.fields}
+
+    missing_left = right_cols - left_cols
+    missing_right = left_cols - right_cols
+
+    def add_nulls(
+        missing_cols: set[str], to_df: snowpark.DataFrame, from_df: snowpark.DataFrame
+    ) -> snowpark.DataFrame:
+        dt_map = {field.name: field.datatype for field in from_df.schema.fields}
+        result = to_df.select(
+            "*",
+            *[lit(None).cast(dt_map[col]).alias(col) for col in missing_cols],
+        )
+
+        result_fields = []
+        for field in to_df.schema.fields:
+            result_fields.append(
+                StructField(field.name, field.datatype, field.nullable)
+            )
+        for col_name in missing_cols:
+            from_field = next(
+                field for field in from_df.schema.fields if field.name == col_name
+            )
+            result_fields.append(
+                StructField(col_name, from_field.datatype, from_field.nullable)
+            )
+
+        set_schema_getter(result, lambda: StructType(result_fields))
+
+        return result
+
+    if missing_left or missing_right:
+        if allow_missing_columns:
+            left = left_df
+            right = right_df
+            if missing_left:
+                left = add_nulls(missing_left, left, right)
+            if missing_right:
+                right = add_nulls(missing_right, right, left)
+            result = left._union_by_name_internal(right, is_all=True)
+
+            result_fields = []
+            for field in left_schema.fields:
+                result_fields.append(
+                    StructField(field.name, field.datatype, field.nullable)
+                )
+            for col_name in missing_left:
+                right_field = right_field_map[col_name]
+                result_fields.append(
+                    StructField(col_name, right_field.datatype, right_field.nullable)
+                )
+
+            set_schema_getter(result, lambda: StructType(result_fields))
+            return result
+        else:
+            raise SnowparkClientExceptionMessages.DF_CANNOT_RESOLVE_COLUMN_NAME_AMONG(
+                missing_left, missing_right
+            )
+
+    result = left_df.unionAllByName(
+        right_df, allow_missing_columns=allow_missing_columns
+    )
+    set_schema_getter(result, lambda: left_df.schema)
+    return result

snowflake/snowpark_connect/relation/map_show_string.py

@@ -15,6 +15,9 @@ from snowflake.snowpark_connect.column_name_handler import set_schema_getter
 from snowflake.snowpark_connect.config import global_config
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.relation.map_relation import map_relation
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    filter_metadata_columns,
+)
 
 
 def map_show_string(rel: relation_proto.Relation) -> pandas.DataFrame:
@@ -26,14 +29,17 @@ def map_show_string(rel: relation_proto.Relation) -> pandas.DataFrame:
     Buffer object as a single cell.
     """
     input_df_container: DataFrameContainer = map_relation(rel.show_string.input)
-    raw_input_df = input_df_container.dataframe
-    input_df = _handle_datetype_columns(raw_input_df)
+    filtered_container = filter_metadata_columns(input_df_container)
+    display_df = filtered_container.dataframe
+    display_spark_columns = filtered_container.column_map.get_spark_columns()
+
+    input_df = _handle_datetype_columns(display_df)
 
     show_string = input_df._show_string_spark(
         num_rows=rel.show_string.num_rows,
         truncate=rel.show_string.truncate,
         vertical=rel.show_string.vertical,
-        _spark_column_names=input_df_container.column_map.get_spark_columns(),
+        _spark_column_names=display_spark_columns,
         _spark_session_tz=global_config.spark_sql_session_timeZone,
     )
     return pandas.DataFrame({"show_string": [show_string]})
@@ -44,14 +50,16 @@ def map_repr_html(rel: relation_proto.Relation) -> pandas.DataFrame:
     Generate the html string representation of the input dataframe.
     """
     input_df_container: DataFrameContainer = map_relation(rel.html_string.input)
-    input_df = input_df_container.dataframe
+
+    filtered_container = filter_metadata_columns(input_df_container)
+    input_df = filtered_container.dataframe
 
     input_panda = input_df.toPandas()
     input_panda.rename(
         columns={
             analyzer_utils.unquote_if_quoted(
-                input_df_container.column_map.get_snowpark_columns()[i]
-            ): input_df_container.column_map.get_spark_columns()[i]
+                filtered_container.column_map.get_snowpark_columns()[i]
+            ): filtered_container.column_map.get_spark_columns()[i]
             for i in range(len(input_panda.columns))
         },
         inplace=True,

snowflake/snowpark_connect/relation/map_sql.py

@@ -7,6 +7,7 @@ from collections.abc import MutableMapping, MutableSequence
 from contextlib import contextmanager, suppress
 from contextvars import ContextVar
 from functools import reduce
+from typing import Tuple
 
 import jpype
 import pandas
@@ -31,6 +32,10 @@ from snowflake.snowpark._internal.analyzer.analyzer_utils import (
 from snowflake.snowpark._internal.type_utils import convert_sp_to_sf_type
 from snowflake.snowpark._internal.utils import is_sql_select_statement, quote_name
 from snowflake.snowpark.functions import when_matched, when_not_matched
+from snowflake.snowpark_connect.client import (
+    SQL_PASS_THROUGH_MARKER,
+    calculate_checksum,
+)
 from snowflake.snowpark_connect.config import (
     auto_uppercase_non_column_identifiers,
     check_table_supports_operation,
@@ -397,7 +402,7 @@ def map_sql_to_pandas_df(
     returns a tuple of None for SELECT queries to enable lazy evaluation
     """
 
-    snowpark_connect_sql_passthrough = get_sql_passthrough()
+    snowpark_connect_sql_passthrough, sql_string = is_valid_passthrough_sql(sql_string)
 
     if not snowpark_connect_sql_passthrough:
         logical_plan = sql_parser().parsePlan(sql_string)
@@ -1047,7 +1052,7 @@ def map_sql_to_pandas_df(
                     raise AnalysisException(
                         f"ALTER TABLE RENAME COLUMN is not supported for table '{full_table_identifier}'. "
                         f"This table was created as a v1 table with a data source that doesn't support column renaming. "
-                        f"To enable this operation, set 'enable_snowflake_extension_behavior' to 'true'."
+                        f"To enable this operation, set 'snowpark.connect.enable_snowflake_extension_behavior' to 'true'."
                     )
 
                 column_obj = logical_plan.column()
@@ -1282,6 +1287,14 @@ def map_sql_to_pandas_df(
                         return pandas.DataFrame({"": [""]}), ""
 
                 rows = session.sql(snowflake_sql).collect()
+            case "RefreshTable":
+                table_name_unquoted = ".".join(
+                    str(part)
+                    for part in as_java_list(logical_plan.child().multipartIdentifier())
+                )
+                SNOWFLAKE_CATALOG.refreshTable(table_name_unquoted)
+
+                return pandas.DataFrame({"": [""]}), ""
             case _:
                 execute_logical_plan(logical_plan)
                 return None, None
@@ -1302,6 +1315,27 @@ def get_sql_passthrough() -> bool:
     return get_boolean_session_config_param("snowpark.connect.sql.passthrough")
 
 
+def is_valid_passthrough_sql(sql_stmt: str) -> Tuple[bool, str]:
+    """
+    Checks if :param sql_stmt: should be executed as SQL pass-through. SQL pass-through can be detected in 1 of 2 ways:
+    1) Either Spark config parameter "snowpark.connect.sql.passthrough" is set (legacy mode, to be deprecated)
+    2) If :param sql_stmt: is created through SnowflakeSession and has correct marker + checksum
+    """
+    if get_sql_passthrough():
+        # legacy style pass-through, sql_stmt should be a whole, valid SQL statement
+        return True, sql_stmt
+
+    # check for new style, SnowflakeSession based SQL pass-through
+    sql_parts = sql_stmt.split(" ", 2)
+    if len(sql_parts) == 3:
+        marker, checksum, sql = sql_parts
+        if marker == SQL_PASS_THROUGH_MARKER and checksum == calculate_checksum(sql):
+            return True, sql
+
+    # Not a SQL pass-through
+    return False, sql_stmt
+
+
 def change_default_to_public(name: str) -> str:
     """
     Change the namespace to PUBLIC when given name is DEFAULT
@@ -1397,10 +1431,10 @@ def map_sql(
     In passthough mode as True, SAS calls session.sql() and not calling Spark Parser.
     This is to mitigate any issue not covered by spark logical plan to protobuf conversion.
     """
-    snowpark_connect_sql_passthrough = get_sql_passthrough()
+    snowpark_connect_sql_passthrough, sql_stmt = is_valid_passthrough_sql(rel.sql.query)
 
     if not snowpark_connect_sql_passthrough:
-        logical_plan = sql_parser().parseQuery(rel.sql.query)
+        logical_plan = sql_parser().parseQuery(sql_stmt)
 
         parsed_pos_args = parse_pos_args(logical_plan, rel.sql.pos_args)
         set_sql_args(rel.sql.args, parsed_pos_args)
@@ -1408,7 +1442,7 @@ def map_sql(
         return execute_logical_plan(logical_plan)
     else:
         session = snowpark.Session.get_active_session()
-        sql_df = session.sql(rel.sql.query)
+        sql_df = session.sql(sql_stmt)
         columns = sql_df.columns
         return DataFrameContainer.create_with_column_mapping(
             dataframe=sql_df,

snowflake/snowpark_connect/relation/map_stats.py

@@ -81,7 +81,7 @@ def map_approx_quantile(
     input_df = input_container.dataframe
 
     snowflake_compatible = get_boolean_session_config_param(
-        "enable_snowflake_extension_behavior"
+        "snowpark.connect.enable_snowflake_extension_behavior"
     )
 
     if not snowflake_compatible:

snowflake/snowpark_connect/relation/read/map_read_csv.py

@@ -13,6 +13,10 @@ from snowflake.snowpark.dataframe_reader import DataFrameReader
 from snowflake.snowpark.types import StringType, StructField, StructType
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.relation.read.map_read import CsvReaderConfig
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    add_filename_metadata_to_reader,
+    get_non_metadata_fields,
+)
 from snowflake.snowpark_connect.relation.read.utils import (
     get_spark_column_names_from_snowpark_columns,
     rename_columns_as_snowflake_standard,
@@ -57,12 +61,17 @@ def map_read_csv(
         snowpark_read_options["PATTERN"] = snowpark_options.get("PATTERN", None)
 
         raw_options = rel.read.data_source.options
+
         if schema is None or (
             parse_header and raw_options.get("enforceSchema", "True").lower() == "false"
         ):  # Schema has to equals to header's format
-            reader = session.read.options(snowpark_read_options)
+            reader = add_filename_metadata_to_reader(
+                session.read.options(snowpark_options), raw_options
+            )
         else:
-            reader = session.read.options(snowpark_read_options).schema(schema)
+            reader = add_filename_metadata_to_reader(
+                session.read.options(snowpark_options).schema(schema), raw_options
+            )
         df = read_data(
             reader,
             schema,
@@ -175,14 +184,16 @@ def read_data(
 ) -> snowpark.DataFrame:
     df = reader.csv(path)
     filename = path.strip("/").split("/")[-1]
+    non_metadata_fields = get_non_metadata_fields(df.schema.fields)
+
     if schema is not None:
-        if len(schema.fields) != len(df.schema.fields):
+        if len(schema.fields) != len(non_metadata_fields):
             raise Exception(f"csv load from {filename} failed.")
         if raw_options.get("enforceSchema", "True").lower() == "false":
             for i in range(len(schema.fields)):
                 if (
-                    schema.fields[i].name != df.schema.fields[i].name
-                    and f'"{schema.fields[i].name}"' != df.schema.fields[i].name
+                    schema.fields[i].name != non_metadata_fields[i].name
+                    and f'"{schema.fields[i].name}"' != non_metadata_fields[i].name
                 ):
                     raise Exception("CSV header does not conform to the schema")
         return df
@@ -191,7 +202,7 @@ def read_data(
         session, path, file_format_options, snowpark_read_options
     )
 
-    df_schema_fields = df.schema.fields
+    df_schema_fields = non_metadata_fields
     if len(headers) == len(df_schema_fields) and parse_header:
         return df.select(
             [

snowflake/snowpark_connect/relation/read/map_read_json.py

@@ -29,6 +29,9 @@ from snowflake.snowpark.types import (
 )
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.relation.read.map_read import JsonReaderConfig
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    add_filename_metadata_to_reader,
+)
 from snowflake.snowpark_connect.relation.read.utils import (
     get_spark_column_names_from_snowpark_columns,
     rename_columns_as_snowflake_standard,
@@ -66,19 +69,26 @@ def map_read_json(
         )
     else:
         snowpark_options = options.convert_to_snowpark_args()
+        raw_options = rel.read.data_source.options
         snowpark_options["infer_schema"] = True
 
         rows_to_infer_schema = snowpark_options.pop("rowstoinferschema", 1000)
         dropFieldIfAllNull = snowpark_options.pop("dropfieldifallnull", False)
         batch_size = snowpark_options.pop("batchsize", 1000)
 
-        reader = session.read.options(snowpark_options)
+        reader = add_filename_metadata_to_reader(
+            session.read.options(snowpark_options), raw_options
+        )
 
         df = reader.json(paths[0])
         if len(paths) > 1:
             # TODO: figure out if this is what Spark does.
             for p in paths[1:]:
-                df = df.union_all(session.read.options(snowpark_options).json(p))
+                df = df.union_all(
+                    add_filename_metadata_to_reader(
+                        session.read.options(snowpark_options), raw_options
+                    ).json(p)
+                )
 
         if schema is None:
             schema = copy.deepcopy(df.schema)

snowflake/snowpark_connect/relation/read/map_read_parquet.py

@@ -22,6 +22,9 @@ from snowflake.snowpark._internal.analyzer.analyzer_utils import (
 from snowflake.snowpark.column import METADATA_FILENAME
 from snowflake.snowpark.types import DataType, DoubleType, IntegerType, StringType
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    add_filename_metadata_to_reader,
+)
 from snowflake.snowpark_connect.relation.read.reader_config import ReaderWriterConfig
 from snowflake.snowpark_connect.relation.read.utils import (
     rename_columns_as_snowflake_standard,
@@ -46,10 +49,13 @@ def map_read_parquet(
         )
 
     snowpark_options = options.convert_to_snowpark_args()
+    raw_options = rel.read.data_source.options
     assert schema is None, "Read PARQUET does not support user schema"
     assert len(paths) > 0, "Read PARQUET expects at least one path"
 
-    reader = session.read.options(snowpark_options)
+    reader = add_filename_metadata_to_reader(
+        session.read.options(snowpark_options), raw_options
+    )
 
     if len(paths) == 1:
         df = _read_parquet_with_partitions(session, reader, paths[0])