snowpark-connect 0.29.0__py3-none-any.whl → 0.30.0__py3-none-any.whl

snowflake/snowpark_connect/relation/read/metadata_utils.py

@@ -0,0 +1,159 @@
+#
+# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
+#
+
+"""
+Utilities for handling internal metadata columns in file-based DataFrames.
+"""
+
+import os
+
+import pandas
+from pyspark.errors.exceptions.base import AnalysisException
+
+from snowflake import snowpark
+from snowflake.snowpark.column import METADATA_FILENAME
+from snowflake.snowpark.functions import col
+from snowflake.snowpark.types import StructField
+from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
+
+# Constant for the metadata filename column name
+METADATA_FILENAME_COLUMN = "METADATA$FILENAME"
+
+
+def add_filename_metadata_to_reader(
+    reader: snowpark.DataFrameReader,
+    options: dict | None = None,
+) -> snowpark.DataFrameReader:
+    """
+    Add filename metadata to a DataFrameReader based on configuration.
+
+    Args:
+        reader: Snowpark DataFrameReader instance
+        options: Dictionary of options to check for metadata configuration
+
+    Returns:
+        DataFrameReader with filename metadata enabled if configured, otherwise unchanged
+    """
+    # NOTE: SNOWPARK_POPULATE_FILE_METADATA_DEFAULT is an internal environment variable
+    # used only for CI testing to verify no metadata columns leak in regular file operations.
+    # This environment variable should NOT be exposed to end users. Users should only use snowpark.populateFileMetadata
+    # to enable metadata population.
+    metadata_default = os.environ.get(
+        "SNOWPARK_POPULATE_FILE_METADATA_DEFAULT", "false"
+    )
+
+    populate_metadata = (
+        options.get("snowpark.populateFileMetadata", metadata_default)
+        if options
+        else metadata_default
+    ).lower() == "true"
+
+    if populate_metadata:
+        return reader.with_metadata(METADATA_FILENAME)
+    else:
+        return reader
+
+
+def get_non_metadata_fields(schema_fields: list[StructField]) -> list[StructField]:
+    """
+    Filter out METADATA$FILENAME fields from a list of schema fields.
+
+    Args:
+        schema_fields: List of StructField objects from a DataFrame schema
+
+    Returns:
+        List of StructField objects excluding METADATA$FILENAME
+    """
+    return [field for field in schema_fields if field.name != METADATA_FILENAME_COLUMN]
+
+
+def get_non_metadata_column_names(schema_fields: list[StructField]) -> list[str]:
+    """
+    Get column names from schema fields, excluding METADATA$FILENAME.
+
+    Args:
+        schema_fields: List of StructField objects from a DataFrame schema
+
+    Returns:
+        List of column names (strings) excluding METADATA$FILENAME
+    """
+    return [
+        field.name for field in schema_fields if field.name != METADATA_FILENAME_COLUMN
+    ]
+
+
+def filter_metadata_column_name(column_names: list[str]) -> list[str]:
+    """
+    Get column names from column_names, excluding METADATA$FILENAME.
+
+    Returns:
+        List of column names (strings) excluding METADATA$FILENAME
+    """
+    return [
+        col_name for col_name in column_names if col_name != METADATA_FILENAME_COLUMN
+    ]
+
+
+def filter_metadata_columns(
+    result_container: DataFrameContainer | pandas.DataFrame | None,
+) -> DataFrameContainer | pandas.DataFrame | None:
+    """
+    Filter METADATA$FILENAME from DataFrame container for execution and write operations.
+
+    Args:
+        result_container: DataFrameContainer or pandas DataFrame to filter
+
+    Returns:
+        Filtered container (callers can access dataframe via container.dataframe)
+    """
+    # Handle pandas DataFrame case - return as-is
+    if isinstance(result_container, pandas.DataFrame):
+        return result_container
+
+    if result_container is None:
+        return None
+
+    result_df = result_container.dataframe
+    if not isinstance(result_df, snowpark.DataFrame):
+        return result_container
+
+    df_columns = result_container.column_map.get_snowpark_columns()
+    has_metadata_filename = any(name == METADATA_FILENAME_COLUMN for name in df_columns)
+
+    if not has_metadata_filename:
+        return result_container
+
+    non_metadata_columns = filter_metadata_column_name(df_columns)
+
+    if len(non_metadata_columns) == 0:
+        # DataFrame contains only metadata columns (METADATA$FILENAME), no actual data columns remaining.
+        # We don't have a way to return an empty dataframe.
+        raise AnalysisException(
+            "[DATAFRAME_MISSING_DATA_COLUMNS] Cannot perform operation on DataFrame that contains no data columns."
+        )
+
+    filtered_df = result_df.select([col(name) for name in non_metadata_columns])
+
+    original_spark_columns = result_container.column_map.get_spark_columns()
+    original_snowpark_columns = result_container.column_map.get_snowpark_columns()
+
+    filtered_spark_columns = []
+    filtered_snowpark_columns = []
+
+    for i, colname in enumerate(df_columns):
+        if colname != METADATA_FILENAME_COLUMN:
+            filtered_spark_columns.append(original_spark_columns[i])
+            filtered_snowpark_columns.append(original_snowpark_columns[i])
+
+    new_container = DataFrameContainer.create_with_column_mapping(
+        dataframe=filtered_df,
+        spark_column_names=filtered_spark_columns,
+        snowpark_column_names=filtered_snowpark_columns,
+        column_metadata=result_container.column_map.column_metadata,
+        table_name=result_container.table_name,
+        alias=result_container.alias,
+        partition_hint=result_container.partition_hint,
+    )
+
+    return new_container

snowflake/snowpark_connect/relation/write/map_write.py

@@ -40,6 +40,9 @@ from snowflake.snowpark_connect.relation.io_utils import (
     supported_compressions_for_format,
 )
 from snowflake.snowpark_connect.relation.map_relation import map_relation
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    filter_metadata_columns,
+)
 from snowflake.snowpark_connect.relation.read.reader_config import CsvWriterConfig
 from snowflake.snowpark_connect.relation.stage_locator import get_paths_from_stage
 from snowflake.snowpark_connect.relation.utils import (
@@ -129,6 +132,19 @@ def map_write(request: proto_base.ExecutePlanRequest):
 
     result = map_relation(write_op.input)
     input_df: snowpark.DataFrame = handle_column_names(result, write_op.source)
+
+    # Create updated container with transformed dataframe, then filter METADATA$FILENAME columns
+    # Update the container to use the transformed dataframe from handle_column_names
+    updated_result = DataFrameContainer(
+        dataframe=input_df,
+        column_map=result.column_map,
+        table_name=result.table_name,
+        alias=result.alias,
+        partition_hint=result.partition_hint,
+    )
+    updated_result = filter_metadata_columns(updated_result)
+    input_df = updated_result.dataframe
+
     session: snowpark.Session = get_or_create_snowpark_session()
 
     # Snowflake saveAsTable doesn't support format
@@ -537,6 +553,19 @@ def map_write_v2(request: proto_base.ExecutePlanRequest):
     snowpark_table_name = _spark_to_snowflake(write_op.table_name)
     result = map_relation(write_op.input)
     input_df: snowpark.DataFrame = handle_column_names(result, "table")
+
+    # Create updated container with transformed dataframe, then filter METADATA$FILENAME columns
+    # Update the container to use the transformed dataframe from handle_column_names
+    updated_result = DataFrameContainer(
+        dataframe=input_df,
+        column_map=result.column_map,
+        table_name=result.table_name,
+        alias=result.alias,
+        partition_hint=result.partition_hint,
+    )
+    updated_result = filter_metadata_columns(updated_result)
+    input_df = updated_result.dataframe
+
     session: snowpark.Session = get_or_create_snowpark_session()
 
     if write_op.table_name is None or write_op.table_name == "":

snowflake/snowpark_connect/server.py

@@ -232,12 +232,20 @@ class SnowflakeConnectServicer(proto_base_grpc.SparkConnectServiceServicer):
             match request.WhichOneof("analyze"):
                 case "schema":
                     result = map_relation(request.schema.plan.root)
-                    snowpark_df = result.dataframe
-                    snowpark_schema: snowpark.types.StructType = snowpark_df.schema
+
+                    from snowflake.snowpark_connect.relation.read.metadata_utils import (
+                        filter_metadata_columns,
+                    )
+
+                    filtered_result = filter_metadata_columns(result)
+                    filtered_df = filtered_result.dataframe
+
                     schema = proto_base.AnalyzePlanResponse.Schema(
                         schema=types_proto.DataType(
                             **snowpark_to_proto_type(
-                                snowpark_schema, result.column_map, snowpark_df
+                                filtered_df.schema,
+                                filtered_result.column_map,
+                                filtered_df,
                             )
                         )
                     )

snowflake/snowpark_connect/type_mapping.py

@@ -30,6 +30,10 @@ from snowflake.snowpark_connect.date_time_format_mapping import (
     convert_spark_format_to_snowflake,
 )
 from snowflake.snowpark_connect.expression.literal import get_literal_field_and_name
+from snowflake.snowpark_connect.expression.map_sql_expression import (
+    _INTERVAL_DAYTIME_PATTERN_RE,
+    _INTERVAL_YEARMONTH_PATTERN_RE,
+)
 from snowflake.snowpark_connect.utils.context import get_is_evaluating_sql
 from snowflake.snowpark_connect.utils.snowpark_connect_logging import logger
 from snowflake.snowpark_connect.utils.telemetry import (
@@ -274,6 +278,18 @@ def snowpark_to_proto_type(
         case snowpark.types.VariantType:
             # For now we are returning a string type for variant types.
             return {"string": types_proto.DataType.String()}
+        case snowpark.types.YearMonthIntervalType:
+            return {
+                "year_month_interval": types_proto.DataType.YearMonthInterval(
+                    start_field=data_type.start_field, end_field=data_type.end_field
+                )
+            }
+        case snowpark.types.DayTimeIntervalType:
+            return {
+                "day_time_interval": types_proto.DataType.DayTimeInterval(
+                    start_field=data_type.start_field, end_field=data_type.end_field
+                )
+            }
         case _:
             raise SnowparkConnectNotImplementedError(
                 f"Unsupported snowpark data type: {data_type}"
@@ -328,6 +344,24 @@ def cast_to_match_snowpark_type(
             return str(content)
         case snowpark.types.TimestampType:
             return str(content)
+        case snowpark.types.YearMonthIntervalType:
+            if isinstance(content, (int, float)):
+                total_months = int(content)
+                years = total_months // 12
+                months = total_months % 12
+                return f"INTERVAL '{years}-{months}' YEAR TO MONTH"
+            elif isinstance(content, str) and content.startswith(("+", "-")):
+                # Handle Snowflake's native interval format (e.g., "+11-08" or "-2-3")
+                # Convert to Spark's format: "INTERVAL 'Y-M' YEAR TO MONTH"
+                sign = content[0]
+                interval_part = content[1:]  # Remove sign
+                if sign == "-":
+                    return f"INTERVAL '-{interval_part}' YEAR TO MONTH"
+                else:
+                    return f"INTERVAL '{interval_part}' YEAR TO MONTH"
+            return str(content)
+        case snowpark.types.DayTimeIntervalType:
+            return str(content)
         case _:
             raise SnowparkConnectNotImplementedError(
                 f"Unsupported snowpark data type in casting: {data_type}"
@@ -411,6 +445,18 @@ def proto_to_snowpark_type(
             # For UDT types, return the underlying SQL type
             logger.debug("Returning underlying sql type for udt")
             return proto_to_snowpark_type(data_type.udt.sql_type)
+        case "year_month_interval":
+            # Preserve start_field and end_field from protobuf
+            return snowpark.types.YearMonthIntervalType(
+                start_field=data_type.year_month_interval.start_field,
+                end_field=data_type.year_month_interval.end_field,
+            )
+        case "day_time_interval":
+            # Preserve start_field and end_field from protobuf
+            return snowpark.types.DayTimeIntervalType(
+                start_field=data_type.day_time_interval.start_field,
+                end_field=data_type.day_time_interval.end_field,
+            )
         case _:
             return map_simple_types(data_type.WhichOneof("kind"))
 
@@ -523,6 +569,12 @@ def map_snowpark_types_to_pyarrow_types(
             return pa.timestamp(unit, tz=tz)
         case snowpark.types.VariantType:
             return pa.string()
+        case snowpark.types.YearMonthIntervalType:
+            # Return string type so formatted intervals are preserved in display
+            return pa.string()
+        case snowpark.types.DayTimeIntervalType:
+            # Return string type so formatted intervals are preserved in display
+            return pa.string()
         case _:
             raise SnowparkConnectNotImplementedError(
                 f"Unsupported snowpark data type: {snowpark_type}"
@@ -676,6 +728,14 @@ def map_pyspark_types_to_snowpark_types(
         return snowpark.types.TimestampType()
     if isinstance(type_to_map, pyspark.sql.types.TimestampNTZType):
         return snowpark.types.TimestampType(timezone=TimestampTimeZone.NTZ)
+    if isinstance(type_to_map, pyspark.sql.types.YearMonthIntervalType):
+        return snowpark.types.YearMonthIntervalType(
+            type_to_map.startField, type_to_map.endField
+        )
+    if isinstance(type_to_map, pyspark.sql.types.DayTimeIntervalType):
+        return snowpark.types.DayTimeIntervalType(
+            type_to_map.startField, type_to_map.endField
+        )
     raise SnowparkConnectNotImplementedError(
         f"Unsupported spark data type: {type_to_map}"
     )
@@ -743,6 +803,14 @@ def map_snowpark_to_pyspark_types(
         if type_to_map.tz == snowpark.types.TimestampTimeZone.NTZ:
             return pyspark.sql.types.TimestampNTZType()
         return pyspark.sql.types.TimestampType()
+    if isinstance(type_to_map, snowpark.types.YearMonthIntervalType):
+        return pyspark.sql.types.YearMonthIntervalType(
+            type_to_map.start_field, type_to_map.end_field
+        )
+    if isinstance(type_to_map, snowpark.types.DayTimeIntervalType):
+        return pyspark.sql.types.DayTimeIntervalType(
+            type_to_map.start_field, type_to_map.end_field
+        )
     raise SnowparkConnectNotImplementedError(f"Unsupported data type: {type_to_map}")
 
 
@@ -785,10 +853,14 @@ def map_simple_types(simple_type: str) -> snowpark.types.DataType:
             return snowpark.types.TimestampType(snowpark.types.TimestampTimeZone.NTZ)
         case "timestamp_ltz":
             return snowpark.types.TimestampType(snowpark.types.TimestampTimeZone.LTZ)
+        case "year_month_interval":
+            return snowpark.types.YearMonthIntervalType()
         case "day_time_interval":
-            # this is not a column type in snowflake so there won't be a dataframe column
-            # with this, for now this type won't make any sense
-            return snowpark.types.StringType()
+            return snowpark.types.DayTimeIntervalType()
+        case type_name if _INTERVAL_YEARMONTH_PATTERN_RE.match(type_name):
+            return snowpark.types.YearMonthIntervalType()
+        case type_name if _INTERVAL_DAYTIME_PATTERN_RE.match(type_name):
+            return snowpark.types.DayTimeIntervalType()
         case _:
             if simple_type.startswith("decimal"):
                 precision = int(simple_type.split("(")[1].split(",")[0])

snowflake/snowpark_connect/utils/describe_query_cache.py

@@ -16,7 +16,6 @@ from snowflake.snowpark_connect.utils.concurrent import SynchronizedDict
 from snowflake.snowpark_connect.utils.snowpark_connect_logging import logger
 from snowflake.snowpark_connect.utils.telemetry import telemetry
 
-DESCRIBE_CACHE_TTL_SECONDS = 15
 USE_DESCRIBE_QUERY_CACHE = True
 
 DDL_DETECTION_PATTERN = re.compile(r"\s*(CREATE|ALTER|DROP)\b", re.IGNORECASE)
@@ -51,6 +50,8 @@ class DescribeQueryCache:
         return sql_query
 
     def get(self, sql_query: str) -> list[ResultMetadataV2] | None:
+        from snowflake.snowpark_connect.config import get_describe_cache_ttl_seconds
+
         telemetry.report_describe_query_cache_lookup()
 
         cache_key = self._get_cache_key(sql_query)
@@ -59,7 +60,9 @@ class DescribeQueryCache:
 
         if key in self._cache:
             result, timestamp = self._cache[key]
-            if current_time < timestamp + DESCRIBE_CACHE_TTL_SECONDS:
+
+            expired_by = current_time - (timestamp + get_describe_cache_ttl_seconds())
+            if expired_by < 0:
                 logger.debug(
                     f"Returning query result from cache for query: {sql_query[:20]}"
                 )
@@ -92,7 +95,7 @@ class DescribeQueryCache:
                 telemetry.report_describe_query_cache_hit()
                 return result
             else:
-                telemetry.report_describe_query_cache_expired()
+                telemetry.report_describe_query_cache_expired(expired_by)
                 del self._cache[key]
         return None
 

snowflake/snowpark_connect/utils/session.py

@@ -120,6 +120,7 @@ def configure_snowpark_session(session: snowpark.Session):
         "PYTHON_SNOWPARK_USE_SCOPED_TEMP_OBJECTS": "false",  # this is required for creating udfs from sproc
         "ENABLE_STRUCTURED_TYPES_IN_SNOWPARK_CONNECT_RESPONSE": "true",
         "QUERY_TAG": f"'{query_tag}'",
+        "FEATURE_INTERVAL_TYPES": "enabled",
     }
 
     session.sql(

snowflake/snowpark_connect/utils/telemetry.py

@@ -15,6 +15,7 @@ from enum import Enum, unique
 from typing import Dict
 
 import google.protobuf.message
+import pyspark.sql.connect.proto.base_pb2 as proto_base
 
 from snowflake.connector.cursor import SnowflakeCursor
 from snowflake.connector.telemetry import (
@@ -290,10 +291,7 @@ class Telemetry:
 
         self._request_summary.set(summary)
 
-        if hasattr(request, "plan"):
-            summary["query_plan"] = _protobuf_to_json_with_redaction(
-                request.plan, REDACTED_PLAN_SUFFIXES
-            )
+        _set_query_plan(request, summary)
 
     def _not_in_request(self):
         # we don't want to add things to the summary if it's not initialized
@@ -454,7 +452,7 @@ class Telemetry:
         summary["describe_cache_hits"] += 1
 
     @safe
-    def report_describe_query_cache_expired(self):
+    def report_describe_query_cache_expired(self, expired_by: float):
         """Report a describe query cache hit."""
         if self._not_in_request():
             return
@@ -466,6 +464,11 @@ class Telemetry:
 
         summary["describe_cache_expired"] += 1
 
+        if "describe_cache_expired_by" not in summary:
+            summary["describe_cache_expired_by"] = []
+
+        summary["describe_cache_expired_by"].append(expired_by)
+
     @safe
     def report_describe_query_cache_clear(self, query_prefix: str):
         """Report a describe query cache clear."""
@@ -697,6 +700,28 @@ def _protobuf_to_json_with_redaction(
     )
 
 
+def _set_query_plan(request: google.protobuf.message.Message, summary: dict) -> None:
+    if isinstance(request, proto_base.ExecutePlanRequest):
+        # ExecutePlanRequest has plan at top level
+        if hasattr(request, "plan"):
+            summary["query_plan"] = (
+                _protobuf_to_json_with_redaction(request.plan, REDACTED_PLAN_SUFFIXES),
+            )
+
+    elif isinstance(request, proto_base.AnalyzePlanRequest):
+        # AnalyzePlanRequest has plan under oneof analyze
+        analyze_type = request.WhichOneof("analyze")
+        if not analyze_type:
+            return
+
+        summary["analyze_type"] = analyze_type
+        analyze_field = getattr(request, analyze_type)
+        if hasattr(analyze_field, "plan"):
+            summary["query_plan"] = _protobuf_to_json_with_redaction(
+                analyze_field.plan, REDACTED_PLAN_SUFFIXES
+            )
+
+
 # global telemetry client
 telemetry = Telemetry(is_enabled="SNOWPARK_CONNECT_DISABLE_TELEMETRY" not in os.environ)
 

snowflake/snowpark_connect/version.py

@@ -2,4 +2,4 @@
 #
 # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #
-VERSION = (0,29,0)
+VERSION = (0,30,0)

{snowpark_connect-0.29.0.dist-info → snowpark_connect-0.30.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: snowpark-connect
-Version: 0.29.0
+Version: 0.30.0
 Summary: Snowpark Connect for Spark
 Author: Snowflake, Inc
 License: Apache License, Version 2.0