snowpark-connect 0.29.0__py3-none-any.whl → 0.30.1__py3-none-any.whl

snowflake/snowpark_connect/relation/read/map_read_csv.py

@@ -13,6 +13,10 @@ from snowflake.snowpark.dataframe_reader import DataFrameReader
 from snowflake.snowpark.types import StringType, StructField, StructType
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.relation.read.map_read import CsvReaderConfig
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    add_filename_metadata_to_reader,
+    get_non_metadata_fields,
+)
 from snowflake.snowpark_connect.relation.read.utils import (
     get_spark_column_names_from_snowpark_columns,
     rename_columns_as_snowflake_standard,
@@ -57,12 +61,17 @@ def map_read_csv(
         snowpark_read_options["PATTERN"] = snowpark_options.get("PATTERN", None)
 
         raw_options = rel.read.data_source.options
+
         if schema is None or (
             parse_header and raw_options.get("enforceSchema", "True").lower() == "false"
         ):  # Schema has to equals to header's format
-            reader = session.read.options(snowpark_read_options)
+            reader = add_filename_metadata_to_reader(
+                session.read.options(snowpark_options), raw_options
+            )
         else:
-            reader = session.read.options(snowpark_read_options).schema(schema)
+            reader = add_filename_metadata_to_reader(
+                session.read.options(snowpark_options).schema(schema), raw_options
+            )
         df = read_data(
             reader,
             schema,
@@ -175,14 +184,16 @@ def read_data(
 ) -> snowpark.DataFrame:
     df = reader.csv(path)
     filename = path.strip("/").split("/")[-1]
+    non_metadata_fields = get_non_metadata_fields(df.schema.fields)
+
     if schema is not None:
-        if len(schema.fields) != len(df.schema.fields):
+        if len(schema.fields) != len(non_metadata_fields):
             raise Exception(f"csv load from {filename} failed.")
         if raw_options.get("enforceSchema", "True").lower() == "false":
             for i in range(len(schema.fields)):
                 if (
-                    schema.fields[i].name != df.schema.fields[i].name
-                    and f'"{schema.fields[i].name}"' != df.schema.fields[i].name
+                    schema.fields[i].name != non_metadata_fields[i].name
+                    and f'"{schema.fields[i].name}"' != non_metadata_fields[i].name
                 ):
                     raise Exception("CSV header does not conform to the schema")
         return df
@@ -191,7 +202,7 @@ def read_data(
         session, path, file_format_options, snowpark_read_options
     )
 
-    df_schema_fields = df.schema.fields
+    df_schema_fields = non_metadata_fields
     if len(headers) == len(df_schema_fields) and parse_header:
         return df.select(
             [

snowflake/snowpark_connect/relation/read/map_read_json.py

@@ -29,6 +29,9 @@ from snowflake.snowpark.types import (
 )
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.relation.read.map_read import JsonReaderConfig
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    add_filename_metadata_to_reader,
+)
 from snowflake.snowpark_connect.relation.read.utils import (
     get_spark_column_names_from_snowpark_columns,
     rename_columns_as_snowflake_standard,
@@ -66,19 +69,26 @@ def map_read_json(
         )
     else:
         snowpark_options = options.convert_to_snowpark_args()
+        raw_options = rel.read.data_source.options
         snowpark_options["infer_schema"] = True
 
         rows_to_infer_schema = snowpark_options.pop("rowstoinferschema", 1000)
         dropFieldIfAllNull = snowpark_options.pop("dropfieldifallnull", False)
         batch_size = snowpark_options.pop("batchsize", 1000)
 
-        reader = session.read.options(snowpark_options)
+        reader = add_filename_metadata_to_reader(
+            session.read.options(snowpark_options), raw_options
+        )
 
         df = reader.json(paths[0])
         if len(paths) > 1:
             # TODO: figure out if this is what Spark does.
             for p in paths[1:]:
-                df = df.union_all(session.read.options(snowpark_options).json(p))
+                df = df.union_all(
+                    add_filename_metadata_to_reader(
+                        session.read.options(snowpark_options), raw_options
+                    ).json(p)
+                )
 
         if schema is None:
             schema = copy.deepcopy(df.schema)

snowflake/snowpark_connect/relation/read/map_read_parquet.py

@@ -22,6 +22,9 @@ from snowflake.snowpark._internal.analyzer.analyzer_utils import (
 from snowflake.snowpark.column import METADATA_FILENAME
 from snowflake.snowpark.types import DataType, DoubleType, IntegerType, StringType
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    add_filename_metadata_to_reader,
+)
 from snowflake.snowpark_connect.relation.read.reader_config import ReaderWriterConfig
 from snowflake.snowpark_connect.relation.read.utils import (
     rename_columns_as_snowflake_standard,
@@ -46,10 +49,13 @@ def map_read_parquet(
         )
 
     snowpark_options = options.convert_to_snowpark_args()
+    raw_options = rel.read.data_source.options
     assert schema is None, "Read PARQUET does not support user schema"
     assert len(paths) > 0, "Read PARQUET expects at least one path"
 
-    reader = session.read.options(snowpark_options)
+    reader = add_filename_metadata_to_reader(
+        session.read.options(snowpark_options), raw_options
+    )
 
     if len(paths) == 1:
         df = _read_parquet_with_partitions(session, reader, paths[0])

snowflake/snowpark_connect/relation/read/metadata_utils.py

@@ -0,0 +1,159 @@
+#
+# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
+#
+
+"""
+Utilities for handling internal metadata columns in file-based DataFrames.
+"""
+
+import os
+
+import pandas
+from pyspark.errors.exceptions.base import AnalysisException
+
+from snowflake import snowpark
+from snowflake.snowpark.column import METADATA_FILENAME
+from snowflake.snowpark.functions import col
+from snowflake.snowpark.types import StructField
+from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
+
+# Constant for the metadata filename column name
+METADATA_FILENAME_COLUMN = "METADATA$FILENAME"
+
+
+def add_filename_metadata_to_reader(
+    reader: snowpark.DataFrameReader,
+    options: dict | None = None,
+) -> snowpark.DataFrameReader:
+    """
+    Add filename metadata to a DataFrameReader based on configuration.
+
+    Args:
+        reader: Snowpark DataFrameReader instance
+        options: Dictionary of options to check for metadata configuration
+
+    Returns:
+        DataFrameReader with filename metadata enabled if configured, otherwise unchanged
+    """
+    # NOTE: SNOWPARK_POPULATE_FILE_METADATA_DEFAULT is an internal environment variable
+    # used only for CI testing to verify no metadata columns leak in regular file operations.
+    # This environment variable should NOT be exposed to end users. Users should only use snowpark.populateFileMetadata
+    # to enable metadata population.
+    metadata_default = os.environ.get(
+        "SNOWPARK_POPULATE_FILE_METADATA_DEFAULT", "false"
+    )
+
+    populate_metadata = (
+        options.get("snowpark.populateFileMetadata", metadata_default)
+        if options
+        else metadata_default
+    ).lower() == "true"
+
+    if populate_metadata:
+        return reader.with_metadata(METADATA_FILENAME)
+    else:
+        return reader
+
+
+def get_non_metadata_fields(schema_fields: list[StructField]) -> list[StructField]:
+    """
+    Filter out METADATA$FILENAME fields from a list of schema fields.
+
+    Args:
+        schema_fields: List of StructField objects from a DataFrame schema
+
+    Returns:
+        List of StructField objects excluding METADATA$FILENAME
+    """
+    return [field for field in schema_fields if field.name != METADATA_FILENAME_COLUMN]
+
+
+def get_non_metadata_column_names(schema_fields: list[StructField]) -> list[str]:
+    """
+    Get column names from schema fields, excluding METADATA$FILENAME.
+
+    Args:
+        schema_fields: List of StructField objects from a DataFrame schema
+
+    Returns:
+        List of column names (strings) excluding METADATA$FILENAME
+    """
+    return [
+        field.name for field in schema_fields if field.name != METADATA_FILENAME_COLUMN
+    ]
+
+
+def filter_metadata_column_name(column_names: list[str]) -> list[str]:
+    """
+    Get column names from column_names, excluding METADATA$FILENAME.
+
+    Returns:
+        List of column names (strings) excluding METADATA$FILENAME
+    """
+    return [
+        col_name for col_name in column_names if col_name != METADATA_FILENAME_COLUMN
+    ]
+
+
+def filter_metadata_columns(
+    result_container: DataFrameContainer | pandas.DataFrame | None,
+) -> DataFrameContainer | pandas.DataFrame | None:
+    """
+    Filter METADATA$FILENAME from DataFrame container for execution and write operations.
+
+    Args:
+        result_container: DataFrameContainer or pandas DataFrame to filter
+
+    Returns:
+        Filtered container (callers can access dataframe via container.dataframe)
+    """
+    # Handle pandas DataFrame case - return as-is
+    if isinstance(result_container, pandas.DataFrame):
+        return result_container
+
+    if result_container is None:
+        return None
+
+    result_df = result_container.dataframe
+    if not isinstance(result_df, snowpark.DataFrame):
+        return result_container
+
+    df_columns = result_container.column_map.get_snowpark_columns()
+    has_metadata_filename = any(name == METADATA_FILENAME_COLUMN for name in df_columns)
+
+    if not has_metadata_filename:
+        return result_container
+
+    non_metadata_columns = filter_metadata_column_name(df_columns)
+
+    if len(non_metadata_columns) == 0:
+        # DataFrame contains only metadata columns (METADATA$FILENAME), no actual data columns remaining.
+        # We don't have a way to return an empty dataframe.
+        raise AnalysisException(
+            "[DATAFRAME_MISSING_DATA_COLUMNS] Cannot perform operation on DataFrame that contains no data columns."
+        )
+
+    filtered_df = result_df.select([col(name) for name in non_metadata_columns])
+
+    original_spark_columns = result_container.column_map.get_spark_columns()
+    original_snowpark_columns = result_container.column_map.get_snowpark_columns()
+
+    filtered_spark_columns = []
+    filtered_snowpark_columns = []
+
+    for i, colname in enumerate(df_columns):
+        if colname != METADATA_FILENAME_COLUMN:
+            filtered_spark_columns.append(original_spark_columns[i])
+            filtered_snowpark_columns.append(original_snowpark_columns[i])
+
+    new_container = DataFrameContainer.create_with_column_mapping(
+        dataframe=filtered_df,
+        spark_column_names=filtered_spark_columns,
+        snowpark_column_names=filtered_snowpark_columns,
+        column_metadata=result_container.column_map.column_metadata,
+        table_name=result_container.table_name,
+        alias=result_container.alias,
+        partition_hint=result_container.partition_hint,
+    )
+
+    return new_container

snowflake/snowpark_connect/relation/utils.py

@@ -92,6 +92,21 @@ TYPE_MAP_FOR_TO_SCHEMA = {
 }
 
 
+# This mapping is used to map the compression type to the extension of the file.
+FILE_COMPRESSION_TO_EXTENSION = {
+    "GZIP": "gz",
+    "BZ2": "bz2",
+    "BROTLI": "br",
+    "ZSTD": "zst",
+    "DEFLATE": "deflate",
+    "RAW_DEFLATE": "raw_deflate",
+    "SNAPPY": "snappy",
+    "LZO": "lzo",
+    "LZ4": "lz4",
+    "BZIP2": "bz2",
+}
+
+
 def get_df_with_partition_row_number(
     container: DataFrameContainer,
     plan_id: int | None,
@@ -186,13 +201,15 @@ def generate_spark_compatible_filename(
 
     # Add compression if specified and not 'none'
     if compression and compression.lower() not in ("none", "uncompressed"):
-        compression_part = f".{compression.lower()}"
+        compression_part = f".{FILE_COMPRESSION_TO_EXTENSION.get(compression.upper(), compression.lower())}"
     else:
         compression_part = ""
 
     # Add format extension if specified
-    if format_ext:
+    if format_ext == "parquet":
         return f"{base_name}{compression_part}.{format_ext}"
+    elif format_ext is not None and format_ext != "":
+        return f"{base_name}.{format_ext}{compression_part}"
     else:
         return f"{base_name}{compression_part}"
 

snowflake/snowpark_connect/relation/write/map_write.py

@@ -35,11 +35,13 @@ from snowflake.snowpark_connect.config import (
 from snowflake.snowpark_connect.dataframe_container import DataFrameContainer
 from snowflake.snowpark_connect.relation.io_utils import (
     convert_file_prefix_path,
+    get_compression_for_source_and_options,
     is_cloud_path,
-    is_supported_compression,
-    supported_compressions_for_format,
 )
 from snowflake.snowpark_connect.relation.map_relation import map_relation
+from snowflake.snowpark_connect.relation.read.metadata_utils import (
+    filter_metadata_columns,
+)
 from snowflake.snowpark_connect.relation.read.reader_config import CsvWriterConfig
 from snowflake.snowpark_connect.relation.stage_locator import get_paths_from_stage
 from snowflake.snowpark_connect.relation.utils import (
@@ -129,8 +131,26 @@ def map_write(request: proto_base.ExecutePlanRequest):
 
     result = map_relation(write_op.input)
     input_df: snowpark.DataFrame = handle_column_names(result, write_op.source)
+
+    # Create updated container with transformed dataframe, then filter METADATA$FILENAME columns
+    # Update the container to use the transformed dataframe from handle_column_names
+    updated_result = DataFrameContainer(
+        dataframe=input_df,
+        column_map=result.column_map,
+        table_name=result.table_name,
+        alias=result.alias,
+        partition_hint=result.partition_hint,
+    )
+    updated_result = filter_metadata_columns(updated_result)
+    input_df = updated_result.dataframe
+
     session: snowpark.Session = get_or_create_snowpark_session()
 
+    # Check for partition hint early to determine precedence over single option
+    partition_hint = (
+        result.partition_hint if hasattr(result, "partition_hint") else None
+    )
+
     # Snowflake saveAsTable doesn't support format
     if (
         write_op.HasField("table")
@@ -160,8 +180,11 @@ def map_write(request: proto_base.ExecutePlanRequest):
             # Generate Spark-compatible filename with proper extension
             extension = write_op.source if write_op.source != "text" else "txt"
 
-            # Get compression from options for proper filename generation
-            compression_option = write_op.options.get("compression", "none")
+            compression = get_compression_for_source_and_options(
+                write_op.source, write_op.options, from_read=False
+            )
+            if compression is not None:
+                write_op.options["compression"] = compression
 
             # Generate Spark-compatible filename or prefix
             # we need a random prefix to support "append" mode
@@ -187,12 +210,12 @@ def map_write(request: proto_base.ExecutePlanRequest):
                 except Exception as e:
                     logger.warning(f"Could not clear directory {write_path}: {e}")
 
-            if should_write_to_single_file:
+            if should_write_to_single_file and partition_hint is None:
                 # Single file: generate complete filename with extension
                 spark_filename = generate_spark_compatible_filename(
                     task_id=0,
                     attempt_number=0,
-                    compression=compression_option,
+                    compression=compression,
                     format_ext=extension,
                 )
                 temp_file_prefix_on_stage = f"{write_path}/{spark_filename}"
@@ -201,29 +224,11 @@ def map_write(request: proto_base.ExecutePlanRequest):
                 spark_filename_prefix = generate_spark_compatible_filename(
                     task_id=0,
                     attempt_number=0,
-                    compression=compression_option,
+                    compression=None,
                     format_ext="",  # No extension for prefix
                 )
                 temp_file_prefix_on_stage = f"{write_path}/{spark_filename_prefix}"
 
-            default_compression = "NONE" if write_op.source != "parquet" else "snappy"
-            compression = write_op.options.get(
-                "compression", default_compression
-            ).upper()
-
-            if not is_supported_compression(write_op.source, compression):
-                supported_compressions = supported_compressions_for_format(
-                    write_op.source
-                )
-                raise AnalysisException(
-                    f"Compression {compression} is not supported for {write_op.source} format. "
-                    + (
-                        f"Supported compressions: {sorted(supported_compressions)}"
-                        if supported_compressions
-                        else "No compression supported for this format."
-                    )
-                )
-
             parameters = {
                 "location": temp_file_prefix_on_stage,
                 "file_format_type": write_op.source
@@ -238,9 +243,6 @@ def map_write(request: proto_base.ExecutePlanRequest):
             # Using the base avoids coupling to exact filenames/prefixes.
             download_stage_path = write_path
 
-            # Check for partition hint early to determine precedence over single option
-            partition_hint = result.partition_hint
-
             # Apply max_file_size for both single and multi-file scenarios
             # This helps control when Snowflake splits files into multiple parts
             if max_file_size:
@@ -298,7 +300,7 @@ def map_write(request: proto_base.ExecutePlanRequest):
                     per_part_prefix = generate_spark_compatible_filename(
                         task_id=part_idx,
                         attempt_number=0,
-                        compression=compression_option,
+                        compression=None,
                         format_ext="",  # prefix only; Snowflake appends extension/counters
                     )
                     part_params["location"] = f"{write_path}/{per_part_prefix}"
@@ -537,6 +539,19 @@ def map_write_v2(request: proto_base.ExecutePlanRequest):
     snowpark_table_name = _spark_to_snowflake(write_op.table_name)
     result = map_relation(write_op.input)
     input_df: snowpark.DataFrame = handle_column_names(result, "table")
+
+    # Create updated container with transformed dataframe, then filter METADATA$FILENAME columns
+    # Update the container to use the transformed dataframe from handle_column_names
+    updated_result = DataFrameContainer(
+        dataframe=input_df,
+        column_map=result.column_map,
+        table_name=result.table_name,
+        alias=result.alias,
+        partition_hint=result.partition_hint,
+    )
+    updated_result = filter_metadata_columns(updated_result)
+    input_df = updated_result.dataframe
+
     session: snowpark.Session = get_or_create_snowpark_session()
 
     if write_op.table_name is None or write_op.table_name == "":

snowflake/snowpark_connect/server.py

@@ -232,12 +232,20 @@ class SnowflakeConnectServicer(proto_base_grpc.SparkConnectServiceServicer):
             match request.WhichOneof("analyze"):
                 case "schema":
                     result = map_relation(request.schema.plan.root)
-                    snowpark_df = result.dataframe
-                    snowpark_schema: snowpark.types.StructType = snowpark_df.schema
+
+                    from snowflake.snowpark_connect.relation.read.metadata_utils import (
+                        filter_metadata_columns,
+                    )
+
+                    filtered_result = filter_metadata_columns(result)
+                    filtered_df = filtered_result.dataframe
+
                     schema = proto_base.AnalyzePlanResponse.Schema(
                         schema=types_proto.DataType(
                             **snowpark_to_proto_type(
-                                snowpark_schema, result.column_map, snowpark_df
+                                filtered_df.schema,
+                                filtered_result.column_map,
+                                filtered_df,
                             )
                         )
                     )

snowflake/snowpark_connect/type_mapping.py

@@ -30,6 +30,10 @@ from snowflake.snowpark_connect.date_time_format_mapping import (
     convert_spark_format_to_snowflake,
 )
 from snowflake.snowpark_connect.expression.literal import get_literal_field_and_name
+from snowflake.snowpark_connect.expression.map_sql_expression import (
+    _INTERVAL_DAYTIME_PATTERN_RE,
+    _INTERVAL_YEARMONTH_PATTERN_RE,
+)
 from snowflake.snowpark_connect.utils.context import get_is_evaluating_sql
 from snowflake.snowpark_connect.utils.snowpark_connect_logging import logger
 from snowflake.snowpark_connect.utils.telemetry import (
@@ -274,6 +278,18 @@ def snowpark_to_proto_type(
         case snowpark.types.VariantType:
             # For now we are returning a string type for variant types.
             return {"string": types_proto.DataType.String()}
+        case snowpark.types.YearMonthIntervalType:
+            return {
+                "year_month_interval": types_proto.DataType.YearMonthInterval(
+                    start_field=data_type.start_field, end_field=data_type.end_field
+                )
+            }
+        case snowpark.types.DayTimeIntervalType:
+            return {
+                "day_time_interval": types_proto.DataType.DayTimeInterval(
+                    start_field=data_type.start_field, end_field=data_type.end_field
+                )
+            }
         case _:
             raise SnowparkConnectNotImplementedError(
                 f"Unsupported snowpark data type: {data_type}"
@@ -328,6 +344,24 @@ def cast_to_match_snowpark_type(
             return str(content)
         case snowpark.types.TimestampType:
             return str(content)
+        case snowpark.types.YearMonthIntervalType:
+            if isinstance(content, (int, float)):
+                total_months = int(content)
+                years = total_months // 12
+                months = total_months % 12
+                return f"INTERVAL '{years}-{months}' YEAR TO MONTH"
+            elif isinstance(content, str) and content.startswith(("+", "-")):
+                # Handle Snowflake's native interval format (e.g., "+11-08" or "-2-3")
+                # Convert to Spark's format: "INTERVAL 'Y-M' YEAR TO MONTH"
+                sign = content[0]
+                interval_part = content[1:]  # Remove sign
+                if sign == "-":
+                    return f"INTERVAL '-{interval_part}' YEAR TO MONTH"
+                else:
+                    return f"INTERVAL '{interval_part}' YEAR TO MONTH"
+            return str(content)
+        case snowpark.types.DayTimeIntervalType:
+            return str(content)
         case _:
             raise SnowparkConnectNotImplementedError(
                 f"Unsupported snowpark data type in casting: {data_type}"
@@ -411,6 +445,18 @@ def proto_to_snowpark_type(
             # For UDT types, return the underlying SQL type
             logger.debug("Returning underlying sql type for udt")
             return proto_to_snowpark_type(data_type.udt.sql_type)
+        case "year_month_interval":
+            # Preserve start_field and end_field from protobuf
+            return snowpark.types.YearMonthIntervalType(
+                start_field=data_type.year_month_interval.start_field,
+                end_field=data_type.year_month_interval.end_field,
+            )
+        case "day_time_interval":
+            # Preserve start_field and end_field from protobuf
+            return snowpark.types.DayTimeIntervalType(
+                start_field=data_type.day_time_interval.start_field,
+                end_field=data_type.day_time_interval.end_field,
+            )
         case _:
             return map_simple_types(data_type.WhichOneof("kind"))
 
@@ -523,6 +569,12 @@ def map_snowpark_types_to_pyarrow_types(
             return pa.timestamp(unit, tz=tz)
         case snowpark.types.VariantType:
             return pa.string()
+        case snowpark.types.YearMonthIntervalType:
+            # Return string type so formatted intervals are preserved in display
+            return pa.string()
+        case snowpark.types.DayTimeIntervalType:
+            # Return string type so formatted intervals are preserved in display
+            return pa.string()
         case _:
             raise SnowparkConnectNotImplementedError(
                 f"Unsupported snowpark data type: {snowpark_type}"
@@ -676,6 +728,14 @@ def map_pyspark_types_to_snowpark_types(
         return snowpark.types.TimestampType()
     if isinstance(type_to_map, pyspark.sql.types.TimestampNTZType):
         return snowpark.types.TimestampType(timezone=TimestampTimeZone.NTZ)
+    if isinstance(type_to_map, pyspark.sql.types.YearMonthIntervalType):
+        return snowpark.types.YearMonthIntervalType(
+            type_to_map.startField, type_to_map.endField
+        )
+    if isinstance(type_to_map, pyspark.sql.types.DayTimeIntervalType):
+        return snowpark.types.DayTimeIntervalType(
+            type_to_map.startField, type_to_map.endField
+        )
     raise SnowparkConnectNotImplementedError(
         f"Unsupported spark data type: {type_to_map}"
     )
@@ -743,6 +803,14 @@ def map_snowpark_to_pyspark_types(
         if type_to_map.tz == snowpark.types.TimestampTimeZone.NTZ:
             return pyspark.sql.types.TimestampNTZType()
         return pyspark.sql.types.TimestampType()
+    if isinstance(type_to_map, snowpark.types.YearMonthIntervalType):
+        return pyspark.sql.types.YearMonthIntervalType(
+            type_to_map.start_field, type_to_map.end_field
+        )
+    if isinstance(type_to_map, snowpark.types.DayTimeIntervalType):
+        return pyspark.sql.types.DayTimeIntervalType(
+            type_to_map.start_field, type_to_map.end_field
+        )
     raise SnowparkConnectNotImplementedError(f"Unsupported data type: {type_to_map}")
 
 
@@ -785,10 +853,14 @@ def map_simple_types(simple_type: str) -> snowpark.types.DataType:
             return snowpark.types.TimestampType(snowpark.types.TimestampTimeZone.NTZ)
         case "timestamp_ltz":
             return snowpark.types.TimestampType(snowpark.types.TimestampTimeZone.LTZ)
+        case "year_month_interval":
+            return snowpark.types.YearMonthIntervalType()
         case "day_time_interval":
-            # this is not a column type in snowflake so there won't be a dataframe column
-            # with this, for now this type won't make any sense
-            return snowpark.types.StringType()
+            return snowpark.types.DayTimeIntervalType()
+        case type_name if _INTERVAL_YEARMONTH_PATTERN_RE.match(type_name):
+            return snowpark.types.YearMonthIntervalType()
+        case type_name if _INTERVAL_DAYTIME_PATTERN_RE.match(type_name):
+            return snowpark.types.DayTimeIntervalType()
         case _:
             if simple_type.startswith("decimal"):
                 precision = int(simple_type.split("(")[1].split(",")[0])

snowflake/snowpark_connect/utils/describe_query_cache.py

@@ -16,7 +16,6 @@ from snowflake.snowpark_connect.utils.concurrent import SynchronizedDict
 from snowflake.snowpark_connect.utils.snowpark_connect_logging import logger
 from snowflake.snowpark_connect.utils.telemetry import telemetry
 
-DESCRIBE_CACHE_TTL_SECONDS = 15
 USE_DESCRIBE_QUERY_CACHE = True
 
 DDL_DETECTION_PATTERN = re.compile(r"\s*(CREATE|ALTER|DROP)\b", re.IGNORECASE)
@@ -51,6 +50,8 @@ class DescribeQueryCache:
         return sql_query
 
     def get(self, sql_query: str) -> list[ResultMetadataV2] | None:
+        from snowflake.snowpark_connect.config import get_describe_cache_ttl_seconds
+
         telemetry.report_describe_query_cache_lookup()
 
         cache_key = self._get_cache_key(sql_query)
@@ -59,7 +60,9 @@ class DescribeQueryCache:
 
         if key in self._cache:
             result, timestamp = self._cache[key]
-            if current_time < timestamp + DESCRIBE_CACHE_TTL_SECONDS:
+
+            expired_by = current_time - (timestamp + get_describe_cache_ttl_seconds())
+            if expired_by < 0:
                 logger.debug(
                     f"Returning query result from cache for query: {sql_query[:20]}"
                 )
@@ -92,7 +95,7 @@ class DescribeQueryCache:
                 telemetry.report_describe_query_cache_hit()
                 return result
             else:
-                telemetry.report_describe_query_cache_expired()
+                telemetry.report_describe_query_cache_expired(expired_by)
                 del self._cache[key]
         return None