apache-airflow-providers-snowflake 6.4.0rc1__py3-none-any.whl → 6.5.0__py3-none-any.whl

airflow/providers/snowflake/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "6.4.0"
+__version__ = "6.5.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.10.0"

airflow/providers/snowflake/decorators/snowpark.py

@@ -17,8 +17,8 @@
 
 from __future__ import annotations
 
-from collections.abc import Sequence
-from typing import TYPE_CHECKING, Callable
+from collections.abc import Callable, Sequence
+from typing import TYPE_CHECKING
 
 from airflow.providers.snowflake.version_compat import AIRFLOW_V_3_0_PLUS
 

airflow/providers/snowflake/hooks/snowflake.py

@@ -19,12 +19,12 @@ from __future__ import annotations
 
 import base64
 import os
-from collections.abc import Iterable, Mapping
+from collections.abc import Callable, Iterable, Mapping
 from contextlib import closing, contextmanager
 from functools import cached_property
 from io import StringIO
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, TypeVar, overload
+from typing import TYPE_CHECKING, Any, TypeVar, overload
 from urllib.parse import urlparse
 
 import requests
@@ -617,10 +617,9 @@ class SnowflakeHook(DbApiHook):
 
     def get_openlineage_database_specific_lineage(self, task_instance) -> OperatorLineage | None:
         """
-        Generate OpenLineage metadata for a Snowflake task instance based on executed query IDs.
+        Emit separate OpenLineage events for each Snowflake query, based on executed query IDs.
 
-        If a single query ID is present, attach an `ExternalQueryRunFacet` to the lineage metadata.
-        If multiple query IDs are present, emits separate OpenLineage events for each query.
+        If a single query ID is present, also add an `ExternalQueryRunFacet` to the returned lineage metadata.
 
         Note that `get_openlineage_database_specific_lineage` is usually called after task's execution,
         so if multiple query IDs are present, both START and COMPLETE event for each query will be emitted
@@ -641,13 +640,22 @@ class SnowflakeHook(DbApiHook):
         )
 
         if not self.query_ids:
-            self.log.debug("openlineage: no snowflake query ids found.")
+            self.log.info("OpenLineage could not find snowflake query ids.")
             return None
 
         self.log.debug("openlineage: getting connection to get database info")
         connection = self.get_connection(self.get_conn_id())
         namespace = SQLParser.create_namespace(self.get_openlineage_database_info(connection))
 
+        self.log.info("Separate OpenLineage events will be emitted for each query_id.")
+        emit_openlineage_events_for_snowflake_queries(
+            task_instance=task_instance,
+            hook=self,
+            query_ids=self.query_ids,
+            query_for_extra_metadata=True,
+            query_source_namespace=namespace,
+        )
+
         if len(self.query_ids) == 1:
             self.log.debug("Attaching ExternalQueryRunFacet with single query_id to OpenLineage event.")
             return OperatorLineage(
@@ -658,20 +666,4 @@ class SnowflakeHook(DbApiHook):
                 }
             )
 
-        self.log.info("Multiple query_ids found. Separate OpenLineage event will be emitted for each query.")
-        try:
-            from airflow.providers.openlineage.utils.utils import should_use_external_connection
-
-            use_external_connection = should_use_external_connection(self)
-        except ImportError:
-            # OpenLineage provider release < 1.8.0 - we always use connection
-            use_external_connection = True
-
-        emit_openlineage_events_for_snowflake_queries(
-            query_ids=self.query_ids,
-            query_source_namespace=namespace,
-            task_instance=task_instance,
-            hook=self if use_external_connection else None,
-        )
-
         return None

airflow/providers/snowflake/hooks/snowflake_sql_api.py

@@ -17,6 +17,7 @@
 from __future__ import annotations
 
 import base64
+import time
 import uuid
 import warnings
 from datetime import timedelta
@@ -25,8 +26,18 @@ from typing import Any
 
 import aiohttp
 import requests
+from aiohttp import ClientConnectionError, ClientResponseError
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
+from requests.exceptions import ConnectionError, HTTPError, Timeout
+from tenacity import (
+    AsyncRetrying,
+    Retrying,
+    before_sleep_log,
+    retry_if_exception,
+    stop_after_attempt,
+    wait_exponential,
+)
 
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
@@ -65,6 +76,7 @@ class SnowflakeSqlApiHook(SnowflakeHook):
     :param token_life_time: lifetime of the JWT Token in timedelta
     :param token_renewal_delta: Renewal time of the JWT Token in timedelta
     :param deferrable: Run operator in the deferrable mode.
+    :param api_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` & ``tenacity.AsyncRetrying`` classes.
     """
 
     LIFETIME = timedelta(minutes=59)  # The tokens will have a 59 minute lifetime
@@ -75,15 +87,27 @@ class SnowflakeSqlApiHook(SnowflakeHook):
         snowflake_conn_id: str,
         token_life_time: timedelta = LIFETIME,
         token_renewal_delta: timedelta = RENEWAL_DELTA,
+        api_retry_args: dict[Any, Any] | None = None,  # Optional retry arguments passed to tenacity.retry
         *args: Any,
         **kwargs: Any,
     ):
         self.snowflake_conn_id = snowflake_conn_id
         self.token_life_time = token_life_time
         self.token_renewal_delta = token_renewal_delta
+
         super().__init__(snowflake_conn_id, *args, **kwargs)
         self.private_key: Any = None
 
+        self.retry_config = {
+            "retry": retry_if_exception(self._should_retry_on_error),
+            "wait": wait_exponential(multiplier=1, min=1, max=60),
+            "stop": stop_after_attempt(5),
+            "before_sleep": before_sleep_log(self.log, log_level=20),  # INFO level
+            "reraise": True,
+        }
+        if api_retry_args:
+            self.retry_config.update(api_retry_args)
+
     def get_private_key(self) -> None:
         """Get the private key from snowflake connection."""
         conn = self.get_connection(self.snowflake_conn_id)
@@ -168,13 +192,8 @@ class SnowflakeSqlApiHook(SnowflakeHook):
                 "query_tag": query_tag,
             },
         }
-        response = requests.post(url, json=data, headers=headers, params=params)
-        try:
-            response.raise_for_status()
-        except requests.exceptions.HTTPError as e:  # pragma: no cover
-            msg = f"Response: {e.response.content.decode()} Status Code: {e.response.status_code}"
-            raise AirflowException(msg)
-        json_response = response.json()
+
+        _, json_response = self._make_api_call_with_retries("POST", url, headers, params, data)
         self.log.info("Snowflake SQL POST API response: %s", json_response)
         if "statementHandles" in json_response:
             self.query_ids = json_response["statementHandles"]
@@ -259,13 +278,10 @@ class SnowflakeSqlApiHook(SnowflakeHook):
         """
         for query_id in query_ids:
             header, params, url = self.get_request_url_header_params(query_id)
-            try:
-                response = requests.get(url, headers=header, params=params)
-                response.raise_for_status()
-                self.log.info(response.json())
-            except requests.exceptions.HTTPError as e:
-                msg = f"Response: {e.response.content.decode()}, Status Code: {e.response.status_code}"
-                raise AirflowException(msg)
+            _, response_json = self._make_api_call_with_retries(
+                method="GET", url=url, headers=header, params=params
+            )
+            self.log.info(response_json)
 
     def _process_response(self, status_code, resp):
         self.log.info("Snowflake SQL GET statements status API response: %s", resp)
@@ -295,11 +311,83 @@ class SnowflakeSqlApiHook(SnowflakeHook):
         """
         self.log.info("Retrieving status for query id %s", query_id)
         header, params, url = self.get_request_url_header_params(query_id)
-        response = requests.get(url, params=params, headers=header)
-        status_code = response.status_code
-        resp = response.json()
+        status_code, resp = self._make_api_call_with_retries("GET", url, header, params)
         return self._process_response(status_code, resp)
 
+    def wait_for_query(
+        self, query_id: str, raise_error: bool = False, poll_interval: int = 5, timeout: int = 60
+    ) -> dict[str, str | list[str]]:
+        """
+        Wait for query to finish either successfully or with error.
+
+        :param query_id: statement handle id for the individual statement.
+        :param raise_error: whether to raise an error if the query failed.
+        :param poll_interval: time (in seconds) between checking the query status.
+        :param timeout: max time (in seconds) to wait for the query to finish before raising a TimeoutError.
+
+        :raises RuntimeError: If the query status is 'error' and `raise_error` is True.
+        :raises TimeoutError: If the query doesn't finish within the specified timeout.
+        """
+        start_time = time.time()
+
+        while True:
+            response = self.get_sql_api_query_status(query_id=query_id)
+            self.log.debug("Query status `%s`", response["status"])
+
+            if time.time() - start_time > timeout:
+                raise TimeoutError(
+                    f"Query `{query_id}` did not finish within the timeout period of {timeout} seconds."
+                )
+
+            if response["status"] != "running":
+                self.log.info("Query status `%s`", response["status"])
+                break
+
+            time.sleep(poll_interval)
+
+        if response["status"] == "error" and raise_error:
+            raise RuntimeError(response["message"])
+
+        return response
+
+    def get_result_from_successful_sql_api_query(self, query_id: str) -> list[dict[str, Any]]:
+        """
+        Based on the query id HTTP requests are made to snowflake SQL API and return result data.
+
+        :param query_id: statement handle id for the individual statement.
+
+        :raises RuntimeError: If the query status is not 'success'.
+        """
+        self.log.info("Retrieving data for query id %s", query_id)
+        header, params, url = self.get_request_url_header_params(query_id)
+        status_code, response = self._make_api_call_with_retries("GET", url, header, params)
+
+        if (query_status := self._process_response(status_code, response)["status"]) != "success":
+            msg = f"Query must have status `success` to retrieve data; got `{query_status}`."
+            raise RuntimeError(msg)
+
+        # Below fields should always be present in response, but added some safety checks
+        data = response.get("data", [])
+        if not data:
+            self.log.warning("No data found in the API response.")
+            return []
+        metadata = response.get("resultSetMetaData", {})
+        col_names = [row["name"] for row in metadata.get("rowType", [])]
+        if not col_names:
+            self.log.warning("No column metadata found in the API response.")
+            return []
+
+        num_partitions = len(metadata.get("partitionInfo", []))
+        if num_partitions > 1:
+            self.log.debug("Result data is returned as multiple partitions. Will perform additional queries.")
+            url += "?partition="
+            for partition_no in range(1, num_partitions):  # First partition was already returned
+                self.log.debug("Querying for partition no. %s", partition_no)
+                _, response = self._make_api_call_with_retries("GET", url + str(partition_no), header, params)
+                data.extend(response.get("data", []))
+
+        return [dict(zip(col_names, row)) for row in data]  # Merged column names with data
+
     async def get_sql_api_query_status_async(self, query_id: str) -> dict[str, str | list[str]]:
         """
         Based on the query id async HTTP request is made to snowflake SQL API and return response.
@@ -308,10 +396,81 @@ class SnowflakeSqlApiHook(SnowflakeHook):
         """
         self.log.info("Retrieving status for query id %s", query_id)
         header, params, url = self.get_request_url_header_params(query_id)
-        async with (
-            aiohttp.ClientSession(headers=header) as session,
-            session.get(url, params=params) as response,
+        status_code, resp = await self._make_api_call_with_retries_async("GET", url, header, params)
+        return self._process_response(status_code, resp)
+
+    @staticmethod
+    def _should_retry_on_error(exception) -> bool:
+        """
+        Determine if the exception should trigger a retry based on error type and status code.
+
+        Retries on HTTP errors 429 (Too Many Requests), 503 (Service Unavailable),
+        and 504 (Gateway Timeout) as recommended by Snowflake error handling docs.
+        Retries on connection errors and timeouts.
+
+        :param exception: The exception to check
+        :return: True if the request should be retried, False otherwise
+        """
+        if isinstance(exception, HTTPError):
+            return exception.response.status_code in [429, 503, 504]
+        if isinstance(exception, ClientResponseError):
+            return exception.status in [429, 503, 504]
+        if isinstance(
+            exception,
+            ConnectionError | Timeout | ClientConnectionError,
         ):
-            status_code = response.status
-            resp = await response.json()
-            return self._process_response(status_code, resp)
+            return True
+        return False
+
+    def _make_api_call_with_retries(
+        self, method: str, url: str, headers: dict, params: dict | None = None, json: dict | None = None
+    ):
+        """
+        Make an API call to the Snowflake SQL API with retry logic for specific HTTP errors.
+
+        Error handling implemented based on Snowflake error handling docs:
+        https://docs.snowflake.com/en/developer-guide/sql-api/handling-errors
+
+        :param method: The HTTP method to use for the API call.
+        :param url: The URL for the API endpoint.
+        :param headers: The headers to include in the API call.
+        :param params: (Optional) The query parameters to include in the API call.
+        :param data: (Optional) The data to include in the API call.
+        :return: The response object from the API call.
+        """
+        with requests.Session() as session:
+            for attempt in Retrying(**self.retry_config):  # type: ignore
+                with attempt:
+                    if method.upper() in ("GET", "POST"):
+                        response = session.request(
+                            method=method.lower(), url=url, headers=headers, params=params, json=json
+                        )
+                    else:
+                        raise ValueError(f"Unsupported HTTP method: {method}")
+                    response.raise_for_status()
+                    return response.status_code, response.json()
+
+    async def _make_api_call_with_retries_async(self, method, url, headers, params=None):
+        """
+        Make an API call to the Snowflake SQL API asynchronously with retry logic for specific HTTP errors.
+
+        Error handling implemented based on Snowflake error handling docs:
+        https://docs.snowflake.com/en/developer-guide/sql-api/handling-errors
+
+        :param method: The HTTP method to use for the API call. Only GET is supported as  is synchronous.
+        :param url: The URL for the API endpoint.
+        :param headers: The headers to include in the API call.
+        :param params: (Optional) The query parameters to include in the API call.
+        :return: The response object from the API call.
+        """
+        async with aiohttp.ClientSession(headers=headers) as session:
+            async for attempt in AsyncRetrying(**self.retry_config):  # type: ignore
+                with attempt:
+                    if method.upper() == "GET":
+                        async with session.request(method=method.lower(), url=url, params=params) as response:
+                            response.raise_for_status()
+                            # Return status and json content for async processing
+                            content = await response.json()
+                            return response.status, content
+                    else:
+                        raise ValueError(f"Unsupported HTTP method: {method}")

airflow/providers/snowflake/operators/snowflake.py

@@ -355,6 +355,7 @@ class SnowflakeSqlApiOperator(SQLExecuteQueryOperator):
             When executing the statement, Snowflake replaces placeholders (? and :name) in
             the statement with these specified values.
     :param deferrable: Run operator in the deferrable mode.
+    :param snowflake_api_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` & ``tenacity.AsyncRetrying`` classes.
     """
 
     LIFETIME = timedelta(minutes=59)  # The tokens will have a 59 minutes lifetime
@@ -381,6 +382,7 @@ class SnowflakeSqlApiOperator(SQLExecuteQueryOperator):
         token_renewal_delta: timedelta = RENEWAL_DELTA,
         bindings: dict[str, Any] | None = None,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
+        snowflake_api_retry_args: dict[str, Any] | None = None,
         **kwargs: Any,
     ) -> None:
         self.snowflake_conn_id = snowflake_conn_id
@@ -390,6 +392,7 @@ class SnowflakeSqlApiOperator(SQLExecuteQueryOperator):
         self.token_renewal_delta = token_renewal_delta
         self.bindings = bindings
         self.execute_async = False
+        self.snowflake_api_retry_args = snowflake_api_retry_args or {}
         self.deferrable = deferrable
         self.query_ids: list[str] = []
         if any([warehouse, database, role, schema, authenticator, session_parameters]):  # pragma: no cover
@@ -412,6 +415,7 @@ class SnowflakeSqlApiOperator(SQLExecuteQueryOperator):
             token_life_time=self.token_life_time,
             token_renewal_delta=self.token_renewal_delta,
             deferrable=self.deferrable,
+            api_retry_args=self.snowflake_api_retry_args,
             **self.hook_params,
         )
 

airflow/providers/snowflake/operators/snowpark.py

@@ -17,8 +17,8 @@
 
 from __future__ import annotations
 
-from collections.abc import Collection, Mapping, Sequence
-from typing import Any, Callable
+from collections.abc import Callable, Collection, Mapping, Sequence
+from typing import Any
 
 from airflow.providers.common.compat.standard.operators import PythonOperator, get_current_context
 from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook

airflow/providers/snowflake/transfers/copy_into_snowflake.py

@@ -22,9 +22,9 @@ from __future__ import annotations
 from collections.abc import Sequence
 from typing import Any
 
-from airflow.models import BaseOperator
 from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
 from airflow.providers.snowflake.utils.common import enclose_param
+from airflow.providers.snowflake.version_compat import BaseOperator
 
 
 def _validate_parameter(param_name: str, value: str | None) -> str | None:

airflow/providers/snowflake/utils/openlineage.py

@@ -19,7 +19,7 @@ from __future__ import annotations
 import datetime
 import logging
 from contextlib import closing
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 from urllib.parse import quote, urlparse, urlunparse
 
 from airflow.providers.common.compat.openlineage.check import require_openlineage_version
@@ -31,6 +31,7 @@ if TYPE_CHECKING:
     from openlineage.client.facet_v2 import JobFacet
 
     from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
+    from airflow.providers.snowflake.hooks.snowflake_sql_api import SnowflakeSqlApiHook
 
 
 log = logging.getLogger(__name__)
@@ -204,9 +205,29 @@ def _run_single_query_with_hook(hook: SnowflakeHook, sql: str) -> list[dict]:
     return result
 
 
+def _run_single_query_with_api_hook(hook: SnowflakeSqlApiHook, sql: str) -> list[dict[str, Any]]:
+    """Execute a query against Snowflake API without adding extra logging or instrumentation."""
+    # `hook.execute_query` resets the query_ids, so we need to save them and re-assign after we're done
+    query_ids_before_execution = list(hook.query_ids)
+    try:
+        _query_ids = hook.execute_query(sql=sql, statement_count=0)
+        hook.wait_for_query(query_id=_query_ids[0], raise_error=True, poll_interval=1, timeout=3)
+        return hook.get_result_from_successful_sql_api_query(query_id=_query_ids[0])
+    finally:
+        hook.query_ids = query_ids_before_execution
+
+
+def _process_data_from_api(data: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Convert 'START_TIME' and 'END_TIME' fields to UTC datetime objects."""
+    for row in data:
+        for key in ("START_TIME", "END_TIME"):
+            row[key] = datetime.datetime.fromtimestamp(float(row[key]), timezone.utc)
+    return data
+
+
 def _get_queries_details_from_snowflake(
-    hook: SnowflakeHook, query_ids: list[str]
-) -> dict[str, dict[str, str]]:
+    hook: SnowflakeHook | SnowflakeSqlApiHook, query_ids: list[str]
+) -> dict[str, dict[str, Any]]:
     """Retrieve execution details for specific queries from Snowflake's query history."""
     if not query_ids:
         return {}
@@ -221,7 +242,16 @@ def _get_queries_details_from_snowflake(
         f";"
     )
 
-    result = _run_single_query_with_hook(hook=hook, sql=query)
+    try:
+        # Can't import the SnowflakeSqlApiHook class and do proper isinstance check - circular imports
+        if hook.__class__.__name__ == "SnowflakeSqlApiHook":
+            result = _run_single_query_with_api_hook(hook=hook, sql=query)  # type: ignore[arg-type]
+            result = _process_data_from_api(data=result)
+        else:
+            result = _run_single_query_with_hook(hook=hook, sql=query)
+    except Exception as e:
+        log.warning("OpenLineage could not retrieve extra metadata from Snowflake. Error encountered: %s", e)
+        result = []
 
     return {row["QUERY_ID"]: row for row in result} if result else {}
 
@@ -259,17 +289,18 @@ def _create_snowflake_event_pair(
 
 @require_openlineage_version(provider_min_version="2.3.0")
 def emit_openlineage_events_for_snowflake_queries(
-    query_ids: list[str],
-    query_source_namespace: str,
     task_instance,
-    hook: SnowflakeHook | None = None,
+    hook: SnowflakeHook | SnowflakeSqlApiHook | None = None,
+    query_ids: list[str] | None = None,
+    query_source_namespace: str | None = None,
+    query_for_extra_metadata: bool = False,
     additional_run_facets: dict | None = None,
     additional_job_facets: dict | None = None,
 ) -> None:
     """
     Emit OpenLineage events for executed Snowflake queries.
 
-    Metadata retrieval from Snowflake is attempted only if a `SnowflakeHook` is provided.
+    Metadata retrieval from Snowflake is attempted only if `get_extra_metadata` is True and hook is provided.
     If metadata is available, execution details such as start time, end time, execution status,
     error messages, and SQL text are included in the events. If no metadata is found, the function
     defaults to using the Airflow task instance's state and the current timestamp.
@@ -279,10 +310,16 @@ def emit_openlineage_events_for_snowflake_queries(
     will correspond to actual query execution times.
 
     Args:
-        query_ids: A list of Snowflake query IDs to emit events for.
-        query_source_namespace: The namespace to be included in ExternalQueryRunFacet.
         task_instance: The Airflow task instance that run these queries.
-        hook: A SnowflakeHook instance used to retrieve query metadata if available.
+        hook: A supported Snowflake hook instance used to retrieve query metadata if available.
+        If omitted, `query_ids` and `query_source_namespace` must be provided explicitly and
+        `query_for_extra_metadata` must be `False`.
+        query_ids: A list of Snowflake query IDs to emit events for, can only be None if `hook` is provided
+        and `hook.query_ids` are present.
+        query_source_namespace: The namespace to be included in ExternalQueryRunFacet,
+        can be `None` only if hook is provided.
+        query_for_extra_metadata: Whether to query Snowflake for additional metadata about queries.
+        Must be `False` if `hook` is not provided.
         additional_run_facets: Additional run facets to include in OpenLineage events.
         additional_job_facets: Additional job facets to include in OpenLineage events.
     """
@@ -297,23 +334,49 @@ def emit_openlineage_events_for_snowflake_queries(
     from airflow.providers.openlineage.conf import namespace
     from airflow.providers.openlineage.plugins.listener import get_openlineage_listener
 
-    if not query_ids:
-        log.debug("No Snowflake query IDs provided; skipping OpenLineage event emission.")
-        return
-
-    query_ids = [q for q in query_ids]  # Make a copy to make sure it does not change
+    log.info("OpenLineage will emit events for Snowflake queries.")
 
     if hook:
+        if not query_ids:
+            log.debug("No Snowflake query IDs provided; Checking `hook.query_ids` property.")
+            query_ids = getattr(hook, "query_ids", [])
+            if not query_ids:
+                raise ValueError("No Snowflake query IDs provided and `hook.query_ids` are not present.")
+
+        if not query_source_namespace:
+            log.debug("No Snowflake query namespace provided; Creating one from scratch.")
+            from airflow.providers.openlineage.sqlparser import SQLParser
+
+            connection = hook.get_connection(hook.get_conn_id())
+            query_source_namespace = SQLParser.create_namespace(
+                hook.get_openlineage_database_info(connection)
+            )
+    else:
+        if not query_ids:
+            raise ValueError("If 'hook' is not provided, 'query_ids' must be set.")
+        if not query_source_namespace:
+            raise ValueError("If 'hook' is not provided, 'query_source_namespace' must be set.")
+        if query_for_extra_metadata:
+            raise ValueError("If 'hook' is not provided, 'query_for_extra_metadata' must be False.")
+
+    query_ids = [q for q in query_ids]  # Make a copy to make sure we do not change hook's attribute
+
+    if query_for_extra_metadata and hook:
         log.debug("Retrieving metadata for %s queries from Snowflake.", len(query_ids))
         snowflake_metadata = _get_queries_details_from_snowflake(hook, query_ids)
     else:
-        log.debug("SnowflakeHook not provided. No extra metadata fill be fetched from Snowflake.")
+        log.debug("`query_for_extra_metadata` is False. No extra metadata fill be fetched from Snowflake.")
         snowflake_metadata = {}
 
     # If real metadata is unavailable, we send events with eventTime=now
     default_event_time = timezone.utcnow()
     # If no query metadata is provided, we use task_instance's state when checking for success
-    default_state = task_instance.state.value if hasattr(task_instance, "state") else ""
+    # ti.state has no `value` attr (AF2) when task it's still running, in AF3 we get 'running', in that case
+    # assuming it's user call and query succeeded, so we replace it with success.
+    default_state = (
+        getattr(task_instance.state, "value", "running") if hasattr(task_instance, "state") else ""
+    )
+    default_state = "success" if default_state == "running" else default_state
 
     common_run_facets = {"parent": _get_parent_run_facet(task_instance)}
     common_job_facets: dict[str, JobFacet] = {

airflow/providers/snowflake/utils/snowpark.py

@@ -18,7 +18,8 @@
 from __future__ import annotations
 
 import inspect
-from typing import TYPE_CHECKING, Callable
+from collections.abc import Callable
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from snowflake.snowpark import Session

airflow/providers/snowflake/version_compat.py

@@ -33,3 +33,13 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
 
 
 AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
+
+if AIRFLOW_V_3_0_PLUS:
+    from airflow.sdk import BaseOperator
+else:
+    from airflow.models import BaseOperator  # type: ignore[no-redef]
+
+__all__ = [
+    "AIRFLOW_V_3_0_PLUS",
+    "BaseOperator",
+]

{apache_airflow_providers_snowflake-6.4.0rc1.dist-info → apache_airflow_providers_snowflake-6.5.0.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-snowflake
-Version: 6.4.0rc1
+Version: 6.5.0
 Summary: Provider package apache-airflow-providers-snowflake for Apache Airflow
 Keywords: airflow-provider,snowflake,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
 Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
-Requires-Python: ~=3.9
+Requires-Python: ~=3.10
 Description-Content-Type: text/x-rst
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -15,23 +15,24 @@ Classifier: Intended Audience :: System Administrators
 Classifier: Framework :: Apache Airflow
 Classifier: Framework :: Apache Airflow :: Provider
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.10.0rc1
-Requires-Dist: apache-airflow-providers-common-compat>=1.6.0rc1
-Requires-Dist: apache-airflow-providers-common-sql>=1.21.0rc1
-Requires-Dist: pandas>=2.1.2,<2.2
-Requires-Dist: pyarrow>=14.0.1
+Requires-Dist: apache-airflow>=2.10.0
+Requires-Dist: apache-airflow-providers-common-compat>=1.6.0
+Requires-Dist: apache-airflow-providers-common-sql>=1.21.0
+Requires-Dist: pandas>=2.1.2; python_version <"3.13"
+Requires-Dist: pandas>=2.2.3; python_version >="3.13"
+Requires-Dist: pyarrow>=16.1.0
 Requires-Dist: snowflake-connector-python>=3.7.1
 Requires-Dist: snowflake-sqlalchemy>=1.4.0
 Requires-Dist: snowflake-snowpark-python>=1.17.0;python_version<'3.12'
-Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1 ; extra == "openlineage"
+Requires-Dist: snowflake-snowpark-python>=1.27.0;python_version>='3.12'
+Requires-Dist: apache-airflow-providers-openlineage>=2.3.0 ; extra == "openlineage"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-snowflake/6.4.0/changelog.html
-Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-snowflake/6.4.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.5.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.5.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -63,7 +64,7 @@ Provides-Extra: openlineage
 
 Package ``apache-airflow-providers-snowflake``
 
-Release: ``6.4.0``
+Release: ``6.5.0``
 
 
 `Snowflake <https://www.snowflake.com/>`__
@@ -76,7 +77,7 @@ This is a provider package for ``snowflake`` provider. All classes for this prov
 are in ``airflow.providers.snowflake`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.4.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.5.0/>`_.
 
 Installation
 ------------
@@ -85,23 +86,25 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-snowflake``
 
-The package supports the following python versions: 3.9,3.10,3.11,3.12
+The package supports the following python versions: 3.10,3.11,3.12
 
 Requirements
 ------------
 
-==========================================  =====================================
+==========================================  ======================================
 PIP package                                 Version required
-==========================================  =====================================
+==========================================  ======================================
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-compat``  ``>=1.6.0``
 ``apache-airflow-providers-common-sql``     ``>=1.21.0``
-``pandas``                                  ``>=2.1.2,<2.2``
-``pyarrow``                                 ``>=14.0.1``
+``pandas``                                  ``>=2.1.2; python_version < "3.13"``
+``pandas``                                  ``>=2.2.3; python_version >= "3.13"``
+``pyarrow``                                 ``>=16.1.0``
 ``snowflake-connector-python``              ``>=3.7.1``
 ``snowflake-sqlalchemy``                    ``>=1.4.0``
 ``snowflake-snowpark-python``               ``>=1.17.0; python_version < "3.12"``
-==========================================  =====================================
+``snowflake-snowpark-python``               ``>=1.27.0; python_version >= "3.12"``
+==========================================  ======================================
 
 Cross provider package dependencies
 -----------------------------------
@@ -125,5 +128,5 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.4.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.5.0/changelog.html>`_.
 

apache_airflow_providers_snowflake-6.5.0.dist-info/RECORD

@@ -0,0 +1,25 @@
+airflow/providers/snowflake/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
+airflow/providers/snowflake/__init__.py,sha256=adBe6ltW-rLtYQFNU8hpf6foAV8Pe60l_xFgAkGQCBg,1498
+airflow/providers/snowflake/get_provider_info.py,sha256=NdNRMfulBbpD-I4yFRr8U533m9djD18ijEMvuxOp4_g,3875
+airflow/providers/snowflake/version_compat.py,sha256=IlaClFJYi0uPxuC8cJt0Ro3Kl3tjIGc31ALoKDQbw5Q,1738
+airflow/providers/snowflake/decorators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/snowflake/decorators/snowpark.py,sha256=5ocPY8wrXvKbZJokefV4HDfX0WXzrHmcekXoZjkfHEw,5523
+airflow/providers/snowflake/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/snowflake/hooks/snowflake.py,sha256=RUWO9j_gd26BbktGzavxvgihCbOaBTNKlxyDvTxIBig,28502
+airflow/providers/snowflake/hooks/snowflake_sql_api.py,sha256=bCY3lSar-k3XjiDnrrWVRWTRunJL6U0Kss7fCLxFqTM,22287
+airflow/providers/snowflake/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/snowflake/operators/snowflake.py,sha256=Xfz_bv1Y0M8IWv67dOXcupeYWYlG91kPVgCe_vEqntY,23253
+airflow/providers/snowflake/operators/snowpark.py,sha256=Tfd31My6arGXKo0yfi46HyVfkHO3yeT085l3ymxtGpk,5815
+airflow/providers/snowflake/transfers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/snowflake/transfers/copy_into_snowflake.py,sha256=2WQDhD9U1l38ZoIv7FImsV6S3gT_rSisg_isNi4k08E,13618
+airflow/providers/snowflake/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/snowflake/triggers/snowflake_trigger.py,sha256=QXNLijmtZI7NIdPtOwbvS-4ohgrm8RV_jaBKvekosHQ,4051
+airflow/providers/snowflake/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/snowflake/utils/common.py,sha256=DG-KLy2KpZWAqZqm_XIECm8lmdoUlzwkXv9onmkQThc,1644
+airflow/providers/snowflake/utils/openlineage.py,sha256=5qpLU7D9EFolXGWEKI90py45iU0OPNyUEim7_Y7a2yw,18686
+airflow/providers/snowflake/utils/snowpark.py,sha256=-S6ltYiW-KooqUMGzY0OebmAzpUAu7GIjFWwuYERuk8,1629
+airflow/providers/snowflake/utils/sql_api_generate_jwt.py,sha256=9mR-vHIquv60tfAni87f6FAjKsiRHUDDrsVhzw4M9vM,6762
+apache_airflow_providers_snowflake-6.5.0.dist-info/entry_points.txt,sha256=bCrl5J1PXUMzbgnrKYho61rkbL2gHRT4I6f_1jlxAX4,105
+apache_airflow_providers_snowflake-6.5.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+apache_airflow_providers_snowflake-6.5.0.dist-info/METADATA,sha256=QLJIMF-ReENIcH9744z8e8TcXxny65U6XTPiFx_17Gc,6493
+apache_airflow_providers_snowflake-6.5.0.dist-info/RECORD,,

apache_airflow_providers_snowflake-6.4.0rc1.dist-info/RECORD

@@ -1,25 +0,0 @@
-airflow/providers/snowflake/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/snowflake/__init__.py,sha256=hfjXA59cpm_yb00p5Y7jNTIBCe3BPNoYYXkAVAcF_c4,1498
-airflow/providers/snowflake/get_provider_info.py,sha256=NdNRMfulBbpD-I4yFRr8U533m9djD18ijEMvuxOp4_g,3875
-airflow/providers/snowflake/version_compat.py,sha256=j5PCtXvZ71aBjixu-EFTNtVDPsngzzs7os0ZQDgFVDk,1536
-airflow/providers/snowflake/decorators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/snowflake/decorators/snowpark.py,sha256=tKXOjP8m8SEIu0jx2KSrd0n3jGMaIKDOwG2lMkvk3cI,5523
-airflow/providers/snowflake/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/snowflake/hooks/snowflake.py,sha256=t-sukzbQ1OYMeyZBDrJ9s9DuJbnDZFJKhBMZn_mQLgY,28933
-airflow/providers/snowflake/hooks/snowflake_sql_api.py,sha256=mREZ0nHc6L-9YSHZARYpgqrETqzVJ3Q6EfbWtEy5TV4,14745
-airflow/providers/snowflake/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/snowflake/operators/snowflake.py,sha256=Abu0MVsUPwVxfDVNYn5OtwVoUOhQanUp-YSFtLdcn6c,22915
-airflow/providers/snowflake/operators/snowpark.py,sha256=Wt3wzcsja0ed4q2KE9WyL74XH6mUVSPNZvcCHWEHQtc,5815
-airflow/providers/snowflake/transfers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/snowflake/transfers/copy_into_snowflake.py,sha256=bXmqkNwthJqUo65DsI_pC3mwk_V_Iikwi646oRCyWus,13590
-airflow/providers/snowflake/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/snowflake/triggers/snowflake_trigger.py,sha256=QXNLijmtZI7NIdPtOwbvS-4ohgrm8RV_jaBKvekosHQ,4051
-airflow/providers/snowflake/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/snowflake/utils/common.py,sha256=DG-KLy2KpZWAqZqm_XIECm8lmdoUlzwkXv9onmkQThc,1644
-airflow/providers/snowflake/utils/openlineage.py,sha256=dr57b0fidPo7A451UE6s6d3PnOf4dOv3iM5aoyE4oBI,15067
-airflow/providers/snowflake/utils/snowpark.py,sha256=9kzWRkdgoNQ8f3Wnr92LdZylMpcpRasxefpOXrM30Cw,1602
-airflow/providers/snowflake/utils/sql_api_generate_jwt.py,sha256=9mR-vHIquv60tfAni87f6FAjKsiRHUDDrsVhzw4M9vM,6762
-apache_airflow_providers_snowflake-6.4.0rc1.dist-info/entry_points.txt,sha256=bCrl5J1PXUMzbgnrKYho61rkbL2gHRT4I6f_1jlxAX4,105
-apache_airflow_providers_snowflake-6.4.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-apache_airflow_providers_snowflake-6.4.0rc1.dist-info/METADATA,sha256=mDqBewdnQi0Pa9w3DNrTvh2i0csPs95pMlAsZDp1kX8,6242
-apache_airflow_providers_snowflake-6.4.0rc1.dist-info/RECORD,,