apache-airflow-providers-amazon 9.0.0rc1__tar.gz → 9.1.0__tar.gz

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: apache-airflow-providers-amazon
-Version: 9.0.0rc1
+Version: 9.1.0
 Summary: Provider package apache-airflow-providers-amazon for Apache Airflow
 Keywords: airflow-provider,amazon,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -21,10 +21,10 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: PyAthena>=3.0.10
-Requires-Dist: apache-airflow-providers-common-compat>=1.1.0rc0
-Requires-Dist: apache-airflow-providers-common-sql>=1.3.1rc0
+Requires-Dist: apache-airflow-providers-common-compat>=1.2.1
+Requires-Dist: apache-airflow-providers-common-sql>=1.20.0
 Requires-Dist: apache-airflow-providers-http
-Requires-Dist: apache-airflow>=2.8.0rc0
+Requires-Dist: apache-airflow>=2.8.0
 Requires-Dist: asgiref>=2.3.0
 Requires-Dist: boto3>=1.34.90
 Requires-Dist: botocore>=1.34.90
@@ -33,19 +33,18 @@ Requires-Dist: jmespath>=0.7.0
 Requires-Dist: jsonpath_ng>=1.5.3
 Requires-Dist: python3-saml>=1.16.0
 Requires-Dist: redshift_connector>=2.0.918
-Requires-Dist: sqlalchemy_redshift>=0.8.6
 Requires-Dist: watchtower>=3.0.0,!=3.3.0,<4
 Requires-Dist: aiobotocore[boto3]>=2.13.0 ; extra == "aiobotocore"
-Requires-Dist: apache-airflow-providers-apache-hive ; extra == "apache.hive"
-Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0rc0 ; extra == "cncf.kubernetes"
-Requires-Dist: apache-airflow-providers-common-compat ; extra == "common.compat"
-Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
+Requires-Dist: apache-airflow-providers-apache-hive ; extra == "apache-hive"
+Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0 ; extra == "cncf-kubernetes"
+Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
+Requires-Dist: apache-airflow-providers-common-sql ; extra == "common-sql"
 Requires-Dist: apache-airflow-providers-exasol ; extra == "exasol"
 Requires-Dist: apache-airflow-providers-ftp ; extra == "ftp"
 Requires-Dist: apache-airflow-providers-google ; extra == "google"
 Requires-Dist: apache-airflow-providers-http ; extra == "http"
 Requires-Dist: apache-airflow-providers-imap ; extra == "imap"
-Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft.azure"
+Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft-azure"
 Requires-Dist: apache-airflow-providers-mongo ; extra == "mongo"
 Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
 Requires-Dist: pandas>=2.1.2,<2.2 ; extra == "pandas" and (python_version>="3.9")
@@ -55,23 +54,23 @@ Requires-Dist: s3fs>=2023.10.0 ; extra == "s3fs"
 Requires-Dist: apache-airflow-providers-salesforce ; extra == "salesforce"
 Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.0.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.0.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.1.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.1.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 Provides-Extra: aiobotocore
-Provides-Extra: apache.hive
-Provides-Extra: cncf.kubernetes
-Provides-Extra: common.compat
-Provides-Extra: common.sql
+Provides-Extra: apache-hive
+Provides-Extra: cncf-kubernetes
+Provides-Extra: common-compat
+Provides-Extra: common-sql
 Provides-Extra: exasol
 Provides-Extra: ftp
 Provides-Extra: google
 Provides-Extra: http
 Provides-Extra: imap
-Provides-Extra: microsoft.azure
+Provides-Extra: microsoft-azure
 Provides-Extra: mongo
 Provides-Extra: openlineage
 Provides-Extra: pandas
@@ -124,7 +123,7 @@ Provides-Extra: ssh
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``9.0.0.rc1``
+Release: ``9.1.0``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -137,7 +136,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.0.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.1.0/>`_.
 
 Installation
 ------------
@@ -155,8 +154,8 @@ Requirements
 PIP package                                 Version required
 ==========================================  ======================
 ``apache-airflow``                          ``>=2.8.0``
-``apache-airflow-providers-common-compat``  ``>=1.1.0``
-``apache-airflow-providers-common-sql``     ``>=1.3.1``
+``apache-airflow-providers-common-compat``  ``>=1.2.1``
+``apache-airflow-providers-common-sql``     ``>=1.20.0``
 ``apache-airflow-providers-http``
 ``boto3``                                   ``>=1.34.90``
 ``botocore``                                ``>=1.34.90``
@@ -164,7 +163,6 @@ PIP package                                 Version required
 ``watchtower``                              ``>=3.0.0,!=3.3.0,<4``
 ``jsonpath_ng``                             ``>=1.5.3``
 ``redshift_connector``                      ``>=2.0.918``
-``sqlalchemy_redshift``                     ``>=0.8.6``
 ``asgiref``                                 ``>=2.3.0``
 ``PyAthena``                                ``>=3.0.10``
 ``jmespath``                                ``>=0.7.0``
@@ -204,4 +202,4 @@ Dependent package
 ======================================================================================================================  ===================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.0.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.1.0/changelog.html>`_.

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``9.0.0.rc1``
+Release: ``9.1.0``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -55,7 +55,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.0.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.1.0/>`_.
 
 Installation
 ------------
@@ -73,8 +73,8 @@ Requirements
 PIP package                                 Version required
 ==========================================  ======================
 ``apache-airflow``                          ``>=2.8.0``
-``apache-airflow-providers-common-compat``  ``>=1.1.0``
-``apache-airflow-providers-common-sql``     ``>=1.3.1``
+``apache-airflow-providers-common-compat``  ``>=1.2.1``
+``apache-airflow-providers-common-sql``     ``>=1.20.0``
 ``apache-airflow-providers-http``
 ``boto3``                                   ``>=1.34.90``
 ``botocore``                                ``>=1.34.90``
@@ -82,7 +82,6 @@ PIP package                                 Version required
 ``watchtower``                              ``>=3.0.0,!=3.3.0,<4``
 ``jsonpath_ng``                             ``>=1.5.3``
 ``redshift_connector``                      ``>=2.0.918``
-``sqlalchemy_redshift``                     ``>=0.8.6``
 ``asgiref``                                 ``>=2.3.0``
 ``PyAthena``                                ``>=3.0.10``
 ``jmespath``                                ``>=0.7.0``
@@ -122,4 +121,4 @@ Dependent package
 ======================================================================================================================  ===================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.0.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.1.0/changelog.html>`_.

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/airflow/providers/amazon/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "9.0.0"
+__version__ = "9.1.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.8.0"

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/airflow/providers/amazon/aws/assets/s3.py

@@ -19,16 +19,14 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from airflow.providers.amazon.aws.hooks.s3 import S3Hook
-
-try:
-    from airflow.assets import Asset
-except ModuleNotFoundError:
-    from airflow.datasets import Dataset as Asset  # type: ignore[no-redef]
+from airflow.providers.common.compat.assets import Asset
 
 if TYPE_CHECKING:
     from urllib.parse import SplitResult
 
-    from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset
+    from airflow.providers.common.compat.openlineage.facet import (
+        Dataset as OpenLineageDataset,
+    )
 
 
 def create_asset(*, bucket: str, key: str, extra=None) -> Asset:
@@ -43,7 +41,9 @@ def sanitize_uri(uri: SplitResult) -> SplitResult:
 
 def convert_asset_to_openlineage(asset: Asset, lineage_context) -> OpenLineageDataset:
     """Translate Asset with valid AIP-60 uri to OpenLineage with assistance from the hook."""
-    from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset
+    from airflow.providers.common.compat.openlineage.facet import (
+        Dataset as OpenLineageDataset,
+    )
 
     bucket, key = S3Hook.parse_s3_url(asset.uri)
     return OpenLineageDataset(namespace=f"s3://{bucket}", name=key if key else "/")

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py

@@ -17,6 +17,7 @@
 from __future__ import annotations
 
 import argparse
+import warnings
 from collections import defaultdict
 from functools import cached_property
 from typing import TYPE_CHECKING, Container, Sequence, cast
@@ -24,7 +25,7 @@ from typing import TYPE_CHECKING, Container, Sequence, cast
 from flask import session, url_for
 
 from airflow.cli.cli_config import CLICommand, DefaultHelpParser, GroupCommand
-from airflow.exceptions import AirflowOptionalProviderFeatureException
+from airflow.exceptions import AirflowOptionalProviderFeatureException, AirflowProviderDeprecationWarning
 from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities
 from airflow.providers.amazon.aws.auth_manager.avp.facade import (
     AwsAuthManagerAmazonVerifiedPermissionsFacade,
@@ -166,6 +167,16 @@ class AwsAuthManager(BaseAuthManager):
             method=method, entity_type=AvpEntities.ASSET, user=user or self.get_user(), entity_id=asset_uri
         )
 
+    def is_authorized_dataset(
+        self, *, method: ResourceMethod, details: AssetDetails | None = None, user: BaseUser | None = None
+    ) -> bool:
+        warnings.warn(
+            "is_authorized_dataset will be renamed as is_authorized_asset in Airflow 3 and will be removed when the minimum Airflow version is set to 3.0 for the amazon provider",
+            AirflowProviderDeprecationWarning,
+            stacklevel=2,
+        )
+        return self.is_authorized_asset(method=method, user=user)
+
     def is_authorized_pool(
         self, *, method: ResourceMethod, details: PoolDetails | None = None, user: BaseUser | None = None
     ) -> bool:

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/airflow/providers/amazon/aws/hooks/athena.py

@@ -155,14 +155,15 @@ class AthenaHook(AwsBaseHook):
         state = None
         try:
             state = response["QueryExecution"]["Status"]["State"]
-        except Exception:
-            self.log.exception(
-                "Exception while getting query state. Query execution id: %s", query_execution_id
-            )
-        finally:
+        except Exception as e:
             # The error is being absorbed here and is being handled by the caller.
             # The error is being absorbed to implement retries.
-            return state
+            self.log.exception(
+                "Exception while getting query state. Query execution id: %s, Exception: %s",
+                query_execution_id,
+                e,
+            )
+        return state
 
     def get_state_change_reason(self, query_execution_id: str, use_cache: bool = False) -> str | None:
         """
@@ -177,15 +178,15 @@ class AthenaHook(AwsBaseHook):
         reason = None
         try:
             reason = response["QueryExecution"]["Status"]["StateChangeReason"]
-        except Exception:
+        except Exception as e:
+            # The error is being absorbed here and is being handled by the caller.
+            # The error is being absorbed to implement retries.
             self.log.exception(
-                "Exception while getting query state change reason. Query execution id: %s",
+                "Exception while getting query state change reason. Query execution id: %s, Exception: %s",
                 query_execution_id,
+                e,
             )
-        finally:
-            # The error is being absorbed here and is being handled by the caller.
-            # The error is being absorbed to implement retries.
-            return reason
+        return reason
 
     def get_query_results(
         self, query_execution_id: str, next_token_id: str | None = None, max_results: int = 1000
@@ -287,9 +288,18 @@ class AthenaHook(AwsBaseHook):
             )
         except AirflowException as error:
             # this function does not raise errors to keep previous behavior.
-            self.log.warning(error)
-        finally:
-            return self.check_query_status(query_execution_id)
+            self.log.warning(
+                "AirflowException while polling query status. Query execution id: %s, Exception: %s",
+                query_execution_id,
+                error,
+            )
+        except Exception as e:
+            self.log.warning(
+                "Unexpected exception while polling query status. Query execution id: %s, Exception: %s",
+                query_execution_id,
+                e,
+            )
+        return self.check_query_status(query_execution_id)
 
     def get_output_location(self, query_execution_id: str) -> str:
         """

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/airflow/providers/amazon/aws/hooks/eks.py

@@ -85,8 +85,8 @@ COMMAND = """
                 exit 1
             fi
 
-            expiration_timestamp=$(echo "$output" | grep -oP 'expirationTimestamp:\s*\K[^,]+')
-            token=$(echo "$output" | grep -oP 'token:\s*\K[^,]+')
+            expiration_timestamp=$(echo "$output" | grep -oP 'expirationTimestamp: \\K[^,]+')
+            token=$(echo "$output" | grep -oP 'token: \\K[^,]+')
 
             json_string=$(printf '{{"kind": "ExecCredential","apiVersion": \
                 "client.authentication.k8s.io/v1alpha1","spec": {{}},"status": \

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/airflow/providers/amazon/aws/hooks/glue.py

@@ -282,13 +282,16 @@ class GlueJobHook(AwsBaseHook):
             log_group_error, continuation_tokens.error_stream_continuation
         )
 
-    def job_completion(self, job_name: str, run_id: str, verbose: bool = False) -> dict[str, str]:
+    def job_completion(
+        self, job_name: str, run_id: str, verbose: bool = False, sleep_before_return: int = 0
+    ) -> dict[str, str]:
         """
         Wait until Glue job with job_name finishes; return final state if finished or raises AirflowException.
 
         :param job_name: unique job name per AWS account
         :param run_id: The job-run ID of the predecessor job run
         :param verbose: If True, more Glue Job Run logs show in the Airflow Task Logs.  (default: False)
+        :param sleep_before_return: time in seconds to wait before returning final status.
         :return: Dict of JobRunState and JobRunId
         """
         next_log_tokens = self.LogContinuationTokens()
@@ -296,6 +299,7 @@ class GlueJobHook(AwsBaseHook):
             job_run_state = self.get_job_state(job_name, run_id)
             ret = self._handle_state(job_run_state, job_name, run_id, verbose, next_log_tokens)
             if ret:
+                time.sleep(sleep_before_return)
                 return ret
             else:
                 time.sleep(self.job_poll_interval)

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/airflow/providers/amazon/aws/hooks/redshift_sql.py

@@ -163,7 +163,7 @@ class RedshiftSQLHook(DbApiHook):
         # Compatibility: The 'create' factory method was added in SQLAlchemy 1.4
         # to replace calling the default URL constructor directly.
         create_url = getattr(URL, "create", URL)
-        return str(create_url(drivername="redshift+redshift_connector", **conn_params))
+        return str(create_url(drivername="postgresql", **conn_params))
 
     def get_sqlalchemy_engine(self, engine_kwargs=None):
         """Overridden to pass Redshift-specific arguments."""

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/airflow/providers/amazon/aws/hooks/s3.py

@@ -41,14 +41,16 @@ from urllib.parse import urlsplit
 from uuid import uuid4
 
 if TYPE_CHECKING:
-    from mypy_boto3_s3.service_resource import Bucket as S3Bucket, Object as S3ResourceObject
+    from mypy_boto3_s3.service_resource import (
+        Bucket as S3Bucket,
+        Object as S3ResourceObject,
+    )
 
     from airflow.utils.types import ArgNotSet
 
     with suppress(ImportError):
         from aiobotocore.client import AioBaseClient
 
-from importlib.util import find_spec
 
 from asgiref.sync import sync_to_async
 from boto3.s3.transfer import S3Transfer, TransferConfig
@@ -58,15 +60,9 @@ from airflow.exceptions import AirflowException, AirflowNotFoundException
 from airflow.providers.amazon.aws.exceptions import S3HookUriParseFailure
 from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
 from airflow.providers.amazon.aws.utils.tags import format_tags
+from airflow.providers.common.compat.lineage.hook import get_hook_lineage_collector
 from airflow.utils.helpers import chunks
 
-if find_spec("airflow.assets"):
-    from airflow.lineage.hook import get_hook_lineage_collector
-else:
-    # TODO: import from common.compat directly after common.compat providers with
-    # asset_compat_lineage_collector released
-    from airflow.providers.amazon.aws.utils.asset_compat_lineage_collector import get_hook_lineage_collector
-
 logger = logging.getLogger(__name__)
 
 
@@ -90,7 +86,7 @@ def provide_bucket_name(func: Callable) -> Callable:
     async def maybe_add_bucket_name(*args, **kwargs):
         bound_args = function_signature.bind(*args, **kwargs)
 
-        if "bucket_name" not in bound_args.arguments:
+        if not bound_args.arguments.get("bucket_name"):
             self = args[0]
             if self.aws_conn_id:
                 connection = await sync_to_async(self.get_connection)(self.aws_conn_id)
@@ -120,7 +116,7 @@ def provide_bucket_name(func: Callable) -> Callable:
         def wrapper(*args, **kwargs) -> Callable:
             bound_args = function_signature.bind(*args, **kwargs)
 
-            if "bucket_name" not in bound_args.arguments:
+            if not bound_args.arguments.get("bucket_name"):
                 self = args[0]
 
                 if "bucket_name" in self.service_config:
@@ -148,9 +144,10 @@ def unify_bucket_name_and_key(func: Callable) -> Callable:
 
         if "bucket_name" not in bound_args.arguments:
             with suppress(S3HookUriParseFailure):
-                bound_args.arguments["bucket_name"], bound_args.arguments[key_name] = S3Hook.parse_s3_url(
-                    bound_args.arguments[key_name]
-                )
+                (
+                    bound_args.arguments["bucket_name"],
+                    bound_args.arguments[key_name],
+                ) = S3Hook.parse_s3_url(bound_args.arguments[key_name])
 
         return func(*bound_args.args, **bound_args.kwargs)
 
@@ -318,7 +315,8 @@ class S3Hook(AwsBaseHook):
                 self.log.info('Bucket "%s" does not exist', bucket_name)
             elif return_code == 403:
                 self.log.error(
-                    'Access to bucket "%s" is forbidden or there was an error with the request', bucket_name
+                    'Access to bucket "%s" is forbidden or there was an error with the request',
+                    bucket_name,
                 )
                 self.log.error(e)
             return False
@@ -359,7 +357,8 @@ class S3Hook(AwsBaseHook):
             self.get_conn().create_bucket(Bucket=bucket_name)
         else:
             self.get_conn().create_bucket(
-                Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": region_name}
+                Bucket=bucket_name,
+                CreateBucketConfiguration={"LocationConstraint": region_name},
             )
 
     @provide_bucket_name
@@ -410,7 +409,10 @@ class S3Hook(AwsBaseHook):
 
         paginator = self.get_conn().get_paginator("list_objects_v2")
         response = paginator.paginate(
-            Bucket=bucket_name, Prefix=prefix, Delimiter=delimiter, PaginationConfig=config
+            Bucket=bucket_name,
+            Prefix=prefix,
+            Delimiter=delimiter,
+            PaginationConfig=config,
         )
 
         prefixes: list[str] = []
@@ -471,7 +473,10 @@ class S3Hook(AwsBaseHook):
 
         paginator = client.get_paginator("list_objects_v2")
         response = paginator.paginate(
-            Bucket=bucket_name, Prefix=prefix, Delimiter=delimiter, PaginationConfig=config
+            Bucket=bucket_name,
+            Prefix=prefix,
+            Delimiter=delimiter,
+            PaginationConfig=config,
         )
 
         prefixes = []
@@ -569,7 +574,11 @@ class S3Hook(AwsBaseHook):
         return await self._check_key_async(client, bucket, wildcard_match, bucket_keys, use_regex)
 
     async def check_for_prefix_async(
-        self, client: AioBaseClient, prefix: str, delimiter: str, bucket_name: str | None = None
+        self,
+        client: AioBaseClient,
+        prefix: str,
+        delimiter: str,
+        bucket_name: str | None = None,
     ) -> bool:
         """
         Check that a prefix exists in a bucket.
@@ -587,7 +596,11 @@ class S3Hook(AwsBaseHook):
         return prefix in plist
 
     async def _check_for_prefix_async(
-        self, client: AioBaseClient, prefix: str, delimiter: str, bucket_name: str | None = None
+        self,
+        client: AioBaseClient,
+        prefix: str,
+        delimiter: str,
+        bucket_name: str | None = None,
     ) -> bool:
         return await self.check_for_prefix_async(
             client, prefix=prefix, delimiter=delimiter, bucket_name=bucket_name
@@ -643,7 +656,10 @@ class S3Hook(AwsBaseHook):
 
         paginator = client.get_paginator("list_objects_v2")
         response = paginator.paginate(
-            Bucket=bucket_name, Prefix=prefix, Delimiter=delimiter, PaginationConfig=config
+            Bucket=bucket_name,
+            Prefix=prefix,
+            Delimiter=delimiter,
+            PaginationConfig=config,
         )
 
         keys = []
@@ -655,7 +671,10 @@ class S3Hook(AwsBaseHook):
         return keys
 
     def _list_key_object_filter(
-        self, keys: list, from_datetime: datetime | None = None, to_datetime: datetime | None = None
+        self,
+        keys: list,
+        from_datetime: datetime | None = None,
+        to_datetime: datetime | None = None,
     ) -> list:
         def _is_in_period(input_date: datetime) -> bool:
             if from_datetime is not None and input_date <= from_datetime:
@@ -766,7 +785,10 @@ class S3Hook(AwsBaseHook):
                     "message": success_message,
                 }
 
-            self.log.error("FAILURE: Inactivity Period passed, not enough objects found in %s", path)
+            self.log.error(
+                "FAILURE: Inactivity Period passed, not enough objects found in %s",
+                path,
+            )
             return {
                 "status": "error",
                 "message": f"FAILURE: Inactivity Period passed, not enough objects found in {path}",
@@ -1109,7 +1131,13 @@ class S3Hook(AwsBaseHook):
             extra_args["ACL"] = acl_policy
 
         client = self.get_conn()
-        client.upload_file(filename, bucket_name, key, ExtraArgs=extra_args, Config=self.transfer_config)
+        client.upload_file(
+            filename,
+            bucket_name,
+            key,
+            ExtraArgs=extra_args,
+            Config=self.transfer_config,
+        )
         get_hook_lineage_collector().add_input_asset(
             context=self, scheme="file", asset_kwargs={"path": filename}
         )
@@ -1308,18 +1336,32 @@ class S3Hook(AwsBaseHook):
         )
 
         source_bucket_name, source_bucket_key = self.get_s3_bucket_key(
-            source_bucket_name, source_bucket_key, "source_bucket_name", "source_bucket_key"
+            source_bucket_name,
+            source_bucket_key,
+            "source_bucket_name",
+            "source_bucket_key",
         )
 
-        copy_source = {"Bucket": source_bucket_name, "Key": source_bucket_key, "VersionId": source_version_id}
+        copy_source = {
+            "Bucket": source_bucket_name,
+            "Key": source_bucket_key,
+            "VersionId": source_version_id,
+        }
         response = self.get_conn().copy_object(
-            Bucket=dest_bucket_name, Key=dest_bucket_key, CopySource=copy_source, **kwargs
+            Bucket=dest_bucket_name,
+            Key=dest_bucket_key,
+            CopySource=copy_source,
+            **kwargs,
         )
         get_hook_lineage_collector().add_input_asset(
-            context=self, scheme="s3", asset_kwargs={"bucket": source_bucket_name, "key": source_bucket_key}
+            context=self,
+            scheme="s3",
+            asset_kwargs={"bucket": source_bucket_name, "key": source_bucket_key},
         )
         get_hook_lineage_collector().add_output_asset(
-            context=self, scheme="s3", asset_kwargs={"bucket": dest_bucket_name, "key": dest_bucket_key}
+            context=self,
+            scheme="s3",
+            asset_kwargs={"bucket": dest_bucket_name, "key": dest_bucket_key},
         )
         return response
 
@@ -1435,7 +1477,10 @@ class S3Hook(AwsBaseHook):
             file_path = Path(local_dir, subdir, filename_in_s3)
 
             if file_path.is_file():
-                self.log.error("file '%s' already exists. Failing the task and not overwriting it", file_path)
+                self.log.error(
+                    "file '%s' already exists. Failing the task and not overwriting it",
+                    file_path,
+                )
                 raise FileExistsError
 
             file_path.parent.mkdir(exist_ok=True, parents=True)
@@ -1484,7 +1529,10 @@ class S3Hook(AwsBaseHook):
         s3_client = self.get_conn()
         try:
             return s3_client.generate_presigned_url(
-                ClientMethod=client_method, Params=params, ExpiresIn=expires_in, HttpMethod=http_method
+                ClientMethod=client_method,
+                Params=params,
+                ExpiresIn=expires_in,
+                HttpMethod=http_method,
             )
 
         except ClientError as e:

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/airflow/providers/amazon/aws/hooks/sagemaker.py

@@ -153,7 +153,9 @@ class SageMakerHook(AwsBaseHook):
     non_terminal_states = {"InProgress", "Stopping"}
     endpoint_non_terminal_states = {"Creating", "Updating", "SystemUpdating", "RollingBack", "Deleting"}
     pipeline_non_terminal_states = {"Executing", "Stopping"}
+    processing_job_non_terminal_states = {"InProgress", "Stopping"}
     failed_states = {"Failed"}
+    processing_job_failed_states = {*failed_states, "Stopped"}
     training_failed_states = {*failed_states, "Stopped"}
 
     def __init__(self, *args, **kwargs):

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/airflow/providers/amazon/aws/operators/appflow.py

@@ -21,11 +21,11 @@ from datetime import datetime, timedelta
 from typing import TYPE_CHECKING, cast
 
 from airflow.exceptions import AirflowException
-from airflow.operators.python import ShortCircuitOperator
 from airflow.providers.amazon.aws.hooks.appflow import AppflowHook
 from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
 from airflow.providers.amazon.aws.utils import datetime_to_epoch_ms
 from airflow.providers.amazon.aws.utils.mixins import AwsBaseHookMixin, AwsHookParams, aws_template_fields
+from airflow.providers.common.compat.standard.operators import ShortCircuitOperator
 
 if TYPE_CHECKING:
     from mypy_boto3_appflow.type_defs import (

{apache_airflow_providers_amazon-9.0.0rc1 → apache_airflow_providers_amazon-9.1.0}/airflow/providers/amazon/aws/operators/athena.py

@@ -311,7 +311,9 @@ class AthenaOperator(AwsBaseOperator[AthenaHook]):
             }
             fields = [
                 SchemaDatasetFacetFields(
-                    name=column["Name"], type=column["Type"], description=column["Comment"]
+                    name=column["Name"],
+                    type=column["Type"],
+                    description=column.get("Comment"),
                 )
                 for column in table_metadata["TableMetadata"]["Columns"]
             ]