apache-airflow-providers-amazon 8.0.0rc1__py3-none-any.whl → 8.0.0rc2__py3-none-any.whl

airflow/providers/amazon/aws/hooks/base_aws.py

@@ -53,10 +53,10 @@ from airflow.compat.functools import cached_property
 from airflow.configuration import conf
 from airflow.exceptions import (
     AirflowException,
+    AirflowNotFoundException,
 )
 from airflow.hooks.base import BaseHook
 from airflow.providers.amazon.aws.utils.connection_wrapper import AwsConnectionWrapper
-from airflow.providers.amazon.aws.waiters.base_waiter import BaseBotoWaiter
 from airflow.providers_manager import ProvidersManager
 from airflow.utils.helpers import exactly_one
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -70,12 +70,15 @@ if TYPE_CHECKING:
 
 class BaseSessionFactory(LoggingMixin):
     """
-    Base AWS Session Factory class to handle boto3 session creation.
+    Base AWS Session Factory class to handle synchronous and async boto session creation.
     It can handle most of the AWS supported authentication methods.
 
     User can also derive from this class to have full control of boto3 session
     creation or to support custom federation.
 
+    Note: Not all features implemented for synchronous sessions are available for async
+    sessions.
+
     .. seealso::
         - :ref:`howto/connection:aws:session-factory`
     """
@@ -125,17 +128,50 @@ class BaseSessionFactory(LoggingMixin):
         """Assume Role ARN from AWS Connection"""
         return self.conn.role_arn
 
-    def create_session(self) -> boto3.session.Session:
-        """Create boto3 Session from connection config."""
+    def _apply_session_kwargs(self, session):
+        if self.conn.session_kwargs.get("profile_name", None) is not None:
+            session.set_config_variable("profile", self.conn.session_kwargs["profile_name"])
+
+        if (
+            self.conn.session_kwargs.get("aws_access_key_id", None)
+            or self.conn.session_kwargs.get("aws_secret_access_key", None)
+            or self.conn.session_kwargs.get("aws_session_token", None)
+        ):
+            session.set_credentials(
+                self.conn.session_kwargs["aws_access_key_id"],
+                self.conn.session_kwargs["aws_secret_access_key"],
+                self.conn.session_kwargs["aws_session_token"],
+            )
+
+        if self.conn.session_kwargs.get("region_name", None) is not None:
+            session.set_config_variable("region", self.conn.session_kwargs["region_name"])
+
+    def get_async_session(self):
+        from aiobotocore.session import get_session as async_get_session
+
+        return async_get_session()
+
+    def create_session(self, deferrable: bool = False) -> boto3.session.Session:
+        """Create boto3 or aiobotocore Session from connection config."""
         if not self.conn:
             self.log.info(
                 "No connection ID provided. Fallback on boto3 credential strategy (region_name=%r). "
                 "See: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html",
                 self.region_name,
             )
-            return boto3.session.Session(region_name=self.region_name)
+            if deferrable:
+                session = self.get_async_session()
+                self._apply_session_kwargs(session)
+                return session
+            else:
+                return boto3.session.Session(region_name=self.region_name)
         elif not self.role_arn:
-            return self.basic_session
+            if deferrable:
+                session = self.get_async_session()
+                self._apply_session_kwargs(session)
+                return session
+            else:
+                return self.basic_session
 
         # Values stored in ``AwsConnectionWrapper.session_kwargs`` are intended to be used only
         # to create the initial boto3 session.
@@ -148,12 +184,18 @@ class BaseSessionFactory(LoggingMixin):
         assume_session_kwargs = {}
         if self.conn.region_name:
             assume_session_kwargs["region_name"] = self.conn.region_name
-        return self._create_session_with_assume_role(session_kwargs=assume_session_kwargs)
+        return self._create_session_with_assume_role(
+            session_kwargs=assume_session_kwargs, deferrable=deferrable
+        )
 
     def _create_basic_session(self, session_kwargs: dict[str, Any]) -> boto3.session.Session:
         return boto3.session.Session(**session_kwargs)
 
-    def _create_session_with_assume_role(self, session_kwargs: dict[str, Any]) -> boto3.session.Session:
+    def _create_session_with_assume_role(
+        self, session_kwargs: dict[str, Any], deferrable: bool = False
+    ) -> boto3.session.Session:
+        from aiobotocore.session import get_session as async_get_session
+
         if self.conn.assume_role_method == "assume_role_with_web_identity":
             # Deferred credentials have no initial credentials
             credential_fetcher = self._get_web_identity_credential_fetcher()
@@ -170,10 +212,10 @@ class BaseSessionFactory(LoggingMixin):
                 method="sts-assume-role",
             )
 
-        session = botocore.session.get_session()
+        session = async_get_session() if deferrable else botocore.session.get_session()
+
         session._credentials = credentials
-        region_name = self.basic_session.region_name
-        session.set_config_variable("region", region_name)
+        session.set_config_variable("region", self.basic_session.region_name)
 
         return boto3.session.Session(botocore_session=session, **session_kwargs)
 
@@ -498,7 +540,12 @@ class AwsGenericHook(BaseHook, Generic[BaseAwsConnection]):
         """Get the Airflow Connection object and wrap it in helper (cached)."""
         connection = None
         if self.aws_conn_id:
-            connection = self.get_connection(self.aws_conn_id)
+            try:
+                connection = self.get_connection(self.aws_conn_id)
+            except AirflowNotFoundException:
+                self.log.warning(
+                    "Unable to find AWS Connection ID '%s', switching to empty.", self.aws_conn_id
+                )
 
         return AwsConnectionWrapper(
             conn=connection, region_name=self._region_name, botocore_config=self._config, verify=self._verify
@@ -524,11 +571,11 @@ class AwsGenericHook(BaseHook, Generic[BaseAwsConnection]):
         """Verify or not SSL certificates boto3 client/resource read-only property."""
         return self.conn_config.verify
 
-    def get_session(self, region_name: str | None = None) -> boto3.session.Session:
+    def get_session(self, region_name: str | None = None, deferrable: bool = False) -> boto3.session.Session:
         """Get the underlying boto3.session.Session(region_name=region_name)."""
         return SessionFactory(
             conn=self.conn_config, region_name=region_name, config=self.config
-        ).create_session()
+        ).create_session(deferrable=deferrable)
 
     def _get_config(self, config: Config | None = None) -> Config:
         """
@@ -551,10 +598,19 @@ class AwsGenericHook(BaseHook, Generic[BaseAwsConnection]):
         self,
         region_name: str | None = None,
         config: Config | None = None,
+        deferrable: bool = False,
     ) -> boto3.client:
         """Get the underlying boto3 client using boto3 session"""
         client_type = self.client_type
-        session = self.get_session(region_name=region_name)
+        session = self.get_session(region_name=region_name, deferrable=deferrable)
+        if not isinstance(session, boto3.session.Session):
+            return session.create_client(
+                client_type,
+                endpoint_url=self.conn_config.endpoint_url,
+                config=self._get_config(config),
+                verify=self.verify,
+            )
+
         return session.client(
             client_type,
             endpoint_url=self.conn_config.endpoint_url,
@@ -594,6 +650,14 @@ class AwsGenericHook(BaseHook, Generic[BaseAwsConnection]):
         else:
             return self.get_resource_type(region_name=self.region_name)
 
+    @property
+    def async_conn(self):
+        """Get an aiobotocore client to use for async operations."""
+        if not self.client_type:
+            raise ValueError("client_type must be specified.")
+
+        return self.get_client_type(region_name=self.region_name, deferrable=True)
+
     @cached_property
     def conn_client_meta(self) -> ClientMeta:
         """Get botocore client metadata from Hook connection (cached)."""
@@ -747,18 +811,35 @@ class AwsGenericHook(BaseHook, Generic[BaseAwsConnection]):
         path = Path(__file__).parents[1].joinpath(f"waiters/{filename}.json").resolve()
         return path if path.exists() else None
 
-    def get_waiter(self, waiter_name: str, parameters: dict[str, str] | None = None) -> Waiter:
+    def get_waiter(
+        self,
+        waiter_name: str,
+        parameters: dict[str, str] | None = None,
+        deferrable: bool = False,
+        client=None,
+    ) -> Waiter:
         """
         First checks if there is a custom waiter with the provided waiter_name and
         uses that if it exists, otherwise it will check the service client for a
         waiter that matches the name and pass that through.
 
+        If `deferrable` is True, the waiter will be an AIOWaiter, generated from the
+        client that is passed as a parameter. If `deferrable` is True, `client` must be
+        provided.
+
         :param waiter_name: The name of the waiter.  The name should exactly match the
             name of the key in the waiter model file (typically this is CamelCase).
         :param parameters: will scan the waiter config for the keys of that dict, and replace them with the
             corresponding value. If a custom waiter has such keys to be expanded, they need to be provided
             here.
+        :param deferrable: If True, the waiter is going to be an async custom waiter.
+
         """
+        from airflow.providers.amazon.aws.waiters.base_waiter import BaseBotoWaiter
+
+        if deferrable and not client:
+            raise ValueError("client must be provided for a deferrable waiter.")
+        client = client or self.conn
         if self.waiter_path and (waiter_name in self._list_custom_waiters()):
             # Technically if waiter_name is in custom_waiters then self.waiter_path must
             # exist but MyPy doesn't like the fact that self.waiter_path could be None.
@@ -766,7 +847,9 @@ class AwsGenericHook(BaseHook, Generic[BaseAwsConnection]):
                 config = json.loads(config_file.read())
 
             config = self._apply_parameters_value(config, waiter_name, parameters)
-            return BaseBotoWaiter(client=self.conn, model_config=config).waiter(waiter_name)
+            return BaseBotoWaiter(client=client, model_config=config, deferrable=deferrable).waiter(
+                waiter_name
+            )
         # If there is no custom waiter found for the provided name,
         # then try checking the service's official waiters.
         return self.conn.get_waiter(waiter_name)
@@ -935,7 +1018,7 @@ class BaseAsyncSessionFactory(BaseSessionFactory):
             aio_session.set_config_variable("region", region_name)
         return aio_session
 
-    def create_session(self) -> AioSession:
+    def create_session(self, deferrable: bool = False) -> AioSession:
         """Create aiobotocore Session from connection and config."""
         if not self._conn:
             self.log.info("No connection ID provided. Fallback on boto3 credential strategy")

airflow/providers/amazon/aws/hooks/batch_waiters.py

@@ -138,7 +138,9 @@ class BatchWaitersHook(BatchClientHook):
         """
         return self._waiter_model
 
-    def get_waiter(self, waiter_name: str, _: dict[str, str] | None = None) -> botocore.waiter.Waiter:
+    def get_waiter(
+        self, waiter_name: str, _: dict[str, str] | None = None, deferrable: bool = False, client=None
+    ) -> botocore.waiter.Waiter:
         """
         Get an AWS Batch service waiter, using the configured ``.waiter_model``.
 

airflow/providers/amazon/aws/hooks/emr.py

@@ -24,7 +24,6 @@ from typing import Any
 
 from botocore.exceptions import ClientError
 
-from airflow.compat.functools import cached_property
 from airflow.exceptions import AirflowException, AirflowNotFoundException
 from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
 from airflow.utils.helpers import prune_dict
@@ -253,10 +252,41 @@ class EmrServerlessHook(AwsBaseHook):
         kwargs["client_type"] = "emr-serverless"
         super().__init__(*args, **kwargs)
 
-    @cached_property
-    def conn(self):
-        """Get the underlying boto3 EmrServerlessAPIService client (cached)"""
-        return super().conn
+    def cancel_running_jobs(self, application_id: str, waiter_config: dict = {}):
+        """
+        List all jobs in an intermediate state and cancel them.
+        Then wait for those jobs to reach a terminal state.
+        Note: if new jobs are triggered while this operation is ongoing,
+        it's going to time out and return an error.
+        """
+        paginator = self.conn.get_paginator("list_job_runs")
+        results_per_response = 50
+        iterator = paginator.paginate(
+            applicationId=application_id,
+            states=list(self.JOB_INTERMEDIATE_STATES),
+            PaginationConfig={
+                "PageSize": results_per_response,
+            },
+        )
+        count = 0
+        for r in iterator:
+            job_ids = [jr["id"] for jr in r["jobRuns"]]
+            count += len(job_ids)
+            if len(job_ids) > 0:
+                self.log.info(
+                    "Cancelling %s pending job(s) for the application %s so that it can be stopped",
+                    len(job_ids),
+                    application_id,
+                )
+                for job_id in job_ids:
+                    self.conn.cancel_job_run(applicationId=application_id, jobRunId=job_id)
+        if count > 0:
+            self.log.info("now waiting for the %s cancelled job(s) to terminate", count)
+            self.get_waiter("no_job_running").wait(
+                applicationId=application_id,
+                states=list(self.JOB_INTERMEDIATE_STATES.union({"CANCELLING"})),
+                WaiterConfig=waiter_config,
+            )
 
 
 class EmrContainerHook(AwsBaseHook):

airflow/providers/amazon/aws/links/emr.py

@@ -24,9 +24,7 @@ class EmrClusterLink(BaseAwsLink):
 
     name = "EMR Cluster"
     key = "emr_cluster"
-    format_str = (
-        BASE_AWS_CONSOLE_LINK + "/elasticmapreduce/home?region={region_name}#cluster-details:{job_flow_id}"
-    )
+    format_str = BASE_AWS_CONSOLE_LINK + "/emr/home?region={region_name}#/clusterDetails/{job_flow_id}"
 
 
 class EmrLogsLink(BaseAwsLink):

airflow/providers/amazon/aws/operators/emr.py

@@ -1025,6 +1025,10 @@ class EmrServerlessStopApplicationOperator(BaseOperator):
         the application be stopped. Defaults to 5 minutes.
     :param waiter_check_interval_seconds: Number of seconds between polling the state of the application.
         Defaults to 30 seconds.
+    :param force_stop: If set to True, any job for that app that is not in a terminal state will be cancelled.
+        Otherwise, trying to stop an app with running jobs will return an error.
+        If you want to wait for the jobs to finish gracefully, use
+        :class:`airflow.providers.amazon.aws.sensors.emr.EmrServerlessJobSensor`
     """
 
     template_fields: Sequence[str] = ("application_id",)
@@ -1036,6 +1040,7 @@ class EmrServerlessStopApplicationOperator(BaseOperator):
         aws_conn_id: str = "aws_default",
         waiter_countdown: int = 5 * 60,
         waiter_check_interval_seconds: int = 30,
+        force_stop: bool = False,
         **kwargs,
     ):
         self.aws_conn_id = aws_conn_id
@@ -1043,6 +1048,7 @@ class EmrServerlessStopApplicationOperator(BaseOperator):
         self.wait_for_completion = wait_for_completion
         self.waiter_countdown = waiter_countdown
         self.waiter_check_interval_seconds = waiter_check_interval_seconds
+        self.force_stop = force_stop
         super().__init__(**kwargs)
 
     @cached_property
@@ -1052,6 +1058,16 @@ class EmrServerlessStopApplicationOperator(BaseOperator):
 
     def execute(self, context: Context) -> None:
         self.log.info("Stopping application: %s", self.application_id)
+
+        if self.force_stop:
+            self.hook.cancel_running_jobs(
+                self.application_id,
+                waiter_config={
+                    "Delay": self.waiter_check_interval_seconds,
+                    "MaxAttempts": self.waiter_countdown / self.waiter_check_interval_seconds,
+                },
+            )
+
         self.hook.conn.stop_application(applicationId=self.application_id)
 
         if self.wait_for_completion:
@@ -1088,6 +1104,10 @@ class EmrServerlessDeleteApplicationOperator(EmrServerlessStopApplicationOperato
         the application to be stopped, and then deleted. Defaults to 25 minutes.
     :param waiter_check_interval_seconds: Number of seconds between polling the state of the application.
         Defaults to 60 seconds.
+    :param force_stop: If set to True, any job for that app that is not in a terminal state will be cancelled.
+        Otherwise, trying to delete an app with running jobs will return an error.
+        If you want to wait for the jobs to finish gracefully, use
+        :class:`airflow.providers.amazon.aws.sensors.emr.EmrServerlessJobSensor`
     """
 
     template_fields: Sequence[str] = ("application_id",)
@@ -1099,6 +1119,7 @@ class EmrServerlessDeleteApplicationOperator(EmrServerlessStopApplicationOperato
         aws_conn_id: str = "aws_default",
         waiter_countdown: int = 25 * 60,
         waiter_check_interval_seconds: int = 60,
+        force_stop: bool = False,
         **kwargs,
     ):
         self.wait_for_delete_completion = wait_for_completion
@@ -1110,6 +1131,7 @@ class EmrServerlessDeleteApplicationOperator(EmrServerlessStopApplicationOperato
             aws_conn_id=aws_conn_id,
             waiter_countdown=waiter_countdown,
             waiter_check_interval_seconds=waiter_check_interval_seconds,
+            force_stop=force_stop,
             **kwargs,
         )
 

airflow/providers/amazon/aws/operators/rds.py

@@ -80,7 +80,7 @@ class RdsCreateDbSnapshotOperator(RdsBaseOperator):
         db_snapshot_identifier: str,
         tags: Sequence[TagTypeDef] | dict | None = None,
         wait_for_completion: bool = True,
-        aws_conn_id: str = "aws_conn_id",
+        aws_conn_id: str = "aws_default",
         **kwargs,
     ):
         super().__init__(aws_conn_id=aws_conn_id, **kwargs)

airflow/providers/amazon/aws/operators/redshift_cluster.py

@@ -22,7 +22,10 @@ from typing import TYPE_CHECKING, Any, Sequence
 from airflow.exceptions import AirflowException
 from airflow.models import BaseOperator
 from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftHook
-from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftClusterTrigger
+from airflow.providers.amazon.aws.triggers.redshift_cluster import (
+    RedshiftClusterTrigger,
+    RedshiftCreateClusterTrigger,
+)
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context
@@ -88,6 +91,7 @@ class RedshiftCreateClusterOperator(BaseOperator):
     :param wait_for_completion: Whether wait for the cluster to be in ``available`` state
     :param max_attempt: The maximum number of attempts to be made. Default: 5
     :param poll_interval: The amount of time in seconds to wait between attempts. Default: 60
+    :param deferrable: If True, the operator will run in deferrable mode
     """
 
     template_fields: Sequence[str] = (
@@ -140,6 +144,7 @@ class RedshiftCreateClusterOperator(BaseOperator):
         wait_for_completion: bool = False,
         max_attempt: int = 5,
         poll_interval: int = 60,
+        deferrable: bool = False,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -180,6 +185,7 @@ class RedshiftCreateClusterOperator(BaseOperator):
         self.wait_for_completion = wait_for_completion
         self.max_attempt = max_attempt
         self.poll_interval = poll_interval
+        self.deferrable = deferrable
         self.kwargs = kwargs
 
     def execute(self, context: Context):
@@ -252,6 +258,16 @@ class RedshiftCreateClusterOperator(BaseOperator):
             self.master_user_password,
             params,
         )
+        if self.deferrable:
+            self.defer(
+                trigger=RedshiftCreateClusterTrigger(
+                    cluster_identifier=self.cluster_identifier,
+                    poll_interval=self.poll_interval,
+                    max_attempt=self.max_attempt,
+                    aws_conn_id=self.aws_conn_id,
+                ),
+                method_name="execute_complete",
+            )
         if self.wait_for_completion:
             redshift_hook.get_conn().get_waiter("cluster_available").wait(
                 ClusterIdentifier=self.cluster_identifier,
@@ -264,6 +280,11 @@ class RedshiftCreateClusterOperator(BaseOperator):
         self.log.info("Created Redshift cluster %s", self.cluster_identifier)
         self.log.info(cluster)
 
+    def execute_complete(self, context, event=None):
+        if event["status"] != "success":
+            raise AirflowException(f"Error creating cluster: {event}")
+        return
+
 
 class RedshiftCreateClusterSnapshotOperator(BaseOperator):
     """

airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py

@@ -87,7 +87,7 @@ class DynamoDBToS3Operator(AwsToAwsBaseOperator):
     :param dynamodb_scan_kwargs: kwargs pass to <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.scan>
     :param s3_key_prefix: Prefix of s3 object key
     :param process_func: How we transforms a dynamodb item to bytes. By default we dump the json
-    """  # noqa: E501
+    """
 
     template_fields: Sequence[str] = (
         *AwsToAwsBaseOperator.template_fields,

airflow/providers/amazon/aws/triggers/redshift_cluster.py

@@ -18,7 +18,8 @@ from __future__ import annotations
 
 from typing import Any, AsyncIterator
 
-from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook
+from airflow.compat.functools import cached_property
+from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook, RedshiftHook
 from airflow.triggers.base import BaseTrigger, TriggerEvent
 
 
@@ -55,7 +56,7 @@ class RedshiftClusterTrigger(BaseTrigger):
             },
         )
 
-    async def run(self) -> AsyncIterator["TriggerEvent"]:
+    async def run(self) -> AsyncIterator[TriggerEvent]:
         hook = RedshiftAsyncHook(aws_conn_id=self.aws_conn_id)
         while self.attempts >= 1:
             self.attempts = self.attempts - 1
@@ -85,3 +86,54 @@ class RedshiftClusterTrigger(BaseTrigger):
             except Exception as e:
                 if self.attempts < 1:
                     yield TriggerEvent({"status": "error", "message": str(e)})
+
+
+class RedshiftCreateClusterTrigger(BaseTrigger):
+    """
+    Trigger for RedshiftCreateClusterOperator.
+    The trigger will asynchronously poll the boto3 API and wait for the
+    Redshift cluster to be in the `available` state.
+
+    :param cluster_identifier:  A unique identifier for the cluster.
+    :param poll_interval: The amount of time in seconds to wait between attempts.
+    :param max_attempt: The maximum number of attempts to be made.
+    :param aws_conn_id: The Airflow connection used for AWS credentials.
+    """
+
+    def __init__(
+        self,
+        cluster_identifier: str,
+        poll_interval: int,
+        max_attempt: int,
+        aws_conn_id: str,
+    ):
+        self.cluster_identifier = cluster_identifier
+        self.poll_interval = poll_interval
+        self.max_attempt = max_attempt
+        self.aws_conn_id = aws_conn_id
+
+    def serialize(self) -> tuple[str, dict[str, Any]]:
+        return (
+            "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftCreateClusterTrigger",
+            {
+                "cluster_identifier": str(self.cluster_identifier),
+                "poll_interval": str(self.poll_interval),
+                "max_attempt": str(self.max_attempt),
+                "aws_conn_id": str(self.aws_conn_id),
+            },
+        )
+
+    @cached_property
+    def hook(self) -> RedshiftHook:
+        return RedshiftHook(aws_conn_id=self.aws_conn_id)
+
+    async def run(self):
+        async with self.hook.async_conn as client:
+            await client.get_waiter("cluster_available").wait(
+                ClusterIdentifier=self.cluster_identifier,
+                WaiterConfig={
+                    "Delay": int(self.poll_interval),
+                    "MaxAttempts": int(self.max_attempt),
+                },
+            )
+        yield TriggerEvent({"status": "success", "message": "Cluster Created"})

airflow/providers/amazon/aws/waiters/base_waiter.py

@@ -28,9 +28,20 @@ class BaseBotoWaiter:
     For more details, see airflow/providers/amazon/aws/waiters/README.md
     """
 
-    def __init__(self, client: boto3.client, model_config: dict) -> None:
+    def __init__(self, client: boto3.client, model_config: dict, deferrable: bool = False) -> None:
         self.model = WaiterModel(model_config)
         self.client = client
+        self.deferrable = deferrable
+
+    def _get_async_waiter_with_client(self, waiter_name: str):
+        from aiobotocore.waiter import create_waiter_with_client as create_async_waiter_with_client
+
+        return create_async_waiter_with_client(
+            waiter_name=waiter_name, waiter_model=self.model, client=self.client
+        )
 
     def waiter(self, waiter_name: str) -> Waiter:
+        if self.deferrable:
+            return self._get_async_waiter_with_client(waiter_name=waiter_name)
+
         return create_waiter_with_client(waiter_name=waiter_name, waiter_model=self.model, client=self.client)

airflow/providers/amazon/aws/waiters/emr-serverless.json

@@ -0,0 +1,18 @@
+{
+    "version": 2,
+    "waiters": {
+        "no_job_running": {
+            "operation": "ListJobRuns",
+            "delay": 10,
+            "maxAttempts": 60,
+            "acceptors": [
+                {
+                    "matcher": "path",
+                    "argument": "length(jobRuns) == `0`",
+                    "expected": true,
+                    "state": "success"
+                }
+            ]
+        }
+    }
+}

airflow/providers/amazon/get_provider_info.py

@@ -74,6 +74,7 @@ def get_provider_info():
             "mypy-boto3-rds>=1.24.0",
             "mypy-boto3-redshift-data>=1.24.0",
             "mypy-boto3-appflow>=1.24.0",
+            "aiobotocore[boto3]>=2.2.0",
         ],
         "integrations": [
             {

{apache_airflow_providers_amazon-8.0.0rc1.dist-info → apache_airflow_providers_amazon-8.0.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-amazon
-Version: 8.0.0rc1
+Version: 8.0.0rc2
 Summary: Provider for Apache Airflow. Implements apache-airflow-providers-amazon package
 Home-page: https://airflow.apache.org/
 Download-URL: https://archive.apache.org/dist/airflow/providers
@@ -30,6 +30,7 @@ Requires-Python: ~=3.7
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 License-File: NOTICE
+Requires-Dist: aiobotocore[boto3] (>=2.2.0)
 Requires-Dist: apache-airflow-providers-common-sql (>=1.3.1.dev0)
 Requires-Dist: apache-airflow (>=2.3.0.dev0)
 Requires-Dist: asgiref
@@ -87,7 +88,7 @@ Requires-Dist: apache-airflow-providers-ssh ; extra == 'ssh'
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``8.0.0rc1``
+Release: ``8.0.0rc2``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -129,6 +130,7 @@ PIP package                              Version required
 ``mypy-boto3-rds``                       ``>=1.24.0``
 ``mypy-boto3-redshift-data``             ``>=1.24.0``
 ``mypy-boto3-appflow``                   ``>=1.24.0``
+``aiobotocore[boto3]``                   ``>=2.2.0``
 =======================================  ==================
 
 Cross provider package dependencies
@@ -199,8 +201,6 @@ Breaking changes
 
   Removed deprecated parameter ``max_tries`` from the Athena & EMR hook & operators in favor of ``max_polling_attempts``.
 
-  Disabled deprecated behavior of switching to an empty aws connection ID on error. You can set it to None explicitly.
-
   Removed deprecated method ``waiter`` from emr hook in favor of the more generic ``airflow.providers.amazon.aws.utils.waiter.waiter``
 
   Removed deprecated unused parameter ``cluster_identifier`` from Redshift Cluster's hook method ``get_cluster_snapshot_status``
@@ -234,6 +234,8 @@ Features
 * ``add a stop operator to emr serverless (#30720)``
 * ``SqlToS3Operator - Add feature to partition SQL table (#30460)``
 * ``New AWS sensor — DynamoDBValueSensor (#28338)``
+* ``Add a "force" option to emr serverless stop/delete operator (#30757)``
+* ``Add support for deferrable operators in AMPP (#30032)``
 
 Bug Fixes
 ~~~~~~~~~
@@ -250,10 +252,15 @@ Misc
 * ``Remove @poke_mode_only from EmrStepSensor (#30774)``
 * ``Organize Amazon providers docs index (#30541)``
 * ``Remove duplicate param docstring in EksPodOperator (#30634)``
+* ``Update AWS EMR Cluster Link to use the new dashboard (#30844)``
 
 .. Below changes are excluded from the changelog. Move them to
    appropriate section above if needed. Do not delete the lines(!):
    * ``Decouple "job runner" from BaseJob ORM model (#30255)``
+   * ``Upgrade ruff to 0.0.262 (#30809)``
+   * ``fixes to system tests following obsolete cleanup (#30804)``
+   * ``restore fallback to empty connection behavior (#30806)``
+   * ``Prepare docs for adhoc release of providers (#30787)``
 
 7.4.1
 .....

{apache_airflow_providers_amazon-8.0.0rc1.dist-info → apache_airflow_providers_amazon-8.0.0rc2.dist-info}/RECORD

@@ -1,14 +1,14 @@
 airflow/providers/amazon/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/amazon/get_provider_info.py,sha256=kviXDXhiXXWsgPFSGd5xgojwyjSzFpMiHhuI1bsW4CI,36826
+airflow/providers/amazon/get_provider_info.py,sha256=gk7Cmp-kyVMF5C5pPpionAknFR7OahOSn3C6u3uf_QI,36867
 airflow/providers/amazon/aws/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/amazon/aws/exceptions.py,sha256=UVoxpfQEdWI1319h0U78Z_r5wRFQL6DN14hJw3G1Rgo,1731
 airflow/providers/amazon/aws/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/amazon/aws/hooks/appflow.py,sha256=oJQSg6kawjjUmbx7YGEK3if_8CNc7g7uk9SRRdxjN0A,4856
 airflow/providers/amazon/aws/hooks/athena.py,sha256=rFRXRwxxO4hk1kN9rD0fRl9dPnJiLyJhHgyNyWj-yYI,12240
-airflow/providers/amazon/aws/hooks/base_aws.py,sha256=Hg2ssKZTM14L00GsmzR3CyTrcZ9HZifshqOPuN7K_ZQ,42181
+airflow/providers/amazon/aws/hooks/base_aws.py,sha256=7xpTAxrZtVba0_Hdo0nefjVeZjbVAs8t_fwpFTRH4SM,45421
 airflow/providers/amazon/aws/hooks/batch_client.py,sha256=NmVxHHK4FMOhCHqbAddsVeeDlWlmVBBFImxCOWUTqow,20629
 airflow/providers/amazon/aws/hooks/batch_waiters.json,sha256=eoN5YDgeTNZ2Xz17TrbKBPhd7z9-6KD3RhaDKXXOvqU,2511
-airflow/providers/amazon/aws/hooks/batch_waiters.py,sha256=IljZE22oJdiZ6wRbJha_w5mbGhyNUXZzMqwQ1j6Y0NE,9644
+airflow/providers/amazon/aws/hooks/batch_waiters.py,sha256=u-PlSje_eIxcI-brKrUj8JDEExZg2nyU2h10Cl4O0-I,9697
 airflow/providers/amazon/aws/hooks/cloud_formation.py,sha256=1IY_btHjNZJr5yMPCo3nYBMJKIHO5z60ywaRqm2irUs,3376
 airflow/providers/amazon/aws/hooks/datasync.py,sha256=AAFLNqrrCwumDaHPtG8dluzd7pp5jXQK4wkP_6rs88M,14133
 airflow/providers/amazon/aws/hooks/dms.py,sha256=iPQ9lyQcJLmD_mv8OYi8JihDDyQ-PWIcHUFJBl3jueA,7906
@@ -18,7 +18,7 @@ airflow/providers/amazon/aws/hooks/ecr.py,sha256=OWXMoqKDAmwnwuA_fiVBradMpopqxST
 airflow/providers/amazon/aws/hooks/ecs.py,sha256=2HxP4a5gUE7aoEWfZrkibDpSgaoBfapRzp4vBTNhnvE,9194
 airflow/providers/amazon/aws/hooks/eks.py,sha256=AHJr-9gPDa9pISD3vX1hkGXRVaqoc78S4SaD8tIr3_s,23957
 airflow/providers/amazon/aws/hooks/elasticache_replication_group.py,sha256=5Zr9T8_-Nc-bxvsfrZoWG4GTfaQnL0iYjhFAsYcdnEc,12114
-airflow/providers/amazon/aws/hooks/emr.py,sha256=EeVYyLUZJ_v6ikvdZV2nYm4u22N0e3GBV7SrVAazQnc,18990
+airflow/providers/amazon/aws/hooks/emr.py,sha256=E59iIckiK5bf_xLIsRChg2oKMYkjOTQaReTlGlaWYB8,20348
 airflow/providers/amazon/aws/hooks/glacier.py,sha256=4d8U13e3KCsyf07Zd0H7AkELWQXH8Z6EZhT4X_HxTAw,3518
 airflow/providers/amazon/aws/hooks/glue.py,sha256=sH2VUuoMKc5gfz5ORRMuvqYsQMz9L9jFyjoWYjdswWY,13816
 airflow/providers/amazon/aws/hooks/glue_catalog.py,sha256=-GbwxS8wlxr9kQkIkNtqApHriISBntsS-sGEmr4a0Xk,7611
@@ -43,7 +43,7 @@ airflow/providers/amazon/aws/hooks/sts.py,sha256=OcE-Lo-yKvTrocLmok5GMB6qWMDTz2i
 airflow/providers/amazon/aws/links/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/amazon/aws/links/base_aws.py,sha256=Uj1BjQOfizqpoCGyH0RyhRyf2PRhmWPJ8a3nZrP_jxE,2940
 airflow/providers/amazon/aws/links/batch.py,sha256=0nqICmZA77YpK0WgTZ6pJUCmOqZxrvJxbxA8ERNV_Qw,1767
-airflow/providers/amazon/aws/links/emr.py,sha256=Xg6j4sNAYYLXKL11w9IYbor1pCR__RJVx71QLksZOzg,1445
+airflow/providers/amazon/aws/links/emr.py,sha256=vlZjjPwC2dP9xwX5w64xlcdW0CSFowkUE00_ob4GyAI,1416
 airflow/providers/amazon/aws/links/glue.py,sha256=sac5NCWhE4yNdUpmHB_OfWYg81rM00BkIKgPGnyplCE,1228
 airflow/providers/amazon/aws/links/logs.py,sha256=BWIU7YQxKt03sUHukr2Z9T99Dzxmwi3pyxHBlWPOi00,1607
 airflow/providers/amazon/aws/log/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
@@ -59,14 +59,14 @@ airflow/providers/amazon/aws/operators/dms.py,sha256=WvLNzBYglB8iKxEHu3eD2NbJ1N8
 airflow/providers/amazon/aws/operators/ec2.py,sha256=iM-DP-RRIYIbIL1mkVr3Eo61U4s6wCBdps_m465mfNg,9589
 airflow/providers/amazon/aws/operators/ecs.py,sha256=1oZSO52u0DrzkRbtWWHHek7SrloipMl7z3HomxPlZYY,29018
 airflow/providers/amazon/aws/operators/eks.py,sha256=QlZCVymQPZUdJw91B31s5VNc3kmO_H-G6InSpsSAOeA,31753
-airflow/providers/amazon/aws/operators/emr.py,sha256=1W_SZ0kZDFoN9EUoYHcYQCzXZZNfyUMgFtbLLrqJBnE,49023
+airflow/providers/amazon/aws/operators/emr.py,sha256=EDeZwfJdAAR3pkea0jqm6SXunITd_fhcmNvTpHswqRo,50180
 airflow/providers/amazon/aws/operators/glacier.py,sha256=FY07RQgiYuJI6wfrnXFzShvVRfQh9Qe4MJKh3lJGPY4,3705
 airflow/providers/amazon/aws/operators/glue.py,sha256=cw9_5TVLLXargUtgNfn4OHDrx1GPU_F_PsIEI9QNfgw,7471
 airflow/providers/amazon/aws/operators/glue_crawler.py,sha256=l1KbD59PO9RlGm_eCUrkIOP76ZvxHvZa0152vlkoY_I,3306
 airflow/providers/amazon/aws/operators/lambda_function.py,sha256=aFWRwBa5FgGpKGS402QWtwOdetg8xXwHIRPfwCmoelU,7755
 airflow/providers/amazon/aws/operators/quicksight.py,sha256=ZzppR2QONsxmOjTDHHOFFqL7SwPKXEVccoC2-EoWu9A,3972
-airflow/providers/amazon/aws/operators/rds.py,sha256=UPtw9SgzaWZFMiSNJt5ab8rUSfFhUIDDqxZJ3JDjMB0,29868
-airflow/providers/amazon/aws/operators/redshift_cluster.py,sha256=Dgu0Br1kS0sLBiYcQ4uIlcM-5Lsog_IZHpqrzhAmPIU,28218
+airflow/providers/amazon/aws/operators/rds.py,sha256=uhNAzp5bCvbnYAaIRVUSwGu-uYDCfZuLc72zqQSD8ok,29868
+airflow/providers/amazon/aws/operators/redshift_cluster.py,sha256=Ay_JHBWQnHwraFRSCXqXAuCaqEDj551lrW6lVYVKYq4,28991
 airflow/providers/amazon/aws/operators/redshift_data.py,sha256=gy3pkrlMKl1X5KHAy9OyiOVUd4F1pOAwA5xAFztKTgY,5559
 airflow/providers/amazon/aws/operators/s3.py,sha256=xHMdAktBsz73tm9D-x3bn-YH-O_7nJommmfxE8hsYtM,29966
 airflow/providers/amazon/aws/operators/sagemaker.py,sha256=d_T4y0AnCt06QhojahsSoh4b1_84Bc3r6hZvupZX_lo,49835
@@ -100,7 +100,7 @@ airflow/providers/amazon/aws/sensors/sqs.py,sha256=3q4ADyIxg3ad4eJDN7iS5DE0NZgSj
 airflow/providers/amazon/aws/sensors/step_function.py,sha256=viLmKvH66ay9D5NBCYIYgYISKSSuwfl60doEYiNCcmY,3400
 airflow/providers/amazon/aws/transfers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/amazon/aws/transfers/base.py,sha256=J1_Zfu4l5hD6d2YscGxk9fIBa6VlBiz8Pa2Bv8lE_zY,2768
-airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py,sha256=2mooMubDviUIxvM-50-wocqBP4Rhbh4RzYYEm4-B4TU,5857
+airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py,sha256=mdstDij3G6VyGxvjHYaZuXR1SIV_7PVpCi_NEoLtduY,5843
 airflow/providers/amazon/aws/transfers/exasol_to_s3.py,sha256=Hwh684UFRIfhYxHlapNuAiIse770ZmeB8Le_KMd6o04,4410
 airflow/providers/amazon/aws/transfers/ftp_to_s3.py,sha256=tj9W9432lBcbty6bpB5lt_6OccsDieTRWCZiZBJ7k74,6435
 airflow/providers/amazon/aws/transfers/gcs_to_s3.py,sha256=4uQMPZBxbi8j9sFZ3KhoanbNx-JoNfFqU3w4C_yg_k0,7676
@@ -119,7 +119,7 @@ airflow/providers/amazon/aws/transfers/salesforce_to_s3.py,sha256=3sYK78G9vHPv9g
 airflow/providers/amazon/aws/transfers/sftp_to_s3.py,sha256=zlgbY6C9JzZCDaZT9xOU8J4_hO7oW16g_OQjSuOH5u4,3679
 airflow/providers/amazon/aws/transfers/sql_to_s3.py,sha256=NktjDIdFoU0T7FQ_ZlfqAtPClEgktCleur0vVwDLmks,8475
 airflow/providers/amazon/aws/triggers/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/amazon/aws/triggers/redshift_cluster.py,sha256=zzgVK4c88MUbEbp4mNsy1G_OQrlD0C3-vuXPNF8-7is,3640
+airflow/providers/amazon/aws/triggers/redshift_cluster.py,sha256=gayq9dNO265b0a9xfuGlZL7rpix6gSGemGiR20xkpQE,5579
 airflow/providers/amazon/aws/utils/__init__.py,sha256=-cre0aYXXsfV5D5an_oLOphKhr-k_uh-dYPvteOrTaE,2215
 airflow/providers/amazon/aws/utils/connection_wrapper.py,sha256=rM3PzrHJlrT4QczMIeOhJxw9Js6tGOizDLUepzb5kvA,20348
 airflow/providers/amazon/aws/utils/eks_get_token.py,sha256=HkntWy4xaASjPRGj2of1leLDyDK4JEwRctqOD4nvOt4,2561
@@ -131,14 +131,15 @@ airflow/providers/amazon/aws/utils/tags.py,sha256=apnh_6MBcNzSCNPXW5BqFjJwgih_t2
 airflow/providers/amazon/aws/utils/waiter.py,sha256=gMtCiIBINvE0wSMzjpDwZw7VMzhGt4do6CoNAR-z1wE,3583
 airflow/providers/amazon/aws/waiters/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/amazon/aws/waiters/appflow.json,sha256=9Ye1dItlQGd9fWX63GriQ_VAYbF16cSGcXy4MxU5MGg,1002
-airflow/providers/amazon/aws/waiters/base_waiter.py,sha256=wIZgp3KG54qqldXs2hIcjsI-TDVGbSiVm6fF47RvMYE,1378
+airflow/providers/amazon/aws/waiters/base_waiter.py,sha256=4hEgo0BvSpwlKX1Jo3eQqc43hQA_FVKAN82gGlWat1g,1853
 airflow/providers/amazon/aws/waiters/ecs.json,sha256=q1_9vRtdZG0g86HHMvkjmBtC2CNaMujA0fzMRGrCLxQ,2720
 airflow/providers/amazon/aws/waiters/eks.json,sha256=8Pl_P_KjEXkrKrA3Enzux4ve_SA8acvvV5Dch8YLgIU,659
+airflow/providers/amazon/aws/waiters/emr-serverless.json,sha256=EiqhDMZLkY101m-9KMbphDQLPXydBRKqXrDJjpjjZsw,430
 airflow/providers/amazon/aws/waiters/emr.json,sha256=_h2F-sZ2DJGYfV9Nj0RD8Bl5m02Z09MO6X_U8bgmOjA,2559
-apache_airflow_providers_amazon-8.0.0rc1.dist-info/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-apache_airflow_providers_amazon-8.0.0rc1.dist-info/METADATA,sha256=HW344Kuq3KNg8Nm0xV1fG354gVq0iZP0o4jMbHO1_x8,57941
-apache_airflow_providers_amazon-8.0.0rc1.dist-info/NOTICE,sha256=m-6s2XynUxVSUIxO4rVablAZCvFq-wmLrqV91DotRBw,240
-apache_airflow_providers_amazon-8.0.0rc1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-apache_airflow_providers_amazon-8.0.0rc1.dist-info/entry_points.txt,sha256=HNciwBqNoIpFPDUUu9_tL6HyozCAxlpCifH810uQTOQ,103
-apache_airflow_providers_amazon-8.0.0rc1.dist-info/top_level.txt,sha256=OeMVH5md7fr2QQWpnZoOWWxWO-0WH1IP70lpTVwopPg,8
-apache_airflow_providers_amazon-8.0.0rc1.dist-info/RECORD,,
+apache_airflow_providers_amazon-8.0.0rc2.dist-info/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
+apache_airflow_providers_amazon-8.0.0rc2.dist-info/METADATA,sha256=ykU7JcUvyRp6KZNkAZKFg2bC21tgkZa3i5_VfRkWuho,58356
+apache_airflow_providers_amazon-8.0.0rc2.dist-info/NOTICE,sha256=m-6s2XynUxVSUIxO4rVablAZCvFq-wmLrqV91DotRBw,240
+apache_airflow_providers_amazon-8.0.0rc2.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+apache_airflow_providers_amazon-8.0.0rc2.dist-info/entry_points.txt,sha256=HNciwBqNoIpFPDUUu9_tL6HyozCAxlpCifH810uQTOQ,103
+apache_airflow_providers_amazon-8.0.0rc2.dist-info/top_level.txt,sha256=OeMVH5md7fr2QQWpnZoOWWxWO-0WH1IP70lpTVwopPg,8
+apache_airflow_providers_amazon-8.0.0rc2.dist-info/RECORD,,