apache-airflow-providers-amazon 8.26.0rc1__tar.gz → 8.27.0__tar.gz

{apache_airflow_providers_amazon-8.26.0rc1 → apache_airflow_providers_amazon-8.27.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-amazon
-Version: 8.26.0rc1
+Version: 8.27.0
 Summary: Provider package apache-airflow-providers-amazon for Apache Airflow
 Keywords: airflow-provider,amazon,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -22,9 +22,11 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: PyAthena>=3.0.10
-Requires-Dist: apache-airflow-providers-common-sql>=1.3.1rc0
+Requires-Dist: apache-airflow-providers-common-compat>=1.1.0
+Requires-Dist: apache-airflow-providers-common-compat>=1.1.0
+Requires-Dist: apache-airflow-providers-common-sql>=1.3.1
 Requires-Dist: apache-airflow-providers-http
-Requires-Dist: apache-airflow>=2.7.0rc0
+Requires-Dist: apache-airflow>=2.7.0
 Requires-Dist: asgiref>=2.3.0
 Requires-Dist: boto3>=1.34.90
 Requires-Dist: botocore>=1.34.90
@@ -36,7 +38,8 @@ Requires-Dist: sqlalchemy_redshift>=0.8.6
 Requires-Dist: watchtower>=3.0.0,<4
 Requires-Dist: aiobotocore[boto3]>=2.13.0 ; extra == "aiobotocore"
 Requires-Dist: apache-airflow-providers-apache-hive ; extra == "apache.hive"
-Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0rc0 ; extra == "cncf.kubernetes"
+Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0 ; extra == "cncf.kubernetes"
+Requires-Dist: apache-airflow-providers-common-compat ; extra == "common.compat"
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Requires-Dist: apache-airflow-providers-exasol ; extra == "exasol"
 Requires-Dist: apache-airflow-providers-ftp ; extra == "ftp"
@@ -53,8 +56,8 @@ Requires-Dist: s3fs>=2023.10.0 ; extra == "s3fs"
 Requires-Dist: apache-airflow-providers-salesforce ; extra == "salesforce"
 Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.26.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.26.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.27.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.27.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -62,6 +65,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 Provides-Extra: aiobotocore
 Provides-Extra: apache.hive
 Provides-Extra: cncf.kubernetes
+Provides-Extra: common.compat
 Provides-Extra: common.sql
 Provides-Extra: exasol
 Provides-Extra: ftp
@@ -121,7 +125,7 @@ Provides-Extra: ssh
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``8.26.0.rc1``
+Release: ``8.27.0``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -134,7 +138,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.26.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.27.0/>`_.
 
 Installation
 ------------
@@ -148,23 +152,25 @@ The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================================  ==================
-PIP package                              Version required
-=======================================  ==================
-``apache-airflow``                       ``>=2.7.0``
-``apache-airflow-providers-common-sql``  ``>=1.3.1``
+==========================================  ==================
+PIP package                                 Version required
+==========================================  ==================
+``apache-airflow``                          ``>=2.7.0``
+``apache-airflow-providers-common-compat``  ``>=1.1.0``
+``apache-airflow-providers-common-sql``     ``>=1.3.1``
 ``apache-airflow-providers-http``
-``boto3``                                ``>=1.34.90``
-``botocore``                             ``>=1.34.90``
-``inflection``                           ``>=0.5.1``
-``watchtower``                           ``>=3.0.0,<4``
-``jsonpath_ng``                          ``>=1.5.3``
-``redshift_connector``                   ``>=2.0.918``
-``sqlalchemy_redshift``                  ``>=0.8.6``
-``asgiref``                              ``>=2.3.0``
-``PyAthena``                             ``>=3.0.10``
-``jmespath``                             ``>=0.7.0``
-=======================================  ==================
+``apache-airflow-providers-common-compat``  ``>=1.1.0``
+``boto3``                                   ``>=1.34.90``
+``botocore``                                ``>=1.34.90``
+``inflection``                              ``>=0.5.1``
+``watchtower``                              ``>=3.0.0,<4``
+``jsonpath_ng``                             ``>=1.5.3``
+``redshift_connector``                      ``>=2.0.918``
+``sqlalchemy_redshift``                     ``>=0.8.6``
+``asgiref``                                 ``>=2.3.0``
+``PyAthena``                                ``>=3.0.10``
+``jmespath``                                ``>=0.7.0``
+==========================================  ==================
 
 Cross provider package dependencies
 -----------------------------------
@@ -184,6 +190,7 @@ Dependent package
 ======================================================================================================================  ===================
 `apache-airflow-providers-apache-hive <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive>`_          ``apache.hive``
 `apache-airflow-providers-cncf-kubernetes <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes>`_  ``cncf.kubernetes``
+`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_      ``common.compat``
 `apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_            ``common.sql``
 `apache-airflow-providers-exasol <https://airflow.apache.org/docs/apache-airflow-providers-exasol>`_                    ``exasol``
 `apache-airflow-providers-ftp <https://airflow.apache.org/docs/apache-airflow-providers-ftp>`_                          ``ftp``
@@ -198,4 +205,4 @@ Dependent package
 ======================================================================================================================  ===================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.26.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.27.0/changelog.html>`_.

{apache_airflow_providers_amazon-8.26.0rc1 → apache_airflow_providers_amazon-8.27.0}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``8.26.0.rc1``
+Release: ``8.27.0``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -55,7 +55,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.26.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.27.0/>`_.
 
 Installation
 ------------
@@ -69,23 +69,25 @@ The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================================  ==================
-PIP package                              Version required
-=======================================  ==================
-``apache-airflow``                       ``>=2.7.0``
-``apache-airflow-providers-common-sql``  ``>=1.3.1``
+==========================================  ==================
+PIP package                                 Version required
+==========================================  ==================
+``apache-airflow``                          ``>=2.7.0``
+``apache-airflow-providers-common-compat``  ``>=1.1.0``
+``apache-airflow-providers-common-sql``     ``>=1.3.1``
 ``apache-airflow-providers-http``
-``boto3``                                ``>=1.34.90``
-``botocore``                             ``>=1.34.90``
-``inflection``                           ``>=0.5.1``
-``watchtower``                           ``>=3.0.0,<4``
-``jsonpath_ng``                          ``>=1.5.3``
-``redshift_connector``                   ``>=2.0.918``
-``sqlalchemy_redshift``                  ``>=0.8.6``
-``asgiref``                              ``>=2.3.0``
-``PyAthena``                             ``>=3.0.10``
-``jmespath``                             ``>=0.7.0``
-=======================================  ==================
+``apache-airflow-providers-common-compat``  ``>=1.1.0``
+``boto3``                                   ``>=1.34.90``
+``botocore``                                ``>=1.34.90``
+``inflection``                              ``>=0.5.1``
+``watchtower``                              ``>=3.0.0,<4``
+``jsonpath_ng``                             ``>=1.5.3``
+``redshift_connector``                      ``>=2.0.918``
+``sqlalchemy_redshift``                     ``>=0.8.6``
+``asgiref``                                 ``>=2.3.0``
+``PyAthena``                                ``>=3.0.10``
+``jmespath``                                ``>=0.7.0``
+==========================================  ==================
 
 Cross provider package dependencies
 -----------------------------------
@@ -105,6 +107,7 @@ Dependent package
 ======================================================================================================================  ===================
 `apache-airflow-providers-apache-hive <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive>`_          ``apache.hive``
 `apache-airflow-providers-cncf-kubernetes <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes>`_  ``cncf.kubernetes``
+`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_      ``common.compat``
 `apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_            ``common.sql``
 `apache-airflow-providers-exasol <https://airflow.apache.org/docs/apache-airflow-providers-exasol>`_                    ``exasol``
 `apache-airflow-providers-ftp <https://airflow.apache.org/docs/apache-airflow-providers-ftp>`_                          ``ftp``
@@ -119,4 +122,4 @@ Dependent package
 ======================================================================================================================  ===================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.26.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.27.0/changelog.html>`_.

{apache_airflow_providers_amazon-8.26.0rc1 → apache_airflow_providers_amazon-8.27.0}/airflow/providers/amazon/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "8.26.0"
+__version__ = "8.27.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.7.0"

{apache_airflow_providers_amazon-8.26.0rc1 → apache_airflow_providers_amazon-8.27.0}/airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py

@@ -81,6 +81,16 @@ class AwsAuthManager(BaseAuthManager):
     """
 
     def __init__(self, appbuilder: AirflowAppBuilder) -> None:
+        from packaging.version import Version
+
+        from airflow.version import version
+
+        # TODO: remove this if block when min_airflow_version is set to higher than 2.9.0
+        if Version(version) < Version("2.9"):
+            raise AirflowOptionalProviderFeatureException(
+                "``AwsAuthManager`` is compatible with Airflow versions >= 2.9."
+            )
+
         super().__init__(appbuilder)
         self._check_avp_schema_version()
 

apache_airflow_providers_amazon-8.27.0/airflow/providers/amazon/aws/datasets/s3.py

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from airflow.datasets import Dataset
+from airflow.providers.amazon.aws.hooks.s3 import S3Hook
+
+if TYPE_CHECKING:
+    from urllib.parse import SplitResult
+
+    from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset
+
+
+def create_dataset(*, bucket: str, key: str, extra=None) -> Dataset:
+    return Dataset(uri=f"s3://{bucket}/{key}", extra=extra)
+
+
+def sanitize_uri(uri: SplitResult) -> SplitResult:
+    if not uri.netloc:
+        raise ValueError("URI format s3:// must contain a bucket name")
+    return uri
+
+
+def convert_dataset_to_openlineage(dataset: Dataset, lineage_context) -> OpenLineageDataset:
+    """Translate Dataset with valid AIP-60 uri to OpenLineage with assistance from the hook."""
+    from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset
+
+    bucket, key = S3Hook.parse_s3_url(dataset.uri)
+    return OpenLineageDataset(namespace=f"s3://{bucket}", name=key if key else "/")

{apache_airflow_providers_amazon-8.26.0rc1 → apache_airflow_providers_amazon-8.27.0}/airflow/providers/amazon/aws/executors/batch/batch_executor.py

@@ -19,9 +19,9 @@
 
 from __future__ import annotations
 
-import contextlib
 import time
-from collections import defaultdict, deque
+from collections import deque
+from contextlib import suppress
 from copy import deepcopy
 from typing import TYPE_CHECKING, Any, Dict, List, Sequence
 
@@ -264,7 +264,6 @@ class AwsBatchExecutor(BaseExecutor):
         in the next iteration of the sync() method, unless it has exceeded the maximum number of
         attempts. If a job exceeds the maximum number of attempts, it is removed from the queue.
         """
-        failure_reasons = defaultdict(int)
         for _ in range(len(self.pending_jobs)):
             batch_job = self.pending_jobs.popleft()
             key = batch_job.key
@@ -272,7 +271,7 @@ class AwsBatchExecutor(BaseExecutor):
             queue = batch_job.queue
             exec_config = batch_job.executor_config
             attempt_number = batch_job.attempt_number
-            _failure_reason = []
+            failure_reason: str | None = None
             if timezone.utcnow() < batch_job.next_attempt_time:
                 self.pending_jobs.append(batch_job)
                 continue
@@ -286,18 +285,25 @@ class AwsBatchExecutor(BaseExecutor):
                 if error_code in INVALID_CREDENTIALS_EXCEPTIONS:
                     self.pending_jobs.append(batch_job)
                     raise
-                _failure_reason.append(str(e))
+                failure_reason = str(e)
             except Exception as e:
-                _failure_reason.append(str(e))
-
-            if _failure_reason:
-                for reason in _failure_reason:
-                    failure_reasons[reason] += 1
+                failure_reason = str(e)
 
+            if failure_reason:
                 if attempt_number >= int(self.__class__.MAX_SUBMIT_JOB_ATTEMPTS):
                     self.log.error(
-                        "This job has been unsuccessfully attempted too many times (%s). Dropping the task.",
+                        (
+                            "This job has been unsuccessfully attempted too many times (%s). "
+                            "Dropping the task. Reason: %s"
+                        ),
                         attempt_number,
+                        failure_reason,
+                    )
+                    self.log_task_event(
+                        event="batch job submit failure",
+                        extra=f"This job has been unsuccessfully attempted too many times ({attempt_number}). "
+                        f"Dropping the task. Reason: {failure_reason}",
+                        ti_key=key,
                     )
                     self.fail(key=key)
                 else:
@@ -317,16 +323,11 @@ class AwsBatchExecutor(BaseExecutor):
                     exec_config=exec_config,
                     attempt_number=attempt_number,
                 )
-                with contextlib.suppress(AttributeError):
+                with suppress(AttributeError):
                     # TODO: Remove this when min_airflow_version is 2.10.0 or higher in Amazon provider.
                     # running_state is added in Airflow 2.10 and only needed to support task adoption
                     # (an optional executor feature).
                     self.running_state(key, job_id)
-        if failure_reasons:
-            self.log.error(
-                "Pending Batch jobs failed to launch for the following reasons: %s. Retrying later.",
-                dict(failure_reasons),
-            )
 
     def _describe_jobs(self, job_ids) -> list[BatchJob]:
         all_jobs = []
@@ -462,3 +463,12 @@ class AwsBatchExecutor(BaseExecutor):
 
             not_adopted_tis = [ti for ti in tis if ti not in adopted_tis]
             return not_adopted_tis
+
+    def log_task_event(self, *, event: str, extra: str, ti_key: TaskInstanceKey):
+        # TODO: remove this method when min_airflow_version is set to higher than 2.10.0
+        with suppress(AttributeError):
+            super().log_task_event(
+                event=event,
+                extra=extra,
+                ti_key=ti_key,
+            )

{apache_airflow_providers_amazon-8.26.0rc1 → apache_airflow_providers_amazon-8.27.0}/airflow/providers/amazon/aws/executors/ecs/ecs_executor.py

@@ -25,6 +25,7 @@ from __future__ import annotations
 
 import time
 from collections import defaultdict, deque
+from contextlib import suppress
 from copy import deepcopy
 from typing import TYPE_CHECKING, Sequence
 
@@ -347,7 +348,7 @@ class AwsEcsExecutor(BaseExecutor):
             queue = ecs_task.queue
             exec_config = ecs_task.executor_config
             attempt_number = ecs_task.attempt_number
-            _failure_reasons = []
+            failure_reasons = []
             if timezone.utcnow() < ecs_task.next_attempt_time:
                 self.pending_tasks.append(ecs_task)
                 continue
@@ -361,23 +362,21 @@ class AwsEcsExecutor(BaseExecutor):
                 if error_code in INVALID_CREDENTIALS_EXCEPTIONS:
                     self.pending_tasks.append(ecs_task)
                     raise
-                _failure_reasons.append(str(e))
+                failure_reasons.append(str(e))
             except Exception as e:
                 # Failed to even get a response back from the Boto3 API or something else went
                 # wrong.  For any possible failure we want to add the exception reasons to the
                 # failure list so that it is logged to the user and most importantly the task is
                 # added back to the pending list to be retried later.
-                _failure_reasons.append(str(e))
+                failure_reasons.append(str(e))
             else:
                 # We got a response back, check if there were failures. If so, add them to the
                 # failures list so that it is logged to the user and most importantly the task
                 # is added back to the pending list to be retried later.
                 if run_task_response["failures"]:
-                    _failure_reasons.extend([f["reason"] for f in run_task_response["failures"]])
+                    failure_reasons.extend([f["reason"] for f in run_task_response["failures"]])
 
-            if _failure_reasons:
-                for reason in _failure_reasons:
-                    failure_reasons[reason] += 1
+            if failure_reasons:
                 # Make sure the number of attempts does not exceed MAX_RUN_TASK_ATTEMPTS
                 if int(attempt_number) < int(self.__class__.MAX_RUN_TASK_ATTEMPTS):
                     ecs_task.attempt_number += 1
@@ -386,14 +385,29 @@ class AwsEcsExecutor(BaseExecutor):
                     )
                     self.pending_tasks.append(ecs_task)
                 else:
+                    reasons_str = ", ".join(failure_reasons)
                     self.log.error(
-                        "ECS task %s has failed a maximum of %s times. Marking as failed",
+                        "ECS task %s has failed a maximum of %s times. Marking as failed. Reasons: %s",
                         task_key,
                         attempt_number,
+                        reasons_str,
+                    )
+                    self.log_task_event(
+                        event="ecs task submit failure",
+                        ti_key=task_key,
+                        extra=(
+                            f"Task could not be queued after {attempt_number} attempts. "
+                            f"Marking as failed. Reasons: {reasons_str}"
+                        ),
                     )
                     self.fail(task_key)
             elif not run_task_response["tasks"]:
                 self.log.error("ECS RunTask Response: %s", run_task_response)
+                self.log_task_event(
+                    event="ecs task submit failure",
+                    extra=f"ECS RunTask Response: {run_task_response}",
+                    ti_key=task_key,
+                )
                 raise EcsExecutorException(
                     "No failures and no ECS tasks provided in response. This should never happen."
                 )
@@ -407,11 +421,6 @@ class AwsEcsExecutor(BaseExecutor):
                     # executor feature).
                     # TODO: remove when min airflow version >= 2.9.2
                     pass
-        if failure_reasons:
-            self.log.error(
-                "Pending ECS tasks failed to launch for the following reasons: %s. Retrying later.",
-                dict(failure_reasons),
-            )
 
     def _run_task(
         self, task_id: TaskInstanceKey, cmd: CommandType, queue: str, exec_config: ExecutorConfigType
@@ -543,3 +552,12 @@ class AwsEcsExecutor(BaseExecutor):
 
             not_adopted_tis = [ti for ti in tis if ti not in adopted_tis]
             return not_adopted_tis
+
+    def log_task_event(self, *, event: str, extra: str, ti_key: TaskInstanceKey):
+        # TODO: remove this method when min_airflow_version is set to higher than 2.10.0
+        with suppress(AttributeError):
+            super().log_task_event(
+                event=event,
+                extra=extra,
+                ti_key=ti_key,
+            )

apache_airflow_providers_amazon-8.27.0/airflow/providers/amazon/aws/hooks/kinesis_analytics.py

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
+
+
+class KinesisAnalyticsV2Hook(AwsBaseHook):
+    """
+    Interact with Amazon Kinesis Analytics V2.
+
+    Provide thin wrapper around :external+boto3:py:class:`boto3.client("kinesisanalyticsv2") <KinesisAnalyticsV2.Client>`.
+
+    Additional arguments (such as ``aws_conn_id``) may be specified and
+    are passed down to the underlying AwsBaseHook.
+
+    .. seealso::
+        - :class:`airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook`
+    """
+
+    APPLICATION_START_INTERMEDIATE_STATES: tuple[str, ...] = ("STARTING", "UPDATING", "AUTOSCALING")
+    APPLICATION_START_FAILURE_STATES: tuple[str, ...] = (
+        "DELETING",
+        "STOPPING",
+        "READY",
+        "FORCE_STOPPING",
+        "ROLLING_BACK",
+        "MAINTENANCE",
+        "ROLLED_BACK",
+    )
+    APPLICATION_START_SUCCESS_STATES: tuple[str, ...] = ("RUNNING",)
+
+    APPLICATION_STOP_INTERMEDIATE_STATES: tuple[str, ...] = (
+        "STARTING",
+        "UPDATING",
+        "AUTOSCALING",
+        "RUNNING",
+        "STOPPING",
+        "FORCE_STOPPING",
+    )
+    APPLICATION_STOP_FAILURE_STATES: tuple[str, ...] = (
+        "DELETING",
+        "ROLLING_BACK",
+        "MAINTENANCE",
+        "ROLLED_BACK",
+    )
+    APPLICATION_STOP_SUCCESS_STATES: tuple[str, ...] = ("READY",)
+
+    def __init__(self, *args, **kwargs) -> None:
+        kwargs["client_type"] = "kinesisanalyticsv2"
+        super().__init__(*args, **kwargs)

{apache_airflow_providers_amazon-8.26.0rc1 → apache_airflow_providers_amazon-8.27.0}/airflow/providers/amazon/aws/hooks/rds.py

@@ -259,7 +259,7 @@ class RdsHook(AwsGenericHook["RDSClient"]):
             return self.get_db_instance_state(db_instance_id)
 
         target_state = target_state.lower()
-        if target_state in ("available", "deleted"):
+        if target_state in ("available", "deleted", "stopped"):
             waiter = self.conn.get_waiter(f"db_instance_{target_state}")  # type: ignore
             wait(
                 waiter=waiter,
@@ -272,7 +272,7 @@ class RdsHook(AwsGenericHook["RDSClient"]):
             )
         else:
             self._wait_for_state(poke, target_state, check_interval, max_attempts)
-            self.log.info("DB cluster snapshot '%s' reached the '%s' state", db_instance_id, target_state)
+            self.log.info("DB cluster '%s' reached the '%s' state", db_instance_id, target_state)
 
     def get_db_cluster_state(self, db_cluster_id: str) -> str:
         """
@@ -310,7 +310,7 @@ class RdsHook(AwsGenericHook["RDSClient"]):
             return self.get_db_cluster_state(db_cluster_id)
 
         target_state = target_state.lower()
-        if target_state in ("available", "deleted"):
+        if target_state in ("available", "deleted", "stopped"):
             waiter = self.conn.get_waiter(f"db_cluster_{target_state}")  # type: ignore
             waiter.wait(
                 DBClusterIdentifier=db_cluster_id,

{apache_airflow_providers_amazon-8.26.0rc1 → apache_airflow_providers_amazon-8.27.0}/airflow/providers/amazon/aws/hooks/s3.py

@@ -41,6 +41,8 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Callable
 from urllib.parse import urlsplit
 from uuid import uuid4
 
+from airflow.providers.common.compat.lineage.hook import get_hook_lineage_collector
+
 if TYPE_CHECKING:
     from mypy_boto3_s3.service_resource import Bucket as S3Bucket, Object as S3ResourceObject
 
@@ -1111,6 +1113,12 @@ class S3Hook(AwsBaseHook):
 
         client = self.get_conn()
         client.upload_file(filename, bucket_name, key, ExtraArgs=extra_args, Config=self.transfer_config)
+        get_hook_lineage_collector().add_input_dataset(
+            context=self, scheme="file", dataset_kwargs={"path": filename}
+        )
+        get_hook_lineage_collector().add_output_dataset(
+            context=self, scheme="s3", dataset_kwargs={"bucket": bucket_name, "key": key}
+        )
 
     @unify_bucket_name_and_key
     @provide_bucket_name
@@ -1251,6 +1259,10 @@ class S3Hook(AwsBaseHook):
             ExtraArgs=extra_args,
             Config=self.transfer_config,
         )
+        # No input because file_obj can be anything - handle in calling function if possible
+        get_hook_lineage_collector().add_output_dataset(
+            context=self, scheme="s3", dataset_kwargs={"bucket": bucket_name, "key": key}
+        )
 
     def copy_object(
         self,
@@ -1306,6 +1318,12 @@ class S3Hook(AwsBaseHook):
         response = self.get_conn().copy_object(
             Bucket=dest_bucket_name, Key=dest_bucket_key, CopySource=copy_source, **kwargs
         )
+        get_hook_lineage_collector().add_input_dataset(
+            context=self, scheme="s3", dataset_kwargs={"bucket": source_bucket_name, "key": source_bucket_key}
+        )
+        get_hook_lineage_collector().add_output_dataset(
+            context=self, scheme="s3", dataset_kwargs={"bucket": dest_bucket_name, "key": dest_bucket_key}
+        )
         return response
 
     @provide_bucket_name
@@ -1425,6 +1443,11 @@ class S3Hook(AwsBaseHook):
 
             file_path.parent.mkdir(exist_ok=True, parents=True)
 
+            get_hook_lineage_collector().add_output_dataset(
+                context=self,
+                scheme="file",
+                dataset_kwargs={"path": file_path if file_path.is_absolute() else file_path.absolute()},
+            )
             file = open(file_path, "wb")
         else:
             file = NamedTemporaryFile(dir=local_path, prefix="airflow_tmp_", delete=False)  # type: ignore
@@ -1435,7 +1458,9 @@ class S3Hook(AwsBaseHook):
                 ExtraArgs=self.extra_args,
                 Config=self.transfer_config,
             )
-
+        get_hook_lineage_collector().add_input_dataset(
+            context=self, scheme="s3", dataset_kwargs={"bucket": bucket_name, "key": key}
+        )
         return file.name
 
     def generate_presigned_url(