apache-airflow-providers-amazon 8.26.0rc2__py3-none-any.whl → 8.27.0__py3-none-any.whl

airflow/providers/amazon/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "8.26.0"
+__version__ = "8.27.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.7.0"

airflow/providers/amazon/aws/datasets/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/amazon/aws/datasets/s3.py

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from airflow.datasets import Dataset
+from airflow.providers.amazon.aws.hooks.s3 import S3Hook
+
+if TYPE_CHECKING:
+    from urllib.parse import SplitResult
+
+    from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset
+
+
+def create_dataset(*, bucket: str, key: str, extra=None) -> Dataset:
+    return Dataset(uri=f"s3://{bucket}/{key}", extra=extra)
+
+
+def sanitize_uri(uri: SplitResult) -> SplitResult:
+    if not uri.netloc:
+        raise ValueError("URI format s3:// must contain a bucket name")
+    return uri
+
+
+def convert_dataset_to_openlineage(dataset: Dataset, lineage_context) -> OpenLineageDataset:
+    """Translate Dataset with valid AIP-60 uri to OpenLineage with assistance from the hook."""
+    from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset
+
+    bucket, key = S3Hook.parse_s3_url(dataset.uri)
+    return OpenLineageDataset(namespace=f"s3://{bucket}", name=key if key else "/")

airflow/providers/amazon/aws/executors/batch/batch_executor.py

@@ -19,10 +19,9 @@
 
 from __future__ import annotations
 
-import contextlib
-import logging
 import time
 from collections import deque
+from contextlib import suppress
 from copy import deepcopy
 from typing import TYPE_CHECKING, Any, Dict, List, Sequence
 
@@ -292,12 +291,19 @@ class AwsBatchExecutor(BaseExecutor):
 
             if failure_reason:
                 if attempt_number >= int(self.__class__.MAX_SUBMIT_JOB_ATTEMPTS):
-                    self.send_message_to_task_logs(
-                        logging.ERROR,
-                        "This job has been unsuccessfully attempted too many times (%s). Dropping the task. Reason: %s",
+                    self.log.error(
+                        (
+                            "This job has been unsuccessfully attempted too many times (%s). "
+                            "Dropping the task. Reason: %s"
+                        ),
                         attempt_number,
                         failure_reason,
-                        ti=key,
+                    )
+                    self.log_task_event(
+                        event="batch job submit failure",
+                        extra=f"This job has been unsuccessfully attempted too many times ({attempt_number}). "
+                        f"Dropping the task. Reason: {failure_reason}",
+                        ti_key=key,
                     )
                     self.fail(key=key)
                 else:
@@ -317,7 +323,7 @@ class AwsBatchExecutor(BaseExecutor):
                     exec_config=exec_config,
                     attempt_number=attempt_number,
                 )
-                with contextlib.suppress(AttributeError):
+                with suppress(AttributeError):
                     # TODO: Remove this when min_airflow_version is 2.10.0 or higher in Amazon provider.
                     # running_state is added in Airflow 2.10 and only needed to support task adoption
                     # (an optional executor feature).
@@ -458,10 +464,11 @@ class AwsBatchExecutor(BaseExecutor):
             not_adopted_tis = [ti for ti in tis if ti not in adopted_tis]
             return not_adopted_tis
 
-    def send_message_to_task_logs(self, level: int, msg: str, *args, ti: TaskInstance | TaskInstanceKey):
+    def log_task_event(self, *, event: str, extra: str, ti_key: TaskInstanceKey):
         # TODO: remove this method when min_airflow_version is set to higher than 2.10.0
-        try:
-            super().send_message_to_task_logs(level, msg, *args, ti=ti)
-        except AttributeError:
-            # ``send_message_to_task_logs`` is added in 2.10.0
-            self.log.error(msg, *args)
+        with suppress(AttributeError):
+            super().log_task_event(
+                event=event,
+                extra=extra,
+                ti_key=ti_key,
+            )

airflow/providers/amazon/aws/executors/ecs/ecs_executor.py

@@ -23,9 +23,9 @@ Each Airflow task gets delegated out to an Amazon ECS Task.
 
 from __future__ import annotations
 
-import logging
 import time
 from collections import defaultdict, deque
+from contextlib import suppress
 from copy import deepcopy
 from typing import TYPE_CHECKING, Sequence
 
@@ -385,18 +385,28 @@ class AwsEcsExecutor(BaseExecutor):
                     )
                     self.pending_tasks.append(ecs_task)
                 else:
-                    self.send_message_to_task_logs(
-                        logging.ERROR,
+                    reasons_str = ", ".join(failure_reasons)
+                    self.log.error(
                         "ECS task %s has failed a maximum of %s times. Marking as failed. Reasons: %s",
                         task_key,
                         attempt_number,
-                        ", ".join(failure_reasons),
-                        ti=task_key,
+                        reasons_str,
+                    )
+                    self.log_task_event(
+                        event="ecs task submit failure",
+                        ti_key=task_key,
+                        extra=(
+                            f"Task could not be queued after {attempt_number} attempts. "
+                            f"Marking as failed. Reasons: {reasons_str}"
+                        ),
                     )
                     self.fail(task_key)
             elif not run_task_response["tasks"]:
-                self.send_message_to_task_logs(
-                    logging.ERROR, "ECS RunTask Response: %s", run_task_response, ti=task_key
+                self.log.error("ECS RunTask Response: %s", run_task_response)
+                self.log_task_event(
+                    event="ecs task submit failure",
+                    extra=f"ECS RunTask Response: {run_task_response}",
+                    ti_key=task_key,
                 )
                 raise EcsExecutorException(
                     "No failures and no ECS tasks provided in response. This should never happen."
@@ -543,10 +553,11 @@ class AwsEcsExecutor(BaseExecutor):
             not_adopted_tis = [ti for ti in tis if ti not in adopted_tis]
             return not_adopted_tis
 
-    def send_message_to_task_logs(self, level: int, msg: str, *args, ti: TaskInstance | TaskInstanceKey):
+    def log_task_event(self, *, event: str, extra: str, ti_key: TaskInstanceKey):
         # TODO: remove this method when min_airflow_version is set to higher than 2.10.0
-        try:
-            super().send_message_to_task_logs(level, msg, *args, ti=ti)
-        except AttributeError:
-            # ``send_message_to_task_logs`` is added in 2.10.0
-            self.log.error(msg, *args)
+        with suppress(AttributeError):
+            super().log_task_event(
+                event=event,
+                extra=extra,
+                ti_key=ti_key,
+            )

airflow/providers/amazon/aws/hooks/kinesis_analytics.py

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
+
+
+class KinesisAnalyticsV2Hook(AwsBaseHook):
+    """
+    Interact with Amazon Kinesis Analytics V2.
+
+    Provide thin wrapper around :external+boto3:py:class:`boto3.client("kinesisanalyticsv2") <KinesisAnalyticsV2.Client>`.
+
+    Additional arguments (such as ``aws_conn_id``) may be specified and
+    are passed down to the underlying AwsBaseHook.
+
+    .. seealso::
+        - :class:`airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook`
+    """
+
+    APPLICATION_START_INTERMEDIATE_STATES: tuple[str, ...] = ("STARTING", "UPDATING", "AUTOSCALING")
+    APPLICATION_START_FAILURE_STATES: tuple[str, ...] = (
+        "DELETING",
+        "STOPPING",
+        "READY",
+        "FORCE_STOPPING",
+        "ROLLING_BACK",
+        "MAINTENANCE",
+        "ROLLED_BACK",
+    )
+    APPLICATION_START_SUCCESS_STATES: tuple[str, ...] = ("RUNNING",)
+
+    APPLICATION_STOP_INTERMEDIATE_STATES: tuple[str, ...] = (
+        "STARTING",
+        "UPDATING",
+        "AUTOSCALING",
+        "RUNNING",
+        "STOPPING",
+        "FORCE_STOPPING",
+    )
+    APPLICATION_STOP_FAILURE_STATES: tuple[str, ...] = (
+        "DELETING",
+        "ROLLING_BACK",
+        "MAINTENANCE",
+        "ROLLED_BACK",
+    )
+    APPLICATION_STOP_SUCCESS_STATES: tuple[str, ...] = ("READY",)
+
+    def __init__(self, *args, **kwargs) -> None:
+        kwargs["client_type"] = "kinesisanalyticsv2"
+        super().__init__(*args, **kwargs)

airflow/providers/amazon/aws/hooks/rds.py

@@ -259,7 +259,7 @@ class RdsHook(AwsGenericHook["RDSClient"]):
             return self.get_db_instance_state(db_instance_id)
 
         target_state = target_state.lower()
-        if target_state in ("available", "deleted"):
+        if target_state in ("available", "deleted", "stopped"):
             waiter = self.conn.get_waiter(f"db_instance_{target_state}")  # type: ignore
             wait(
                 waiter=waiter,
@@ -272,7 +272,7 @@ class RdsHook(AwsGenericHook["RDSClient"]):
             )
         else:
             self._wait_for_state(poke, target_state, check_interval, max_attempts)
-            self.log.info("DB cluster snapshot '%s' reached the '%s' state", db_instance_id, target_state)
+            self.log.info("DB cluster '%s' reached the '%s' state", db_instance_id, target_state)
 
     def get_db_cluster_state(self, db_cluster_id: str) -> str:
         """
@@ -310,7 +310,7 @@ class RdsHook(AwsGenericHook["RDSClient"]):
             return self.get_db_cluster_state(db_cluster_id)
 
         target_state = target_state.lower()
-        if target_state in ("available", "deleted"):
+        if target_state in ("available", "deleted", "stopped"):
             waiter = self.conn.get_waiter(f"db_cluster_{target_state}")  # type: ignore
             waiter.wait(
                 DBClusterIdentifier=db_cluster_id,

airflow/providers/amazon/aws/hooks/s3.py

@@ -41,6 +41,8 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Callable
 from urllib.parse import urlsplit
 from uuid import uuid4
 
+from airflow.providers.common.compat.lineage.hook import get_hook_lineage_collector
+
 if TYPE_CHECKING:
     from mypy_boto3_s3.service_resource import Bucket as S3Bucket, Object as S3ResourceObject
 
@@ -1111,6 +1113,12 @@ class S3Hook(AwsBaseHook):
 
         client = self.get_conn()
         client.upload_file(filename, bucket_name, key, ExtraArgs=extra_args, Config=self.transfer_config)
+        get_hook_lineage_collector().add_input_dataset(
+            context=self, scheme="file", dataset_kwargs={"path": filename}
+        )
+        get_hook_lineage_collector().add_output_dataset(
+            context=self, scheme="s3", dataset_kwargs={"bucket": bucket_name, "key": key}
+        )
 
     @unify_bucket_name_and_key
     @provide_bucket_name
@@ -1251,6 +1259,10 @@ class S3Hook(AwsBaseHook):
             ExtraArgs=extra_args,
             Config=self.transfer_config,
         )
+        # No input because file_obj can be anything - handle in calling function if possible
+        get_hook_lineage_collector().add_output_dataset(
+            context=self, scheme="s3", dataset_kwargs={"bucket": bucket_name, "key": key}
+        )
 
     def copy_object(
         self,
@@ -1306,6 +1318,12 @@ class S3Hook(AwsBaseHook):
         response = self.get_conn().copy_object(
             Bucket=dest_bucket_name, Key=dest_bucket_key, CopySource=copy_source, **kwargs
         )
+        get_hook_lineage_collector().add_input_dataset(
+            context=self, scheme="s3", dataset_kwargs={"bucket": source_bucket_name, "key": source_bucket_key}
+        )
+        get_hook_lineage_collector().add_output_dataset(
+            context=self, scheme="s3", dataset_kwargs={"bucket": dest_bucket_name, "key": dest_bucket_key}
+        )
         return response
 
     @provide_bucket_name
@@ -1425,6 +1443,11 @@ class S3Hook(AwsBaseHook):
 
             file_path.parent.mkdir(exist_ok=True, parents=True)
 
+            get_hook_lineage_collector().add_output_dataset(
+                context=self,
+                scheme="file",
+                dataset_kwargs={"path": file_path if file_path.is_absolute() else file_path.absolute()},
+            )
             file = open(file_path, "wb")
         else:
             file = NamedTemporaryFile(dir=local_path, prefix="airflow_tmp_", delete=False)  # type: ignore
@@ -1435,7 +1458,9 @@ class S3Hook(AwsBaseHook):
                 ExtraArgs=self.extra_args,
                 Config=self.transfer_config,
             )
-
+        get_hook_lineage_collector().add_input_dataset(
+            context=self, scheme="s3", dataset_kwargs={"bucket": bucket_name, "key": key}
+        )
         return file.name
 
     def generate_presigned_url(

airflow/providers/amazon/aws/hooks/step_function.py

@@ -18,6 +18,7 @@ from __future__ import annotations
 
 import json
 
+from airflow.exceptions import AirflowFailException
 from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
 
 
@@ -43,6 +44,7 @@ class StepFunctionHook(AwsBaseHook):
         state_machine_arn: str,
         name: str | None = None,
         state_machine_input: dict | str | None = None,
+        is_redrive_execution: bool = False,
     ) -> str:
         """
         Start Execution of the State Machine.
@@ -51,10 +53,26 @@ class StepFunctionHook(AwsBaseHook):
             - :external+boto3:py:meth:`SFN.Client.start_execution`
 
         :param state_machine_arn: AWS Step Function State Machine ARN.
+        :param is_redrive_execution: Restarts unsuccessful executions of Standard workflows that did not
+            complete successfully in the last 14 days.
         :param name: The name of the execution.
         :param state_machine_input: JSON data input to pass to the State Machine.
         :return: Execution ARN.
         """
+        if is_redrive_execution:
+            if not name:
+                raise AirflowFailException(
+                    "Execution name is required to start RedriveExecution for %s.", state_machine_arn
+                )
+            elements = state_machine_arn.split(":stateMachine:")
+            execution_arn = f"{elements[0]}:execution:{elements[1]}:{name}"
+            self.conn.redrive_execution(executionArn=execution_arn)
+            self.log.info(
+                "Successfully started RedriveExecution for Step Function State Machine: %s.",
+                state_machine_arn,
+            )
+            return execution_arn
+
         execution_args = {"stateMachineArn": state_machine_arn}
         if name is not None:
             execution_args["name"] = name

airflow/providers/amazon/aws/operators/athena.py

@@ -30,9 +30,7 @@ from airflow.providers.amazon.aws.utils import validate_execute_complete_event
 from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
 
 if TYPE_CHECKING:
-    from openlineage.client.facet import BaseFacet
-    from openlineage.client.run import Dataset
-
+    from airflow.providers.common.compat.openlineage.facet import BaseFacet, Dataset, DatasetFacet
     from airflow.providers.openlineage.extractors.base import OperatorLineage
     from airflow.utils.context import Context
 
@@ -217,20 +215,19 @@ class AthenaOperator(AwsBaseOperator[AthenaHook]):
         path where the results are saved (user's prefix + some UUID), we are creating a dataset with the
         user-provided path only. This should make it easier to match this dataset across different processes.
         """
-        from openlineage.client.facet import (
+        from airflow.providers.common.compat.openlineage.facet import (
+            Dataset,
+            Error,
             ExternalQueryRunFacet,
-            ExtractionError,
             ExtractionErrorRunFacet,
-            SqlJobFacet,
+            SQLJobFacet,
         )
-        from openlineage.client.run import Dataset
-
         from airflow.providers.openlineage.extractors.base import OperatorLineage
         from airflow.providers.openlineage.sqlparser import SQLParser
 
         sql_parser = SQLParser(dialect="generic")
 
-        job_facets: dict[str, BaseFacet] = {"sql": SqlJobFacet(query=sql_parser.normalize_sql(self.query))}
+        job_facets: dict[str, BaseFacet] = {"sql": SQLJobFacet(query=sql_parser.normalize_sql(self.query))}
         parse_result = sql_parser.parse(sql=self.query)
 
         if not parse_result:
@@ -242,7 +239,7 @@ class AthenaOperator(AwsBaseOperator[AthenaHook]):
                 totalTasks=len(self.query) if isinstance(self.query, list) else 1,
                 failedTasks=len(parse_result.errors),
                 errors=[
-                    ExtractionError(
+                    Error(
                         errorMessage=error.message,
                         stackTrace=None,
                         task=error.origin_statement,
@@ -284,13 +281,13 @@ class AthenaOperator(AwsBaseOperator[AthenaHook]):
         return OperatorLineage(job_facets=job_facets, run_facets=run_facets, inputs=inputs, outputs=outputs)
 
     def get_openlineage_dataset(self, database, table) -> Dataset | None:
-        from openlineage.client.facet import (
+        from airflow.providers.common.compat.openlineage.facet import (
+            Dataset,
+            Identifier,
             SchemaDatasetFacet,
-            SchemaField,
+            SchemaDatasetFacetFields,
             SymlinksDatasetFacet,
-            SymlinksDatasetFacetIdentifiers,
         )
-        from openlineage.client.run import Dataset
 
         client = self.hook.get_conn()
         try:
@@ -301,10 +298,10 @@ class AthenaOperator(AwsBaseOperator[AthenaHook]):
             # Dataset has also its' physical location which we can add in symlink facet.
             s3_location = table_metadata["TableMetadata"]["Parameters"]["location"]
             parsed_path = urlparse(s3_location)
-            facets: dict[str, BaseFacet] = {
+            facets: dict[str, DatasetFacet] = {
                 "symlinks": SymlinksDatasetFacet(
                     identifiers=[
-                        SymlinksDatasetFacetIdentifiers(
+                        Identifier(
                             namespace=f"{parsed_path.scheme}://{parsed_path.netloc}",
                             name=str(parsed_path.path),
                             type="TABLE",
@@ -313,7 +310,9 @@ class AthenaOperator(AwsBaseOperator[AthenaHook]):
                 )
             }
             fields = [
-                SchemaField(name=column["Name"], type=column["Type"], description=column.get("Comment"))
+                SchemaDatasetFacetFields(
+                    name=column["Name"], type=column["Type"], description=column["Comment"]
+                )
                 for column in table_metadata["TableMetadata"]["Columns"]
             ]
             if fields:

airflow/providers/amazon/aws/operators/emr.py

@@ -1382,30 +1382,30 @@ class EmrServerlessStartJobOperator(BaseOperator):
 
         self.persist_links(context)
 
-        if self.deferrable:
-            self.defer(
-                trigger=EmrServerlessStartJobTrigger(
-                    application_id=self.application_id,
-                    job_id=self.job_id,
-                    waiter_delay=self.waiter_delay,
-                    waiter_max_attempts=self.waiter_max_attempts,
-                    aws_conn_id=self.aws_conn_id,
-                ),
-                method_name="execute_complete",
-                timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay),
-            )
-
         if self.wait_for_completion:
-            waiter = self.hook.get_waiter("serverless_job_completed")
-            wait(
-                waiter=waiter,
-                waiter_max_attempts=self.waiter_max_attempts,
-                waiter_delay=self.waiter_delay,
-                args={"applicationId": self.application_id, "jobRunId": self.job_id},
-                failure_message="Serverless Job failed",
-                status_message="Serverless Job status is",
-                status_args=["jobRun.state", "jobRun.stateDetails"],
-            )
+            if self.deferrable:
+                self.defer(
+                    trigger=EmrServerlessStartJobTrigger(
+                        application_id=self.application_id,
+                        job_id=self.job_id,
+                        waiter_delay=self.waiter_delay,
+                        waiter_max_attempts=self.waiter_max_attempts,
+                        aws_conn_id=self.aws_conn_id,
+                    ),
+                    method_name="execute_complete",
+                    timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay),
+                )
+            else:
+                waiter = self.hook.get_waiter("serverless_job_completed")
+                wait(
+                    waiter=waiter,
+                    waiter_max_attempts=self.waiter_max_attempts,
+                    waiter_delay=self.waiter_delay,
+                    args={"applicationId": self.application_id, "jobRunId": self.job_id},
+                    failure_message="Serverless Job failed",
+                    status_message="Serverless Job status is",
+                    status_args=["jobRun.state", "jobRun.stateDetails"],
+                )
 
         return self.job_id