apache-airflow-providers-amazon 8.17.0rc2__py3-none-any.whl → 8.18.0rc2__py3-none-any.whl

airflow/providers/amazon/aws/operators/batch.py

@@ -44,7 +44,7 @@ from airflow.providers.amazon.aws.triggers.batch import (
     BatchCreateComputeEnvironmentTrigger,
     BatchJobTrigger,
 )
-from airflow.providers.amazon.aws.utils import trim_none_values
+from airflow.providers.amazon.aws.utils import trim_none_values, validate_execute_complete_event
 from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
 
 if TYPE_CHECKING:
@@ -269,10 +269,7 @@ class BatchOperator(BaseOperator):
         return self.job_id
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        if event is None:
-            err_msg = "Trigger error: event is None"
-            self.log.info(err_msg)
-            raise AirflowException(err_msg)
+        event = validate_execute_complete_event(event)
 
         if event["status"] != "success":
             raise AirflowException(f"Error while running job: {event}")
@@ -541,7 +538,9 @@ class BatchCreateComputeEnvironmentOperator(BaseOperator):
         self.log.info("AWS Batch compute environment created successfully")
         return arn
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error while waiting for the compute environment to be ready: {event}")
         return event["value"]

airflow/providers/amazon/aws/operators/ecs.py

@@ -21,7 +21,7 @@ import re
 import warnings
 from datetime import timedelta
 from functools import cached_property
-from typing import TYPE_CHECKING, Sequence
+from typing import TYPE_CHECKING, Any, Sequence
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
@@ -35,6 +35,7 @@ from airflow.providers.amazon.aws.triggers.ecs import (
     ClusterInactiveTrigger,
     TaskDoneTrigger,
 )
+from airflow.providers.amazon.aws.utils import validate_execute_complete_event
 from airflow.providers.amazon.aws.utils.identifiers import generate_uuid
 from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
 from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
@@ -580,7 +581,9 @@ class EcsRunTaskOperator(EcsBaseOperator):
         else:
             return None
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str | None:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error in task execution: {event}")
         self.arn = event["task_arn"]  # restore arn to its updated value, needed for next steps
@@ -596,6 +599,7 @@ class EcsRunTaskOperator(EcsBaseOperator):
             )
             if len(one_log["events"]) > 0:
                 return one_log["events"][0]["message"]
+        return None
 
     def _after_execution(self):
         self._check_success_task()

airflow/providers/amazon/aws/operators/eks.py

@@ -25,6 +25,7 @@ from functools import cached_property
 from typing import TYPE_CHECKING, Any, List, Sequence, cast
 
 from botocore.exceptions import ClientError, WaiterError
+from deprecated import deprecated
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
@@ -38,6 +39,7 @@ from airflow.providers.amazon.aws.triggers.eks import (
     EksDeleteFargateProfileTrigger,
     EksDeleteNodegroupTrigger,
 )
+from airflow.providers.amazon.aws.utils import validate_execute_complete_event
 from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
 from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction
 
@@ -263,13 +265,14 @@ class EksCreateClusterOperator(BaseOperator):
         return EksHook(aws_conn_id=self.aws_conn_id, region_name=self.region)
 
     @property
-    def eks_hook(self):
-        warnings.warn(
+    @deprecated(
+        reason=(
             "`eks_hook` property is deprecated and will be removed in the future. "
-            "Please use `hook` property instead.",
-            AirflowProviderDeprecationWarning,
-            stacklevel=2,
-        )
+            "Please use `hook` property instead."
+        ),
+        category=AirflowProviderDeprecationWarning,
+    )
+    def eks_hook(self):
         return self.hook
 
     def execute(self, context: Context):
@@ -419,11 +422,10 @@ class EksCreateClusterOperator(BaseOperator):
         raise AirflowException("Error creating cluster")
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
+        event = validate_execute_complete_event(event)
+
         resource = "fargate profile" if self.compute == "fargate" else self.compute
-        if event is None:
-            self.log.info("Trigger error: event is None")
-            raise AirflowException("Trigger error: event is None")
-        elif event["status"] != "success":
+        if event["status"] != "success":
             raise AirflowException(f"Error creating {resource}: {event}")
 
         self.log.info("%s created successfully", resource)
@@ -545,10 +547,11 @@ class EksCreateNodegroupOperator(BaseOperator):
                 timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay + 60),
             )
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error creating nodegroup: {event}")
-        return
 
 
 class EksCreateFargateProfileOperator(BaseOperator):
@@ -654,12 +657,13 @@ class EksCreateFargateProfileOperator(BaseOperator):
                 timeout=timedelta(seconds=(self.waiter_max_attempts * self.waiter_delay + 60)),
             )
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error creating Fargate profile: {event}")
-        else:
-            self.log.info("Fargate profile created successfully")
-        return
+
+        self.log.info("Fargate profile created successfully")
 
 
 class EksDeleteClusterOperator(BaseOperator):
@@ -786,10 +790,9 @@ class EksDeleteClusterOperator(BaseOperator):
         self.log.info(SUCCESS_MSG.format(compute=FARGATE_FULL_NAME))
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
-        if event is None:
-            self.log.error("Trigger error. Event is None")
-            raise AirflowException("Trigger error. Event is None")
-        elif event["status"] == "success":
+        event = validate_execute_complete_event(event)
+
+        if event["status"] == "success":
             self.log.info("Cluster deleted successfully.")
 
 
@@ -877,10 +880,11 @@ class EksDeleteNodegroupOperator(BaseOperator):
                 clusterName=self.cluster_name, nodegroupName=self.nodegroup_name
             )
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error deleting nodegroup: {event}")
-        return
 
 
 class EksDeleteFargateProfileOperator(BaseOperator):
@@ -970,12 +974,13 @@ class EksDeleteFargateProfileOperator(BaseOperator):
                 WaiterConfig={"Delay": self.waiter_delay, "MaxAttempts": self.waiter_max_attempts},
             )
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error deleting Fargate profile: {event}")
-        else:
-            self.log.info("Fargate profile deleted successfully")
-        return
+
+        self.log.info("Fargate profile deleted successfully")
 
 
 class EksPodOperator(KubernetesPodOperator):

airflow/providers/amazon/aws/operators/emr.py

@@ -27,8 +27,17 @@ from uuid import uuid4
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.models import BaseOperator
+from airflow.models.mappedoperator import MappedOperator
 from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook, EmrHook, EmrServerlessHook
-from airflow.providers.amazon.aws.links.emr import EmrClusterLink, EmrLogsLink, get_log_uri
+from airflow.providers.amazon.aws.links.emr import (
+    EmrClusterLink,
+    EmrLogsLink,
+    EmrServerlessCloudWatchLogsLink,
+    EmrServerlessDashboardLink,
+    EmrServerlessLogsLink,
+    EmrServerlessS3LogsLink,
+    get_log_uri,
+)
 from airflow.providers.amazon.aws.triggers.emr import (
     EmrAddStepsTrigger,
     EmrContainerTrigger,
@@ -41,6 +50,7 @@ from airflow.providers.amazon.aws.triggers.emr import (
     EmrServerlessStopApplicationTrigger,
     EmrTerminateJobFlowTrigger,
 )
+from airflow.providers.amazon.aws.utils import validate_execute_complete_event
 from airflow.providers.amazon.aws.utils.waiter import waiter
 from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
 from airflow.utils.helpers import exactly_one, prune_dict
@@ -180,11 +190,13 @@ class EmrAddStepsOperator(BaseOperator):
 
         return step_ids
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error while running steps: {event}")
-        else:
-            self.log.info("Steps completed successfully")
+
+        self.log.info("Steps completed successfully")
         return event["value"]
 
 
@@ -494,6 +506,8 @@ class EmrContainerOperator(BaseOperator):
     :param max_tries: Deprecated - use max_polling_attempts instead.
     :param max_polling_attempts: Maximum number of times to wait for the job run to finish.
         Defaults to None, which will poll until the job is *not* in a pending, submitted, or running state.
+    :param job_retry_max_attempts: Maximum number of times to retry when the EMR job fails.
+        Defaults to None, which disable the retry.
     :param tags: The tags assigned to job runs.
         Defaults to None
     :param deferrable: Run operator in the deferrable mode.
@@ -525,6 +539,7 @@ class EmrContainerOperator(BaseOperator):
         max_tries: int | None = None,
         tags: dict | None = None,
         max_polling_attempts: int | None = None,
+        job_retry_max_attempts: int | None = None,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         **kwargs: Any,
     ) -> None:
@@ -540,6 +555,7 @@ class EmrContainerOperator(BaseOperator):
         self.wait_for_completion = wait_for_completion
         self.poll_interval = poll_interval
         self.max_polling_attempts = max_polling_attempts
+        self.job_retry_max_attempts = job_retry_max_attempts
         self.tags = tags
         self.job_id: str | None = None
         self.deferrable = deferrable
@@ -574,6 +590,7 @@ class EmrContainerOperator(BaseOperator):
             self.configuration_overrides,
             self.client_request_token,
             self.tags,
+            self.job_retry_max_attempts,
         )
         if self.deferrable:
             query_status = self.hook.check_query_status(job_id=self.job_id)
@@ -619,7 +636,9 @@ class EmrContainerOperator(BaseOperator):
                 f"query_execution_id is {self.job_id}. Error: {error_message}"
             )
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error while running job: {event}")
 
@@ -806,11 +825,13 @@ class EmrCreateJobFlowOperator(BaseOperator):
             )
         return self._job_flow_id
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error creating jobFlow: {event}")
-        else:
-            self.log.info("JobFlow created successfully")
+
+        self.log.info("JobFlow created successfully")
         return event["job_flow_id"]
 
     def on_kill(self) -> None:
@@ -969,12 +990,13 @@ class EmrTerminateJobFlowOperator(BaseOperator):
                 timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay + 60),
             )
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error terminating JobFlow: {event}")
-        else:
-            self.log.info("Jobflow terminated successfully.")
-        return
+
+        self.log.info("Jobflow terminated successfully.")
 
 
 class EmrServerlessCreateApplicationOperator(BaseOperator):
@@ -1135,7 +1157,9 @@ class EmrServerlessCreateApplicationOperator(BaseOperator):
         )
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
-        if event is None or event["status"] != "success":
+        event = validate_execute_complete_event(event)
+
+        if event["status"] != "success":
             raise AirflowException(f"Trigger error: Application failed to start, event is {event}")
 
         self.log.info("Application %s started", event["application_id"])
@@ -1172,6 +1196,9 @@ class EmrServerlessStartJobOperator(BaseOperator):
     :param deferrable: If True, the operator will wait asynchronously for the crawl to complete.
         This implies waiting for completion. This mode requires aiobotocore module to be installed.
         (default: False, but can be overridden in config file by setting default_deferrable to True)
+    :param enable_application_ui_links: If True, the operator will generate one-time links to EMR Serverless
+        application UIs. The generated links will allow any user with access to the DAG to see the Spark or
+        Tez UI or Spark stdout logs. Defaults to False.
     """
 
     template_fields: Sequence[str] = (
@@ -1181,6 +1208,7 @@ class EmrServerlessStartJobOperator(BaseOperator):
         "job_driver",
         "configuration_overrides",
         "name",
+        "aws_conn_id",
     )
 
     template_fields_renderers = {
@@ -1188,12 +1216,48 @@ class EmrServerlessStartJobOperator(BaseOperator):
         "configuration_overrides": "json",
     }
 
+    @property
+    def operator_extra_links(self):
+        """
+        Dynamically add extra links depending on the job type and if they're enabled.
+
+        If S3 or CloudWatch monitoring configurations exist, add links directly to the relevant consoles.
+        Only add dashboard links if they're explicitly enabled. These are one-time links that any user
+        can access, but expire on first click or one hour, whichever comes first.
+        """
+        op_extra_links = []
+
+        if isinstance(self, MappedOperator):
+            enable_application_ui_links = self.partial_kwargs.get(
+                "enable_application_ui_links"
+            ) or self.expand_input.value.get("enable_application_ui_links")
+            job_driver = self.partial_kwargs.get("job_driver") or self.expand_input.value.get("job_driver")
+            configuration_overrides = self.partial_kwargs.get(
+                "configuration_overrides"
+            ) or self.expand_input.value.get("configuration_overrides")
+
+        else:
+            enable_application_ui_links = self.enable_application_ui_links
+            configuration_overrides = self.configuration_overrides
+            job_driver = self.job_driver
+
+        if enable_application_ui_links:
+            op_extra_links.extend([EmrServerlessDashboardLink()])
+            if "sparkSubmit" in job_driver:
+                op_extra_links.extend([EmrServerlessLogsLink()])
+        if self.is_monitoring_in_job_override("s3MonitoringConfiguration", configuration_overrides):
+            op_extra_links.extend([EmrServerlessS3LogsLink()])
+        if self.is_monitoring_in_job_override("cloudWatchLoggingConfiguration", configuration_overrides):
+            op_extra_links.extend([EmrServerlessCloudWatchLogsLink()])
+
+        return tuple(op_extra_links)
+
     def __init__(
         self,
         application_id: str,
         execution_role_arn: str,
         job_driver: dict,
-        configuration_overrides: dict | None,
+        configuration_overrides: dict | None = None,
         client_request_token: str = "",
         config: dict | None = None,
         wait_for_completion: bool = True,
@@ -1204,6 +1268,7 @@ class EmrServerlessStartJobOperator(BaseOperator):
         waiter_max_attempts: int | ArgNotSet = NOTSET,
         waiter_delay: int | ArgNotSet = NOTSET,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
+        enable_application_ui_links: bool = False,
         **kwargs,
     ):
         if waiter_check_interval_seconds is NOTSET:
@@ -1243,6 +1308,7 @@ class EmrServerlessStartJobOperator(BaseOperator):
         self.waiter_delay = int(waiter_delay)  # type: ignore[arg-type]
         self.job_id: str | None = None
         self.deferrable = deferrable
+        self.enable_application_ui_links = enable_application_ui_links
         super().__init__(**kwargs)
 
         self.client_request_token = client_request_token or str(uuid4())
@@ -1300,6 +1366,9 @@ class EmrServerlessStartJobOperator(BaseOperator):
 
         self.job_id = response["jobRunId"]
         self.log.info("EMR serverless job started: %s", self.job_id)
+
+        self.persist_links(context)
+
         if self.deferrable:
             self.defer(
                 trigger=EmrServerlessStartJobTrigger(
@@ -1312,6 +1381,7 @@ class EmrServerlessStartJobOperator(BaseOperator):
                 method_name="execute_complete",
                 timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay),
             )
+
         if self.wait_for_completion:
             waiter = self.hook.get_waiter("serverless_job_completed")
             wait(
@@ -1327,10 +1397,9 @@ class EmrServerlessStartJobOperator(BaseOperator):
         return self.job_id
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
-        if event is None:
-            self.log.error("Trigger error: event is None")
-            raise AirflowException("Trigger error: event is None")
-        elif event["status"] == "success":
+        event = validate_execute_complete_event(event)
+
+        if event["status"] == "success":
             self.log.info("Serverless job completed")
             return event["job_id"]
 
@@ -1369,6 +1438,105 @@ class EmrServerlessStartJobOperator(BaseOperator):
                 check_interval_seconds=self.waiter_delay,
             )
 
+    def is_monitoring_in_job_override(self, config_key: str, job_override: dict | None) -> bool:
+        """
+        Check if monitoring is enabled for the job.
+
+        Note: This is not compatible with application defaults:
+        https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/default-configs.html
+
+        This is used to determine what extra links should be shown.
+        """
+        monitoring_config = (job_override or {}).get("monitoringConfiguration")
+        if monitoring_config is None or config_key not in monitoring_config:
+            return False
+
+        # CloudWatch can have an "enabled" flag set to False
+        if config_key == "cloudWatchLoggingConfiguration":
+            return monitoring_config.get(config_key).get("enabled") is True
+
+        return config_key in monitoring_config
+
+    def persist_links(self, context: Context):
+        """Populate the relevant extra links for the EMR Serverless jobs."""
+        # Persist the EMR Serverless Dashboard link (Spark/Tez UI)
+        if self.enable_application_ui_links:
+            EmrServerlessDashboardLink.persist(
+                context=context,
+                operator=self,
+                region_name=self.hook.conn_region_name,
+                aws_partition=self.hook.conn_partition,
+                conn_id=self.hook.aws_conn_id,
+                application_id=self.application_id,
+                job_run_id=self.job_id,
+            )
+
+        # If this is a Spark job, persist the EMR Serverless logs link (Driver stdout)
+        if self.enable_application_ui_links and "sparkSubmit" in self.job_driver:
+            EmrServerlessLogsLink.persist(
+                context=context,
+                operator=self,
+                region_name=self.hook.conn_region_name,
+                aws_partition=self.hook.conn_partition,
+                conn_id=self.hook.aws_conn_id,
+                application_id=self.application_id,
+                job_run_id=self.job_id,
+            )
+
+        # Add S3 and/or CloudWatch links if either is enabled
+        if self.is_monitoring_in_job_override("s3MonitoringConfiguration", self.configuration_overrides):
+            log_uri = (
+                (self.configuration_overrides or {})
+                .get("monitoringConfiguration", {})
+                .get("s3MonitoringConfiguration", {})
+                .get("logUri")
+            )
+            EmrServerlessS3LogsLink.persist(
+                context=context,
+                operator=self,
+                region_name=self.hook.conn_region_name,
+                aws_partition=self.hook.conn_partition,
+                log_uri=log_uri,
+                application_id=self.application_id,
+                job_run_id=self.job_id,
+            )
+            emrs_s3_url = EmrServerlessS3LogsLink().format_link(
+                aws_domain=EmrServerlessCloudWatchLogsLink.get_aws_domain(self.hook.conn_partition),
+                region_name=self.hook.conn_region_name,
+                aws_partition=self.hook.conn_partition,
+                log_uri=log_uri,
+                application_id=self.application_id,
+                job_run_id=self.job_id,
+            )
+            self.log.info("S3 logs available at: %s", emrs_s3_url)
+
+        if self.is_monitoring_in_job_override("cloudWatchLoggingConfiguration", self.configuration_overrides):
+            cloudwatch_config = (
+                (self.configuration_overrides or {})
+                .get("monitoringConfiguration", {})
+                .get("cloudWatchLoggingConfiguration", {})
+            )
+            log_group_name = cloudwatch_config.get("logGroupName", "/aws/emr-serverless")
+            log_stream_prefix = cloudwatch_config.get("logStreamNamePrefix", "")
+            log_stream_prefix = f"{log_stream_prefix}/applications/{self.application_id}/jobs/{self.job_id}"
+
+            EmrServerlessCloudWatchLogsLink.persist(
+                context=context,
+                operator=self,
+                region_name=self.hook.conn_region_name,
+                aws_partition=self.hook.conn_partition,
+                awslogs_group=log_group_name,
+                stream_prefix=log_stream_prefix,
+            )
+            emrs_cloudwatch_url = EmrServerlessCloudWatchLogsLink().format_link(
+                aws_domain=EmrServerlessCloudWatchLogsLink.get_aws_domain(self.hook.conn_partition),
+                region_name=self.hook.conn_region_name,
+                aws_partition=self.hook.conn_partition,
+                awslogs_group=log_group_name,
+                stream_prefix=log_stream_prefix,
+            )
+            self.log.info("CloudWatch logs available at: %s", emrs_cloudwatch_url)
+
 
 class EmrServerlessStopApplicationOperator(BaseOperator):
     """
@@ -1527,10 +1695,9 @@ class EmrServerlessStopApplicationOperator(BaseOperator):
             )
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
-        if event is None:
-            self.log.error("Trigger error: event is None")
-            raise AirflowException("Trigger error: event is None")
-        elif event["status"] == "success":
+        event = validate_execute_complete_event(event)
+
+        if event["status"] == "success":
             self.log.info("EMR serverless application %s stopped successfully", self.application_id)
 
 
@@ -1656,8 +1823,7 @@ class EmrServerlessDeleteApplicationOperator(EmrServerlessStopApplicationOperato
         self.log.info("EMR serverless application deleted")
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
-        if event is None:
-            self.log.error("Trigger error: event is None")
-            raise AirflowException("Trigger error: event is None")
-        elif event["status"] == "success":
+        event = validate_execute_complete_event(event)
+
+        if event["status"] == "success":
             self.log.info("EMR serverless application %s deleted successfully", self.application_id)

airflow/providers/amazon/aws/operators/glue.py

@@ -20,7 +20,7 @@ from __future__ import annotations
 import os
 import urllib.parse
 from functools import cached_property
-from typing import TYPE_CHECKING, Sequence
+from typing import TYPE_CHECKING, Any, Sequence
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException
@@ -29,6 +29,7 @@ from airflow.providers.amazon.aws.hooks.glue import GlueJobHook
 from airflow.providers.amazon.aws.hooks.s3 import S3Hook
 from airflow.providers.amazon.aws.links.glue import GlueJobRunDetailsLink
 from airflow.providers.amazon.aws.triggers.glue import GlueJobCompleteTrigger
+from airflow.providers.amazon.aws.utils import validate_execute_complete_event
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context
@@ -215,7 +216,9 @@ class GlueJobOperator(BaseOperator):
             self.log.info("AWS Glue Job: %s. Run Id: %s", self.job_name, self._job_run_id)
         return self._job_run_id
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error in glue job: {event}")
         return event["value"]

airflow/providers/amazon/aws/operators/glue_crawler.py

@@ -18,11 +18,12 @@
 from __future__ import annotations
 
 from functools import cached_property
-from typing import TYPE_CHECKING, Sequence
+from typing import TYPE_CHECKING, Any, Sequence
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.triggers.glue_crawler import GlueCrawlerCompleteTrigger
+from airflow.providers.amazon.aws.utils import validate_execute_complete_event
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context
@@ -107,7 +108,9 @@ class GlueCrawlerOperator(BaseOperator):
 
         return crawler_name
 
-    def execute_complete(self, context, event=None):
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
+        event = validate_execute_complete_event(event)
+
         if event["status"] != "success":
             raise AirflowException(f"Error in glue crawl: {event}")
         return self.config["Name"]

airflow/providers/amazon/aws/operators/glue_databrew.py

@@ -18,12 +18,13 @@
 from __future__ import annotations
 
 from functools import cached_property
-from typing import TYPE_CHECKING, Sequence
+from typing import TYPE_CHECKING, Any, Sequence
 
 from airflow.configuration import conf
 from airflow.models import BaseOperator
 from airflow.providers.amazon.aws.hooks.glue_databrew import GlueDataBrewHook
 from airflow.providers.amazon.aws.triggers.glue_databrew import GlueDataBrewJobCompleteTrigger
+from airflow.providers.amazon.aws.utils import validate_execute_complete_event
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context
@@ -101,7 +102,9 @@ class GlueDataBrewStartJobOperator(BaseOperator):
 
         return {"run_id": run_id}
 
-    def execute_complete(self, context: Context, event=None) -> dict[str, str]:
+    def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, str]:
+        event = validate_execute_complete_event(event)
+
         run_id = event.get("run_id", "")
         status = event.get("status", "")
 

airflow/providers/amazon/aws/operators/lambda_function.py

@@ -26,6 +26,7 @@ from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.hooks.lambda_function import LambdaHook
 from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
 from airflow.providers.amazon.aws.triggers.lambda_function import LambdaCreateFunctionCompleteTrigger
+from airflow.providers.amazon.aws.utils import validate_execute_complete_event
 from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
 
 if TYPE_CHECKING:
@@ -143,6 +144,8 @@ class LambdaCreateFunctionOperator(AwsBaseOperator[LambdaHook]):
         return response.get("FunctionArn")
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
+        event = validate_execute_complete_event(event)
+
         if not event or event["status"] != "success":
             raise AirflowException(f"Trigger error: event is {event}")