apache-airflow-providers-amazon 8.18.0rc1__py3-none-any.whl → 8.18.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. airflow/providers/amazon/aws/auth_manager/cli/avp_commands.py +2 -2
  2. airflow/providers/amazon/aws/auth_manager/cli/definition.py +14 -0
  3. airflow/providers/amazon/aws/auth_manager/cli/idc_commands.py +148 -0
  4. airflow/providers/amazon/aws/hooks/base_aws.py +2 -2
  5. airflow/providers/amazon/aws/hooks/emr.py +6 -0
  6. airflow/providers/amazon/aws/hooks/redshift_cluster.py +1 -1
  7. airflow/providers/amazon/aws/links/emr.py +122 -2
  8. airflow/providers/amazon/aws/log/cloudwatch_task_handler.py +2 -2
  9. airflow/providers/amazon/aws/operators/athena.py +4 -1
  10. airflow/providers/amazon/aws/operators/batch.py +5 -6
  11. airflow/providers/amazon/aws/operators/ecs.py +6 -2
  12. airflow/providers/amazon/aws/operators/eks.py +23 -20
  13. airflow/providers/amazon/aws/operators/emr.py +192 -26
  14. airflow/providers/amazon/aws/operators/glue.py +5 -2
  15. airflow/providers/amazon/aws/operators/glue_crawler.py +5 -2
  16. airflow/providers/amazon/aws/operators/glue_databrew.py +5 -2
  17. airflow/providers/amazon/aws/operators/lambda_function.py +3 -0
  18. airflow/providers/amazon/aws/operators/rds.py +21 -12
  19. airflow/providers/amazon/aws/operators/redshift_cluster.py +12 -18
  20. airflow/providers/amazon/aws/operators/redshift_data.py +2 -4
  21. airflow/providers/amazon/aws/operators/sagemaker.py +24 -20
  22. airflow/providers/amazon/aws/operators/step_function.py +4 -1
  23. airflow/providers/amazon/aws/sensors/ec2.py +4 -2
  24. airflow/providers/amazon/aws/sensors/emr.py +13 -6
  25. airflow/providers/amazon/aws/sensors/glue_catalog_partition.py +4 -1
  26. airflow/providers/amazon/aws/sensors/redshift_cluster.py +2 -4
  27. airflow/providers/amazon/aws/sensors/s3.py +3 -0
  28. airflow/providers/amazon/aws/sensors/sqs.py +4 -1
  29. airflow/providers/amazon/aws/utils/__init__.py +10 -0
  30. airflow/providers/amazon/aws/utils/task_log_fetcher.py +2 -2
  31. airflow/providers/amazon/get_provider_info.py +4 -0
  32. {apache_airflow_providers_amazon-8.18.0rc1.dist-info → apache_airflow_providers_amazon-8.18.0rc2.dist-info}/METADATA +2 -2
  33. {apache_airflow_providers_amazon-8.18.0rc1.dist-info → apache_airflow_providers_amazon-8.18.0rc2.dist-info}/RECORD +35 -34
  34. {apache_airflow_providers_amazon-8.18.0rc1.dist-info → apache_airflow_providers_amazon-8.18.0rc2.dist-info}/WHEEL +0 -0
  35. {apache_airflow_providers_amazon-8.18.0rc1.dist-info → apache_airflow_providers_amazon-8.18.0rc2.dist-info}/entry_points.txt +0 -0
@@ -55,7 +55,7 @@ def init_avp(args):
55
55
  if not is_new_policy_store:
56
56
  print(
57
57
  f"Since an existing policy store with description '{args.policy_store_description}' has been found in Amazon Verified Permissions, "
58
- "the CLI nade no changes to this policy store for security reasons. "
58
+ "the CLI made no changes to this policy store for security reasons. "
59
59
  "Any modification to this policy store must be done manually.",
60
60
  )
61
61
  else:
@@ -115,7 +115,7 @@ def _create_policy_store(client: BaseClient, args) -> tuple[str | None, bool]:
115
115
  print(f"No policy store with description '{args.policy_store_description}' found, creating one.")
116
116
  if args.dry_run:
117
117
  print(
118
- "Dry run, not creating the policy store with description '{args.policy_store_description}'."
118
+ f"Dry run, not creating the policy store with description '{args.policy_store_description}'."
119
119
  )
120
120
  return None, True
121
121
 
@@ -35,6 +35,14 @@ ARG_DRY_RUN = Arg(
35
35
  action="store_true",
36
36
  )
37
37
 
38
+ # AWS IAM Identity Center
39
+ ARG_INSTANCE_NAME = Arg(("--instance-name",), help="Instance name in Identity Center", default="Airflow")
40
+
41
+ ARG_APPLICATION_NAME = Arg(
42
+ ("--application-name",), help="Application name in Identity Center", default="Airflow"
43
+ )
44
+
45
+
38
46
  # Amazon Verified Permissions
39
47
  ARG_POLICY_STORE_DESCRIPTION = Arg(
40
48
  ("--policy-store-description",), help="Policy store description", default="Airflow"
@@ -47,6 +55,12 @@ ARG_POLICY_STORE_ID = Arg(("--policy-store-id",), help="Policy store ID")
47
55
  ################
48
56
 
49
57
  AWS_AUTH_MANAGER_COMMANDS = (
58
+ ActionCommand(
59
+ name="init-identity-center",
60
+ help="Initialize AWS IAM identity Center resources to be used by AWS manager",
61
+ func=lazy_load_command("airflow.providers.amazon.aws.auth_manager.cli.idc_commands.init_idc"),
62
+ args=(ARG_INSTANCE_NAME, ARG_APPLICATION_NAME, ARG_DRY_RUN, ARG_VERBOSE),
63
+ ),
50
64
  ActionCommand(
51
65
  name="init-avp",
52
66
  help="Initialize Amazon Verified resources to be used by AWS manager",
@@ -0,0 +1,148 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ """User sub-commands."""
18
+ from __future__ import annotations
19
+
20
+ import logging
21
+ from typing import TYPE_CHECKING
22
+
23
+ import boto3
24
+ from botocore.exceptions import ClientError
25
+
26
+ from airflow.configuration import conf
27
+ from airflow.exceptions import AirflowOptionalProviderFeatureException
28
+ from airflow.providers.amazon.aws.auth_manager.constants import CONF_REGION_NAME_KEY, CONF_SECTION_NAME
29
+ from airflow.utils import cli as cli_utils
30
+
31
+ try:
32
+ from airflow.utils.providers_configuration_loader import providers_configuration_loaded
33
+ except ImportError:
34
+ raise AirflowOptionalProviderFeatureException(
35
+ "Failed to import avp_commands. This feature is only available in Airflow "
36
+ "version >= 2.8.0 where Auth Managers are introduced."
37
+ )
38
+
39
+ if TYPE_CHECKING:
40
+ from botocore.client import BaseClient
41
+
42
+ log = logging.getLogger(__name__)
43
+
44
+
45
+ @cli_utils.action_cli
46
+ @providers_configuration_loaded
47
+ def init_idc(args):
48
+ """Initialize AWS IAM Identity Center resources."""
49
+ client = _get_client()
50
+
51
+ # Create the instance if needed
52
+ instance_arn = _create_instance(client, args)
53
+
54
+ # Create the application if needed
55
+ _create_application(client, instance_arn, args)
56
+
57
+ if not args.dry_run:
58
+ print("AWS IAM Identity Center resources created successfully.")
59
+
60
+
61
+ def _get_client():
62
+ """Return AWS IAM Identity Center client."""
63
+ region_name = conf.get(CONF_SECTION_NAME, CONF_REGION_NAME_KEY)
64
+ return boto3.client("sso-admin", region_name=region_name)
65
+
66
+
67
+ def _create_instance(client: BaseClient, args) -> str | None:
68
+ """Create if needed AWS IAM Identity Center instance."""
69
+ instances = client.list_instances()
70
+
71
+ if args.verbose:
72
+ log.debug("Instances found: %s", instances)
73
+
74
+ if len(instances["Instances"]) > 0:
75
+ print(
76
+ f"There is already an instance configured in AWS IAM Identity Center: '{instances['Instances'][0]['InstanceArn']}'. "
77
+ "No need to create a new one."
78
+ )
79
+ return instances["Instances"][0]["InstanceArn"]
80
+ else:
81
+ print("No instance configured in AWS IAM Identity Center, creating one.")
82
+ if args.dry_run:
83
+ print("Dry run, not creating the instance.")
84
+ return None
85
+
86
+ response = client.create_instance(Name=args.instance_name)
87
+ if args.verbose:
88
+ log.debug("Response from create_instance: %s", response)
89
+
90
+ print(f"Instance created: '{response['InstanceArn']}'")
91
+
92
+ return response["InstanceArn"]
93
+
94
+
95
+ def _create_application(client: BaseClient, instance_arn: str | None, args) -> str | None:
96
+ """Create if needed AWS IAM identity Center application."""
97
+ paginator = client.get_paginator("list_applications")
98
+ pages = paginator.paginate(InstanceArn=instance_arn or "")
99
+ applications = [application for page in pages for application in page["Applications"]]
100
+ existing_applications = [
101
+ application for application in applications if application["Name"] == args.application_name
102
+ ]
103
+
104
+ if args.verbose:
105
+ log.debug("Applications found: %s", applications)
106
+ log.debug("Existing applications found: %s", existing_applications)
107
+
108
+ if len(existing_applications) > 0:
109
+ print(
110
+ f"There is already an application named '{args.application_name}' in AWS IAM Identity Center: '{existing_applications[0]['ApplicationArn']}'. "
111
+ "Using this application."
112
+ )
113
+ return existing_applications[0]["ApplicationArn"]
114
+ else:
115
+ print(f"No application named {args.application_name} found, creating one.")
116
+ if args.dry_run:
117
+ print("Dry run, not creating the application.")
118
+ return None
119
+
120
+ try:
121
+ response = client.create_application(
122
+ ApplicationProviderArn="arn:aws:sso::aws:applicationProvider/custom-saml",
123
+ Description="Application automatically created through the Airflow CLI. This application is used to access Airflow environment.",
124
+ InstanceArn=instance_arn,
125
+ Name=args.application_name,
126
+ PortalOptions={
127
+ "SignInOptions": {
128
+ "Origin": "IDENTITY_CENTER",
129
+ },
130
+ "Visibility": "ENABLED",
131
+ },
132
+ Status="ENABLED",
133
+ )
134
+ if args.verbose:
135
+ log.debug("Response from create_application: %s", response)
136
+ except ClientError as e:
137
+ # This is needed because as of today, the create_application in AWS Identity Center does not support SAML application
138
+ # Remove this part when it is supported
139
+ if "is not supported for this action" in e.response["Error"]["Message"]:
140
+ print(
141
+ "Creation of SAML applications is only supported in AWS console today. "
142
+ "Please create the application through the console."
143
+ )
144
+ raise
145
+
146
+ print(f"Application created: '{response['ApplicationArn']}'")
147
+
148
+ return response["ApplicationArn"]
@@ -1022,7 +1022,7 @@ except ImportError:
1022
1022
 
1023
1023
  @deprecated(
1024
1024
  reason=(
1025
- "airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory "
1025
+ "`airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory` "
1026
1026
  "has been deprecated and will be removed in future"
1027
1027
  ),
1028
1028
  category=AirflowProviderDeprecationWarning,
@@ -1116,7 +1116,7 @@ class BaseAsyncSessionFactory(BaseSessionFactory):
1116
1116
 
1117
1117
  @deprecated(
1118
1118
  reason=(
1119
- "airflow.providers.amazon.aws.hook.base_aws.AwsBaseAsyncHook "
1119
+ "`airflow.providers.amazon.aws.hook.base_aws.AwsBaseAsyncHook` "
1120
1120
  "has been deprecated and will be removed in future"
1121
1121
  ),
1122
1122
  category=AirflowProviderDeprecationWarning,
@@ -383,6 +383,7 @@ class EmrContainerHook(AwsBaseHook):
383
383
  configuration_overrides: dict | None = None,
384
384
  client_request_token: str | None = None,
385
385
  tags: dict | None = None,
386
+ retry_max_attempts: int | None = None,
386
387
  ) -> str:
387
388
  """
388
389
  Submit a job to the EMR Containers API and return the job ID.
@@ -402,6 +403,7 @@ class EmrContainerHook(AwsBaseHook):
402
403
  :param client_request_token: The client idempotency token of the job run request.
403
404
  Use this if you want to specify a unique ID to prevent two jobs from getting started.
404
405
  :param tags: The tags assigned to job runs.
406
+ :param retry_max_attempts: The maximum number of attempts on the job's driver.
405
407
  :return: The ID of the job run request.
406
408
  """
407
409
  params = {
@@ -415,6 +417,10 @@ class EmrContainerHook(AwsBaseHook):
415
417
  }
416
418
  if client_request_token:
417
419
  params["clientToken"] = client_request_token
420
+ if retry_max_attempts:
421
+ params["retryPolicyConfiguration"] = {
422
+ "maxAttempts": retry_max_attempts,
423
+ }
418
424
 
419
425
  response = self.conn.start_job_run(**params)
420
426
 
@@ -197,7 +197,7 @@ class RedshiftHook(AwsBaseHook):
197
197
 
198
198
  @deprecated(
199
199
  reason=(
200
- "airflow.providers.amazon.aws.hook.base_aws.RedshiftAsyncHook "
200
+ "`airflow.providers.amazon.aws.hook.base_aws.RedshiftAsyncHook` "
201
201
  "has been deprecated and will be removed in future"
202
202
  ),
203
203
  category=AirflowProviderDeprecationWarning,
@@ -17,8 +17,10 @@
17
17
  from __future__ import annotations
18
18
 
19
19
  from typing import TYPE_CHECKING, Any
20
+ from urllib.parse import ParseResult, quote_plus, urlparse
20
21
 
21
22
  from airflow.exceptions import AirflowException
23
+ from airflow.providers.amazon.aws.hooks.emr import EmrServerlessHook
22
24
  from airflow.providers.amazon.aws.hooks.s3 import S3Hook
23
25
  from airflow.providers.amazon.aws.links.base_aws import BASE_AWS_CONSOLE_LINK, BaseAwsLink
24
26
  from airflow.utils.helpers import exactly_one
@@ -28,7 +30,7 @@ if TYPE_CHECKING:
28
30
 
29
31
 
30
32
  class EmrClusterLink(BaseAwsLink):
31
- """Helper class for constructing AWS EMR Cluster Link."""
33
+ """Helper class for constructing Amazon EMR Cluster Link."""
32
34
 
33
35
  name = "EMR Cluster"
34
36
  key = "emr_cluster"
@@ -36,7 +38,7 @@ class EmrClusterLink(BaseAwsLink):
36
38
 
37
39
 
38
40
  class EmrLogsLink(BaseAwsLink):
39
- """Helper class for constructing AWS EMR Logs Link."""
41
+ """Helper class for constructing Amazon EMR Logs Link."""
40
42
 
41
43
  name = "EMR Cluster Logs"
42
44
  key = "emr_logs"
@@ -48,6 +50,49 @@ class EmrLogsLink(BaseAwsLink):
48
50
  return super().format_link(**kwargs)
49
51
 
50
52
 
53
+ def get_serverless_log_uri(*, s3_log_uri: str, application_id: str, job_run_id: str) -> str:
54
+ """
55
+ Retrieve the S3 URI to EMR Serverless Job logs.
56
+
57
+ Any EMR Serverless job may have a different S3 logging location (or none), which is an S3 URI.
58
+ The logging location is then {s3_uri}/applications/{application_id}/jobs/{job_run_id}.
59
+ """
60
+ return f"{s3_log_uri}/applications/{application_id}/jobs/{job_run_id}"
61
+
62
+
63
+ def get_serverless_dashboard_url(
64
+ *,
65
+ aws_conn_id: str | None = None,
66
+ emr_serverless_client: boto3.client = None,
67
+ application_id: str,
68
+ job_run_id: str,
69
+ ) -> ParseResult | None:
70
+ """
71
+ Retrieve the URL to EMR Serverless dashboard.
72
+
73
+ The URL is a one-use, ephemeral link that expires in 1 hour and is accessible without authentication.
74
+
75
+ Either an AWS connection ID or existing EMR Serverless client must be passed.
76
+ If the connection ID is passed, a client is generated using that connection.
77
+ """
78
+ if not exactly_one(aws_conn_id, emr_serverless_client):
79
+ raise AirflowException("Requires either an AWS connection ID or an EMR Serverless Client.")
80
+
81
+ if aws_conn_id:
82
+ # If get_dashboard_for_job_run fails for whatever reason, fail after 1 attempt
83
+ # so that the rest of the links load in a reasonable time frame.
84
+ hook = EmrServerlessHook(aws_conn_id=aws_conn_id, config={"retries": {"total_max_attempts": 1}})
85
+ emr_serverless_client = hook.conn
86
+
87
+ response = emr_serverless_client.get_dashboard_for_job_run(
88
+ applicationId=application_id, jobRunId=job_run_id
89
+ )
90
+ if "url" not in response:
91
+ return None
92
+ log_uri = urlparse(response["url"])
93
+ return log_uri
94
+
95
+
51
96
  def get_log_uri(
52
97
  *, cluster: dict[str, Any] | None = None, emr_client: boto3.client = None, job_flow_id: str | None = None
53
98
  ) -> str | None:
@@ -66,3 +111,78 @@ def get_log_uri(
66
111
  return None
67
112
  log_uri = S3Hook.parse_s3_url(cluster_info["LogUri"])
68
113
  return "/".join(log_uri)
114
+
115
+
116
+ class EmrServerlessLogsLink(BaseAwsLink):
117
+ """Helper class for constructing Amazon EMR Serverless link to Spark stdout logs."""
118
+
119
+ name = "Spark Driver stdout"
120
+ key = "emr_serverless_logs"
121
+
122
+ def format_link(self, application_id: str | None = None, job_run_id: str | None = None, **kwargs) -> str:
123
+ if not application_id or not job_run_id:
124
+ return ""
125
+ url = get_serverless_dashboard_url(
126
+ aws_conn_id=kwargs.get("conn_id"), application_id=application_id, job_run_id=job_run_id
127
+ )
128
+ if url:
129
+ return url._replace(path="/logs/SPARK_DRIVER/stdout.gz").geturl()
130
+ else:
131
+ return ""
132
+
133
+
134
+ class EmrServerlessDashboardLink(BaseAwsLink):
135
+ """Helper class for constructing Amazon EMR Serverless Dashboard Link."""
136
+
137
+ name = "EMR Serverless Dashboard"
138
+ key = "emr_serverless_dashboard"
139
+
140
+ def format_link(self, application_id: str | None = None, job_run_id: str | None = None, **kwargs) -> str:
141
+ if not application_id or not job_run_id:
142
+ return ""
143
+ url = get_serverless_dashboard_url(
144
+ aws_conn_id=kwargs.get("conn_id"), application_id=application_id, job_run_id=job_run_id
145
+ )
146
+ if url:
147
+ return url.geturl()
148
+ else:
149
+ return ""
150
+
151
+
152
+ class EmrServerlessS3LogsLink(BaseAwsLink):
153
+ """Helper class for constructing link to S3 console for Amazon EMR Serverless Logs."""
154
+
155
+ name = "S3 Logs"
156
+ key = "emr_serverless_s3_logs"
157
+ format_str = BASE_AWS_CONSOLE_LINK + (
158
+ "/s3/buckets/{bucket_name}?region={region_name}"
159
+ "&prefix={prefix}/applications/{application_id}/jobs/{job_run_id}/"
160
+ )
161
+
162
+ def format_link(self, **kwargs) -> str:
163
+ bucket, prefix = S3Hook.parse_s3_url(kwargs["log_uri"])
164
+ kwargs["bucket_name"] = bucket
165
+ kwargs["prefix"] = prefix.rstrip("/")
166
+ return super().format_link(**kwargs)
167
+
168
+
169
+ class EmrServerlessCloudWatchLogsLink(BaseAwsLink):
170
+ """
171
+ Helper class for constructing link to CloudWatch console for Amazon EMR Serverless Logs.
172
+
173
+ This is a deep link that filters on a specific job run.
174
+ """
175
+
176
+ name = "CloudWatch Logs"
177
+ key = "emr_serverless_cloudwatch_logs"
178
+ format_str = (
179
+ BASE_AWS_CONSOLE_LINK
180
+ + "/cloudwatch/home?region={region_name}#logsV2:log-groups/log-group/{awslogs_group}{stream_prefix}"
181
+ )
182
+
183
+ def format_link(self, **kwargs) -> str:
184
+ kwargs["awslogs_group"] = quote_plus(kwargs["awslogs_group"])
185
+ kwargs["stream_prefix"] = quote_plus("?logStreamNameFilter=").replace("%", "$") + quote_plus(
186
+ kwargs["stream_prefix"]
187
+ )
188
+ return super().format_link(**kwargs)
@@ -17,7 +17,7 @@
17
17
  # under the License.
18
18
  from __future__ import annotations
19
19
 
20
- from datetime import date, datetime, timedelta
20
+ from datetime import date, datetime, timedelta, timezone
21
21
  from functools import cached_property
22
22
  from typing import TYPE_CHECKING, Any
23
23
 
@@ -163,7 +163,7 @@ class CloudwatchTaskHandler(FileTaskHandler, LoggingMixin):
163
163
  return "\n".join(self._event_to_str(event) for event in events)
164
164
 
165
165
  def _event_to_str(self, event: dict) -> str:
166
- event_dt = datetime.utcfromtimestamp(event["timestamp"] / 1000.0)
166
+ event_dt = datetime.fromtimestamp(event["timestamp"] / 1000.0, tz=timezone.utc)
167
167
  formatted_event_dt = event_dt.strftime("%Y-%m-%d %H:%M:%S,%f")[:-3]
168
168
  message = event["message"]
169
169
  return f"[{formatted_event_dt}] {message}"
@@ -26,6 +26,7 @@ from airflow.providers.amazon.aws.hooks.athena import AthenaHook
26
26
  from airflow.providers.amazon.aws.links.athena import AthenaQueryResultsLink
27
27
  from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
28
28
  from airflow.providers.amazon.aws.triggers.athena import AthenaTrigger
29
+ from airflow.providers.amazon.aws.utils import validate_execute_complete_event
29
30
  from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
30
31
 
31
32
  if TYPE_CHECKING:
@@ -179,7 +180,9 @@ class AthenaOperator(AwsBaseOperator[AthenaHook]):
179
180
 
180
181
  return self.query_execution_id
181
182
 
182
- def execute_complete(self, context, event=None):
183
+ def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
184
+ event = validate_execute_complete_event(event)
185
+
183
186
  if event["status"] != "success":
184
187
  raise AirflowException(f"Error while waiting for operation on cluster to complete: {event}")
185
188
  return event["value"]
@@ -44,7 +44,7 @@ from airflow.providers.amazon.aws.triggers.batch import (
44
44
  BatchCreateComputeEnvironmentTrigger,
45
45
  BatchJobTrigger,
46
46
  )
47
- from airflow.providers.amazon.aws.utils import trim_none_values
47
+ from airflow.providers.amazon.aws.utils import trim_none_values, validate_execute_complete_event
48
48
  from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
49
49
 
50
50
  if TYPE_CHECKING:
@@ -269,10 +269,7 @@ class BatchOperator(BaseOperator):
269
269
  return self.job_id
270
270
 
271
271
  def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
272
- if event is None:
273
- err_msg = "Trigger error: event is None"
274
- self.log.info(err_msg)
275
- raise AirflowException(err_msg)
272
+ event = validate_execute_complete_event(event)
276
273
 
277
274
  if event["status"] != "success":
278
275
  raise AirflowException(f"Error while running job: {event}")
@@ -541,7 +538,9 @@ class BatchCreateComputeEnvironmentOperator(BaseOperator):
541
538
  self.log.info("AWS Batch compute environment created successfully")
542
539
  return arn
543
540
 
544
- def execute_complete(self, context, event=None):
541
+ def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
542
+ event = validate_execute_complete_event(event)
543
+
545
544
  if event["status"] != "success":
546
545
  raise AirflowException(f"Error while waiting for the compute environment to be ready: {event}")
547
546
  return event["value"]
@@ -21,7 +21,7 @@ import re
21
21
  import warnings
22
22
  from datetime import timedelta
23
23
  from functools import cached_property
24
- from typing import TYPE_CHECKING, Sequence
24
+ from typing import TYPE_CHECKING, Any, Sequence
25
25
 
26
26
  from airflow.configuration import conf
27
27
  from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
@@ -35,6 +35,7 @@ from airflow.providers.amazon.aws.triggers.ecs import (
35
35
  ClusterInactiveTrigger,
36
36
  TaskDoneTrigger,
37
37
  )
38
+ from airflow.providers.amazon.aws.utils import validate_execute_complete_event
38
39
  from airflow.providers.amazon.aws.utils.identifiers import generate_uuid
39
40
  from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
40
41
  from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
@@ -580,7 +581,9 @@ class EcsRunTaskOperator(EcsBaseOperator):
580
581
  else:
581
582
  return None
582
583
 
583
- def execute_complete(self, context, event=None):
584
+ def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str | None:
585
+ event = validate_execute_complete_event(event)
586
+
584
587
  if event["status"] != "success":
585
588
  raise AirflowException(f"Error in task execution: {event}")
586
589
  self.arn = event["task_arn"] # restore arn to its updated value, needed for next steps
@@ -596,6 +599,7 @@ class EcsRunTaskOperator(EcsBaseOperator):
596
599
  )
597
600
  if len(one_log["events"]) > 0:
598
601
  return one_log["events"][0]["message"]
602
+ return None
599
603
 
600
604
  def _after_execution(self):
601
605
  self._check_success_task()
@@ -39,6 +39,7 @@ from airflow.providers.amazon.aws.triggers.eks import (
39
39
  EksDeleteFargateProfileTrigger,
40
40
  EksDeleteNodegroupTrigger,
41
41
  )
42
+ from airflow.providers.amazon.aws.utils import validate_execute_complete_event
42
43
  from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
43
44
  from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction
44
45
 
@@ -421,11 +422,10 @@ class EksCreateClusterOperator(BaseOperator):
421
422
  raise AirflowException("Error creating cluster")
422
423
 
423
424
  def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
425
+ event = validate_execute_complete_event(event)
426
+
424
427
  resource = "fargate profile" if self.compute == "fargate" else self.compute
425
- if event is None:
426
- self.log.info("Trigger error: event is None")
427
- raise AirflowException("Trigger error: event is None")
428
- elif event["status"] != "success":
428
+ if event["status"] != "success":
429
429
  raise AirflowException(f"Error creating {resource}: {event}")
430
430
 
431
431
  self.log.info("%s created successfully", resource)
@@ -547,10 +547,11 @@ class EksCreateNodegroupOperator(BaseOperator):
547
547
  timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay + 60),
548
548
  )
549
549
 
550
- def execute_complete(self, context, event=None):
550
+ def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
551
+ event = validate_execute_complete_event(event)
552
+
551
553
  if event["status"] != "success":
552
554
  raise AirflowException(f"Error creating nodegroup: {event}")
553
- return
554
555
 
555
556
 
556
557
  class EksCreateFargateProfileOperator(BaseOperator):
@@ -656,12 +657,13 @@ class EksCreateFargateProfileOperator(BaseOperator):
656
657
  timeout=timedelta(seconds=(self.waiter_max_attempts * self.waiter_delay + 60)),
657
658
  )
658
659
 
659
- def execute_complete(self, context, event=None):
660
+ def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
661
+ event = validate_execute_complete_event(event)
662
+
660
663
  if event["status"] != "success":
661
664
  raise AirflowException(f"Error creating Fargate profile: {event}")
662
- else:
663
- self.log.info("Fargate profile created successfully")
664
- return
665
+
666
+ self.log.info("Fargate profile created successfully")
665
667
 
666
668
 
667
669
  class EksDeleteClusterOperator(BaseOperator):
@@ -788,10 +790,9 @@ class EksDeleteClusterOperator(BaseOperator):
788
790
  self.log.info(SUCCESS_MSG.format(compute=FARGATE_FULL_NAME))
789
791
 
790
792
  def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
791
- if event is None:
792
- self.log.error("Trigger error. Event is None")
793
- raise AirflowException("Trigger error. Event is None")
794
- elif event["status"] == "success":
793
+ event = validate_execute_complete_event(event)
794
+
795
+ if event["status"] == "success":
795
796
  self.log.info("Cluster deleted successfully.")
796
797
 
797
798
 
@@ -879,10 +880,11 @@ class EksDeleteNodegroupOperator(BaseOperator):
879
880
  clusterName=self.cluster_name, nodegroupName=self.nodegroup_name
880
881
  )
881
882
 
882
- def execute_complete(self, context, event=None):
883
+ def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
884
+ event = validate_execute_complete_event(event)
885
+
883
886
  if event["status"] != "success":
884
887
  raise AirflowException(f"Error deleting nodegroup: {event}")
885
- return
886
888
 
887
889
 
888
890
  class EksDeleteFargateProfileOperator(BaseOperator):
@@ -972,12 +974,13 @@ class EksDeleteFargateProfileOperator(BaseOperator):
972
974
  WaiterConfig={"Delay": self.waiter_delay, "MaxAttempts": self.waiter_max_attempts},
973
975
  )
974
976
 
975
- def execute_complete(self, context, event=None):
977
+ def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
978
+ event = validate_execute_complete_event(event)
979
+
976
980
  if event["status"] != "success":
977
981
  raise AirflowException(f"Error deleting Fargate profile: {event}")
978
- else:
979
- self.log.info("Fargate profile deleted successfully")
980
- return
982
+
983
+ self.log.info("Fargate profile deleted successfully")
981
984
 
982
985
 
983
986
  class EksPodOperator(KubernetesPodOperator):