apache-airflow-providers-amazon 8.18.0rc1__py3-none-any.whl → 8.18.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/amazon/aws/auth_manager/cli/avp_commands.py +2 -2
- airflow/providers/amazon/aws/auth_manager/cli/definition.py +14 -0
- airflow/providers/amazon/aws/auth_manager/cli/idc_commands.py +148 -0
- airflow/providers/amazon/aws/hooks/base_aws.py +2 -2
- airflow/providers/amazon/aws/hooks/emr.py +6 -0
- airflow/providers/amazon/aws/hooks/redshift_cluster.py +1 -1
- airflow/providers/amazon/aws/links/emr.py +122 -2
- airflow/providers/amazon/aws/log/cloudwatch_task_handler.py +2 -2
- airflow/providers/amazon/aws/operators/athena.py +4 -1
- airflow/providers/amazon/aws/operators/batch.py +5 -6
- airflow/providers/amazon/aws/operators/ecs.py +6 -2
- airflow/providers/amazon/aws/operators/eks.py +23 -20
- airflow/providers/amazon/aws/operators/emr.py +192 -26
- airflow/providers/amazon/aws/operators/glue.py +5 -2
- airflow/providers/amazon/aws/operators/glue_crawler.py +5 -2
- airflow/providers/amazon/aws/operators/glue_databrew.py +5 -2
- airflow/providers/amazon/aws/operators/lambda_function.py +3 -0
- airflow/providers/amazon/aws/operators/rds.py +21 -12
- airflow/providers/amazon/aws/operators/redshift_cluster.py +12 -18
- airflow/providers/amazon/aws/operators/redshift_data.py +2 -4
- airflow/providers/amazon/aws/operators/sagemaker.py +24 -20
- airflow/providers/amazon/aws/operators/step_function.py +4 -1
- airflow/providers/amazon/aws/sensors/ec2.py +4 -2
- airflow/providers/amazon/aws/sensors/emr.py +13 -6
- airflow/providers/amazon/aws/sensors/glue_catalog_partition.py +4 -1
- airflow/providers/amazon/aws/sensors/redshift_cluster.py +2 -4
- airflow/providers/amazon/aws/sensors/s3.py +3 -0
- airflow/providers/amazon/aws/sensors/sqs.py +4 -1
- airflow/providers/amazon/aws/utils/__init__.py +10 -0
- airflow/providers/amazon/aws/utils/task_log_fetcher.py +2 -2
- airflow/providers/amazon/get_provider_info.py +4 -0
- {apache_airflow_providers_amazon-8.18.0rc1.dist-info → apache_airflow_providers_amazon-8.18.0rc2.dist-info}/METADATA +2 -2
- {apache_airflow_providers_amazon-8.18.0rc1.dist-info → apache_airflow_providers_amazon-8.18.0rc2.dist-info}/RECORD +35 -34
- {apache_airflow_providers_amazon-8.18.0rc1.dist-info → apache_airflow_providers_amazon-8.18.0rc2.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_amazon-8.18.0rc1.dist-info → apache_airflow_providers_amazon-8.18.0rc2.dist-info}/entry_points.txt +0 -0
@@ -55,7 +55,7 @@ def init_avp(args):
|
|
55
55
|
if not is_new_policy_store:
|
56
56
|
print(
|
57
57
|
f"Since an existing policy store with description '{args.policy_store_description}' has been found in Amazon Verified Permissions, "
|
58
|
-
"the CLI
|
58
|
+
"the CLI made no changes to this policy store for security reasons. "
|
59
59
|
"Any modification to this policy store must be done manually.",
|
60
60
|
)
|
61
61
|
else:
|
@@ -115,7 +115,7 @@ def _create_policy_store(client: BaseClient, args) -> tuple[str | None, bool]:
|
|
115
115
|
print(f"No policy store with description '{args.policy_store_description}' found, creating one.")
|
116
116
|
if args.dry_run:
|
117
117
|
print(
|
118
|
-
"Dry run, not creating the policy store with description '{args.policy_store_description}'."
|
118
|
+
f"Dry run, not creating the policy store with description '{args.policy_store_description}'."
|
119
119
|
)
|
120
120
|
return None, True
|
121
121
|
|
@@ -35,6 +35,14 @@ ARG_DRY_RUN = Arg(
|
|
35
35
|
action="store_true",
|
36
36
|
)
|
37
37
|
|
38
|
+
# AWS IAM Identity Center
|
39
|
+
ARG_INSTANCE_NAME = Arg(("--instance-name",), help="Instance name in Identity Center", default="Airflow")
|
40
|
+
|
41
|
+
ARG_APPLICATION_NAME = Arg(
|
42
|
+
("--application-name",), help="Application name in Identity Center", default="Airflow"
|
43
|
+
)
|
44
|
+
|
45
|
+
|
38
46
|
# Amazon Verified Permissions
|
39
47
|
ARG_POLICY_STORE_DESCRIPTION = Arg(
|
40
48
|
("--policy-store-description",), help="Policy store description", default="Airflow"
|
@@ -47,6 +55,12 @@ ARG_POLICY_STORE_ID = Arg(("--policy-store-id",), help="Policy store ID")
|
|
47
55
|
################
|
48
56
|
|
49
57
|
AWS_AUTH_MANAGER_COMMANDS = (
|
58
|
+
ActionCommand(
|
59
|
+
name="init-identity-center",
|
60
|
+
help="Initialize AWS IAM identity Center resources to be used by AWS manager",
|
61
|
+
func=lazy_load_command("airflow.providers.amazon.aws.auth_manager.cli.idc_commands.init_idc"),
|
62
|
+
args=(ARG_INSTANCE_NAME, ARG_APPLICATION_NAME, ARG_DRY_RUN, ARG_VERBOSE),
|
63
|
+
),
|
50
64
|
ActionCommand(
|
51
65
|
name="init-avp",
|
52
66
|
help="Initialize Amazon Verified resources to be used by AWS manager",
|
@@ -0,0 +1,148 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
"""User sub-commands."""
|
18
|
+
from __future__ import annotations
|
19
|
+
|
20
|
+
import logging
|
21
|
+
from typing import TYPE_CHECKING
|
22
|
+
|
23
|
+
import boto3
|
24
|
+
from botocore.exceptions import ClientError
|
25
|
+
|
26
|
+
from airflow.configuration import conf
|
27
|
+
from airflow.exceptions import AirflowOptionalProviderFeatureException
|
28
|
+
from airflow.providers.amazon.aws.auth_manager.constants import CONF_REGION_NAME_KEY, CONF_SECTION_NAME
|
29
|
+
from airflow.utils import cli as cli_utils
|
30
|
+
|
31
|
+
try:
|
32
|
+
from airflow.utils.providers_configuration_loader import providers_configuration_loaded
|
33
|
+
except ImportError:
|
34
|
+
raise AirflowOptionalProviderFeatureException(
|
35
|
+
"Failed to import avp_commands. This feature is only available in Airflow "
|
36
|
+
"version >= 2.8.0 where Auth Managers are introduced."
|
37
|
+
)
|
38
|
+
|
39
|
+
if TYPE_CHECKING:
|
40
|
+
from botocore.client import BaseClient
|
41
|
+
|
42
|
+
log = logging.getLogger(__name__)
|
43
|
+
|
44
|
+
|
45
|
+
@cli_utils.action_cli
|
46
|
+
@providers_configuration_loaded
|
47
|
+
def init_idc(args):
|
48
|
+
"""Initialize AWS IAM Identity Center resources."""
|
49
|
+
client = _get_client()
|
50
|
+
|
51
|
+
# Create the instance if needed
|
52
|
+
instance_arn = _create_instance(client, args)
|
53
|
+
|
54
|
+
# Create the application if needed
|
55
|
+
_create_application(client, instance_arn, args)
|
56
|
+
|
57
|
+
if not args.dry_run:
|
58
|
+
print("AWS IAM Identity Center resources created successfully.")
|
59
|
+
|
60
|
+
|
61
|
+
def _get_client():
|
62
|
+
"""Return AWS IAM Identity Center client."""
|
63
|
+
region_name = conf.get(CONF_SECTION_NAME, CONF_REGION_NAME_KEY)
|
64
|
+
return boto3.client("sso-admin", region_name=region_name)
|
65
|
+
|
66
|
+
|
67
|
+
def _create_instance(client: BaseClient, args) -> str | None:
|
68
|
+
"""Create if needed AWS IAM Identity Center instance."""
|
69
|
+
instances = client.list_instances()
|
70
|
+
|
71
|
+
if args.verbose:
|
72
|
+
log.debug("Instances found: %s", instances)
|
73
|
+
|
74
|
+
if len(instances["Instances"]) > 0:
|
75
|
+
print(
|
76
|
+
f"There is already an instance configured in AWS IAM Identity Center: '{instances['Instances'][0]['InstanceArn']}'. "
|
77
|
+
"No need to create a new one."
|
78
|
+
)
|
79
|
+
return instances["Instances"][0]["InstanceArn"]
|
80
|
+
else:
|
81
|
+
print("No instance configured in AWS IAM Identity Center, creating one.")
|
82
|
+
if args.dry_run:
|
83
|
+
print("Dry run, not creating the instance.")
|
84
|
+
return None
|
85
|
+
|
86
|
+
response = client.create_instance(Name=args.instance_name)
|
87
|
+
if args.verbose:
|
88
|
+
log.debug("Response from create_instance: %s", response)
|
89
|
+
|
90
|
+
print(f"Instance created: '{response['InstanceArn']}'")
|
91
|
+
|
92
|
+
return response["InstanceArn"]
|
93
|
+
|
94
|
+
|
95
|
+
def _create_application(client: BaseClient, instance_arn: str | None, args) -> str | None:
|
96
|
+
"""Create if needed AWS IAM identity Center application."""
|
97
|
+
paginator = client.get_paginator("list_applications")
|
98
|
+
pages = paginator.paginate(InstanceArn=instance_arn or "")
|
99
|
+
applications = [application for page in pages for application in page["Applications"]]
|
100
|
+
existing_applications = [
|
101
|
+
application for application in applications if application["Name"] == args.application_name
|
102
|
+
]
|
103
|
+
|
104
|
+
if args.verbose:
|
105
|
+
log.debug("Applications found: %s", applications)
|
106
|
+
log.debug("Existing applications found: %s", existing_applications)
|
107
|
+
|
108
|
+
if len(existing_applications) > 0:
|
109
|
+
print(
|
110
|
+
f"There is already an application named '{args.application_name}' in AWS IAM Identity Center: '{existing_applications[0]['ApplicationArn']}'. "
|
111
|
+
"Using this application."
|
112
|
+
)
|
113
|
+
return existing_applications[0]["ApplicationArn"]
|
114
|
+
else:
|
115
|
+
print(f"No application named {args.application_name} found, creating one.")
|
116
|
+
if args.dry_run:
|
117
|
+
print("Dry run, not creating the application.")
|
118
|
+
return None
|
119
|
+
|
120
|
+
try:
|
121
|
+
response = client.create_application(
|
122
|
+
ApplicationProviderArn="arn:aws:sso::aws:applicationProvider/custom-saml",
|
123
|
+
Description="Application automatically created through the Airflow CLI. This application is used to access Airflow environment.",
|
124
|
+
InstanceArn=instance_arn,
|
125
|
+
Name=args.application_name,
|
126
|
+
PortalOptions={
|
127
|
+
"SignInOptions": {
|
128
|
+
"Origin": "IDENTITY_CENTER",
|
129
|
+
},
|
130
|
+
"Visibility": "ENABLED",
|
131
|
+
},
|
132
|
+
Status="ENABLED",
|
133
|
+
)
|
134
|
+
if args.verbose:
|
135
|
+
log.debug("Response from create_application: %s", response)
|
136
|
+
except ClientError as e:
|
137
|
+
# This is needed because as of today, the create_application in AWS Identity Center does not support SAML application
|
138
|
+
# Remove this part when it is supported
|
139
|
+
if "is not supported for this action" in e.response["Error"]["Message"]:
|
140
|
+
print(
|
141
|
+
"Creation of SAML applications is only supported in AWS console today. "
|
142
|
+
"Please create the application through the console."
|
143
|
+
)
|
144
|
+
raise
|
145
|
+
|
146
|
+
print(f"Application created: '{response['ApplicationArn']}'")
|
147
|
+
|
148
|
+
return response["ApplicationArn"]
|
@@ -1022,7 +1022,7 @@ except ImportError:
|
|
1022
1022
|
|
1023
1023
|
@deprecated(
|
1024
1024
|
reason=(
|
1025
|
-
"airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory "
|
1025
|
+
"`airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory` "
|
1026
1026
|
"has been deprecated and will be removed in future"
|
1027
1027
|
),
|
1028
1028
|
category=AirflowProviderDeprecationWarning,
|
@@ -1116,7 +1116,7 @@ class BaseAsyncSessionFactory(BaseSessionFactory):
|
|
1116
1116
|
|
1117
1117
|
@deprecated(
|
1118
1118
|
reason=(
|
1119
|
-
"airflow.providers.amazon.aws.hook.base_aws.AwsBaseAsyncHook "
|
1119
|
+
"`airflow.providers.amazon.aws.hook.base_aws.AwsBaseAsyncHook` "
|
1120
1120
|
"has been deprecated and will be removed in future"
|
1121
1121
|
),
|
1122
1122
|
category=AirflowProviderDeprecationWarning,
|
@@ -383,6 +383,7 @@ class EmrContainerHook(AwsBaseHook):
|
|
383
383
|
configuration_overrides: dict | None = None,
|
384
384
|
client_request_token: str | None = None,
|
385
385
|
tags: dict | None = None,
|
386
|
+
retry_max_attempts: int | None = None,
|
386
387
|
) -> str:
|
387
388
|
"""
|
388
389
|
Submit a job to the EMR Containers API and return the job ID.
|
@@ -402,6 +403,7 @@ class EmrContainerHook(AwsBaseHook):
|
|
402
403
|
:param client_request_token: The client idempotency token of the job run request.
|
403
404
|
Use this if you want to specify a unique ID to prevent two jobs from getting started.
|
404
405
|
:param tags: The tags assigned to job runs.
|
406
|
+
:param retry_max_attempts: The maximum number of attempts on the job's driver.
|
405
407
|
:return: The ID of the job run request.
|
406
408
|
"""
|
407
409
|
params = {
|
@@ -415,6 +417,10 @@ class EmrContainerHook(AwsBaseHook):
|
|
415
417
|
}
|
416
418
|
if client_request_token:
|
417
419
|
params["clientToken"] = client_request_token
|
420
|
+
if retry_max_attempts:
|
421
|
+
params["retryPolicyConfiguration"] = {
|
422
|
+
"maxAttempts": retry_max_attempts,
|
423
|
+
}
|
418
424
|
|
419
425
|
response = self.conn.start_job_run(**params)
|
420
426
|
|
@@ -197,7 +197,7 @@ class RedshiftHook(AwsBaseHook):
|
|
197
197
|
|
198
198
|
@deprecated(
|
199
199
|
reason=(
|
200
|
-
"airflow.providers.amazon.aws.hook.base_aws.RedshiftAsyncHook "
|
200
|
+
"`airflow.providers.amazon.aws.hook.base_aws.RedshiftAsyncHook` "
|
201
201
|
"has been deprecated and will be removed in future"
|
202
202
|
),
|
203
203
|
category=AirflowProviderDeprecationWarning,
|
@@ -17,8 +17,10 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
from typing import TYPE_CHECKING, Any
|
20
|
+
from urllib.parse import ParseResult, quote_plus, urlparse
|
20
21
|
|
21
22
|
from airflow.exceptions import AirflowException
|
23
|
+
from airflow.providers.amazon.aws.hooks.emr import EmrServerlessHook
|
22
24
|
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
|
23
25
|
from airflow.providers.amazon.aws.links.base_aws import BASE_AWS_CONSOLE_LINK, BaseAwsLink
|
24
26
|
from airflow.utils.helpers import exactly_one
|
@@ -28,7 +30,7 @@ if TYPE_CHECKING:
|
|
28
30
|
|
29
31
|
|
30
32
|
class EmrClusterLink(BaseAwsLink):
|
31
|
-
"""Helper class for constructing
|
33
|
+
"""Helper class for constructing Amazon EMR Cluster Link."""
|
32
34
|
|
33
35
|
name = "EMR Cluster"
|
34
36
|
key = "emr_cluster"
|
@@ -36,7 +38,7 @@ class EmrClusterLink(BaseAwsLink):
|
|
36
38
|
|
37
39
|
|
38
40
|
class EmrLogsLink(BaseAwsLink):
|
39
|
-
"""Helper class for constructing
|
41
|
+
"""Helper class for constructing Amazon EMR Logs Link."""
|
40
42
|
|
41
43
|
name = "EMR Cluster Logs"
|
42
44
|
key = "emr_logs"
|
@@ -48,6 +50,49 @@ class EmrLogsLink(BaseAwsLink):
|
|
48
50
|
return super().format_link(**kwargs)
|
49
51
|
|
50
52
|
|
53
|
+
def get_serverless_log_uri(*, s3_log_uri: str, application_id: str, job_run_id: str) -> str:
|
54
|
+
"""
|
55
|
+
Retrieve the S3 URI to EMR Serverless Job logs.
|
56
|
+
|
57
|
+
Any EMR Serverless job may have a different S3 logging location (or none), which is an S3 URI.
|
58
|
+
The logging location is then {s3_uri}/applications/{application_id}/jobs/{job_run_id}.
|
59
|
+
"""
|
60
|
+
return f"{s3_log_uri}/applications/{application_id}/jobs/{job_run_id}"
|
61
|
+
|
62
|
+
|
63
|
+
def get_serverless_dashboard_url(
|
64
|
+
*,
|
65
|
+
aws_conn_id: str | None = None,
|
66
|
+
emr_serverless_client: boto3.client = None,
|
67
|
+
application_id: str,
|
68
|
+
job_run_id: str,
|
69
|
+
) -> ParseResult | None:
|
70
|
+
"""
|
71
|
+
Retrieve the URL to EMR Serverless dashboard.
|
72
|
+
|
73
|
+
The URL is a one-use, ephemeral link that expires in 1 hour and is accessible without authentication.
|
74
|
+
|
75
|
+
Either an AWS connection ID or existing EMR Serverless client must be passed.
|
76
|
+
If the connection ID is passed, a client is generated using that connection.
|
77
|
+
"""
|
78
|
+
if not exactly_one(aws_conn_id, emr_serverless_client):
|
79
|
+
raise AirflowException("Requires either an AWS connection ID or an EMR Serverless Client.")
|
80
|
+
|
81
|
+
if aws_conn_id:
|
82
|
+
# If get_dashboard_for_job_run fails for whatever reason, fail after 1 attempt
|
83
|
+
# so that the rest of the links load in a reasonable time frame.
|
84
|
+
hook = EmrServerlessHook(aws_conn_id=aws_conn_id, config={"retries": {"total_max_attempts": 1}})
|
85
|
+
emr_serverless_client = hook.conn
|
86
|
+
|
87
|
+
response = emr_serverless_client.get_dashboard_for_job_run(
|
88
|
+
applicationId=application_id, jobRunId=job_run_id
|
89
|
+
)
|
90
|
+
if "url" not in response:
|
91
|
+
return None
|
92
|
+
log_uri = urlparse(response["url"])
|
93
|
+
return log_uri
|
94
|
+
|
95
|
+
|
51
96
|
def get_log_uri(
|
52
97
|
*, cluster: dict[str, Any] | None = None, emr_client: boto3.client = None, job_flow_id: str | None = None
|
53
98
|
) -> str | None:
|
@@ -66,3 +111,78 @@ def get_log_uri(
|
|
66
111
|
return None
|
67
112
|
log_uri = S3Hook.parse_s3_url(cluster_info["LogUri"])
|
68
113
|
return "/".join(log_uri)
|
114
|
+
|
115
|
+
|
116
|
+
class EmrServerlessLogsLink(BaseAwsLink):
|
117
|
+
"""Helper class for constructing Amazon EMR Serverless link to Spark stdout logs."""
|
118
|
+
|
119
|
+
name = "Spark Driver stdout"
|
120
|
+
key = "emr_serverless_logs"
|
121
|
+
|
122
|
+
def format_link(self, application_id: str | None = None, job_run_id: str | None = None, **kwargs) -> str:
|
123
|
+
if not application_id or not job_run_id:
|
124
|
+
return ""
|
125
|
+
url = get_serverless_dashboard_url(
|
126
|
+
aws_conn_id=kwargs.get("conn_id"), application_id=application_id, job_run_id=job_run_id
|
127
|
+
)
|
128
|
+
if url:
|
129
|
+
return url._replace(path="/logs/SPARK_DRIVER/stdout.gz").geturl()
|
130
|
+
else:
|
131
|
+
return ""
|
132
|
+
|
133
|
+
|
134
|
+
class EmrServerlessDashboardLink(BaseAwsLink):
|
135
|
+
"""Helper class for constructing Amazon EMR Serverless Dashboard Link."""
|
136
|
+
|
137
|
+
name = "EMR Serverless Dashboard"
|
138
|
+
key = "emr_serverless_dashboard"
|
139
|
+
|
140
|
+
def format_link(self, application_id: str | None = None, job_run_id: str | None = None, **kwargs) -> str:
|
141
|
+
if not application_id or not job_run_id:
|
142
|
+
return ""
|
143
|
+
url = get_serverless_dashboard_url(
|
144
|
+
aws_conn_id=kwargs.get("conn_id"), application_id=application_id, job_run_id=job_run_id
|
145
|
+
)
|
146
|
+
if url:
|
147
|
+
return url.geturl()
|
148
|
+
else:
|
149
|
+
return ""
|
150
|
+
|
151
|
+
|
152
|
+
class EmrServerlessS3LogsLink(BaseAwsLink):
|
153
|
+
"""Helper class for constructing link to S3 console for Amazon EMR Serverless Logs."""
|
154
|
+
|
155
|
+
name = "S3 Logs"
|
156
|
+
key = "emr_serverless_s3_logs"
|
157
|
+
format_str = BASE_AWS_CONSOLE_LINK + (
|
158
|
+
"/s3/buckets/{bucket_name}?region={region_name}"
|
159
|
+
"&prefix={prefix}/applications/{application_id}/jobs/{job_run_id}/"
|
160
|
+
)
|
161
|
+
|
162
|
+
def format_link(self, **kwargs) -> str:
|
163
|
+
bucket, prefix = S3Hook.parse_s3_url(kwargs["log_uri"])
|
164
|
+
kwargs["bucket_name"] = bucket
|
165
|
+
kwargs["prefix"] = prefix.rstrip("/")
|
166
|
+
return super().format_link(**kwargs)
|
167
|
+
|
168
|
+
|
169
|
+
class EmrServerlessCloudWatchLogsLink(BaseAwsLink):
|
170
|
+
"""
|
171
|
+
Helper class for constructing link to CloudWatch console for Amazon EMR Serverless Logs.
|
172
|
+
|
173
|
+
This is a deep link that filters on a specific job run.
|
174
|
+
"""
|
175
|
+
|
176
|
+
name = "CloudWatch Logs"
|
177
|
+
key = "emr_serverless_cloudwatch_logs"
|
178
|
+
format_str = (
|
179
|
+
BASE_AWS_CONSOLE_LINK
|
180
|
+
+ "/cloudwatch/home?region={region_name}#logsV2:log-groups/log-group/{awslogs_group}{stream_prefix}"
|
181
|
+
)
|
182
|
+
|
183
|
+
def format_link(self, **kwargs) -> str:
|
184
|
+
kwargs["awslogs_group"] = quote_plus(kwargs["awslogs_group"])
|
185
|
+
kwargs["stream_prefix"] = quote_plus("?logStreamNameFilter=").replace("%", "$") + quote_plus(
|
186
|
+
kwargs["stream_prefix"]
|
187
|
+
)
|
188
|
+
return super().format_link(**kwargs)
|
@@ -17,7 +17,7 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from datetime import date, datetime, timedelta
|
20
|
+
from datetime import date, datetime, timedelta, timezone
|
21
21
|
from functools import cached_property
|
22
22
|
from typing import TYPE_CHECKING, Any
|
23
23
|
|
@@ -163,7 +163,7 @@ class CloudwatchTaskHandler(FileTaskHandler, LoggingMixin):
|
|
163
163
|
return "\n".join(self._event_to_str(event) for event in events)
|
164
164
|
|
165
165
|
def _event_to_str(self, event: dict) -> str:
|
166
|
-
event_dt = datetime.
|
166
|
+
event_dt = datetime.fromtimestamp(event["timestamp"] / 1000.0, tz=timezone.utc)
|
167
167
|
formatted_event_dt = event_dt.strftime("%Y-%m-%d %H:%M:%S,%f")[:-3]
|
168
168
|
message = event["message"]
|
169
169
|
return f"[{formatted_event_dt}] {message}"
|
@@ -26,6 +26,7 @@ from airflow.providers.amazon.aws.hooks.athena import AthenaHook
|
|
26
26
|
from airflow.providers.amazon.aws.links.athena import AthenaQueryResultsLink
|
27
27
|
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
28
28
|
from airflow.providers.amazon.aws.triggers.athena import AthenaTrigger
|
29
|
+
from airflow.providers.amazon.aws.utils import validate_execute_complete_event
|
29
30
|
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
30
31
|
|
31
32
|
if TYPE_CHECKING:
|
@@ -179,7 +180,9 @@ class AthenaOperator(AwsBaseOperator[AthenaHook]):
|
|
179
180
|
|
180
181
|
return self.query_execution_id
|
181
182
|
|
182
|
-
def execute_complete(self, context, event=None):
|
183
|
+
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
|
184
|
+
event = validate_execute_complete_event(event)
|
185
|
+
|
183
186
|
if event["status"] != "success":
|
184
187
|
raise AirflowException(f"Error while waiting for operation on cluster to complete: {event}")
|
185
188
|
return event["value"]
|
@@ -44,7 +44,7 @@ from airflow.providers.amazon.aws.triggers.batch import (
|
|
44
44
|
BatchCreateComputeEnvironmentTrigger,
|
45
45
|
BatchJobTrigger,
|
46
46
|
)
|
47
|
-
from airflow.providers.amazon.aws.utils import trim_none_values
|
47
|
+
from airflow.providers.amazon.aws.utils import trim_none_values, validate_execute_complete_event
|
48
48
|
from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
|
49
49
|
|
50
50
|
if TYPE_CHECKING:
|
@@ -269,10 +269,7 @@ class BatchOperator(BaseOperator):
|
|
269
269
|
return self.job_id
|
270
270
|
|
271
271
|
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
|
272
|
-
|
273
|
-
err_msg = "Trigger error: event is None"
|
274
|
-
self.log.info(err_msg)
|
275
|
-
raise AirflowException(err_msg)
|
272
|
+
event = validate_execute_complete_event(event)
|
276
273
|
|
277
274
|
if event["status"] != "success":
|
278
275
|
raise AirflowException(f"Error while running job: {event}")
|
@@ -541,7 +538,9 @@ class BatchCreateComputeEnvironmentOperator(BaseOperator):
|
|
541
538
|
self.log.info("AWS Batch compute environment created successfully")
|
542
539
|
return arn
|
543
540
|
|
544
|
-
def execute_complete(self, context, event=None):
|
541
|
+
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
|
542
|
+
event = validate_execute_complete_event(event)
|
543
|
+
|
545
544
|
if event["status"] != "success":
|
546
545
|
raise AirflowException(f"Error while waiting for the compute environment to be ready: {event}")
|
547
546
|
return event["value"]
|
@@ -21,7 +21,7 @@ import re
|
|
21
21
|
import warnings
|
22
22
|
from datetime import timedelta
|
23
23
|
from functools import cached_property
|
24
|
-
from typing import TYPE_CHECKING, Sequence
|
24
|
+
from typing import TYPE_CHECKING, Any, Sequence
|
25
25
|
|
26
26
|
from airflow.configuration import conf
|
27
27
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
@@ -35,6 +35,7 @@ from airflow.providers.amazon.aws.triggers.ecs import (
|
|
35
35
|
ClusterInactiveTrigger,
|
36
36
|
TaskDoneTrigger,
|
37
37
|
)
|
38
|
+
from airflow.providers.amazon.aws.utils import validate_execute_complete_event
|
38
39
|
from airflow.providers.amazon.aws.utils.identifiers import generate_uuid
|
39
40
|
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
40
41
|
from airflow.providers.amazon.aws.utils.task_log_fetcher import AwsTaskLogFetcher
|
@@ -580,7 +581,9 @@ class EcsRunTaskOperator(EcsBaseOperator):
|
|
580
581
|
else:
|
581
582
|
return None
|
582
583
|
|
583
|
-
def execute_complete(self, context, event=None):
|
584
|
+
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str | None:
|
585
|
+
event = validate_execute_complete_event(event)
|
586
|
+
|
584
587
|
if event["status"] != "success":
|
585
588
|
raise AirflowException(f"Error in task execution: {event}")
|
586
589
|
self.arn = event["task_arn"] # restore arn to its updated value, needed for next steps
|
@@ -596,6 +599,7 @@ class EcsRunTaskOperator(EcsBaseOperator):
|
|
596
599
|
)
|
597
600
|
if len(one_log["events"]) > 0:
|
598
601
|
return one_log["events"][0]["message"]
|
602
|
+
return None
|
599
603
|
|
600
604
|
def _after_execution(self):
|
601
605
|
self._check_success_task()
|
@@ -39,6 +39,7 @@ from airflow.providers.amazon.aws.triggers.eks import (
|
|
39
39
|
EksDeleteFargateProfileTrigger,
|
40
40
|
EksDeleteNodegroupTrigger,
|
41
41
|
)
|
42
|
+
from airflow.providers.amazon.aws.utils import validate_execute_complete_event
|
42
43
|
from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
|
43
44
|
from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction
|
44
45
|
|
@@ -421,11 +422,10 @@ class EksCreateClusterOperator(BaseOperator):
|
|
421
422
|
raise AirflowException("Error creating cluster")
|
422
423
|
|
423
424
|
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
|
425
|
+
event = validate_execute_complete_event(event)
|
426
|
+
|
424
427
|
resource = "fargate profile" if self.compute == "fargate" else self.compute
|
425
|
-
if event
|
426
|
-
self.log.info("Trigger error: event is None")
|
427
|
-
raise AirflowException("Trigger error: event is None")
|
428
|
-
elif event["status"] != "success":
|
428
|
+
if event["status"] != "success":
|
429
429
|
raise AirflowException(f"Error creating {resource}: {event}")
|
430
430
|
|
431
431
|
self.log.info("%s created successfully", resource)
|
@@ -547,10 +547,11 @@ class EksCreateNodegroupOperator(BaseOperator):
|
|
547
547
|
timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay + 60),
|
548
548
|
)
|
549
549
|
|
550
|
-
def execute_complete(self, context, event=None):
|
550
|
+
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
|
551
|
+
event = validate_execute_complete_event(event)
|
552
|
+
|
551
553
|
if event["status"] != "success":
|
552
554
|
raise AirflowException(f"Error creating nodegroup: {event}")
|
553
|
-
return
|
554
555
|
|
555
556
|
|
556
557
|
class EksCreateFargateProfileOperator(BaseOperator):
|
@@ -656,12 +657,13 @@ class EksCreateFargateProfileOperator(BaseOperator):
|
|
656
657
|
timeout=timedelta(seconds=(self.waiter_max_attempts * self.waiter_delay + 60)),
|
657
658
|
)
|
658
659
|
|
659
|
-
def execute_complete(self, context, event=None):
|
660
|
+
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
|
661
|
+
event = validate_execute_complete_event(event)
|
662
|
+
|
660
663
|
if event["status"] != "success":
|
661
664
|
raise AirflowException(f"Error creating Fargate profile: {event}")
|
662
|
-
|
663
|
-
|
664
|
-
return
|
665
|
+
|
666
|
+
self.log.info("Fargate profile created successfully")
|
665
667
|
|
666
668
|
|
667
669
|
class EksDeleteClusterOperator(BaseOperator):
|
@@ -788,10 +790,9 @@ class EksDeleteClusterOperator(BaseOperator):
|
|
788
790
|
self.log.info(SUCCESS_MSG.format(compute=FARGATE_FULL_NAME))
|
789
791
|
|
790
792
|
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
|
791
|
-
|
792
|
-
|
793
|
-
|
794
|
-
elif event["status"] == "success":
|
793
|
+
event = validate_execute_complete_event(event)
|
794
|
+
|
795
|
+
if event["status"] == "success":
|
795
796
|
self.log.info("Cluster deleted successfully.")
|
796
797
|
|
797
798
|
|
@@ -879,10 +880,11 @@ class EksDeleteNodegroupOperator(BaseOperator):
|
|
879
880
|
clusterName=self.cluster_name, nodegroupName=self.nodegroup_name
|
880
881
|
)
|
881
882
|
|
882
|
-
def execute_complete(self, context, event=None):
|
883
|
+
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
|
884
|
+
event = validate_execute_complete_event(event)
|
885
|
+
|
883
886
|
if event["status"] != "success":
|
884
887
|
raise AirflowException(f"Error deleting nodegroup: {event}")
|
885
|
-
return
|
886
888
|
|
887
889
|
|
888
890
|
class EksDeleteFargateProfileOperator(BaseOperator):
|
@@ -972,12 +974,13 @@ class EksDeleteFargateProfileOperator(BaseOperator):
|
|
972
974
|
WaiterConfig={"Delay": self.waiter_delay, "MaxAttempts": self.waiter_max_attempts},
|
973
975
|
)
|
974
976
|
|
975
|
-
def execute_complete(self, context, event=None):
|
977
|
+
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
|
978
|
+
event = validate_execute_complete_event(event)
|
979
|
+
|
976
980
|
if event["status"] != "success":
|
977
981
|
raise AirflowException(f"Error deleting Fargate profile: {event}")
|
978
|
-
|
979
|
-
|
980
|
-
return
|
982
|
+
|
983
|
+
self.log.info("Fargate profile deleted successfully")
|
981
984
|
|
982
985
|
|
983
986
|
class EksPodOperator(KubernetesPodOperator):
|