apache-airflow-providers-amazon 9.2.0rc2__py3-none-any.whl → 9.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/amazon/LICENSE +0 -52
- airflow/providers/amazon/__init__.py +1 -1
- airflow/providers/amazon/aws/auth_manager/avp/facade.py +1 -4
- airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py +90 -106
- airflow/providers/amazon/aws/auth_manager/router/login.py +124 -0
- airflow/providers/amazon/aws/executors/batch/batch_executor.py +2 -2
- airflow/providers/amazon/aws/executors/ecs/boto_schema.py +1 -1
- airflow/providers/amazon/aws/executors/ecs/utils.py +2 -1
- airflow/providers/amazon/aws/hooks/base_aws.py +6 -1
- airflow/providers/amazon/aws/hooks/batch_client.py +1 -2
- airflow/providers/amazon/aws/hooks/ecr.py +7 -1
- airflow/providers/amazon/aws/hooks/ecs.py +1 -2
- airflow/providers/amazon/aws/hooks/eks.py +10 -3
- airflow/providers/amazon/aws/hooks/emr.py +20 -0
- airflow/providers/amazon/aws/hooks/mwaa.py +85 -0
- airflow/providers/amazon/aws/hooks/sqs.py +4 -0
- airflow/providers/amazon/aws/hooks/ssm.py +10 -1
- airflow/providers/amazon/aws/links/comprehend.py +41 -0
- airflow/providers/amazon/aws/links/datasync.py +37 -0
- airflow/providers/amazon/aws/links/ec2.py +46 -0
- airflow/providers/amazon/aws/links/sagemaker.py +27 -0
- airflow/providers/amazon/aws/operators/athena.py +7 -5
- airflow/providers/amazon/aws/operators/batch.py +16 -8
- airflow/providers/amazon/aws/operators/bedrock.py +20 -18
- airflow/providers/amazon/aws/operators/comprehend.py +52 -11
- airflow/providers/amazon/aws/operators/datasync.py +40 -2
- airflow/providers/amazon/aws/operators/dms.py +0 -4
- airflow/providers/amazon/aws/operators/ec2.py +50 -0
- airflow/providers/amazon/aws/operators/ecs.py +11 -7
- airflow/providers/amazon/aws/operators/eks.py +17 -17
- airflow/providers/amazon/aws/operators/emr.py +27 -27
- airflow/providers/amazon/aws/operators/glue.py +16 -14
- airflow/providers/amazon/aws/operators/glue_crawler.py +3 -3
- airflow/providers/amazon/aws/operators/glue_databrew.py +5 -5
- airflow/providers/amazon/aws/operators/kinesis_analytics.py +9 -9
- airflow/providers/amazon/aws/operators/lambda_function.py +4 -4
- airflow/providers/amazon/aws/operators/mwaa.py +109 -0
- airflow/providers/amazon/aws/operators/rds.py +16 -16
- airflow/providers/amazon/aws/operators/redshift_cluster.py +15 -15
- airflow/providers/amazon/aws/operators/redshift_data.py +4 -4
- airflow/providers/amazon/aws/operators/sagemaker.py +52 -29
- airflow/providers/amazon/aws/operators/sqs.py +6 -0
- airflow/providers/amazon/aws/operators/step_function.py +4 -4
- airflow/providers/amazon/aws/sensors/ec2.py +3 -3
- airflow/providers/amazon/aws/sensors/emr.py +9 -9
- airflow/providers/amazon/aws/sensors/glue.py +7 -7
- airflow/providers/amazon/aws/sensors/glue_catalog_partition.py +3 -3
- airflow/providers/amazon/aws/sensors/redshift_cluster.py +3 -3
- airflow/providers/amazon/aws/sensors/sqs.py +6 -5
- airflow/providers/amazon/aws/transfers/google_api_to_s3.py +8 -3
- airflow/providers/amazon/aws/triggers/README.md +1 -1
- airflow/providers/amazon/aws/triggers/opensearch_serverless.py +2 -1
- airflow/providers/amazon/aws/triggers/sqs.py +2 -1
- airflow/providers/amazon/aws/utils/sqs.py +6 -4
- airflow/providers/amazon/aws/waiters/dms.json +12 -0
- airflow/providers/amazon/get_provider_info.py +106 -87
- {apache_airflow_providers_amazon-9.2.0rc2.dist-info → apache_airflow_providers_amazon-9.3.0.dist-info}/METADATA +18 -36
- {apache_airflow_providers_amazon-9.2.0rc2.dist-info → apache_airflow_providers_amazon-9.3.0.dist-info}/RECORD +61 -55
- airflow/providers/amazon/aws/auth_manager/views/auth.py +0 -151
- /airflow/providers/amazon/aws/auth_manager/{views → router}/__init__.py +0 -0
- {apache_airflow_providers_amazon-9.2.0rc2.dist-info → apache_airflow_providers_amazon-9.3.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_amazon-9.2.0rc2.dist-info → apache_airflow_providers_amazon-9.3.0.dist-info}/entry_points.txt +0 -0
airflow/providers/amazon/LICENSE
CHANGED
@@ -199,55 +199,3 @@ distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
199
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
200
200
|
See the License for the specific language governing permissions and
|
201
201
|
limitations under the License.
|
202
|
-
|
203
|
-
============================================================================
|
204
|
-
APACHE AIRFLOW SUBCOMPONENTS:
|
205
|
-
|
206
|
-
The Apache Airflow project contains subcomponents with separate copyright
|
207
|
-
notices and license terms. Your use of the source code for the these
|
208
|
-
subcomponents is subject to the terms and conditions of the following
|
209
|
-
licenses.
|
210
|
-
|
211
|
-
|
212
|
-
========================================================================
|
213
|
-
Third party Apache 2.0 licenses
|
214
|
-
========================================================================
|
215
|
-
|
216
|
-
The following components are provided under the Apache 2.0 License.
|
217
|
-
See project link for details. The text of each license is also included
|
218
|
-
at 3rd-party-licenses/LICENSE-[project].txt.
|
219
|
-
|
220
|
-
(ALv2 License) hue v4.3.0 (https://github.com/cloudera/hue/)
|
221
|
-
(ALv2 License) jqclock v2.3.0 (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin)
|
222
|
-
(ALv2 License) bootstrap3-typeahead v4.0.2 (https://github.com/bassjobsen/Bootstrap-3-Typeahead)
|
223
|
-
(ALv2 License) connexion v2.7.0 (https://github.com/zalando/connexion)
|
224
|
-
|
225
|
-
========================================================================
|
226
|
-
MIT licenses
|
227
|
-
========================================================================
|
228
|
-
|
229
|
-
The following components are provided under the MIT License. See project link for details.
|
230
|
-
The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
|
231
|
-
|
232
|
-
(MIT License) jquery v3.5.1 (https://jquery.org/license/)
|
233
|
-
(MIT License) dagre-d3 v0.6.4 (https://github.com/cpettitt/dagre-d3)
|
234
|
-
(MIT License) bootstrap v3.4.1 (https://github.com/twbs/bootstrap/)
|
235
|
-
(MIT License) d3-tip v0.9.1 (https://github.com/Caged/d3-tip)
|
236
|
-
(MIT License) dataTables v1.10.25 (https://datatables.net)
|
237
|
-
(MIT License) normalize.css v3.0.2 (http://necolas.github.io/normalize.css/)
|
238
|
-
(MIT License) ElasticMock v1.3.2 (https://github.com/vrcmarcos/elasticmock)
|
239
|
-
(MIT License) MomentJS v2.24.0 (http://momentjs.com/)
|
240
|
-
(MIT License) eonasdan-bootstrap-datetimepicker v4.17.49 (https://github.com/eonasdan/bootstrap-datetimepicker/)
|
241
|
-
|
242
|
-
========================================================================
|
243
|
-
BSD 3-Clause licenses
|
244
|
-
========================================================================
|
245
|
-
The following components are provided under the BSD 3-Clause license. See project links for details.
|
246
|
-
The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
|
247
|
-
|
248
|
-
(BSD 3 License) d3 v5.16.0 (https://d3js.org)
|
249
|
-
(BSD 3 License) d3-shape v2.1.0 (https://github.com/d3/d3-shape)
|
250
|
-
(BSD 3 License) cgroupspy 0.2.1 (https://github.com/cloudsigma/cgroupspy)
|
251
|
-
|
252
|
-
========================================================================
|
253
|
-
See 3rd-party-licenses/LICENSES-ui.txt for packages used in `/airflow/www`
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
29
29
|
|
30
30
|
__all__ = ["__version__"]
|
31
31
|
|
32
|
-
__version__ = "9.
|
32
|
+
__version__ = "9.3.0"
|
33
33
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
35
35
|
"2.9.0"
|
@@ -78,7 +78,7 @@ class AwsAuthManagerAmazonVerifiedPermissionsFacade(LoggingMixin):
|
|
78
78
|
*,
|
79
79
|
method: ResourceMethod | str,
|
80
80
|
entity_type: AvpEntities,
|
81
|
-
user: AwsAuthManagerUser
|
81
|
+
user: AwsAuthManagerUser,
|
82
82
|
entity_id: str | None = None,
|
83
83
|
context: dict | None = None,
|
84
84
|
) -> bool:
|
@@ -97,9 +97,6 @@ class AwsAuthManagerAmazonVerifiedPermissionsFacade(LoggingMixin):
|
|
97
97
|
considered.
|
98
98
|
:param context: optional additional context to pass to Amazon Verified Permissions.
|
99
99
|
"""
|
100
|
-
if user is None:
|
101
|
-
return False
|
102
|
-
|
103
100
|
entity_list = self._get_user_group_entities(user)
|
104
101
|
|
105
102
|
self.log.debug(
|
@@ -17,16 +17,26 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import argparse
|
20
|
-
import warnings
|
21
20
|
from collections import defaultdict
|
22
21
|
from collections.abc import Container, Sequence
|
23
22
|
from functools import cached_property
|
24
|
-
from typing import TYPE_CHECKING, cast
|
25
|
-
|
26
|
-
from
|
27
|
-
|
23
|
+
from typing import TYPE_CHECKING, Any, cast
|
24
|
+
|
25
|
+
from fastapi import FastAPI
|
26
|
+
from flask import session
|
27
|
+
|
28
|
+
from airflow.auth.managers.base_auth_manager import BaseAuthManager
|
29
|
+
from airflow.auth.managers.models.resource_details import (
|
30
|
+
AccessView,
|
31
|
+
ConnectionDetails,
|
32
|
+
DagAccessEntity,
|
33
|
+
DagDetails,
|
34
|
+
PoolDetails,
|
35
|
+
VariableDetails,
|
36
|
+
)
|
28
37
|
from airflow.cli.cli_config import CLICommand, DefaultHelpParser, GroupCommand
|
29
|
-
from airflow.
|
38
|
+
from airflow.configuration import conf
|
39
|
+
from airflow.exceptions import AirflowOptionalProviderFeatureException
|
30
40
|
from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities
|
31
41
|
from airflow.providers.amazon.aws.auth_manager.avp.facade import (
|
32
42
|
AwsAuthManagerAmazonVerifiedPermissionsFacade,
|
@@ -35,30 +45,13 @@ from airflow.providers.amazon.aws.auth_manager.avp.facade import (
|
|
35
45
|
from airflow.providers.amazon.aws.auth_manager.cli.definition import (
|
36
46
|
AWS_AUTH_MANAGER_COMMANDS,
|
37
47
|
)
|
38
|
-
from airflow.providers.amazon.aws.auth_manager.
|
39
|
-
|
40
|
-
)
|
41
|
-
from airflow.providers.amazon.aws.auth_manager.views.auth import AwsAuthManagerAuthenticationViews
|
42
|
-
|
43
|
-
try:
|
44
|
-
from airflow.auth.managers.base_auth_manager import BaseAuthManager, ResourceMethod
|
45
|
-
from airflow.auth.managers.models.resource_details import (
|
46
|
-
AccessView,
|
47
|
-
ConnectionDetails,
|
48
|
-
DagAccessEntity,
|
49
|
-
DagDetails,
|
50
|
-
PoolDetails,
|
51
|
-
VariableDetails,
|
52
|
-
)
|
53
|
-
except ImportError:
|
54
|
-
raise AirflowOptionalProviderFeatureException(
|
55
|
-
"Failed to import BaseUser. This feature is only available in Airflow versions >= 2.8.0"
|
56
|
-
)
|
48
|
+
from airflow.providers.amazon.aws.auth_manager.user import AwsAuthManagerUser
|
49
|
+
from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
|
57
50
|
|
58
51
|
if TYPE_CHECKING:
|
59
52
|
from flask_appbuilder.menu import MenuItem
|
60
53
|
|
61
|
-
from airflow.auth.managers.
|
54
|
+
from airflow.auth.managers.base_auth_manager import ResourceMethod
|
62
55
|
from airflow.auth.managers.models.batch_apis import (
|
63
56
|
IsAuthorizedConnectionRequest,
|
64
57
|
IsAuthorizedDagRequest,
|
@@ -66,46 +59,62 @@ if TYPE_CHECKING:
|
|
66
59
|
IsAuthorizedVariableRequest,
|
67
60
|
)
|
68
61
|
from airflow.auth.managers.models.resource_details import AssetDetails, ConfigurationDetails
|
69
|
-
from airflow.providers.amazon.aws.auth_manager.user import AwsAuthManagerUser
|
70
|
-
from airflow.www.extensions.init_appbuilder import AirflowAppBuilder
|
71
62
|
|
72
63
|
|
73
|
-
class AwsAuthManager(BaseAuthManager):
|
64
|
+
class AwsAuthManager(BaseAuthManager[AwsAuthManagerUser]):
|
74
65
|
"""
|
75
66
|
AWS auth manager.
|
76
67
|
|
77
68
|
Leverages AWS services such as Amazon Identity Center and Amazon Verified Permissions to perform
|
78
69
|
authentication and authorization in Airflow.
|
79
|
-
|
80
|
-
:param appbuilder: the flask app builder
|
81
70
|
"""
|
82
71
|
|
83
|
-
def __init__(self
|
84
|
-
|
72
|
+
def __init__(self) -> None:
|
73
|
+
if not AIRFLOW_V_3_0_PLUS:
|
74
|
+
raise AirflowOptionalProviderFeatureException(
|
75
|
+
"AWS auth manager is only compatible with Airflow versions >= 3.0.0"
|
76
|
+
)
|
77
|
+
|
78
|
+
super().__init__()
|
85
79
|
self._check_avp_schema_version()
|
86
80
|
|
87
81
|
@cached_property
|
88
82
|
def avp_facade(self):
|
89
83
|
return AwsAuthManagerAmazonVerifiedPermissionsFacade()
|
90
84
|
|
85
|
+
@cached_property
|
86
|
+
def fastapi_endpoint(self) -> str:
|
87
|
+
return conf.get("fastapi", "base_url")
|
88
|
+
|
91
89
|
def get_user(self) -> AwsAuthManagerUser | None:
|
92
90
|
return session["aws_user"] if self.is_logged_in() else None
|
93
91
|
|
94
92
|
def is_logged_in(self) -> bool:
|
95
93
|
return "aws_user" in session
|
96
94
|
|
95
|
+
def deserialize_user(self, token: dict[str, Any]) -> AwsAuthManagerUser:
|
96
|
+
return AwsAuthManagerUser(**token)
|
97
|
+
|
98
|
+
def serialize_user(self, user: AwsAuthManagerUser) -> dict[str, Any]:
|
99
|
+
return {
|
100
|
+
"user_id": user.get_id(),
|
101
|
+
"groups": user.get_groups(),
|
102
|
+
"username": user.username,
|
103
|
+
"email": user.email,
|
104
|
+
}
|
105
|
+
|
97
106
|
def is_authorized_configuration(
|
98
107
|
self,
|
99
108
|
*,
|
100
109
|
method: ResourceMethod,
|
110
|
+
user: AwsAuthManagerUser,
|
101
111
|
details: ConfigurationDetails | None = None,
|
102
|
-
user: BaseUser | None = None,
|
103
112
|
) -> bool:
|
104
113
|
config_section = details.section if details else None
|
105
114
|
return self.avp_facade.is_authorized(
|
106
115
|
method=method,
|
107
116
|
entity_type=AvpEntities.CONFIGURATION,
|
108
|
-
user=user
|
117
|
+
user=user,
|
109
118
|
entity_id=config_section,
|
110
119
|
)
|
111
120
|
|
@@ -113,14 +122,14 @@ class AwsAuthManager(BaseAuthManager):
|
|
113
122
|
self,
|
114
123
|
*,
|
115
124
|
method: ResourceMethod,
|
125
|
+
user: AwsAuthManagerUser,
|
116
126
|
details: ConnectionDetails | None = None,
|
117
|
-
user: BaseUser | None = None,
|
118
127
|
) -> bool:
|
119
128
|
connection_id = details.conn_id if details else None
|
120
129
|
return self.avp_facade.is_authorized(
|
121
130
|
method=method,
|
122
131
|
entity_type=AvpEntities.CONNECTION,
|
123
|
-
user=user
|
132
|
+
user=user,
|
124
133
|
entity_id=connection_id,
|
125
134
|
)
|
126
135
|
|
@@ -128,9 +137,9 @@ class AwsAuthManager(BaseAuthManager):
|
|
128
137
|
self,
|
129
138
|
*,
|
130
139
|
method: ResourceMethod,
|
140
|
+
user: AwsAuthManagerUser,
|
131
141
|
access_entity: DagAccessEntity | None = None,
|
132
142
|
details: DagDetails | None = None,
|
133
|
-
user: BaseUser | None = None,
|
134
143
|
) -> bool:
|
135
144
|
dag_id = details.id if details else None
|
136
145
|
context = (
|
@@ -145,48 +154,38 @@ class AwsAuthManager(BaseAuthManager):
|
|
145
154
|
return self.avp_facade.is_authorized(
|
146
155
|
method=method,
|
147
156
|
entity_type=AvpEntities.DAG,
|
148
|
-
user=user
|
157
|
+
user=user,
|
149
158
|
entity_id=dag_id,
|
150
159
|
context=context,
|
151
160
|
)
|
152
161
|
|
153
162
|
def is_authorized_asset(
|
154
|
-
self, *, method: ResourceMethod,
|
163
|
+
self, *, method: ResourceMethod, user: AwsAuthManagerUser, details: AssetDetails | None = None
|
155
164
|
) -> bool:
|
156
165
|
asset_uri = details.uri if details else None
|
157
166
|
return self.avp_facade.is_authorized(
|
158
|
-
method=method, entity_type=AvpEntities.ASSET, user=user
|
159
|
-
)
|
160
|
-
|
161
|
-
def is_authorized_dataset(
|
162
|
-
self, *, method: ResourceMethod, details: AssetDetails | None = None, user: BaseUser | None = None
|
163
|
-
) -> bool:
|
164
|
-
warnings.warn(
|
165
|
-
"is_authorized_dataset will be renamed as is_authorized_asset in Airflow 3 and will be removed when the minimum Airflow version is set to 3.0 for the amazon provider",
|
166
|
-
AirflowProviderDeprecationWarning,
|
167
|
-
stacklevel=2,
|
167
|
+
method=method, entity_type=AvpEntities.ASSET, user=user, entity_id=asset_uri
|
168
168
|
)
|
169
|
-
return self.is_authorized_asset(method=method, user=user)
|
170
169
|
|
171
170
|
def is_authorized_pool(
|
172
|
-
self, *, method: ResourceMethod,
|
171
|
+
self, *, method: ResourceMethod, user: AwsAuthManagerUser, details: PoolDetails | None = None
|
173
172
|
) -> bool:
|
174
173
|
pool_name = details.name if details else None
|
175
174
|
return self.avp_facade.is_authorized(
|
176
175
|
method=method,
|
177
176
|
entity_type=AvpEntities.POOL,
|
178
|
-
user=user
|
177
|
+
user=user,
|
179
178
|
entity_id=pool_name,
|
180
179
|
)
|
181
180
|
|
182
181
|
def is_authorized_variable(
|
183
|
-
self, *, method: ResourceMethod,
|
182
|
+
self, *, method: ResourceMethod, user: AwsAuthManagerUser, details: VariableDetails | None = None
|
184
183
|
) -> bool:
|
185
184
|
variable_key = details.key if details else None
|
186
185
|
return self.avp_facade.is_authorized(
|
187
186
|
method=method,
|
188
187
|
entity_type=AvpEntities.VARIABLE,
|
189
|
-
user=user
|
188
|
+
user=user,
|
190
189
|
entity_id=variable_key,
|
191
190
|
)
|
192
191
|
|
@@ -194,34 +193,31 @@ class AwsAuthManager(BaseAuthManager):
|
|
194
193
|
self,
|
195
194
|
*,
|
196
195
|
access_view: AccessView,
|
197
|
-
user:
|
196
|
+
user: AwsAuthManagerUser,
|
198
197
|
) -> bool:
|
199
198
|
return self.avp_facade.is_authorized(
|
200
199
|
method="GET",
|
201
200
|
entity_type=AvpEntities.VIEW,
|
202
|
-
user=user
|
201
|
+
user=user,
|
203
202
|
entity_id=access_view.value,
|
204
203
|
)
|
205
204
|
|
206
205
|
def is_authorized_custom_view(
|
207
|
-
self, *, method: ResourceMethod | str, resource_name: str, user:
|
206
|
+
self, *, method: ResourceMethod | str, resource_name: str, user: AwsAuthManagerUser
|
208
207
|
):
|
209
208
|
return self.avp_facade.is_authorized(
|
210
209
|
method=method,
|
211
210
|
entity_type=AvpEntities.CUSTOM,
|
212
|
-
user=user
|
211
|
+
user=user,
|
213
212
|
entity_id=resource_name,
|
214
213
|
)
|
215
214
|
|
216
215
|
def batch_is_authorized_connection(
|
217
216
|
self,
|
218
217
|
requests: Sequence[IsAuthorizedConnectionRequest],
|
218
|
+
*,
|
219
|
+
user: AwsAuthManagerUser,
|
219
220
|
) -> bool:
|
220
|
-
"""
|
221
|
-
Batch version of ``is_authorized_connection``.
|
222
|
-
|
223
|
-
:param requests: a list of requests containing the parameters for ``is_authorized_connection``
|
224
|
-
"""
|
225
221
|
facade_requests: Sequence[IsAuthorizedRequest] = [
|
226
222
|
{
|
227
223
|
"method": request["method"],
|
@@ -232,17 +228,14 @@ class AwsAuthManager(BaseAuthManager):
|
|
232
228
|
}
|
233
229
|
for request in requests
|
234
230
|
]
|
235
|
-
return self.avp_facade.batch_is_authorized(requests=facade_requests, user=
|
231
|
+
return self.avp_facade.batch_is_authorized(requests=facade_requests, user=user)
|
236
232
|
|
237
233
|
def batch_is_authorized_dag(
|
238
234
|
self,
|
239
235
|
requests: Sequence[IsAuthorizedDagRequest],
|
236
|
+
*,
|
237
|
+
user: AwsAuthManagerUser,
|
240
238
|
) -> bool:
|
241
|
-
"""
|
242
|
-
Batch version of ``is_authorized_dag``.
|
243
|
-
|
244
|
-
:param requests: a list of requests containing the parameters for ``is_authorized_dag``
|
245
|
-
"""
|
246
239
|
facade_requests: Sequence[IsAuthorizedRequest] = [
|
247
240
|
{
|
248
241
|
"method": request["method"],
|
@@ -258,17 +251,14 @@ class AwsAuthManager(BaseAuthManager):
|
|
258
251
|
}
|
259
252
|
for request in requests
|
260
253
|
]
|
261
|
-
return self.avp_facade.batch_is_authorized(requests=facade_requests, user=
|
254
|
+
return self.avp_facade.batch_is_authorized(requests=facade_requests, user=user)
|
262
255
|
|
263
256
|
def batch_is_authorized_pool(
|
264
257
|
self,
|
265
258
|
requests: Sequence[IsAuthorizedPoolRequest],
|
259
|
+
*,
|
260
|
+
user: AwsAuthManagerUser,
|
266
261
|
) -> bool:
|
267
|
-
"""
|
268
|
-
Batch version of ``is_authorized_pool``.
|
269
|
-
|
270
|
-
:param requests: a list of requests containing the parameters for ``is_authorized_pool``
|
271
|
-
"""
|
272
262
|
facade_requests: Sequence[IsAuthorizedRequest] = [
|
273
263
|
{
|
274
264
|
"method": request["method"],
|
@@ -277,17 +267,14 @@ class AwsAuthManager(BaseAuthManager):
|
|
277
267
|
}
|
278
268
|
for request in requests
|
279
269
|
]
|
280
|
-
return self.avp_facade.batch_is_authorized(requests=facade_requests, user=
|
270
|
+
return self.avp_facade.batch_is_authorized(requests=facade_requests, user=user)
|
281
271
|
|
282
272
|
def batch_is_authorized_variable(
|
283
273
|
self,
|
284
274
|
requests: Sequence[IsAuthorizedVariableRequest],
|
275
|
+
*,
|
276
|
+
user: AwsAuthManagerUser,
|
285
277
|
) -> bool:
|
286
|
-
"""
|
287
|
-
Batch version of ``is_authorized_variable``.
|
288
|
-
|
289
|
-
:param requests: a list of requests containing the parameters for ``is_authorized_variable``
|
290
|
-
"""
|
291
278
|
facade_requests: Sequence[IsAuthorizedRequest] = [
|
292
279
|
{
|
293
280
|
"method": request["method"],
|
@@ -298,39 +285,29 @@ class AwsAuthManager(BaseAuthManager):
|
|
298
285
|
}
|
299
286
|
for request in requests
|
300
287
|
]
|
301
|
-
return self.avp_facade.batch_is_authorized(requests=facade_requests, user=
|
288
|
+
return self.avp_facade.batch_is_authorized(requests=facade_requests, user=user)
|
302
289
|
|
303
290
|
def filter_permitted_dag_ids(
|
304
291
|
self,
|
305
292
|
*,
|
306
293
|
dag_ids: set[str],
|
294
|
+
user: AwsAuthManagerUser,
|
307
295
|
methods: Container[ResourceMethod] | None = None,
|
308
|
-
user=None,
|
309
296
|
):
|
310
|
-
"""
|
311
|
-
Filter readable or writable DAGs for user.
|
312
|
-
|
313
|
-
:param dag_ids: the list of DAG ids
|
314
|
-
:param methods: whether filter readable or writable
|
315
|
-
:param user: the current user
|
316
|
-
"""
|
317
297
|
if not methods:
|
318
298
|
methods = ["PUT", "GET"]
|
319
299
|
|
320
|
-
if not user:
|
321
|
-
user = self.get_user()
|
322
|
-
|
323
300
|
requests: dict[str, dict[ResourceMethod, IsAuthorizedRequest]] = defaultdict(dict)
|
324
301
|
requests_list: list[IsAuthorizedRequest] = []
|
325
302
|
for dag_id in dag_ids:
|
326
303
|
for method in ["GET", "PUT"]:
|
327
304
|
if method in methods:
|
328
305
|
request: IsAuthorizedRequest = {
|
329
|
-
"method": cast(ResourceMethod, method),
|
306
|
+
"method": cast("ResourceMethod", method),
|
330
307
|
"entity_type": AvpEntities.DAG,
|
331
308
|
"entity_id": dag_id,
|
332
309
|
}
|
333
|
-
requests[dag_id][cast(ResourceMethod, method)] = request
|
310
|
+
requests[dag_id][cast("ResourceMethod", method)] = request
|
334
311
|
requests_list.append(request)
|
335
312
|
|
336
313
|
batch_is_authorized_results = self.avp_facade.get_batch_is_authorized_results(
|
@@ -400,14 +377,10 @@ class AwsAuthManager(BaseAuthManager):
|
|
400
377
|
return accessible_items
|
401
378
|
|
402
379
|
def get_url_login(self, **kwargs) -> str:
|
403
|
-
return
|
380
|
+
return f"{self.fastapi_endpoint}/auth/login"
|
404
381
|
|
405
382
|
def get_url_logout(self) -> str:
|
406
|
-
|
407
|
-
|
408
|
-
@cached_property
|
409
|
-
def security_manager(self) -> AwsSecurityManagerOverride:
|
410
|
-
return AwsSecurityManagerOverride(self.appbuilder)
|
383
|
+
raise NotImplementedError()
|
411
384
|
|
412
385
|
@staticmethod
|
413
386
|
def get_cli_commands() -> list[CLICommand]:
|
@@ -420,9 +393,20 @@ class AwsAuthManager(BaseAuthManager):
|
|
420
393
|
),
|
421
394
|
]
|
422
395
|
|
423
|
-
def
|
424
|
-
|
425
|
-
|
396
|
+
def get_fastapi_app(self) -> FastAPI | None:
|
397
|
+
from airflow.providers.amazon.aws.auth_manager.router.login import login_router
|
398
|
+
|
399
|
+
app = FastAPI(
|
400
|
+
title="AWS auth manager sub application",
|
401
|
+
description=(
|
402
|
+
"This is the AWS auth manager fastapi sub application. This API is only available if the "
|
403
|
+
"auth manager used in the Airflow environment is AWS auth manager. "
|
404
|
+
"This sub application provides login routes."
|
405
|
+
),
|
406
|
+
)
|
407
|
+
app.include_router(login_router)
|
408
|
+
|
409
|
+
return app
|
426
410
|
|
427
411
|
@staticmethod
|
428
412
|
def _get_menu_item_request(resource_name: str) -> IsAuthorizedRequest:
|
@@ -436,7 +420,7 @@ class AwsAuthManager(BaseAuthManager):
|
|
436
420
|
if not self.avp_facade.is_policy_store_schema_up_to_date():
|
437
421
|
self.log.warning(
|
438
422
|
"The Amazon Verified Permissions policy store schema is different from the latest version "
|
439
|
-
"(https://github.com/apache/airflow/blob/main/airflow/providers/amazon/aws/auth_manager/avp/schema.json). "
|
423
|
+
"(https://github.com/apache/airflow/blob/main/providers/amazon/aws/src/airflow/providers/amazon/aws/auth_manager/avp/schema.json). "
|
440
424
|
"Please update it to its latest version. "
|
441
425
|
"See doc: https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/auth-manager/setup/amazon-verified-permissions.html#update-the-policy-store-schema."
|
442
426
|
)
|
@@ -0,0 +1,124 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
from __future__ import annotations
|
19
|
+
|
20
|
+
import logging
|
21
|
+
from typing import Any
|
22
|
+
|
23
|
+
import anyio
|
24
|
+
from fastapi import HTTPException, Request
|
25
|
+
from starlette import status
|
26
|
+
from starlette.responses import RedirectResponse
|
27
|
+
|
28
|
+
from airflow.api_fastapi.app import get_auth_manager
|
29
|
+
from airflow.api_fastapi.common.router import AirflowRouter
|
30
|
+
from airflow.configuration import conf
|
31
|
+
from airflow.providers.amazon.aws.auth_manager.constants import CONF_SAML_METADATA_URL_KEY, CONF_SECTION_NAME
|
32
|
+
from airflow.providers.amazon.aws.auth_manager.user import AwsAuthManagerUser
|
33
|
+
|
34
|
+
try:
|
35
|
+
from onelogin.saml2.auth import OneLogin_Saml2_Auth
|
36
|
+
from onelogin.saml2.errors import OneLogin_Saml2_Error
|
37
|
+
from onelogin.saml2.idp_metadata_parser import OneLogin_Saml2_IdPMetadataParser
|
38
|
+
except ImportError:
|
39
|
+
raise ImportError(
|
40
|
+
"AWS auth manager requires the python3-saml library but it is not installed by default. "
|
41
|
+
"Please install the python3-saml library by running: "
|
42
|
+
"pip install apache-airflow-providers-amazon[python3-saml]"
|
43
|
+
)
|
44
|
+
|
45
|
+
log = logging.getLogger(__name__)
|
46
|
+
login_router = AirflowRouter(tags=["AWSAuthManagerLogin"])
|
47
|
+
|
48
|
+
|
49
|
+
@login_router.get("/login")
|
50
|
+
def login(request: Request):
|
51
|
+
"""Authenticate the user."""
|
52
|
+
saml_auth = _init_saml_auth(request)
|
53
|
+
callback_url = saml_auth.login()
|
54
|
+
return RedirectResponse(url=callback_url)
|
55
|
+
|
56
|
+
|
57
|
+
@login_router.post("/login_callback")
|
58
|
+
def login_callback(request: Request):
|
59
|
+
"""Authenticate the user."""
|
60
|
+
saml_auth = _init_saml_auth(request)
|
61
|
+
try:
|
62
|
+
saml_auth.process_response()
|
63
|
+
except OneLogin_Saml2_Error as e:
|
64
|
+
log.exception(e)
|
65
|
+
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, "Failed to authenticate")
|
66
|
+
errors = saml_auth.get_errors()
|
67
|
+
is_authenticated = saml_auth.is_authenticated()
|
68
|
+
if not is_authenticated:
|
69
|
+
error_reason = saml_auth.get_last_error_reason()
|
70
|
+
log.error("Failed to authenticate")
|
71
|
+
log.error("Errors: %s", errors)
|
72
|
+
log.error("Error reason: %s", error_reason)
|
73
|
+
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, f"Failed to authenticate: {error_reason}")
|
74
|
+
|
75
|
+
attributes = saml_auth.get_attributes()
|
76
|
+
user = AwsAuthManagerUser(
|
77
|
+
user_id=attributes["id"][0],
|
78
|
+
groups=attributes["groups"],
|
79
|
+
username=saml_auth.get_nameid(),
|
80
|
+
email=attributes["email"][0] if "email" in attributes else None,
|
81
|
+
)
|
82
|
+
return RedirectResponse(url=f"/webapp?token={get_auth_manager().get_jwt_token(user)}", status_code=303)
|
83
|
+
|
84
|
+
|
85
|
+
def _init_saml_auth(request: Request) -> OneLogin_Saml2_Auth:
|
86
|
+
request_data = _prepare_request(request)
|
87
|
+
base_url = conf.get(section="fastapi", key="base_url")
|
88
|
+
settings = {
|
89
|
+
# We want to keep this flag on in case of errors.
|
90
|
+
# It provides an error reasons, if turned off, it does not
|
91
|
+
"debug": True,
|
92
|
+
"sp": {
|
93
|
+
"entityId": "aws-auth-manager-saml-client",
|
94
|
+
"assertionConsumerService": {
|
95
|
+
"url": f"{base_url}/auth/login_callback",
|
96
|
+
"binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST",
|
97
|
+
},
|
98
|
+
},
|
99
|
+
}
|
100
|
+
merged_settings = OneLogin_Saml2_IdPMetadataParser.merge_settings(_get_idp_data(), settings)
|
101
|
+
return OneLogin_Saml2_Auth(request_data, merged_settings)
|
102
|
+
|
103
|
+
|
104
|
+
def _prepare_request(request: Request) -> dict:
|
105
|
+
host = request.headers.get("host", request.client.host if request.client else "localhost")
|
106
|
+
data: dict[str, Any] = {
|
107
|
+
"https": "on" if request.url.scheme == "https" else "off",
|
108
|
+
"http_host": host,
|
109
|
+
"server_port": request.url.port,
|
110
|
+
"script_name": request.url.path,
|
111
|
+
"get_data": request.query_params,
|
112
|
+
"post_data": {},
|
113
|
+
}
|
114
|
+
form_data = anyio.from_thread.run(request.form)
|
115
|
+
if "SAMLResponse" in form_data:
|
116
|
+
data["post_data"]["SAMLResponse"] = form_data["SAMLResponse"]
|
117
|
+
if "RelayState" in form_data:
|
118
|
+
data["post_data"]["RelayState"] = form_data["RelayState"]
|
119
|
+
return data
|
120
|
+
|
121
|
+
|
122
|
+
def _get_idp_data() -> dict:
|
123
|
+
saml_metadata_url = conf.get_mandatory_value(CONF_SECTION_NAME, CONF_SAML_METADATA_URL_KEY)
|
124
|
+
return OneLogin_Saml2_IdPMetadataParser.parse_remote(saml_metadata_url)
|
@@ -56,7 +56,7 @@ from airflow.providers.amazon.aws.executors.batch.utils import (
|
|
56
56
|
)
|
57
57
|
from airflow.utils.state import State
|
58
58
|
|
59
|
-
CommandType =
|
59
|
+
CommandType = Sequence[str]
|
60
60
|
ExecutorConfigType = dict[str, Any]
|
61
61
|
|
62
62
|
INVALID_CREDENTIALS_EXCEPTIONS = [
|
@@ -350,7 +350,7 @@ class AwsBatchExecutor(BaseExecutor):
|
|
350
350
|
self.pending_jobs.append(
|
351
351
|
BatchQueuedJob(
|
352
352
|
key=key,
|
353
|
-
command=command,
|
353
|
+
command=list(command),
|
354
354
|
queue=queue,
|
355
355
|
executor_config=executor_config or {},
|
356
356
|
attempt_number=1,
|
@@ -56,7 +56,7 @@ class BotoTaskSchema(Schema):
|
|
56
56
|
last_status = fields.String(data_key="lastStatus", required=True)
|
57
57
|
desired_status = fields.String(data_key="desiredStatus", required=True)
|
58
58
|
containers = fields.List(fields.Nested(BotoContainerSchema), required=True)
|
59
|
-
started_at = fields.
|
59
|
+
started_at = fields.Raw(data_key="startedAt")
|
60
60
|
stopped_reason = fields.String(data_key="stoppedReason")
|
61
61
|
|
62
62
|
@post_load
|
@@ -25,6 +25,7 @@ from __future__ import annotations
|
|
25
25
|
|
26
26
|
import datetime
|
27
27
|
from collections import defaultdict
|
28
|
+
from collections.abc import Sequence
|
28
29
|
from dataclasses import dataclass
|
29
30
|
from typing import TYPE_CHECKING, Any, Callable
|
30
31
|
|
@@ -36,7 +37,7 @@ from airflow.utils.state import State
|
|
36
37
|
if TYPE_CHECKING:
|
37
38
|
from airflow.models.taskinstance import TaskInstanceKey
|
38
39
|
|
39
|
-
CommandType =
|
40
|
+
CommandType = Sequence[str]
|
40
41
|
ExecutorConfigFunctionType = Callable[[CommandType], dict]
|
41
42
|
ExecutorConfigType = dict[str, Any]
|
42
43
|
|