qontract-reconcile 0.10.2.dev51__py3-none-any.whl → 0.10.2.dev52__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qontract_reconcile-0.10.2.dev51.dist-info → qontract_reconcile-0.10.2.dev52.dist-info}/METADATA +1 -1
- {qontract_reconcile-0.10.2.dev51.dist-info → qontract_reconcile-0.10.2.dev52.dist-info}/RECORD +25 -26
- reconcile/aws_cloudwatch_log_retention/integration.py +10 -17
- reconcile/dashdotdb_dora.py +5 -4
- reconcile/gitlab_housekeeping.py +10 -6
- reconcile/terraform_tgw_attachments.py +5 -5
- reconcile/terraform_vpc_peerings.py +1 -1
- reconcile/utils/aggregated_list.py +30 -20
- reconcile/utils/aws_api.py +596 -169
- reconcile/utils/aws_helper.py +7 -7
- reconcile/utils/binary.py +14 -7
- reconcile/utils/config.py +9 -6
- reconcile/utils/defer.py +4 -2
- reconcile/utils/elasticsearch_exceptions.py +7 -4
- reconcile/utils/environ.py +5 -3
- reconcile/utils/exceptions.py +5 -2
- reconcile/utils/git.py +6 -4
- reconcile/utils/gitlab_api.py +103 -82
- reconcile/utils/mr/base.py +6 -3
- reconcile/utils/mr/update_access_report_base.py +2 -2
- reconcile/utils/vcs.py +5 -3
- reconcile/vpc_peerings_validator.py +21 -15
- tools/qontract_cli.py +26 -17
- reconcile/utils/data_structures.py +0 -13
- {qontract_reconcile-0.10.2.dev51.dist-info → qontract_reconcile-0.10.2.dev52.dist-info}/WHEEL +0 -0
- {qontract_reconcile-0.10.2.dev51.dist-info → qontract_reconcile-0.10.2.dev52.dist-info}/entry_points.txt +0 -0
reconcile/utils/aws_api.py
CHANGED
@@ -16,19 +16,24 @@ from typing import (
|
|
16
16
|
TYPE_CHECKING,
|
17
17
|
Any,
|
18
18
|
Literal,
|
19
|
+
Self,
|
20
|
+
cast,
|
21
|
+
overload,
|
19
22
|
)
|
20
23
|
|
21
24
|
import botocore
|
22
25
|
from boto3 import Session
|
26
|
+
from botocore.client import BaseClient
|
23
27
|
from botocore.config import Config
|
24
28
|
from pydantic import BaseModel
|
25
29
|
from sretoolbox.utils import threaded
|
26
30
|
|
27
31
|
import reconcile.utils.aws_helper as awsh
|
28
32
|
import reconcile.utils.lean_terraform_client as terraform
|
29
|
-
from reconcile.utils.secret_reader import SecretReader
|
33
|
+
from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
|
30
34
|
|
31
35
|
if TYPE_CHECKING:
|
36
|
+
from mypy_boto3_dynamodb import DynamoDBClient, DynamoDBServiceResource
|
32
37
|
from mypy_boto3_ec2 import (
|
33
38
|
EC2Client,
|
34
39
|
EC2ServiceResource,
|
@@ -45,12 +50,17 @@ if TYPE_CHECKING:
|
|
45
50
|
VpcEndpointTypeDef,
|
46
51
|
VpcTypeDef,
|
47
52
|
)
|
48
|
-
from
|
53
|
+
from mypy_boto3_ecr import ECRClient
|
54
|
+
from mypy_boto3_elb import ElasticLoadBalancingClient
|
55
|
+
from mypy_boto3_iam import IAMClient, IAMServiceResource
|
49
56
|
from mypy_boto3_iam.type_defs import AccessKeyMetadataTypeDef
|
57
|
+
from mypy_boto3_logs import CloudWatchLogsClient
|
58
|
+
from mypy_boto3_logs.type_defs import LogGroupTypeDef
|
50
59
|
from mypy_boto3_organizations import OrganizationsClient
|
51
60
|
from mypy_boto3_rds import RDSClient
|
52
61
|
from mypy_boto3_rds.type_defs import (
|
53
62
|
DBInstanceMessageTypeDef,
|
63
|
+
DBRecommendationsMessageTypeDef,
|
54
64
|
UpgradeTargetTypeDef,
|
55
65
|
)
|
56
66
|
from mypy_boto3_route53 import Route53Client
|
@@ -59,19 +69,30 @@ if TYPE_CHECKING:
|
|
59
69
|
ResourceRecordSetTypeDef,
|
60
70
|
ResourceRecordTypeDef,
|
61
71
|
)
|
62
|
-
from mypy_boto3_s3 import S3Client
|
72
|
+
from mypy_boto3_s3 import S3Client, S3ServiceResource
|
73
|
+
from mypy_boto3_sqs import SQSClient, SQSServiceResource
|
74
|
+
from mypy_boto3_sts import STSClient
|
75
|
+
from mypy_boto3_support import SupportClient
|
76
|
+
from mypy_boto3_support.type_defs import CaseDetailsTypeDef
|
77
|
+
|
63
78
|
else:
|
64
|
-
|
65
|
-
|
66
|
-
) =
|
67
|
-
|
68
|
-
) =
|
69
|
-
FilterTypeDef
|
70
|
-
) = Route53Client = ResourceRecordSetTypeDef = ResourceRecordTypeDef = (
|
79
|
+
AccessKeyMetadataTypeDef = CaseDetailsTypeDef = CloudWatchLogsClient = (
|
80
|
+
DBInstanceMessageTypeDef
|
81
|
+
) = DBRecommendationsMessageTypeDef = DynamoDBClient = DynamoDBServiceResource = (
|
82
|
+
EC2Client
|
83
|
+
) = EC2ServiceResource = ECRClient = ElasticLoadBalancingClient = FilterTypeDef = (
|
71
84
|
HostedZoneTypeDef
|
72
|
-
) =
|
73
|
-
|
74
|
-
) =
|
85
|
+
) = IAMClient = IAMServiceResource = ImageTypeDef = (
|
86
|
+
LaunchPermissionModificationsTypeDef
|
87
|
+
) = LogGroupTypeDef = OrganizationsClient = RDSClient = ResourceRecordSetTypeDef = (
|
88
|
+
ResourceRecordTypeDef
|
89
|
+
) = Route53Client = RouteTableTypeDef = S3Client = S3ServiceResource = SQSClient = (
|
90
|
+
SQSServiceResource
|
91
|
+
) = STSClient = SubnetTypeDef = SupportClient = TagTypeDef = (
|
92
|
+
TransitGatewayTypeDef
|
93
|
+
) = TransitGatewayVpcAttachmentTypeDef = UpgradeTargetTypeDef = (
|
94
|
+
VpcEndpointTypeDef
|
95
|
+
) = VpcTypeDef = object
|
75
96
|
|
76
97
|
|
77
98
|
class InvalidResourceTypeError(Exception):
|
@@ -82,7 +103,7 @@ class MissingARNError(Exception):
|
|
82
103
|
pass
|
83
104
|
|
84
105
|
|
85
|
-
KeyStatus = Literal["Active"
|
106
|
+
KeyStatus = Literal["Active", "Inactive"]
|
86
107
|
|
87
108
|
GOVCLOUD_PARTITION = "aws-us-gov"
|
88
109
|
|
@@ -92,19 +113,50 @@ class AmiTag(BaseModel):
|
|
92
113
|
value: str
|
93
114
|
|
94
115
|
|
95
|
-
|
116
|
+
SERVICE_NAME = Literal[
|
117
|
+
"dynamodb",
|
118
|
+
"ec2",
|
119
|
+
"ecr",
|
120
|
+
"elb",
|
121
|
+
"iam",
|
122
|
+
"logs",
|
123
|
+
"organizations",
|
124
|
+
"rds",
|
125
|
+
"route53",
|
126
|
+
"s3",
|
127
|
+
"sqs",
|
128
|
+
"sts",
|
129
|
+
"support",
|
130
|
+
]
|
131
|
+
RESOURCE_NAME = Literal[
|
132
|
+
"dynamodb",
|
133
|
+
"ec2",
|
134
|
+
"iam",
|
135
|
+
"s3",
|
136
|
+
"sqs",
|
137
|
+
]
|
138
|
+
RESOURCE_TYPE = Literal[
|
139
|
+
"dynamodb",
|
140
|
+
"rds",
|
141
|
+
"rds_snapshots",
|
142
|
+
"s3",
|
143
|
+
"sqs",
|
144
|
+
]
|
145
|
+
|
146
|
+
|
147
|
+
class AWSApi:
|
96
148
|
"""Wrapper around AWS SDK"""
|
97
149
|
|
98
150
|
def __init__(
|
99
151
|
self,
|
100
|
-
thread_pool_size,
|
101
|
-
accounts,
|
102
|
-
settings=None,
|
103
|
-
secret_reader=None,
|
104
|
-
init_ecr_auth_tokens=False,
|
105
|
-
init_users=True,
|
106
|
-
):
|
107
|
-
self._session_clients = []
|
152
|
+
thread_pool_size: int,
|
153
|
+
accounts: Iterable[awsh.Account],
|
154
|
+
settings: Mapping | None = None,
|
155
|
+
secret_reader: SecretReaderBase | None = None,
|
156
|
+
init_ecr_auth_tokens: bool = False,
|
157
|
+
init_users: bool = True,
|
158
|
+
) -> None:
|
159
|
+
self._session_clients: list[BaseClient] = []
|
108
160
|
self.thread_pool_size = thread_pool_size
|
109
161
|
if secret_reader:
|
110
162
|
self.secret_reader = secret_reader
|
@@ -116,7 +168,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
116
168
|
if init_users:
|
117
169
|
self.init_users()
|
118
170
|
self._lock = Lock()
|
119
|
-
self.resource_types = [
|
171
|
+
self.resource_types: list[RESOURCE_TYPE] = [
|
172
|
+
"s3",
|
173
|
+
"sqs",
|
174
|
+
"dynamodb",
|
175
|
+
"rds",
|
176
|
+
"rds_snapshots",
|
177
|
+
]
|
120
178
|
|
121
179
|
# store the app-interface accounts in a dictionary indexed by name
|
122
180
|
self.accounts = {acc["name"]: acc for acc in accounts}
|
@@ -125,21 +183,21 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
125
183
|
# https://stackoverflow.com/questions/33672412/python-functools-lru-cache-with-class-methods-release-object
|
126
184
|
# using @lru_cache decorators on methods would lek AWSApi instances
|
127
185
|
# since the cache keeps a reference to self.
|
128
|
-
self._get_assume_role_session = lru_cache()(self._get_assume_role_session)
|
129
|
-
self._get_session_resource = lru_cache()(self._get_session_resource)
|
130
|
-
self.get_account_amis = lru_cache()(self.get_account_amis)
|
131
|
-
self.get_account_vpcs = lru_cache()(self.get_account_vpcs)
|
132
|
-
self.get_session_client = lru_cache()(self.get_session_client)
|
133
|
-
self.get_transit_gateway_vpc_attachments = lru_cache()(
|
186
|
+
self._get_assume_role_session = lru_cache()(self._get_assume_role_session) # type: ignore[method-assign]
|
187
|
+
self._get_session_resource = lru_cache()(self._get_session_resource) # type: ignore[method-assign, assignment]
|
188
|
+
self.get_account_amis = lru_cache()(self.get_account_amis) # type: ignore[method-assign]
|
189
|
+
self.get_account_vpcs = lru_cache()(self.get_account_vpcs) # type: ignore[method-assign]
|
190
|
+
self.get_session_client = lru_cache()(self.get_session_client) # type: ignore[method-assign, assignment]
|
191
|
+
self.get_transit_gateway_vpc_attachments = lru_cache()( # type: ignore[method-assign]
|
134
192
|
self.get_transit_gateway_vpc_attachments
|
135
193
|
)
|
136
|
-
self.get_transit_gateways = lru_cache()(self.get_transit_gateways)
|
137
|
-
self.get_vpc_default_sg_id = lru_cache()(self.get_vpc_default_sg_id)
|
138
|
-
self.get_vpc_route_tables = lru_cache()(self.get_vpc_route_tables)
|
139
|
-
self.get_vpc_subnets = lru_cache()(self.get_vpc_subnets)
|
140
|
-
self._get_vpc_endpoints = lru_cache()(self._get_vpc_endpoints)
|
194
|
+
self.get_transit_gateways = lru_cache()(self.get_transit_gateways) # type: ignore[method-assign]
|
195
|
+
self.get_vpc_default_sg_id = lru_cache()(self.get_vpc_default_sg_id) # type: ignore[method-assign]
|
196
|
+
self.get_vpc_route_tables = lru_cache()(self.get_vpc_route_tables) # type: ignore[method-assign]
|
197
|
+
self.get_vpc_subnets = lru_cache()(self.get_vpc_subnets) # type: ignore[method-assign]
|
198
|
+
self._get_vpc_endpoints = lru_cache()(self._get_vpc_endpoints) # type: ignore[method-assign]
|
141
199
|
|
142
|
-
def init_sessions_and_resources(self, accounts: Iterable[awsh.Account]):
|
200
|
+
def init_sessions_and_resources(self, accounts: Iterable[awsh.Account]) -> None:
|
143
201
|
results = threaded.run(
|
144
202
|
awsh.get_tf_secrets,
|
145
203
|
accounts,
|
@@ -168,13 +226,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
168
226
|
self.sessions[account_name] = session
|
169
227
|
self.resources[account_name] = {}
|
170
228
|
|
171
|
-
def __enter__(self):
|
229
|
+
def __enter__(self) -> Self:
|
172
230
|
return self
|
173
231
|
|
174
|
-
def __exit__(self, *exc):
|
232
|
+
def __exit__(self, *exc: Any) -> None:
|
175
233
|
self.cleanup()
|
176
234
|
|
177
|
-
def cleanup(self):
|
235
|
+
def cleanup(self) -> None:
|
178
236
|
"""
|
179
237
|
Close all session clients
|
180
238
|
:return:
|
@@ -185,12 +243,129 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
185
243
|
def get_session(self, account: str) -> Session:
|
186
244
|
return self.sessions[account]
|
187
245
|
|
188
|
-
|
246
|
+
@overload
|
247
|
+
def get_session_client(
|
248
|
+
self,
|
249
|
+
session: Session,
|
250
|
+
service_name: Literal["ec2"],
|
251
|
+
region_name: str | None = None,
|
252
|
+
) -> EC2Client: ...
|
253
|
+
|
254
|
+
@overload
|
255
|
+
def get_session_client(
|
256
|
+
self,
|
257
|
+
session: Session,
|
258
|
+
service_name: Literal["elb"],
|
259
|
+
region_name: str | None = None,
|
260
|
+
) -> ElasticLoadBalancingClient: ...
|
261
|
+
|
262
|
+
@overload
|
263
|
+
def get_session_client(
|
264
|
+
self,
|
265
|
+
session: Session,
|
266
|
+
service_name: Literal["route53"],
|
267
|
+
region_name: str | None = None,
|
268
|
+
) -> Route53Client: ...
|
269
|
+
|
270
|
+
@overload
|
271
|
+
def get_session_client(
|
272
|
+
self,
|
273
|
+
session: Session,
|
274
|
+
service_name: Literal["rds"],
|
275
|
+
region_name: str | None = None,
|
276
|
+
) -> RDSClient: ...
|
277
|
+
|
278
|
+
@overload
|
279
|
+
def get_session_client(
|
280
|
+
self,
|
281
|
+
session: Session,
|
282
|
+
service_name: Literal["logs"],
|
283
|
+
region_name: str | None = None,
|
284
|
+
) -> CloudWatchLogsClient: ...
|
285
|
+
|
286
|
+
@overload
|
287
|
+
def get_session_client(
|
288
|
+
self,
|
289
|
+
session: Session,
|
290
|
+
service_name: Literal["organizations"],
|
291
|
+
region_name: str | None = None,
|
292
|
+
) -> OrganizationsClient: ...
|
293
|
+
|
294
|
+
@overload
|
295
|
+
def get_session_client(
|
296
|
+
self,
|
297
|
+
session: Session,
|
298
|
+
service_name: Literal["s3"],
|
299
|
+
region_name: str | None = None,
|
300
|
+
) -> S3Client: ...
|
301
|
+
|
302
|
+
@overload
|
303
|
+
def get_session_client(
|
304
|
+
self,
|
305
|
+
session: Session,
|
306
|
+
service_name: Literal["iam"],
|
307
|
+
region_name: str | None = None,
|
308
|
+
) -> IAMClient: ...
|
309
|
+
|
310
|
+
@overload
|
311
|
+
def get_session_client(
|
312
|
+
self,
|
313
|
+
session: Session,
|
314
|
+
service_name: Literal["sqs"],
|
315
|
+
region_name: str | None = None,
|
316
|
+
) -> SQSClient: ...
|
317
|
+
|
318
|
+
@overload
|
319
|
+
def get_session_client(
|
320
|
+
self,
|
321
|
+
session: Session,
|
322
|
+
service_name: Literal["dynamodb"],
|
323
|
+
region_name: str | None = None,
|
324
|
+
) -> DynamoDBClient: ...
|
325
|
+
|
326
|
+
@overload
|
327
|
+
def get_session_client(
|
328
|
+
self,
|
329
|
+
session: Session,
|
330
|
+
service_name: Literal["ecr"],
|
331
|
+
region_name: str | None = None,
|
332
|
+
) -> ECRClient: ...
|
333
|
+
|
334
|
+
@overload
|
189
335
|
def get_session_client(
|
190
336
|
self,
|
191
337
|
session: Session,
|
192
|
-
service_name,
|
338
|
+
service_name: Literal["support"],
|
193
339
|
region_name: str | None = None,
|
340
|
+
) -> SupportClient: ...
|
341
|
+
|
342
|
+
@overload
|
343
|
+
def get_session_client(
|
344
|
+
self,
|
345
|
+
session: Session,
|
346
|
+
service_name: Literal["sts"],
|
347
|
+
region_name: str | None = None,
|
348
|
+
) -> STSClient: ...
|
349
|
+
|
350
|
+
def get_session_client(
|
351
|
+
self,
|
352
|
+
session: Session,
|
353
|
+
service_name: SERVICE_NAME,
|
354
|
+
region_name: str | None = None,
|
355
|
+
) -> (
|
356
|
+
CloudWatchLogsClient
|
357
|
+
| DynamoDBClient
|
358
|
+
| EC2Client
|
359
|
+
| ECRClient
|
360
|
+
| ElasticLoadBalancingClient
|
361
|
+
| IAMClient
|
362
|
+
| OrganizationsClient
|
363
|
+
| RDSClient
|
364
|
+
| Route53Client
|
365
|
+
| S3Client
|
366
|
+
| SQSClient
|
367
|
+
| STSClient
|
368
|
+
| SupportClient
|
194
369
|
):
|
195
370
|
region = region_name or session.region_name
|
196
371
|
client = session.client(
|
@@ -201,10 +376,55 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
201
376
|
self._session_clients.append(client)
|
202
377
|
return client
|
203
378
|
|
379
|
+
@overload
|
380
|
+
@staticmethod
|
381
|
+
def _get_session_resource(
|
382
|
+
session: Session,
|
383
|
+
service_name: Literal["dynamodb"],
|
384
|
+
region_name: str | None = None,
|
385
|
+
) -> DynamoDBServiceResource: ...
|
386
|
+
|
387
|
+
@overload
|
388
|
+
@staticmethod
|
389
|
+
def _get_session_resource(
|
390
|
+
session: Session,
|
391
|
+
service_name: Literal["ec2"],
|
392
|
+
region_name: str | None = None,
|
393
|
+
) -> EC2ServiceResource: ...
|
394
|
+
|
395
|
+
@overload
|
396
|
+
@staticmethod
|
397
|
+
def _get_session_resource(
|
398
|
+
session: Session,
|
399
|
+
service_name: Literal["iam"],
|
400
|
+
region_name: str | None = None,
|
401
|
+
) -> IAMServiceResource: ...
|
402
|
+
|
403
|
+
@overload
|
204
404
|
@staticmethod
|
205
|
-
# pylint: disable=method-hidden
|
206
405
|
def _get_session_resource(
|
207
|
-
session: Session,
|
406
|
+
session: Session,
|
407
|
+
service_name: Literal["s3"],
|
408
|
+
region_name: str | None = None,
|
409
|
+
) -> S3ServiceResource: ...
|
410
|
+
|
411
|
+
@overload
|
412
|
+
@staticmethod
|
413
|
+
def _get_session_resource(
|
414
|
+
session: Session,
|
415
|
+
service_name: Literal["sqs"],
|
416
|
+
region_name: str | None = None,
|
417
|
+
) -> SQSServiceResource: ...
|
418
|
+
|
419
|
+
@staticmethod
|
420
|
+
def _get_session_resource(
|
421
|
+
session: Session, service_name: RESOURCE_NAME, region_name: str | None = None
|
422
|
+
) -> (
|
423
|
+
DynamoDBServiceResource
|
424
|
+
| EC2ServiceResource
|
425
|
+
| IAMServiceResource
|
426
|
+
| S3ServiceResource
|
427
|
+
| SQSServiceResource
|
208
428
|
):
|
209
429
|
region = region_name or session.region_name
|
210
430
|
return session.resource(service_name, region_name=region)
|
@@ -235,7 +455,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
235
455
|
|
236
456
|
def _account_cloudwatch_client(
|
237
457
|
self, account_name: str, region_name: str | None = None
|
238
|
-
):
|
458
|
+
) -> CloudWatchLogsClient:
|
239
459
|
session = self.get_session(account_name)
|
240
460
|
return self.get_session_client(session, "logs", region_name)
|
241
461
|
|
@@ -249,9 +469,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
249
469
|
self, account_name: str, region_name: str | None = None
|
250
470
|
) -> S3Client:
|
251
471
|
session = self.get_session(account_name)
|
252
|
-
return self.get_session_client(session, "s3", region_name)
|
472
|
+
return cast(S3Client, self.get_session_client(session, "s3", region_name))
|
253
473
|
|
254
|
-
def init_users(self):
|
474
|
+
def init_users(self) -> None:
|
255
475
|
self.users = {}
|
256
476
|
for account, s in self.sessions.items():
|
257
477
|
iam = self.get_session_client(s, "iam")
|
@@ -259,26 +479,29 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
259
479
|
users = [u["UserName"] for u in users]
|
260
480
|
self.users[account] = users
|
261
481
|
|
262
|
-
def map_resources(self):
|
482
|
+
def map_resources(self) -> None:
|
263
483
|
threaded.run(self.map_resource, self.resource_types, self.thread_pool_size)
|
264
484
|
|
265
|
-
def map_resource(self, resource_type):
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
485
|
+
def map_resource(self, resource_type: str) -> None:
|
486
|
+
match resource_type:
|
487
|
+
case "s3":
|
488
|
+
self.map_s3_resources()
|
489
|
+
case "sqs":
|
490
|
+
self.map_sqs_resources()
|
491
|
+
case "dynamodb":
|
492
|
+
self.map_dynamodb_resources()
|
493
|
+
case "rds":
|
494
|
+
self.map_rds_resources()
|
495
|
+
case "rds_snapshots":
|
496
|
+
self.map_rds_snapshots()
|
497
|
+
case "route53":
|
498
|
+
self.map_route53_resources()
|
499
|
+
case "ecr":
|
500
|
+
self.map_ecr_resources()
|
501
|
+
case _:
|
502
|
+
raise InvalidResourceTypeError(resource_type)
|
503
|
+
|
504
|
+
def map_s3_resources(self) -> None:
|
282
505
|
for account, s in self.sessions.items():
|
283
506
|
s3 = self.get_session_client(s, "s3")
|
284
507
|
buckets_list = s3.list_buckets()
|
@@ -292,7 +515,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
292
515
|
)
|
293
516
|
self.set_resouces(account, "s3_no_owner", unfiltered_buckets)
|
294
517
|
|
295
|
-
def map_sqs_resources(self):
|
518
|
+
def map_sqs_resources(self) -> None:
|
296
519
|
for account, s in self.sessions.items():
|
297
520
|
sqs = self.get_session_client(s, "sqs")
|
298
521
|
queues_list = sqs.list_queues()
|
@@ -306,7 +529,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
306
529
|
)
|
307
530
|
self.set_resouces(account, "sqs_no_owner", unfiltered_queues)
|
308
531
|
|
309
|
-
def map_dynamodb_resources(self):
|
532
|
+
def map_dynamodb_resources(self) -> None:
|
310
533
|
for account, s in self.sessions.items():
|
311
534
|
dynamodb = self.get_session_client(s, "dynamodb")
|
312
535
|
tables = self.paginate(dynamodb, "list_tables", "TableNames")
|
@@ -317,7 +540,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
317
540
|
)
|
318
541
|
self.set_resouces(account, "dynamodb_no_owner", unfiltered_tables)
|
319
542
|
|
320
|
-
def map_rds_resources(self):
|
543
|
+
def map_rds_resources(self) -> None:
|
321
544
|
for account, s in self.sessions.items():
|
322
545
|
rds = self.get_session_client(s, "rds")
|
323
546
|
results = self.paginate(rds, "describe_db_instances", "DBInstances")
|
@@ -331,7 +554,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
331
554
|
)
|
332
555
|
self.set_resouces(account, "rds_no_owner", unfiltered_instances)
|
333
556
|
|
334
|
-
def map_rds_snapshots(self):
|
557
|
+
def map_rds_snapshots(self) -> None:
|
335
558
|
self.wait_for_resource("rds")
|
336
559
|
for account, s in self.sessions.items():
|
337
560
|
rds = self.get_session_client(s, "rds")
|
@@ -348,7 +571,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
348
571
|
)
|
349
572
|
self.set_resouces(account, "rds_snapshots_no_owner", unfiltered_snapshots)
|
350
573
|
|
351
|
-
def map_route53_resources(self):
|
574
|
+
def map_route53_resources(self) -> None:
|
352
575
|
for account, s in self.sessions.items():
|
353
576
|
client = self.get_session_client(s, "route53")
|
354
577
|
results = self.paginate(client, "list_hosted_zones", "HostedZones")
|
@@ -363,7 +586,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
363
586
|
zone["records"] = results
|
364
587
|
self.set_resouces(account, "route53", zones)
|
365
588
|
|
366
|
-
def map_ecr_resources(self):
|
589
|
+
def map_ecr_resources(self) -> None:
|
367
590
|
for account, s in self.sessions.items():
|
368
591
|
client = self.get_session_client(s, "ecr")
|
369
592
|
repositories = self.paginate(
|
@@ -372,7 +595,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
372
595
|
self.set_resouces(account, "ecr", repositories)
|
373
596
|
|
374
597
|
@staticmethod
|
375
|
-
def paginate(
|
598
|
+
def paginate(
|
599
|
+
client: BaseClient, method: str, key: str, params: Mapping | None = None
|
600
|
+
) -> Iterable:
|
376
601
|
"""paginate returns an aggregated list of the specified key
|
377
602
|
from all pages returned by executing the client's specified method."""
|
378
603
|
if params is None:
|
@@ -384,7 +609,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
384
609
|
for values in page.get(key, [])
|
385
610
|
]
|
386
611
|
|
387
|
-
def wait_for_resource(self, resource):
|
612
|
+
def wait_for_resource(self, resource: str) -> None:
|
388
613
|
"""wait_for_resource waits until the specified resource type
|
389
614
|
is ready for all accounts.
|
390
615
|
When we have more resource types then threads,
|
@@ -398,14 +623,16 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
398
623
|
if wait:
|
399
624
|
time.sleep(2)
|
400
625
|
|
401
|
-
def set_resouces(self, account, key, value):
|
626
|
+
def set_resouces(self, account: str, key: str, value: Any) -> None:
|
402
627
|
with self._lock:
|
403
628
|
self.resources[account][key] = value
|
404
629
|
|
405
|
-
def get_resources_without_owner(
|
630
|
+
def get_resources_without_owner(
|
631
|
+
self, account: str, resources: Iterable[str]
|
632
|
+
) -> list[str]:
|
406
633
|
return [r for r in resources if not self.has_owner(account, r)]
|
407
634
|
|
408
|
-
def has_owner(self, account, resource):
|
635
|
+
def has_owner(self, account: str, resource: str) -> bool:
|
409
636
|
has_owner = False
|
410
637
|
for u in self.users[account]:
|
411
638
|
if resource.lower().startswith(u.lower()):
|
@@ -417,20 +644,24 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
417
644
|
break
|
418
645
|
return has_owner
|
419
646
|
|
420
|
-
def custom_s3_filter(
|
647
|
+
def custom_s3_filter(
|
648
|
+
self, account: str, s3: S3Client, buckets: Iterable[str]
|
649
|
+
) -> list[str]:
|
421
650
|
type = "s3 bucket"
|
422
651
|
unfiltered_buckets = []
|
423
652
|
for b in buckets:
|
424
653
|
try:
|
425
654
|
tags = s3.get_bucket_tagging(Bucket=b)
|
426
655
|
except botocore.exceptions.ClientError:
|
427
|
-
tags = {}
|
656
|
+
tags = {} # type: ignore
|
428
657
|
if not self.should_filter(account, type, b, tags, "TagSet"):
|
429
658
|
unfiltered_buckets.append(b)
|
430
659
|
|
431
660
|
return unfiltered_buckets
|
432
661
|
|
433
|
-
def custom_sqs_filter(
|
662
|
+
def custom_sqs_filter(
|
663
|
+
self, account: str, sqs: SQSClient, queues: Iterable[str]
|
664
|
+
) -> list[str]:
|
434
665
|
type = "sqs queue"
|
435
666
|
unfiltered_queues = []
|
436
667
|
for q in queues:
|
@@ -440,7 +671,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
440
671
|
|
441
672
|
return unfiltered_queues
|
442
673
|
|
443
|
-
def custom_dynamodb_filter(
|
674
|
+
def custom_dynamodb_filter(
|
675
|
+
self,
|
676
|
+
account: str,
|
677
|
+
session: Session,
|
678
|
+
dynamodb: DynamoDBClient,
|
679
|
+
tables: Iterable[str],
|
680
|
+
) -> list[str]:
|
444
681
|
type = "dynamodb table"
|
445
682
|
dynamodb_resource = self._get_session_resource(session, "dynamodb")
|
446
683
|
unfiltered_tables = []
|
@@ -452,7 +689,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
452
689
|
|
453
690
|
return unfiltered_tables
|
454
691
|
|
455
|
-
def custom_rds_filter(
|
692
|
+
def custom_rds_filter(
|
693
|
+
self, account: str, rds: RDSClient, instances: Iterable[str]
|
694
|
+
) -> list[str]:
|
456
695
|
type = "rds instance"
|
457
696
|
unfiltered_instances = []
|
458
697
|
for i in instances:
|
@@ -464,7 +703,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
464
703
|
|
465
704
|
return unfiltered_instances
|
466
705
|
|
467
|
-
def custom_rds_snapshot_filter(
|
706
|
+
def custom_rds_snapshot_filter(
|
707
|
+
self, account: str, rds: RDSClient, snapshots: Iterable[str]
|
708
|
+
) -> list[str]:
|
468
709
|
type = "rds snapshots"
|
469
710
|
unfiltered_snapshots = []
|
470
711
|
for s in snapshots:
|
@@ -477,8 +718,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
477
718
|
return unfiltered_snapshots
|
478
719
|
|
479
720
|
def should_filter(
|
480
|
-
self,
|
481
|
-
|
721
|
+
self,
|
722
|
+
account: str,
|
723
|
+
resource_type: str,
|
724
|
+
resource_name: str,
|
725
|
+
resource_tags: Mapping,
|
726
|
+
tags_key: str,
|
727
|
+
) -> bool:
|
482
728
|
if self.resource_has_special_name(account, resource_type, resource_name):
|
483
729
|
return True
|
484
730
|
if tags_key in resource_tags:
|
@@ -491,7 +737,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
491
737
|
return False
|
492
738
|
|
493
739
|
@staticmethod
|
494
|
-
def resource_has_special_name(account, type, resource):
|
740
|
+
def resource_has_special_name(account: str, type: str, resource: str) -> bool:
|
495
741
|
skip_msg = f"[{account}] skipping {type} " + "({} related) {}"
|
496
742
|
|
497
743
|
ignore_names = {
|
@@ -508,7 +754,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
508
754
|
|
509
755
|
return False
|
510
756
|
|
511
|
-
def resource_has_special_tags(
|
757
|
+
def resource_has_special_tags(
|
758
|
+
self, account: str, type: str, resource: str, tags: Mapping | list[Mapping]
|
759
|
+
) -> bool:
|
512
760
|
skip_msg = f"[{account}] skipping {type} " + "({}={}) {}"
|
513
761
|
|
514
762
|
ignore_tags = {
|
@@ -529,7 +777,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
529
777
|
return False
|
530
778
|
|
531
779
|
@staticmethod
|
532
|
-
def get_tag_value(tags, tag):
|
780
|
+
def get_tag_value(tags: Mapping | list[Mapping], tag: str) -> str:
|
533
781
|
if isinstance(tags, dict):
|
534
782
|
return tags.get(tag, "")
|
535
783
|
if isinstance(tags, list):
|
@@ -539,7 +787,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
539
787
|
|
540
788
|
return ""
|
541
789
|
|
542
|
-
def delete_resources_without_owner(self, dry_run):
|
790
|
+
def delete_resources_without_owner(self, dry_run: bool) -> None:
|
543
791
|
for account, s in self.sessions.items():
|
544
792
|
for rt in self.resource_types:
|
545
793
|
for r in self.resources[account].get(rt + "_no_owner", []):
|
@@ -547,42 +795,50 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
547
795
|
if not dry_run:
|
548
796
|
self.delete_resource(s, rt, r)
|
549
797
|
|
550
|
-
def delete_resource(
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
565
|
-
|
566
|
-
|
567
|
-
|
798
|
+
def delete_resource(
|
799
|
+
self, session: Session, resource_type: RESOURCE_TYPE, resource_name: str
|
800
|
+
) -> None:
|
801
|
+
match resource_type:
|
802
|
+
case "s3":
|
803
|
+
self.delete_bucket(
|
804
|
+
self._get_session_resource(session, resource_type), resource_name
|
805
|
+
)
|
806
|
+
case "sqs":
|
807
|
+
self.delete_queue(
|
808
|
+
self.get_session_client(session, resource_type), resource_name
|
809
|
+
)
|
810
|
+
case "dynamodb":
|
811
|
+
self.delete_table(
|
812
|
+
self._get_session_resource(session, resource_type), resource_name
|
813
|
+
)
|
814
|
+
case "rds":
|
815
|
+
self.delete_instance(
|
816
|
+
self.get_session_client(session, resource_type), resource_name
|
817
|
+
)
|
818
|
+
case "rds_snapshots":
|
819
|
+
self.delete_snapshot(
|
820
|
+
self.get_session_client(session, "rds"), resource_name
|
821
|
+
)
|
822
|
+
case _:
|
823
|
+
raise InvalidResourceTypeError(resource_type)
|
568
824
|
|
569
825
|
@staticmethod
|
570
|
-
def delete_bucket(s3, bucket_name):
|
826
|
+
def delete_bucket(s3: S3ServiceResource, bucket_name: str) -> None:
|
571
827
|
bucket = s3.Bucket(bucket_name)
|
572
828
|
bucket.object_versions.delete()
|
573
829
|
bucket.delete()
|
574
830
|
|
575
831
|
@staticmethod
|
576
|
-
def delete_queue(sqs, queue_url):
|
832
|
+
def delete_queue(sqs: SQSClient, queue_url: str) -> None:
|
577
833
|
sqs.delete_queue(QueueUrl=queue_url)
|
578
834
|
|
579
835
|
@staticmethod
|
580
|
-
def delete_table(dynamodb, table_name):
|
836
|
+
def delete_table(dynamodb: DynamoDBServiceResource, table_name: str) -> None:
|
581
837
|
table = dynamodb.Table(table_name)
|
582
838
|
table.delete()
|
583
839
|
|
584
840
|
@staticmethod
|
585
|
-
def delete_instance(rds, instance_name):
|
841
|
+
def delete_instance(rds: RDSClient, instance_name: str) -> None:
|
586
842
|
rds.delete_db_instance(
|
587
843
|
DBInstanceIdentifier=instance_name,
|
588
844
|
SkipFinalSnapshot=True,
|
@@ -590,11 +846,11 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
590
846
|
)
|
591
847
|
|
592
848
|
@staticmethod
|
593
|
-
def delete_snapshot(rds, snapshot_identifier):
|
849
|
+
def delete_snapshot(rds: RDSClient, snapshot_identifier: str) -> None:
|
594
850
|
rds.delete_db_snapshot(DBSnapshotIdentifier=snapshot_identifier)
|
595
851
|
|
596
852
|
@staticmethod
|
597
|
-
def determine_key_type(iam, user):
|
853
|
+
def determine_key_type(iam: IAMClient, user: str) -> str:
|
598
854
|
tags = iam.list_user_tags(UserName=user)["Tags"]
|
599
855
|
managed_by_integration_tag = [
|
600
856
|
t["Value"] for t in tags if t["Key"] == "managed_by_integration"
|
@@ -621,8 +877,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
621
877
|
raise InvalidResourceTypeError(huh)
|
622
878
|
|
623
879
|
def delete_keys(
|
624
|
-
self,
|
625
|
-
|
880
|
+
self,
|
881
|
+
dry_run: bool,
|
882
|
+
keys_to_delete: Mapping,
|
883
|
+
working_dirs: Mapping[str, str],
|
884
|
+
disable_service_account_keys: bool,
|
885
|
+
) -> tuple[bool, bool]:
|
626
886
|
error = False
|
627
887
|
service_account_recycle_complete = True
|
628
888
|
users_keys = self.get_users_keys()
|
@@ -637,11 +897,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
637
897
|
]
|
638
898
|
if not user_and_user_keys:
|
639
899
|
continue
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
900
|
+
|
901
|
+
if len(user_and_user_keys) > 1:
|
902
|
+
raise RuntimeError(
|
903
|
+
f"key {key} returned multiple users: {user_and_user_keys}"
|
904
|
+
)
|
905
|
+
user = user_and_user_keys[0][0]
|
906
|
+
user_keys = user_and_user_keys[0][1]
|
645
907
|
key_type = self.determine_key_type(iam, user)
|
646
908
|
key_status = self.get_user_key_status(iam, user, key)
|
647
909
|
if key_type == "unmanaged" and key_status == "Active":
|
@@ -702,7 +964,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
702
964
|
|
703
965
|
return error, service_account_recycle_complete
|
704
966
|
|
705
|
-
def get_users_keys(self):
|
967
|
+
def get_users_keys(self) -> dict:
|
706
968
|
users_keys = {}
|
707
969
|
for account, s in self.sessions.items():
|
708
970
|
iam = self.get_session_client(s, "iam")
|
@@ -712,12 +974,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
712
974
|
|
713
975
|
return users_keys
|
714
976
|
|
715
|
-
def reset_password(self, account, user_name):
|
977
|
+
def reset_password(self, account: str, user_name: str) -> None:
|
716
978
|
s = self.sessions[account]
|
717
979
|
iam = self.get_session_client(s, "iam")
|
718
980
|
iam.delete_login_profile(UserName=user_name)
|
719
981
|
|
720
|
-
def reset_mfa(self, account, user_name):
|
982
|
+
def reset_mfa(self, account: str, user_name: str) -> None:
|
721
983
|
s = self.sessions[account]
|
722
984
|
iam = self.get_session_client(s, "iam")
|
723
985
|
mfa_devices = iam.list_mfa_devices(UserName=user_name)["MFADevices"]
|
@@ -741,7 +1003,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
741
1003
|
key_list = self._get_user_key_list(iam, user)
|
742
1004
|
return next(k["Status"] for k in key_list if k["AccessKeyId"] == key)
|
743
1005
|
|
744
|
-
def get_support_cases(self):
|
1006
|
+
def get_support_cases(self) -> dict[str, list[CaseDetailsTypeDef]]:
|
745
1007
|
all_support_cases = {}
|
746
1008
|
for account, s in self.sessions.items():
|
747
1009
|
if not self.accounts[account].get("premiumSupport"):
|
@@ -826,9 +1088,8 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
826
1088
|
return (account["name"], account.get("assume_role"), account["assume_region"])
|
827
1089
|
|
828
1090
|
@staticmethod
|
829
|
-
# pylint: disable=method-hidden
|
830
1091
|
def _get_assume_role_session(
|
831
|
-
sts, account_name: str, assume_role: str, assume_region: str
|
1092
|
+
sts: STSClient, account_name: str, assume_role: str, assume_region: str
|
832
1093
|
) -> Session:
|
833
1094
|
"""
|
834
1095
|
Returns a session for a supplied role to assume:
|
@@ -859,13 +1120,144 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
859
1120
|
|
860
1121
|
return assumed_session
|
861
1122
|
|
1123
|
+
@overload
|
862
1124
|
def _get_assumed_role_client(
|
863
1125
|
self,
|
864
1126
|
account_name: str,
|
865
1127
|
assume_role: str | None,
|
866
1128
|
assume_region: str,
|
867
|
-
client_type
|
868
|
-
) ->
|
1129
|
+
client_type: Literal["logs"],
|
1130
|
+
) -> CloudWatchLogsClient: ...
|
1131
|
+
|
1132
|
+
@overload
|
1133
|
+
def _get_assumed_role_client(
|
1134
|
+
self,
|
1135
|
+
account_name: str,
|
1136
|
+
assume_role: str | None,
|
1137
|
+
assume_region: str,
|
1138
|
+
client_type: Literal["dynamodb"],
|
1139
|
+
) -> DynamoDBClient: ...
|
1140
|
+
|
1141
|
+
@overload
|
1142
|
+
def _get_assumed_role_client(
|
1143
|
+
self,
|
1144
|
+
account_name: str,
|
1145
|
+
assume_role: str | None,
|
1146
|
+
assume_region: str,
|
1147
|
+
client_type: Literal["ec2"],
|
1148
|
+
) -> EC2Client: ...
|
1149
|
+
|
1150
|
+
@overload
|
1151
|
+
def _get_assumed_role_client(
|
1152
|
+
self,
|
1153
|
+
account_name: str,
|
1154
|
+
assume_role: str | None,
|
1155
|
+
assume_region: str,
|
1156
|
+
client_type: Literal["ecr"],
|
1157
|
+
) -> ECRClient: ...
|
1158
|
+
|
1159
|
+
@overload
|
1160
|
+
def _get_assumed_role_client(
|
1161
|
+
self,
|
1162
|
+
account_name: str,
|
1163
|
+
assume_role: str | None,
|
1164
|
+
assume_region: str,
|
1165
|
+
client_type: Literal["elb"],
|
1166
|
+
) -> ElasticLoadBalancingClient: ...
|
1167
|
+
|
1168
|
+
@overload
|
1169
|
+
def _get_assumed_role_client(
|
1170
|
+
self,
|
1171
|
+
account_name: str,
|
1172
|
+
assume_role: str | None,
|
1173
|
+
assume_region: str,
|
1174
|
+
client_type: Literal["iam"],
|
1175
|
+
) -> IAMClient: ...
|
1176
|
+
|
1177
|
+
@overload
|
1178
|
+
def _get_assumed_role_client(
|
1179
|
+
self,
|
1180
|
+
account_name: str,
|
1181
|
+
assume_role: str | None,
|
1182
|
+
assume_region: str,
|
1183
|
+
client_type: Literal["organizations"],
|
1184
|
+
) -> OrganizationsClient: ...
|
1185
|
+
|
1186
|
+
@overload
|
1187
|
+
def _get_assumed_role_client(
|
1188
|
+
self,
|
1189
|
+
account_name: str,
|
1190
|
+
assume_role: str | None,
|
1191
|
+
assume_region: str,
|
1192
|
+
client_type: Literal["rds"],
|
1193
|
+
) -> RDSClient: ...
|
1194
|
+
|
1195
|
+
@overload
|
1196
|
+
def _get_assumed_role_client(
|
1197
|
+
self,
|
1198
|
+
account_name: str,
|
1199
|
+
assume_role: str | None,
|
1200
|
+
assume_region: str,
|
1201
|
+
client_type: Literal["route53"],
|
1202
|
+
) -> Route53Client: ...
|
1203
|
+
|
1204
|
+
@overload
|
1205
|
+
def _get_assumed_role_client(
|
1206
|
+
self,
|
1207
|
+
account_name: str,
|
1208
|
+
assume_role: str | None,
|
1209
|
+
assume_region: str,
|
1210
|
+
client_type: Literal["s3"],
|
1211
|
+
) -> S3Client: ...
|
1212
|
+
|
1213
|
+
@overload
|
1214
|
+
def _get_assumed_role_client(
|
1215
|
+
self,
|
1216
|
+
account_name: str,
|
1217
|
+
assume_role: str | None,
|
1218
|
+
assume_region: str,
|
1219
|
+
client_type: Literal["sqs"],
|
1220
|
+
) -> SQSClient: ...
|
1221
|
+
|
1222
|
+
@overload
|
1223
|
+
def _get_assumed_role_client(
|
1224
|
+
self,
|
1225
|
+
account_name: str,
|
1226
|
+
assume_role: str | None,
|
1227
|
+
assume_region: str,
|
1228
|
+
client_type: Literal["sts"],
|
1229
|
+
) -> STSClient: ...
|
1230
|
+
|
1231
|
+
@overload
|
1232
|
+
def _get_assumed_role_client(
|
1233
|
+
self,
|
1234
|
+
account_name: str,
|
1235
|
+
assume_role: str | None,
|
1236
|
+
assume_region: str,
|
1237
|
+
client_type: Literal["support"],
|
1238
|
+
) -> SupportClient: ...
|
1239
|
+
|
1240
|
+
def _get_assumed_role_client(
|
1241
|
+
self,
|
1242
|
+
account_name: str,
|
1243
|
+
assume_role: str | None,
|
1244
|
+
assume_region: str,
|
1245
|
+
client_type: SERVICE_NAME = "ec2",
|
1246
|
+
) -> (
|
1247
|
+
CloudWatchLogsClient
|
1248
|
+
| DynamoDBClient
|
1249
|
+
| EC2Client
|
1250
|
+
| ECRClient
|
1251
|
+
| ElasticLoadBalancingClient
|
1252
|
+
| IAMClient
|
1253
|
+
| OrganizationsClient
|
1254
|
+
| RDSClient
|
1255
|
+
| Route53Client
|
1256
|
+
| S3Client
|
1257
|
+
| SQSClient
|
1258
|
+
| STSClient
|
1259
|
+
| SupportClient
|
1260
|
+
):
|
869
1261
|
session = self.get_session(account_name)
|
870
1262
|
if not assume_role:
|
871
1263
|
return self.get_session_client(
|
@@ -878,13 +1270,11 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
878
1270
|
return self.get_session_client(assumed_session, client_type)
|
879
1271
|
|
880
1272
|
@staticmethod
|
881
|
-
# pylint: disable=method-hidden
|
882
1273
|
def get_account_vpcs(ec2: EC2Client) -> list[VpcTypeDef]:
|
883
1274
|
vpcs = ec2.describe_vpcs()
|
884
1275
|
return vpcs.get("Vpcs", [])
|
885
1276
|
|
886
1277
|
@staticmethod
|
887
|
-
# pylint: disable=method-hidden
|
888
1278
|
def get_account_amis(ec2: EC2Client, owner: str) -> list[ImageTypeDef]:
|
889
1279
|
amis = ec2.describe_images(Owners=[owner])
|
890
1280
|
return amis.get("Images", [])
|
@@ -904,7 +1294,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
904
1294
|
return res
|
905
1295
|
|
906
1296
|
@staticmethod
|
907
|
-
# pylint: disable=method-hidden
|
908
1297
|
def get_vpc_route_tables(vpc_id: str, ec2: EC2Client) -> list[RouteTableTypeDef]:
|
909
1298
|
rts = ec2.describe_route_tables(
|
910
1299
|
Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
|
@@ -912,14 +1301,17 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
912
1301
|
return rts.get("RouteTables", [])
|
913
1302
|
|
914
1303
|
@staticmethod
|
915
|
-
# pylint: disable=method-hidden
|
916
1304
|
def get_vpc_subnets(vpc_id: str, ec2: EC2Client) -> list[SubnetTypeDef]:
|
917
1305
|
subnets = ec2.describe_subnets(Filters=[{"Name": "vpc-id", "Values": [vpc_id]}])
|
918
1306
|
return subnets.get("Subnets", [])
|
919
1307
|
|
920
1308
|
def get_cluster_vpc_details(
|
921
|
-
self,
|
922
|
-
|
1309
|
+
self,
|
1310
|
+
account: awsh.Account,
|
1311
|
+
route_tables: bool = False,
|
1312
|
+
subnets: bool = False,
|
1313
|
+
hcp_vpc_endpoint_sg: bool = False,
|
1314
|
+
) -> tuple[str | None, list[str] | None, list[dict[str, str]] | None, str | None]:
|
923
1315
|
"""
|
924
1316
|
Returns a cluster VPC details:
|
925
1317
|
- VPC ID
|
@@ -935,7 +1327,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
935
1327
|
use to find the matching VPC
|
936
1328
|
"""
|
937
1329
|
assume_role_data = self._get_account_assume_data(account)
|
938
|
-
assumed_ec2 = self._get_assumed_role_client(
|
1330
|
+
assumed_ec2 = self._get_assumed_role_client(
|
1331
|
+
account_name=assume_role_data[0],
|
1332
|
+
assume_role=assume_role_data[1],
|
1333
|
+
assume_region=assume_role_data[2],
|
1334
|
+
client_type="ec2",
|
1335
|
+
)
|
939
1336
|
vpcs = self.get_account_vpcs(assumed_ec2)
|
940
1337
|
vpc_id = None
|
941
1338
|
for vpc in vpcs:
|
@@ -963,7 +1360,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
963
1360
|
|
964
1361
|
return vpc_id, route_table_ids, subnets_id_az, api_security_group_id
|
965
1362
|
|
966
|
-
def _get_api_security_group_id(
|
1363
|
+
def _get_api_security_group_id(
|
1364
|
+
self, assumed_ec2: EC2Client, vpc_id: str
|
1365
|
+
) -> str | None:
|
967
1366
|
endpoints = AWSApi._get_vpc_endpoints(
|
968
1367
|
[
|
969
1368
|
{"Name": "vpc-id", "Values": [vpc_id]},
|
@@ -997,9 +1396,16 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
997
1396
|
)
|
998
1397
|
return security_groups[0]["GroupId"]
|
999
1398
|
|
1000
|
-
def get_cluster_nat_gateways_egress_ips(
|
1001
|
-
|
1002
|
-
|
1399
|
+
def get_cluster_nat_gateways_egress_ips(
|
1400
|
+
self, account: dict[str, Any], vpc_id: str
|
1401
|
+
) -> set[str]:
|
1402
|
+
assume_role_data = self._get_account_assume_data(account)
|
1403
|
+
assumed_ec2 = self._get_assumed_role_client(
|
1404
|
+
account_name=assume_role_data[0],
|
1405
|
+
assume_role=assume_role_data[1],
|
1406
|
+
assume_region=assume_role_data[2],
|
1407
|
+
client_type="ec2",
|
1408
|
+
)
|
1003
1409
|
nat_gateways = assumed_ec2.describe_nat_gateways()
|
1004
1410
|
egress_ips: set[str] = set()
|
1005
1411
|
for nat in nat_gateways.get("NatGateways") or []:
|
@@ -1011,7 +1417,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1011
1417
|
|
1012
1418
|
return egress_ips
|
1013
1419
|
|
1014
|
-
def get_vpcs_details(
|
1420
|
+
def get_vpcs_details(
|
1421
|
+
self,
|
1422
|
+
account: awsh.Account,
|
1423
|
+
tags: Mapping[str, str] | None = None,
|
1424
|
+
route_tables: bool = False,
|
1425
|
+
) -> list[dict[str, Any]]:
|
1015
1426
|
results = []
|
1016
1427
|
ec2 = self._account_ec2_client(account["name"])
|
1017
1428
|
regions = [r["RegionName"] for r in ec2.describe_regions()["Regions"]]
|
@@ -1076,7 +1487,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1076
1487
|
share_account_uid: str,
|
1077
1488
|
image_id: str,
|
1078
1489
|
region: str | None = None,
|
1079
|
-
):
|
1490
|
+
) -> None:
|
1080
1491
|
ec2 = self._account_ec2_resource(account["name"], region)
|
1081
1492
|
image = ec2.Image(image_id)
|
1082
1493
|
launch_permission: LaunchPermissionModificationsTypeDef = {
|
@@ -1106,7 +1517,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1106
1517
|
self,
|
1107
1518
|
account_name: str,
|
1108
1519
|
region_name: str | None = None,
|
1109
|
-
) -> Iterator[
|
1520
|
+
) -> Iterator[LogGroupTypeDef]:
|
1110
1521
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1111
1522
|
paginator = client.get_paginator("describe_log_groups")
|
1112
1523
|
for page in paginator.paginate():
|
@@ -1130,7 +1541,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1130
1541
|
group_name: str,
|
1131
1542
|
retention_days: int,
|
1132
1543
|
region_name: str | None = None,
|
1133
|
-
):
|
1544
|
+
) -> None:
|
1134
1545
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1135
1546
|
client.put_retention_policy(
|
1136
1547
|
logGroupName=group_name, retentionInDays=retention_days
|
@@ -1141,21 +1552,33 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1141
1552
|
account_name: str,
|
1142
1553
|
group_name: str,
|
1143
1554
|
region_name: str | None = None,
|
1144
|
-
):
|
1555
|
+
) -> None:
|
1145
1556
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1146
1557
|
client.delete_log_group(logGroupName=group_name)
|
1147
1558
|
|
1148
1559
|
def create_tag(
|
1149
1560
|
self, account: Mapping[str, Any], resource_id: str, tag: Mapping[str, str]
|
1150
|
-
):
|
1561
|
+
) -> None:
|
1151
1562
|
ec2 = self._account_ec2_client(account["name"])
|
1152
1563
|
tag_type_def: TagTypeDef = {"Key": tag["Key"], "Value": tag["Value"]}
|
1153
1564
|
ec2.create_tags(Resources=[resource_id], Tags=[tag_type_def])
|
1154
1565
|
|
1155
|
-
def get_alb_network_interface_ips(
|
1566
|
+
def get_alb_network_interface_ips(
|
1567
|
+
self, account: awsh.Account, service_name: str
|
1568
|
+
) -> set[str]:
|
1156
1569
|
assumed_role_data = self._get_account_assume_data(account)
|
1157
|
-
ec2_client = self._get_assumed_role_client(
|
1158
|
-
|
1570
|
+
ec2_client = self._get_assumed_role_client(
|
1571
|
+
account_name=assumed_role_data[0],
|
1572
|
+
assume_role=assumed_role_data[1],
|
1573
|
+
assume_region=assumed_role_data[2],
|
1574
|
+
client_type="ec2",
|
1575
|
+
)
|
1576
|
+
elb_client = self._get_assumed_role_client(
|
1577
|
+
account_name=assumed_role_data[0],
|
1578
|
+
assume_role=assumed_role_data[1],
|
1579
|
+
assume_region=assumed_role_data[2],
|
1580
|
+
client_type="elb",
|
1581
|
+
)
|
1159
1582
|
service_tag = {"Key": "kubernetes.io/service-name", "Value": service_name}
|
1160
1583
|
nis = ec2_client.describe_network_interfaces()["NetworkInterfaces"]
|
1161
1584
|
lbs = elb_client.describe_load_balancers()["LoadBalancerDescriptions"]
|
@@ -1183,7 +1606,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1183
1606
|
return result_ips
|
1184
1607
|
|
1185
1608
|
@staticmethod
|
1186
|
-
# pylint: disable=method-hidden
|
1187
1609
|
def get_vpc_default_sg_id(vpc_id: str, ec2: EC2Client) -> str | None:
|
1188
1610
|
vpc_security_groups = ec2.describe_security_groups(
|
1189
1611
|
Filters=[
|
@@ -1198,7 +1620,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1198
1620
|
return None
|
1199
1621
|
|
1200
1622
|
@staticmethod
|
1201
|
-
# pylint: disable=method-hidden
|
1202
1623
|
def get_transit_gateways(ec2: EC2Client) -> list[TransitGatewayTypeDef]:
|
1203
1624
|
tgws = ec2.describe_transit_gateways()
|
1204
1625
|
return tgws.get("TransitGateways", [])
|
@@ -1221,7 +1642,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1221
1642
|
return None
|
1222
1643
|
|
1223
1644
|
@staticmethod
|
1224
|
-
# pylint: disable=method-hidden
|
1225
1645
|
def get_transit_gateway_vpc_attachments(
|
1226
1646
|
tgw_id: str, ec2: EC2Client
|
1227
1647
|
) -> list[TransitGatewayVpcAttachmentTypeDef]:
|
@@ -1232,23 +1652,23 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1232
1652
|
|
1233
1653
|
def get_tgws_details(
|
1234
1654
|
self,
|
1235
|
-
account,
|
1236
|
-
region_name,
|
1237
|
-
routes_cidr_block,
|
1238
|
-
tags
|
1239
|
-
route_tables=False,
|
1240
|
-
security_groups=False,
|
1241
|
-
route53_associations=False,
|
1242
|
-
):
|
1655
|
+
account: awsh.Account,
|
1656
|
+
region_name: str,
|
1657
|
+
routes_cidr_block: str,
|
1658
|
+
tags: Mapping,
|
1659
|
+
route_tables: bool = False,
|
1660
|
+
security_groups: bool = False,
|
1661
|
+
route53_associations: bool = False,
|
1662
|
+
) -> list[dict[str, Any]]:
|
1243
1663
|
results = []
|
1244
1664
|
ec2 = self._account_ec2_client(account["name"], region_name)
|
1245
1665
|
tgws = ec2.describe_transit_gateways(
|
1246
1666
|
Filters=[{"Name": f"tag:{k}", "Values": [v]} for k, v in tags.items()]
|
1247
1667
|
)
|
1248
|
-
for tgw in tgws.get("TransitGateways"):
|
1668
|
+
for tgw in tgws.get("TransitGateways") or []:
|
1249
1669
|
tgw_id = tgw["TransitGatewayId"]
|
1250
1670
|
tgw_arn = tgw["TransitGatewayArn"]
|
1251
|
-
item = {
|
1671
|
+
item: dict[str, str | list[str] | list[dict]] = {
|
1252
1672
|
"tgw_id": tgw_id,
|
1253
1673
|
"tgw_arn": tgw_arn,
|
1254
1674
|
"region": region_name,
|
@@ -1280,7 +1700,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1280
1700
|
attachments = ec2.describe_transit_gateway_peering_attachments(
|
1281
1701
|
Filters=[{"Name": "transit-gateway-id", "Values": [tgw_id]}]
|
1282
1702
|
)
|
1283
|
-
for a in attachments.get("TransitGatewayPeeringAttachments"):
|
1703
|
+
for a in attachments.get("TransitGatewayPeeringAttachments") or []:
|
1284
1704
|
tgw_attachment_id = a["TransitGatewayAttachmentId"]
|
1285
1705
|
tgw_attachment_state = a["State"]
|
1286
1706
|
if tgw_attachment_state != "available":
|
@@ -1377,7 +1797,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1377
1797
|
return results
|
1378
1798
|
|
1379
1799
|
@staticmethod
|
1380
|
-
# pylint: disable=method-hidden
|
1381
1800
|
def _get_vpc_endpoints(
|
1382
1801
|
filters: Sequence[FilterTypeDef], ec2: EC2Client
|
1383
1802
|
) -> list["VpcEndpointTypeDef"]:
|
@@ -1411,11 +1830,15 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1411
1830
|
]
|
1412
1831
|
|
1413
1832
|
@staticmethod
|
1414
|
-
def _extract_records(
|
1833
|
+
def _extract_records(
|
1834
|
+
resource_records: Iterable[ResourceRecordTypeDef],
|
1835
|
+
) -> list[str]:
|
1415
1836
|
# [{'Value': 'ns.example.com.'}, ...]
|
1416
1837
|
return [r["Value"].rstrip(".") for r in resource_records]
|
1417
1838
|
|
1418
|
-
def get_route53_zone_ns_records(
|
1839
|
+
def get_route53_zone_ns_records(
|
1840
|
+
self, account_name: str, zone_name: str, region: str
|
1841
|
+
) -> list[str]:
|
1419
1842
|
route53 = self._account_route53_client(account_name, region)
|
1420
1843
|
record_sets = self._get_hosted_zone_record_sets(route53, zone_name)
|
1421
1844
|
filtered_record_sets = self._filter_record_sets(record_sets, zone_name, "NS")
|
@@ -1425,7 +1848,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1425
1848
|
ns_records = self._extract_records(resource_records)
|
1426
1849
|
return ns_records
|
1427
1850
|
|
1428
|
-
def get_route53_zones(self):
|
1851
|
+
def get_route53_zones(self) -> dict[str, list[dict[str, str]]]:
|
1429
1852
|
"""
|
1430
1853
|
Return a list of (str, dict) representing Route53 DNS zones per account
|
1431
1854
|
|
@@ -1437,7 +1860,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1437
1860
|
for account, _ in self.sessions.items()
|
1438
1861
|
}
|
1439
1862
|
|
1440
|
-
def create_route53_zone(self, account_name, zone_name):
|
1863
|
+
def create_route53_zone(self, account_name: str, zone_name: str) -> None:
|
1441
1864
|
"""
|
1442
1865
|
Create a Route53 DNS zone
|
1443
1866
|
|
@@ -1467,7 +1890,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1467
1890
|
except Exception as e:
|
1468
1891
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1469
1892
|
|
1470
|
-
def delete_route53_zone(self, account_name, zone_id):
|
1893
|
+
def delete_route53_zone(self, account_name: str, zone_id: str) -> None:
|
1471
1894
|
"""
|
1472
1895
|
Delete a Route53 DNS zone
|
1473
1896
|
|
@@ -1492,7 +1915,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1492
1915
|
except Exception as e:
|
1493
1916
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1494
1917
|
|
1495
|
-
def delete_route53_record(
|
1918
|
+
def delete_route53_record(
|
1919
|
+
self, account_name: str, zone_id: str, awsdata: ResourceRecordSetTypeDef
|
1920
|
+
) -> None:
|
1496
1921
|
"""
|
1497
1922
|
Delete a Route53 DNS zone record
|
1498
1923
|
|
@@ -1526,7 +1951,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1526
1951
|
except Exception as e:
|
1527
1952
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1528
1953
|
|
1529
|
-
def upsert_route53_record(
|
1954
|
+
def upsert_route53_record(
|
1955
|
+
self, account_name: str, zone_id: str, recordset: ResourceRecordSetTypeDef
|
1956
|
+
) -> None:
|
1530
1957
|
"""
|
1531
1958
|
Upsert a Route53 DNS zone record
|
1532
1959
|
|
@@ -1612,7 +2039,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1612
2039
|
self,
|
1613
2040
|
account_name: str,
|
1614
2041
|
region_name: str | None = None,
|
1615
|
-
):
|
2042
|
+
) -> DBRecommendationsMessageTypeDef:
|
1616
2043
|
rds = self._account_rds_client(account_name, region_name)
|
1617
2044
|
return rds.describe_db_recommendations()
|
1618
2045
|
|