qontract-reconcile 0.10.2.dev55__py3-none-any.whl → 0.10.2.dev57__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qontract_reconcile-0.10.2.dev55.dist-info → qontract_reconcile-0.10.2.dev57.dist-info}/METADATA +1 -1
- {qontract_reconcile-0.10.2.dev55.dist-info → qontract_reconcile-0.10.2.dev57.dist-info}/RECORD +26 -25
- reconcile/aws_cloudwatch_log_retention/integration.py +17 -10
- reconcile/dashdotdb_dora.py +4 -5
- reconcile/gitlab_housekeeping.py +6 -10
- reconcile/terraform_tgw_attachments.py +5 -5
- reconcile/terraform_vpc_peerings.py +1 -1
- reconcile/utils/aggregated_list.py +20 -30
- reconcile/utils/aws_api.py +168 -595
- reconcile/utils/aws_helper.py +7 -7
- reconcile/utils/binary.py +7 -14
- reconcile/utils/config.py +6 -9
- reconcile/utils/data_structures.py +13 -0
- reconcile/utils/defer.py +2 -4
- reconcile/utils/elasticsearch_exceptions.py +4 -7
- reconcile/utils/environ.py +3 -5
- reconcile/utils/exceptions.py +2 -5
- reconcile/utils/git.py +4 -6
- reconcile/utils/gitlab_api.py +82 -103
- reconcile/utils/mr/base.py +3 -6
- reconcile/utils/mr/update_access_report_base.py +2 -2
- reconcile/utils/vcs.py +3 -5
- reconcile/vpc_peerings_validator.py +15 -21
- tools/qontract_cli.py +19 -27
- {qontract_reconcile-0.10.2.dev55.dist-info → qontract_reconcile-0.10.2.dev57.dist-info}/WHEEL +0 -0
- {qontract_reconcile-0.10.2.dev55.dist-info → qontract_reconcile-0.10.2.dev57.dist-info}/entry_points.txt +0 -0
reconcile/utils/aws_api.py
CHANGED
@@ -16,24 +16,19 @@ from typing import (
|
|
16
16
|
TYPE_CHECKING,
|
17
17
|
Any,
|
18
18
|
Literal,
|
19
|
-
Self,
|
20
|
-
cast,
|
21
|
-
overload,
|
22
19
|
)
|
23
20
|
|
24
21
|
import botocore
|
25
22
|
from boto3 import Session
|
26
|
-
from botocore.client import BaseClient
|
27
23
|
from botocore.config import Config
|
28
24
|
from pydantic import BaseModel
|
29
25
|
from sretoolbox.utils import threaded
|
30
26
|
|
31
27
|
import reconcile.utils.aws_helper as awsh
|
32
28
|
import reconcile.utils.lean_terraform_client as terraform
|
33
|
-
from reconcile.utils.secret_reader import SecretReader
|
29
|
+
from reconcile.utils.secret_reader import SecretReader
|
34
30
|
|
35
31
|
if TYPE_CHECKING:
|
36
|
-
from mypy_boto3_dynamodb import DynamoDBClient, DynamoDBServiceResource
|
37
32
|
from mypy_boto3_ec2 import (
|
38
33
|
EC2Client,
|
39
34
|
EC2ServiceResource,
|
@@ -50,17 +45,12 @@ if TYPE_CHECKING:
|
|
50
45
|
VpcEndpointTypeDef,
|
51
46
|
VpcTypeDef,
|
52
47
|
)
|
53
|
-
from
|
54
|
-
from mypy_boto3_elb import ElasticLoadBalancingClient
|
55
|
-
from mypy_boto3_iam import IAMClient, IAMServiceResource
|
48
|
+
from mypy_boto3_iam import IAMClient
|
56
49
|
from mypy_boto3_iam.type_defs import AccessKeyMetadataTypeDef
|
57
|
-
from mypy_boto3_logs import CloudWatchLogsClient
|
58
|
-
from mypy_boto3_logs.type_defs import LogGroupTypeDef
|
59
50
|
from mypy_boto3_organizations import OrganizationsClient
|
60
51
|
from mypy_boto3_rds import RDSClient
|
61
52
|
from mypy_boto3_rds.type_defs import (
|
62
53
|
DBInstanceMessageTypeDef,
|
63
|
-
DBRecommendationsMessageTypeDef,
|
64
54
|
UpgradeTargetTypeDef,
|
65
55
|
)
|
66
56
|
from mypy_boto3_route53 import Route53Client
|
@@ -69,30 +59,19 @@ if TYPE_CHECKING:
|
|
69
59
|
ResourceRecordSetTypeDef,
|
70
60
|
ResourceRecordTypeDef,
|
71
61
|
)
|
72
|
-
from mypy_boto3_s3 import S3Client
|
73
|
-
from mypy_boto3_sqs import SQSClient, SQSServiceResource
|
74
|
-
from mypy_boto3_sts import STSClient
|
75
|
-
from mypy_boto3_support import SupportClient
|
76
|
-
from mypy_boto3_support.type_defs import CaseDetailsTypeDef
|
77
|
-
|
62
|
+
from mypy_boto3_s3 import S3Client
|
78
63
|
else:
|
79
|
-
|
80
|
-
DBInstanceMessageTypeDef
|
81
|
-
) = DBRecommendationsMessageTypeDef = DynamoDBClient = DynamoDBServiceResource = (
|
82
|
-
EC2Client
|
83
|
-
) = EC2ServiceResource = ECRClient = ElasticLoadBalancingClient = FilterTypeDef = (
|
84
|
-
HostedZoneTypeDef
|
85
|
-
) = IAMClient = IAMServiceResource = ImageTypeDef = (
|
86
|
-
LaunchPermissionModificationsTypeDef
|
87
|
-
) = LogGroupTypeDef = OrganizationsClient = RDSClient = ResourceRecordSetTypeDef = (
|
88
|
-
ResourceRecordTypeDef
|
89
|
-
) = Route53Client = RouteTableTypeDef = S3Client = S3ServiceResource = SQSClient = (
|
90
|
-
SQSServiceResource
|
91
|
-
) = STSClient = SubnetTypeDef = SupportClient = TagTypeDef = (
|
64
|
+
EC2Client = EC2ServiceResource = RouteTableTypeDef = SubnetTypeDef = (
|
92
65
|
TransitGatewayTypeDef
|
93
|
-
) = TransitGatewayVpcAttachmentTypeDef =
|
94
|
-
|
95
|
-
) =
|
66
|
+
) = TransitGatewayVpcAttachmentTypeDef = VpcTypeDef = IAMClient = (
|
67
|
+
AccessKeyMetadataTypeDef
|
68
|
+
) = ImageTypeDef = TagTypeDef = LaunchPermissionModificationsTypeDef = (
|
69
|
+
FilterTypeDef
|
70
|
+
) = Route53Client = ResourceRecordSetTypeDef = ResourceRecordTypeDef = (
|
71
|
+
HostedZoneTypeDef
|
72
|
+
) = RDSClient = DBInstanceMessageTypeDef = UpgradeTargetTypeDef = (
|
73
|
+
OrganizationsClient
|
74
|
+
) = S3Client = object
|
96
75
|
|
97
76
|
|
98
77
|
class InvalidResourceTypeError(Exception):
|
@@ -103,7 +82,7 @@ class MissingARNError(Exception):
|
|
103
82
|
pass
|
104
83
|
|
105
84
|
|
106
|
-
KeyStatus = Literal["Active"
|
85
|
+
KeyStatus = Literal["Active"] | Literal["Inactive"]
|
107
86
|
|
108
87
|
GOVCLOUD_PARTITION = "aws-us-gov"
|
109
88
|
|
@@ -113,50 +92,19 @@ class AmiTag(BaseModel):
|
|
113
92
|
value: str
|
114
93
|
|
115
94
|
|
116
|
-
|
117
|
-
"dynamodb",
|
118
|
-
"ec2",
|
119
|
-
"ecr",
|
120
|
-
"elb",
|
121
|
-
"iam",
|
122
|
-
"logs",
|
123
|
-
"organizations",
|
124
|
-
"rds",
|
125
|
-
"route53",
|
126
|
-
"s3",
|
127
|
-
"sqs",
|
128
|
-
"sts",
|
129
|
-
"support",
|
130
|
-
]
|
131
|
-
RESOURCE_NAME = Literal[
|
132
|
-
"dynamodb",
|
133
|
-
"ec2",
|
134
|
-
"iam",
|
135
|
-
"s3",
|
136
|
-
"sqs",
|
137
|
-
]
|
138
|
-
RESOURCE_TYPE = Literal[
|
139
|
-
"dynamodb",
|
140
|
-
"rds",
|
141
|
-
"rds_snapshots",
|
142
|
-
"s3",
|
143
|
-
"sqs",
|
144
|
-
]
|
145
|
-
|
146
|
-
|
147
|
-
class AWSApi:
|
95
|
+
class AWSApi: # pylint: disable=too-many-public-methods
|
148
96
|
"""Wrapper around AWS SDK"""
|
149
97
|
|
150
98
|
def __init__(
|
151
99
|
self,
|
152
|
-
thread_pool_size
|
153
|
-
accounts
|
154
|
-
settings
|
155
|
-
secret_reader
|
156
|
-
init_ecr_auth_tokens
|
157
|
-
init_users
|
158
|
-
)
|
159
|
-
self._session_clients
|
100
|
+
thread_pool_size,
|
101
|
+
accounts,
|
102
|
+
settings=None,
|
103
|
+
secret_reader=None,
|
104
|
+
init_ecr_auth_tokens=False,
|
105
|
+
init_users=True,
|
106
|
+
):
|
107
|
+
self._session_clients = []
|
160
108
|
self.thread_pool_size = thread_pool_size
|
161
109
|
if secret_reader:
|
162
110
|
self.secret_reader = secret_reader
|
@@ -166,13 +114,7 @@ class AWSApi:
|
|
166
114
|
if init_ecr_auth_tokens:
|
167
115
|
self.init_ecr_auth_tokens(accounts)
|
168
116
|
self._lock = Lock()
|
169
|
-
self.resource_types
|
170
|
-
"s3",
|
171
|
-
"sqs",
|
172
|
-
"dynamodb",
|
173
|
-
"rds",
|
174
|
-
"rds_snapshots",
|
175
|
-
]
|
117
|
+
self.resource_types = ["s3", "sqs", "dynamodb", "rds", "rds_snapshots"]
|
176
118
|
|
177
119
|
# store the app-interface accounts in a dictionary indexed by name
|
178
120
|
self.accounts = {acc["name"]: acc for acc in accounts}
|
@@ -181,19 +123,19 @@ class AWSApi:
|
|
181
123
|
# https://stackoverflow.com/questions/33672412/python-functools-lru-cache-with-class-methods-release-object
|
182
124
|
# using @lru_cache decorators on methods would lek AWSApi instances
|
183
125
|
# since the cache keeps a reference to self.
|
184
|
-
self._get_assume_role_session = lru_cache()(self._get_assume_role_session)
|
185
|
-
self._get_session_resource = lru_cache()(self._get_session_resource)
|
186
|
-
self.get_account_amis = lru_cache()(self.get_account_amis)
|
187
|
-
self.get_account_vpcs = lru_cache()(self.get_account_vpcs)
|
188
|
-
self.get_session_client = lru_cache()(self.get_session_client)
|
189
|
-
self.get_transit_gateway_vpc_attachments = lru_cache()(
|
126
|
+
self._get_assume_role_session = lru_cache()(self._get_assume_role_session)
|
127
|
+
self._get_session_resource = lru_cache()(self._get_session_resource)
|
128
|
+
self.get_account_amis = lru_cache()(self.get_account_amis)
|
129
|
+
self.get_account_vpcs = lru_cache()(self.get_account_vpcs)
|
130
|
+
self.get_session_client = lru_cache()(self.get_session_client)
|
131
|
+
self.get_transit_gateway_vpc_attachments = lru_cache()(
|
190
132
|
self.get_transit_gateway_vpc_attachments
|
191
133
|
)
|
192
|
-
self.get_transit_gateways = lru_cache()(self.get_transit_gateways)
|
193
|
-
self.get_vpc_default_sg_id = lru_cache()(self.get_vpc_default_sg_id)
|
194
|
-
self.get_vpc_route_tables = lru_cache()(self.get_vpc_route_tables)
|
195
|
-
self.get_vpc_subnets = lru_cache()(self.get_vpc_subnets)
|
196
|
-
self._get_vpc_endpoints = lru_cache()(self._get_vpc_endpoints)
|
134
|
+
self.get_transit_gateways = lru_cache()(self.get_transit_gateways)
|
135
|
+
self.get_vpc_default_sg_id = lru_cache()(self.get_vpc_default_sg_id)
|
136
|
+
self.get_vpc_route_tables = lru_cache()(self.get_vpc_route_tables)
|
137
|
+
self.get_vpc_subnets = lru_cache()(self.get_vpc_subnets)
|
138
|
+
self._get_vpc_endpoints = lru_cache()(self._get_vpc_endpoints)
|
197
139
|
|
198
140
|
if init_users:
|
199
141
|
self.init_users()
|
@@ -227,13 +169,13 @@ class AWSApi:
|
|
227
169
|
self.sessions[account_name] = session
|
228
170
|
self.resources[account_name] = {}
|
229
171
|
|
230
|
-
def __enter__(self)
|
172
|
+
def __enter__(self):
|
231
173
|
return self
|
232
174
|
|
233
|
-
def __exit__(self, *exc
|
175
|
+
def __exit__(self, *exc):
|
234
176
|
self.cleanup()
|
235
177
|
|
236
|
-
def cleanup(self)
|
178
|
+
def cleanup(self):
|
237
179
|
"""
|
238
180
|
Close all session clients
|
239
181
|
:return:
|
@@ -244,129 +186,12 @@ class AWSApi:
|
|
244
186
|
def get_session(self, account: str) -> Session:
|
245
187
|
return self.sessions[account]
|
246
188
|
|
247
|
-
|
248
|
-
def get_session_client(
|
249
|
-
self,
|
250
|
-
session: Session,
|
251
|
-
service_name: Literal["ec2"],
|
252
|
-
region_name: str | None = None,
|
253
|
-
) -> EC2Client: ...
|
254
|
-
|
255
|
-
@overload
|
256
|
-
def get_session_client(
|
257
|
-
self,
|
258
|
-
session: Session,
|
259
|
-
service_name: Literal["elb"],
|
260
|
-
region_name: str | None = None,
|
261
|
-
) -> ElasticLoadBalancingClient: ...
|
262
|
-
|
263
|
-
@overload
|
264
|
-
def get_session_client(
|
265
|
-
self,
|
266
|
-
session: Session,
|
267
|
-
service_name: Literal["route53"],
|
268
|
-
region_name: str | None = None,
|
269
|
-
) -> Route53Client: ...
|
270
|
-
|
271
|
-
@overload
|
272
|
-
def get_session_client(
|
273
|
-
self,
|
274
|
-
session: Session,
|
275
|
-
service_name: Literal["rds"],
|
276
|
-
region_name: str | None = None,
|
277
|
-
) -> RDSClient: ...
|
278
|
-
|
279
|
-
@overload
|
280
|
-
def get_session_client(
|
281
|
-
self,
|
282
|
-
session: Session,
|
283
|
-
service_name: Literal["logs"],
|
284
|
-
region_name: str | None = None,
|
285
|
-
) -> CloudWatchLogsClient: ...
|
286
|
-
|
287
|
-
@overload
|
288
|
-
def get_session_client(
|
289
|
-
self,
|
290
|
-
session: Session,
|
291
|
-
service_name: Literal["organizations"],
|
292
|
-
region_name: str | None = None,
|
293
|
-
) -> OrganizationsClient: ...
|
294
|
-
|
295
|
-
@overload
|
296
|
-
def get_session_client(
|
297
|
-
self,
|
298
|
-
session: Session,
|
299
|
-
service_name: Literal["s3"],
|
300
|
-
region_name: str | None = None,
|
301
|
-
) -> S3Client: ...
|
302
|
-
|
303
|
-
@overload
|
304
|
-
def get_session_client(
|
305
|
-
self,
|
306
|
-
session: Session,
|
307
|
-
service_name: Literal["iam"],
|
308
|
-
region_name: str | None = None,
|
309
|
-
) -> IAMClient: ...
|
310
|
-
|
311
|
-
@overload
|
312
|
-
def get_session_client(
|
313
|
-
self,
|
314
|
-
session: Session,
|
315
|
-
service_name: Literal["sqs"],
|
316
|
-
region_name: str | None = None,
|
317
|
-
) -> SQSClient: ...
|
318
|
-
|
319
|
-
@overload
|
320
|
-
def get_session_client(
|
321
|
-
self,
|
322
|
-
session: Session,
|
323
|
-
service_name: Literal["dynamodb"],
|
324
|
-
region_name: str | None = None,
|
325
|
-
) -> DynamoDBClient: ...
|
326
|
-
|
327
|
-
@overload
|
328
|
-
def get_session_client(
|
329
|
-
self,
|
330
|
-
session: Session,
|
331
|
-
service_name: Literal["ecr"],
|
332
|
-
region_name: str | None = None,
|
333
|
-
) -> ECRClient: ...
|
334
|
-
|
335
|
-
@overload
|
189
|
+
# pylint: disable=method-hidden
|
336
190
|
def get_session_client(
|
337
191
|
self,
|
338
192
|
session: Session,
|
339
|
-
service_name
|
193
|
+
service_name,
|
340
194
|
region_name: str | None = None,
|
341
|
-
) -> SupportClient: ...
|
342
|
-
|
343
|
-
@overload
|
344
|
-
def get_session_client(
|
345
|
-
self,
|
346
|
-
session: Session,
|
347
|
-
service_name: Literal["sts"],
|
348
|
-
region_name: str | None = None,
|
349
|
-
) -> STSClient: ...
|
350
|
-
|
351
|
-
def get_session_client(
|
352
|
-
self,
|
353
|
-
session: Session,
|
354
|
-
service_name: SERVICE_NAME,
|
355
|
-
region_name: str | None = None,
|
356
|
-
) -> (
|
357
|
-
CloudWatchLogsClient
|
358
|
-
| DynamoDBClient
|
359
|
-
| EC2Client
|
360
|
-
| ECRClient
|
361
|
-
| ElasticLoadBalancingClient
|
362
|
-
| IAMClient
|
363
|
-
| OrganizationsClient
|
364
|
-
| RDSClient
|
365
|
-
| Route53Client
|
366
|
-
| S3Client
|
367
|
-
| SQSClient
|
368
|
-
| STSClient
|
369
|
-
| SupportClient
|
370
195
|
):
|
371
196
|
region = region_name or session.region_name
|
372
197
|
client = session.client(
|
@@ -377,55 +202,10 @@ class AWSApi:
|
|
377
202
|
self._session_clients.append(client)
|
378
203
|
return client
|
379
204
|
|
380
|
-
@overload
|
381
|
-
@staticmethod
|
382
|
-
def _get_session_resource(
|
383
|
-
session: Session,
|
384
|
-
service_name: Literal["dynamodb"],
|
385
|
-
region_name: str | None = None,
|
386
|
-
) -> DynamoDBServiceResource: ...
|
387
|
-
|
388
|
-
@overload
|
389
|
-
@staticmethod
|
390
|
-
def _get_session_resource(
|
391
|
-
session: Session,
|
392
|
-
service_name: Literal["ec2"],
|
393
|
-
region_name: str | None = None,
|
394
|
-
) -> EC2ServiceResource: ...
|
395
|
-
|
396
|
-
@overload
|
397
|
-
@staticmethod
|
398
|
-
def _get_session_resource(
|
399
|
-
session: Session,
|
400
|
-
service_name: Literal["iam"],
|
401
|
-
region_name: str | None = None,
|
402
|
-
) -> IAMServiceResource: ...
|
403
|
-
|
404
|
-
@overload
|
405
205
|
@staticmethod
|
206
|
+
# pylint: disable=method-hidden
|
406
207
|
def _get_session_resource(
|
407
|
-
session: Session,
|
408
|
-
service_name: Literal["s3"],
|
409
|
-
region_name: str | None = None,
|
410
|
-
) -> S3ServiceResource: ...
|
411
|
-
|
412
|
-
@overload
|
413
|
-
@staticmethod
|
414
|
-
def _get_session_resource(
|
415
|
-
session: Session,
|
416
|
-
service_name: Literal["sqs"],
|
417
|
-
region_name: str | None = None,
|
418
|
-
) -> SQSServiceResource: ...
|
419
|
-
|
420
|
-
@staticmethod
|
421
|
-
def _get_session_resource(
|
422
|
-
session: Session, service_name: RESOURCE_NAME, region_name: str | None = None
|
423
|
-
) -> (
|
424
|
-
DynamoDBServiceResource
|
425
|
-
| EC2ServiceResource
|
426
|
-
| IAMServiceResource
|
427
|
-
| S3ServiceResource
|
428
|
-
| SQSServiceResource
|
208
|
+
session: Session, service_name, region_name: str | None = None
|
429
209
|
):
|
430
210
|
region = region_name or session.region_name
|
431
211
|
return session.resource(service_name, region_name=region)
|
@@ -456,7 +236,7 @@ class AWSApi:
|
|
456
236
|
|
457
237
|
def _account_cloudwatch_client(
|
458
238
|
self, account_name: str, region_name: str | None = None
|
459
|
-
)
|
239
|
+
):
|
460
240
|
session = self.get_session(account_name)
|
461
241
|
return self.get_session_client(session, "logs", region_name)
|
462
242
|
|
@@ -470,9 +250,9 @@ class AWSApi:
|
|
470
250
|
self, account_name: str, region_name: str | None = None
|
471
251
|
) -> S3Client:
|
472
252
|
session = self.get_session(account_name)
|
473
|
-
return
|
253
|
+
return self.get_session_client(session, "s3", region_name)
|
474
254
|
|
475
|
-
def init_users(self)
|
255
|
+
def init_users(self):
|
476
256
|
self.users = {}
|
477
257
|
for account, s in self.sessions.items():
|
478
258
|
iam = self.get_session_client(s, "iam")
|
@@ -480,29 +260,26 @@ class AWSApi:
|
|
480
260
|
users = [u["UserName"] for u in users]
|
481
261
|
self.users[account] = users
|
482
262
|
|
483
|
-
def map_resources(self)
|
263
|
+
def map_resources(self):
|
484
264
|
threaded.run(self.map_resource, self.resource_types, self.thread_pool_size)
|
485
265
|
|
486
|
-
def map_resource(self, resource_type
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
raise InvalidResourceTypeError(resource_type)
|
504
|
-
|
505
|
-
def map_s3_resources(self) -> None:
|
266
|
+
def map_resource(self, resource_type):
|
267
|
+
if resource_type == "s3":
|
268
|
+
self.map_s3_resources()
|
269
|
+
elif resource_type == "sqs":
|
270
|
+
self.map_sqs_resources()
|
271
|
+
elif resource_type == "dynamodb":
|
272
|
+
self.map_dynamodb_resources()
|
273
|
+
elif resource_type == "rds":
|
274
|
+
self.map_rds_resources()
|
275
|
+
elif resource_type == "rds_snapshots":
|
276
|
+
self.map_rds_snapshots()
|
277
|
+
elif resource_type == "route53":
|
278
|
+
self.map_route53_resources()
|
279
|
+
else:
|
280
|
+
raise InvalidResourceTypeError(resource_type)
|
281
|
+
|
282
|
+
def map_s3_resources(self):
|
506
283
|
for account, s in self.sessions.items():
|
507
284
|
s3 = self.get_session_client(s, "s3")
|
508
285
|
buckets_list = s3.list_buckets()
|
@@ -516,7 +293,7 @@ class AWSApi:
|
|
516
293
|
)
|
517
294
|
self.set_resouces(account, "s3_no_owner", unfiltered_buckets)
|
518
295
|
|
519
|
-
def map_sqs_resources(self)
|
296
|
+
def map_sqs_resources(self):
|
520
297
|
for account, s in self.sessions.items():
|
521
298
|
sqs = self.get_session_client(s, "sqs")
|
522
299
|
queues_list = sqs.list_queues()
|
@@ -530,7 +307,7 @@ class AWSApi:
|
|
530
307
|
)
|
531
308
|
self.set_resouces(account, "sqs_no_owner", unfiltered_queues)
|
532
309
|
|
533
|
-
def map_dynamodb_resources(self)
|
310
|
+
def map_dynamodb_resources(self):
|
534
311
|
for account, s in self.sessions.items():
|
535
312
|
dynamodb = self.get_session_client(s, "dynamodb")
|
536
313
|
tables = self.paginate(dynamodb, "list_tables", "TableNames")
|
@@ -541,7 +318,7 @@ class AWSApi:
|
|
541
318
|
)
|
542
319
|
self.set_resouces(account, "dynamodb_no_owner", unfiltered_tables)
|
543
320
|
|
544
|
-
def map_rds_resources(self)
|
321
|
+
def map_rds_resources(self):
|
545
322
|
for account, s in self.sessions.items():
|
546
323
|
rds = self.get_session_client(s, "rds")
|
547
324
|
results = self.paginate(rds, "describe_db_instances", "DBInstances")
|
@@ -555,7 +332,7 @@ class AWSApi:
|
|
555
332
|
)
|
556
333
|
self.set_resouces(account, "rds_no_owner", unfiltered_instances)
|
557
334
|
|
558
|
-
def map_rds_snapshots(self)
|
335
|
+
def map_rds_snapshots(self):
|
559
336
|
self.wait_for_resource("rds")
|
560
337
|
for account, s in self.sessions.items():
|
561
338
|
rds = self.get_session_client(s, "rds")
|
@@ -572,7 +349,7 @@ class AWSApi:
|
|
572
349
|
)
|
573
350
|
self.set_resouces(account, "rds_snapshots_no_owner", unfiltered_snapshots)
|
574
351
|
|
575
|
-
def map_route53_resources(self)
|
352
|
+
def map_route53_resources(self):
|
576
353
|
for account, s in self.sessions.items():
|
577
354
|
client = self.get_session_client(s, "route53")
|
578
355
|
results = self.paginate(client, "list_hosted_zones", "HostedZones")
|
@@ -587,7 +364,7 @@ class AWSApi:
|
|
587
364
|
zone["records"] = results
|
588
365
|
self.set_resouces(account, "route53", zones)
|
589
366
|
|
590
|
-
def map_ecr_resources(self)
|
367
|
+
def map_ecr_resources(self):
|
591
368
|
for account, s in self.sessions.items():
|
592
369
|
client = self.get_session_client(s, "ecr")
|
593
370
|
repositories = self.paginate(
|
@@ -596,9 +373,7 @@ class AWSApi:
|
|
596
373
|
self.set_resouces(account, "ecr", repositories)
|
597
374
|
|
598
375
|
@staticmethod
|
599
|
-
def paginate(
|
600
|
-
client: BaseClient, method: str, key: str, params: Mapping | None = None
|
601
|
-
) -> Iterable:
|
376
|
+
def paginate(client, method, key, params=None):
|
602
377
|
"""paginate returns an aggregated list of the specified key
|
603
378
|
from all pages returned by executing the client's specified method."""
|
604
379
|
if params is None:
|
@@ -610,7 +385,7 @@ class AWSApi:
|
|
610
385
|
for values in page.get(key, [])
|
611
386
|
]
|
612
387
|
|
613
|
-
def wait_for_resource(self, resource
|
388
|
+
def wait_for_resource(self, resource):
|
614
389
|
"""wait_for_resource waits until the specified resource type
|
615
390
|
is ready for all accounts.
|
616
391
|
When we have more resource types then threads,
|
@@ -624,16 +399,14 @@ class AWSApi:
|
|
624
399
|
if wait:
|
625
400
|
time.sleep(2)
|
626
401
|
|
627
|
-
def set_resouces(self, account
|
402
|
+
def set_resouces(self, account, key, value):
|
628
403
|
with self._lock:
|
629
404
|
self.resources[account][key] = value
|
630
405
|
|
631
|
-
def get_resources_without_owner(
|
632
|
-
self, account: str, resources: Iterable[str]
|
633
|
-
) -> list[str]:
|
406
|
+
def get_resources_without_owner(self, account, resources):
|
634
407
|
return [r for r in resources if not self.has_owner(account, r)]
|
635
408
|
|
636
|
-
def has_owner(self, account
|
409
|
+
def has_owner(self, account, resource):
|
637
410
|
has_owner = False
|
638
411
|
for u in self.users[account]:
|
639
412
|
if resource.lower().startswith(u.lower()):
|
@@ -645,24 +418,20 @@ class AWSApi:
|
|
645
418
|
break
|
646
419
|
return has_owner
|
647
420
|
|
648
|
-
def custom_s3_filter(
|
649
|
-
self, account: str, s3: S3Client, buckets: Iterable[str]
|
650
|
-
) -> list[str]:
|
421
|
+
def custom_s3_filter(self, account, s3, buckets):
|
651
422
|
type = "s3 bucket"
|
652
423
|
unfiltered_buckets = []
|
653
424
|
for b in buckets:
|
654
425
|
try:
|
655
426
|
tags = s3.get_bucket_tagging(Bucket=b)
|
656
427
|
except botocore.exceptions.ClientError:
|
657
|
-
tags = {}
|
428
|
+
tags = {}
|
658
429
|
if not self.should_filter(account, type, b, tags, "TagSet"):
|
659
430
|
unfiltered_buckets.append(b)
|
660
431
|
|
661
432
|
return unfiltered_buckets
|
662
433
|
|
663
|
-
def custom_sqs_filter(
|
664
|
-
self, account: str, sqs: SQSClient, queues: Iterable[str]
|
665
|
-
) -> list[str]:
|
434
|
+
def custom_sqs_filter(self, account, sqs, queues):
|
666
435
|
type = "sqs queue"
|
667
436
|
unfiltered_queues = []
|
668
437
|
for q in queues:
|
@@ -672,13 +441,7 @@ class AWSApi:
|
|
672
441
|
|
673
442
|
return unfiltered_queues
|
674
443
|
|
675
|
-
def custom_dynamodb_filter(
|
676
|
-
self,
|
677
|
-
account: str,
|
678
|
-
session: Session,
|
679
|
-
dynamodb: DynamoDBClient,
|
680
|
-
tables: Iterable[str],
|
681
|
-
) -> list[str]:
|
444
|
+
def custom_dynamodb_filter(self, account, session, dynamodb, tables):
|
682
445
|
type = "dynamodb table"
|
683
446
|
dynamodb_resource = self._get_session_resource(session, "dynamodb")
|
684
447
|
unfiltered_tables = []
|
@@ -690,9 +453,7 @@ class AWSApi:
|
|
690
453
|
|
691
454
|
return unfiltered_tables
|
692
455
|
|
693
|
-
def custom_rds_filter(
|
694
|
-
self, account: str, rds: RDSClient, instances: Iterable[str]
|
695
|
-
) -> list[str]:
|
456
|
+
def custom_rds_filter(self, account, rds, instances):
|
696
457
|
type = "rds instance"
|
697
458
|
unfiltered_instances = []
|
698
459
|
for i in instances:
|
@@ -704,9 +465,7 @@ class AWSApi:
|
|
704
465
|
|
705
466
|
return unfiltered_instances
|
706
467
|
|
707
|
-
def custom_rds_snapshot_filter(
|
708
|
-
self, account: str, rds: RDSClient, snapshots: Iterable[str]
|
709
|
-
) -> list[str]:
|
468
|
+
def custom_rds_snapshot_filter(self, account, rds, snapshots):
|
710
469
|
type = "rds snapshots"
|
711
470
|
unfiltered_snapshots = []
|
712
471
|
for s in snapshots:
|
@@ -719,13 +478,8 @@ class AWSApi:
|
|
719
478
|
return unfiltered_snapshots
|
720
479
|
|
721
480
|
def should_filter(
|
722
|
-
self,
|
723
|
-
|
724
|
-
resource_type: str,
|
725
|
-
resource_name: str,
|
726
|
-
resource_tags: Mapping,
|
727
|
-
tags_key: str,
|
728
|
-
) -> bool:
|
481
|
+
self, account, resource_type, resource_name, resource_tags, tags_key
|
482
|
+
):
|
729
483
|
if self.resource_has_special_name(account, resource_type, resource_name):
|
730
484
|
return True
|
731
485
|
if tags_key in resource_tags:
|
@@ -738,7 +492,7 @@ class AWSApi:
|
|
738
492
|
return False
|
739
493
|
|
740
494
|
@staticmethod
|
741
|
-
def resource_has_special_name(account
|
495
|
+
def resource_has_special_name(account, type, resource):
|
742
496
|
skip_msg = f"[{account}] skipping {type} " + "({} related) {}"
|
743
497
|
|
744
498
|
ignore_names = {
|
@@ -755,9 +509,7 @@ class AWSApi:
|
|
755
509
|
|
756
510
|
return False
|
757
511
|
|
758
|
-
def resource_has_special_tags(
|
759
|
-
self, account: str, type: str, resource: str, tags: Mapping | list[Mapping]
|
760
|
-
) -> bool:
|
512
|
+
def resource_has_special_tags(self, account, type, resource, tags):
|
761
513
|
skip_msg = f"[{account}] skipping {type} " + "({}={}) {}"
|
762
514
|
|
763
515
|
ignore_tags = {
|
@@ -778,7 +530,7 @@ class AWSApi:
|
|
778
530
|
return False
|
779
531
|
|
780
532
|
@staticmethod
|
781
|
-
def get_tag_value(tags
|
533
|
+
def get_tag_value(tags, tag):
|
782
534
|
if isinstance(tags, dict):
|
783
535
|
return tags.get(tag, "")
|
784
536
|
if isinstance(tags, list):
|
@@ -788,7 +540,7 @@ class AWSApi:
|
|
788
540
|
|
789
541
|
return ""
|
790
542
|
|
791
|
-
def delete_resources_without_owner(self, dry_run
|
543
|
+
def delete_resources_without_owner(self, dry_run):
|
792
544
|
for account, s in self.sessions.items():
|
793
545
|
for rt in self.resource_types:
|
794
546
|
for r in self.resources[account].get(rt + "_no_owner", []):
|
@@ -796,50 +548,42 @@ class AWSApi:
|
|
796
548
|
if not dry_run:
|
797
549
|
self.delete_resource(s, rt, r)
|
798
550
|
|
799
|
-
def delete_resource(
|
800
|
-
|
801
|
-
|
802
|
-
|
803
|
-
|
804
|
-
|
805
|
-
|
806
|
-
|
807
|
-
|
808
|
-
|
809
|
-
|
810
|
-
|
811
|
-
|
812
|
-
|
813
|
-
|
814
|
-
|
815
|
-
|
816
|
-
|
817
|
-
self.get_session_client(session, resource_type), resource_name
|
818
|
-
)
|
819
|
-
case "rds_snapshots":
|
820
|
-
self.delete_snapshot(
|
821
|
-
self.get_session_client(session, "rds"), resource_name
|
822
|
-
)
|
823
|
-
case _:
|
824
|
-
raise InvalidResourceTypeError(resource_type)
|
551
|
+
def delete_resource(self, session, resource_type, resource_name):
|
552
|
+
if resource_type == "s3":
|
553
|
+
resource = self._get_session_resource(session, resource_type)
|
554
|
+
self.delete_bucket(resource, resource_name)
|
555
|
+
elif resource_type == "sqs":
|
556
|
+
client = self.get_session_client(session, resource_type)
|
557
|
+
self.delete_queue(client, resource_name)
|
558
|
+
elif resource_type == "dynamodb":
|
559
|
+
resource = self._get_session_resource(session, resource_type)
|
560
|
+
self.delete_table(resource, resource_name)
|
561
|
+
elif resource_type == "rds":
|
562
|
+
client = self.get_session_client(session, resource_type)
|
563
|
+
self.delete_instance(client, resource_name)
|
564
|
+
elif resource_type == "rds_snapshots":
|
565
|
+
client = self.get_session_client(session, resource_type)
|
566
|
+
self.delete_snapshot(client, resource_name)
|
567
|
+
else:
|
568
|
+
raise InvalidResourceTypeError(resource_type)
|
825
569
|
|
826
570
|
@staticmethod
|
827
|
-
def delete_bucket(s3
|
571
|
+
def delete_bucket(s3, bucket_name):
|
828
572
|
bucket = s3.Bucket(bucket_name)
|
829
573
|
bucket.object_versions.delete()
|
830
574
|
bucket.delete()
|
831
575
|
|
832
576
|
@staticmethod
|
833
|
-
def delete_queue(sqs
|
577
|
+
def delete_queue(sqs, queue_url):
|
834
578
|
sqs.delete_queue(QueueUrl=queue_url)
|
835
579
|
|
836
580
|
@staticmethod
|
837
|
-
def delete_table(dynamodb
|
581
|
+
def delete_table(dynamodb, table_name):
|
838
582
|
table = dynamodb.Table(table_name)
|
839
583
|
table.delete()
|
840
584
|
|
841
585
|
@staticmethod
|
842
|
-
def delete_instance(rds
|
586
|
+
def delete_instance(rds, instance_name):
|
843
587
|
rds.delete_db_instance(
|
844
588
|
DBInstanceIdentifier=instance_name,
|
845
589
|
SkipFinalSnapshot=True,
|
@@ -847,11 +591,11 @@ class AWSApi:
|
|
847
591
|
)
|
848
592
|
|
849
593
|
@staticmethod
|
850
|
-
def delete_snapshot(rds
|
594
|
+
def delete_snapshot(rds, snapshot_identifier):
|
851
595
|
rds.delete_db_snapshot(DBSnapshotIdentifier=snapshot_identifier)
|
852
596
|
|
853
597
|
@staticmethod
|
854
|
-
def determine_key_type(iam
|
598
|
+
def determine_key_type(iam, user):
|
855
599
|
tags = iam.list_user_tags(UserName=user)["Tags"]
|
856
600
|
managed_by_integration_tag = [
|
857
601
|
t["Value"] for t in tags if t["Key"] == "managed_by_integration"
|
@@ -878,12 +622,8 @@ class AWSApi:
|
|
878
622
|
raise InvalidResourceTypeError(huh)
|
879
623
|
|
880
624
|
def delete_keys(
|
881
|
-
self,
|
882
|
-
|
883
|
-
keys_to_delete: Mapping,
|
884
|
-
working_dirs: Mapping[str, str],
|
885
|
-
disable_service_account_keys: bool,
|
886
|
-
) -> tuple[bool, bool]:
|
625
|
+
self, dry_run, keys_to_delete, working_dirs, disable_service_account_keys
|
626
|
+
):
|
887
627
|
error = False
|
888
628
|
service_account_recycle_complete = True
|
889
629
|
users_keys = self.get_users_keys()
|
@@ -898,13 +638,11 @@ class AWSApi:
|
|
898
638
|
]
|
899
639
|
if not user_and_user_keys:
|
900
640
|
continue
|
901
|
-
|
902
|
-
|
903
|
-
|
904
|
-
|
905
|
-
|
906
|
-
user = user_and_user_keys[0][0]
|
907
|
-
user_keys = user_and_user_keys[0][1]
|
641
|
+
# unpack single item from sequence
|
642
|
+
# since only a single user can have a given key
|
643
|
+
[user_and_user_keys] = user_and_user_keys
|
644
|
+
user = user_and_user_keys[0]
|
645
|
+
user_keys = user_and_user_keys[1]
|
908
646
|
key_type = self.determine_key_type(iam, user)
|
909
647
|
key_status = self.get_user_key_status(iam, user, key)
|
910
648
|
if key_type == "unmanaged" and key_status == "Active":
|
@@ -965,7 +703,7 @@ class AWSApi:
|
|
965
703
|
|
966
704
|
return error, service_account_recycle_complete
|
967
705
|
|
968
|
-
def get_users_keys(self)
|
706
|
+
def get_users_keys(self):
|
969
707
|
users_keys = {}
|
970
708
|
for account, s in self.sessions.items():
|
971
709
|
iam = self.get_session_client(s, "iam")
|
@@ -975,12 +713,12 @@ class AWSApi:
|
|
975
713
|
|
976
714
|
return users_keys
|
977
715
|
|
978
|
-
def reset_password(self, account
|
716
|
+
def reset_password(self, account, user_name):
|
979
717
|
s = self.sessions[account]
|
980
718
|
iam = self.get_session_client(s, "iam")
|
981
719
|
iam.delete_login_profile(UserName=user_name)
|
982
720
|
|
983
|
-
def reset_mfa(self, account
|
721
|
+
def reset_mfa(self, account, user_name):
|
984
722
|
s = self.sessions[account]
|
985
723
|
iam = self.get_session_client(s, "iam")
|
986
724
|
mfa_devices = iam.list_mfa_devices(UserName=user_name)["MFADevices"]
|
@@ -1004,7 +742,7 @@ class AWSApi:
|
|
1004
742
|
key_list = self._get_user_key_list(iam, user)
|
1005
743
|
return next(k["Status"] for k in key_list if k["AccessKeyId"] == key)
|
1006
744
|
|
1007
|
-
def get_support_cases(self)
|
745
|
+
def get_support_cases(self):
|
1008
746
|
all_support_cases = {}
|
1009
747
|
for account, s in self.sessions.items():
|
1010
748
|
if not self.accounts[account].get("premiumSupport"):
|
@@ -1089,8 +827,9 @@ class AWSApi:
|
|
1089
827
|
return (account["name"], account.get("assume_role"), account["assume_region"])
|
1090
828
|
|
1091
829
|
@staticmethod
|
830
|
+
# pylint: disable=method-hidden
|
1092
831
|
def _get_assume_role_session(
|
1093
|
-
sts
|
832
|
+
sts, account_name: str, assume_role: str, assume_region: str
|
1094
833
|
) -> Session:
|
1095
834
|
"""
|
1096
835
|
Returns a session for a supplied role to assume:
|
@@ -1121,144 +860,13 @@ class AWSApi:
|
|
1121
860
|
|
1122
861
|
return assumed_session
|
1123
862
|
|
1124
|
-
@overload
|
1125
|
-
def _get_assumed_role_client(
|
1126
|
-
self,
|
1127
|
-
account_name: str,
|
1128
|
-
assume_role: str | None,
|
1129
|
-
assume_region: str,
|
1130
|
-
client_type: Literal["logs"],
|
1131
|
-
) -> CloudWatchLogsClient: ...
|
1132
|
-
|
1133
|
-
@overload
|
1134
|
-
def _get_assumed_role_client(
|
1135
|
-
self,
|
1136
|
-
account_name: str,
|
1137
|
-
assume_role: str | None,
|
1138
|
-
assume_region: str,
|
1139
|
-
client_type: Literal["dynamodb"],
|
1140
|
-
) -> DynamoDBClient: ...
|
1141
|
-
|
1142
|
-
@overload
|
1143
|
-
def _get_assumed_role_client(
|
1144
|
-
self,
|
1145
|
-
account_name: str,
|
1146
|
-
assume_role: str | None,
|
1147
|
-
assume_region: str,
|
1148
|
-
client_type: Literal["ec2"],
|
1149
|
-
) -> EC2Client: ...
|
1150
|
-
|
1151
|
-
@overload
|
1152
|
-
def _get_assumed_role_client(
|
1153
|
-
self,
|
1154
|
-
account_name: str,
|
1155
|
-
assume_role: str | None,
|
1156
|
-
assume_region: str,
|
1157
|
-
client_type: Literal["ecr"],
|
1158
|
-
) -> ECRClient: ...
|
1159
|
-
|
1160
|
-
@overload
|
1161
|
-
def _get_assumed_role_client(
|
1162
|
-
self,
|
1163
|
-
account_name: str,
|
1164
|
-
assume_role: str | None,
|
1165
|
-
assume_region: str,
|
1166
|
-
client_type: Literal["elb"],
|
1167
|
-
) -> ElasticLoadBalancingClient: ...
|
1168
|
-
|
1169
|
-
@overload
|
1170
|
-
def _get_assumed_role_client(
|
1171
|
-
self,
|
1172
|
-
account_name: str,
|
1173
|
-
assume_role: str | None,
|
1174
|
-
assume_region: str,
|
1175
|
-
client_type: Literal["iam"],
|
1176
|
-
) -> IAMClient: ...
|
1177
|
-
|
1178
|
-
@overload
|
1179
|
-
def _get_assumed_role_client(
|
1180
|
-
self,
|
1181
|
-
account_name: str,
|
1182
|
-
assume_role: str | None,
|
1183
|
-
assume_region: str,
|
1184
|
-
client_type: Literal["organizations"],
|
1185
|
-
) -> OrganizationsClient: ...
|
1186
|
-
|
1187
|
-
@overload
|
1188
|
-
def _get_assumed_role_client(
|
1189
|
-
self,
|
1190
|
-
account_name: str,
|
1191
|
-
assume_role: str | None,
|
1192
|
-
assume_region: str,
|
1193
|
-
client_type: Literal["rds"],
|
1194
|
-
) -> RDSClient: ...
|
1195
|
-
|
1196
|
-
@overload
|
1197
|
-
def _get_assumed_role_client(
|
1198
|
-
self,
|
1199
|
-
account_name: str,
|
1200
|
-
assume_role: str | None,
|
1201
|
-
assume_region: str,
|
1202
|
-
client_type: Literal["route53"],
|
1203
|
-
) -> Route53Client: ...
|
1204
|
-
|
1205
|
-
@overload
|
1206
|
-
def _get_assumed_role_client(
|
1207
|
-
self,
|
1208
|
-
account_name: str,
|
1209
|
-
assume_role: str | None,
|
1210
|
-
assume_region: str,
|
1211
|
-
client_type: Literal["s3"],
|
1212
|
-
) -> S3Client: ...
|
1213
|
-
|
1214
|
-
@overload
|
1215
863
|
def _get_assumed_role_client(
|
1216
864
|
self,
|
1217
865
|
account_name: str,
|
1218
866
|
assume_role: str | None,
|
1219
867
|
assume_region: str,
|
1220
|
-
client_type
|
1221
|
-
) ->
|
1222
|
-
|
1223
|
-
@overload
|
1224
|
-
def _get_assumed_role_client(
|
1225
|
-
self,
|
1226
|
-
account_name: str,
|
1227
|
-
assume_role: str | None,
|
1228
|
-
assume_region: str,
|
1229
|
-
client_type: Literal["sts"],
|
1230
|
-
) -> STSClient: ...
|
1231
|
-
|
1232
|
-
@overload
|
1233
|
-
def _get_assumed_role_client(
|
1234
|
-
self,
|
1235
|
-
account_name: str,
|
1236
|
-
assume_role: str | None,
|
1237
|
-
assume_region: str,
|
1238
|
-
client_type: Literal["support"],
|
1239
|
-
) -> SupportClient: ...
|
1240
|
-
|
1241
|
-
def _get_assumed_role_client(
|
1242
|
-
self,
|
1243
|
-
account_name: str,
|
1244
|
-
assume_role: str | None,
|
1245
|
-
assume_region: str,
|
1246
|
-
client_type: SERVICE_NAME = "ec2",
|
1247
|
-
) -> (
|
1248
|
-
CloudWatchLogsClient
|
1249
|
-
| DynamoDBClient
|
1250
|
-
| EC2Client
|
1251
|
-
| ECRClient
|
1252
|
-
| ElasticLoadBalancingClient
|
1253
|
-
| IAMClient
|
1254
|
-
| OrganizationsClient
|
1255
|
-
| RDSClient
|
1256
|
-
| Route53Client
|
1257
|
-
| S3Client
|
1258
|
-
| SQSClient
|
1259
|
-
| STSClient
|
1260
|
-
| SupportClient
|
1261
|
-
):
|
868
|
+
client_type="ec2",
|
869
|
+
) -> EC2Client:
|
1262
870
|
session = self.get_session(account_name)
|
1263
871
|
if not assume_role:
|
1264
872
|
return self.get_session_client(
|
@@ -1271,11 +879,13 @@ class AWSApi:
|
|
1271
879
|
return self.get_session_client(assumed_session, client_type)
|
1272
880
|
|
1273
881
|
@staticmethod
|
882
|
+
# pylint: disable=method-hidden
|
1274
883
|
def get_account_vpcs(ec2: EC2Client) -> list[VpcTypeDef]:
|
1275
884
|
vpcs = ec2.describe_vpcs()
|
1276
885
|
return vpcs.get("Vpcs", [])
|
1277
886
|
|
1278
887
|
@staticmethod
|
888
|
+
# pylint: disable=method-hidden
|
1279
889
|
def get_account_amis(ec2: EC2Client, owner: str) -> list[ImageTypeDef]:
|
1280
890
|
amis = ec2.describe_images(Owners=[owner])
|
1281
891
|
return amis.get("Images", [])
|
@@ -1295,6 +905,7 @@ class AWSApi:
|
|
1295
905
|
return res
|
1296
906
|
|
1297
907
|
@staticmethod
|
908
|
+
# pylint: disable=method-hidden
|
1298
909
|
def get_vpc_route_tables(vpc_id: str, ec2: EC2Client) -> list[RouteTableTypeDef]:
|
1299
910
|
rts = ec2.describe_route_tables(
|
1300
911
|
Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
|
@@ -1302,17 +913,14 @@ class AWSApi:
|
|
1302
913
|
return rts.get("RouteTables", [])
|
1303
914
|
|
1304
915
|
@staticmethod
|
916
|
+
# pylint: disable=method-hidden
|
1305
917
|
def get_vpc_subnets(vpc_id: str, ec2: EC2Client) -> list[SubnetTypeDef]:
|
1306
918
|
subnets = ec2.describe_subnets(Filters=[{"Name": "vpc-id", "Values": [vpc_id]}])
|
1307
919
|
return subnets.get("Subnets", [])
|
1308
920
|
|
1309
921
|
def get_cluster_vpc_details(
|
1310
|
-
self,
|
1311
|
-
|
1312
|
-
route_tables: bool = False,
|
1313
|
-
subnets: bool = False,
|
1314
|
-
hcp_vpc_endpoint_sg: bool = False,
|
1315
|
-
) -> tuple[str | None, list[str] | None, list[dict[str, str]] | None, str | None]:
|
922
|
+
self, account, route_tables=False, subnets=False, hcp_vpc_endpoint_sg=False
|
923
|
+
):
|
1316
924
|
"""
|
1317
925
|
Returns a cluster VPC details:
|
1318
926
|
- VPC ID
|
@@ -1328,12 +936,7 @@ class AWSApi:
|
|
1328
936
|
use to find the matching VPC
|
1329
937
|
"""
|
1330
938
|
assume_role_data = self._get_account_assume_data(account)
|
1331
|
-
assumed_ec2 = self._get_assumed_role_client(
|
1332
|
-
account_name=assume_role_data[0],
|
1333
|
-
assume_role=assume_role_data[1],
|
1334
|
-
assume_region=assume_role_data[2],
|
1335
|
-
client_type="ec2",
|
1336
|
-
)
|
939
|
+
assumed_ec2 = self._get_assumed_role_client(*assume_role_data)
|
1337
940
|
vpcs = self.get_account_vpcs(assumed_ec2)
|
1338
941
|
vpc_id = None
|
1339
942
|
for vpc in vpcs:
|
@@ -1361,9 +964,7 @@ class AWSApi:
|
|
1361
964
|
|
1362
965
|
return vpc_id, route_table_ids, subnets_id_az, api_security_group_id
|
1363
966
|
|
1364
|
-
def _get_api_security_group_id(
|
1365
|
-
self, assumed_ec2: EC2Client, vpc_id: str
|
1366
|
-
) -> str | None:
|
967
|
+
def _get_api_security_group_id(self, assumed_ec2, vpc_id):
|
1367
968
|
endpoints = AWSApi._get_vpc_endpoints(
|
1368
969
|
[
|
1369
970
|
{"Name": "vpc-id", "Values": [vpc_id]},
|
@@ -1397,16 +998,9 @@ class AWSApi:
|
|
1397
998
|
)
|
1398
999
|
return security_groups[0]["GroupId"]
|
1399
1000
|
|
1400
|
-
def get_cluster_nat_gateways_egress_ips(
|
1401
|
-
self
|
1402
|
-
|
1403
|
-
assume_role_data = self._get_account_assume_data(account)
|
1404
|
-
assumed_ec2 = self._get_assumed_role_client(
|
1405
|
-
account_name=assume_role_data[0],
|
1406
|
-
assume_role=assume_role_data[1],
|
1407
|
-
assume_region=assume_role_data[2],
|
1408
|
-
client_type="ec2",
|
1409
|
-
)
|
1001
|
+
def get_cluster_nat_gateways_egress_ips(self, account: dict[str, Any], vpc_id: str):
|
1002
|
+
assumed_role_data = self._get_account_assume_data(account)
|
1003
|
+
assumed_ec2 = self._get_assumed_role_client(*assumed_role_data)
|
1410
1004
|
nat_gateways = assumed_ec2.describe_nat_gateways()
|
1411
1005
|
egress_ips: set[str] = set()
|
1412
1006
|
for nat in nat_gateways.get("NatGateways") or []:
|
@@ -1418,12 +1012,7 @@ class AWSApi:
|
|
1418
1012
|
|
1419
1013
|
return egress_ips
|
1420
1014
|
|
1421
|
-
def get_vpcs_details(
|
1422
|
-
self,
|
1423
|
-
account: awsh.Account,
|
1424
|
-
tags: Mapping[str, str] | None = None,
|
1425
|
-
route_tables: bool = False,
|
1426
|
-
) -> list[dict[str, Any]]:
|
1015
|
+
def get_vpcs_details(self, account, tags=None, route_tables=False):
|
1427
1016
|
results = []
|
1428
1017
|
ec2 = self._account_ec2_client(account["name"])
|
1429
1018
|
regions = [r["RegionName"] for r in ec2.describe_regions()["Regions"]]
|
@@ -1488,7 +1077,7 @@ class AWSApi:
|
|
1488
1077
|
share_account_uid: str,
|
1489
1078
|
image_id: str,
|
1490
1079
|
region: str | None = None,
|
1491
|
-
)
|
1080
|
+
):
|
1492
1081
|
ec2 = self._account_ec2_resource(account["name"], region)
|
1493
1082
|
image = ec2.Image(image_id)
|
1494
1083
|
launch_permission: LaunchPermissionModificationsTypeDef = {
|
@@ -1518,7 +1107,7 @@ class AWSApi:
|
|
1518
1107
|
self,
|
1519
1108
|
account_name: str,
|
1520
1109
|
region_name: str | None = None,
|
1521
|
-
) -> Iterator[
|
1110
|
+
) -> Iterator[dict]:
|
1522
1111
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1523
1112
|
paginator = client.get_paginator("describe_log_groups")
|
1524
1113
|
for page in paginator.paginate():
|
@@ -1542,7 +1131,7 @@ class AWSApi:
|
|
1542
1131
|
group_name: str,
|
1543
1132
|
retention_days: int,
|
1544
1133
|
region_name: str | None = None,
|
1545
|
-
)
|
1134
|
+
):
|
1546
1135
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1547
1136
|
client.put_retention_policy(
|
1548
1137
|
logGroupName=group_name, retentionInDays=retention_days
|
@@ -1553,33 +1142,21 @@ class AWSApi:
|
|
1553
1142
|
account_name: str,
|
1554
1143
|
group_name: str,
|
1555
1144
|
region_name: str | None = None,
|
1556
|
-
)
|
1145
|
+
):
|
1557
1146
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1558
1147
|
client.delete_log_group(logGroupName=group_name)
|
1559
1148
|
|
1560
1149
|
def create_tag(
|
1561
1150
|
self, account: Mapping[str, Any], resource_id: str, tag: Mapping[str, str]
|
1562
|
-
)
|
1151
|
+
):
|
1563
1152
|
ec2 = self._account_ec2_client(account["name"])
|
1564
1153
|
tag_type_def: TagTypeDef = {"Key": tag["Key"], "Value": tag["Value"]}
|
1565
1154
|
ec2.create_tags(Resources=[resource_id], Tags=[tag_type_def])
|
1566
1155
|
|
1567
|
-
def get_alb_network_interface_ips(
|
1568
|
-
self, account: awsh.Account, service_name: str
|
1569
|
-
) -> set[str]:
|
1156
|
+
def get_alb_network_interface_ips(self, account, service_name):
|
1570
1157
|
assumed_role_data = self._get_account_assume_data(account)
|
1571
|
-
ec2_client = self._get_assumed_role_client(
|
1572
|
-
|
1573
|
-
assume_role=assumed_role_data[1],
|
1574
|
-
assume_region=assumed_role_data[2],
|
1575
|
-
client_type="ec2",
|
1576
|
-
)
|
1577
|
-
elb_client = self._get_assumed_role_client(
|
1578
|
-
account_name=assumed_role_data[0],
|
1579
|
-
assume_role=assumed_role_data[1],
|
1580
|
-
assume_region=assumed_role_data[2],
|
1581
|
-
client_type="elb",
|
1582
|
-
)
|
1158
|
+
ec2_client = self._get_assumed_role_client(*assumed_role_data, "ec2")
|
1159
|
+
elb_client = self._get_assumed_role_client(*assumed_role_data, "elb")
|
1583
1160
|
service_tag = {"Key": "kubernetes.io/service-name", "Value": service_name}
|
1584
1161
|
nis = ec2_client.describe_network_interfaces()["NetworkInterfaces"]
|
1585
1162
|
lbs = elb_client.describe_load_balancers()["LoadBalancerDescriptions"]
|
@@ -1607,6 +1184,7 @@ class AWSApi:
|
|
1607
1184
|
return result_ips
|
1608
1185
|
|
1609
1186
|
@staticmethod
|
1187
|
+
# pylint: disable=method-hidden
|
1610
1188
|
def get_vpc_default_sg_id(vpc_id: str, ec2: EC2Client) -> str | None:
|
1611
1189
|
vpc_security_groups = ec2.describe_security_groups(
|
1612
1190
|
Filters=[
|
@@ -1621,6 +1199,7 @@ class AWSApi:
|
|
1621
1199
|
return None
|
1622
1200
|
|
1623
1201
|
@staticmethod
|
1202
|
+
# pylint: disable=method-hidden
|
1624
1203
|
def get_transit_gateways(ec2: EC2Client) -> list[TransitGatewayTypeDef]:
|
1625
1204
|
tgws = ec2.describe_transit_gateways()
|
1626
1205
|
return tgws.get("TransitGateways", [])
|
@@ -1643,6 +1222,7 @@ class AWSApi:
|
|
1643
1222
|
return None
|
1644
1223
|
|
1645
1224
|
@staticmethod
|
1225
|
+
# pylint: disable=method-hidden
|
1646
1226
|
def get_transit_gateway_vpc_attachments(
|
1647
1227
|
tgw_id: str, ec2: EC2Client
|
1648
1228
|
) -> list[TransitGatewayVpcAttachmentTypeDef]:
|
@@ -1653,23 +1233,23 @@ class AWSApi:
|
|
1653
1233
|
|
1654
1234
|
def get_tgws_details(
|
1655
1235
|
self,
|
1656
|
-
account
|
1657
|
-
region_name
|
1658
|
-
routes_cidr_block
|
1659
|
-
tags
|
1660
|
-
route_tables
|
1661
|
-
security_groups
|
1662
|
-
route53_associations
|
1663
|
-
)
|
1236
|
+
account,
|
1237
|
+
region_name,
|
1238
|
+
routes_cidr_block,
|
1239
|
+
tags=None,
|
1240
|
+
route_tables=False,
|
1241
|
+
security_groups=False,
|
1242
|
+
route53_associations=False,
|
1243
|
+
):
|
1664
1244
|
results = []
|
1665
1245
|
ec2 = self._account_ec2_client(account["name"], region_name)
|
1666
1246
|
tgws = ec2.describe_transit_gateways(
|
1667
1247
|
Filters=[{"Name": f"tag:{k}", "Values": [v]} for k, v in tags.items()]
|
1668
1248
|
)
|
1669
|
-
for tgw in tgws.get("TransitGateways")
|
1249
|
+
for tgw in tgws.get("TransitGateways"):
|
1670
1250
|
tgw_id = tgw["TransitGatewayId"]
|
1671
1251
|
tgw_arn = tgw["TransitGatewayArn"]
|
1672
|
-
item
|
1252
|
+
item = {
|
1673
1253
|
"tgw_id": tgw_id,
|
1674
1254
|
"tgw_arn": tgw_arn,
|
1675
1255
|
"region": region_name,
|
@@ -1701,7 +1281,7 @@ class AWSApi:
|
|
1701
1281
|
attachments = ec2.describe_transit_gateway_peering_attachments(
|
1702
1282
|
Filters=[{"Name": "transit-gateway-id", "Values": [tgw_id]}]
|
1703
1283
|
)
|
1704
|
-
for a in attachments.get("TransitGatewayPeeringAttachments")
|
1284
|
+
for a in attachments.get("TransitGatewayPeeringAttachments"):
|
1705
1285
|
tgw_attachment_id = a["TransitGatewayAttachmentId"]
|
1706
1286
|
tgw_attachment_state = a["State"]
|
1707
1287
|
if tgw_attachment_state != "available":
|
@@ -1798,6 +1378,7 @@ class AWSApi:
|
|
1798
1378
|
return results
|
1799
1379
|
|
1800
1380
|
@staticmethod
|
1381
|
+
# pylint: disable=method-hidden
|
1801
1382
|
def _get_vpc_endpoints(
|
1802
1383
|
filters: Sequence[FilterTypeDef], ec2: EC2Client
|
1803
1384
|
) -> list["VpcEndpointTypeDef"]:
|
@@ -1831,15 +1412,11 @@ class AWSApi:
|
|
1831
1412
|
]
|
1832
1413
|
|
1833
1414
|
@staticmethod
|
1834
|
-
def _extract_records(
|
1835
|
-
resource_records: Iterable[ResourceRecordTypeDef],
|
1836
|
-
) -> list[str]:
|
1415
|
+
def _extract_records(resource_records: list[ResourceRecordTypeDef]) -> list[str]:
|
1837
1416
|
# [{'Value': 'ns.example.com.'}, ...]
|
1838
1417
|
return [r["Value"].rstrip(".") for r in resource_records]
|
1839
1418
|
|
1840
|
-
def get_route53_zone_ns_records(
|
1841
|
-
self, account_name: str, zone_name: str, region: str
|
1842
|
-
) -> list[str]:
|
1419
|
+
def get_route53_zone_ns_records(self, account_name, zone_name, region):
|
1843
1420
|
route53 = self._account_route53_client(account_name, region)
|
1844
1421
|
record_sets = self._get_hosted_zone_record_sets(route53, zone_name)
|
1845
1422
|
filtered_record_sets = self._filter_record_sets(record_sets, zone_name, "NS")
|
@@ -1849,7 +1426,7 @@ class AWSApi:
|
|
1849
1426
|
ns_records = self._extract_records(resource_records)
|
1850
1427
|
return ns_records
|
1851
1428
|
|
1852
|
-
def get_route53_zones(self)
|
1429
|
+
def get_route53_zones(self):
|
1853
1430
|
"""
|
1854
1431
|
Return a list of (str, dict) representing Route53 DNS zones per account
|
1855
1432
|
|
@@ -1861,7 +1438,7 @@ class AWSApi:
|
|
1861
1438
|
for account, _ in self.sessions.items()
|
1862
1439
|
}
|
1863
1440
|
|
1864
|
-
def create_route53_zone(self, account_name
|
1441
|
+
def create_route53_zone(self, account_name, zone_name):
|
1865
1442
|
"""
|
1866
1443
|
Create a Route53 DNS zone
|
1867
1444
|
|
@@ -1891,7 +1468,7 @@ class AWSApi:
|
|
1891
1468
|
except Exception as e:
|
1892
1469
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1893
1470
|
|
1894
|
-
def delete_route53_zone(self, account_name
|
1471
|
+
def delete_route53_zone(self, account_name, zone_id):
|
1895
1472
|
"""
|
1896
1473
|
Delete a Route53 DNS zone
|
1897
1474
|
|
@@ -1916,9 +1493,7 @@ class AWSApi:
|
|
1916
1493
|
except Exception as e:
|
1917
1494
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1918
1495
|
|
1919
|
-
def delete_route53_record(
|
1920
|
-
self, account_name: str, zone_id: str, awsdata: ResourceRecordSetTypeDef
|
1921
|
-
) -> None:
|
1496
|
+
def delete_route53_record(self, account_name, zone_id, awsdata):
|
1922
1497
|
"""
|
1923
1498
|
Delete a Route53 DNS zone record
|
1924
1499
|
|
@@ -1952,9 +1527,7 @@ class AWSApi:
|
|
1952
1527
|
except Exception as e:
|
1953
1528
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1954
1529
|
|
1955
|
-
def upsert_route53_record(
|
1956
|
-
self, account_name: str, zone_id: str, recordset: ResourceRecordSetTypeDef
|
1957
|
-
) -> None:
|
1530
|
+
def upsert_route53_record(self, account_name, zone_id, recordset):
|
1958
1531
|
"""
|
1959
1532
|
Upsert a Route53 DNS zone record
|
1960
1533
|
|
@@ -2040,7 +1613,7 @@ class AWSApi:
|
|
2040
1613
|
self,
|
2041
1614
|
account_name: str,
|
2042
1615
|
region_name: str | None = None,
|
2043
|
-
)
|
1616
|
+
):
|
2044
1617
|
rds = self._account_rds_client(account_name, region_name)
|
2045
1618
|
return rds.describe_db_recommendations()
|
2046
1619
|
|