qontract-reconcile 0.10.2.dev57__py3-none-any.whl → 0.10.2.dev59__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qontract_reconcile-0.10.2.dev57.dist-info → qontract_reconcile-0.10.2.dev59.dist-info}/METADATA +1 -1
- {qontract_reconcile-0.10.2.dev57.dist-info → qontract_reconcile-0.10.2.dev59.dist-info}/RECORD +29 -30
- reconcile/aws_cloudwatch_log_retention/integration.py +10 -17
- reconcile/dashdotdb_dora.py +5 -4
- reconcile/gitlab_housekeeping.py +10 -6
- reconcile/terraform_tgw_attachments.py +5 -5
- reconcile/terraform_vpc_peerings.py +1 -1
- reconcile/utils/aggregated_list.py +30 -20
- reconcile/utils/aws_api.py +595 -168
- reconcile/utils/aws_helper.py +7 -7
- reconcile/utils/binary.py +14 -7
- reconcile/utils/config.py +9 -6
- reconcile/utils/defer.py +4 -2
- reconcile/utils/elasticsearch_exceptions.py +7 -4
- reconcile/utils/environ.py +5 -3
- reconcile/utils/exceptions.py +5 -2
- reconcile/utils/git.py +6 -4
- reconcile/utils/gitlab_api.py +103 -82
- reconcile/utils/mr/base.py +6 -3
- reconcile/utils/mr/update_access_report_base.py +2 -2
- reconcile/utils/output.py +6 -3
- reconcile/utils/terrascript_aws_client.py +25 -0
- reconcile/utils/vcs.py +5 -3
- reconcile/vpc_peerings_validator.py +21 -15
- tools/app_interface_reporter.py +70 -44
- tools/cli_commands/gpg_encrypt.py +2 -2
- tools/qontract_cli.py +255 -307
- reconcile/utils/data_structures.py +0 -13
- {qontract_reconcile-0.10.2.dev57.dist-info → qontract_reconcile-0.10.2.dev59.dist-info}/WHEEL +0 -0
- {qontract_reconcile-0.10.2.dev57.dist-info → qontract_reconcile-0.10.2.dev59.dist-info}/entry_points.txt +0 -0
reconcile/utils/aws_api.py
CHANGED
@@ -16,19 +16,24 @@ from typing import (
|
|
16
16
|
TYPE_CHECKING,
|
17
17
|
Any,
|
18
18
|
Literal,
|
19
|
+
Self,
|
20
|
+
cast,
|
21
|
+
overload,
|
19
22
|
)
|
20
23
|
|
21
24
|
import botocore
|
22
25
|
from boto3 import Session
|
26
|
+
from botocore.client import BaseClient
|
23
27
|
from botocore.config import Config
|
24
28
|
from pydantic import BaseModel
|
25
29
|
from sretoolbox.utils import threaded
|
26
30
|
|
27
31
|
import reconcile.utils.aws_helper as awsh
|
28
32
|
import reconcile.utils.lean_terraform_client as terraform
|
29
|
-
from reconcile.utils.secret_reader import SecretReader
|
33
|
+
from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
|
30
34
|
|
31
35
|
if TYPE_CHECKING:
|
36
|
+
from mypy_boto3_dynamodb import DynamoDBClient, DynamoDBServiceResource
|
32
37
|
from mypy_boto3_ec2 import (
|
33
38
|
EC2Client,
|
34
39
|
EC2ServiceResource,
|
@@ -45,12 +50,17 @@ if TYPE_CHECKING:
|
|
45
50
|
VpcEndpointTypeDef,
|
46
51
|
VpcTypeDef,
|
47
52
|
)
|
48
|
-
from
|
53
|
+
from mypy_boto3_ecr import ECRClient
|
54
|
+
from mypy_boto3_elb import ElasticLoadBalancingClient
|
55
|
+
from mypy_boto3_iam import IAMClient, IAMServiceResource
|
49
56
|
from mypy_boto3_iam.type_defs import AccessKeyMetadataTypeDef
|
57
|
+
from mypy_boto3_logs import CloudWatchLogsClient
|
58
|
+
from mypy_boto3_logs.type_defs import LogGroupTypeDef
|
50
59
|
from mypy_boto3_organizations import OrganizationsClient
|
51
60
|
from mypy_boto3_rds import RDSClient
|
52
61
|
from mypy_boto3_rds.type_defs import (
|
53
62
|
DBInstanceMessageTypeDef,
|
63
|
+
DBRecommendationsMessageTypeDef,
|
54
64
|
UpgradeTargetTypeDef,
|
55
65
|
)
|
56
66
|
from mypy_boto3_route53 import Route53Client
|
@@ -59,19 +69,30 @@ if TYPE_CHECKING:
|
|
59
69
|
ResourceRecordSetTypeDef,
|
60
70
|
ResourceRecordTypeDef,
|
61
71
|
)
|
62
|
-
from mypy_boto3_s3 import S3Client
|
72
|
+
from mypy_boto3_s3 import S3Client, S3ServiceResource
|
73
|
+
from mypy_boto3_sqs import SQSClient, SQSServiceResource
|
74
|
+
from mypy_boto3_sts import STSClient
|
75
|
+
from mypy_boto3_support import SupportClient
|
76
|
+
from mypy_boto3_support.type_defs import CaseDetailsTypeDef
|
77
|
+
|
63
78
|
else:
|
64
|
-
|
65
|
-
|
66
|
-
) =
|
67
|
-
|
68
|
-
) =
|
69
|
-
FilterTypeDef
|
70
|
-
) = Route53Client = ResourceRecordSetTypeDef = ResourceRecordTypeDef = (
|
79
|
+
AccessKeyMetadataTypeDef = CaseDetailsTypeDef = CloudWatchLogsClient = (
|
80
|
+
DBInstanceMessageTypeDef
|
81
|
+
) = DBRecommendationsMessageTypeDef = DynamoDBClient = DynamoDBServiceResource = (
|
82
|
+
EC2Client
|
83
|
+
) = EC2ServiceResource = ECRClient = ElasticLoadBalancingClient = FilterTypeDef = (
|
71
84
|
HostedZoneTypeDef
|
72
|
-
) =
|
73
|
-
|
74
|
-
) =
|
85
|
+
) = IAMClient = IAMServiceResource = ImageTypeDef = (
|
86
|
+
LaunchPermissionModificationsTypeDef
|
87
|
+
) = LogGroupTypeDef = OrganizationsClient = RDSClient = ResourceRecordSetTypeDef = (
|
88
|
+
ResourceRecordTypeDef
|
89
|
+
) = Route53Client = RouteTableTypeDef = S3Client = S3ServiceResource = SQSClient = (
|
90
|
+
SQSServiceResource
|
91
|
+
) = STSClient = SubnetTypeDef = SupportClient = TagTypeDef = (
|
92
|
+
TransitGatewayTypeDef
|
93
|
+
) = TransitGatewayVpcAttachmentTypeDef = UpgradeTargetTypeDef = (
|
94
|
+
VpcEndpointTypeDef
|
95
|
+
) = VpcTypeDef = object
|
75
96
|
|
76
97
|
|
77
98
|
class InvalidResourceTypeError(Exception):
|
@@ -82,7 +103,7 @@ class MissingARNError(Exception):
|
|
82
103
|
pass
|
83
104
|
|
84
105
|
|
85
|
-
KeyStatus = Literal["Active"
|
106
|
+
KeyStatus = Literal["Active", "Inactive"]
|
86
107
|
|
87
108
|
GOVCLOUD_PARTITION = "aws-us-gov"
|
88
109
|
|
@@ -92,19 +113,50 @@ class AmiTag(BaseModel):
|
|
92
113
|
value: str
|
93
114
|
|
94
115
|
|
95
|
-
|
116
|
+
SERVICE_NAME = Literal[
|
117
|
+
"dynamodb",
|
118
|
+
"ec2",
|
119
|
+
"ecr",
|
120
|
+
"elb",
|
121
|
+
"iam",
|
122
|
+
"logs",
|
123
|
+
"organizations",
|
124
|
+
"rds",
|
125
|
+
"route53",
|
126
|
+
"s3",
|
127
|
+
"sqs",
|
128
|
+
"sts",
|
129
|
+
"support",
|
130
|
+
]
|
131
|
+
RESOURCE_NAME = Literal[
|
132
|
+
"dynamodb",
|
133
|
+
"ec2",
|
134
|
+
"iam",
|
135
|
+
"s3",
|
136
|
+
"sqs",
|
137
|
+
]
|
138
|
+
RESOURCE_TYPE = Literal[
|
139
|
+
"dynamodb",
|
140
|
+
"rds",
|
141
|
+
"rds_snapshots",
|
142
|
+
"s3",
|
143
|
+
"sqs",
|
144
|
+
]
|
145
|
+
|
146
|
+
|
147
|
+
class AWSApi:
|
96
148
|
"""Wrapper around AWS SDK"""
|
97
149
|
|
98
150
|
def __init__(
|
99
151
|
self,
|
100
|
-
thread_pool_size,
|
101
|
-
accounts,
|
102
|
-
settings=None,
|
103
|
-
secret_reader=None,
|
104
|
-
init_ecr_auth_tokens=False,
|
105
|
-
init_users=True,
|
106
|
-
):
|
107
|
-
self._session_clients = []
|
152
|
+
thread_pool_size: int,
|
153
|
+
accounts: Iterable[awsh.Account],
|
154
|
+
settings: Mapping | None = None,
|
155
|
+
secret_reader: SecretReaderBase | None = None,
|
156
|
+
init_ecr_auth_tokens: bool = False,
|
157
|
+
init_users: bool = True,
|
158
|
+
) -> None:
|
159
|
+
self._session_clients: list[BaseClient] = []
|
108
160
|
self.thread_pool_size = thread_pool_size
|
109
161
|
if secret_reader:
|
110
162
|
self.secret_reader = secret_reader
|
@@ -114,7 +166,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
114
166
|
if init_ecr_auth_tokens:
|
115
167
|
self.init_ecr_auth_tokens(accounts)
|
116
168
|
self._lock = Lock()
|
117
|
-
self.resource_types = [
|
169
|
+
self.resource_types: list[RESOURCE_TYPE] = [
|
170
|
+
"s3",
|
171
|
+
"sqs",
|
172
|
+
"dynamodb",
|
173
|
+
"rds",
|
174
|
+
"rds_snapshots",
|
175
|
+
]
|
118
176
|
|
119
177
|
# store the app-interface accounts in a dictionary indexed by name
|
120
178
|
self.accounts = {acc["name"]: acc for acc in accounts}
|
@@ -123,19 +181,19 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
123
181
|
# https://stackoverflow.com/questions/33672412/python-functools-lru-cache-with-class-methods-release-object
|
124
182
|
# using @lru_cache decorators on methods would lek AWSApi instances
|
125
183
|
# since the cache keeps a reference to self.
|
126
|
-
self._get_assume_role_session = lru_cache()(self._get_assume_role_session)
|
127
|
-
self._get_session_resource = lru_cache()(self._get_session_resource)
|
128
|
-
self.get_account_amis = lru_cache()(self.get_account_amis)
|
129
|
-
self.get_account_vpcs = lru_cache()(self.get_account_vpcs)
|
130
|
-
self.get_session_client = lru_cache()(self.get_session_client)
|
131
|
-
self.get_transit_gateway_vpc_attachments = lru_cache()(
|
184
|
+
self._get_assume_role_session = lru_cache()(self._get_assume_role_session) # type: ignore[method-assign]
|
185
|
+
self._get_session_resource = lru_cache()(self._get_session_resource) # type: ignore[method-assign, assignment]
|
186
|
+
self.get_account_amis = lru_cache()(self.get_account_amis) # type: ignore[method-assign]
|
187
|
+
self.get_account_vpcs = lru_cache()(self.get_account_vpcs) # type: ignore[method-assign]
|
188
|
+
self.get_session_client = lru_cache()(self.get_session_client) # type: ignore[method-assign, assignment]
|
189
|
+
self.get_transit_gateway_vpc_attachments = lru_cache()( # type: ignore[method-assign]
|
132
190
|
self.get_transit_gateway_vpc_attachments
|
133
191
|
)
|
134
|
-
self.get_transit_gateways = lru_cache()(self.get_transit_gateways)
|
135
|
-
self.get_vpc_default_sg_id = lru_cache()(self.get_vpc_default_sg_id)
|
136
|
-
self.get_vpc_route_tables = lru_cache()(self.get_vpc_route_tables)
|
137
|
-
self.get_vpc_subnets = lru_cache()(self.get_vpc_subnets)
|
138
|
-
self._get_vpc_endpoints = lru_cache()(self._get_vpc_endpoints)
|
192
|
+
self.get_transit_gateways = lru_cache()(self.get_transit_gateways) # type: ignore[method-assign]
|
193
|
+
self.get_vpc_default_sg_id = lru_cache()(self.get_vpc_default_sg_id) # type: ignore[method-assign]
|
194
|
+
self.get_vpc_route_tables = lru_cache()(self.get_vpc_route_tables) # type: ignore[method-assign]
|
195
|
+
self.get_vpc_subnets = lru_cache()(self.get_vpc_subnets) # type: ignore[method-assign]
|
196
|
+
self._get_vpc_endpoints = lru_cache()(self._get_vpc_endpoints) # type: ignore[method-assign]
|
139
197
|
|
140
198
|
if init_users:
|
141
199
|
self.init_users()
|
@@ -169,13 +227,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
169
227
|
self.sessions[account_name] = session
|
170
228
|
self.resources[account_name] = {}
|
171
229
|
|
172
|
-
def __enter__(self):
|
230
|
+
def __enter__(self) -> Self:
|
173
231
|
return self
|
174
232
|
|
175
|
-
def __exit__(self, *exc):
|
233
|
+
def __exit__(self, *exc: Any) -> None:
|
176
234
|
self.cleanup()
|
177
235
|
|
178
|
-
def cleanup(self):
|
236
|
+
def cleanup(self) -> None:
|
179
237
|
"""
|
180
238
|
Close all session clients
|
181
239
|
:return:
|
@@ -186,12 +244,129 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
186
244
|
def get_session(self, account: str) -> Session:
|
187
245
|
return self.sessions[account]
|
188
246
|
|
189
|
-
|
247
|
+
@overload
|
248
|
+
def get_session_client(
|
249
|
+
self,
|
250
|
+
session: Session,
|
251
|
+
service_name: Literal["ec2"],
|
252
|
+
region_name: str | None = None,
|
253
|
+
) -> EC2Client: ...
|
254
|
+
|
255
|
+
@overload
|
256
|
+
def get_session_client(
|
257
|
+
self,
|
258
|
+
session: Session,
|
259
|
+
service_name: Literal["elb"],
|
260
|
+
region_name: str | None = None,
|
261
|
+
) -> ElasticLoadBalancingClient: ...
|
262
|
+
|
263
|
+
@overload
|
264
|
+
def get_session_client(
|
265
|
+
self,
|
266
|
+
session: Session,
|
267
|
+
service_name: Literal["route53"],
|
268
|
+
region_name: str | None = None,
|
269
|
+
) -> Route53Client: ...
|
270
|
+
|
271
|
+
@overload
|
272
|
+
def get_session_client(
|
273
|
+
self,
|
274
|
+
session: Session,
|
275
|
+
service_name: Literal["rds"],
|
276
|
+
region_name: str | None = None,
|
277
|
+
) -> RDSClient: ...
|
278
|
+
|
279
|
+
@overload
|
280
|
+
def get_session_client(
|
281
|
+
self,
|
282
|
+
session: Session,
|
283
|
+
service_name: Literal["logs"],
|
284
|
+
region_name: str | None = None,
|
285
|
+
) -> CloudWatchLogsClient: ...
|
286
|
+
|
287
|
+
@overload
|
288
|
+
def get_session_client(
|
289
|
+
self,
|
290
|
+
session: Session,
|
291
|
+
service_name: Literal["organizations"],
|
292
|
+
region_name: str | None = None,
|
293
|
+
) -> OrganizationsClient: ...
|
294
|
+
|
295
|
+
@overload
|
296
|
+
def get_session_client(
|
297
|
+
self,
|
298
|
+
session: Session,
|
299
|
+
service_name: Literal["s3"],
|
300
|
+
region_name: str | None = None,
|
301
|
+
) -> S3Client: ...
|
302
|
+
|
303
|
+
@overload
|
304
|
+
def get_session_client(
|
305
|
+
self,
|
306
|
+
session: Session,
|
307
|
+
service_name: Literal["iam"],
|
308
|
+
region_name: str | None = None,
|
309
|
+
) -> IAMClient: ...
|
310
|
+
|
311
|
+
@overload
|
312
|
+
def get_session_client(
|
313
|
+
self,
|
314
|
+
session: Session,
|
315
|
+
service_name: Literal["sqs"],
|
316
|
+
region_name: str | None = None,
|
317
|
+
) -> SQSClient: ...
|
318
|
+
|
319
|
+
@overload
|
320
|
+
def get_session_client(
|
321
|
+
self,
|
322
|
+
session: Session,
|
323
|
+
service_name: Literal["dynamodb"],
|
324
|
+
region_name: str | None = None,
|
325
|
+
) -> DynamoDBClient: ...
|
326
|
+
|
327
|
+
@overload
|
328
|
+
def get_session_client(
|
329
|
+
self,
|
330
|
+
session: Session,
|
331
|
+
service_name: Literal["ecr"],
|
332
|
+
region_name: str | None = None,
|
333
|
+
) -> ECRClient: ...
|
334
|
+
|
335
|
+
@overload
|
190
336
|
def get_session_client(
|
191
337
|
self,
|
192
338
|
session: Session,
|
193
|
-
service_name,
|
339
|
+
service_name: Literal["support"],
|
194
340
|
region_name: str | None = None,
|
341
|
+
) -> SupportClient: ...
|
342
|
+
|
343
|
+
@overload
|
344
|
+
def get_session_client(
|
345
|
+
self,
|
346
|
+
session: Session,
|
347
|
+
service_name: Literal["sts"],
|
348
|
+
region_name: str | None = None,
|
349
|
+
) -> STSClient: ...
|
350
|
+
|
351
|
+
def get_session_client(
|
352
|
+
self,
|
353
|
+
session: Session,
|
354
|
+
service_name: SERVICE_NAME,
|
355
|
+
region_name: str | None = None,
|
356
|
+
) -> (
|
357
|
+
CloudWatchLogsClient
|
358
|
+
| DynamoDBClient
|
359
|
+
| EC2Client
|
360
|
+
| ECRClient
|
361
|
+
| ElasticLoadBalancingClient
|
362
|
+
| IAMClient
|
363
|
+
| OrganizationsClient
|
364
|
+
| RDSClient
|
365
|
+
| Route53Client
|
366
|
+
| S3Client
|
367
|
+
| SQSClient
|
368
|
+
| STSClient
|
369
|
+
| SupportClient
|
195
370
|
):
|
196
371
|
region = region_name or session.region_name
|
197
372
|
client = session.client(
|
@@ -202,10 +377,55 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
202
377
|
self._session_clients.append(client)
|
203
378
|
return client
|
204
379
|
|
380
|
+
@overload
|
381
|
+
@staticmethod
|
382
|
+
def _get_session_resource(
|
383
|
+
session: Session,
|
384
|
+
service_name: Literal["dynamodb"],
|
385
|
+
region_name: str | None = None,
|
386
|
+
) -> DynamoDBServiceResource: ...
|
387
|
+
|
388
|
+
@overload
|
389
|
+
@staticmethod
|
390
|
+
def _get_session_resource(
|
391
|
+
session: Session,
|
392
|
+
service_name: Literal["ec2"],
|
393
|
+
region_name: str | None = None,
|
394
|
+
) -> EC2ServiceResource: ...
|
395
|
+
|
396
|
+
@overload
|
397
|
+
@staticmethod
|
398
|
+
def _get_session_resource(
|
399
|
+
session: Session,
|
400
|
+
service_name: Literal["iam"],
|
401
|
+
region_name: str | None = None,
|
402
|
+
) -> IAMServiceResource: ...
|
403
|
+
|
404
|
+
@overload
|
205
405
|
@staticmethod
|
206
|
-
# pylint: disable=method-hidden
|
207
406
|
def _get_session_resource(
|
208
|
-
session: Session,
|
407
|
+
session: Session,
|
408
|
+
service_name: Literal["s3"],
|
409
|
+
region_name: str | None = None,
|
410
|
+
) -> S3ServiceResource: ...
|
411
|
+
|
412
|
+
@overload
|
413
|
+
@staticmethod
|
414
|
+
def _get_session_resource(
|
415
|
+
session: Session,
|
416
|
+
service_name: Literal["sqs"],
|
417
|
+
region_name: str | None = None,
|
418
|
+
) -> SQSServiceResource: ...
|
419
|
+
|
420
|
+
@staticmethod
|
421
|
+
def _get_session_resource(
|
422
|
+
session: Session, service_name: RESOURCE_NAME, region_name: str | None = None
|
423
|
+
) -> (
|
424
|
+
DynamoDBServiceResource
|
425
|
+
| EC2ServiceResource
|
426
|
+
| IAMServiceResource
|
427
|
+
| S3ServiceResource
|
428
|
+
| SQSServiceResource
|
209
429
|
):
|
210
430
|
region = region_name or session.region_name
|
211
431
|
return session.resource(service_name, region_name=region)
|
@@ -236,7 +456,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
236
456
|
|
237
457
|
def _account_cloudwatch_client(
|
238
458
|
self, account_name: str, region_name: str | None = None
|
239
|
-
):
|
459
|
+
) -> CloudWatchLogsClient:
|
240
460
|
session = self.get_session(account_name)
|
241
461
|
return self.get_session_client(session, "logs", region_name)
|
242
462
|
|
@@ -250,9 +470,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
250
470
|
self, account_name: str, region_name: str | None = None
|
251
471
|
) -> S3Client:
|
252
472
|
session = self.get_session(account_name)
|
253
|
-
return self.get_session_client(session, "s3", region_name)
|
473
|
+
return cast(S3Client, self.get_session_client(session, "s3", region_name))
|
254
474
|
|
255
|
-
def init_users(self):
|
475
|
+
def init_users(self) -> None:
|
256
476
|
self.users = {}
|
257
477
|
for account, s in self.sessions.items():
|
258
478
|
iam = self.get_session_client(s, "iam")
|
@@ -260,26 +480,29 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
260
480
|
users = [u["UserName"] for u in users]
|
261
481
|
self.users[account] = users
|
262
482
|
|
263
|
-
def map_resources(self):
|
483
|
+
def map_resources(self) -> None:
|
264
484
|
threaded.run(self.map_resource, self.resource_types, self.thread_pool_size)
|
265
485
|
|
266
|
-
def map_resource(self, resource_type):
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
486
|
+
def map_resource(self, resource_type: str) -> None:
|
487
|
+
match resource_type:
|
488
|
+
case "s3":
|
489
|
+
self.map_s3_resources()
|
490
|
+
case "sqs":
|
491
|
+
self.map_sqs_resources()
|
492
|
+
case "dynamodb":
|
493
|
+
self.map_dynamodb_resources()
|
494
|
+
case "rds":
|
495
|
+
self.map_rds_resources()
|
496
|
+
case "rds_snapshots":
|
497
|
+
self.map_rds_snapshots()
|
498
|
+
case "route53":
|
499
|
+
self.map_route53_resources()
|
500
|
+
case "ecr":
|
501
|
+
self.map_ecr_resources()
|
502
|
+
case _:
|
503
|
+
raise InvalidResourceTypeError(resource_type)
|
504
|
+
|
505
|
+
def map_s3_resources(self) -> None:
|
283
506
|
for account, s in self.sessions.items():
|
284
507
|
s3 = self.get_session_client(s, "s3")
|
285
508
|
buckets_list = s3.list_buckets()
|
@@ -293,7 +516,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
293
516
|
)
|
294
517
|
self.set_resouces(account, "s3_no_owner", unfiltered_buckets)
|
295
518
|
|
296
|
-
def map_sqs_resources(self):
|
519
|
+
def map_sqs_resources(self) -> None:
|
297
520
|
for account, s in self.sessions.items():
|
298
521
|
sqs = self.get_session_client(s, "sqs")
|
299
522
|
queues_list = sqs.list_queues()
|
@@ -307,7 +530,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
307
530
|
)
|
308
531
|
self.set_resouces(account, "sqs_no_owner", unfiltered_queues)
|
309
532
|
|
310
|
-
def map_dynamodb_resources(self):
|
533
|
+
def map_dynamodb_resources(self) -> None:
|
311
534
|
for account, s in self.sessions.items():
|
312
535
|
dynamodb = self.get_session_client(s, "dynamodb")
|
313
536
|
tables = self.paginate(dynamodb, "list_tables", "TableNames")
|
@@ -318,7 +541,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
318
541
|
)
|
319
542
|
self.set_resouces(account, "dynamodb_no_owner", unfiltered_tables)
|
320
543
|
|
321
|
-
def map_rds_resources(self):
|
544
|
+
def map_rds_resources(self) -> None:
|
322
545
|
for account, s in self.sessions.items():
|
323
546
|
rds = self.get_session_client(s, "rds")
|
324
547
|
results = self.paginate(rds, "describe_db_instances", "DBInstances")
|
@@ -332,7 +555,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
332
555
|
)
|
333
556
|
self.set_resouces(account, "rds_no_owner", unfiltered_instances)
|
334
557
|
|
335
|
-
def map_rds_snapshots(self):
|
558
|
+
def map_rds_snapshots(self) -> None:
|
336
559
|
self.wait_for_resource("rds")
|
337
560
|
for account, s in self.sessions.items():
|
338
561
|
rds = self.get_session_client(s, "rds")
|
@@ -349,7 +572,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
349
572
|
)
|
350
573
|
self.set_resouces(account, "rds_snapshots_no_owner", unfiltered_snapshots)
|
351
574
|
|
352
|
-
def map_route53_resources(self):
|
575
|
+
def map_route53_resources(self) -> None:
|
353
576
|
for account, s in self.sessions.items():
|
354
577
|
client = self.get_session_client(s, "route53")
|
355
578
|
results = self.paginate(client, "list_hosted_zones", "HostedZones")
|
@@ -364,7 +587,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
364
587
|
zone["records"] = results
|
365
588
|
self.set_resouces(account, "route53", zones)
|
366
589
|
|
367
|
-
def map_ecr_resources(self):
|
590
|
+
def map_ecr_resources(self) -> None:
|
368
591
|
for account, s in self.sessions.items():
|
369
592
|
client = self.get_session_client(s, "ecr")
|
370
593
|
repositories = self.paginate(
|
@@ -373,7 +596,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
373
596
|
self.set_resouces(account, "ecr", repositories)
|
374
597
|
|
375
598
|
@staticmethod
|
376
|
-
def paginate(
|
599
|
+
def paginate(
|
600
|
+
client: BaseClient, method: str, key: str, params: Mapping | None = None
|
601
|
+
) -> Iterable:
|
377
602
|
"""paginate returns an aggregated list of the specified key
|
378
603
|
from all pages returned by executing the client's specified method."""
|
379
604
|
if params is None:
|
@@ -385,7 +610,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
385
610
|
for values in page.get(key, [])
|
386
611
|
]
|
387
612
|
|
388
|
-
def wait_for_resource(self, resource):
|
613
|
+
def wait_for_resource(self, resource: str) -> None:
|
389
614
|
"""wait_for_resource waits until the specified resource type
|
390
615
|
is ready for all accounts.
|
391
616
|
When we have more resource types then threads,
|
@@ -399,14 +624,16 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
399
624
|
if wait:
|
400
625
|
time.sleep(2)
|
401
626
|
|
402
|
-
def set_resouces(self, account, key, value):
|
627
|
+
def set_resouces(self, account: str, key: str, value: Any) -> None:
|
403
628
|
with self._lock:
|
404
629
|
self.resources[account][key] = value
|
405
630
|
|
406
|
-
def get_resources_without_owner(
|
631
|
+
def get_resources_without_owner(
|
632
|
+
self, account: str, resources: Iterable[str]
|
633
|
+
) -> list[str]:
|
407
634
|
return [r for r in resources if not self.has_owner(account, r)]
|
408
635
|
|
409
|
-
def has_owner(self, account, resource):
|
636
|
+
def has_owner(self, account: str, resource: str) -> bool:
|
410
637
|
has_owner = False
|
411
638
|
for u in self.users[account]:
|
412
639
|
if resource.lower().startswith(u.lower()):
|
@@ -418,20 +645,24 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
418
645
|
break
|
419
646
|
return has_owner
|
420
647
|
|
421
|
-
def custom_s3_filter(
|
648
|
+
def custom_s3_filter(
|
649
|
+
self, account: str, s3: S3Client, buckets: Iterable[str]
|
650
|
+
) -> list[str]:
|
422
651
|
type = "s3 bucket"
|
423
652
|
unfiltered_buckets = []
|
424
653
|
for b in buckets:
|
425
654
|
try:
|
426
655
|
tags = s3.get_bucket_tagging(Bucket=b)
|
427
656
|
except botocore.exceptions.ClientError:
|
428
|
-
tags = {}
|
657
|
+
tags = {} # type: ignore
|
429
658
|
if not self.should_filter(account, type, b, tags, "TagSet"):
|
430
659
|
unfiltered_buckets.append(b)
|
431
660
|
|
432
661
|
return unfiltered_buckets
|
433
662
|
|
434
|
-
def custom_sqs_filter(
|
663
|
+
def custom_sqs_filter(
|
664
|
+
self, account: str, sqs: SQSClient, queues: Iterable[str]
|
665
|
+
) -> list[str]:
|
435
666
|
type = "sqs queue"
|
436
667
|
unfiltered_queues = []
|
437
668
|
for q in queues:
|
@@ -441,7 +672,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
441
672
|
|
442
673
|
return unfiltered_queues
|
443
674
|
|
444
|
-
def custom_dynamodb_filter(
|
675
|
+
def custom_dynamodb_filter(
|
676
|
+
self,
|
677
|
+
account: str,
|
678
|
+
session: Session,
|
679
|
+
dynamodb: DynamoDBClient,
|
680
|
+
tables: Iterable[str],
|
681
|
+
) -> list[str]:
|
445
682
|
type = "dynamodb table"
|
446
683
|
dynamodb_resource = self._get_session_resource(session, "dynamodb")
|
447
684
|
unfiltered_tables = []
|
@@ -453,7 +690,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
453
690
|
|
454
691
|
return unfiltered_tables
|
455
692
|
|
456
|
-
def custom_rds_filter(
|
693
|
+
def custom_rds_filter(
|
694
|
+
self, account: str, rds: RDSClient, instances: Iterable[str]
|
695
|
+
) -> list[str]:
|
457
696
|
type = "rds instance"
|
458
697
|
unfiltered_instances = []
|
459
698
|
for i in instances:
|
@@ -465,7 +704,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
465
704
|
|
466
705
|
return unfiltered_instances
|
467
706
|
|
468
|
-
def custom_rds_snapshot_filter(
|
707
|
+
def custom_rds_snapshot_filter(
|
708
|
+
self, account: str, rds: RDSClient, snapshots: Iterable[str]
|
709
|
+
) -> list[str]:
|
469
710
|
type = "rds snapshots"
|
470
711
|
unfiltered_snapshots = []
|
471
712
|
for s in snapshots:
|
@@ -478,8 +719,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
478
719
|
return unfiltered_snapshots
|
479
720
|
|
480
721
|
def should_filter(
|
481
|
-
self,
|
482
|
-
|
722
|
+
self,
|
723
|
+
account: str,
|
724
|
+
resource_type: str,
|
725
|
+
resource_name: str,
|
726
|
+
resource_tags: Mapping,
|
727
|
+
tags_key: str,
|
728
|
+
) -> bool:
|
483
729
|
if self.resource_has_special_name(account, resource_type, resource_name):
|
484
730
|
return True
|
485
731
|
if tags_key in resource_tags:
|
@@ -492,7 +738,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
492
738
|
return False
|
493
739
|
|
494
740
|
@staticmethod
|
495
|
-
def resource_has_special_name(account, type, resource):
|
741
|
+
def resource_has_special_name(account: str, type: str, resource: str) -> bool:
|
496
742
|
skip_msg = f"[{account}] skipping {type} " + "({} related) {}"
|
497
743
|
|
498
744
|
ignore_names = {
|
@@ -509,7 +755,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
509
755
|
|
510
756
|
return False
|
511
757
|
|
512
|
-
def resource_has_special_tags(
|
758
|
+
def resource_has_special_tags(
|
759
|
+
self, account: str, type: str, resource: str, tags: Mapping | list[Mapping]
|
760
|
+
) -> bool:
|
513
761
|
skip_msg = f"[{account}] skipping {type} " + "({}={}) {}"
|
514
762
|
|
515
763
|
ignore_tags = {
|
@@ -530,7 +778,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
530
778
|
return False
|
531
779
|
|
532
780
|
@staticmethod
|
533
|
-
def get_tag_value(tags, tag):
|
781
|
+
def get_tag_value(tags: Mapping | list[Mapping], tag: str) -> str:
|
534
782
|
if isinstance(tags, dict):
|
535
783
|
return tags.get(tag, "")
|
536
784
|
if isinstance(tags, list):
|
@@ -540,7 +788,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
540
788
|
|
541
789
|
return ""
|
542
790
|
|
543
|
-
def delete_resources_without_owner(self, dry_run):
|
791
|
+
def delete_resources_without_owner(self, dry_run: bool) -> None:
|
544
792
|
for account, s in self.sessions.items():
|
545
793
|
for rt in self.resource_types:
|
546
794
|
for r in self.resources[account].get(rt + "_no_owner", []):
|
@@ -548,42 +796,50 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
548
796
|
if not dry_run:
|
549
797
|
self.delete_resource(s, rt, r)
|
550
798
|
|
551
|
-
def delete_resource(
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
565
|
-
|
566
|
-
|
567
|
-
|
568
|
-
|
799
|
+
def delete_resource(
|
800
|
+
self, session: Session, resource_type: RESOURCE_TYPE, resource_name: str
|
801
|
+
) -> None:
|
802
|
+
match resource_type:
|
803
|
+
case "s3":
|
804
|
+
self.delete_bucket(
|
805
|
+
self._get_session_resource(session, resource_type), resource_name
|
806
|
+
)
|
807
|
+
case "sqs":
|
808
|
+
self.delete_queue(
|
809
|
+
self.get_session_client(session, resource_type), resource_name
|
810
|
+
)
|
811
|
+
case "dynamodb":
|
812
|
+
self.delete_table(
|
813
|
+
self._get_session_resource(session, resource_type), resource_name
|
814
|
+
)
|
815
|
+
case "rds":
|
816
|
+
self.delete_instance(
|
817
|
+
self.get_session_client(session, resource_type), resource_name
|
818
|
+
)
|
819
|
+
case "rds_snapshots":
|
820
|
+
self.delete_snapshot(
|
821
|
+
self.get_session_client(session, "rds"), resource_name
|
822
|
+
)
|
823
|
+
case _:
|
824
|
+
raise InvalidResourceTypeError(resource_type)
|
569
825
|
|
570
826
|
@staticmethod
|
571
|
-
def delete_bucket(s3, bucket_name):
|
827
|
+
def delete_bucket(s3: S3ServiceResource, bucket_name: str) -> None:
|
572
828
|
bucket = s3.Bucket(bucket_name)
|
573
829
|
bucket.object_versions.delete()
|
574
830
|
bucket.delete()
|
575
831
|
|
576
832
|
@staticmethod
|
577
|
-
def delete_queue(sqs, queue_url):
|
833
|
+
def delete_queue(sqs: SQSClient, queue_url: str) -> None:
|
578
834
|
sqs.delete_queue(QueueUrl=queue_url)
|
579
835
|
|
580
836
|
@staticmethod
|
581
|
-
def delete_table(dynamodb, table_name):
|
837
|
+
def delete_table(dynamodb: DynamoDBServiceResource, table_name: str) -> None:
|
582
838
|
table = dynamodb.Table(table_name)
|
583
839
|
table.delete()
|
584
840
|
|
585
841
|
@staticmethod
|
586
|
-
def delete_instance(rds, instance_name):
|
842
|
+
def delete_instance(rds: RDSClient, instance_name: str) -> None:
|
587
843
|
rds.delete_db_instance(
|
588
844
|
DBInstanceIdentifier=instance_name,
|
589
845
|
SkipFinalSnapshot=True,
|
@@ -591,11 +847,11 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
591
847
|
)
|
592
848
|
|
593
849
|
@staticmethod
|
594
|
-
def delete_snapshot(rds, snapshot_identifier):
|
850
|
+
def delete_snapshot(rds: RDSClient, snapshot_identifier: str) -> None:
|
595
851
|
rds.delete_db_snapshot(DBSnapshotIdentifier=snapshot_identifier)
|
596
852
|
|
597
853
|
@staticmethod
|
598
|
-
def determine_key_type(iam, user):
|
854
|
+
def determine_key_type(iam: IAMClient, user: str) -> str:
|
599
855
|
tags = iam.list_user_tags(UserName=user)["Tags"]
|
600
856
|
managed_by_integration_tag = [
|
601
857
|
t["Value"] for t in tags if t["Key"] == "managed_by_integration"
|
@@ -622,8 +878,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
622
878
|
raise InvalidResourceTypeError(huh)
|
623
879
|
|
624
880
|
def delete_keys(
|
625
|
-
self,
|
626
|
-
|
881
|
+
self,
|
882
|
+
dry_run: bool,
|
883
|
+
keys_to_delete: Mapping,
|
884
|
+
working_dirs: Mapping[str, str],
|
885
|
+
disable_service_account_keys: bool,
|
886
|
+
) -> tuple[bool, bool]:
|
627
887
|
error = False
|
628
888
|
service_account_recycle_complete = True
|
629
889
|
users_keys = self.get_users_keys()
|
@@ -638,11 +898,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
638
898
|
]
|
639
899
|
if not user_and_user_keys:
|
640
900
|
continue
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
901
|
+
|
902
|
+
if len(user_and_user_keys) > 1:
|
903
|
+
raise RuntimeError(
|
904
|
+
f"key {key} returned multiple users: {user_and_user_keys}"
|
905
|
+
)
|
906
|
+
user = user_and_user_keys[0][0]
|
907
|
+
user_keys = user_and_user_keys[0][1]
|
646
908
|
key_type = self.determine_key_type(iam, user)
|
647
909
|
key_status = self.get_user_key_status(iam, user, key)
|
648
910
|
if key_type == "unmanaged" and key_status == "Active":
|
@@ -703,7 +965,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
703
965
|
|
704
966
|
return error, service_account_recycle_complete
|
705
967
|
|
706
|
-
def get_users_keys(self):
|
968
|
+
def get_users_keys(self) -> dict:
|
707
969
|
users_keys = {}
|
708
970
|
for account, s in self.sessions.items():
|
709
971
|
iam = self.get_session_client(s, "iam")
|
@@ -713,12 +975,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
713
975
|
|
714
976
|
return users_keys
|
715
977
|
|
716
|
-
def reset_password(self, account, user_name):
|
978
|
+
def reset_password(self, account: str, user_name: str) -> None:
|
717
979
|
s = self.sessions[account]
|
718
980
|
iam = self.get_session_client(s, "iam")
|
719
981
|
iam.delete_login_profile(UserName=user_name)
|
720
982
|
|
721
|
-
def reset_mfa(self, account, user_name):
|
983
|
+
def reset_mfa(self, account: str, user_name: str) -> None:
|
722
984
|
s = self.sessions[account]
|
723
985
|
iam = self.get_session_client(s, "iam")
|
724
986
|
mfa_devices = iam.list_mfa_devices(UserName=user_name)["MFADevices"]
|
@@ -742,7 +1004,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
742
1004
|
key_list = self._get_user_key_list(iam, user)
|
743
1005
|
return next(k["Status"] for k in key_list if k["AccessKeyId"] == key)
|
744
1006
|
|
745
|
-
def get_support_cases(self):
|
1007
|
+
def get_support_cases(self) -> dict[str, list[CaseDetailsTypeDef]]:
|
746
1008
|
all_support_cases = {}
|
747
1009
|
for account, s in self.sessions.items():
|
748
1010
|
if not self.accounts[account].get("premiumSupport"):
|
@@ -827,9 +1089,8 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
827
1089
|
return (account["name"], account.get("assume_role"), account["assume_region"])
|
828
1090
|
|
829
1091
|
@staticmethod
|
830
|
-
# pylint: disable=method-hidden
|
831
1092
|
def _get_assume_role_session(
|
832
|
-
sts, account_name: str, assume_role: str, assume_region: str
|
1093
|
+
sts: STSClient, account_name: str, assume_role: str, assume_region: str
|
833
1094
|
) -> Session:
|
834
1095
|
"""
|
835
1096
|
Returns a session for a supplied role to assume:
|
@@ -860,13 +1121,144 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
860
1121
|
|
861
1122
|
return assumed_session
|
862
1123
|
|
1124
|
+
@overload
|
863
1125
|
def _get_assumed_role_client(
|
864
1126
|
self,
|
865
1127
|
account_name: str,
|
866
1128
|
assume_role: str | None,
|
867
1129
|
assume_region: str,
|
868
|
-
client_type
|
869
|
-
) ->
|
1130
|
+
client_type: Literal["logs"],
|
1131
|
+
) -> CloudWatchLogsClient: ...
|
1132
|
+
|
1133
|
+
@overload
|
1134
|
+
def _get_assumed_role_client(
|
1135
|
+
self,
|
1136
|
+
account_name: str,
|
1137
|
+
assume_role: str | None,
|
1138
|
+
assume_region: str,
|
1139
|
+
client_type: Literal["dynamodb"],
|
1140
|
+
) -> DynamoDBClient: ...
|
1141
|
+
|
1142
|
+
@overload
|
1143
|
+
def _get_assumed_role_client(
|
1144
|
+
self,
|
1145
|
+
account_name: str,
|
1146
|
+
assume_role: str | None,
|
1147
|
+
assume_region: str,
|
1148
|
+
client_type: Literal["ec2"],
|
1149
|
+
) -> EC2Client: ...
|
1150
|
+
|
1151
|
+
@overload
|
1152
|
+
def _get_assumed_role_client(
|
1153
|
+
self,
|
1154
|
+
account_name: str,
|
1155
|
+
assume_role: str | None,
|
1156
|
+
assume_region: str,
|
1157
|
+
client_type: Literal["ecr"],
|
1158
|
+
) -> ECRClient: ...
|
1159
|
+
|
1160
|
+
@overload
|
1161
|
+
def _get_assumed_role_client(
|
1162
|
+
self,
|
1163
|
+
account_name: str,
|
1164
|
+
assume_role: str | None,
|
1165
|
+
assume_region: str,
|
1166
|
+
client_type: Literal["elb"],
|
1167
|
+
) -> ElasticLoadBalancingClient: ...
|
1168
|
+
|
1169
|
+
@overload
|
1170
|
+
def _get_assumed_role_client(
|
1171
|
+
self,
|
1172
|
+
account_name: str,
|
1173
|
+
assume_role: str | None,
|
1174
|
+
assume_region: str,
|
1175
|
+
client_type: Literal["iam"],
|
1176
|
+
) -> IAMClient: ...
|
1177
|
+
|
1178
|
+
@overload
|
1179
|
+
def _get_assumed_role_client(
|
1180
|
+
self,
|
1181
|
+
account_name: str,
|
1182
|
+
assume_role: str | None,
|
1183
|
+
assume_region: str,
|
1184
|
+
client_type: Literal["organizations"],
|
1185
|
+
) -> OrganizationsClient: ...
|
1186
|
+
|
1187
|
+
@overload
|
1188
|
+
def _get_assumed_role_client(
|
1189
|
+
self,
|
1190
|
+
account_name: str,
|
1191
|
+
assume_role: str | None,
|
1192
|
+
assume_region: str,
|
1193
|
+
client_type: Literal["rds"],
|
1194
|
+
) -> RDSClient: ...
|
1195
|
+
|
1196
|
+
@overload
|
1197
|
+
def _get_assumed_role_client(
|
1198
|
+
self,
|
1199
|
+
account_name: str,
|
1200
|
+
assume_role: str | None,
|
1201
|
+
assume_region: str,
|
1202
|
+
client_type: Literal["route53"],
|
1203
|
+
) -> Route53Client: ...
|
1204
|
+
|
1205
|
+
@overload
|
1206
|
+
def _get_assumed_role_client(
|
1207
|
+
self,
|
1208
|
+
account_name: str,
|
1209
|
+
assume_role: str | None,
|
1210
|
+
assume_region: str,
|
1211
|
+
client_type: Literal["s3"],
|
1212
|
+
) -> S3Client: ...
|
1213
|
+
|
1214
|
+
@overload
|
1215
|
+
def _get_assumed_role_client(
|
1216
|
+
self,
|
1217
|
+
account_name: str,
|
1218
|
+
assume_role: str | None,
|
1219
|
+
assume_region: str,
|
1220
|
+
client_type: Literal["sqs"],
|
1221
|
+
) -> SQSClient: ...
|
1222
|
+
|
1223
|
+
@overload
|
1224
|
+
def _get_assumed_role_client(
|
1225
|
+
self,
|
1226
|
+
account_name: str,
|
1227
|
+
assume_role: str | None,
|
1228
|
+
assume_region: str,
|
1229
|
+
client_type: Literal["sts"],
|
1230
|
+
) -> STSClient: ...
|
1231
|
+
|
1232
|
+
@overload
|
1233
|
+
def _get_assumed_role_client(
|
1234
|
+
self,
|
1235
|
+
account_name: str,
|
1236
|
+
assume_role: str | None,
|
1237
|
+
assume_region: str,
|
1238
|
+
client_type: Literal["support"],
|
1239
|
+
) -> SupportClient: ...
|
1240
|
+
|
1241
|
+
def _get_assumed_role_client(
|
1242
|
+
self,
|
1243
|
+
account_name: str,
|
1244
|
+
assume_role: str | None,
|
1245
|
+
assume_region: str,
|
1246
|
+
client_type: SERVICE_NAME = "ec2",
|
1247
|
+
) -> (
|
1248
|
+
CloudWatchLogsClient
|
1249
|
+
| DynamoDBClient
|
1250
|
+
| EC2Client
|
1251
|
+
| ECRClient
|
1252
|
+
| ElasticLoadBalancingClient
|
1253
|
+
| IAMClient
|
1254
|
+
| OrganizationsClient
|
1255
|
+
| RDSClient
|
1256
|
+
| Route53Client
|
1257
|
+
| S3Client
|
1258
|
+
| SQSClient
|
1259
|
+
| STSClient
|
1260
|
+
| SupportClient
|
1261
|
+
):
|
870
1262
|
session = self.get_session(account_name)
|
871
1263
|
if not assume_role:
|
872
1264
|
return self.get_session_client(
|
@@ -879,13 +1271,11 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
879
1271
|
return self.get_session_client(assumed_session, client_type)
|
880
1272
|
|
881
1273
|
@staticmethod
|
882
|
-
# pylint: disable=method-hidden
|
883
1274
|
def get_account_vpcs(ec2: EC2Client) -> list[VpcTypeDef]:
|
884
1275
|
vpcs = ec2.describe_vpcs()
|
885
1276
|
return vpcs.get("Vpcs", [])
|
886
1277
|
|
887
1278
|
@staticmethod
|
888
|
-
# pylint: disable=method-hidden
|
889
1279
|
def get_account_amis(ec2: EC2Client, owner: str) -> list[ImageTypeDef]:
|
890
1280
|
amis = ec2.describe_images(Owners=[owner])
|
891
1281
|
return amis.get("Images", [])
|
@@ -905,7 +1295,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
905
1295
|
return res
|
906
1296
|
|
907
1297
|
@staticmethod
|
908
|
-
# pylint: disable=method-hidden
|
909
1298
|
def get_vpc_route_tables(vpc_id: str, ec2: EC2Client) -> list[RouteTableTypeDef]:
|
910
1299
|
rts = ec2.describe_route_tables(
|
911
1300
|
Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
|
@@ -913,14 +1302,17 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
913
1302
|
return rts.get("RouteTables", [])
|
914
1303
|
|
915
1304
|
@staticmethod
|
916
|
-
# pylint: disable=method-hidden
|
917
1305
|
def get_vpc_subnets(vpc_id: str, ec2: EC2Client) -> list[SubnetTypeDef]:
|
918
1306
|
subnets = ec2.describe_subnets(Filters=[{"Name": "vpc-id", "Values": [vpc_id]}])
|
919
1307
|
return subnets.get("Subnets", [])
|
920
1308
|
|
921
1309
|
def get_cluster_vpc_details(
|
922
|
-
self,
|
923
|
-
|
1310
|
+
self,
|
1311
|
+
account: awsh.Account,
|
1312
|
+
route_tables: bool = False,
|
1313
|
+
subnets: bool = False,
|
1314
|
+
hcp_vpc_endpoint_sg: bool = False,
|
1315
|
+
) -> tuple[str | None, list[str] | None, list[dict[str, str]] | None, str | None]:
|
924
1316
|
"""
|
925
1317
|
Returns a cluster VPC details:
|
926
1318
|
- VPC ID
|
@@ -936,7 +1328,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
936
1328
|
use to find the matching VPC
|
937
1329
|
"""
|
938
1330
|
assume_role_data = self._get_account_assume_data(account)
|
939
|
-
assumed_ec2 = self._get_assumed_role_client(
|
1331
|
+
assumed_ec2 = self._get_assumed_role_client(
|
1332
|
+
account_name=assume_role_data[0],
|
1333
|
+
assume_role=assume_role_data[1],
|
1334
|
+
assume_region=assume_role_data[2],
|
1335
|
+
client_type="ec2",
|
1336
|
+
)
|
940
1337
|
vpcs = self.get_account_vpcs(assumed_ec2)
|
941
1338
|
vpc_id = None
|
942
1339
|
for vpc in vpcs:
|
@@ -964,7 +1361,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
964
1361
|
|
965
1362
|
return vpc_id, route_table_ids, subnets_id_az, api_security_group_id
|
966
1363
|
|
967
|
-
def _get_api_security_group_id(
|
1364
|
+
def _get_api_security_group_id(
|
1365
|
+
self, assumed_ec2: EC2Client, vpc_id: str
|
1366
|
+
) -> str | None:
|
968
1367
|
endpoints = AWSApi._get_vpc_endpoints(
|
969
1368
|
[
|
970
1369
|
{"Name": "vpc-id", "Values": [vpc_id]},
|
@@ -998,9 +1397,16 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
998
1397
|
)
|
999
1398
|
return security_groups[0]["GroupId"]
|
1000
1399
|
|
1001
|
-
def get_cluster_nat_gateways_egress_ips(
|
1002
|
-
|
1003
|
-
|
1400
|
+
def get_cluster_nat_gateways_egress_ips(
|
1401
|
+
self, account: dict[str, Any], vpc_id: str
|
1402
|
+
) -> set[str]:
|
1403
|
+
assume_role_data = self._get_account_assume_data(account)
|
1404
|
+
assumed_ec2 = self._get_assumed_role_client(
|
1405
|
+
account_name=assume_role_data[0],
|
1406
|
+
assume_role=assume_role_data[1],
|
1407
|
+
assume_region=assume_role_data[2],
|
1408
|
+
client_type="ec2",
|
1409
|
+
)
|
1004
1410
|
nat_gateways = assumed_ec2.describe_nat_gateways()
|
1005
1411
|
egress_ips: set[str] = set()
|
1006
1412
|
for nat in nat_gateways.get("NatGateways") or []:
|
@@ -1012,7 +1418,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1012
1418
|
|
1013
1419
|
return egress_ips
|
1014
1420
|
|
1015
|
-
def get_vpcs_details(
|
1421
|
+
def get_vpcs_details(
|
1422
|
+
self,
|
1423
|
+
account: awsh.Account,
|
1424
|
+
tags: Mapping[str, str] | None = None,
|
1425
|
+
route_tables: bool = False,
|
1426
|
+
) -> list[dict[str, Any]]:
|
1016
1427
|
results = []
|
1017
1428
|
ec2 = self._account_ec2_client(account["name"])
|
1018
1429
|
regions = [r["RegionName"] for r in ec2.describe_regions()["Regions"]]
|
@@ -1077,7 +1488,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1077
1488
|
share_account_uid: str,
|
1078
1489
|
image_id: str,
|
1079
1490
|
region: str | None = None,
|
1080
|
-
):
|
1491
|
+
) -> None:
|
1081
1492
|
ec2 = self._account_ec2_resource(account["name"], region)
|
1082
1493
|
image = ec2.Image(image_id)
|
1083
1494
|
launch_permission: LaunchPermissionModificationsTypeDef = {
|
@@ -1107,7 +1518,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1107
1518
|
self,
|
1108
1519
|
account_name: str,
|
1109
1520
|
region_name: str | None = None,
|
1110
|
-
) -> Iterator[
|
1521
|
+
) -> Iterator[LogGroupTypeDef]:
|
1111
1522
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1112
1523
|
paginator = client.get_paginator("describe_log_groups")
|
1113
1524
|
for page in paginator.paginate():
|
@@ -1131,7 +1542,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1131
1542
|
group_name: str,
|
1132
1543
|
retention_days: int,
|
1133
1544
|
region_name: str | None = None,
|
1134
|
-
):
|
1545
|
+
) -> None:
|
1135
1546
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1136
1547
|
client.put_retention_policy(
|
1137
1548
|
logGroupName=group_name, retentionInDays=retention_days
|
@@ -1142,21 +1553,33 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1142
1553
|
account_name: str,
|
1143
1554
|
group_name: str,
|
1144
1555
|
region_name: str | None = None,
|
1145
|
-
):
|
1556
|
+
) -> None:
|
1146
1557
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1147
1558
|
client.delete_log_group(logGroupName=group_name)
|
1148
1559
|
|
1149
1560
|
def create_tag(
|
1150
1561
|
self, account: Mapping[str, Any], resource_id: str, tag: Mapping[str, str]
|
1151
|
-
):
|
1562
|
+
) -> None:
|
1152
1563
|
ec2 = self._account_ec2_client(account["name"])
|
1153
1564
|
tag_type_def: TagTypeDef = {"Key": tag["Key"], "Value": tag["Value"]}
|
1154
1565
|
ec2.create_tags(Resources=[resource_id], Tags=[tag_type_def])
|
1155
1566
|
|
1156
|
-
def get_alb_network_interface_ips(
|
1567
|
+
def get_alb_network_interface_ips(
|
1568
|
+
self, account: awsh.Account, service_name: str
|
1569
|
+
) -> set[str]:
|
1157
1570
|
assumed_role_data = self._get_account_assume_data(account)
|
1158
|
-
ec2_client = self._get_assumed_role_client(
|
1159
|
-
|
1571
|
+
ec2_client = self._get_assumed_role_client(
|
1572
|
+
account_name=assumed_role_data[0],
|
1573
|
+
assume_role=assumed_role_data[1],
|
1574
|
+
assume_region=assumed_role_data[2],
|
1575
|
+
client_type="ec2",
|
1576
|
+
)
|
1577
|
+
elb_client = self._get_assumed_role_client(
|
1578
|
+
account_name=assumed_role_data[0],
|
1579
|
+
assume_role=assumed_role_data[1],
|
1580
|
+
assume_region=assumed_role_data[2],
|
1581
|
+
client_type="elb",
|
1582
|
+
)
|
1160
1583
|
service_tag = {"Key": "kubernetes.io/service-name", "Value": service_name}
|
1161
1584
|
nis = ec2_client.describe_network_interfaces()["NetworkInterfaces"]
|
1162
1585
|
lbs = elb_client.describe_load_balancers()["LoadBalancerDescriptions"]
|
@@ -1184,7 +1607,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1184
1607
|
return result_ips
|
1185
1608
|
|
1186
1609
|
@staticmethod
|
1187
|
-
# pylint: disable=method-hidden
|
1188
1610
|
def get_vpc_default_sg_id(vpc_id: str, ec2: EC2Client) -> str | None:
|
1189
1611
|
vpc_security_groups = ec2.describe_security_groups(
|
1190
1612
|
Filters=[
|
@@ -1199,7 +1621,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1199
1621
|
return None
|
1200
1622
|
|
1201
1623
|
@staticmethod
|
1202
|
-
# pylint: disable=method-hidden
|
1203
1624
|
def get_transit_gateways(ec2: EC2Client) -> list[TransitGatewayTypeDef]:
|
1204
1625
|
tgws = ec2.describe_transit_gateways()
|
1205
1626
|
return tgws.get("TransitGateways", [])
|
@@ -1222,7 +1643,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1222
1643
|
return None
|
1223
1644
|
|
1224
1645
|
@staticmethod
|
1225
|
-
# pylint: disable=method-hidden
|
1226
1646
|
def get_transit_gateway_vpc_attachments(
|
1227
1647
|
tgw_id: str, ec2: EC2Client
|
1228
1648
|
) -> list[TransitGatewayVpcAttachmentTypeDef]:
|
@@ -1233,23 +1653,23 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1233
1653
|
|
1234
1654
|
def get_tgws_details(
|
1235
1655
|
self,
|
1236
|
-
account,
|
1237
|
-
region_name,
|
1238
|
-
routes_cidr_block,
|
1239
|
-
tags
|
1240
|
-
route_tables=False,
|
1241
|
-
security_groups=False,
|
1242
|
-
route53_associations=False,
|
1243
|
-
):
|
1656
|
+
account: awsh.Account,
|
1657
|
+
region_name: str,
|
1658
|
+
routes_cidr_block: str,
|
1659
|
+
tags: Mapping,
|
1660
|
+
route_tables: bool = False,
|
1661
|
+
security_groups: bool = False,
|
1662
|
+
route53_associations: bool = False,
|
1663
|
+
) -> list[dict[str, Any]]:
|
1244
1664
|
results = []
|
1245
1665
|
ec2 = self._account_ec2_client(account["name"], region_name)
|
1246
1666
|
tgws = ec2.describe_transit_gateways(
|
1247
1667
|
Filters=[{"Name": f"tag:{k}", "Values": [v]} for k, v in tags.items()]
|
1248
1668
|
)
|
1249
|
-
for tgw in tgws.get("TransitGateways"):
|
1669
|
+
for tgw in tgws.get("TransitGateways") or []:
|
1250
1670
|
tgw_id = tgw["TransitGatewayId"]
|
1251
1671
|
tgw_arn = tgw["TransitGatewayArn"]
|
1252
|
-
item = {
|
1672
|
+
item: dict[str, str | list[str] | list[dict]] = {
|
1253
1673
|
"tgw_id": tgw_id,
|
1254
1674
|
"tgw_arn": tgw_arn,
|
1255
1675
|
"region": region_name,
|
@@ -1281,7 +1701,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1281
1701
|
attachments = ec2.describe_transit_gateway_peering_attachments(
|
1282
1702
|
Filters=[{"Name": "transit-gateway-id", "Values": [tgw_id]}]
|
1283
1703
|
)
|
1284
|
-
for a in attachments.get("TransitGatewayPeeringAttachments"):
|
1704
|
+
for a in attachments.get("TransitGatewayPeeringAttachments") or []:
|
1285
1705
|
tgw_attachment_id = a["TransitGatewayAttachmentId"]
|
1286
1706
|
tgw_attachment_state = a["State"]
|
1287
1707
|
if tgw_attachment_state != "available":
|
@@ -1378,7 +1798,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1378
1798
|
return results
|
1379
1799
|
|
1380
1800
|
@staticmethod
|
1381
|
-
# pylint: disable=method-hidden
|
1382
1801
|
def _get_vpc_endpoints(
|
1383
1802
|
filters: Sequence[FilterTypeDef], ec2: EC2Client
|
1384
1803
|
) -> list["VpcEndpointTypeDef"]:
|
@@ -1412,11 +1831,15 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1412
1831
|
]
|
1413
1832
|
|
1414
1833
|
@staticmethod
|
1415
|
-
def _extract_records(
|
1834
|
+
def _extract_records(
|
1835
|
+
resource_records: Iterable[ResourceRecordTypeDef],
|
1836
|
+
) -> list[str]:
|
1416
1837
|
# [{'Value': 'ns.example.com.'}, ...]
|
1417
1838
|
return [r["Value"].rstrip(".") for r in resource_records]
|
1418
1839
|
|
1419
|
-
def get_route53_zone_ns_records(
|
1840
|
+
def get_route53_zone_ns_records(
|
1841
|
+
self, account_name: str, zone_name: str, region: str
|
1842
|
+
) -> list[str]:
|
1420
1843
|
route53 = self._account_route53_client(account_name, region)
|
1421
1844
|
record_sets = self._get_hosted_zone_record_sets(route53, zone_name)
|
1422
1845
|
filtered_record_sets = self._filter_record_sets(record_sets, zone_name, "NS")
|
@@ -1426,7 +1849,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1426
1849
|
ns_records = self._extract_records(resource_records)
|
1427
1850
|
return ns_records
|
1428
1851
|
|
1429
|
-
def get_route53_zones(self):
|
1852
|
+
def get_route53_zones(self) -> dict[str, list[dict[str, str]]]:
|
1430
1853
|
"""
|
1431
1854
|
Return a list of (str, dict) representing Route53 DNS zones per account
|
1432
1855
|
|
@@ -1438,7 +1861,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1438
1861
|
for account, _ in self.sessions.items()
|
1439
1862
|
}
|
1440
1863
|
|
1441
|
-
def create_route53_zone(self, account_name, zone_name):
|
1864
|
+
def create_route53_zone(self, account_name: str, zone_name: str) -> None:
|
1442
1865
|
"""
|
1443
1866
|
Create a Route53 DNS zone
|
1444
1867
|
|
@@ -1468,7 +1891,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1468
1891
|
except Exception as e:
|
1469
1892
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1470
1893
|
|
1471
|
-
def delete_route53_zone(self, account_name, zone_id):
|
1894
|
+
def delete_route53_zone(self, account_name: str, zone_id: str) -> None:
|
1472
1895
|
"""
|
1473
1896
|
Delete a Route53 DNS zone
|
1474
1897
|
|
@@ -1493,7 +1916,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1493
1916
|
except Exception as e:
|
1494
1917
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1495
1918
|
|
1496
|
-
def delete_route53_record(
|
1919
|
+
def delete_route53_record(
|
1920
|
+
self, account_name: str, zone_id: str, awsdata: ResourceRecordSetTypeDef
|
1921
|
+
) -> None:
|
1497
1922
|
"""
|
1498
1923
|
Delete a Route53 DNS zone record
|
1499
1924
|
|
@@ -1527,7 +1952,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1527
1952
|
except Exception as e:
|
1528
1953
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1529
1954
|
|
1530
|
-
def upsert_route53_record(
|
1955
|
+
def upsert_route53_record(
|
1956
|
+
self, account_name: str, zone_id: str, recordset: ResourceRecordSetTypeDef
|
1957
|
+
) -> None:
|
1531
1958
|
"""
|
1532
1959
|
Upsert a Route53 DNS zone record
|
1533
1960
|
|
@@ -1613,7 +2040,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1613
2040
|
self,
|
1614
2041
|
account_name: str,
|
1615
2042
|
region_name: str | None = None,
|
1616
|
-
):
|
2043
|
+
) -> DBRecommendationsMessageTypeDef:
|
1617
2044
|
rds = self._account_rds_client(account_name, region_name)
|
1618
2045
|
return rds.describe_db_recommendations()
|
1619
2046
|
|