cloud-drift-report 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
aws_inventory.py ADDED
@@ -0,0 +1,1029 @@
1
+ """Read-only AWS inventory: find resources that exist in AWS but no Terraform state or
2
+ CloudFormation stack you point at lists ("unmanaged", or "not in IaC").
3
+
4
+ How it works:
5
+ 1. List resources per region, mostly through the Cloud Control API (ListResources), and
6
+ through the service's own list call where Cloud Control lacks the fields needed to tell
7
+ an AWS default (default VPC, service-linked role, AWS-managed key) from a customer's.
8
+ 2. Drop what AWS creates in every account or for another service.
9
+ 3. Collect identifiers from Terraform state files and CloudFormation stack resources.
10
+ 4. Report every listed resource no state or stack accounts for.
11
+
12
+ Only List/Describe/Get calls are made. Nothing is written to AWS or to state.
13
+ Licence: Apache-2.0.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ import os
20
+ import re
21
+ import subprocess
22
+ import sys
23
+ import time
24
+ from concurrent.futures import ThreadPoolExecutor
25
+ from dataclasses import dataclass, field
26
+ from typing import Any, Callable, Iterable
27
+
28
+ # --- what is listed --------------------------------------------------------
29
+
30
+ # Cloud Control type -> Terraform type. Each Terraform type's state id (or name/arn) is
31
+ # what Cloud Control returns as the primary identifier, so the two can be matched.
32
+ CLOUDCONTROL_TO_TERRAFORM: dict[str, str] = {
33
+ # core services
34
+ "AWS::EC2::VPC": "aws_vpc",
35
+ "AWS::EC2::Subnet": "aws_subnet",
36
+ "AWS::EC2::SecurityGroup": "aws_security_group",
37
+ "AWS::EC2::Instance": "aws_instance",
38
+ "AWS::EC2::InternetGateway": "aws_internet_gateway",
39
+ "AWS::EC2::DHCPOptions": "aws_vpc_dhcp_options",
40
+ "AWS::EC2::RouteTable": "aws_route_table",
41
+ "AWS::EC2::NetworkAcl": "aws_network_acl",
42
+ "AWS::EC2::Volume": "aws_ebs_volume",
43
+ "AWS::EC2::VPCEndpoint": "aws_vpc_endpoint",
44
+ "AWS::S3::Bucket": "aws_s3_bucket",
45
+ "AWS::Lambda::Function": "aws_lambda_function",
46
+ "AWS::RDS::DBInstance": "aws_db_instance",
47
+ "AWS::RDS::DBCluster": "aws_rds_cluster",
48
+ "AWS::IAM::Role": "aws_iam_role",
49
+ "AWS::SQS::Queue": "aws_sqs_queue",
50
+ "AWS::SNS::Topic": "aws_sns_topic",
51
+ "AWS::DynamoDB::Table": "aws_dynamodb_table",
52
+ "AWS::ECS::Cluster": "aws_ecs_cluster",
53
+ "AWS::ECS::Service": "aws_ecs_service",
54
+ "AWS::EKS::Cluster": "aws_eks_cluster",
55
+ "AWS::EKS::Nodegroup": "aws_eks_node_group",
56
+ "AWS::ElasticLoadBalancingV2::LoadBalancer": "aws_lb",
57
+ "AWS::ElasticLoadBalancing::LoadBalancer": "aws_elb",
58
+ "AWS::KMS::Key": "aws_kms_key",
59
+ "AWS::Logs::LogGroup": "aws_cloudwatch_log_group",
60
+ "AWS::SecretsManager::Secret": "aws_secretsmanager_secret",
61
+ "AWS::ECR::Repository": "aws_ecr_repository",
62
+ "AWS::Route53::HostedZone": "aws_route53_zone",
63
+ "AWS::CloudFront::Distribution": "aws_cloudfront_distribution",
64
+ "AWS::StepFunctions::StateMachine": "aws_sfn_state_machine",
65
+ "AWS::Events::EventBus": "aws_cloudwatch_event_bus",
66
+ "AWS::Events::Rule": "aws_cloudwatch_event_rule",
67
+ "AWS::ApiGateway::RestApi": "aws_api_gateway_rest_api",
68
+ "AWS::ApiGatewayV2::Api": "aws_apigatewayv2_api",
69
+ "AWS::Kinesis::Stream": "aws_kinesis_stream",
70
+ "AWS::KinesisFirehose::DeliveryStream": "aws_kinesis_firehose_delivery_stream",
71
+ "AWS::Cognito::UserPool": "aws_cognito_user_pool",
72
+ "AWS::EFS::FileSystem": "aws_efs_file_system",
73
+ "AWS::SSM::Parameter": "aws_ssm_parameter",
74
+ "AWS::CodeBuild::Project": "aws_codebuild_project",
75
+ "AWS::CodePipeline::Pipeline": "aws_codepipeline",
76
+ "AWS::Glue::Crawler": "aws_glue_crawler",
77
+ "AWS::OpenSearchService::Domain": "aws_opensearch_domain",
78
+ "AWS::MSK::Cluster": "aws_msk_cluster",
79
+ "AWS::Redshift::Cluster": "aws_redshift_cluster",
80
+ "AWS::ElastiCache::ReplicationGroup": "aws_elasticache_replication_group",
81
+ "AWS::ElastiCache::CacheCluster": "aws_elasticache_cluster",
82
+ "AWS::WAFv2::WebACL": "aws_wafv2_web_acl",
83
+ "AWS::CloudTrail::Trail": "aws_cloudtrail",
84
+ # long tail (copied from OrchesTerra's Cloud Control discovery list)
85
+ "AWS::CertificateManager::Certificate": "aws_acm_certificate",
86
+ "AWS::ACMPCA::CertificateAuthority": "aws_acmpca_certificate_authority",
87
+ "AWS::AmazonMQ::Broker": "aws_mq_broker",
88
+ "AWS::ApiGateway::ApiKey": "aws_api_gateway_api_key",
89
+ "AWS::ApiGateway::DomainName": "aws_api_gateway_domain_name",
90
+ "AWS::ApiGateway::UsagePlan": "aws_api_gateway_usage_plan",
91
+ "AWS::ApiGateway::VpcLink": "aws_api_gateway_vpc_link",
92
+ "AWS::AppConfig::Application": "aws_appconfig_application",
93
+ "AWS::AppConfig::ConfigurationProfile": "aws_appconfig_configuration_profile",
94
+ "AWS::AppConfig::Environment": "aws_appconfig_environment",
95
+ "AWS::AppSync::GraphQLApi": "aws_appsync_graphql_api",
96
+ "AWS::ApplicationAutoScaling::ScalableTarget": "aws_appautoscaling_target",
97
+ "AWS::Athena::DataCatalog": "aws_athena_data_catalog",
98
+ "AWS::Athena::NamedQuery": "aws_athena_named_query",
99
+ "AWS::Athena::WorkGroup": "aws_athena_workgroup",
100
+ "AWS::AutoScaling::AutoScalingGroup": "aws_autoscaling_group",
101
+ "AWS::AutoScaling::LaunchConfiguration": "aws_launch_configuration",
102
+ "AWS::Backup::BackupPlan": "aws_backup_plan",
103
+ "AWS::Backup::BackupVault": "aws_backup_vault",
104
+ "AWS::Batch::ComputeEnvironment": "aws_batch_compute_environment",
105
+ "AWS::Batch::JobQueue": "aws_batch_job_queue",
106
+ "AWS::Budgets::Budget": "aws_budgets_budget",
107
+ "AWS::Cloud9::EnvironmentEC2": "aws_cloud9_environment_ec2",
108
+ "AWS::CloudFront::CachePolicy": "aws_cloudfront_cache_policy",
109
+ "AWS::CloudFront::Function": "aws_cloudfront_function",
110
+ "AWS::CloudFront::OriginAccessControl": "aws_cloudfront_origin_access_control",
111
+ "AWS::CloudFront::OriginRequestPolicy": "aws_cloudfront_origin_request_policy",
112
+ "AWS::CloudFront::ResponseHeadersPolicy": "aws_cloudfront_response_headers_policy",
113
+ "AWS::CloudWatch::Alarm": "aws_cloudwatch_metric_alarm",
114
+ "AWS::CloudWatch::CompositeAlarm": "aws_cloudwatch_composite_alarm",
115
+ "AWS::CloudWatch::Dashboard": "aws_cloudwatch_dashboard",
116
+ "AWS::CodeArtifact::Domain": "aws_codeartifact_domain",
117
+ "AWS::CodeArtifact::Repository": "aws_codeartifact_repository",
118
+ "AWS::CodeCommit::Repository": "aws_codecommit_repository",
119
+ "AWS::CodeStarConnections::Connection": "aws_codestarconnections_connection",
120
+ "AWS::Cognito::IdentityPool": "aws_cognito_identity_pool",
121
+ "AWS::Config::ConfigurationAggregator": "aws_config_configuration_aggregator",
122
+ "AWS::DataSync::Task": "aws_datasync_task",
123
+ "AWS::DMS::ReplicationInstance": "aws_dms_replication_instance",
124
+ "AWS::EC2::CustomerGateway": "aws_customer_gateway",
125
+ "AWS::EC2::EIP": "aws_eip",
126
+ "AWS::EC2::FlowLog": "aws_flow_log",
127
+ "AWS::EC2::LaunchTemplate": "aws_launch_template",
128
+ "AWS::EC2::NatGateway": "aws_nat_gateway",
129
+ "AWS::EC2::PlacementGroup": "aws_placement_group",
130
+ "AWS::EC2::PrefixList": "aws_ec2_managed_prefix_list",
131
+ "AWS::EC2::TransitGateway": "aws_ec2_transit_gateway",
132
+ "AWS::EC2::TransitGatewayAttachment": "aws_ec2_transit_gateway_vpc_attachment",
133
+ "AWS::EC2::VPCPeeringConnection": "aws_vpc_peering_connection",
134
+ "AWS::EC2::VPNGateway": "aws_vpn_gateway",
135
+ "AWS::ECR::PullThroughCacheRule": "aws_ecr_pull_through_cache_rule",
136
+ "AWS::ECS::TaskDefinition": "aws_ecs_task_definition",
137
+ "AWS::EFS::AccessPoint": "aws_efs_access_point",
138
+ "AWS::EKS::Addon": "aws_eks_addon",
139
+ "AWS::EKS::FargateProfile": "aws_eks_fargate_profile",
140
+ "AWS::ElasticLoadBalancingV2::Listener": "aws_lb_listener",
141
+ "AWS::ElasticLoadBalancingV2::TargetGroup": "aws_lb_target_group",
142
+ "AWS::EMR::Studio": "aws_emr_studio",
143
+ "AWS::Events::ApiDestination": "aws_cloudwatch_event_api_destination",
144
+ "AWS::Events::Archive": "aws_cloudwatch_event_archive",
145
+ "AWS::Events::Connection": "aws_cloudwatch_event_connection",
146
+ "AWS::Glue::Job": "aws_glue_job",
147
+ "AWS::Glue::Registry": "aws_glue_registry",
148
+ "AWS::Glue::Trigger": "aws_glue_trigger",
149
+ "AWS::GuardDuty::Detector": "aws_guardduty_detector",
150
+ "AWS::IAM::Group": "aws_iam_group",
151
+ "AWS::IAM::InstanceProfile": "aws_iam_instance_profile",
152
+ "AWS::IAM::ManagedPolicy": "aws_iam_policy",
153
+ "AWS::IAM::OIDCProvider": "aws_iam_openid_connect_provider",
154
+ "AWS::IAM::SAMLProvider": "aws_iam_saml_provider",
155
+ "AWS::IAM::User": "aws_iam_user",
156
+ "AWS::ImageBuilder::Component": "aws_imagebuilder_component",
157
+ "AWS::ImageBuilder::ImagePipeline": "aws_imagebuilder_image_pipeline",
158
+ "AWS::IoT::Thing": "aws_iot_thing",
159
+ "AWS::IoT::TopicRule": "aws_iot_topic_rule",
160
+ "AWS::KinesisAnalyticsV2::Application": "aws_kinesisanalyticsv2_application",
161
+ "AWS::KMS::Alias": "aws_kms_alias",
162
+ "AWS::Lambda::EventSourceMapping": "aws_lambda_event_source_mapping",
163
+ "AWS::Lambda::LayerVersion": "aws_lambda_layer_version",
164
+ "AWS::Logs::Destination": "aws_cloudwatch_log_destination",
165
+ "AWS::Logs::QueryDefinition": "aws_cloudwatch_query_definition",
166
+ "AWS::Logs::ResourcePolicy": "aws_cloudwatch_log_resource_policy",
167
+ "AWS::Logs::SubscriptionFilter": "aws_cloudwatch_log_subscription_filter",
168
+ "AWS::MemoryDB::Cluster": "aws_memorydb_cluster",
169
+ "AWS::MWAA::Environment": "aws_mwaa_environment",
170
+ "AWS::NetworkFirewall::Firewall": "aws_networkfirewall_firewall",
171
+ "AWS::NetworkFirewall::FirewallPolicy": "aws_networkfirewall_firewall_policy",
172
+ "AWS::NetworkFirewall::RuleGroup": "aws_networkfirewall_rule_group",
173
+ "AWS::Pipes::Pipe": "aws_pipes_pipe",
174
+ "AWS::RDS::DBClusterParameterGroup": "aws_rds_cluster_parameter_group",
175
+ "AWS::RDS::DBParameterGroup": "aws_db_parameter_group",
176
+ "AWS::RDS::DBSubnetGroup": "aws_db_subnet_group",
177
+ "AWS::RDS::OptionGroup": "aws_db_option_group",
178
+ "AWS::Route53::HealthCheck": "aws_route53_health_check",
179
+ "AWS::Route53Resolver::ResolverEndpoint": "aws_route53_resolver_endpoint",
180
+ "AWS::Route53Resolver::ResolverRule": "aws_route53_resolver_rule",
181
+ "AWS::S3::AccessPoint": "aws_s3_access_point",
182
+ "AWS::SageMaker::Domain": "aws_sagemaker_domain",
183
+ "AWS::SageMaker::NotebookInstance": "aws_sagemaker_notebook_instance",
184
+ "AWS::Scheduler::Schedule": "aws_scheduler_schedule",
185
+ "AWS::Scheduler::ScheduleGroup": "aws_scheduler_schedule_group",
186
+ "AWS::SecretsManager::RotationSchedule": "aws_secretsmanager_secret_rotation",
187
+ "AWS::SES::ConfigurationSet": "aws_sesv2_configuration_set",
188
+ "AWS::SES::EmailIdentity": "aws_sesv2_email_identity",
189
+ "AWS::SES::Template": "aws_ses_template",
190
+ "AWS::SNS::Subscription": "aws_sns_topic_subscription",
191
+ "AWS::SSM::Document": "aws_ssm_document",
192
+ "AWS::SSM::MaintenanceWindow": "aws_ssm_maintenance_window",
193
+ "AWS::SSM::PatchBaseline": "aws_ssm_patch_baseline",
194
+ "AWS::StepFunctions::Activity": "aws_sfn_activity",
195
+ "AWS::Synthetics::Canary": "aws_synthetics_canary",
196
+ "AWS::Transfer::Server": "aws_transfer_server",
197
+ "AWS::WAFv2::IPSet": "aws_wafv2_ip_set",
198
+ "AWS::WAFv2::RegexPatternSet": "aws_wafv2_regex_pattern_set",
199
+ "AWS::WAFv2::RuleGroup": "aws_wafv2_rule_group",
200
+ "AWS::XRay::Group": "aws_xray_group",
201
+ "AWS::XRay::SamplingRule": "aws_xray_sampling_rule",
202
+ }
203
+
204
+ # Terraform type aliases that name the same AWS resource.
205
+ TERRAFORM_TYPE_ALIASES = {
206
+ "aws_alb": "aws_lb",
207
+ "aws_alb_listener": "aws_lb_listener",
208
+ "aws_alb_target_group": "aws_lb_target_group",
209
+ "aws_docdb_cluster": "aws_rds_cluster",
210
+ "aws_neptune_cluster": "aws_rds_cluster",
211
+ "aws_elasticsearch_domain": "aws_opensearch_domain",
212
+ }
213
+
214
+ # Account-wide services: listed once, in the first region scanned.
215
+ _GLOBAL_PREFIXES = ("AWS::IAM::", "AWS::CloudFront::", "AWS::Route53::", "AWS::Budgets::", "AWS::S3::Bucket")
216
+
217
+ MAX_PER_TYPE = 1000
218
+
219
+ _SKIP_ERRORS = (
220
+ "UnsupportedActionException", "TypeNotFoundException", "InvalidRequestException",
221
+ "GeneralServiceException", "AccessDenied", "UnauthorizedOperation", "not authorized",
222
+ "is not supported", "Could not connect to the endpoint", "OptInRequired",
223
+ "SubscriptionRequiredException", "UnrecognizedClientException",
224
+ )
225
+
226
+ SECURITY_TYPES = ("aws_security_group", "aws_iam_", "aws_kms_", "aws_network_acl", "aws_wafv2_",
227
+ "aws_networkfirewall_", "aws_secretsmanager_", "aws_acm", "aws_cognito_")
228
+
229
+
230
+ def terraform_type(cc_type: str) -> str:
231
+ return CLOUDCONTROL_TO_TERRAFORM.get(cc_type, "")
232
+
233
+
234
+ def is_global(cc_type: str) -> bool:
235
+ return cc_type.startswith(_GLOBAL_PREFIXES)
236
+
237
+
238
+ # --- live resources --------------------------------------------------------
239
+
240
+ @dataclass
241
+ class LiveResource:
242
+ cc_type: str
243
+ tf_type: str
244
+ identifier: str
245
+ region: str
246
+ name: str = ""
247
+ arn: str = ""
248
+ props: dict[str, Any] = field(default_factory=dict)
249
+
250
+ def keys(self) -> list[frozenset[str]]:
251
+ """Alternative ways IaC may record this resource. A key matches when all its parts appear."""
252
+ out = []
253
+ ident = self.identifier
254
+ if "|" in ident:
255
+ out.append(frozenset(p for p in ident.split("|") if p))
256
+ elif ident:
257
+ out.append(frozenset([ident]))
258
+ parts = set(ident.split("|"))
259
+ for extra in (self.arn, self.name):
260
+ # A name that is only one part of a composite id (a parent's id) would match a sibling.
261
+ if extra and extra != ident and extra not in parts:
262
+ out.append(frozenset([extra]))
263
+ for alt in self.props.get("_aliases") or []:
264
+ out.append(frozenset(alt) if isinstance(alt, (list, tuple, set, frozenset)) else frozenset([alt]))
265
+ return [k for k in out if k]
266
+
267
+
268
+ def _props(raw: Any) -> dict[str, Any]:
269
+ if isinstance(raw, dict):
270
+ return raw
271
+ try:
272
+ parsed = json.loads(raw or "{}")
273
+ except (TypeError, ValueError):
274
+ return {}
275
+ return parsed if isinstance(parsed, dict) else {}
276
+
277
+
278
+ def _tag_map(raw: Any) -> dict[str, str]:
279
+ if isinstance(raw, dict):
280
+ return {str(k): str(v) for k, v in raw.items()}
281
+ out = {}
282
+ for item in raw or []:
283
+ if isinstance(item, dict) and item.get("Key") is not None:
284
+ out[str(item["Key"])] = str(item.get("Value") or "")
285
+ return out
286
+
287
+
288
+ def _name(identifier: str, props: dict[str, Any]) -> str:
289
+ for key in ("Name", "DomainName", "FunctionName", "GroupName", "RoleName", "UserName", "PolicyName",
290
+ "AliasName", "BucketName", "TableName", "QueueName", "TopicName", "ClusterName",
291
+ "LogGroupName", "RepositoryName", "StateMachineName", "PrefixListName", "RuleName",
292
+ "DBInstanceIdentifier", "LoadBalancerName"):
293
+ value = props.get(key)
294
+ if isinstance(value, str) and value:
295
+ return value
296
+ tags = _tag_map(props.get("Tags"))
297
+ if tags.get("Name"):
298
+ return tags["Name"]
299
+ if identifier.startswith("arn:"):
300
+ return re.split(r"[:/]", identifier)[-1]
301
+ if identifier.startswith("https://"):
302
+ return identifier.rstrip("/").rsplit("/", 1)[-1]
303
+ return identifier.rsplit("|", 1)[-1]
304
+
305
+
306
+ def _arn_region(arn: str) -> str:
307
+ parts = arn.split(":")
308
+ return parts[3] if len(parts) > 5 and arn.startswith("arn:") else ""
309
+
310
+
311
+ def _paginate(client: Any, method: str, key: str, **kwargs: Any) -> list[dict[str, Any]]:
312
+ out: list[dict[str, Any]] = []
313
+ if client.can_paginate(method):
314
+ for page in client.get_paginator(method).paginate(**kwargs):
315
+ out.extend(page.get(key) or [])
316
+ return out
317
+ token = None
318
+ while True:
319
+ page = getattr(client, method)(**({**kwargs, "NextToken": token} if token else kwargs)) or {}
320
+ out.extend(page.get(key) or [])
321
+ token = page.get("NextToken")
322
+ if not token:
323
+ return out
324
+
325
+
326
+ def list_cloudcontrol(client: Any, cc_type: str, model: dict | None = None) -> list[dict[str, Any]]:
327
+ """ResourceDescriptions for one type. Unsupported types and denied calls raise Skipped."""
328
+ out: list[dict[str, Any]] = []
329
+ token = None
330
+ while len(out) < MAX_PER_TYPE:
331
+ kwargs: dict[str, Any] = {"TypeName": cc_type, "MaxResults": 100}
332
+ if model:
333
+ kwargs["ResourceModel"] = json.dumps(model)
334
+ if token:
335
+ kwargs["NextToken"] = token
336
+ try:
337
+ page = client.list_resources(**kwargs)
338
+ except Exception as exc:
339
+ text = f"{type(exc).__name__} {exc}"
340
+ if any(marker in text for marker in _SKIP_ERRORS):
341
+ raise Skipped(text) from exc
342
+ raise
343
+ out.extend(page.get("ResourceDescriptions") or [])
344
+ token = page.get("NextToken")
345
+ if not token:
346
+ break
347
+ return out
348
+
349
+
350
+ class Skipped(Exception):
351
+ """A type that could not be listed (not supported in the region, or access denied)."""
352
+
353
+
354
+ # Native listers: (session, region) -> [{"Identifier": ..., "Properties": {...}}].
355
+ # Used where Cloud Control's model lacks what tells an AWS default from a customer's resource,
356
+ # or where ListResources needs a parent identifier.
357
+
358
+ def _vpcs(s, r):
359
+ ec2 = s.client("ec2", region_name=r)
360
+ return [{"Identifier": v["VpcId"], "Properties": {"IsDefault": v.get("IsDefault"), "Tags": v.get("Tags") or []}}
361
+ for v in _paginate(ec2, "describe_vpcs", "Vpcs")]
362
+
363
+
364
+ def _default_vpc_ids(ec2) -> set[str]:
365
+ return {v["VpcId"] for v in _paginate(ec2, "describe_vpcs", "Vpcs", Filters=[{"Name": "is-default", "Values": ["true"]}])}
366
+
367
+
368
+ def _subnets(s, r):
369
+ ec2 = s.client("ec2", region_name=r)
370
+ return [{"Identifier": n["SubnetId"], "Properties": {"DefaultForAz": n.get("DefaultForAz"), "VpcId": n.get("VpcId"),
371
+ "Tags": n.get("Tags") or []}}
372
+ for n in _paginate(ec2, "describe_subnets", "Subnets")]
373
+
374
+
375
+ def _security_groups(s, r):
376
+ ec2 = s.client("ec2", region_name=r)
377
+ return [{"Identifier": g["GroupId"], "Properties": {"GroupName": g.get("GroupName"), "Description": g.get("Description"),
378
+ "VpcId": g.get("VpcId"), "Tags": g.get("Tags") or []}}
379
+ for g in _paginate(ec2, "describe_security_groups", "SecurityGroups")]
380
+
381
+
382
+ def _internet_gateways(s, r):
383
+ ec2 = s.client("ec2", region_name=r)
384
+ defaults = _default_vpc_ids(ec2)
385
+ out = []
386
+ for g in _paginate(ec2, "describe_internet_gateways", "InternetGateways"):
387
+ vpcs = {a.get("VpcId") for a in g.get("Attachments") or []}
388
+ out.append({"Identifier": g["InternetGatewayId"],
389
+ "Properties": {"IsDefault": bool(vpcs & defaults), "Tags": g.get("Tags") or []}})
390
+ return out
391
+
392
+
393
+ def _dhcp_options(s, r):
394
+ ec2 = s.client("ec2", region_name=r)
395
+ vpcs = _paginate(ec2, "describe_vpcs", "Vpcs")
396
+ in_default = {v.get("DhcpOptionsId") for v in vpcs if v.get("IsDefault")}
397
+ in_custom = {v.get("DhcpOptionsId") for v in vpcs if not v.get("IsDefault")}
398
+ out = []
399
+ for d in _paginate(ec2, "describe_dhcp_options", "DhcpOptions"):
400
+ did = d["DhcpOptionsId"]
401
+ # The default set is created with the default VPC; custom VPCs share it unless told otherwise.
402
+ out.append({"Identifier": did, "Properties": {"IsDefault": did in in_default and not d.get("Tags"),
403
+ "SharedWithCustomVpc": did in in_custom, "Tags": d.get("Tags") or []}})
404
+ return out
405
+
406
+
407
+ def _route_tables(s, r):
408
+ ec2 = s.client("ec2", region_name=r)
409
+ return [{"Identifier": t["RouteTableId"],
410
+ "Properties": {"Main": any(a.get("Main") for a in t.get("Associations") or []), "VpcId": t.get("VpcId"),
411
+ "Tags": t.get("Tags") or []}}
412
+ for t in _paginate(ec2, "describe_route_tables", "RouteTables")]
413
+
414
+
415
+ def _network_acls(s, r):
416
+ ec2 = s.client("ec2", region_name=r)
417
+ return [{"Identifier": a["NetworkAclId"], "Properties": {"IsDefault": a.get("IsDefault"), "Tags": a.get("Tags") or []}}
418
+ for a in _paginate(ec2, "describe_network_acls", "NetworkAcls")]
419
+
420
+
421
+ def _instances(s, r):
422
+ ec2 = s.client("ec2", region_name=r)
423
+ out = []
424
+ for res in _paginate(ec2, "describe_instances", "Reservations",
425
+ Filters=[{"Name": "instance-state-name", "Values": ["pending", "running", "stopping", "stopped"]}]):
426
+ for i in res.get("Instances") or []:
427
+ out.append({"Identifier": i["InstanceId"], "Properties": {"Tags": i.get("Tags") or []}})
428
+ return out
429
+
430
+
431
+ def _volumes(s, r):
432
+ ec2 = s.client("ec2", region_name=r)
433
+ out = []
434
+ for v in _paginate(ec2, "describe_volumes", "Volumes"):
435
+ # A volume launched with an instance (root or block-device mapping) is part of that instance.
436
+ with_instance = any(a.get("DeleteOnTermination") for a in v.get("Attachments") or [])
437
+ out.append({"Identifier": v["VolumeId"], "Properties": {"LaunchedWithInstance": with_instance, "Tags": v.get("Tags") or []}})
438
+ return out
439
+
440
+
441
+ def _vpc_endpoints(s, r):
442
+ ec2 = s.client("ec2", region_name=r)
443
+ return [{"Identifier": e["VpcEndpointId"], "Properties": {"RequesterManaged": e.get("RequesterManaged"),
444
+ "VpcEndpointType": e.get("VpcEndpointType"), "Tags": e.get("Tags") or []}}
445
+ for e in _paginate(ec2, "describe_vpc_endpoints", "VpcEndpoints")]
446
+
447
+
448
+ def _buckets(s, r):
449
+ s3 = s.client("s3", region_name=r)
450
+ out = []
451
+ for b in s3.list_buckets().get("Buckets") or []:
452
+ region = b.get("BucketRegion")
453
+ if not region:
454
+ try:
455
+ region = s3.get_bucket_location(Bucket=b["Name"]).get("LocationConstraint") or "us-east-1"
456
+ except Exception:
457
+ region = ""
458
+ out.append({"Identifier": b["Name"], "Properties": {"BucketName": b["Name"], "_region": region}})
459
+ return out
460
+
461
+
462
+ def _iam_roles(s, r):
463
+ iam = s.client("iam", region_name=r)
464
+ return [{"Identifier": x["RoleName"], "Properties": {"RoleName": x["RoleName"], "Path": x.get("Path"), "Arn": x.get("Arn")}}
465
+ for x in _paginate(iam, "list_roles", "Roles")]
466
+
467
+
468
+ def _iam_policies(s, r):
469
+ iam = s.client("iam", region_name=r) # Scope=Local skips ~1,600 AWS managed policies
470
+ return [{"Identifier": x["Arn"], "Properties": {"PolicyName": x.get("PolicyName"), "Arn": x["Arn"]}}
471
+ for x in _paginate(iam, "list_policies", "Policies", Scope="Local")]
472
+
473
+
474
+ def _instance_profiles(s, r):
475
+ iam = s.client("iam", region_name=r)
476
+ return [{"Identifier": x["InstanceProfileName"], "Properties": {"Arn": x.get("Arn"), "Path": x.get("Path")}}
477
+ for x in _paginate(iam, "list_instance_profiles", "InstanceProfiles")]
478
+
479
+
480
+ def _kms_keys(s, r):
481
+ kms = s.client("kms", region_name=r)
482
+ out = []
483
+ for k in _paginate(kms, "list_keys", "Keys"):
484
+ try:
485
+ meta = kms.describe_key(KeyId=k["KeyId"])["KeyMetadata"]
486
+ except Exception:
487
+ continue
488
+ if meta.get("KeyState") in ("PendingDeletion", "PendingReplicaDeletion"):
489
+ continue
490
+ out.append({"Identifier": k["KeyId"], "Properties": {"KeyManager": meta.get("KeyManager"), "Arn": meta.get("Arn")}})
491
+ return out
492
+
493
+
494
+ def _ecs_services(s, r):
495
+ ecs = s.client("ecs", region_name=r)
496
+ out = []
497
+ for cluster in _paginate(ecs, "list_clusters", "clusterArns"):
498
+ for arn in _paginate(ecs, "list_services", "serviceArns", cluster=cluster):
499
+ out.append({"Identifier": arn, "Properties": {"Arn": arn, "_aliases": [[cluster.rsplit("/", 1)[-1], arn.rsplit("/", 1)[-1]]]}})
500
+ return out
501
+
502
+
503
+ def _eks_nodegroups(s, r):
504
+ eks = s.client("eks", region_name=r)
505
+ out = []
506
+ for cluster in _paginate(eks, "list_clusters", "clusters"):
507
+ for ng in _paginate(eks, "list_nodegroups", "nodegroups", clusterName=cluster):
508
+ out.append({"Identifier": f"{cluster}:{ng}", "Properties": {"Name": ng, "_aliases": [[cluster, ng]]}})
509
+ return out
510
+
511
+
512
+ def _rds_clusters(s, r):
513
+ rds = s.client("rds", region_name=r)
514
+ return [{"Identifier": c["DBClusterIdentifier"], "Properties": {"Engine": c.get("Engine"), "Arn": c.get("DBClusterArn")}}
515
+ for c in _paginate(rds, "describe_db_clusters", "DBClusters")]
516
+
517
+
518
+ def _event_rules(s, r):
519
+ ev = s.client("events", region_name=r)
520
+ out = []
521
+ for bus in _paginate(ev, "list_event_buses", "EventBuses"):
522
+ for rule in _paginate(ev, "list_rules", "Rules", EventBusName=bus["Name"]):
523
+ ident = rule["Name"] if bus["Name"] == "default" else f"{bus['Name']}/{rule['Name']}"
524
+ out.append({"Identifier": ident, "Properties": {"Name": rule["Name"], "Arn": rule.get("Arn"),
525
+ "ManagedBy": rule.get("ManagedBy")}})
526
+ return out
527
+
528
+
529
+ def _elasticache_groups(s, r):
530
+ ec = s.client("elasticache", region_name=r)
531
+ return [{"Identifier": g["ReplicationGroupId"], "Properties": {"Arn": g.get("ARN")}}
532
+ for g in _paginate(ec, "describe_replication_groups", "ReplicationGroups")]
533
+
534
+
535
+ def _elasticache_clusters(s, r):
536
+ ec = s.client("elasticache", region_name=r) # members of a replication group belong to the group
537
+ return [{"Identifier": c["CacheClusterId"], "Properties": {"Arn": c.get("ARN")}}
538
+ for c in _paginate(ec, "describe_cache_clusters", "CacheClusters") if not c.get("ReplicationGroupId")]
539
+
540
+
541
+ def _classic_elbs(s, r):
542
+ elb = s.client("elb", region_name=r)
543
+ return [{"Identifier": x["LoadBalancerName"], "Properties": {}}
544
+ for x in _paginate(elb, "describe_load_balancers", "LoadBalancerDescriptions")]
545
+
546
+
547
+ def _cloudfront_policies(method, list_key, item_key):
548
+ # Type=custom leaves out CloudFront's managed policies, which every account sees.
549
+ def _list(s, r):
550
+ cf = s.client("cloudfront", region_name="us-east-1")
551
+ out, marker = [], None
552
+ while True:
553
+ page = getattr(cf, method)(Type="custom", **({"Marker": marker} if marker else {})).get(list_key) or {}
554
+ for item in page.get("Items") or []:
555
+ pol = item.get(item_key) or {}
556
+ conf = next((v for k, v in pol.items() if k.endswith("Config")), {}) or {}
557
+ out.append({"Identifier": pol["Id"], "Properties": {"Name": conf.get("Name")}})
558
+ marker = page.get("NextMarker")
559
+ if not marker:
560
+ return out
561
+ return _list
562
+
563
+
564
+ def _lb_listeners(s, r):
565
+ elb = s.client("elbv2", region_name=r)
566
+ out = []
567
+ for lb in _paginate(elb, "describe_load_balancers", "LoadBalancers"):
568
+ for ln in _paginate(elb, "describe_listeners", "Listeners", LoadBalancerArn=lb["LoadBalancerArn"]):
569
+ out.append({"Identifier": ln["ListenerArn"], "Properties": {"Name": f"{lb.get('LoadBalancerName')}:{ln.get('Port')}"}})
570
+ return out
571
+
572
+
573
+ def _eks_children(kind):
574
+ method, key, arg = {"addon": ("list_addons", "addons", "addonName"),
575
+ "fargate": ("list_fargate_profiles", "fargateProfileNames", "fargateProfileName")}[kind]
576
+
577
+ def _list(s, r):
578
+ eks = s.client("eks", region_name=r)
579
+ out = []
580
+ for cluster in _paginate(eks, "list_clusters", "clusters"):
581
+ for name in _paginate(eks, method, key, clusterName=cluster):
582
+ out.append({"Identifier": f"{cluster}:{name}", "Properties": {"Name": name, "_aliases": [[cluster, name]]}})
583
+ return out
584
+ return _list
585
+
586
+
587
+ def _opensearch_domains(s, r):
588
+ client = s.client("opensearch", region_name=r)
589
+ return [{"Identifier": d["DomainName"], "Properties": {"DomainName": d["DomainName"]}}
590
+ for d in client.list_domain_names().get("DomainNames") or []]
591
+
592
+
593
+ def _appconfig_children(kind):
594
+ method, id_key = {"environment": ("list_environments", "EnvironmentId"),
595
+ "profile": ("list_configuration_profiles", "ConfigurationProfileId")}[kind]
596
+
597
+ def _list(s, r):
598
+ ac = s.client("appconfig", region_name=r)
599
+ out = []
600
+ for app in _paginate(ac, "list_applications", "Items"):
601
+ for item in _paginate(ac, method, "Items", ApplicationId=app["Id"]):
602
+ # Terraform's id is "<child id>:<application id>"
603
+ out.append({"Identifier": f"{app['Id']}|{item['Id']}", "Properties": {"Name": item.get("Name")}})
604
+ return out
605
+ return _list
606
+
607
+
608
+ def _ssm_documents(s, r):
609
+ ssm = s.client("ssm", region_name=r)
610
+ return [{"Identifier": d["Name"], "Properties": {"Name": d["Name"]}}
611
+ for d in _paginate(ssm, "list_documents", "DocumentIdentifiers", Filters=[{"Key": "Owner", "Values": ["Self"]}])]
612
+
613
+
614
+ NATIVE_LISTERS: dict[str, Callable[[Any, str], list[dict[str, Any]]]] = {
615
+ "AWS::EC2::VPC": _vpcs,
616
+ "AWS::EC2::Subnet": _subnets,
617
+ "AWS::EC2::SecurityGroup": _security_groups,
618
+ "AWS::EC2::InternetGateway": _internet_gateways,
619
+ "AWS::EC2::DHCPOptions": _dhcp_options,
620
+ "AWS::EC2::RouteTable": _route_tables,
621
+ "AWS::EC2::NetworkAcl": _network_acls,
622
+ "AWS::EC2::Instance": _instances,
623
+ "AWS::EC2::Volume": _volumes,
624
+ "AWS::EC2::VPCEndpoint": _vpc_endpoints,
625
+ "AWS::S3::Bucket": _buckets,
626
+ "AWS::IAM::Role": _iam_roles,
627
+ "AWS::IAM::ManagedPolicy": _iam_policies,
628
+ "AWS::IAM::InstanceProfile": _instance_profiles,
629
+ "AWS::KMS::Key": _kms_keys,
630
+ "AWS::ECS::Service": _ecs_services,
631
+ "AWS::EKS::Nodegroup": _eks_nodegroups,
632
+ "AWS::RDS::DBCluster": _rds_clusters,
633
+ "AWS::Events::Rule": _event_rules,
634
+ "AWS::ElastiCache::ReplicationGroup": _elasticache_groups,
635
+ "AWS::ElastiCache::CacheCluster": _elasticache_clusters,
636
+ "AWS::ElasticLoadBalancing::LoadBalancer": _classic_elbs,
637
+ "AWS::SSM::Document": _ssm_documents,
638
+ "AWS::CloudFront::CachePolicy": _cloudfront_policies("list_cache_policies", "CachePolicyList", "CachePolicy"),
639
+ "AWS::CloudFront::OriginRequestPolicy": _cloudfront_policies(
640
+ "list_origin_request_policies", "OriginRequestPolicyList", "OriginRequestPolicy"),
641
+ "AWS::CloudFront::ResponseHeadersPolicy": _cloudfront_policies(
642
+ "list_response_headers_policies", "ResponseHeadersPolicyList", "ResponseHeadersPolicy"),
643
+ "AWS::ElasticLoadBalancingV2::Listener": _lb_listeners,
644
+ "AWS::EKS::Addon": _eks_children("addon"),
645
+ "AWS::EKS::FargateProfile": _eks_children("fargate"),
646
+ "AWS::OpenSearchService::Domain": _opensearch_domains,
647
+ "AWS::AppConfig::Environment": _appconfig_children("environment"),
648
+ "AWS::AppConfig::ConfigurationProfile": _appconfig_children("profile"),
649
+ }
650
+
651
+ # Cloud Control types that need a ResourceModel to be listed.
652
+ _MODELS = {
653
+ "AWS::WAFv2::WebACL": {"Scope": "REGIONAL"},
654
+ "AWS::WAFv2::IPSet": {"Scope": "REGIONAL"},
655
+ "AWS::WAFv2::RegexPatternSet": {"Scope": "REGIONAL"},
656
+ "AWS::WAFv2::RuleGroup": {"Scope": "REGIONAL"},
657
+ }
658
+
659
+ # RDS's cluster API also returns DocumentDB and Neptune clusters.
660
+ _CLUSTER_ENGINE_TYPES = {"docdb": "aws_docdb_cluster", "neptune": "aws_neptune_cluster"}
661
+
662
+
663
+ # --- resources AWS owns ----------------------------------------------------
664
+ # Copied from OrchesTerra's rules. Each uses evidence a customer cannot reproduce.
665
+
666
+ _CLOUDFRONT_MANAGED_IDS = frozenset({
667
+ "658327ea-f89d-4fab-a63d-7e88639e58f6", "4135ea2d-6df8-44a3-9df3-4b5a84be39ad", "b2884449-e4de-46a7-ac36-70bc7f1ddd6d",
668
+ "08627262-05a9-4f76-9ded-b50ca2e3a84f", "83da9c7e-98b4-4e11-a168-04f0df8e2c65", "216adef6-5c7f-47e4-b989-5492eafa07d3",
669
+ "88a5eaf4-2fd4-4709-b370-b4c650ea3fcf", "acba4595-bd28-49b8-b9fe-13317c0390fa", "59781a5b-3903-41f3-afcb-af62929ccde1",
670
+ "775133bc-15f2-49f9-abea-afb2e0bf67d2", "b689b0a8-53d0-40ab-baf2-68738e2966ac", "33f36d7e-f396-46d9-90e0-52428a34d9dc",
671
+ "5cc3b908-e619-4b99-88e5-2cf7f45965bd", "60669652-455b-4ae9-85a4-c4c02393f86c", "67f7725c-6f97-4210-82d7-5512b31e9d03",
672
+ "e61eb60c-9c35-4d20-a928-2b84e02af89c", "eaab4381-ed33-4a86-88ca-d9558dc6cd63",
673
+ })
674
+ _EKS_CLUSTER_SG = re.compile(r"^eks-cluster-sg-.+-\d+$")
675
+ _CLOUDHSM_SG = re.compile(r"^cloudhsm-cluster-[a-z0-9-]+-sg$")
676
+ _DIRECTORY_SG = re.compile(r"^d-[0-9a-f]{10}_(controllers|workspacesMembers)$")
677
+ _EMR_SG_NAMES = {"ElasticMapReduce-master", "ElasticMapReduce-slave", "ElasticMapReduce-ServiceAccess"}
678
+
679
+
680
+ def aws_owned_reason(res: LiveResource) -> str:
681
+ """Why AWS owns this resource (so it is never "not in IaC"), or "" when it is the customer's."""
682
+ p, t, n, ident = res.props, res.tf_type, res.name, res.identifier
683
+ tags = _tag_map(p.get("Tags"))
684
+ arn = res.arn or (ident if ident.startswith("arn:") else "")
685
+ if t in ("aws_vpc", "aws_network_acl", "aws_internet_gateway") and p.get("IsDefault"):
686
+ return "default VPC resource"
687
+ if t == "aws_subnet" and p.get("DefaultForAz"):
688
+ return "default subnet"
689
+ if t == "aws_route_table" and p.get("Main"):
690
+ return "VPC main route table"
691
+ if t == "aws_vpc_dhcp_options" and p.get("IsDefault"):
692
+ return "default DHCP options"
693
+ if t == "aws_security_group":
694
+ g, d = str(p.get("GroupName") or n), str(p.get("Description") or "")
695
+ if g == "default":
696
+ return "VPC default security group"
697
+ if tags.get("aws:eks:cluster-name") or (_EKS_CLUSTER_SG.match(g) and d.startswith("EKS created")):
698
+ return "created by Amazon EKS"
699
+ if _CLOUDHSM_SG.match(g):
700
+ return "created by AWS CloudHSM"
701
+ if _DIRECTORY_SG.match(g) and d.startswith("AWS created"):
702
+ return "created by AWS Directory Service"
703
+ if g in _EMR_SG_NAMES:
704
+ return "created by Amazon EMR"
705
+ if t == "aws_instance" and (tags.get("aws:autoscaling:groupName") or tags.get("eks:nodegroup-name")):
706
+ return "launched by an Auto Scaling group"
707
+ if t == "aws_ebs_volume" and p.get("LaunchedWithInstance"):
708
+ return "launched with an EC2 instance"
709
+ if t == "aws_vpc_endpoint" and p.get("RequesterManaged"):
710
+ return "requester-managed endpoint (created by an AWS service)"
711
+ if t == "aws_iam_role" and str(p.get("Path") or "").startswith(("/aws-service-role/", "/aws-reserved/")):
712
+ return "service-linked or AWS SSO role"
713
+ if t == "aws_iam_policy" and ":aws:policy/" in arn:
714
+ return "AWS managed policy"
715
+ if t == "aws_kms_key" and p.get("KeyManager") == "AWS":
716
+ return "AWS managed key"
717
+ if t == "aws_kms_alias" and (n.startswith("alias/aws/") or ident.startswith("alias/aws/")):
718
+ return "AWS managed key alias"
719
+ if t == "aws_cloudwatch_event_bus" and (n == "default" or ident.endswith("/default") or ident == "default"):
720
+ return "default event bus"
721
+ if t == "aws_cloudwatch_event_rule" and p.get("ManagedBy"):
722
+ return f"managed by {p['ManagedBy']}"
723
+ if t == "aws_cloudwatch_metric_alarm" and n.startswith(("TargetTracking-", "AutoScaling-")):
724
+ return "created by Application Auto Scaling"
725
+ if t == "aws_secretsmanager_secret" and ("!" in n or "!" in ident or tags.get("aws:secretsmanager:owningService")):
726
+ return "created by an AWS service (managed secret)"
727
+ if t == "aws_ssm_document" and n.lower().startswith(("aws", "amazon", "amzn")) and "/" not in n:
728
+ return "Amazon-owned SSM document"
729
+ if t == "aws_ssm_patch_baseline" and (n.startswith("AWS-") or ident.startswith("arn:")):
730
+ return "AWS patch baseline"
731
+ if t == "aws_ec2_managed_prefix_list" and (n.startswith("com.amazonaws.") or ":aws:" in arn):
732
+ return "AWS managed prefix list"
733
+ if t.startswith("aws_cloudfront_") and t.endswith("_policy"):
734
+ if ident in _CLOUDFRONT_MANAGED_IDS or n.startswith("Managed-") or n.lower().startswith("useorigincachecontrolheaders"):
735
+ return "CloudFront managed policy"
736
+ if t in ("aws_xray_group", "aws_xray_sampling_rule") and (n == "Default" or ident.endswith("/Default")):
737
+ return "X-Ray default"
738
+ if t in ("aws_db_option_group", "aws_db_parameter_group", "aws_rds_cluster_parameter_group") and \
739
+ (n.lower().startswith(("default.", "default:")) or ident.lower().startswith(("default.", "default:"))):
740
+ return "RDS default group"
741
+ if t == "aws_db_subnet_group" and n.lower() == "default":
742
+ return "RDS default subnet group"
743
+ if t == "aws_athena_data_catalog" and ident == "AwsDataCatalog":
744
+ return "Athena default catalog"
745
+ if t == "aws_athena_workgroup" and ident == "primary":
746
+ return "Athena primary workgroup"
747
+ if t == "aws_scheduler_schedule_group" and ident == "default":
748
+ return "default schedule group"
749
+ if t == "aws_backup_vault" and (ident == "Default" or ident.startswith("aws/")):
750
+ return "AWS Backup default vault"
751
+ if t == "aws_route53_resolver_rule" and "rslvr-autodefined" in (ident + arn):
752
+ return "Route 53 Resolver default rule"
753
+ if t == "aws_launch_template" and (tags.get("eks:nodegroup-name") or n.startswith("eks-")):
754
+ return "created by Amazon EKS"
755
+ return ""
756
+
757
+
758
+ # --- listing ---------------------------------------------------------------
759
+
760
+ @dataclass
761
+ class Inventory:
762
+ resources: list[LiveResource] = field(default_factory=list)
763
+ owned: list[tuple[LiveResource, str]] = field(default_factory=list)
764
+ not_listed: dict[str, str] = field(default_factory=dict) # "type (region)" -> reason
765
+
766
+
767
+ def _list_one(session: Any, region: str, cc_type: str) -> list[dict[str, Any]]:
768
+ lister = NATIVE_LISTERS.get(cc_type)
769
+ if lister is not None:
770
+ try:
771
+ return lister(session, region)
772
+ except Exception as exc:
773
+ text = f"{type(exc).__name__} {exc}"
774
+ if any(marker in text for marker in _SKIP_ERRORS):
775
+ raise Skipped(text) from exc
776
+ raise
777
+ client = session.client("cloudcontrol", region_name=region)
778
+ return list_cloudcontrol(client, cc_type, _MODELS.get(cc_type))
779
+
780
+
781
+ def _to_live(cc_type: str, region: str, row: dict[str, Any]) -> LiveResource | None:
782
+ ident = str(row.get("Identifier") or "").strip()
783
+ if not ident:
784
+ return None
785
+ props = _props(row.get("Properties"))
786
+ tf = terraform_type(cc_type)
787
+ if cc_type == "AWS::RDS::DBCluster":
788
+ tf = _CLUSTER_ENGINE_TYPES.get(str(props.get("Engine") or "").lower(), tf)
789
+ if tf == "aws_eip" and "|" in ident: # Cloud Control keys an EIP as "<ip>|<allocation id>"
790
+ ident = next((p for p in ident.split("|") if p.startswith("eipalloc-")), ident)
791
+ arn = str(props.get("Arn") or props.get("ARN") or "") or (ident if ident.startswith("arn:") else "")
792
+ res_region = props.pop("_region", None) or ("" if cc_type.startswith(("AWS::IAM::", "AWS::CloudFront::", "AWS::Route53::", "AWS::Budgets::")) else region)
793
+ return LiveResource(cc_type, tf, ident, res_region, _name(ident, props), arn, props)
794
+
795
+
796
+ class _RetrySession:
797
+ """Session whose clients retry throttled calls with backoff (Cloud Control throttles parallel listing)."""
798
+
799
+ def __init__(self, session: Any):
800
+ self._session = session
801
+ self.region_name = getattr(session, "region_name", None)
802
+ try:
803
+ from botocore.config import Config
804
+ self._config = Config(retries={"mode": "adaptive", "max_attempts": 12})
805
+ except ImportError: # fake sessions in tests
806
+ self._config = None
807
+
808
+ def client(self, service: str, region_name: str | None = None):
809
+ if self._config is None:
810
+ return self._session.client(service, region_name=region_name)
811
+ return self._session.client(service, region_name=region_name, config=self._config)
812
+
813
+
814
+ def list_inventory(session: Any, regions: list[str], types: Iterable[str] | None = None,
815
+ workers: int = 4, log: Callable[..., None] = print) -> Inventory:
816
+ """List every chosen type in every region (global types once)."""
817
+ session = _RetrySession(session)
818
+ wanted = [t for t in (types or CLOUDCONTROL_TO_TERRAFORM)]
819
+ jobs: list[tuple[str, str]] = []
820
+ for i, region in enumerate(regions):
821
+ for cc_type in wanted:
822
+ if is_global(cc_type) and i > 0:
823
+ continue
824
+ jobs.append((region, cc_type))
825
+ inv = Inventory()
826
+ seen: set[tuple[str, str, str]] = set()
827
+ started = time.monotonic()
828
+
829
+ def run(job):
830
+ region, cc_type = job
831
+ try:
832
+ return job, _list_one(session, region, cc_type), ""
833
+ except Skipped as exc:
834
+ return job, [], _short_reason(str(exc))
835
+ except Exception as exc: # keep going; one bad type shouldn't sink the scan
836
+ return job, [], _short_reason(f"{type(exc).__name__}: {exc}")
837
+
838
+ log(f"listing {len(jobs)} resource types across {len(regions)} region(s) …", file=sys.stderr)
839
+ with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
840
+ for (region, cc_type), rows, reason in pool.map(run, jobs):
841
+ if reason:
842
+ inv.not_listed[f"{cc_type} ({'global' if is_global(cc_type) else region})"] = reason
843
+ continue
844
+ for row in rows:
845
+ res = _to_live(cc_type, region, row)
846
+ if res is None or (res.tf_type, res.identifier, res.region) in seen:
847
+ continue
848
+ seen.add((res.tf_type, res.identifier, res.region))
849
+ if res.region and res.region not in regions and cc_type == "AWS::S3::Bucket":
850
+ continue # a bucket in a region the scan did not cover
851
+ why = aws_owned_reason(res)
852
+ if why:
853
+ inv.owned.append((res, why))
854
+ else:
855
+ inv.resources.append(res)
856
+ log(f"listed {len(inv.resources)} resources ({len(inv.owned)} AWS-owned skipped) in "
857
+ f"{time.monotonic() - started:.0f}s", file=sys.stderr)
858
+ return inv
859
+
860
+
861
+ def _short_reason(text: str) -> str:
862
+ for marker, label in (("Throttling", "throttled by AWS; rerun with fewer --workers"),
863
+ ("Rate exceeded", "throttled by AWS; rerun with fewer --workers"),
864
+ ("ResourceModel", "needs a parent resource to list; not checked"),
865
+ ("Handler returned", "the service returned an error"),
866
+ ("AccessDenied", "access denied"), ("not authorized", "access denied"),
867
+ ("UnauthorizedOperation", "access denied"), ("TypeNotFound", "type not available"),
868
+ ("UnsupportedAction", "listing not supported"), ("is not supported", "listing not supported"),
869
+ ("OptInRequired", "region or service not enabled"), ("Could not connect", "service not in region"),
870
+ ("SubscriptionRequired", "service not enabled")):
871
+ if marker in text:
872
+ return label
873
+ return text.splitlines()[0][:160]
874
+
875
+
876
+ # --- what IaC knows about --------------------------------------------------
877
+
878
+ _STATE_KEYS = ("id", "arn", "name", "identifier", "bucket", "url", "key_id", "function_name")
879
+
880
+
881
+ @dataclass
882
+ class IacEntry:
883
+ tf_type: str # Terraform type (normalized), or "" for a CloudFormation resource
884
+ cc_type: str # CloudFormation type, or "" for Terraform
885
+ tokens: frozenset[str]
886
+ region: str
887
+ where: str # state file or stack
888
+
889
+
890
+ def _state_tokens(attrs: dict[str, Any]) -> tuple[frozenset[str], str]:
891
+ tokens: set[str] = set()
892
+ region = ""
893
+ for key, value in attrs.items():
894
+ if not isinstance(value, str) or not value:
895
+ continue
896
+ if key in _STATE_KEYS or (key.endswith(("_name", "_identifier", "_arn"))
897
+ and key not in ("display_name", "role_arn", "kms_key_arn", "execution_role_arn")):
898
+ tokens.add(value)
899
+ if value.startswith("arn:"):
900
+ region = region or _arn_region(value)
901
+ tokens.add(re.split(r"[:/]", value)[-1])
902
+ ident = attrs.get("id")
903
+ if isinstance(ident, str):
904
+ for sep in ("|", ":", ",", "/"):
905
+ if sep in ident and not ident.startswith(("arn:", "https://")):
906
+ tokens.update(p for p in ident.split(sep) if p)
907
+ return frozenset(tokens), region
908
+
909
+
910
+ def parse_state(doc: dict[str, Any], where: str) -> list[IacEntry]:
911
+ """Managed resources in a Terraform state file (format version 4)."""
912
+ out = []
913
+ for res in doc.get("resources") or []:
914
+ if res.get("mode") != "managed":
915
+ continue
916
+ tf_type = TERRAFORM_TYPE_ALIASES.get(res.get("type", ""), res.get("type", ""))
917
+ for inst in res.get("instances") or []:
918
+ tokens, region = _state_tokens(inst.get("attributes") or {})
919
+ if tokens:
920
+ out.append(IacEntry(tf_type, "", tokens, region, where))
921
+ return out
922
+
923
+
924
+ def read_state_source(source: str, session: Any = None, binary: str | None = None,
925
+ runner=subprocess.run) -> list[tuple[str, dict[str, Any]]]:
926
+ """(label, state doc) for a local file, s3://bucket/key, s3://bucket/prefix/, or a Terraform directory."""
927
+ if source.startswith("s3://"):
928
+ bucket, _, key = source[5:].partition("/")
929
+ s3 = session.client("s3")
930
+ keys = [key] if key.endswith(".tfstate") else [
931
+ o["Key"] for o in _paginate(s3, "list_objects_v2", "Contents", Bucket=bucket, Prefix=key)
932
+ if o["Key"].endswith(".tfstate")]
933
+ docs = []
934
+ for k in keys:
935
+ body = s3.get_object(Bucket=bucket, Key=k)["Body"].read()
936
+ docs.append((f"s3://{bucket}/{k}", json.loads(body)))
937
+ return docs
938
+ if os.path.isdir(source):
939
+ tf = binary or next((b for b in ("tofu", "terraform") if _which(b)), None)
940
+ if not tf:
941
+ raise RuntimeError("Neither `tofu` nor `terraform` was found on PATH to read state from a directory.")
942
+ proc = runner([tf, f"-chdir={source}", "state", "pull"], capture_output=True, text=True)
943
+ if proc.returncode != 0:
944
+ raise RuntimeError(f"`{tf} state pull` in {source} failed:\n{proc.stderr or proc.stdout}")
945
+ return [(source, json.loads(proc.stdout or "{}"))]
946
+ with open(source, encoding="utf-8") as fh:
947
+ return [(source, json.load(fh))]
948
+
949
+
950
+ def _which(name: str) -> bool:
951
+ import shutil
952
+ return shutil.which(name) is not None
953
+
954
+
955
+ def stack_entries(session: Any, region: str, log: Callable[..., None] = print) -> list[IacEntry]:
956
+ """Every resource in every CloudFormation stack in a region."""
957
+ cfn = session.client("cloudformation", region_name=region)
958
+ out = []
959
+ statuses = ["CREATE_COMPLETE", "UPDATE_COMPLETE", "UPDATE_ROLLBACK_COMPLETE", "IMPORT_COMPLETE",
960
+ "IMPORT_ROLLBACK_COMPLETE", "ROLLBACK_COMPLETE", "UPDATE_ROLLBACK_FAILED",
961
+ "CREATE_IN_PROGRESS", "UPDATE_IN_PROGRESS", "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS"]
962
+ try:
963
+ stacks = [s["StackName"] for s in _paginate(cfn, "list_stacks", "StackSummaries", StackStatusFilter=statuses)]
964
+ except Exception as exc:
965
+ log(f"warning: could not list CloudFormation stacks in {region}: {_short_reason(str(exc))}", file=sys.stderr)
966
+ return out
967
+ for name in stacks:
968
+ try:
969
+ rows = _paginate(cfn, "list_stack_resources", "StackResourceSummaries", StackName=name)
970
+ except Exception as exc:
971
+ log(f"warning: could not read stack {name}: {_short_reason(str(exc))}", file=sys.stderr)
972
+ continue
973
+ for row in rows:
974
+ phys = str(row.get("PhysicalResourceId") or "")
975
+ if not phys:
976
+ continue
977
+ tokens = {phys}
978
+ if phys.startswith("arn:"):
979
+ tokens.add(re.split(r"[:/]", phys)[-1])
980
+ elif phys.startswith("https://"):
981
+ tokens.add(phys.rstrip("/").rsplit("/", 1)[-1])
982
+ if "|" in phys:
983
+ tokens.update(p for p in phys.split("|") if p)
984
+ out.append(IacEntry("", row.get("ResourceType", ""), frozenset(tokens), region, f"stack {name} ({region})"))
985
+ return out
986
+
987
+
988
+ # --- matching --------------------------------------------------------------
989
+
990
+ @dataclass
991
+ class Match:
992
+ unmanaged: list[LiveResource]
993
+ managed: list[tuple[LiveResource, str]]
994
+
995
+
996
+ def match(inventory: Inventory, iac: list[IacEntry]) -> Match:
997
+ by_tf: dict[str, list[IacEntry]] = {}
998
+ by_cc: dict[str, list[IacEntry]] = {}
999
+ for e in iac:
1000
+ if e.tf_type:
1001
+ by_tf.setdefault(e.tf_type, []).append(e)
1002
+ if e.cc_type:
1003
+ by_cc.setdefault(e.cc_type, []).append(e)
1004
+ unmanaged, managed = [], []
1005
+ for res in inventory.resources:
1006
+ tf = TERRAFORM_TYPE_ALIASES.get(res.tf_type, res.tf_type)
1007
+ candidates = by_tf.get(tf, []) + by_cc.get(res.cc_type, [])
1008
+ hit = _find(res, candidates)
1009
+ if hit:
1010
+ managed.append((res, hit.where))
1011
+ else:
1012
+ unmanaged.append(res)
1013
+ return Match(unmanaged, managed)
1014
+
1015
+
1016
+ def _find(res: LiveResource, candidates: list[IacEntry]) -> IacEntry | None:
1017
+ keys = res.keys()
1018
+ for entry in candidates:
1019
+ if entry.region and res.region and entry.region != res.region:
1020
+ continue
1021
+ if any(key <= entry.tokens for key in keys):
1022
+ return entry
1023
+ return None
1024
+
1025
+
1026
+ def enabled_regions(session: Any) -> list[str]:
1027
+ ec2 = session.client("ec2", region_name=session.region_name or "us-east-1")
1028
+ rows = ec2.describe_regions(Filters=[{"Name": "opt-in-status", "Values": ["opt-in-not-required", "opted-in"]}])
1029
+ return sorted(r["RegionName"] for r in rows.get("Regions") or [])