cartography 0.102.0rc1__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

Files changed (251) hide show
  1. cartography/__main__.py +1 -2
  2. cartography/_version.py +2 -2
  3. cartography/cli.py +302 -253
  4. cartography/client/core/tx.py +39 -18
  5. cartography/config.py +4 -0
  6. cartography/driftdetect/__main__.py +1 -2
  7. cartography/driftdetect/add_shortcut.py +10 -2
  8. cartography/driftdetect/cli.py +71 -75
  9. cartography/driftdetect/detect_deviations.py +7 -3
  10. cartography/driftdetect/get_states.py +20 -8
  11. cartography/driftdetect/model.py +5 -5
  12. cartography/driftdetect/serializers.py +8 -6
  13. cartography/driftdetect/storage.py +2 -2
  14. cartography/graph/cleanupbuilder.py +35 -15
  15. cartography/graph/job.py +46 -17
  16. cartography/graph/querybuilder.py +165 -80
  17. cartography/graph/statement.py +35 -26
  18. cartography/intel/analysis.py +4 -1
  19. cartography/intel/aws/__init__.py +114 -55
  20. cartography/intel/aws/apigateway.py +134 -63
  21. cartography/intel/aws/cloudtrail.py +127 -0
  22. cartography/intel/aws/config.py +56 -20
  23. cartography/intel/aws/dynamodb.py +108 -40
  24. cartography/intel/aws/ec2/__init__.py +2 -2
  25. cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
  26. cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
  27. cartography/intel/aws/ec2/images.py +49 -20
  28. cartography/intel/aws/ec2/instances.py +234 -136
  29. cartography/intel/aws/ec2/internet_gateways.py +40 -11
  30. cartography/intel/aws/ec2/key_pairs.py +44 -20
  31. cartography/intel/aws/ec2/launch_templates.py +101 -59
  32. cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
  33. cartography/intel/aws/ec2/load_balancers.py +82 -42
  34. cartography/intel/aws/ec2/network_acls.py +89 -65
  35. cartography/intel/aws/ec2/network_interfaces.py +146 -87
  36. cartography/intel/aws/ec2/reserved_instances.py +45 -16
  37. cartography/intel/aws/ec2/route_tables.py +327 -0
  38. cartography/intel/aws/ec2/security_groups.py +71 -21
  39. cartography/intel/aws/ec2/snapshots.py +61 -22
  40. cartography/intel/aws/ec2/subnets.py +54 -18
  41. cartography/intel/aws/ec2/tgw.py +100 -34
  42. cartography/intel/aws/ec2/util.py +1 -1
  43. cartography/intel/aws/ec2/volumes.py +69 -41
  44. cartography/intel/aws/ec2/vpc.py +37 -12
  45. cartography/intel/aws/ec2/vpc_peerings.py +83 -24
  46. cartography/intel/aws/ecr.py +88 -32
  47. cartography/intel/aws/ecs.py +83 -47
  48. cartography/intel/aws/eks.py +55 -29
  49. cartography/intel/aws/elasticache.py +42 -18
  50. cartography/intel/aws/elasticsearch.py +57 -20
  51. cartography/intel/aws/emr.py +61 -23
  52. cartography/intel/aws/iam.py +401 -145
  53. cartography/intel/aws/iam_instance_profiles.py +22 -22
  54. cartography/intel/aws/identitycenter.py +71 -37
  55. cartography/intel/aws/inspector.py +159 -89
  56. cartography/intel/aws/kms.py +92 -38
  57. cartography/intel/aws/lambda_function.py +103 -34
  58. cartography/intel/aws/organizations.py +30 -10
  59. cartography/intel/aws/permission_relationships.py +133 -51
  60. cartography/intel/aws/rds.py +249 -85
  61. cartography/intel/aws/redshift.py +107 -46
  62. cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
  63. cartography/intel/aws/resources.py +53 -44
  64. cartography/intel/aws/route53.py +108 -61
  65. cartography/intel/aws/s3.py +168 -83
  66. cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
  67. cartography/intel/aws/secretsmanager.py +24 -12
  68. cartography/intel/aws/securityhub.py +20 -9
  69. cartography/intel/aws/sns.py +166 -0
  70. cartography/intel/aws/sqs.py +60 -28
  71. cartography/intel/aws/ssm.py +70 -30
  72. cartography/intel/aws/util/arns.py +7 -7
  73. cartography/intel/aws/util/common.py +31 -4
  74. cartography/intel/azure/__init__.py +78 -19
  75. cartography/intel/azure/compute.py +101 -27
  76. cartography/intel/azure/cosmosdb.py +496 -170
  77. cartography/intel/azure/sql.py +296 -105
  78. cartography/intel/azure/storage.py +322 -113
  79. cartography/intel/azure/subscription.py +39 -23
  80. cartography/intel/azure/tenant.py +13 -4
  81. cartography/intel/azure/util/credentials.py +95 -55
  82. cartography/intel/bigfix/__init__.py +2 -2
  83. cartography/intel/bigfix/computers.py +93 -65
  84. cartography/intel/create_indexes.py +3 -2
  85. cartography/intel/crowdstrike/__init__.py +11 -9
  86. cartography/intel/crowdstrike/endpoints.py +5 -1
  87. cartography/intel/crowdstrike/spotlight.py +8 -3
  88. cartography/intel/cve/__init__.py +46 -13
  89. cartography/intel/cve/feed.py +48 -12
  90. cartography/intel/digitalocean/__init__.py +22 -13
  91. cartography/intel/digitalocean/compute.py +75 -108
  92. cartography/intel/digitalocean/management.py +44 -80
  93. cartography/intel/digitalocean/platform.py +48 -43
  94. cartography/intel/dns.py +36 -10
  95. cartography/intel/duo/__init__.py +21 -16
  96. cartography/intel/duo/api_host.py +14 -9
  97. cartography/intel/duo/endpoints.py +50 -45
  98. cartography/intel/duo/groups.py +18 -14
  99. cartography/intel/duo/phones.py +37 -34
  100. cartography/intel/duo/tokens.py +26 -23
  101. cartography/intel/duo/users.py +54 -50
  102. cartography/intel/duo/web_authn_credentials.py +30 -25
  103. cartography/intel/entra/__init__.py +25 -7
  104. cartography/intel/entra/ou.py +112 -0
  105. cartography/intel/entra/users.py +69 -63
  106. cartography/intel/gcp/__init__.py +185 -49
  107. cartography/intel/gcp/compute.py +418 -231
  108. cartography/intel/gcp/crm.py +96 -43
  109. cartography/intel/gcp/dns.py +60 -19
  110. cartography/intel/gcp/gke.py +72 -38
  111. cartography/intel/gcp/iam.py +61 -41
  112. cartography/intel/gcp/storage.py +84 -55
  113. cartography/intel/github/__init__.py +13 -11
  114. cartography/intel/github/repos.py +270 -137
  115. cartography/intel/github/teams.py +170 -88
  116. cartography/intel/github/users.py +70 -39
  117. cartography/intel/github/util.py +36 -34
  118. cartography/intel/gsuite/__init__.py +47 -26
  119. cartography/intel/gsuite/api.py +73 -30
  120. cartography/intel/jamf/__init__.py +19 -1
  121. cartography/intel/jamf/computers.py +30 -7
  122. cartography/intel/jamf/util.py +7 -2
  123. cartography/intel/kandji/__init__.py +6 -3
  124. cartography/intel/kandji/devices.py +14 -8
  125. cartography/intel/kubernetes/namespaces.py +7 -4
  126. cartography/intel/kubernetes/pods.py +7 -4
  127. cartography/intel/kubernetes/services.py +8 -4
  128. cartography/intel/lastpass/__init__.py +2 -2
  129. cartography/intel/lastpass/users.py +23 -12
  130. cartography/intel/oci/__init__.py +44 -11
  131. cartography/intel/oci/iam.py +134 -38
  132. cartography/intel/oci/organizations.py +13 -6
  133. cartography/intel/oci/utils.py +43 -20
  134. cartography/intel/okta/__init__.py +66 -15
  135. cartography/intel/okta/applications.py +42 -20
  136. cartography/intel/okta/awssaml.py +93 -33
  137. cartography/intel/okta/factors.py +16 -4
  138. cartography/intel/okta/groups.py +56 -29
  139. cartography/intel/okta/organization.py +5 -1
  140. cartography/intel/okta/origins.py +6 -2
  141. cartography/intel/okta/roles.py +15 -5
  142. cartography/intel/okta/users.py +20 -8
  143. cartography/intel/okta/utils.py +6 -4
  144. cartography/intel/pagerduty/__init__.py +8 -7
  145. cartography/intel/pagerduty/escalation_policies.py +18 -6
  146. cartography/intel/pagerduty/schedules.py +12 -4
  147. cartography/intel/pagerduty/services.py +11 -4
  148. cartography/intel/pagerduty/teams.py +8 -3
  149. cartography/intel/pagerduty/users.py +3 -1
  150. cartography/intel/pagerduty/vendors.py +3 -1
  151. cartography/intel/semgrep/__init__.py +24 -6
  152. cartography/intel/semgrep/dependencies.py +50 -28
  153. cartography/intel/semgrep/deployment.py +3 -1
  154. cartography/intel/semgrep/findings.py +42 -18
  155. cartography/intel/snipeit/__init__.py +17 -3
  156. cartography/intel/snipeit/asset.py +12 -6
  157. cartography/intel/snipeit/user.py +8 -5
  158. cartography/intel/snipeit/util.py +9 -4
  159. cartography/models/aws/apigateway.py +21 -17
  160. cartography/models/aws/apigatewaycertificate.py +28 -22
  161. cartography/models/aws/apigatewayresource.py +28 -20
  162. cartography/models/aws/apigatewaystage.py +33 -25
  163. cartography/models/aws/cloudtrail/__init__.py +0 -0
  164. cartography/models/aws/cloudtrail/trail.py +61 -0
  165. cartography/models/aws/dynamodb/gsi.py +30 -22
  166. cartography/models/aws/dynamodb/tables.py +25 -17
  167. cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
  168. cartography/models/aws/ec2/images.py +36 -34
  169. cartography/models/aws/ec2/instances.py +51 -45
  170. cartography/models/aws/ec2/keypair.py +21 -16
  171. cartography/models/aws/ec2/keypair_instance.py +28 -21
  172. cartography/models/aws/ec2/launch_configurations.py +30 -26
  173. cartography/models/aws/ec2/launch_template_versions.py +48 -38
  174. cartography/models/aws/ec2/launch_templates.py +21 -17
  175. cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
  176. cartography/models/aws/ec2/load_balancers.py +47 -37
  177. cartography/models/aws/ec2/network_acl_rules.py +38 -30
  178. cartography/models/aws/ec2/network_acls.py +38 -29
  179. cartography/models/aws/ec2/networkinterface_instance.py +52 -39
  180. cartography/models/aws/ec2/networkinterfaces.py +53 -37
  181. cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
  182. cartography/models/aws/ec2/reservations.py +18 -14
  183. cartography/models/aws/ec2/route_table_associations.py +97 -0
  184. cartography/models/aws/ec2/route_tables.py +128 -0
  185. cartography/models/aws/ec2/routes.py +85 -0
  186. cartography/models/aws/ec2/securitygroup_instance.py +29 -20
  187. cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
  188. cartography/models/aws/ec2/subnet_instance.py +24 -19
  189. cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
  190. cartography/models/aws/ec2/volumes.py +47 -40
  191. cartography/models/aws/eks/clusters.py +23 -21
  192. cartography/models/aws/emr.py +32 -30
  193. cartography/models/aws/iam/instanceprofile.py +33 -24
  194. cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
  195. cartography/models/aws/identitycenter/awspermissionset.py +37 -29
  196. cartography/models/aws/identitycenter/awsssouser.py +23 -21
  197. cartography/models/aws/inspector/findings.py +77 -65
  198. cartography/models/aws/inspector/packages.py +35 -29
  199. cartography/models/aws/s3/__init__.py +0 -0
  200. cartography/models/aws/s3/account_public_access_block.py +51 -0
  201. cartography/models/aws/sns/__init__.py +0 -0
  202. cartography/models/aws/sns/topic.py +50 -0
  203. cartography/models/aws/ssm/instance_information.py +51 -39
  204. cartography/models/aws/ssm/instance_patch.py +32 -26
  205. cartography/models/bigfix/bigfix_computer.py +42 -38
  206. cartography/models/bigfix/bigfix_root.py +3 -3
  207. cartography/models/core/common.py +12 -10
  208. cartography/models/core/nodes.py +5 -2
  209. cartography/models/core/relationships.py +14 -6
  210. cartography/models/crowdstrike/hosts.py +37 -35
  211. cartography/models/cve/cve.py +34 -32
  212. cartography/models/cve/cve_feed.py +6 -6
  213. cartography/models/digitalocean/__init__.py +0 -0
  214. cartography/models/digitalocean/account.py +21 -0
  215. cartography/models/digitalocean/droplet.py +56 -0
  216. cartography/models/digitalocean/project.py +48 -0
  217. cartography/models/duo/api_host.py +3 -3
  218. cartography/models/duo/endpoint.py +43 -41
  219. cartography/models/duo/group.py +14 -14
  220. cartography/models/duo/phone.py +27 -27
  221. cartography/models/duo/token.py +16 -16
  222. cartography/models/duo/user.py +46 -44
  223. cartography/models/duo/web_authn_credential.py +27 -19
  224. cartography/models/entra/ou.py +48 -0
  225. cartography/models/entra/tenant.py +24 -18
  226. cartography/models/entra/user.py +64 -48
  227. cartography/models/gcp/iam.py +23 -23
  228. cartography/models/github/orgs.py +5 -4
  229. cartography/models/github/teams.py +37 -31
  230. cartography/models/github/users.py +34 -23
  231. cartography/models/kandji/device.py +22 -16
  232. cartography/models/kandji/tenant.py +6 -4
  233. cartography/models/lastpass/tenant.py +3 -3
  234. cartography/models/lastpass/user.py +32 -28
  235. cartography/models/semgrep/dependencies.py +36 -24
  236. cartography/models/semgrep/deployment.py +5 -5
  237. cartography/models/semgrep/findings.py +58 -42
  238. cartography/models/semgrep/locations.py +27 -21
  239. cartography/models/snipeit/asset.py +30 -21
  240. cartography/models/snipeit/tenant.py +6 -4
  241. cartography/models/snipeit/user.py +19 -12
  242. cartography/stats.py +3 -3
  243. cartography/sync.py +107 -31
  244. cartography/util.py +84 -62
  245. {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/METADATA +3 -14
  246. cartography-0.103.0rc1.dist-info/RECORD +396 -0
  247. {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/WHEEL +1 -1
  248. cartography-0.102.0rc1.dist-info/RECORD +0 -377
  249. {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/entry_points.txt +0 -0
  250. {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/licenses/LICENSE +0 -0
  251. {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/top_level.txt +0 -0
@@ -22,12 +22,19 @@ stat_handler = get_stats_client(__name__)
22
22
 
23
23
  @timeit
24
24
  @aws_handle_regions
25
- def get_dynamodb_tables(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
26
- client = boto3_session.client('dynamodb', region_name=region, config=get_botocore_config())
27
- paginator = client.get_paginator('list_tables')
25
+ def get_dynamodb_tables(
26
+ boto3_session: boto3.session.Session,
27
+ region: str,
28
+ ) -> List[Dict]:
29
+ client = boto3_session.client(
30
+ "dynamodb",
31
+ region_name=region,
32
+ config=get_botocore_config(),
33
+ )
34
+ paginator = client.get_paginator("list_tables")
28
35
  dynamodb_tables = []
29
36
  for page in paginator.paginate():
30
- for table_name in page['TableNames']:
37
+ for table_name in page["TableNames"]:
31
38
  dynamodb_tables.append(client.describe_table(TableName=table_name))
32
39
  return dynamodb_tables
33
40
 
@@ -38,33 +45,50 @@ def transform_dynamodb_tables(dynamodb_tables: List, region: str) -> Any:
38
45
  ddb_gsi_data: List[Dict[str, Any]] = []
39
46
 
40
47
  for table in dynamodb_tables:
41
- ddb_table_data.append({
42
- 'Arn': table['Table']['TableArn'],
43
- 'TableName': table['Table']['TableName'],
44
- 'Region': region,
45
- 'Rows': table['Table']['ItemCount'],
46
- 'Size': table['Table']['TableSizeBytes'],
47
- 'ProvisionedThroughputReadCapacityUnits': table['Table']['ProvisionedThroughput']['ReadCapacityUnits'],
48
- 'ProvisionedThroughputWriteCapacityUnits': table['Table']['ProvisionedThroughput']['WriteCapacityUnits'],
49
- })
50
- for gsi in table['Table'].get('GlobalSecondaryIndexes', []):
51
- ddb_gsi_data.append({
52
- 'Arn': gsi['IndexArn'],
53
- 'TableArn': table['Table']['TableArn'],
54
- 'Region': region,
55
- 'ProvisionedThroughputReadCapacityUnits': gsi['ProvisionedThroughput']['ReadCapacityUnits'],
56
- 'ProvisionedThroughputWriteCapacityUnits': gsi['ProvisionedThroughput']['WriteCapacityUnits'],
57
- 'GSIName': gsi['IndexName'],
58
- })
48
+ ddb_table_data.append(
49
+ {
50
+ "Arn": table["Table"]["TableArn"],
51
+ "TableName": table["Table"]["TableName"],
52
+ "Region": region,
53
+ "Rows": table["Table"]["ItemCount"],
54
+ "Size": table["Table"]["TableSizeBytes"],
55
+ "ProvisionedThroughputReadCapacityUnits": table["Table"][
56
+ "ProvisionedThroughput"
57
+ ]["ReadCapacityUnits"],
58
+ "ProvisionedThroughputWriteCapacityUnits": table["Table"][
59
+ "ProvisionedThroughput"
60
+ ]["WriteCapacityUnits"],
61
+ },
62
+ )
63
+ for gsi in table["Table"].get("GlobalSecondaryIndexes", []):
64
+ ddb_gsi_data.append(
65
+ {
66
+ "Arn": gsi["IndexArn"],
67
+ "TableArn": table["Table"]["TableArn"],
68
+ "Region": region,
69
+ "ProvisionedThroughputReadCapacityUnits": gsi[
70
+ "ProvisionedThroughput"
71
+ ]["ReadCapacityUnits"],
72
+ "ProvisionedThroughputWriteCapacityUnits": gsi[
73
+ "ProvisionedThroughput"
74
+ ]["WriteCapacityUnits"],
75
+ "GSIName": gsi["IndexName"],
76
+ },
77
+ )
59
78
  return ddb_table_data, ddb_gsi_data
60
79
 
61
80
 
62
81
  @timeit
63
82
  def load_dynamodb_tables(
64
- neo4j_session: neo4j.Session, tables_data: List[Dict[str, Any]], region: str, current_aws_account_id: str,
83
+ neo4j_session: neo4j.Session,
84
+ tables_data: List[Dict[str, Any]],
85
+ region: str,
86
+ current_aws_account_id: str,
65
87
  aws_update_tag: int,
66
88
  ) -> None:
67
- logger.info(f"Loading Dynamo DB tables {len(tables_data)} for region '{region}' into graph.")
89
+ logger.info(
90
+ f"Loading Dynamo DB tables {len(tables_data)} for region '{region}' into graph.",
91
+ )
68
92
  load(
69
93
  neo4j_session,
70
94
  DynamoDBTableSchema(),
@@ -77,10 +101,15 @@ def load_dynamodb_tables(
77
101
 
78
102
  @timeit
79
103
  def load_dynamodb_gsi(
80
- neo4j_session: neo4j.Session, gsi_data: List[Dict[str, Any]], region: str, current_aws_account_id: str,
104
+ neo4j_session: neo4j.Session,
105
+ gsi_data: List[Dict[str, Any]],
106
+ region: str,
107
+ current_aws_account_id: str,
81
108
  aws_update_tag: int,
82
109
  ) -> None:
83
- logger.info(f"Loading Dynamo DB GSI {len(gsi_data)} for region '{region}' into graph.")
110
+ logger.info(
111
+ f"Loading Dynamo DB GSI {len(gsi_data)} for region '{region}' into graph.",
112
+ )
84
113
  load(
85
114
  neo4j_session,
86
115
  DynamoDBGSISchema(),
@@ -92,38 +121,77 @@ def load_dynamodb_gsi(
92
121
 
93
122
 
94
123
  @timeit
95
- def cleanup_dynamodb_tables(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
96
- GraphJob.from_node_schema(DynamoDBTableSchema(), common_job_parameters).run(neo4j_session)
97
- GraphJob.from_node_schema(DynamoDBGSISchema(), common_job_parameters).run(neo4j_session)
124
+ def cleanup_dynamodb_tables(
125
+ neo4j_session: neo4j.Session,
126
+ common_job_parameters: Dict,
127
+ ) -> None:
128
+ GraphJob.from_node_schema(DynamoDBTableSchema(), common_job_parameters).run(
129
+ neo4j_session,
130
+ )
131
+ GraphJob.from_node_schema(DynamoDBGSISchema(), common_job_parameters).run(
132
+ neo4j_session,
133
+ )
98
134
 
99
135
 
100
136
  @timeit
101
137
  def sync_dynamodb_tables(
102
- neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
103
- aws_update_tag: int, common_job_parameters: Dict,
138
+ neo4j_session: neo4j.Session,
139
+ boto3_session: boto3.session.Session,
140
+ regions: List[str],
141
+ current_aws_account_id: str,
142
+ aws_update_tag: int,
143
+ common_job_parameters: Dict,
104
144
  ) -> None:
105
145
  for region in regions:
106
- logger.info("Syncing DynamoDB for region in '%s' in account '%s'.", region, current_aws_account_id)
146
+ logger.info(
147
+ "Syncing DynamoDB for region in '%s' in account '%s'.",
148
+ region,
149
+ current_aws_account_id,
150
+ )
107
151
  dynamodb_tables = get_dynamodb_tables(boto3_session, region)
108
- ddb_table_data, ddb_gsi_data = transform_dynamodb_tables(dynamodb_tables, region)
109
- load_dynamodb_tables(neo4j_session, ddb_table_data, region, current_aws_account_id, aws_update_tag)
110
- load_dynamodb_gsi(neo4j_session, ddb_gsi_data, region, current_aws_account_id, aws_update_tag)
152
+ ddb_table_data, ddb_gsi_data = transform_dynamodb_tables(
153
+ dynamodb_tables,
154
+ region,
155
+ )
156
+ load_dynamodb_tables(
157
+ neo4j_session,
158
+ ddb_table_data,
159
+ region,
160
+ current_aws_account_id,
161
+ aws_update_tag,
162
+ )
163
+ load_dynamodb_gsi(
164
+ neo4j_session,
165
+ ddb_gsi_data,
166
+ region,
167
+ current_aws_account_id,
168
+ aws_update_tag,
169
+ )
111
170
  cleanup_dynamodb_tables(neo4j_session, common_job_parameters)
112
171
 
113
172
 
114
173
  @timeit
115
174
  def sync(
116
- neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
117
- update_tag: int, common_job_parameters: Dict,
175
+ neo4j_session: neo4j.Session,
176
+ boto3_session: boto3.session.Session,
177
+ regions: List[str],
178
+ current_aws_account_id: str,
179
+ update_tag: int,
180
+ common_job_parameters: Dict,
118
181
  ) -> None:
119
182
  sync_dynamodb_tables(
120
- neo4j_session, boto3_session, regions, current_aws_account_id, update_tag, common_job_parameters,
183
+ neo4j_session,
184
+ boto3_session,
185
+ regions,
186
+ current_aws_account_id,
187
+ update_tag,
188
+ common_job_parameters,
121
189
  )
122
190
  merge_module_sync_metadata(
123
191
  neo4j_session,
124
- group_type='AWSAccount',
192
+ group_type="AWSAccount",
125
193
  group_id=current_aws_account_id,
126
- synced_type='DynamoDBTable',
194
+ synced_type="DynamoDBTable",
127
195
  update_tag=update_tag,
128
196
  stat_handler=stat_handler,
129
197
  )
@@ -10,6 +10,6 @@ logger = logging.getLogger(__name__)
10
10
 
11
11
  @timeit
12
12
  def get_ec2_regions(boto3_session: boto3.session.Session) -> List[str]:
13
- client = boto3_session.client('ec2')
13
+ client = boto3_session.client("ec2")
14
14
  result = client.describe_regions()
15
- return [r['RegionName'] for r in result['Regions']]
15
+ return [r["RegionName"] for r in result["Regions"]]
@@ -5,20 +5,26 @@ from typing import Any
5
5
  import boto3
6
6
  import neo4j
7
7
 
8
- from .util import get_botocore_config
9
8
  from cartography.client.core.tx import load
10
9
  from cartography.graph.job import GraphJob
11
10
  from cartography.models.aws.ec2.auto_scaling_groups import AutoScalingGroupSchema
12
- from cartography.models.aws.ec2.auto_scaling_groups import EC2InstanceAutoScalingGroupSchema
13
- from cartography.models.aws.ec2.auto_scaling_groups import EC2SubnetAutoScalingGroupSchema
11
+ from cartography.models.aws.ec2.auto_scaling_groups import (
12
+ EC2InstanceAutoScalingGroupSchema,
13
+ )
14
+ from cartography.models.aws.ec2.auto_scaling_groups import (
15
+ EC2SubnetAutoScalingGroupSchema,
16
+ )
14
17
  from cartography.models.aws.ec2.launch_configurations import LaunchConfigurationSchema
15
18
  from cartography.util import aws_handle_regions
16
19
  from cartography.util import timeit
17
20
 
21
+ from .util import get_botocore_config
22
+
18
23
  logger = logging.getLogger(__name__)
19
24
 
20
25
  AsgData = namedtuple(
21
- 'AsgData', [
26
+ "AsgData",
27
+ [
22
28
  "group_list",
23
29
  "instance_list",
24
30
  "subnet_list",
@@ -28,46 +34,66 @@ AsgData = namedtuple(
28
34
 
29
35
  @timeit
30
36
  @aws_handle_regions
31
- def get_ec2_auto_scaling_groups(boto3_session: boto3.session.Session, region: str) -> list[dict]:
32
- client = boto3_session.client('autoscaling', region_name=region, config=get_botocore_config())
33
- paginator = client.get_paginator('describe_auto_scaling_groups')
37
+ def get_ec2_auto_scaling_groups(
38
+ boto3_session: boto3.session.Session,
39
+ region: str,
40
+ ) -> list[dict]:
41
+ client = boto3_session.client(
42
+ "autoscaling",
43
+ region_name=region,
44
+ config=get_botocore_config(),
45
+ )
46
+ paginator = client.get_paginator("describe_auto_scaling_groups")
34
47
  asgs: list[dict] = []
35
48
  for page in paginator.paginate():
36
- asgs.extend(page['AutoScalingGroups'])
49
+ asgs.extend(page["AutoScalingGroups"])
37
50
  return asgs
38
51
 
39
52
 
40
53
  @timeit
41
54
  @aws_handle_regions
42
- def get_launch_configurations(boto3_session: boto3.session.Session, region: str) -> list[dict]:
43
- client = boto3_session.client('autoscaling', region_name=region, config=get_botocore_config())
44
- paginator = client.get_paginator('describe_launch_configurations')
55
+ def get_launch_configurations(
56
+ boto3_session: boto3.session.Session,
57
+ region: str,
58
+ ) -> list[dict]:
59
+ client = boto3_session.client(
60
+ "autoscaling",
61
+ region_name=region,
62
+ config=get_botocore_config(),
63
+ )
64
+ paginator = client.get_paginator("describe_launch_configurations")
45
65
  lcs: list[dict] = []
46
66
  for page in paginator.paginate():
47
- lcs.extend(page['LaunchConfigurations'])
67
+ lcs.extend(page["LaunchConfigurations"])
48
68
  return lcs
49
69
 
50
70
 
51
- def transform_launch_configurations(configurations: list[dict[str, Any]]) -> list[dict[str, Any]]:
71
+ def transform_launch_configurations(
72
+ configurations: list[dict[str, Any]],
73
+ ) -> list[dict[str, Any]]:
52
74
  transformed_configurations = []
53
75
  for config in configurations:
54
- transformed_configurations.append({
55
- 'AssociatePublicIpAddress': config.get('AssociatePublicIpAddress'),
56
- 'LaunchConfigurationARN': config.get('LaunchConfigurationARN'),
57
- 'LaunchConfigurationName': config.get('LaunchConfigurationName'),
58
- 'CreatedTime': config.get('CreatedTime'),
59
- 'ImageId': config.get('ImageId'),
60
- 'KeyName': config.get('KeyName'),
61
- 'SecurityGroups': config.get('SecurityGroups'),
62
- 'InstanceType': config.get('InstanceType'),
63
- 'KernelId': config.get('KernelId'),
64
- 'RamdiskId': config.get('RamdiskId'),
65
- 'InstanceMonitoring': config.get('InstanceMonitoring', {}).get('Enabled'),
66
- 'SpotPrice': config.get('SpotPrice'),
67
- 'IamInstanceProfile': config.get('IamInstanceProfile'),
68
- 'EbsOptimized': config.get('EbsOptimized'),
69
- 'PlacementTenancy': config.get('PlacementTenancy'),
70
- })
76
+ transformed_configurations.append(
77
+ {
78
+ "AssociatePublicIpAddress": config.get("AssociatePublicIpAddress"),
79
+ "LaunchConfigurationARN": config.get("LaunchConfigurationARN"),
80
+ "LaunchConfigurationName": config.get("LaunchConfigurationName"),
81
+ "CreatedTime": config.get("CreatedTime"),
82
+ "ImageId": config.get("ImageId"),
83
+ "KeyName": config.get("KeyName"),
84
+ "SecurityGroups": config.get("SecurityGroups"),
85
+ "InstanceType": config.get("InstanceType"),
86
+ "KernelId": config.get("KernelId"),
87
+ "RamdiskId": config.get("RamdiskId"),
88
+ "InstanceMonitoring": config.get("InstanceMonitoring", {}).get(
89
+ "Enabled",
90
+ ),
91
+ "SpotPrice": config.get("SpotPrice"),
92
+ "IamInstanceProfile": config.get("IamInstanceProfile"),
93
+ "EbsOptimized": config.get("EbsOptimized"),
94
+ "PlacementTenancy": config.get("PlacementTenancy"),
95
+ },
96
+ )
71
97
  return transformed_configurations
72
98
 
73
99
 
@@ -76,42 +102,54 @@ def transform_auto_scaling_groups(groups: list[dict[str, Any]]) -> AsgData:
76
102
  related_vpcs = []
77
103
  related_instances = []
78
104
  for group in groups:
79
- transformed_groups.append({
80
- 'AutoScalingGroupARN': group['AutoScalingGroupARN'],
81
- 'CapacityRebalance': group.get('CapacityRebalance'),
82
- 'CreatedTime': str(group.get('CreatedTime')),
83
- 'DefaultCooldown': group.get('DefaultCooldown'),
84
- 'DesiredCapacity': group.get('DesiredCapacity'),
85
- 'HealthCheckGracePeriod': group.get('HealthCheckGracePeriod'),
86
- 'HealthCheckType': group.get('HealthCheckType'),
87
- 'LaunchConfigurationName': group.get('LaunchConfigurationName'),
88
- 'LaunchTemplateName': group.get('LaunchTemplate', {}).get('LaunchTemplateName'),
89
- 'LaunchTemplateId': group.get('LaunchTemplate', {}).get('LaunchTemplateId'),
90
- 'LaunchTemplateVersion': group.get('LaunchTemplate', {}).get('Version'),
91
- 'MaxInstanceLifetime': group.get('MaxInstanceLifetime'),
92
- 'MaxSize': group.get('MaxSize'),
93
- 'MinSize': group.get('MinSize'),
94
- 'AutoScalingGroupName': group.get('AutoScalingGroupName'),
95
- 'NewInstancesProtectedFromScaleIn': group.get('NewInstancesProtectedFromScaleIn'),
96
- 'Status': group.get('Status'),
97
- })
98
-
99
- if group.get('VPCZoneIdentifier', None):
100
- vpclist = group['VPCZoneIdentifier']
101
- subnet_ids = vpclist.split(',') if ',' in vpclist else [vpclist]
105
+ transformed_groups.append(
106
+ {
107
+ "AutoScalingGroupARN": group["AutoScalingGroupARN"],
108
+ "CapacityRebalance": group.get("CapacityRebalance"),
109
+ "CreatedTime": str(group.get("CreatedTime")),
110
+ "DefaultCooldown": group.get("DefaultCooldown"),
111
+ "DesiredCapacity": group.get("DesiredCapacity"),
112
+ "HealthCheckGracePeriod": group.get("HealthCheckGracePeriod"),
113
+ "HealthCheckType": group.get("HealthCheckType"),
114
+ "LaunchConfigurationName": group.get("LaunchConfigurationName"),
115
+ "LaunchTemplateName": group.get("LaunchTemplate", {}).get(
116
+ "LaunchTemplateName",
117
+ ),
118
+ "LaunchTemplateId": group.get("LaunchTemplate", {}).get(
119
+ "LaunchTemplateId",
120
+ ),
121
+ "LaunchTemplateVersion": group.get("LaunchTemplate", {}).get("Version"),
122
+ "MaxInstanceLifetime": group.get("MaxInstanceLifetime"),
123
+ "MaxSize": group.get("MaxSize"),
124
+ "MinSize": group.get("MinSize"),
125
+ "AutoScalingGroupName": group.get("AutoScalingGroupName"),
126
+ "NewInstancesProtectedFromScaleIn": group.get(
127
+ "NewInstancesProtectedFromScaleIn",
128
+ ),
129
+ "Status": group.get("Status"),
130
+ },
131
+ )
132
+
133
+ if group.get("VPCZoneIdentifier", None):
134
+ vpclist = group["VPCZoneIdentifier"]
135
+ subnet_ids = vpclist.split(",") if "," in vpclist else [vpclist]
102
136
  subnets = []
103
137
  for subnet_id in subnet_ids:
104
- subnets.append({
105
- 'VPCZoneIdentifier': subnet_id,
106
- 'AutoScalingGroupARN': group['AutoScalingGroupARN'],
107
- })
138
+ subnets.append(
139
+ {
140
+ "VPCZoneIdentifier": subnet_id,
141
+ "AutoScalingGroupARN": group["AutoScalingGroupARN"],
142
+ },
143
+ )
108
144
  related_vpcs.extend(subnets)
109
145
 
110
- for instance_data in group.get('Instances', []):
111
- related_instances.append({
112
- 'InstanceId': instance_data['InstanceId'],
113
- 'AutoScalingGroupARN': group['AutoScalingGroupARN'],
114
- })
146
+ for instance_data in group.get("Instances", []):
147
+ related_instances.append(
148
+ {
149
+ "InstanceId": instance_data["InstanceId"],
150
+ "AutoScalingGroupARN": group["AutoScalingGroupARN"],
151
+ },
152
+ )
115
153
 
116
154
  return AsgData(
117
155
  group_list=transformed_groups,
@@ -122,7 +160,11 @@ def transform_auto_scaling_groups(groups: list[dict[str, Any]]) -> AsgData:
122
160
 
123
161
  @timeit
124
162
  def load_launch_configurations(
125
- neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
163
+ neo4j_session: neo4j.Session,
164
+ data: list[dict],
165
+ region: str,
166
+ current_aws_account_id: str,
167
+ update_tag: int,
126
168
  ) -> None:
127
169
  load(
128
170
  neo4j_session,
@@ -135,7 +177,11 @@ def load_launch_configurations(
135
177
 
136
178
 
137
179
  def load_groups(
138
- neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
180
+ neo4j_session: neo4j.Session,
181
+ data: list[dict],
182
+ region: str,
183
+ current_aws_account_id: str,
184
+ update_tag: int,
139
185
  ) -> None:
140
186
  load(
141
187
  neo4j_session,
@@ -148,7 +194,11 @@ def load_groups(
148
194
 
149
195
 
150
196
  def load_asg_subnets(
151
- neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
197
+ neo4j_session: neo4j.Session,
198
+ data: list[dict],
199
+ region: str,
200
+ current_aws_account_id: str,
201
+ update_tag: int,
152
202
  ) -> None:
153
203
  load(
154
204
  neo4j_session,
@@ -161,7 +211,11 @@ def load_asg_subnets(
161
211
 
162
212
 
163
213
  def load_asg_instances(
164
- neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int,
214
+ neo4j_session: neo4j.Session,
215
+ data: list[dict],
216
+ region: str,
217
+ current_aws_account_id: str,
218
+ update_tag: int,
165
219
  ) -> None:
166
220
  load(
167
221
  neo4j_session,
@@ -175,31 +229,80 @@ def load_asg_instances(
175
229
 
176
230
  @timeit
177
231
  def load_auto_scaling_groups(
178
- neo4j_session: neo4j.Session, data: AsgData, region: str, current_aws_account_id: str, update_tag: int,
232
+ neo4j_session: neo4j.Session,
233
+ data: AsgData,
234
+ region: str,
235
+ current_aws_account_id: str,
236
+ update_tag: int,
179
237
  ) -> None:
180
- load_groups(neo4j_session, data.group_list, region, current_aws_account_id, update_tag)
181
- load_asg_instances(neo4j_session, data.instance_list, region, current_aws_account_id, update_tag)
182
- load_asg_subnets(neo4j_session, data.subnet_list, region, current_aws_account_id, update_tag)
238
+ load_groups(
239
+ neo4j_session,
240
+ data.group_list,
241
+ region,
242
+ current_aws_account_id,
243
+ update_tag,
244
+ )
245
+ load_asg_instances(
246
+ neo4j_session,
247
+ data.instance_list,
248
+ region,
249
+ current_aws_account_id,
250
+ update_tag,
251
+ )
252
+ load_asg_subnets(
253
+ neo4j_session,
254
+ data.subnet_list,
255
+ region,
256
+ current_aws_account_id,
257
+ update_tag,
258
+ )
183
259
 
184
260
 
185
261
  @timeit
186
- def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
262
+ def cleanup(
263
+ neo4j_session: neo4j.Session,
264
+ common_job_parameters: dict[str, Any],
265
+ ) -> None:
187
266
  logger.debug("Running EC2 instance cleanup")
188
- GraphJob.from_node_schema(AutoScalingGroupSchema(), common_job_parameters).run(neo4j_session)
189
- GraphJob.from_node_schema(LaunchConfigurationSchema(), common_job_parameters).run(neo4j_session)
267
+ GraphJob.from_node_schema(AutoScalingGroupSchema(), common_job_parameters).run(
268
+ neo4j_session,
269
+ )
270
+ GraphJob.from_node_schema(LaunchConfigurationSchema(), common_job_parameters).run(
271
+ neo4j_session,
272
+ )
190
273
 
191
274
 
192
275
  @timeit
193
276
  def sync_ec2_auto_scaling_groups(
194
- neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: list[str],
195
- current_aws_account_id: str, update_tag: int, common_job_parameters: dict,
277
+ neo4j_session: neo4j.Session,
278
+ boto3_session: boto3.session.Session,
279
+ regions: list[str],
280
+ current_aws_account_id: str,
281
+ update_tag: int,
282
+ common_job_parameters: dict,
196
283
  ) -> None:
197
284
  for region in regions:
198
- logger.debug("Syncing auto scaling groups for region '%s' in account '%s'.", region, current_aws_account_id)
285
+ logger.debug(
286
+ "Syncing auto scaling groups for region '%s' in account '%s'.",
287
+ region,
288
+ current_aws_account_id,
289
+ )
199
290
  lc_data = get_launch_configurations(boto3_session, region)
200
291
  asg_data = get_ec2_auto_scaling_groups(boto3_session, region)
201
292
  lc_transformed = transform_launch_configurations(lc_data)
202
293
  asg_transformed = transform_auto_scaling_groups(asg_data)
203
- load_launch_configurations(neo4j_session, lc_transformed, region, current_aws_account_id, update_tag)
204
- load_auto_scaling_groups(neo4j_session, asg_transformed, region, current_aws_account_id, update_tag)
294
+ load_launch_configurations(
295
+ neo4j_session,
296
+ lc_transformed,
297
+ region,
298
+ current_aws_account_id,
299
+ update_tag,
300
+ )
301
+ load_auto_scaling_groups(
302
+ neo4j_session,
303
+ asg_transformed,
304
+ region,
305
+ current_aws_account_id,
306
+ update_tag,
307
+ )
205
308
  cleanup(neo4j_session, common_job_parameters)
@@ -6,20 +6,28 @@ import boto3
6
6
  import neo4j
7
7
  from botocore.exceptions import ClientError
8
8
 
9
- from .util import get_botocore_config
10
9
  from cartography.util import aws_handle_regions
11
10
  from cartography.util import run_cleanup_job
12
11
  from cartography.util import timeit
13
12
 
13
+ from .util import get_botocore_config
14
+
14
15
  logger = logging.getLogger(__name__)
15
16
 
16
17
 
17
18
  @timeit
18
19
  @aws_handle_regions
19
- def get_elastic_ip_addresses(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
20
- client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
20
+ def get_elastic_ip_addresses(
21
+ boto3_session: boto3.session.Session,
22
+ region: str,
23
+ ) -> List[Dict]:
24
+ client = boto3_session.client(
25
+ "ec2",
26
+ region_name=region,
27
+ config=get_botocore_config(),
28
+ )
21
29
  try:
22
- addresses = client.describe_addresses()['Addresses']
30
+ addresses = client.describe_addresses()["Addresses"]
23
31
  except ClientError as e:
24
32
  logger.warning(f"Failed retrieve address for region - {region}. Error - {e}")
25
33
  raise
@@ -28,8 +36,11 @@ def get_elastic_ip_addresses(boto3_session: boto3.session.Session, region: str)
28
36
 
29
37
  @timeit
30
38
  def load_elastic_ip_addresses(
31
- neo4j_session: neo4j.Session, elastic_ip_addresses: List[Dict], region: str,
32
- current_aws_account_id: str, update_tag: int,
39
+ neo4j_session: neo4j.Session,
40
+ elastic_ip_addresses: List[Dict],
41
+ region: str,
42
+ current_aws_account_id: str,
43
+ update_tag: int,
33
44
  ) -> None:
34
45
  """
35
46
  Creates (:ElasticIpAddress)
@@ -37,7 +48,9 @@ def load_elastic_ip_addresses(
37
48
  (:EC2Instance)-[:ELASTIC_IP_ADDRESS]->(:ElasticIpAddress),
38
49
  (:NetworkInterface)-[:ELASTIC_IP_ADDRESS]->(:ElasticIpAddress),
39
50
  """
40
- logger.info(f"Loading {len(elastic_ip_addresses)} Elastic IP Addresses in {region}.")
51
+ logger.info(
52
+ f"Loading {len(elastic_ip_addresses)} Elastic IP Addresses in {region}.",
53
+ )
41
54
  ingest_addresses = """
42
55
  UNWIND $elastic_ip_addresses as eia
43
56
  MERGE (address: ElasticIPAddress{id: eia.PublicIp})
@@ -80,9 +93,12 @@ def load_elastic_ip_addresses(
80
93
 
81
94
 
82
95
  @timeit
83
- def cleanup_elastic_ip_addresses(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
96
+ def cleanup_elastic_ip_addresses(
97
+ neo4j_session: neo4j.Session,
98
+ common_job_parameters: Dict,
99
+ ) -> None:
84
100
  run_cleanup_job(
85
- 'aws_import_elastic_ip_addresses_cleanup.json',
101
+ "aws_import_elastic_ip_addresses_cleanup.json",
86
102
  neo4j_session,
87
103
  common_job_parameters,
88
104
  )
@@ -90,11 +106,23 @@ def cleanup_elastic_ip_addresses(neo4j_session: neo4j.Session, common_job_parame
90
106
 
91
107
  @timeit
92
108
  def sync_elastic_ip_addresses(
93
- neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str],
94
- current_aws_account_id: str, update_tag: int, common_job_parameters: Dict,
109
+ neo4j_session: neo4j.Session,
110
+ boto3_session: boto3.session.Session,
111
+ regions: List[str],
112
+ current_aws_account_id: str,
113
+ update_tag: int,
114
+ common_job_parameters: Dict,
95
115
  ) -> None:
96
116
  for region in regions:
97
- logger.info(f"Syncing Elastic IP Addresses for region {region} in account {current_aws_account_id}.")
117
+ logger.info(
118
+ f"Syncing Elastic IP Addresses for region {region} in account {current_aws_account_id}.",
119
+ )
98
120
  addresses = get_elastic_ip_addresses(boto3_session, region)
99
- load_elastic_ip_addresses(neo4j_session, addresses, region, current_aws_account_id, update_tag)
121
+ load_elastic_ip_addresses(
122
+ neo4j_session,
123
+ addresses,
124
+ region,
125
+ current_aws_account_id,
126
+ update_tag,
127
+ )
100
128
  cleanup_elastic_ip_addresses(neo4j_session, common_job_parameters)