cartography 0.102.0rc1__py3-none-any.whl → 0.103.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

Files changed (297) hide show
  1. cartography/__main__.py +1 -2
  2. cartography/_version.py +2 -2
  3. cartography/cli.py +376 -249
  4. cartography/client/core/tx.py +39 -18
  5. cartography/config.py +28 -0
  6. cartography/driftdetect/__main__.py +1 -2
  7. cartography/driftdetect/add_shortcut.py +10 -2
  8. cartography/driftdetect/cli.py +71 -75
  9. cartography/driftdetect/detect_deviations.py +7 -3
  10. cartography/driftdetect/get_states.py +20 -8
  11. cartography/driftdetect/model.py +5 -5
  12. cartography/driftdetect/serializers.py +8 -6
  13. cartography/driftdetect/storage.py +2 -2
  14. cartography/graph/cleanupbuilder.py +35 -15
  15. cartography/graph/job.py +46 -17
  16. cartography/graph/querybuilder.py +165 -80
  17. cartography/graph/statement.py +35 -26
  18. cartography/intel/analysis.py +4 -1
  19. cartography/intel/aws/__init__.py +114 -55
  20. cartography/intel/aws/apigateway.py +134 -63
  21. cartography/intel/aws/cloudtrail.py +127 -0
  22. cartography/intel/aws/cloudwatch.py +93 -0
  23. cartography/intel/aws/config.py +56 -20
  24. cartography/intel/aws/dynamodb.py +108 -40
  25. cartography/intel/aws/ec2/__init__.py +2 -2
  26. cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
  27. cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
  28. cartography/intel/aws/ec2/images.py +49 -20
  29. cartography/intel/aws/ec2/instances.py +234 -136
  30. cartography/intel/aws/ec2/internet_gateways.py +40 -11
  31. cartography/intel/aws/ec2/key_pairs.py +44 -20
  32. cartography/intel/aws/ec2/launch_templates.py +101 -59
  33. cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
  34. cartography/intel/aws/ec2/load_balancers.py +82 -42
  35. cartography/intel/aws/ec2/network_acls.py +89 -65
  36. cartography/intel/aws/ec2/network_interfaces.py +146 -87
  37. cartography/intel/aws/ec2/reserved_instances.py +45 -16
  38. cartography/intel/aws/ec2/route_tables.py +327 -0
  39. cartography/intel/aws/ec2/security_groups.py +71 -21
  40. cartography/intel/aws/ec2/snapshots.py +61 -22
  41. cartography/intel/aws/ec2/subnets.py +54 -18
  42. cartography/intel/aws/ec2/tgw.py +100 -34
  43. cartography/intel/aws/ec2/util.py +1 -1
  44. cartography/intel/aws/ec2/volumes.py +69 -41
  45. cartography/intel/aws/ec2/vpc.py +37 -12
  46. cartography/intel/aws/ec2/vpc_peerings.py +83 -24
  47. cartography/intel/aws/ecr.py +88 -32
  48. cartography/intel/aws/ecs.py +83 -47
  49. cartography/intel/aws/efs.py +93 -0
  50. cartography/intel/aws/eks.py +55 -29
  51. cartography/intel/aws/elasticache.py +42 -18
  52. cartography/intel/aws/elasticsearch.py +57 -20
  53. cartography/intel/aws/emr.py +61 -23
  54. cartography/intel/aws/iam.py +401 -145
  55. cartography/intel/aws/iam_instance_profiles.py +22 -22
  56. cartography/intel/aws/identitycenter.py +71 -37
  57. cartography/intel/aws/inspector.py +159 -89
  58. cartography/intel/aws/kms.py +92 -38
  59. cartography/intel/aws/lambda_function.py +103 -34
  60. cartography/intel/aws/organizations.py +30 -10
  61. cartography/intel/aws/permission_relationships.py +133 -51
  62. cartography/intel/aws/rds.py +249 -85
  63. cartography/intel/aws/redshift.py +107 -46
  64. cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
  65. cartography/intel/aws/resources.py +57 -44
  66. cartography/intel/aws/route53.py +108 -61
  67. cartography/intel/aws/s3.py +168 -83
  68. cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
  69. cartography/intel/aws/secretsmanager.py +24 -12
  70. cartography/intel/aws/securityhub.py +20 -9
  71. cartography/intel/aws/sns.py +166 -0
  72. cartography/intel/aws/sqs.py +60 -28
  73. cartography/intel/aws/ssm.py +70 -30
  74. cartography/intel/aws/util/arns.py +7 -7
  75. cartography/intel/aws/util/common.py +31 -4
  76. cartography/intel/azure/__init__.py +78 -19
  77. cartography/intel/azure/compute.py +101 -27
  78. cartography/intel/azure/cosmosdb.py +496 -170
  79. cartography/intel/azure/sql.py +296 -105
  80. cartography/intel/azure/storage.py +322 -113
  81. cartography/intel/azure/subscription.py +39 -23
  82. cartography/intel/azure/tenant.py +13 -4
  83. cartography/intel/azure/util/credentials.py +95 -55
  84. cartography/intel/bigfix/__init__.py +2 -2
  85. cartography/intel/bigfix/computers.py +93 -65
  86. cartography/intel/cloudflare/__init__.py +74 -0
  87. cartography/intel/cloudflare/accounts.py +57 -0
  88. cartography/intel/cloudflare/dnsrecords.py +64 -0
  89. cartography/intel/cloudflare/members.py +75 -0
  90. cartography/intel/cloudflare/roles.py +65 -0
  91. cartography/intel/cloudflare/zones.py +64 -0
  92. cartography/intel/create_indexes.py +3 -2
  93. cartography/intel/crowdstrike/__init__.py +11 -9
  94. cartography/intel/crowdstrike/endpoints.py +5 -1
  95. cartography/intel/crowdstrike/spotlight.py +8 -3
  96. cartography/intel/cve/__init__.py +46 -13
  97. cartography/intel/cve/feed.py +48 -12
  98. cartography/intel/digitalocean/__init__.py +22 -13
  99. cartography/intel/digitalocean/compute.py +75 -108
  100. cartography/intel/digitalocean/management.py +44 -80
  101. cartography/intel/digitalocean/platform.py +48 -43
  102. cartography/intel/dns.py +36 -10
  103. cartography/intel/duo/__init__.py +21 -16
  104. cartography/intel/duo/api_host.py +14 -9
  105. cartography/intel/duo/endpoints.py +50 -45
  106. cartography/intel/duo/groups.py +18 -14
  107. cartography/intel/duo/phones.py +37 -34
  108. cartography/intel/duo/tokens.py +26 -23
  109. cartography/intel/duo/users.py +54 -50
  110. cartography/intel/duo/web_authn_credentials.py +30 -25
  111. cartography/intel/entra/__init__.py +25 -7
  112. cartography/intel/entra/ou.py +112 -0
  113. cartography/intel/entra/users.py +69 -63
  114. cartography/intel/gcp/__init__.py +185 -49
  115. cartography/intel/gcp/compute.py +418 -231
  116. cartography/intel/gcp/crm.py +96 -43
  117. cartography/intel/gcp/dns.py +60 -19
  118. cartography/intel/gcp/gke.py +72 -38
  119. cartography/intel/gcp/iam.py +61 -41
  120. cartography/intel/gcp/storage.py +84 -55
  121. cartography/intel/github/__init__.py +13 -11
  122. cartography/intel/github/repos.py +270 -137
  123. cartography/intel/github/teams.py +170 -88
  124. cartography/intel/github/users.py +70 -39
  125. cartography/intel/github/util.py +36 -34
  126. cartography/intel/gsuite/__init__.py +47 -26
  127. cartography/intel/gsuite/api.py +73 -30
  128. cartography/intel/jamf/__init__.py +19 -1
  129. cartography/intel/jamf/computers.py +30 -7
  130. cartography/intel/jamf/util.py +7 -2
  131. cartography/intel/kandji/__init__.py +6 -3
  132. cartography/intel/kandji/devices.py +14 -8
  133. cartography/intel/kubernetes/namespaces.py +7 -4
  134. cartography/intel/kubernetes/pods.py +7 -4
  135. cartography/intel/kubernetes/services.py +8 -4
  136. cartography/intel/lastpass/__init__.py +2 -2
  137. cartography/intel/lastpass/users.py +23 -12
  138. cartography/intel/oci/__init__.py +44 -11
  139. cartography/intel/oci/iam.py +134 -38
  140. cartography/intel/oci/organizations.py +13 -6
  141. cartography/intel/oci/utils.py +43 -20
  142. cartography/intel/okta/__init__.py +66 -15
  143. cartography/intel/okta/applications.py +42 -20
  144. cartography/intel/okta/awssaml.py +93 -33
  145. cartography/intel/okta/factors.py +16 -4
  146. cartography/intel/okta/groups.py +56 -29
  147. cartography/intel/okta/organization.py +5 -1
  148. cartography/intel/okta/origins.py +6 -2
  149. cartography/intel/okta/roles.py +15 -5
  150. cartography/intel/okta/users.py +20 -8
  151. cartography/intel/okta/utils.py +6 -4
  152. cartography/intel/openai/__init__.py +86 -0
  153. cartography/intel/openai/adminapikeys.py +90 -0
  154. cartography/intel/openai/apikeys.py +96 -0
  155. cartography/intel/openai/projects.py +94 -0
  156. cartography/intel/openai/serviceaccounts.py +82 -0
  157. cartography/intel/openai/users.py +78 -0
  158. cartography/intel/openai/util.py +29 -0
  159. cartography/intel/pagerduty/__init__.py +8 -7
  160. cartography/intel/pagerduty/escalation_policies.py +18 -6
  161. cartography/intel/pagerduty/schedules.py +12 -4
  162. cartography/intel/pagerduty/services.py +11 -4
  163. cartography/intel/pagerduty/teams.py +8 -3
  164. cartography/intel/pagerduty/users.py +3 -1
  165. cartography/intel/pagerduty/vendors.py +3 -1
  166. cartography/intel/semgrep/__init__.py +24 -6
  167. cartography/intel/semgrep/dependencies.py +50 -28
  168. cartography/intel/semgrep/deployment.py +3 -1
  169. cartography/intel/semgrep/findings.py +42 -18
  170. cartography/intel/snipeit/__init__.py +17 -3
  171. cartography/intel/snipeit/asset.py +12 -6
  172. cartography/intel/snipeit/user.py +8 -5
  173. cartography/intel/snipeit/util.py +9 -4
  174. cartography/intel/tailscale/__init__.py +77 -0
  175. cartography/intel/tailscale/acls.py +146 -0
  176. cartography/intel/tailscale/devices.py +127 -0
  177. cartography/intel/tailscale/postureintegrations.py +81 -0
  178. cartography/intel/tailscale/tailnets.py +76 -0
  179. cartography/intel/tailscale/users.py +80 -0
  180. cartography/intel/tailscale/utils.py +132 -0
  181. cartography/models/aws/apigateway.py +21 -17
  182. cartography/models/aws/apigatewaycertificate.py +28 -22
  183. cartography/models/aws/apigatewayresource.py +28 -20
  184. cartography/models/aws/apigatewaystage.py +33 -25
  185. cartography/models/aws/cloudtrail/__init__.py +0 -0
  186. cartography/models/aws/cloudtrail/trail.py +61 -0
  187. cartography/models/aws/cloudwatch/__init__.py +0 -0
  188. cartography/models/aws/cloudwatch/loggroup.py +52 -0
  189. cartography/models/aws/dynamodb/gsi.py +30 -22
  190. cartography/models/aws/dynamodb/tables.py +25 -17
  191. cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
  192. cartography/models/aws/ec2/images.py +36 -34
  193. cartography/models/aws/ec2/instances.py +51 -45
  194. cartography/models/aws/ec2/keypair.py +21 -16
  195. cartography/models/aws/ec2/keypair_instance.py +28 -21
  196. cartography/models/aws/ec2/launch_configurations.py +30 -26
  197. cartography/models/aws/ec2/launch_template_versions.py +48 -38
  198. cartography/models/aws/ec2/launch_templates.py +21 -17
  199. cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
  200. cartography/models/aws/ec2/load_balancers.py +47 -37
  201. cartography/models/aws/ec2/network_acl_rules.py +38 -30
  202. cartography/models/aws/ec2/network_acls.py +38 -29
  203. cartography/models/aws/ec2/networkinterface_instance.py +52 -39
  204. cartography/models/aws/ec2/networkinterfaces.py +53 -37
  205. cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
  206. cartography/models/aws/ec2/reservations.py +18 -14
  207. cartography/models/aws/ec2/route_table_associations.py +97 -0
  208. cartography/models/aws/ec2/route_tables.py +128 -0
  209. cartography/models/aws/ec2/routes.py +85 -0
  210. cartography/models/aws/ec2/securitygroup_instance.py +29 -20
  211. cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
  212. cartography/models/aws/ec2/subnet_instance.py +24 -19
  213. cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
  214. cartography/models/aws/ec2/volumes.py +47 -40
  215. cartography/models/aws/efs/__init__.py +0 -0
  216. cartography/models/aws/efs/mount_target.py +52 -0
  217. cartography/models/aws/eks/clusters.py +23 -21
  218. cartography/models/aws/emr.py +32 -30
  219. cartography/models/aws/iam/instanceprofile.py +33 -24
  220. cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
  221. cartography/models/aws/identitycenter/awspermissionset.py +37 -29
  222. cartography/models/aws/identitycenter/awsssouser.py +23 -21
  223. cartography/models/aws/inspector/findings.py +77 -65
  224. cartography/models/aws/inspector/packages.py +35 -29
  225. cartography/models/aws/s3/__init__.py +0 -0
  226. cartography/models/aws/s3/account_public_access_block.py +51 -0
  227. cartography/models/aws/sns/__init__.py +0 -0
  228. cartography/models/aws/sns/topic.py +50 -0
  229. cartography/models/aws/ssm/instance_information.py +51 -39
  230. cartography/models/aws/ssm/instance_patch.py +32 -26
  231. cartography/models/bigfix/bigfix_computer.py +42 -38
  232. cartography/models/bigfix/bigfix_root.py +3 -3
  233. cartography/models/cloudflare/__init__.py +0 -0
  234. cartography/models/cloudflare/account.py +25 -0
  235. cartography/models/cloudflare/dnsrecord.py +55 -0
  236. cartography/models/cloudflare/member.py +82 -0
  237. cartography/models/cloudflare/role.py +44 -0
  238. cartography/models/cloudflare/zone.py +59 -0
  239. cartography/models/core/common.py +12 -10
  240. cartography/models/core/nodes.py +5 -2
  241. cartography/models/core/relationships.py +14 -6
  242. cartography/models/crowdstrike/hosts.py +37 -35
  243. cartography/models/cve/cve.py +34 -32
  244. cartography/models/cve/cve_feed.py +6 -6
  245. cartography/models/digitalocean/__init__.py +0 -0
  246. cartography/models/digitalocean/account.py +21 -0
  247. cartography/models/digitalocean/droplet.py +56 -0
  248. cartography/models/digitalocean/project.py +48 -0
  249. cartography/models/duo/api_host.py +3 -3
  250. cartography/models/duo/endpoint.py +43 -41
  251. cartography/models/duo/group.py +14 -14
  252. cartography/models/duo/phone.py +27 -27
  253. cartography/models/duo/token.py +16 -16
  254. cartography/models/duo/user.py +46 -44
  255. cartography/models/duo/web_authn_credential.py +27 -19
  256. cartography/models/entra/ou.py +48 -0
  257. cartography/models/entra/tenant.py +24 -18
  258. cartography/models/entra/user.py +64 -48
  259. cartography/models/gcp/iam.py +23 -23
  260. cartography/models/github/orgs.py +5 -4
  261. cartography/models/github/teams.py +37 -31
  262. cartography/models/github/users.py +34 -23
  263. cartography/models/kandji/device.py +22 -16
  264. cartography/models/kandji/tenant.py +6 -4
  265. cartography/models/lastpass/tenant.py +3 -3
  266. cartography/models/lastpass/user.py +32 -28
  267. cartography/models/openai/__init__.py +0 -0
  268. cartography/models/openai/adminapikey.py +90 -0
  269. cartography/models/openai/apikey.py +84 -0
  270. cartography/models/openai/organization.py +17 -0
  271. cartography/models/openai/project.py +70 -0
  272. cartography/models/openai/serviceaccount.py +50 -0
  273. cartography/models/openai/user.py +49 -0
  274. cartography/models/semgrep/dependencies.py +36 -24
  275. cartography/models/semgrep/deployment.py +5 -5
  276. cartography/models/semgrep/findings.py +58 -42
  277. cartography/models/semgrep/locations.py +27 -21
  278. cartography/models/snipeit/asset.py +30 -21
  279. cartography/models/snipeit/tenant.py +6 -4
  280. cartography/models/snipeit/user.py +19 -12
  281. cartography/models/tailscale/__init__.py +0 -0
  282. cartography/models/tailscale/device.py +95 -0
  283. cartography/models/tailscale/group.py +86 -0
  284. cartography/models/tailscale/postureintegration.py +58 -0
  285. cartography/models/tailscale/tag.py +102 -0
  286. cartography/models/tailscale/tailnet.py +29 -0
  287. cartography/models/tailscale/user.py +52 -0
  288. cartography/stats.py +3 -3
  289. cartography/sync.py +113 -31
  290. cartography/util.py +84 -62
  291. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/METADATA +8 -15
  292. cartography-0.103.0.dist-info/RECORD +442 -0
  293. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/WHEEL +1 -1
  294. cartography-0.102.0rc1.dist-info/RECORD +0 -377
  295. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/entry_points.txt +0 -0
  296. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/licenses/LICENSE +0 -0
  297. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/top_level.txt +0 -0
@@ -5,25 +5,36 @@ from typing import List
5
5
  import boto3
6
6
  import neo4j
7
7
 
8
- from .util import get_botocore_config
9
8
  from cartography.util import aws_handle_regions
10
9
  from cartography.util import run_cleanup_job
11
10
  from cartography.util import timeit
12
11
 
12
+ from .util import get_botocore_config
13
+
13
14
  logger = logging.getLogger(__name__)
14
15
 
15
16
 
16
17
  @timeit
17
18
  @aws_handle_regions
18
- def get_vpc_peerings_data(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
19
- client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
20
- return client.describe_vpc_peering_connections()['VpcPeeringConnections']
19
+ def get_vpc_peerings_data(
20
+ boto3_session: boto3.session.Session,
21
+ region: str,
22
+ ) -> List[Dict]:
23
+ client = boto3_session.client(
24
+ "ec2",
25
+ region_name=region,
26
+ config=get_botocore_config(),
27
+ )
28
+ return client.describe_vpc_peering_connections()["VpcPeeringConnections"]
21
29
 
22
30
 
23
31
  @timeit
24
32
  def load_vpc_peerings(
25
- neo4j_session: neo4j.Session, data: List[Dict], region: str,
26
- aws_account_id: str, update_tag: int,
33
+ neo4j_session: neo4j.Session,
34
+ data: List[Dict],
35
+ region: str,
36
+ aws_account_id: str,
37
+ update_tag: int,
27
38
  ) -> None:
28
39
  ingest_vpc_peerings = """
29
40
  UNWIND $vpc_peerings AS vpc_peering
@@ -77,15 +88,21 @@ def load_vpc_peerings(
77
88
  """
78
89
 
79
90
  neo4j_session.run(
80
- ingest_vpc_peerings, vpc_peerings=data, update_tag=update_tag,
81
- region=region, aws_account_id=aws_account_id,
91
+ ingest_vpc_peerings,
92
+ vpc_peerings=data,
93
+ update_tag=update_tag,
94
+ region=region,
95
+ aws_account_id=aws_account_id,
82
96
  )
83
97
 
84
98
 
85
99
  @timeit
86
100
  def load_accepter_cidrs(
87
- neo4j_session: neo4j.Session, data: List[Dict], region: str,
88
- aws_account_id: str, update_tag: int,
101
+ neo4j_session: neo4j.Session,
102
+ data: List[Dict],
103
+ region: str,
104
+ aws_account_id: str,
105
+ update_tag: int,
89
106
  ) -> None:
90
107
 
91
108
  ingest_accepter_cidr = """
@@ -110,15 +127,21 @@ def load_accepter_cidrs(
110
127
  """
111
128
 
112
129
  neo4j_session.run(
113
- ingest_accepter_cidr, vpc_peerings=data, update_tag=update_tag,
114
- region=region, aws_account_id=aws_account_id,
130
+ ingest_accepter_cidr,
131
+ vpc_peerings=data,
132
+ update_tag=update_tag,
133
+ region=region,
134
+ aws_account_id=aws_account_id,
115
135
  )
116
136
 
117
137
 
118
138
  @timeit
119
139
  def load_requester_cidrs(
120
- neo4j_session: neo4j.Session, data: List[Dict], region: str,
121
- aws_account_id: str, update_tag: int,
140
+ neo4j_session: neo4j.Session,
141
+ data: List[Dict],
142
+ region: str,
143
+ aws_account_id: str,
144
+ update_tag: int,
122
145
  ) -> None:
123
146
 
124
147
  ingest_requester_cidr = """
@@ -143,25 +166,61 @@ def load_requester_cidrs(
143
166
  """
144
167
 
145
168
  neo4j_session.run(
146
- ingest_requester_cidr, vpc_peerings=data, update_tag=update_tag,
147
- region=region, aws_account_id=aws_account_id,
169
+ ingest_requester_cidr,
170
+ vpc_peerings=data,
171
+ update_tag=update_tag,
172
+ region=region,
173
+ aws_account_id=aws_account_id,
148
174
  )
149
175
 
150
176
 
151
177
  @timeit
152
- def cleanup_vpc_peerings(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
153
- run_cleanup_job('aws_import_vpc_peering_cleanup.json', neo4j_session, common_job_parameters)
178
+ def cleanup_vpc_peerings(
179
+ neo4j_session: neo4j.Session,
180
+ common_job_parameters: Dict,
181
+ ) -> None:
182
+ run_cleanup_job(
183
+ "aws_import_vpc_peering_cleanup.json",
184
+ neo4j_session,
185
+ common_job_parameters,
186
+ )
154
187
 
155
188
 
156
189
  @timeit
157
190
  def sync_vpc_peerings(
158
- neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str],
159
- current_aws_account_id: str, update_tag: int, common_job_parameters: Dict,
191
+ neo4j_session: neo4j.Session,
192
+ boto3_session: boto3.session.Session,
193
+ regions: List[str],
194
+ current_aws_account_id: str,
195
+ update_tag: int,
196
+ common_job_parameters: Dict,
160
197
  ) -> None:
161
198
  for region in regions:
162
- logger.debug("Syncing EC2 VPC peering for region '%s' in account '%s'.", region, current_aws_account_id)
199
+ logger.debug(
200
+ "Syncing EC2 VPC peering for region '%s' in account '%s'.",
201
+ region,
202
+ current_aws_account_id,
203
+ )
163
204
  data = get_vpc_peerings_data(boto3_session, region)
164
- load_vpc_peerings(neo4j_session, data, region, current_aws_account_id, update_tag)
165
- load_accepter_cidrs(neo4j_session, data, region, current_aws_account_id, update_tag)
166
- load_requester_cidrs(neo4j_session, data, region, current_aws_account_id, update_tag)
205
+ load_vpc_peerings(
206
+ neo4j_session,
207
+ data,
208
+ region,
209
+ current_aws_account_id,
210
+ update_tag,
211
+ )
212
+ load_accepter_cidrs(
213
+ neo4j_session,
214
+ data,
215
+ region,
216
+ current_aws_account_id,
217
+ update_tag,
218
+ )
219
+ load_requester_cidrs(
220
+ neo4j_session,
221
+ data,
222
+ region,
223
+ current_aws_account_id,
224
+ update_tag,
225
+ )
167
226
  cleanup_vpc_peerings(neo4j_session, common_job_parameters)
@@ -18,33 +18,48 @@ logger = logging.getLogger(__name__)
18
18
 
19
19
  @timeit
20
20
  @aws_handle_regions
21
- def get_ecr_repositories(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
21
+ def get_ecr_repositories(
22
+ boto3_session: boto3.session.Session,
23
+ region: str,
24
+ ) -> List[Dict]:
22
25
  logger.info("Getting ECR repositories for region '%s'.", region)
23
- client = boto3_session.client('ecr', region_name=region)
24
- paginator = client.get_paginator('describe_repositories')
26
+ client = boto3_session.client("ecr", region_name=region)
27
+ paginator = client.get_paginator("describe_repositories")
25
28
  ecr_repositories: List[Dict] = []
26
29
  for page in paginator.paginate():
27
- ecr_repositories.extend(page['repositories'])
30
+ ecr_repositories.extend(page["repositories"])
28
31
  return ecr_repositories
29
32
 
30
33
 
31
34
  @timeit
32
35
  @aws_handle_regions
33
- def get_ecr_repository_images(boto3_session: boto3.session.Session, region: str, repository_name: str) -> List[Dict]:
34
- logger.debug("Getting ECR images in repository '%s' for region '%s'.", repository_name, region)
35
- client = boto3_session.client('ecr', region_name=region)
36
- list_paginator = client.get_paginator('list_images')
36
+ def get_ecr_repository_images(
37
+ boto3_session: boto3.session.Session, region: str, repository_name: str
38
+ ) -> List[Dict]:
39
+ logger.debug(
40
+ "Getting ECR images in repository '%s' for region '%s'.",
41
+ repository_name,
42
+ region,
43
+ )
44
+ client = boto3_session.client("ecr", region_name=region)
45
+ list_paginator = client.get_paginator("list_images")
37
46
  ecr_repository_images: List[Dict] = []
38
47
  for page in list_paginator.paginate(repositoryName=repository_name):
39
- image_ids = page['imageIds']
48
+ image_ids = page["imageIds"]
40
49
  if not image_ids:
41
50
  continue
42
- describe_paginator = client.get_paginator('describe_images')
43
- describe_response = describe_paginator.paginate(repositoryName=repository_name, imageIds=image_ids)
51
+ describe_paginator = client.get_paginator("describe_images")
52
+ describe_response = describe_paginator.paginate(
53
+ repositoryName=repository_name, imageIds=image_ids
54
+ )
44
55
  for response in describe_response:
45
- image_details = response['imageDetails']
56
+ image_details = response["imageDetails"]
46
57
  image_details = [
47
- {**detail, 'imageTag': detail['imageTags'][0]} if detail.get('imageTags') else detail
58
+ (
59
+ {**detail, "imageTag": detail["imageTags"][0]}
60
+ if detail.get("imageTags")
61
+ else detail
62
+ )
48
63
  for detail in image_details
49
64
  ]
50
65
  ecr_repository_images.extend(image_details)
@@ -53,7 +68,10 @@ def get_ecr_repository_images(boto3_session: boto3.session.Session, region: str,
53
68
 
54
69
  @timeit
55
70
  def load_ecr_repositories(
56
- neo4j_session: neo4j.Session, repos: List[Dict], region: str, current_aws_account_id: str,
71
+ neo4j_session: neo4j.Session,
72
+ repos: List[Dict],
73
+ region: str,
74
+ current_aws_account_id: str,
57
75
  aws_update_tag: int,
58
76
  ) -> None:
59
77
  query = """
@@ -73,7 +91,9 @@ def load_ecr_repositories(
73
91
  ON CREATE SET r.firstseen = timestamp()
74
92
  SET r.lastupdated = $aws_update_tag
75
93
  """
76
- logger.info(f"Loading {len(repos)} ECR repositories for region {region} into graph.")
94
+ logger.info(
95
+ f"Loading {len(repos)} ECR repositories for region {region} into graph.",
96
+ )
77
97
  neo4j_session.run(
78
98
  query,
79
99
  Repositories=repos,
@@ -91,21 +111,23 @@ def transform_ecr_repository_images(repo_data: Dict) -> List[Dict]:
91
111
  repo_images_list = []
92
112
  for repo_uri, repo_images in repo_data.items():
93
113
  for img in repo_images:
94
- if 'imageDigest' in img and img['imageDigest']:
95
- img['repo_uri'] = repo_uri
114
+ if "imageDigest" in img and img["imageDigest"]:
115
+ img["repo_uri"] = repo_uri
96
116
  repo_images_list.append(img)
97
117
  else:
98
118
  logger.warning(
99
119
  "Repo %s has an image that has no imageDigest. Its tag is %s. Continuing on.",
100
120
  repo_uri,
101
- img.get('imageTag'),
121
+ img.get("imageTag"),
102
122
  )
103
123
 
104
124
  return repo_images_list
105
125
 
106
126
 
107
127
  def _load_ecr_repo_img_tx(
108
- tx: neo4j.Transaction, repo_images_list: List[Dict], aws_update_tag: int,
128
+ tx: neo4j.Transaction,
129
+ repo_images_list: List[Dict],
130
+ aws_update_tag: int,
109
131
  region: str,
110
132
  ) -> None:
111
133
  query = """
@@ -139,23 +161,37 @@ def _load_ecr_repo_img_tx(
139
161
  ON CREATE SET r2.firstseen = timestamp()
140
162
  SET r2.lastupdated = $aws_update_tag
141
163
  """
142
- tx.run(query, RepoList=repo_images_list, Region=region, aws_update_tag=aws_update_tag)
164
+ tx.run(
165
+ query,
166
+ RepoList=repo_images_list,
167
+ Region=region,
168
+ aws_update_tag=aws_update_tag,
169
+ )
143
170
 
144
171
 
145
172
  @timeit
146
173
  def load_ecr_repository_images(
147
- neo4j_session: neo4j.Session, repo_images_list: List[Dict], region: str,
174
+ neo4j_session: neo4j.Session,
175
+ repo_images_list: List[Dict],
176
+ region: str,
148
177
  aws_update_tag: int,
149
178
  ) -> None:
150
- logger.info(f"Loading {len(repo_images_list)} ECR repository images in {region} into graph.")
179
+ logger.info(
180
+ f"Loading {len(repo_images_list)} ECR repository images in {region} into graph.",
181
+ )
151
182
  for repo_image_batch in batch(repo_images_list, size=10000):
152
- neo4j_session.write_transaction(_load_ecr_repo_img_tx, repo_image_batch, aws_update_tag, region)
183
+ neo4j_session.write_transaction(
184
+ _load_ecr_repo_img_tx,
185
+ repo_image_batch,
186
+ aws_update_tag,
187
+ region,
188
+ )
153
189
 
154
190
 
155
191
  @timeit
156
192
  def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
157
193
  logger.debug("Running ECR cleanup job.")
158
- run_cleanup_job('aws_import_ecr_cleanup.json', neo4j_session, common_job_parameters)
194
+ run_cleanup_job("aws_import_ecr_cleanup.json", neo4j_session, common_job_parameters)
159
195
 
160
196
 
161
197
  def _get_image_data(
@@ -163,15 +199,21 @@ def _get_image_data(
163
199
  region: str,
164
200
  repositories: List[Dict[str, Any]],
165
201
  ) -> Dict[str, Any]:
166
- '''
202
+ """
167
203
  Given a list of repositories, get the image data for each repository,
168
204
  return as a mapping from repositoryUri to image object
169
- '''
205
+ """
170
206
  image_data = {}
171
207
 
172
208
  async def async_get_images(repo: Dict[str, Any]) -> None:
173
- repo_image_obj = await to_asynchronous(get_ecr_repository_images, boto3_session, region, repo['repositoryName'])
174
- image_data[repo['repositoryUri']] = repo_image_obj
209
+ repo_image_obj = await to_asynchronous(
210
+ get_ecr_repository_images,
211
+ boto3_session,
212
+ region,
213
+ repo["repositoryName"],
214
+ )
215
+ image_data[repo["repositoryUri"]] = repo_image_obj
216
+
175
217
  to_synchronous(*[async_get_images(repo) for repo in repositories])
176
218
 
177
219
  return image_data
@@ -179,15 +221,29 @@ def _get_image_data(
179
221
 
180
222
  @timeit
181
223
  def sync(
182
- neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
183
- update_tag: int, common_job_parameters: Dict,
224
+ neo4j_session: neo4j.Session,
225
+ boto3_session: boto3.session.Session,
226
+ regions: List[str],
227
+ current_aws_account_id: str,
228
+ update_tag: int,
229
+ common_job_parameters: Dict,
184
230
  ) -> None:
185
231
  for region in regions:
186
- logger.info("Syncing ECR for region '%s' in account '%s'.", region, current_aws_account_id)
232
+ logger.info(
233
+ "Syncing ECR for region '%s' in account '%s'.",
234
+ region,
235
+ current_aws_account_id,
236
+ )
187
237
  image_data = {}
188
238
  repositories = get_ecr_repositories(boto3_session, region)
189
239
  image_data = _get_image_data(boto3_session, region, repositories)
190
- load_ecr_repositories(neo4j_session, repositories, region, current_aws_account_id, update_tag)
240
+ load_ecr_repositories(
241
+ neo4j_session,
242
+ repositories,
243
+ region,
244
+ current_aws_account_id,
245
+ update_tag,
246
+ )
191
247
  repo_images_list = transform_ecr_repository_images(image_data)
192
248
  load_ecr_repository_images(neo4j_session, repo_images_list, region, update_tag)
193
249
  cleanup(neo4j_session, common_job_parameters)
@@ -17,12 +17,15 @@ logger = logging.getLogger(__name__)
17
17
 
18
18
  @timeit
19
19
  @aws_handle_regions
20
- def get_ecs_cluster_arns(boto3_session: boto3.session.Session, region: str) -> List[str]:
21
- client = boto3_session.client('ecs', region_name=region)
22
- paginator = client.get_paginator('list_clusters')
20
+ def get_ecs_cluster_arns(
21
+ boto3_session: boto3.session.Session,
22
+ region: str,
23
+ ) -> List[str]:
24
+ client = boto3_session.client("ecs", region_name=region)
25
+ paginator = client.get_paginator("list_clusters")
23
26
  cluster_arns: List[str] = []
24
27
  for page in paginator.paginate():
25
- cluster_arns.extend(page.get('clusterArns', []))
28
+ cluster_arns.extend(page.get("clusterArns", []))
26
29
  return cluster_arns
27
30
 
28
31
 
@@ -33,15 +36,18 @@ def get_ecs_clusters(
33
36
  region: str,
34
37
  cluster_arns: List[str],
35
38
  ) -> List[Dict[str, Any]]:
36
- client = boto3_session.client('ecs', region_name=region)
39
+ client = boto3_session.client("ecs", region_name=region)
37
40
  # TODO: also include attachment info, and make relationships between the attachements
38
41
  # and the cluster.
39
- includes = ['SETTINGS', 'CONFIGURATIONS']
42
+ includes = ["SETTINGS", "CONFIGURATIONS"]
40
43
  clusters: List[Dict[str, Any]] = []
41
44
  for i in range(0, len(cluster_arns), 100):
42
- cluster_arn_chunk = cluster_arns[i:i + 100]
43
- cluster_chunk = client.describe_clusters(clusters=cluster_arn_chunk, include=includes)
44
- clusters.extend(cluster_chunk.get('clusters', []))
45
+ cluster_arn_chunk = cluster_arns[i : i + 100]
46
+ cluster_chunk = client.describe_clusters(
47
+ clusters=cluster_arn_chunk,
48
+ include=includes,
49
+ )
50
+ clusters.extend(cluster_chunk.get("clusters", []))
45
51
  return clusters
46
52
 
47
53
 
@@ -52,40 +58,46 @@ def get_ecs_container_instances(
52
58
  boto3_session: boto3.session.Session,
53
59
  region: str,
54
60
  ) -> List[Dict[str, Any]]:
55
- client = boto3_session.client('ecs', region_name=region)
56
- paginator = client.get_paginator('list_container_instances')
61
+ client = boto3_session.client("ecs", region_name=region)
62
+ paginator = client.get_paginator("list_container_instances")
57
63
  container_instances: List[Dict[str, Any]] = []
58
64
  container_instance_arns: List[str] = []
59
65
  for page in paginator.paginate(cluster=cluster_arn):
60
- container_instance_arns.extend(page.get('containerInstanceArns', []))
61
- includes = ['CONTAINER_INSTANCE_HEALTH']
66
+ container_instance_arns.extend(page.get("containerInstanceArns", []))
67
+ includes = ["CONTAINER_INSTANCE_HEALTH"]
62
68
  for i in range(0, len(container_instance_arns), 100):
63
- container_instance_arn_chunk = container_instance_arns[i:i + 100]
69
+ container_instance_arn_chunk = container_instance_arns[i : i + 100]
64
70
  container_instance_chunk = client.describe_container_instances(
65
71
  cluster=cluster_arn,
66
72
  containerInstances=container_instance_arn_chunk,
67
73
  include=includes,
68
74
  )
69
- container_instances.extend(container_instance_chunk.get('containerInstances', []))
75
+ container_instances.extend(
76
+ container_instance_chunk.get("containerInstances", []),
77
+ )
70
78
  return container_instances
71
79
 
72
80
 
73
81
  @timeit
74
82
  @aws_handle_regions
75
- def get_ecs_services(cluster_arn: str, boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]:
76
- client = boto3_session.client('ecs', region_name=region)
77
- paginator = client.get_paginator('list_services')
83
+ def get_ecs_services(
84
+ cluster_arn: str,
85
+ boto3_session: boto3.session.Session,
86
+ region: str,
87
+ ) -> List[Dict[str, Any]]:
88
+ client = boto3_session.client("ecs", region_name=region)
89
+ paginator = client.get_paginator("list_services")
78
90
  services: List[Dict[str, Any]] = []
79
91
  service_arns: List[str] = []
80
92
  for page in paginator.paginate(cluster=cluster_arn):
81
- service_arns.extend(page.get('serviceArns', []))
93
+ service_arns.extend(page.get("serviceArns", []))
82
94
  for i in range(0, len(service_arns), 10):
83
- service_arn_chunk = service_arns[i:i + 10]
95
+ service_arn_chunk = service_arns[i : i + 10]
84
96
  service_chunk = client.describe_services(
85
97
  cluster=cluster_arn,
86
98
  services=service_arn_chunk,
87
99
  )
88
- services.extend(service_chunk.get('services', []))
100
+ services.extend(service_chunk.get("services", []))
89
101
  return services
90
102
 
91
103
 
@@ -96,32 +108,36 @@ def get_ecs_task_definitions(
96
108
  region: str,
97
109
  tasks: List[Dict[str, Any]],
98
110
  ) -> List[Dict[str, Any]]:
99
- client = boto3_session.client('ecs', region_name=region)
111
+ client = boto3_session.client("ecs", region_name=region)
100
112
  task_definitions: List[Dict[str, Any]] = []
101
113
  for task in tasks:
102
114
  task_definition = client.describe_task_definition(
103
- taskDefinition=task['taskDefinitionArn'],
115
+ taskDefinition=task["taskDefinitionArn"],
104
116
  )
105
- task_definitions.append(task_definition['taskDefinition'])
117
+ task_definitions.append(task_definition["taskDefinition"])
106
118
  return task_definitions
107
119
 
108
120
 
109
121
  @timeit
110
122
  @aws_handle_regions
111
- def get_ecs_tasks(cluster_arn: str, boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]:
112
- client = boto3_session.client('ecs', region_name=region)
113
- paginator = client.get_paginator('list_tasks')
123
+ def get_ecs_tasks(
124
+ cluster_arn: str,
125
+ boto3_session: boto3.session.Session,
126
+ region: str,
127
+ ) -> List[Dict[str, Any]]:
128
+ client = boto3_session.client("ecs", region_name=region)
129
+ paginator = client.get_paginator("list_tasks")
114
130
  tasks: List[Dict[str, Any]] = []
115
131
  task_arns: List[str] = []
116
132
  for page in paginator.paginate(cluster=cluster_arn):
117
- task_arns.extend(page.get('taskArns', []))
133
+ task_arns.extend(page.get("taskArns", []))
118
134
  for i in range(0, len(task_arns), 100):
119
- task_arn_chunk = task_arns[i:i + 100]
135
+ task_arn_chunk = task_arns[i : i + 100]
120
136
  task_chunk = client.describe_tasks(
121
137
  cluster=cluster_arn,
122
138
  tasks=task_arn_chunk,
123
139
  )
124
- tasks.extend(task_chunk.get('tasks', []))
140
+ tasks.extend(task_chunk.get("tasks", []))
125
141
  return tasks
126
142
 
127
143
 
@@ -207,7 +223,7 @@ def load_ecs_container_instances(
207
223
  """
208
224
  instances: List[Dict[str, Any]] = []
209
225
  for instance in data:
210
- instance['registeredAt'] = dict_date_to_epoch(instance, 'registeredAt')
226
+ instance["registeredAt"] = dict_date_to_epoch(instance, "registeredAt")
211
227
  instances.append(instance)
212
228
 
213
229
  neo4j_session.run(
@@ -269,7 +285,7 @@ def load_ecs_services(
269
285
  """ # noqa:E501
270
286
  services: List[Dict[str, Any]] = []
271
287
  for service in data:
272
- service['createdAt'] = dict_date_to_epoch(service, 'createdAt')
288
+ service["createdAt"] = dict_date_to_epoch(service, "createdAt")
273
289
  services.append(service)
274
290
 
275
291
  neo4j_session.run(
@@ -331,8 +347,14 @@ def load_ecs_task_definitions(
331
347
  container_definitions: List[Dict[str, Any]] = []
332
348
  task_definitions: List[Dict[str, Any]] = []
333
349
  for task_definition in data:
334
- task_definition['registeredAt'] = dict_date_to_epoch(task_definition, 'registeredAt')
335
- task_definition['deregisteredAt'] = dict_date_to_epoch(task_definition, 'deregisteredAt')
350
+ task_definition["registeredAt"] = dict_date_to_epoch(
351
+ task_definition,
352
+ "registeredAt",
353
+ )
354
+ task_definition["deregisteredAt"] = dict_date_to_epoch(
355
+ task_definition,
356
+ "deregisteredAt",
357
+ )
336
358
  for container in task_definition.get("containerDefinitions", []):
337
359
  container["_taskDefinitionArn"] = task_definition["taskDefinitionArn"]
338
360
  container_definitions.append(container)
@@ -418,14 +440,14 @@ def load_ecs_tasks(
418
440
  containers: List[Dict[str, Any]] = []
419
441
  tasks: List[Dict[str, Any]] = []
420
442
  for task in data:
421
- task['connectivityAt'] = dict_date_to_epoch(task, 'connectivityAt')
422
- task['createdAt'] = dict_date_to_epoch(task, 'createdAt')
423
- task['executionStoppedAt'] = dict_date_to_epoch(task, 'executionStoppedAt')
424
- task['pullStartedAt'] = dict_date_to_epoch(task, 'pullStartedAt')
425
- task['pullStoppedAt'] = dict_date_to_epoch(task, 'pullStoppedAt')
426
- task['startedAt'] = dict_date_to_epoch(task, 'startedAt')
427
- task['stoppedAt'] = dict_date_to_epoch(task, 'stoppedAt')
428
- task['stoppingAt'] = dict_date_to_epoch(task, 'stoppingAt')
443
+ task["connectivityAt"] = dict_date_to_epoch(task, "connectivityAt")
444
+ task["createdAt"] = dict_date_to_epoch(task, "createdAt")
445
+ task["executionStoppedAt"] = dict_date_to_epoch(task, "executionStoppedAt")
446
+ task["pullStartedAt"] = dict_date_to_epoch(task, "pullStartedAt")
447
+ task["pullStoppedAt"] = dict_date_to_epoch(task, "pullStoppedAt")
448
+ task["startedAt"] = dict_date_to_epoch(task, "startedAt")
449
+ task["stoppedAt"] = dict_date_to_epoch(task, "stoppedAt")
450
+ task["stoppingAt"] = dict_date_to_epoch(task, "stoppingAt")
429
451
  containers.extend(task["containers"])
430
452
  tasks.append(task)
431
453
 
@@ -542,21 +564,35 @@ def load_ecs_containers(
542
564
 
543
565
  @timeit
544
566
  def cleanup_ecs(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
545
- run_cleanup_job('aws_import_ecs_cleanup.json', neo4j_session, common_job_parameters)
567
+ run_cleanup_job("aws_import_ecs_cleanup.json", neo4j_session, common_job_parameters)
546
568
 
547
569
 
548
570
  @timeit
549
571
  def sync(
550
- neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
551
- update_tag: int, common_job_parameters: Dict,
572
+ neo4j_session: neo4j.Session,
573
+ boto3_session: boto3.session.Session,
574
+ regions: List[str],
575
+ current_aws_account_id: str,
576
+ update_tag: int,
577
+ common_job_parameters: Dict,
552
578
  ) -> None:
553
579
  for region in regions:
554
- logger.info("Syncing ECS for region '%s' in account '%s'.", region, current_aws_account_id)
580
+ logger.info(
581
+ "Syncing ECS for region '%s' in account '%s'.",
582
+ region,
583
+ current_aws_account_id,
584
+ )
555
585
  cluster_arns = get_ecs_cluster_arns(boto3_session, region)
556
586
  clusters = get_ecs_clusters(boto3_session, region, cluster_arns)
557
587
  if len(clusters) == 0:
558
588
  continue
559
- load_ecs_clusters(neo4j_session, clusters, region, current_aws_account_id, update_tag)
589
+ load_ecs_clusters(
590
+ neo4j_session,
591
+ clusters,
592
+ region,
593
+ current_aws_account_id,
594
+ update_tag,
595
+ )
560
596
  for cluster_arn in cluster_arns:
561
597
  cluster_instances = get_ecs_container_instances(
562
598
  cluster_arn,