cartography 0.102.0rc1__py3-none-any.whl → 0.103.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

Files changed (297) hide show
  1. cartography/__main__.py +1 -2
  2. cartography/_version.py +2 -2
  3. cartography/cli.py +376 -249
  4. cartography/client/core/tx.py +39 -18
  5. cartography/config.py +28 -0
  6. cartography/driftdetect/__main__.py +1 -2
  7. cartography/driftdetect/add_shortcut.py +10 -2
  8. cartography/driftdetect/cli.py +71 -75
  9. cartography/driftdetect/detect_deviations.py +7 -3
  10. cartography/driftdetect/get_states.py +20 -8
  11. cartography/driftdetect/model.py +5 -5
  12. cartography/driftdetect/serializers.py +8 -6
  13. cartography/driftdetect/storage.py +2 -2
  14. cartography/graph/cleanupbuilder.py +35 -15
  15. cartography/graph/job.py +46 -17
  16. cartography/graph/querybuilder.py +165 -80
  17. cartography/graph/statement.py +35 -26
  18. cartography/intel/analysis.py +4 -1
  19. cartography/intel/aws/__init__.py +114 -55
  20. cartography/intel/aws/apigateway.py +134 -63
  21. cartography/intel/aws/cloudtrail.py +127 -0
  22. cartography/intel/aws/cloudwatch.py +93 -0
  23. cartography/intel/aws/config.py +56 -20
  24. cartography/intel/aws/dynamodb.py +108 -40
  25. cartography/intel/aws/ec2/__init__.py +2 -2
  26. cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
  27. cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
  28. cartography/intel/aws/ec2/images.py +49 -20
  29. cartography/intel/aws/ec2/instances.py +234 -136
  30. cartography/intel/aws/ec2/internet_gateways.py +40 -11
  31. cartography/intel/aws/ec2/key_pairs.py +44 -20
  32. cartography/intel/aws/ec2/launch_templates.py +101 -59
  33. cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
  34. cartography/intel/aws/ec2/load_balancers.py +82 -42
  35. cartography/intel/aws/ec2/network_acls.py +89 -65
  36. cartography/intel/aws/ec2/network_interfaces.py +146 -87
  37. cartography/intel/aws/ec2/reserved_instances.py +45 -16
  38. cartography/intel/aws/ec2/route_tables.py +327 -0
  39. cartography/intel/aws/ec2/security_groups.py +71 -21
  40. cartography/intel/aws/ec2/snapshots.py +61 -22
  41. cartography/intel/aws/ec2/subnets.py +54 -18
  42. cartography/intel/aws/ec2/tgw.py +100 -34
  43. cartography/intel/aws/ec2/util.py +1 -1
  44. cartography/intel/aws/ec2/volumes.py +69 -41
  45. cartography/intel/aws/ec2/vpc.py +37 -12
  46. cartography/intel/aws/ec2/vpc_peerings.py +83 -24
  47. cartography/intel/aws/ecr.py +88 -32
  48. cartography/intel/aws/ecs.py +83 -47
  49. cartography/intel/aws/efs.py +93 -0
  50. cartography/intel/aws/eks.py +55 -29
  51. cartography/intel/aws/elasticache.py +42 -18
  52. cartography/intel/aws/elasticsearch.py +57 -20
  53. cartography/intel/aws/emr.py +61 -23
  54. cartography/intel/aws/iam.py +401 -145
  55. cartography/intel/aws/iam_instance_profiles.py +22 -22
  56. cartography/intel/aws/identitycenter.py +71 -37
  57. cartography/intel/aws/inspector.py +159 -89
  58. cartography/intel/aws/kms.py +92 -38
  59. cartography/intel/aws/lambda_function.py +103 -34
  60. cartography/intel/aws/organizations.py +30 -10
  61. cartography/intel/aws/permission_relationships.py +133 -51
  62. cartography/intel/aws/rds.py +249 -85
  63. cartography/intel/aws/redshift.py +107 -46
  64. cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
  65. cartography/intel/aws/resources.py +57 -44
  66. cartography/intel/aws/route53.py +108 -61
  67. cartography/intel/aws/s3.py +168 -83
  68. cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
  69. cartography/intel/aws/secretsmanager.py +24 -12
  70. cartography/intel/aws/securityhub.py +20 -9
  71. cartography/intel/aws/sns.py +166 -0
  72. cartography/intel/aws/sqs.py +60 -28
  73. cartography/intel/aws/ssm.py +70 -30
  74. cartography/intel/aws/util/arns.py +7 -7
  75. cartography/intel/aws/util/common.py +31 -4
  76. cartography/intel/azure/__init__.py +78 -19
  77. cartography/intel/azure/compute.py +101 -27
  78. cartography/intel/azure/cosmosdb.py +496 -170
  79. cartography/intel/azure/sql.py +296 -105
  80. cartography/intel/azure/storage.py +322 -113
  81. cartography/intel/azure/subscription.py +39 -23
  82. cartography/intel/azure/tenant.py +13 -4
  83. cartography/intel/azure/util/credentials.py +95 -55
  84. cartography/intel/bigfix/__init__.py +2 -2
  85. cartography/intel/bigfix/computers.py +93 -65
  86. cartography/intel/cloudflare/__init__.py +74 -0
  87. cartography/intel/cloudflare/accounts.py +57 -0
  88. cartography/intel/cloudflare/dnsrecords.py +64 -0
  89. cartography/intel/cloudflare/members.py +75 -0
  90. cartography/intel/cloudflare/roles.py +65 -0
  91. cartography/intel/cloudflare/zones.py +64 -0
  92. cartography/intel/create_indexes.py +3 -2
  93. cartography/intel/crowdstrike/__init__.py +11 -9
  94. cartography/intel/crowdstrike/endpoints.py +5 -1
  95. cartography/intel/crowdstrike/spotlight.py +8 -3
  96. cartography/intel/cve/__init__.py +46 -13
  97. cartography/intel/cve/feed.py +48 -12
  98. cartography/intel/digitalocean/__init__.py +22 -13
  99. cartography/intel/digitalocean/compute.py +75 -108
  100. cartography/intel/digitalocean/management.py +44 -80
  101. cartography/intel/digitalocean/platform.py +48 -43
  102. cartography/intel/dns.py +36 -10
  103. cartography/intel/duo/__init__.py +21 -16
  104. cartography/intel/duo/api_host.py +14 -9
  105. cartography/intel/duo/endpoints.py +50 -45
  106. cartography/intel/duo/groups.py +18 -14
  107. cartography/intel/duo/phones.py +37 -34
  108. cartography/intel/duo/tokens.py +26 -23
  109. cartography/intel/duo/users.py +54 -50
  110. cartography/intel/duo/web_authn_credentials.py +30 -25
  111. cartography/intel/entra/__init__.py +25 -7
  112. cartography/intel/entra/ou.py +112 -0
  113. cartography/intel/entra/users.py +69 -63
  114. cartography/intel/gcp/__init__.py +185 -49
  115. cartography/intel/gcp/compute.py +418 -231
  116. cartography/intel/gcp/crm.py +96 -43
  117. cartography/intel/gcp/dns.py +60 -19
  118. cartography/intel/gcp/gke.py +72 -38
  119. cartography/intel/gcp/iam.py +61 -41
  120. cartography/intel/gcp/storage.py +84 -55
  121. cartography/intel/github/__init__.py +13 -11
  122. cartography/intel/github/repos.py +270 -137
  123. cartography/intel/github/teams.py +170 -88
  124. cartography/intel/github/users.py +70 -39
  125. cartography/intel/github/util.py +36 -34
  126. cartography/intel/gsuite/__init__.py +47 -26
  127. cartography/intel/gsuite/api.py +73 -30
  128. cartography/intel/jamf/__init__.py +19 -1
  129. cartography/intel/jamf/computers.py +30 -7
  130. cartography/intel/jamf/util.py +7 -2
  131. cartography/intel/kandji/__init__.py +6 -3
  132. cartography/intel/kandji/devices.py +14 -8
  133. cartography/intel/kubernetes/namespaces.py +7 -4
  134. cartography/intel/kubernetes/pods.py +7 -4
  135. cartography/intel/kubernetes/services.py +8 -4
  136. cartography/intel/lastpass/__init__.py +2 -2
  137. cartography/intel/lastpass/users.py +23 -12
  138. cartography/intel/oci/__init__.py +44 -11
  139. cartography/intel/oci/iam.py +134 -38
  140. cartography/intel/oci/organizations.py +13 -6
  141. cartography/intel/oci/utils.py +43 -20
  142. cartography/intel/okta/__init__.py +66 -15
  143. cartography/intel/okta/applications.py +42 -20
  144. cartography/intel/okta/awssaml.py +93 -33
  145. cartography/intel/okta/factors.py +16 -4
  146. cartography/intel/okta/groups.py +56 -29
  147. cartography/intel/okta/organization.py +5 -1
  148. cartography/intel/okta/origins.py +6 -2
  149. cartography/intel/okta/roles.py +15 -5
  150. cartography/intel/okta/users.py +20 -8
  151. cartography/intel/okta/utils.py +6 -4
  152. cartography/intel/openai/__init__.py +86 -0
  153. cartography/intel/openai/adminapikeys.py +90 -0
  154. cartography/intel/openai/apikeys.py +96 -0
  155. cartography/intel/openai/projects.py +94 -0
  156. cartography/intel/openai/serviceaccounts.py +82 -0
  157. cartography/intel/openai/users.py +78 -0
  158. cartography/intel/openai/util.py +29 -0
  159. cartography/intel/pagerduty/__init__.py +8 -7
  160. cartography/intel/pagerduty/escalation_policies.py +18 -6
  161. cartography/intel/pagerduty/schedules.py +12 -4
  162. cartography/intel/pagerduty/services.py +11 -4
  163. cartography/intel/pagerduty/teams.py +8 -3
  164. cartography/intel/pagerduty/users.py +3 -1
  165. cartography/intel/pagerduty/vendors.py +3 -1
  166. cartography/intel/semgrep/__init__.py +24 -6
  167. cartography/intel/semgrep/dependencies.py +50 -28
  168. cartography/intel/semgrep/deployment.py +3 -1
  169. cartography/intel/semgrep/findings.py +42 -18
  170. cartography/intel/snipeit/__init__.py +17 -3
  171. cartography/intel/snipeit/asset.py +12 -6
  172. cartography/intel/snipeit/user.py +8 -5
  173. cartography/intel/snipeit/util.py +9 -4
  174. cartography/intel/tailscale/__init__.py +77 -0
  175. cartography/intel/tailscale/acls.py +146 -0
  176. cartography/intel/tailscale/devices.py +127 -0
  177. cartography/intel/tailscale/postureintegrations.py +81 -0
  178. cartography/intel/tailscale/tailnets.py +76 -0
  179. cartography/intel/tailscale/users.py +80 -0
  180. cartography/intel/tailscale/utils.py +132 -0
  181. cartography/models/aws/apigateway.py +21 -17
  182. cartography/models/aws/apigatewaycertificate.py +28 -22
  183. cartography/models/aws/apigatewayresource.py +28 -20
  184. cartography/models/aws/apigatewaystage.py +33 -25
  185. cartography/models/aws/cloudtrail/__init__.py +0 -0
  186. cartography/models/aws/cloudtrail/trail.py +61 -0
  187. cartography/models/aws/cloudwatch/__init__.py +0 -0
  188. cartography/models/aws/cloudwatch/loggroup.py +52 -0
  189. cartography/models/aws/dynamodb/gsi.py +30 -22
  190. cartography/models/aws/dynamodb/tables.py +25 -17
  191. cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
  192. cartography/models/aws/ec2/images.py +36 -34
  193. cartography/models/aws/ec2/instances.py +51 -45
  194. cartography/models/aws/ec2/keypair.py +21 -16
  195. cartography/models/aws/ec2/keypair_instance.py +28 -21
  196. cartography/models/aws/ec2/launch_configurations.py +30 -26
  197. cartography/models/aws/ec2/launch_template_versions.py +48 -38
  198. cartography/models/aws/ec2/launch_templates.py +21 -17
  199. cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
  200. cartography/models/aws/ec2/load_balancers.py +47 -37
  201. cartography/models/aws/ec2/network_acl_rules.py +38 -30
  202. cartography/models/aws/ec2/network_acls.py +38 -29
  203. cartography/models/aws/ec2/networkinterface_instance.py +52 -39
  204. cartography/models/aws/ec2/networkinterfaces.py +53 -37
  205. cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
  206. cartography/models/aws/ec2/reservations.py +18 -14
  207. cartography/models/aws/ec2/route_table_associations.py +97 -0
  208. cartography/models/aws/ec2/route_tables.py +128 -0
  209. cartography/models/aws/ec2/routes.py +85 -0
  210. cartography/models/aws/ec2/securitygroup_instance.py +29 -20
  211. cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
  212. cartography/models/aws/ec2/subnet_instance.py +24 -19
  213. cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
  214. cartography/models/aws/ec2/volumes.py +47 -40
  215. cartography/models/aws/efs/__init__.py +0 -0
  216. cartography/models/aws/efs/mount_target.py +52 -0
  217. cartography/models/aws/eks/clusters.py +23 -21
  218. cartography/models/aws/emr.py +32 -30
  219. cartography/models/aws/iam/instanceprofile.py +33 -24
  220. cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
  221. cartography/models/aws/identitycenter/awspermissionset.py +37 -29
  222. cartography/models/aws/identitycenter/awsssouser.py +23 -21
  223. cartography/models/aws/inspector/findings.py +77 -65
  224. cartography/models/aws/inspector/packages.py +35 -29
  225. cartography/models/aws/s3/__init__.py +0 -0
  226. cartography/models/aws/s3/account_public_access_block.py +51 -0
  227. cartography/models/aws/sns/__init__.py +0 -0
  228. cartography/models/aws/sns/topic.py +50 -0
  229. cartography/models/aws/ssm/instance_information.py +51 -39
  230. cartography/models/aws/ssm/instance_patch.py +32 -26
  231. cartography/models/bigfix/bigfix_computer.py +42 -38
  232. cartography/models/bigfix/bigfix_root.py +3 -3
  233. cartography/models/cloudflare/__init__.py +0 -0
  234. cartography/models/cloudflare/account.py +25 -0
  235. cartography/models/cloudflare/dnsrecord.py +55 -0
  236. cartography/models/cloudflare/member.py +82 -0
  237. cartography/models/cloudflare/role.py +44 -0
  238. cartography/models/cloudflare/zone.py +59 -0
  239. cartography/models/core/common.py +12 -10
  240. cartography/models/core/nodes.py +5 -2
  241. cartography/models/core/relationships.py +14 -6
  242. cartography/models/crowdstrike/hosts.py +37 -35
  243. cartography/models/cve/cve.py +34 -32
  244. cartography/models/cve/cve_feed.py +6 -6
  245. cartography/models/digitalocean/__init__.py +0 -0
  246. cartography/models/digitalocean/account.py +21 -0
  247. cartography/models/digitalocean/droplet.py +56 -0
  248. cartography/models/digitalocean/project.py +48 -0
  249. cartography/models/duo/api_host.py +3 -3
  250. cartography/models/duo/endpoint.py +43 -41
  251. cartography/models/duo/group.py +14 -14
  252. cartography/models/duo/phone.py +27 -27
  253. cartography/models/duo/token.py +16 -16
  254. cartography/models/duo/user.py +46 -44
  255. cartography/models/duo/web_authn_credential.py +27 -19
  256. cartography/models/entra/ou.py +48 -0
  257. cartography/models/entra/tenant.py +24 -18
  258. cartography/models/entra/user.py +64 -48
  259. cartography/models/gcp/iam.py +23 -23
  260. cartography/models/github/orgs.py +5 -4
  261. cartography/models/github/teams.py +37 -31
  262. cartography/models/github/users.py +34 -23
  263. cartography/models/kandji/device.py +22 -16
  264. cartography/models/kandji/tenant.py +6 -4
  265. cartography/models/lastpass/tenant.py +3 -3
  266. cartography/models/lastpass/user.py +32 -28
  267. cartography/models/openai/__init__.py +0 -0
  268. cartography/models/openai/adminapikey.py +90 -0
  269. cartography/models/openai/apikey.py +84 -0
  270. cartography/models/openai/organization.py +17 -0
  271. cartography/models/openai/project.py +70 -0
  272. cartography/models/openai/serviceaccount.py +50 -0
  273. cartography/models/openai/user.py +49 -0
  274. cartography/models/semgrep/dependencies.py +36 -24
  275. cartography/models/semgrep/deployment.py +5 -5
  276. cartography/models/semgrep/findings.py +58 -42
  277. cartography/models/semgrep/locations.py +27 -21
  278. cartography/models/snipeit/asset.py +30 -21
  279. cartography/models/snipeit/tenant.py +6 -4
  280. cartography/models/snipeit/user.py +19 -12
  281. cartography/models/tailscale/__init__.py +0 -0
  282. cartography/models/tailscale/device.py +95 -0
  283. cartography/models/tailscale/group.py +86 -0
  284. cartography/models/tailscale/postureintegration.py +58 -0
  285. cartography/models/tailscale/tag.py +102 -0
  286. cartography/models/tailscale/tailnet.py +29 -0
  287. cartography/models/tailscale/user.py +52 -0
  288. cartography/stats.py +3 -3
  289. cartography/sync.py +113 -31
  290. cartography/util.py +84 -62
  291. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/METADATA +8 -15
  292. cartography-0.103.0.dist-info/RECORD +442 -0
  293. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/WHEEL +1 -1
  294. cartography-0.102.0rc1.dist-info/RECORD +0 -377
  295. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/entry_points.txt +0 -0
  296. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/licenses/LICENSE +0 -0
  297. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/top_level.txt +0 -0
@@ -18,7 +18,7 @@ from cartography.util import run_cleanup_job
18
18
  from cartography.util import timeit
19
19
 
20
20
  logger = logging.getLogger(__name__)
21
- InstanceUriPrefix = namedtuple('InstanceUriPrefix', 'zone_name project_id')
21
+ InstanceUriPrefix = namedtuple("InstanceUriPrefix", "zone_name project_id")
22
22
 
23
23
 
24
24
  def _get_error_reason(http_error: HttpError) -> str:
@@ -33,19 +33,23 @@ def _get_error_reason(http_error: HttpError) -> str:
33
33
  :return: The error reason as a string
34
34
  """
35
35
  try:
36
- data = json.loads(http_error.content.decode('utf-8'))
36
+ data = json.loads(http_error.content.decode("utf-8"))
37
37
  if isinstance(data, dict):
38
- reason = data['error']['errors'][0]['reason']
38
+ reason = data["error"]["errors"][0]["reason"]
39
39
  else:
40
- reason = data[0]['error']['errors']['reason']
40
+ reason = data[0]["error"]["errors"]["reason"]
41
41
  except (UnicodeDecodeError, ValueError, KeyError):
42
42
  logger.warning(f"HttpError: {data}")
43
- return ''
43
+ return ""
44
44
  return reason
45
45
 
46
46
 
47
47
  @timeit
48
- def get_zones_in_project(project_id: str, compute: Resource, max_results: Optional[int] = None) -> Optional[List[Dict]]:
48
+ def get_zones_in_project(
49
+ project_id: str,
50
+ compute: Resource,
51
+ max_results: Optional[int] = None,
52
+ ) -> Optional[List[Dict]]:
49
53
  """
50
54
  Return the zones where the Compute Engine API is enabled for the given project_id.
51
55
  See https://cloud.google.com/compute/docs/reference/rest/v1/zones and
@@ -60,10 +64,10 @@ def get_zones_in_project(project_id: str, compute: Resource, max_results: Option
60
64
  try:
61
65
  req = compute.zones().list(project=project_id, maxResults=max_results)
62
66
  res = req.execute()
63
- return res['items']
67
+ return res["items"]
64
68
  except HttpError as e:
65
69
  reason = _get_error_reason(e)
66
- if reason == 'accessNotConfigured':
70
+ if reason == "accessNotConfigured":
67
71
  logger.info(
68
72
  (
69
73
  "Google Compute Engine API access is not configured for project %s; skipping. "
@@ -73,17 +77,14 @@ def get_zones_in_project(project_id: str, compute: Resource, max_results: Option
73
77
  e,
74
78
  )
75
79
  return None
76
- elif reason == 'notFound':
80
+ elif reason == "notFound":
77
81
  logger.info(
78
- (
79
- "Project %s returned a 404 not found error. "
80
- "Full details: %s"
81
- ),
82
+ ("Project %s returned a 404 not found error. " "Full details: %s"),
82
83
  project_id,
83
84
  e,
84
85
  )
85
86
  return None
86
- elif reason == 'forbidden':
87
+ elif reason == "forbidden":
87
88
  logger.info(
88
89
  (
89
90
  "Your GCP identity does not have the compute.zones.list permission for project %s; skipping "
@@ -98,7 +99,11 @@ def get_zones_in_project(project_id: str, compute: Resource, max_results: Option
98
99
 
99
100
 
100
101
  @timeit
101
- def get_gcp_instance_responses(project_id: str, zones: Optional[List[Dict]], compute: Resource) -> List[Resource]:
102
+ def get_gcp_instance_responses(
103
+ project_id: str,
104
+ zones: Optional[List[Dict]],
105
+ compute: Resource,
106
+ ) -> List[Resource]:
102
107
  """
103
108
  Return list of GCP instance response objects for a given project and list of zones
104
109
  :param project_id: The project ID
@@ -111,7 +116,7 @@ def get_gcp_instance_responses(project_id: str, zones: Optional[List[Dict]], com
111
116
  return []
112
117
  response_objects: List[Resource] = []
113
118
  for zone in zones:
114
- req = compute.instances().list(project=project_id, zone=zone['name'])
119
+ req = compute.instances().list(project=project_id, zone=zone["name"])
115
120
  res = req.execute()
116
121
  response_objects.append(res)
117
122
  return response_objects
@@ -143,7 +148,11 @@ def get_gcp_vpcs(projectid: str, compute: Resource) -> Resource:
143
148
 
144
149
 
145
150
  @timeit
146
- def get_gcp_regional_forwarding_rules(project_id: str, region: str, compute: Resource) -> Resource:
151
+ def get_gcp_regional_forwarding_rules(
152
+ project_id: str,
153
+ region: str,
154
+ compute: Resource,
155
+ ) -> Resource:
147
156
  """
148
157
  Return list of all regional forwarding rules in the given project_id and region
149
158
  :param project_id: The project ID
@@ -189,17 +198,21 @@ def transform_gcp_instances(response_objects: List[Dict]) -> List[Dict]:
189
198
  """
190
199
  instance_list = []
191
200
  for res in response_objects:
192
- prefix = res['id']
201
+ prefix = res["id"]
193
202
  prefix_fields = _parse_instance_uri_prefix(prefix)
194
203
 
195
- for instance in res.get('items', []):
196
- instance['partial_uri'] = f"{prefix}/{instance['name']}"
197
- instance['project_id'] = prefix_fields.project_id
198
- instance['zone_name'] = prefix_fields.zone_name
204
+ for instance in res.get("items", []):
205
+ instance["partial_uri"] = f"{prefix}/{instance['name']}"
206
+ instance["project_id"] = prefix_fields.project_id
207
+ instance["zone_name"] = prefix_fields.zone_name
199
208
 
200
- for nic in instance.get('networkInterfaces', []):
201
- nic['subnet_partial_uri'] = _parse_compute_full_uri_to_partial_uri(nic['subnetwork'])
202
- nic['vpc_partial_uri'] = _parse_compute_full_uri_to_partial_uri(nic['network'])
209
+ for nic in instance.get("networkInterfaces", []):
210
+ nic["subnet_partial_uri"] = _parse_compute_full_uri_to_partial_uri(
211
+ nic["subnetwork"],
212
+ )
213
+ nic["vpc_partial_uri"] = _parse_compute_full_uri_to_partial_uri(
214
+ nic["network"],
215
+ )
203
216
 
204
217
  instance_list.append(instance)
205
218
  return instance_list
@@ -211,7 +224,7 @@ def _parse_instance_uri_prefix(prefix: str) -> InstanceUriPrefix:
211
224
  :param prefix: String of the form `projects/{project}/zones/{zone}/instances`
212
225
  :return: namedtuple with fields project_id and zone_name
213
226
  """
214
- split_list = prefix.split('/')
227
+ split_list = prefix.split("/")
215
228
 
216
229
  return InstanceUriPrefix(
217
230
  project_id=split_list[1],
@@ -219,7 +232,7 @@ def _parse_instance_uri_prefix(prefix: str) -> InstanceUriPrefix:
219
232
  )
220
233
 
221
234
 
222
- def _parse_compute_full_uri_to_partial_uri(full_uri: str, version: str = 'v1') -> str:
235
+ def _parse_compute_full_uri_to_partial_uri(full_uri: str, version: str = "v1") -> str:
223
236
  """
224
237
  Take a GCP Compute object's self_link of the form
225
238
  `https://www.googleapis.com/compute/{version}/projects/{project}/{location specifier}/{subtype}/{resource name}`
@@ -229,7 +242,7 @@ def _parse_compute_full_uri_to_partial_uri(full_uri: str, version: str = 'v1') -
229
242
  :param version: The version number; default to v1 since at the time of this writing v1 is the only Compute API.
230
243
  :return: Partial URI `{project}/{location specifier}/{subtype}/{resource name}`
231
244
  """
232
- return full_uri.split(f'compute/{version}/')[1]
245
+ return full_uri.split(f"compute/{version}/")[1]
233
246
 
234
247
 
235
248
  def _create_gcp_network_tag_id(vpc_partial_uri: str, tag: str) -> str:
@@ -251,19 +264,22 @@ def transform_gcp_vpcs(vpc_res: Dict) -> List[Dict]:
251
264
  vpc_list = []
252
265
 
253
266
  # prefix has the form `projects/{project ID}/global/networks`
254
- prefix = vpc_res['id']
255
- projectid = prefix.split('/')[1]
256
- for v in vpc_res.get('items', []):
267
+ prefix = vpc_res["id"]
268
+ projectid = prefix.split("/")[1]
269
+ for v in vpc_res.get("items", []):
257
270
  vpc = {}
258
271
  partial_uri = f"{prefix}/{v['name']}"
259
272
 
260
- vpc['partial_uri'] = partial_uri
261
- vpc['name'] = v['name']
262
- vpc['self_link'] = v['selfLink']
263
- vpc['project_id'] = projectid
264
- vpc['auto_create_subnetworks'] = v.get('autoCreateSubnetworks', None)
265
- vpc['description'] = v.get('description', None)
266
- vpc['routing_config_routing_mode'] = v.get('routingConfig', {}).get('routingMode', None)
273
+ vpc["partial_uri"] = partial_uri
274
+ vpc["name"] = v["name"]
275
+ vpc["self_link"] = v["selfLink"]
276
+ vpc["project_id"] = projectid
277
+ vpc["auto_create_subnetworks"] = v.get("autoCreateSubnetworks", None)
278
+ vpc["description"] = v.get("description", None)
279
+ vpc["routing_config_routing_mode"] = v.get("routingConfig", {}).get(
280
+ "routingMode",
281
+ None,
282
+ )
267
283
 
268
284
  vpc_list.append(vpc)
269
285
  return vpc_list
@@ -278,29 +294,29 @@ def transform_gcp_subnets(subnet_res: Dict) -> List[Dict]:
278
294
  """
279
295
  # The `id` in the response object has the form `projects/{project}/regions/{region}/subnetworks`.
280
296
  # We can include this in each subnet object in the list to form the partial_uri later on.
281
- prefix = subnet_res['id']
282
- projectid = prefix.split('/')[1]
297
+ prefix = subnet_res["id"]
298
+ projectid = prefix.split("/")[1]
283
299
  subnet_list: List[Dict] = []
284
- for s in subnet_res.get('items', []):
300
+ for s in subnet_res.get("items", []):
285
301
  subnet = {}
286
302
 
287
303
  # Has the form `projects/{project}/regions/{region}/subnetworks/{subnet_name}`
288
304
  partial_uri = f"{prefix}/{s['name']}"
289
- subnet['id'] = partial_uri
290
- subnet['partial_uri'] = partial_uri
305
+ subnet["id"] = partial_uri
306
+ subnet["partial_uri"] = partial_uri
291
307
 
292
308
  # Let's maintain an on-node reference to the VPC that this subnet belongs to.
293
- subnet['vpc_self_link'] = s['network']
294
- subnet['vpc_partial_uri'] = _parse_compute_full_uri_to_partial_uri(s['network'])
309
+ subnet["vpc_self_link"] = s["network"]
310
+ subnet["vpc_partial_uri"] = _parse_compute_full_uri_to_partial_uri(s["network"])
295
311
 
296
- subnet['name'] = s['name']
297
- subnet['project_id'] = projectid
312
+ subnet["name"] = s["name"]
313
+ subnet["project_id"] = projectid
298
314
  # Region looks like "https://www.googleapis.com/compute/v1/projects/{project}/regions/{region name}"
299
- subnet['region'] = s['region'].split('/')[-1]
300
- subnet['gateway_address'] = s.get('gatewayAddress', None)
301
- subnet['ip_cidr_range'] = s.get('ipCidrRange', None)
302
- subnet['self_link'] = s['selfLink']
303
- subnet['private_ip_google_access'] = s.get('privateIpGoogleAccess', None)
315
+ subnet["region"] = s["region"].split("/")[-1]
316
+ subnet["gateway_address"] = s.get("gatewayAddress", None)
317
+ subnet["ip_cidr_range"] = s.get("ipCidrRange", None)
318
+ subnet["self_link"] = s["selfLink"]
319
+ subnet["private_ip_google_access"] = s.get("privateIpGoogleAccess", None)
304
320
 
305
321
  subnet_list.append(subnet)
306
322
  return subnet_list
@@ -314,43 +330,47 @@ def transform_gcp_forwarding_rules(fwd_response: Resource) -> List[Dict]:
314
330
  :return: A transformed fwd_response
315
331
  """
316
332
  fwd_list: List[Dict] = []
317
- prefix = fwd_response['id']
318
- project_id = prefix.split('/')[1]
319
- for fwd in fwd_response.get('items', []):
333
+ prefix = fwd_response["id"]
334
+ project_id = prefix.split("/")[1]
335
+ for fwd in fwd_response.get("items", []):
320
336
  forwarding_rule: Dict[str, Any] = {}
321
337
 
322
338
  fwd_partial_uri = f"{prefix}/{fwd['name']}"
323
- forwarding_rule['id'] = fwd_partial_uri
324
- forwarding_rule['partial_uri'] = fwd_partial_uri
339
+ forwarding_rule["id"] = fwd_partial_uri
340
+ forwarding_rule["partial_uri"] = fwd_partial_uri
325
341
 
326
- forwarding_rule['project_id'] = project_id
342
+ forwarding_rule["project_id"] = project_id
327
343
  # Region looks like "https://www.googleapis.com/compute/v1/projects/{project}/regions/{region name}"
328
- region = fwd.get('region', None)
329
- forwarding_rule['region'] = region.split('/')[-1] if region else None
330
- forwarding_rule['ip_address'] = fwd.get('IPAddress', None)
331
- forwarding_rule['ip_protocol'] = fwd.get('IPProtocol', None)
332
- forwarding_rule['allow_global_access'] = fwd.get('allowGlobalAccess', None)
333
-
334
- forwarding_rule['load_balancing_scheme'] = fwd.get('loadBalancingScheme', None)
335
- forwarding_rule['name'] = fwd.get('name', None)
336
- forwarding_rule['port_range'] = fwd.get('portRange', None)
337
- forwarding_rule['ports'] = fwd.get('ports', None)
338
- forwarding_rule['self_link'] = fwd.get('selfLink', None)
339
- target = fwd.get('target', None)
344
+ region = fwd.get("region", None)
345
+ forwarding_rule["region"] = region.split("/")[-1] if region else None
346
+ forwarding_rule["ip_address"] = fwd.get("IPAddress", None)
347
+ forwarding_rule["ip_protocol"] = fwd.get("IPProtocol", None)
348
+ forwarding_rule["allow_global_access"] = fwd.get("allowGlobalAccess", None)
349
+
350
+ forwarding_rule["load_balancing_scheme"] = fwd.get("loadBalancingScheme", None)
351
+ forwarding_rule["name"] = fwd.get("name", None)
352
+ forwarding_rule["port_range"] = fwd.get("portRange", None)
353
+ forwarding_rule["ports"] = fwd.get("ports", None)
354
+ forwarding_rule["self_link"] = fwd.get("selfLink", None)
355
+ target = fwd.get("target", None)
340
356
  if target:
341
- forwarding_rule['target'] = _parse_compute_full_uri_to_partial_uri(target)
357
+ forwarding_rule["target"] = _parse_compute_full_uri_to_partial_uri(target)
342
358
  else:
343
- forwarding_rule['target'] = None
359
+ forwarding_rule["target"] = None
344
360
 
345
- network = fwd.get('network', None)
361
+ network = fwd.get("network", None)
346
362
  if network:
347
- forwarding_rule['network'] = network
348
- forwarding_rule['network_partial_uri'] = _parse_compute_full_uri_to_partial_uri(network)
363
+ forwarding_rule["network"] = network
364
+ forwarding_rule["network_partial_uri"] = (
365
+ _parse_compute_full_uri_to_partial_uri(network)
366
+ )
349
367
 
350
- subnetwork = fwd.get('subnetwork', None)
368
+ subnetwork = fwd.get("subnetwork", None)
351
369
  if subnetwork:
352
- forwarding_rule['subnetwork'] = subnetwork
353
- forwarding_rule['subnetwork_partial_uri'] = _parse_compute_full_uri_to_partial_uri(subnetwork)
370
+ forwarding_rule["subnetwork"] = subnetwork
371
+ forwarding_rule["subnetwork_partial_uri"] = (
372
+ _parse_compute_full_uri_to_partial_uri(subnetwork)
373
+ )
354
374
 
355
375
  fwd_list.append(forwarding_rule)
356
376
  return fwd_list
@@ -365,31 +385,45 @@ def transform_gcp_firewall(fw_response: Resource) -> List[Dict]:
365
385
  :return: List of transformed firewall rule objects.
366
386
  """
367
387
  fw_list: List[Dict] = []
368
- prefix = fw_response['id']
369
- for fw in fw_response.get('items', []):
388
+ prefix = fw_response["id"]
389
+ for fw in fw_response.get("items", []):
370
390
  fw_partial_uri = f"{prefix}/{fw['name']}"
371
- fw['id'] = fw_partial_uri
372
- fw['vpc_partial_uri'] = _parse_compute_full_uri_to_partial_uri(fw['network'])
391
+ fw["id"] = fw_partial_uri
392
+ fw["vpc_partial_uri"] = _parse_compute_full_uri_to_partial_uri(fw["network"])
373
393
 
374
- fw['transformed_allow_list'] = []
375
- fw['transformed_deny_list'] = []
394
+ fw["transformed_allow_list"] = []
395
+ fw["transformed_deny_list"] = []
376
396
  # Mark whether this FW is defined on a target service account.
377
397
  # In future we will need to ingest GCP IAM objects but for now we simply mark the presence of svc accounts here.
378
- fw['has_target_service_accounts'] = True if 'targetServiceAccounts' in fw else False
398
+ fw["has_target_service_accounts"] = (
399
+ True if "targetServiceAccounts" in fw else False
400
+ )
379
401
 
380
- for allow_rule in fw.get('allowed', []):
381
- transformed_allow_rules = _transform_fw_entry(allow_rule, fw_partial_uri, is_allow_rule=True)
382
- fw['transformed_allow_list'].extend(transformed_allow_rules)
402
+ for allow_rule in fw.get("allowed", []):
403
+ transformed_allow_rules = _transform_fw_entry(
404
+ allow_rule,
405
+ fw_partial_uri,
406
+ is_allow_rule=True,
407
+ )
408
+ fw["transformed_allow_list"].extend(transformed_allow_rules)
383
409
 
384
- for deny_rule in fw.get('denied', []):
385
- transformed_deny_rules = _transform_fw_entry(deny_rule, fw_partial_uri, is_allow_rule=False)
386
- fw['transformed_deny_list'].extend(transformed_deny_rules)
410
+ for deny_rule in fw.get("denied", []):
411
+ transformed_deny_rules = _transform_fw_entry(
412
+ deny_rule,
413
+ fw_partial_uri,
414
+ is_allow_rule=False,
415
+ )
416
+ fw["transformed_deny_list"].extend(transformed_deny_rules)
387
417
 
388
418
  fw_list.append(fw)
389
419
  return fw_list
390
420
 
391
421
 
392
- def _transform_fw_entry(rule: Dict, fw_partial_uri: str, is_allow_rule: bool) -> List[Dict]:
422
+ def _transform_fw_entry(
423
+ rule: Dict,
424
+ fw_partial_uri: str,
425
+ is_allow_rule: bool,
426
+ ) -> List[Dict]:
393
427
  """
394
428
  Takes a rule entry from a GCP firewall object's allow or deny list and converts it to a list of one or more
395
429
  dicts representing a firewall rule for each port and port range. This format is easier to load into Neo4j.
@@ -415,21 +449,31 @@ def _transform_fw_entry(rule: Dict, fw_partial_uri: str, is_allow_rule: bool) ->
415
449
  """
416
450
  result: List[Dict] = []
417
451
  # rule['ruleid'] = f"{fw_partial_uri}/"
418
- protocol = rule['IPProtocol']
452
+ protocol = rule["IPProtocol"]
419
453
 
420
454
  # If the protocol covered is TCP or UDP then we need to handle ports
421
- if protocol == 'tcp' or protocol == 'udp':
455
+ if protocol == "tcp" or protocol == "udp":
422
456
 
423
457
  # If ports are specified then create rules for each port and range
424
- if 'ports' in rule:
425
- for port in rule['ports']:
426
- rule = _parse_port_string_to_rule(port, protocol, fw_partial_uri, is_allow_rule)
458
+ if "ports" in rule:
459
+ for port in rule["ports"]:
460
+ rule = _parse_port_string_to_rule(
461
+ port,
462
+ protocol,
463
+ fw_partial_uri,
464
+ is_allow_rule,
465
+ )
427
466
  result.append(rule)
428
467
  return result
429
468
 
430
469
  # If ports are not specified then the rule applies to every port
431
470
  else:
432
- rule = _parse_port_string_to_rule('0-65535', protocol, fw_partial_uri, is_allow_rule)
471
+ rule = _parse_port_string_to_rule(
472
+ "0-65535",
473
+ protocol,
474
+ fw_partial_uri,
475
+ is_allow_rule,
476
+ )
433
477
  result.append(rule)
434
478
  return result
435
479
 
@@ -440,7 +484,12 @@ def _transform_fw_entry(rule: Dict, fw_partial_uri: str, is_allow_rule: bool) ->
440
484
  return result
441
485
 
442
486
 
443
- def _parse_port_string_to_rule(port: Optional[str], protocol: str, fw_partial_uri: str, is_allow_rule: bool) -> Dict:
487
+ def _parse_port_string_to_rule(
488
+ port: Optional[str],
489
+ protocol: str,
490
+ fw_partial_uri: str,
491
+ is_allow_rule: bool,
492
+ ) -> Dict:
444
493
  """
445
494
  Takes a string argument representing a GCP firewall rule port or port range and returns a dict that is easier to
446
495
  load into Neo4j.
@@ -467,13 +516,13 @@ def _parse_port_string_to_rule(port: Optional[str], protocol: str, fw_partial_ur
467
516
 
468
517
  if port is None:
469
518
  # Keep the port range as the empty string
470
- port_range_str = ''
519
+ port_range_str = ""
471
520
  fromport = None
472
521
  toport = None
473
522
  else:
474
523
  # Case 1 - port range: '12345-12349'.split('-') => ['12345','12349'].
475
524
  # Case 2 - single port: '22'.split('-') => ['22'].
476
- port_split = port.split('-')
525
+ port_split = port.split("-")
477
526
 
478
527
  # Port range
479
528
  if len(port_split) == 2:
@@ -486,18 +535,22 @@ def _parse_port_string_to_rule(port: Optional[str], protocol: str, fw_partial_ur
486
535
  fromport = int(port_split[0])
487
536
  toport = int(port_split[0])
488
537
 
489
- rule_type = 'allow' if is_allow_rule else 'deny'
538
+ rule_type = "allow" if is_allow_rule else "deny"
490
539
 
491
540
  return {
492
- 'ruleid': f"{fw_partial_uri}/{rule_type}/{port_range_str}{protocol}",
493
- 'fromport': fromport,
494
- 'toport': toport,
495
- 'protocol': protocol,
541
+ "ruleid": f"{fw_partial_uri}/{rule_type}/{port_range_str}{protocol}",
542
+ "fromport": fromport,
543
+ "toport": toport,
544
+ "protocol": protocol,
496
545
  }
497
546
 
498
547
 
499
548
  @timeit
500
- def load_gcp_instances(neo4j_session: neo4j.Session, data: List[Dict], gcp_update_tag: int) -> None:
549
+ def load_gcp_instances(
550
+ neo4j_session: neo4j.Session,
551
+ data: List[Dict],
552
+ gcp_update_tag: int,
553
+ ) -> None:
501
554
  """
502
555
  Ingest GCP instance objects to Neo4j
503
556
  :param neo4j_session: The Neo4j session object
@@ -530,22 +583,26 @@ def load_gcp_instances(neo4j_session: neo4j.Session, data: List[Dict], gcp_updat
530
583
  for instance in data:
531
584
  neo4j_session.run(
532
585
  query,
533
- ProjectId=instance['project_id'],
534
- PartialUri=instance['partial_uri'],
535
- SelfLink=instance['selfLink'],
536
- InstanceName=instance['name'],
537
- ZoneName=instance['zone_name'],
538
- Hostname=instance.get('hostname', None),
539
- Status=instance['status'],
586
+ ProjectId=instance["project_id"],
587
+ PartialUri=instance["partial_uri"],
588
+ SelfLink=instance["selfLink"],
589
+ InstanceName=instance["name"],
590
+ ZoneName=instance["zone_name"],
591
+ Hostname=instance.get("hostname", None),
592
+ Status=instance["status"],
540
593
  gcp_update_tag=gcp_update_tag,
541
594
  )
542
595
  _attach_instance_tags(neo4j_session, instance, gcp_update_tag)
543
596
  _attach_gcp_nics(neo4j_session, instance, gcp_update_tag)
544
- _attach_gcp_vpc(neo4j_session, instance['partial_uri'], gcp_update_tag)
597
+ _attach_gcp_vpc(neo4j_session, instance["partial_uri"], gcp_update_tag)
545
598
 
546
599
 
547
600
  @timeit
548
- def load_gcp_vpcs(neo4j_session: neo4j.Session, vpcs: List[Dict], gcp_update_tag: int) -> None:
601
+ def load_gcp_vpcs(
602
+ neo4j_session: neo4j.Session,
603
+ vpcs: List[Dict],
604
+ gcp_update_tag: int,
605
+ ) -> None:
549
606
  """
550
607
  Ingest VPCs to Neo4j
551
608
  :param neo4j_session: The Neo4j session object
@@ -576,19 +633,23 @@ def load_gcp_vpcs(neo4j_session: neo4j.Session, vpcs: List[Dict], gcp_update_tag
576
633
  for vpc in vpcs:
577
634
  neo4j_session.run(
578
635
  query,
579
- ProjectId=vpc['project_id'],
580
- PartialUri=vpc['partial_uri'],
581
- SelfLink=vpc['self_link'],
582
- VpcName=vpc['name'],
583
- AutoCreateSubnetworks=vpc['auto_create_subnetworks'],
584
- RoutingMode=vpc['routing_config_routing_mode'],
585
- Description=vpc['description'],
636
+ ProjectId=vpc["project_id"],
637
+ PartialUri=vpc["partial_uri"],
638
+ SelfLink=vpc["self_link"],
639
+ VpcName=vpc["name"],
640
+ AutoCreateSubnetworks=vpc["auto_create_subnetworks"],
641
+ RoutingMode=vpc["routing_config_routing_mode"],
642
+ Description=vpc["description"],
586
643
  gcp_update_tag=gcp_update_tag,
587
644
  )
588
645
 
589
646
 
590
647
  @timeit
591
- def load_gcp_subnets(neo4j_session: neo4j.Session, subnets: List[Dict], gcp_update_tag: int) -> None:
648
+ def load_gcp_subnets(
649
+ neo4j_session: neo4j.Session,
650
+ subnets: List[Dict],
651
+ gcp_update_tag: int,
652
+ ) -> None:
592
653
  """
593
654
  Ingest GCP subnet data to Neo4j
594
655
  :param neo4j_session: The Neo4j session
@@ -621,22 +682,26 @@ def load_gcp_subnets(neo4j_session: neo4j.Session, subnets: List[Dict], gcp_upda
621
682
  for s in subnets:
622
683
  neo4j_session.run(
623
684
  query,
624
- VpcPartialUri=s['vpc_partial_uri'],
625
- VpcSelfLink=s['vpc_self_link'],
626
- PartialUri=s['partial_uri'],
627
- SubnetSelfLink=s['self_link'],
628
- ProjectId=s['project_id'],
629
- SubnetName=s['name'],
630
- Region=s['region'],
631
- GatewayAddress=s['gateway_address'],
632
- IpCidrRange=s['ip_cidr_range'],
633
- PrivateIpGoogleAccess=s['private_ip_google_access'],
685
+ VpcPartialUri=s["vpc_partial_uri"],
686
+ VpcSelfLink=s["vpc_self_link"],
687
+ PartialUri=s["partial_uri"],
688
+ SubnetSelfLink=s["self_link"],
689
+ ProjectId=s["project_id"],
690
+ SubnetName=s["name"],
691
+ Region=s["region"],
692
+ GatewayAddress=s["gateway_address"],
693
+ IpCidrRange=s["ip_cidr_range"],
694
+ PrivateIpGoogleAccess=s["private_ip_google_access"],
634
695
  gcp_update_tag=gcp_update_tag,
635
696
  )
636
697
 
637
698
 
638
699
  @timeit
639
- def load_gcp_forwarding_rules(neo4j_session: neo4j.Session, fwd_rules: List[Dict], gcp_update_tag: int) -> None:
700
+ def load_gcp_forwarding_rules(
701
+ neo4j_session: neo4j.Session,
702
+ fwd_rules: List[Dict],
703
+ gcp_update_tag: int,
704
+ ) -> None:
640
705
  """
641
706
  Ingest GCP forwarding rules data to Neo4j
642
707
  :param neo4j_session: The Neo4j session
@@ -665,26 +730,26 @@ def load_gcp_forwarding_rules(neo4j_session: neo4j.Session, fwd_rules: List[Dict
665
730
  """
666
731
 
667
732
  for fwd in fwd_rules:
668
- network = fwd.get('network', None)
669
- subnetwork = fwd.get('subnetwork', None)
733
+ network = fwd.get("network", None)
734
+ subnetwork = fwd.get("subnetwork", None)
670
735
 
671
736
  neo4j_session.run(
672
737
  query,
673
- PartialUri=fwd['partial_uri'],
674
- IPAddress=fwd['ip_address'],
675
- IPProtocol=fwd['ip_protocol'],
676
- LoadBalancingScheme=fwd['load_balancing_scheme'],
677
- Name=fwd['name'],
738
+ PartialUri=fwd["partial_uri"],
739
+ IPAddress=fwd["ip_address"],
740
+ IPProtocol=fwd["ip_protocol"],
741
+ LoadBalancingScheme=fwd["load_balancing_scheme"],
742
+ Name=fwd["name"],
678
743
  Network=network,
679
- NetworkPartialUri=fwd.get('network_partial_uri', None),
680
- PortRange=fwd.get('port_range', None),
681
- Ports=fwd.get('ports', None),
682
- ProjectId=fwd['project_id'],
683
- Region=fwd.get('region', None),
684
- SelfLink=fwd['self_link'],
744
+ NetworkPartialUri=fwd.get("network_partial_uri", None),
745
+ PortRange=fwd.get("port_range", None),
746
+ Ports=fwd.get("ports", None),
747
+ ProjectId=fwd["project_id"],
748
+ Region=fwd.get("region", None),
749
+ SelfLink=fwd["self_link"],
685
750
  SubNetwork=subnetwork,
686
- SubNetworkPartialUri=fwd.get('subnetwork_partial_uri', None),
687
- TargetPartialUri=fwd['target'],
751
+ SubNetworkPartialUri=fwd.get("subnetwork_partial_uri", None),
752
+ TargetPartialUri=fwd["target"],
688
753
  gcp_update_tag=gcp_update_tag,
689
754
  )
690
755
 
@@ -695,7 +760,11 @@ def load_gcp_forwarding_rules(neo4j_session: neo4j.Session, fwd_rules: List[Dict
695
760
 
696
761
 
697
762
  @timeit
698
- def _attach_fwd_rule_to_subnet(neo4j_session: neo4j.Session, fwd: Dict, gcp_update_tag: int) -> None:
763
+ def _attach_fwd_rule_to_subnet(
764
+ neo4j_session: neo4j.Session,
765
+ fwd: Dict,
766
+ gcp_update_tag: int,
767
+ ) -> None:
699
768
  query = """
700
769
  MERGE(subnet:GCPSubnet{id:$SubNetworkPartialUri})
701
770
  ON CREATE SET subnet.firstseen = timestamp(),
@@ -712,14 +781,18 @@ def _attach_fwd_rule_to_subnet(neo4j_session: neo4j.Session, fwd: Dict, gcp_upda
712
781
 
713
782
  neo4j_session.run(
714
783
  query,
715
- PartialUri=fwd['partial_uri'],
716
- SubNetworkPartialUri=fwd.get('subnetwork_partial_uri', None),
784
+ PartialUri=fwd["partial_uri"],
785
+ SubNetworkPartialUri=fwd.get("subnetwork_partial_uri", None),
717
786
  gcp_update_tag=gcp_update_tag,
718
787
  )
719
788
 
720
789
 
721
790
  @timeit
722
- def _attach_fwd_rule_to_vpc(neo4j_session: neo4j.Session, fwd: Dict, gcp_update_tag: int) -> None:
791
+ def _attach_fwd_rule_to_vpc(
792
+ neo4j_session: neo4j.Session,
793
+ fwd: Dict,
794
+ gcp_update_tag: int,
795
+ ) -> None:
723
796
  query = """
724
797
  MERGE (vpc:GCPVpc{id:$NetworkPartialUri})
725
798
  ON CREATE SET vpc.firstseen = timestamp(),
@@ -735,14 +808,18 @@ def _attach_fwd_rule_to_vpc(neo4j_session: neo4j.Session, fwd: Dict, gcp_update_
735
808
 
736
809
  neo4j_session.run(
737
810
  query,
738
- PartialUri=fwd['partial_uri'],
739
- NetworkPartialUri=fwd.get('network_partial_uri', None),
811
+ PartialUri=fwd["partial_uri"],
812
+ NetworkPartialUri=fwd.get("network_partial_uri", None),
740
813
  gcp_update_tag=gcp_update_tag,
741
814
  )
742
815
 
743
816
 
744
817
  @timeit
745
- def _attach_instance_tags(neo4j_session: neo4j.Session, instance: Resource, gcp_update_tag: int) -> None:
818
+ def _attach_instance_tags(
819
+ neo4j_session: neo4j.Session,
820
+ instance: Resource,
821
+ gcp_update_tag: int,
822
+ ) -> None:
746
823
  """
747
824
  Attach tags to GCP instance and to the VPCs that they are defined in.
748
825
  :param neo4j_session: The session
@@ -770,21 +847,25 @@ def _attach_instance_tags(neo4j_session: neo4j.Session, instance: Resource, gcp_
770
847
  ON CREATE SET d.firstseen = timestamp()
771
848
  SET d.lastupdated = $gcp_update_tag
772
849
  """
773
- for tag in instance.get('tags', {}).get('items', []):
774
- for nic in instance.get('networkInterfaces', []):
775
- tag_id = _create_gcp_network_tag_id(nic['vpc_partial_uri'], tag)
850
+ for tag in instance.get("tags", {}).get("items", []):
851
+ for nic in instance.get("networkInterfaces", []):
852
+ tag_id = _create_gcp_network_tag_id(nic["vpc_partial_uri"], tag)
776
853
  neo4j_session.run(
777
854
  query,
778
- InstanceId=instance['partial_uri'],
855
+ InstanceId=instance["partial_uri"],
779
856
  TagId=tag_id,
780
857
  TagValue=tag,
781
- VpcPartialUri=nic['vpc_partial_uri'],
858
+ VpcPartialUri=nic["vpc_partial_uri"],
782
859
  gcp_update_tag=gcp_update_tag,
783
860
  )
784
861
 
785
862
 
786
863
  @timeit
787
- def _attach_gcp_nics(neo4j_session: neo4j.Session, instance: Resource, gcp_update_tag: int) -> None:
864
+ def _attach_gcp_nics(
865
+ neo4j_session: neo4j.Session,
866
+ instance: Resource,
867
+ gcp_update_tag: int,
868
+ ) -> None:
788
869
  """
789
870
  Attach GCP Network Interfaces to GCP Instances and GCP Subnets.
790
871
  Then, attach GCP Instances directly to VPCs.
@@ -815,24 +896,27 @@ def _attach_gcp_nics(neo4j_session: neo4j.Session, instance: Resource, gcp_updat
815
896
  ON CREATE SET p.firstseen = timestamp()
816
897
  SET p.lastupdated = $gcp_update_tag
817
898
  """
818
- for nic in instance.get('networkInterfaces', []):
899
+ for nic in instance.get("networkInterfaces", []):
819
900
  # Make an ID for GCPNetworkInterface nodes because GCP doesn't define one but we need to uniquely identify them
820
901
  nic_id = f"{instance['partial_uri']}/networkinterfaces/{nic['name']}"
821
902
  neo4j_session.run(
822
903
  query,
823
- InstanceId=instance['partial_uri'],
904
+ InstanceId=instance["partial_uri"],
824
905
  NicId=nic_id,
825
- NetworkIP=nic.get('networkIP'),
826
- NicName=nic['name'],
906
+ NetworkIP=nic.get("networkIP"),
907
+ NicName=nic["name"],
827
908
  gcp_update_tag=gcp_update_tag,
828
- SubnetPartialUri=nic['subnet_partial_uri'],
909
+ SubnetPartialUri=nic["subnet_partial_uri"],
829
910
  )
830
911
  _attach_gcp_nic_access_configs(neo4j_session, nic_id, nic, gcp_update_tag)
831
912
 
832
913
 
833
914
  @timeit
834
915
  def _attach_gcp_nic_access_configs(
835
- neo4j_session: neo4j.Session, nic_id: str, nic: Resource, gcp_update_tag: int,
916
+ neo4j_session: neo4j.Session,
917
+ nic_id: str,
918
+ nic: Resource,
919
+ gcp_update_tag: int,
836
920
  ) -> None:
837
921
  """
838
922
  Attach an access configuration to the GCP NIC.
@@ -858,25 +942,29 @@ def _attach_gcp_nic_access_configs(
858
942
  ON CREATE SET r.firstseen = timestamp()
859
943
  SET r.lastupdated = $gcp_update_tag
860
944
  """
861
- for ac in nic.get('accessConfigs', []):
945
+ for ac in nic.get("accessConfigs", []):
862
946
  # Make an ID for GCPNicAccessConfig nodes because GCP doesn't define one but we need to uniquely identify them
863
947
  access_config_id = f"{nic_id}/accessconfigs/{ac['type']}"
864
948
  neo4j_session.run(
865
949
  query,
866
950
  NicId=nic_id,
867
951
  AccessConfigId=access_config_id,
868
- Type=ac['type'],
869
- Name=ac['name'],
870
- NatIP=ac.get('natIP', None),
871
- SetPublicPtr=ac.get('setPublicPtr', None),
872
- PublicPtrDomainName=ac.get('publicPtrDomainName', None),
873
- NetworkTier=ac.get('networkTier', None),
952
+ Type=ac["type"],
953
+ Name=ac["name"],
954
+ NatIP=ac.get("natIP", None),
955
+ SetPublicPtr=ac.get("setPublicPtr", None),
956
+ PublicPtrDomainName=ac.get("publicPtrDomainName", None),
957
+ NetworkTier=ac.get("networkTier", None),
874
958
  gcp_update_tag=gcp_update_tag,
875
959
  )
876
960
 
877
961
 
878
962
  @timeit
879
- def _attach_gcp_vpc(neo4j_session: neo4j.Session, instance_id: str, gcp_update_tag: int) -> None:
963
+ def _attach_gcp_vpc(
964
+ neo4j_session: neo4j.Session,
965
+ instance_id: str,
966
+ gcp_update_tag: int,
967
+ ) -> None:
880
968
  """
881
969
  Attach a GCP instance directly to a VPC
882
970
  :param neo4j_session: neo4j_session
@@ -899,7 +987,11 @@ def _attach_gcp_vpc(neo4j_session: neo4j.Session, instance_id: str, gcp_update_t
899
987
 
900
988
 
901
989
  @timeit
902
- def load_gcp_ingress_firewalls(neo4j_session: neo4j.Session, fw_list: List[Resource], gcp_update_tag: int) -> None:
990
+ def load_gcp_ingress_firewalls(
991
+ neo4j_session: neo4j.Session,
992
+ fw_list: List[Resource],
993
+ gcp_update_tag: int,
994
+ ) -> None:
903
995
  """
904
996
  Load the firewall list to Neo4j
905
997
  :param fw_list: The transformed list of firewalls
@@ -929,14 +1021,14 @@ def load_gcp_ingress_firewalls(neo4j_session: neo4j.Session, fw_list: List[Resou
929
1021
  for fw in fw_list:
930
1022
  neo4j_session.run(
931
1023
  query,
932
- FwPartialUri=fw['id'],
933
- Direction=fw['direction'],
934
- Disabled=fw['disabled'],
935
- Name=fw['name'],
936
- Priority=fw['priority'],
937
- SelfLink=fw['selfLink'],
938
- VpcPartialUri=fw['vpc_partial_uri'],
939
- HasTargetServiceAccounts=fw['has_target_service_accounts'],
1024
+ FwPartialUri=fw["id"],
1025
+ Direction=fw["direction"],
1026
+ Disabled=fw["disabled"],
1027
+ Name=fw["name"],
1028
+ Priority=fw["priority"],
1029
+ SelfLink=fw["selfLink"],
1030
+ VpcPartialUri=fw["vpc_partial_uri"],
1031
+ HasTargetServiceAccounts=fw["has_target_service_accounts"],
940
1032
  gcp_update_tag=gcp_update_tag,
941
1033
  )
942
1034
  _attach_firewall_rules(neo4j_session, fw, gcp_update_tag)
@@ -944,7 +1036,11 @@ def load_gcp_ingress_firewalls(neo4j_session: neo4j.Session, fw_list: List[Resou
944
1036
 
945
1037
 
946
1038
  @timeit
947
- def _attach_firewall_rules(neo4j_session: neo4j.Session, fw: Resource, gcp_update_tag: int) -> None:
1039
+ def _attach_firewall_rules(
1040
+ neo4j_session: neo4j.Session,
1041
+ fw: Resource,
1042
+ gcp_update_tag: int,
1043
+ ) -> None:
948
1044
  """
949
1045
  Attach the allow_rules to the Firewall object
950
1046
  :param neo4j_session: The Neo4j session
@@ -952,7 +1048,8 @@ def _attach_firewall_rules(neo4j_session: neo4j.Session, fw: Resource, gcp_updat
952
1048
  :param gcp_update_tag: The timestamp
953
1049
  :return: Nothing
954
1050
  """
955
- template = Template("""
1051
+ template = Template(
1052
+ """
956
1053
  MATCH (fw:GCPFirewall{id:$FwPartialUri})
957
1054
 
958
1055
  MERGE (rule:IpRule:IpPermissionInbound:GCPIpRule{id:$RuleId})
@@ -975,9 +1072,10 @@ def _attach_firewall_rules(neo4j_session: neo4j.Session, fw: Resource, gcp_updat
975
1072
  MERGE (fw)<-[r:$fw_rule_relationship_label]-(rule)
976
1073
  ON CREATE SET r.firstseen = timestamp()
977
1074
  SET r.lastupdated = $gcp_update_tag
978
- """)
979
- for list_type in 'transformed_allow_list', 'transformed_deny_list':
980
- if list_type == 'transformed_allow_list':
1075
+ """,
1076
+ )
1077
+ for list_type in "transformed_allow_list", "transformed_deny_list":
1078
+ if list_type == "transformed_allow_list":
981
1079
  label = "ALLOWED_BY"
982
1080
  else:
983
1081
  label = "DENIED_BY"
@@ -985,21 +1083,25 @@ def _attach_firewall_rules(neo4j_session: neo4j.Session, fw: Resource, gcp_updat
985
1083
  # It is possible for sourceRanges to not be specified for this rule
986
1084
  # If sourceRanges is not specified then the rule must specify sourceTags.
987
1085
  # Since an IP range cannot have a tag applied to it, it is ok if we don't ingest this rule.
988
- for ip_range in fw.get('sourceRanges', []):
1086
+ for ip_range in fw.get("sourceRanges", []):
989
1087
  neo4j_session.run(
990
1088
  template.safe_substitute(fw_rule_relationship_label=label),
991
- FwPartialUri=fw['id'],
992
- RuleId=rule['ruleid'],
993
- Protocol=rule['protocol'],
994
- FromPort=rule.get('fromport'),
995
- ToPort=rule.get('toport'),
1089
+ FwPartialUri=fw["id"],
1090
+ RuleId=rule["ruleid"],
1091
+ Protocol=rule["protocol"],
1092
+ FromPort=rule.get("fromport"),
1093
+ ToPort=rule.get("toport"),
996
1094
  Range=ip_range,
997
1095
  gcp_update_tag=gcp_update_tag,
998
1096
  )
999
1097
 
1000
1098
 
1001
1099
  @timeit
1002
- def _attach_target_tags(neo4j_session: neo4j.Session, fw: Resource, gcp_update_tag: int) -> None:
1100
+ def _attach_target_tags(
1101
+ neo4j_session: neo4j.Session,
1102
+ fw: Resource,
1103
+ gcp_update_tag: int,
1104
+ ) -> None:
1003
1105
  """
1004
1106
  Attach target tags to the firewall object
1005
1107
  :param neo4j_session: The neo4j session
@@ -1020,11 +1122,11 @@ def _attach_target_tags(neo4j_session: neo4j.Session, fw: Resource, gcp_update_t
1020
1122
  ON CREATE SET h.firstseen = timestamp()
1021
1123
  SET h.lastupdated = $gcp_update_tag
1022
1124
  """
1023
- for tag in fw.get('targetTags', []):
1024
- tag_id = _create_gcp_network_tag_id(fw['vpc_partial_uri'], tag)
1125
+ for tag in fw.get("targetTags", []):
1126
+ tag_id = _create_gcp_network_tag_id(fw["vpc_partial_uri"], tag)
1025
1127
  neo4j_session.run(
1026
1128
  query,
1027
- FwPartialUri=fw['id'],
1129
+ FwPartialUri=fw["id"],
1028
1130
  TagId=tag_id,
1029
1131
  TagValue=tag,
1030
1132
  gcp_update_tag=gcp_update_tag,
@@ -1032,14 +1134,21 @@ def _attach_target_tags(neo4j_session: neo4j.Session, fw: Resource, gcp_update_t
1032
1134
 
1033
1135
 
1034
1136
  @timeit
1035
- def cleanup_gcp_instances(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
1137
+ def cleanup_gcp_instances(
1138
+ neo4j_session: neo4j.Session,
1139
+ common_job_parameters: Dict,
1140
+ ) -> None:
1036
1141
  """
1037
1142
  Delete out-of-date GCP instance nodes and relationships
1038
1143
  :param neo4j_session: The Neo4j session
1039
1144
  :param common_job_parameters: dict of other job parameters to pass to Neo4j
1040
1145
  :return: Nothing
1041
1146
  """
1042
- run_cleanup_job('gcp_compute_instance_cleanup.json', neo4j_session, common_job_parameters)
1147
+ run_cleanup_job(
1148
+ "gcp_compute_instance_cleanup.json",
1149
+ neo4j_session,
1150
+ common_job_parameters,
1151
+ )
1043
1152
 
1044
1153
 
1045
1154
  @timeit
@@ -1050,46 +1159,75 @@ def cleanup_gcp_vpcs(neo4j_session: neo4j.Session, common_job_parameters: Dict)
1050
1159
  :param common_job_parameters: dict of other job parameters to pass to Neo4j
1051
1160
  :return: Nothing
1052
1161
  """
1053
- run_cleanup_job('gcp_compute_vpc_cleanup.json', neo4j_session, common_job_parameters)
1162
+ run_cleanup_job(
1163
+ "gcp_compute_vpc_cleanup.json",
1164
+ neo4j_session,
1165
+ common_job_parameters,
1166
+ )
1054
1167
 
1055
1168
 
1056
1169
  @timeit
1057
- def cleanup_gcp_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
1170
+ def cleanup_gcp_subnets(
1171
+ neo4j_session: neo4j.Session,
1172
+ common_job_parameters: Dict,
1173
+ ) -> None:
1058
1174
  """
1059
1175
  Delete out-of-date GCP VPC subnet nodes and relationships
1060
1176
  :param neo4j_session: The Neo4j session
1061
1177
  :param common_job_parameters: dict of other job parameters to pass to Neo4j
1062
1178
  :return: Nothing
1063
1179
  """
1064
- run_cleanup_job('gcp_compute_vpc_subnet_cleanup.json', neo4j_session, common_job_parameters)
1180
+ run_cleanup_job(
1181
+ "gcp_compute_vpc_subnet_cleanup.json",
1182
+ neo4j_session,
1183
+ common_job_parameters,
1184
+ )
1065
1185
 
1066
1186
 
1067
1187
  @timeit
1068
- def cleanup_gcp_forwarding_rules(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
1188
+ def cleanup_gcp_forwarding_rules(
1189
+ neo4j_session: neo4j.Session,
1190
+ common_job_parameters: Dict,
1191
+ ) -> None:
1069
1192
  """
1070
1193
  Delete out-of-date GCP forwarding rules and relationships
1071
1194
  :param neo4j_session: The Neo4j session
1072
1195
  :param common_job_parameters: dict of other job parameters to pass to Neo4j
1073
1196
  :return: Nothing
1074
1197
  """
1075
- run_cleanup_job('gcp_compute_forwarding_rules_cleanup.json', neo4j_session, common_job_parameters)
1198
+ run_cleanup_job(
1199
+ "gcp_compute_forwarding_rules_cleanup.json",
1200
+ neo4j_session,
1201
+ common_job_parameters,
1202
+ )
1076
1203
 
1077
1204
 
1078
1205
  @timeit
1079
- def cleanup_gcp_firewall_rules(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
1206
+ def cleanup_gcp_firewall_rules(
1207
+ neo4j_session: neo4j.Session,
1208
+ common_job_parameters: Dict,
1209
+ ) -> None:
1080
1210
  """
1081
1211
  Delete out of date GCP firewalls and their relationships
1082
1212
  :param neo4j_session: The Neo4j session
1083
1213
  :param common_job_parameters: dict of other job parameters to pass to Neo4j
1084
1214
  :return: Nothing
1085
1215
  """
1086
- run_cleanup_job('gcp_compute_firewall_cleanup.json', neo4j_session, common_job_parameters)
1216
+ run_cleanup_job(
1217
+ "gcp_compute_firewall_cleanup.json",
1218
+ neo4j_session,
1219
+ common_job_parameters,
1220
+ )
1087
1221
 
1088
1222
 
1089
1223
  @timeit
1090
1224
  def sync_gcp_instances(
1091
- neo4j_session: neo4j.Session, compute: Resource, project_id: str, zones: Optional[List[Dict]],
1092
- gcp_update_tag: int, common_job_parameters: Dict,
1225
+ neo4j_session: neo4j.Session,
1226
+ compute: Resource,
1227
+ project_id: str,
1228
+ zones: Optional[List[Dict]],
1229
+ gcp_update_tag: int,
1230
+ common_job_parameters: Dict,
1093
1231
  ) -> None:
1094
1232
  """
1095
1233
  Get GCP instances using the Compute resource object, ingest to Neo4j, and clean up old data.
@@ -1106,13 +1244,16 @@ def sync_gcp_instances(
1106
1244
  instance_responses = get_gcp_instance_responses(project_id, zones, compute)
1107
1245
  instance_list = transform_gcp_instances(instance_responses)
1108
1246
  load_gcp_instances(neo4j_session, instance_list, gcp_update_tag)
1109
- # TODO scope the cleanup to the current project - https://github.com/lyft/cartography/issues/381
1247
+ # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
1110
1248
  cleanup_gcp_instances(neo4j_session, common_job_parameters)
1111
1249
 
1112
1250
 
1113
1251
  @timeit
1114
1252
  def sync_gcp_vpcs(
1115
- neo4j_session: neo4j.Session, compute: Resource, project_id: str, gcp_update_tag: int,
1253
+ neo4j_session: neo4j.Session,
1254
+ compute: Resource,
1255
+ project_id: str,
1256
+ gcp_update_tag: int,
1116
1257
  common_job_parameters: Dict,
1117
1258
  ) -> None:
1118
1259
  """
@@ -1127,27 +1268,34 @@ def sync_gcp_vpcs(
1127
1268
  vpc_res = get_gcp_vpcs(project_id, compute)
1128
1269
  vpcs = transform_gcp_vpcs(vpc_res)
1129
1270
  load_gcp_vpcs(neo4j_session, vpcs, gcp_update_tag)
1130
- # TODO scope the cleanup to the current project - https://github.com/lyft/cartography/issues/381
1271
+ # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
1131
1272
  cleanup_gcp_vpcs(neo4j_session, common_job_parameters)
1132
1273
 
1133
1274
 
1134
1275
  @timeit
1135
1276
  def sync_gcp_subnets(
1136
- neo4j_session: neo4j.Session, compute: Resource, project_id: str, regions: List[str], gcp_update_tag: int,
1277
+ neo4j_session: neo4j.Session,
1278
+ compute: Resource,
1279
+ project_id: str,
1280
+ regions: List[str],
1281
+ gcp_update_tag: int,
1137
1282
  common_job_parameters: Dict,
1138
1283
  ) -> None:
1139
1284
  for r in regions:
1140
1285
  subnet_res = get_gcp_subnets(project_id, r, compute)
1141
1286
  subnets = transform_gcp_subnets(subnet_res)
1142
1287
  load_gcp_subnets(neo4j_session, subnets, gcp_update_tag)
1143
- # TODO scope the cleanup to the current project - https://github.com/lyft/cartography/issues/381
1288
+ # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
1144
1289
  cleanup_gcp_subnets(neo4j_session, common_job_parameters)
1145
1290
 
1146
1291
 
1147
1292
  @timeit
1148
1293
  def sync_gcp_forwarding_rules(
1149
- neo4j_session: neo4j.Session, compute: Resource, project_id: str, regions: List[str], gcp_update_tag: int,
1150
-
1294
+ neo4j_session: neo4j.Session,
1295
+ compute: Resource,
1296
+ project_id: str,
1297
+ regions: List[str],
1298
+ gcp_update_tag: int,
1151
1299
  common_job_parameters: Dict,
1152
1300
  ) -> None:
1153
1301
  """
@@ -1163,20 +1311,23 @@ def sync_gcp_forwarding_rules(
1163
1311
  global_fwd_response = get_gcp_global_forwarding_rules(project_id, compute)
1164
1312
  forwarding_rules = transform_gcp_forwarding_rules(global_fwd_response)
1165
1313
  load_gcp_forwarding_rules(neo4j_session, forwarding_rules, gcp_update_tag)
1166
- # TODO scope the cleanup to the current project - https://github.com/lyft/cartography/issues/381
1314
+ # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
1167
1315
  cleanup_gcp_forwarding_rules(neo4j_session, common_job_parameters)
1168
1316
 
1169
1317
  for r in regions:
1170
1318
  fwd_response = get_gcp_regional_forwarding_rules(project_id, r, compute)
1171
1319
  forwarding_rules = transform_gcp_forwarding_rules(fwd_response)
1172
1320
  load_gcp_forwarding_rules(neo4j_session, forwarding_rules, gcp_update_tag)
1173
- # TODO scope the cleanup to the current project - https://github.com/lyft/cartography/issues/381
1321
+ # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
1174
1322
  cleanup_gcp_forwarding_rules(neo4j_session, common_job_parameters)
1175
1323
 
1176
1324
 
1177
1325
  @timeit
1178
1326
  def sync_gcp_firewall_rules(
1179
- neo4j_session: neo4j.Session, compute: Resource, project_id: str, gcp_update_tag: int,
1327
+ neo4j_session: neo4j.Session,
1328
+ compute: Resource,
1329
+ project_id: str,
1330
+ gcp_update_tag: int,
1180
1331
  common_job_parameters: Dict,
1181
1332
  ) -> None:
1182
1333
  """
@@ -1190,7 +1341,7 @@ def sync_gcp_firewall_rules(
1190
1341
  fw_response = get_gcp_firewall_ingress_rules(project_id, compute)
1191
1342
  fw_list = transform_gcp_firewall(fw_response)
1192
1343
  load_gcp_ingress_firewalls(neo4j_session, fw_list, gcp_update_tag)
1193
- # TODO scope the cleanup to the current project - https://github.com/lyft/cartography/issues/381
1344
+ # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
1194
1345
  cleanup_gcp_firewall_rules(neo4j_session, common_job_parameters)
1195
1346
 
1196
1347
 
@@ -1203,13 +1354,16 @@ def _zones_to_regions(zones: List[str]) -> List[Set]:
1203
1354
  regions = set()
1204
1355
  for zone in zones:
1205
1356
  # Chop off the last 2 chars to turn the zone to a region
1206
- region = zone['name'][:-2] # type: ignore
1357
+ region = zone["name"][:-2] # type: ignore
1207
1358
  regions.add(region)
1208
- return list(regions) # type: ignore
1359
+ return list(regions) # type: ignore
1209
1360
 
1210
1361
 
1211
1362
  def sync(
1212
- neo4j_session: neo4j.Session, compute: Resource, project_id: str, gcp_update_tag: int,
1363
+ neo4j_session: neo4j.Session,
1364
+ compute: Resource,
1365
+ project_id: str,
1366
+ gcp_update_tag: int,
1213
1367
  common_job_parameters: dict,
1214
1368
  ) -> None:
1215
1369
  """
@@ -1230,8 +1384,41 @@ def sync(
1230
1384
  return
1231
1385
  else:
1232
1386
  regions = _zones_to_regions(zones)
1233
- sync_gcp_vpcs(neo4j_session, compute, project_id, gcp_update_tag, common_job_parameters)
1234
- sync_gcp_firewall_rules(neo4j_session, compute, project_id, gcp_update_tag, common_job_parameters)
1235
- sync_gcp_subnets(neo4j_session, compute, project_id, regions, gcp_update_tag, common_job_parameters)
1236
- sync_gcp_instances(neo4j_session, compute, project_id, zones, gcp_update_tag, common_job_parameters)
1237
- sync_gcp_forwarding_rules(neo4j_session, compute, project_id, regions, gcp_update_tag, common_job_parameters)
1387
+ sync_gcp_vpcs(
1388
+ neo4j_session,
1389
+ compute,
1390
+ project_id,
1391
+ gcp_update_tag,
1392
+ common_job_parameters,
1393
+ )
1394
+ sync_gcp_firewall_rules(
1395
+ neo4j_session,
1396
+ compute,
1397
+ project_id,
1398
+ gcp_update_tag,
1399
+ common_job_parameters,
1400
+ )
1401
+ sync_gcp_subnets(
1402
+ neo4j_session,
1403
+ compute,
1404
+ project_id,
1405
+ regions,
1406
+ gcp_update_tag,
1407
+ common_job_parameters,
1408
+ )
1409
+ sync_gcp_instances(
1410
+ neo4j_session,
1411
+ compute,
1412
+ project_id,
1413
+ zones,
1414
+ gcp_update_tag,
1415
+ common_job_parameters,
1416
+ )
1417
+ sync_gcp_forwarding_rules(
1418
+ neo4j_session,
1419
+ compute,
1420
+ project_id,
1421
+ regions,
1422
+ gcp_update_tag,
1423
+ common_job_parameters,
1424
+ )