cartography 0.102.0rc1__py3-none-any.whl → 0.103.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

Files changed (297) hide show
  1. cartography/__main__.py +1 -2
  2. cartography/_version.py +2 -2
  3. cartography/cli.py +376 -249
  4. cartography/client/core/tx.py +39 -18
  5. cartography/config.py +28 -0
  6. cartography/driftdetect/__main__.py +1 -2
  7. cartography/driftdetect/add_shortcut.py +10 -2
  8. cartography/driftdetect/cli.py +71 -75
  9. cartography/driftdetect/detect_deviations.py +7 -3
  10. cartography/driftdetect/get_states.py +20 -8
  11. cartography/driftdetect/model.py +5 -5
  12. cartography/driftdetect/serializers.py +8 -6
  13. cartography/driftdetect/storage.py +2 -2
  14. cartography/graph/cleanupbuilder.py +35 -15
  15. cartography/graph/job.py +46 -17
  16. cartography/graph/querybuilder.py +165 -80
  17. cartography/graph/statement.py +35 -26
  18. cartography/intel/analysis.py +4 -1
  19. cartography/intel/aws/__init__.py +114 -55
  20. cartography/intel/aws/apigateway.py +134 -63
  21. cartography/intel/aws/cloudtrail.py +127 -0
  22. cartography/intel/aws/cloudwatch.py +93 -0
  23. cartography/intel/aws/config.py +56 -20
  24. cartography/intel/aws/dynamodb.py +108 -40
  25. cartography/intel/aws/ec2/__init__.py +2 -2
  26. cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
  27. cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
  28. cartography/intel/aws/ec2/images.py +49 -20
  29. cartography/intel/aws/ec2/instances.py +234 -136
  30. cartography/intel/aws/ec2/internet_gateways.py +40 -11
  31. cartography/intel/aws/ec2/key_pairs.py +44 -20
  32. cartography/intel/aws/ec2/launch_templates.py +101 -59
  33. cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
  34. cartography/intel/aws/ec2/load_balancers.py +82 -42
  35. cartography/intel/aws/ec2/network_acls.py +89 -65
  36. cartography/intel/aws/ec2/network_interfaces.py +146 -87
  37. cartography/intel/aws/ec2/reserved_instances.py +45 -16
  38. cartography/intel/aws/ec2/route_tables.py +327 -0
  39. cartography/intel/aws/ec2/security_groups.py +71 -21
  40. cartography/intel/aws/ec2/snapshots.py +61 -22
  41. cartography/intel/aws/ec2/subnets.py +54 -18
  42. cartography/intel/aws/ec2/tgw.py +100 -34
  43. cartography/intel/aws/ec2/util.py +1 -1
  44. cartography/intel/aws/ec2/volumes.py +69 -41
  45. cartography/intel/aws/ec2/vpc.py +37 -12
  46. cartography/intel/aws/ec2/vpc_peerings.py +83 -24
  47. cartography/intel/aws/ecr.py +88 -32
  48. cartography/intel/aws/ecs.py +83 -47
  49. cartography/intel/aws/efs.py +93 -0
  50. cartography/intel/aws/eks.py +55 -29
  51. cartography/intel/aws/elasticache.py +42 -18
  52. cartography/intel/aws/elasticsearch.py +57 -20
  53. cartography/intel/aws/emr.py +61 -23
  54. cartography/intel/aws/iam.py +401 -145
  55. cartography/intel/aws/iam_instance_profiles.py +22 -22
  56. cartography/intel/aws/identitycenter.py +71 -37
  57. cartography/intel/aws/inspector.py +159 -89
  58. cartography/intel/aws/kms.py +92 -38
  59. cartography/intel/aws/lambda_function.py +103 -34
  60. cartography/intel/aws/organizations.py +30 -10
  61. cartography/intel/aws/permission_relationships.py +133 -51
  62. cartography/intel/aws/rds.py +249 -85
  63. cartography/intel/aws/redshift.py +107 -46
  64. cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
  65. cartography/intel/aws/resources.py +57 -44
  66. cartography/intel/aws/route53.py +108 -61
  67. cartography/intel/aws/s3.py +168 -83
  68. cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
  69. cartography/intel/aws/secretsmanager.py +24 -12
  70. cartography/intel/aws/securityhub.py +20 -9
  71. cartography/intel/aws/sns.py +166 -0
  72. cartography/intel/aws/sqs.py +60 -28
  73. cartography/intel/aws/ssm.py +70 -30
  74. cartography/intel/aws/util/arns.py +7 -7
  75. cartography/intel/aws/util/common.py +31 -4
  76. cartography/intel/azure/__init__.py +78 -19
  77. cartography/intel/azure/compute.py +101 -27
  78. cartography/intel/azure/cosmosdb.py +496 -170
  79. cartography/intel/azure/sql.py +296 -105
  80. cartography/intel/azure/storage.py +322 -113
  81. cartography/intel/azure/subscription.py +39 -23
  82. cartography/intel/azure/tenant.py +13 -4
  83. cartography/intel/azure/util/credentials.py +95 -55
  84. cartography/intel/bigfix/__init__.py +2 -2
  85. cartography/intel/bigfix/computers.py +93 -65
  86. cartography/intel/cloudflare/__init__.py +74 -0
  87. cartography/intel/cloudflare/accounts.py +57 -0
  88. cartography/intel/cloudflare/dnsrecords.py +64 -0
  89. cartography/intel/cloudflare/members.py +75 -0
  90. cartography/intel/cloudflare/roles.py +65 -0
  91. cartography/intel/cloudflare/zones.py +64 -0
  92. cartography/intel/create_indexes.py +3 -2
  93. cartography/intel/crowdstrike/__init__.py +11 -9
  94. cartography/intel/crowdstrike/endpoints.py +5 -1
  95. cartography/intel/crowdstrike/spotlight.py +8 -3
  96. cartography/intel/cve/__init__.py +46 -13
  97. cartography/intel/cve/feed.py +48 -12
  98. cartography/intel/digitalocean/__init__.py +22 -13
  99. cartography/intel/digitalocean/compute.py +75 -108
  100. cartography/intel/digitalocean/management.py +44 -80
  101. cartography/intel/digitalocean/platform.py +48 -43
  102. cartography/intel/dns.py +36 -10
  103. cartography/intel/duo/__init__.py +21 -16
  104. cartography/intel/duo/api_host.py +14 -9
  105. cartography/intel/duo/endpoints.py +50 -45
  106. cartography/intel/duo/groups.py +18 -14
  107. cartography/intel/duo/phones.py +37 -34
  108. cartography/intel/duo/tokens.py +26 -23
  109. cartography/intel/duo/users.py +54 -50
  110. cartography/intel/duo/web_authn_credentials.py +30 -25
  111. cartography/intel/entra/__init__.py +25 -7
  112. cartography/intel/entra/ou.py +112 -0
  113. cartography/intel/entra/users.py +69 -63
  114. cartography/intel/gcp/__init__.py +185 -49
  115. cartography/intel/gcp/compute.py +418 -231
  116. cartography/intel/gcp/crm.py +96 -43
  117. cartography/intel/gcp/dns.py +60 -19
  118. cartography/intel/gcp/gke.py +72 -38
  119. cartography/intel/gcp/iam.py +61 -41
  120. cartography/intel/gcp/storage.py +84 -55
  121. cartography/intel/github/__init__.py +13 -11
  122. cartography/intel/github/repos.py +270 -137
  123. cartography/intel/github/teams.py +170 -88
  124. cartography/intel/github/users.py +70 -39
  125. cartography/intel/github/util.py +36 -34
  126. cartography/intel/gsuite/__init__.py +47 -26
  127. cartography/intel/gsuite/api.py +73 -30
  128. cartography/intel/jamf/__init__.py +19 -1
  129. cartography/intel/jamf/computers.py +30 -7
  130. cartography/intel/jamf/util.py +7 -2
  131. cartography/intel/kandji/__init__.py +6 -3
  132. cartography/intel/kandji/devices.py +14 -8
  133. cartography/intel/kubernetes/namespaces.py +7 -4
  134. cartography/intel/kubernetes/pods.py +7 -4
  135. cartography/intel/kubernetes/services.py +8 -4
  136. cartography/intel/lastpass/__init__.py +2 -2
  137. cartography/intel/lastpass/users.py +23 -12
  138. cartography/intel/oci/__init__.py +44 -11
  139. cartography/intel/oci/iam.py +134 -38
  140. cartography/intel/oci/organizations.py +13 -6
  141. cartography/intel/oci/utils.py +43 -20
  142. cartography/intel/okta/__init__.py +66 -15
  143. cartography/intel/okta/applications.py +42 -20
  144. cartography/intel/okta/awssaml.py +93 -33
  145. cartography/intel/okta/factors.py +16 -4
  146. cartography/intel/okta/groups.py +56 -29
  147. cartography/intel/okta/organization.py +5 -1
  148. cartography/intel/okta/origins.py +6 -2
  149. cartography/intel/okta/roles.py +15 -5
  150. cartography/intel/okta/users.py +20 -8
  151. cartography/intel/okta/utils.py +6 -4
  152. cartography/intel/openai/__init__.py +86 -0
  153. cartography/intel/openai/adminapikeys.py +90 -0
  154. cartography/intel/openai/apikeys.py +96 -0
  155. cartography/intel/openai/projects.py +94 -0
  156. cartography/intel/openai/serviceaccounts.py +82 -0
  157. cartography/intel/openai/users.py +78 -0
  158. cartography/intel/openai/util.py +29 -0
  159. cartography/intel/pagerduty/__init__.py +8 -7
  160. cartography/intel/pagerduty/escalation_policies.py +18 -6
  161. cartography/intel/pagerduty/schedules.py +12 -4
  162. cartography/intel/pagerduty/services.py +11 -4
  163. cartography/intel/pagerduty/teams.py +8 -3
  164. cartography/intel/pagerduty/users.py +3 -1
  165. cartography/intel/pagerduty/vendors.py +3 -1
  166. cartography/intel/semgrep/__init__.py +24 -6
  167. cartography/intel/semgrep/dependencies.py +50 -28
  168. cartography/intel/semgrep/deployment.py +3 -1
  169. cartography/intel/semgrep/findings.py +42 -18
  170. cartography/intel/snipeit/__init__.py +17 -3
  171. cartography/intel/snipeit/asset.py +12 -6
  172. cartography/intel/snipeit/user.py +8 -5
  173. cartography/intel/snipeit/util.py +9 -4
  174. cartography/intel/tailscale/__init__.py +77 -0
  175. cartography/intel/tailscale/acls.py +146 -0
  176. cartography/intel/tailscale/devices.py +127 -0
  177. cartography/intel/tailscale/postureintegrations.py +81 -0
  178. cartography/intel/tailscale/tailnets.py +76 -0
  179. cartography/intel/tailscale/users.py +80 -0
  180. cartography/intel/tailscale/utils.py +132 -0
  181. cartography/models/aws/apigateway.py +21 -17
  182. cartography/models/aws/apigatewaycertificate.py +28 -22
  183. cartography/models/aws/apigatewayresource.py +28 -20
  184. cartography/models/aws/apigatewaystage.py +33 -25
  185. cartography/models/aws/cloudtrail/__init__.py +0 -0
  186. cartography/models/aws/cloudtrail/trail.py +61 -0
  187. cartography/models/aws/cloudwatch/__init__.py +0 -0
  188. cartography/models/aws/cloudwatch/loggroup.py +52 -0
  189. cartography/models/aws/dynamodb/gsi.py +30 -22
  190. cartography/models/aws/dynamodb/tables.py +25 -17
  191. cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
  192. cartography/models/aws/ec2/images.py +36 -34
  193. cartography/models/aws/ec2/instances.py +51 -45
  194. cartography/models/aws/ec2/keypair.py +21 -16
  195. cartography/models/aws/ec2/keypair_instance.py +28 -21
  196. cartography/models/aws/ec2/launch_configurations.py +30 -26
  197. cartography/models/aws/ec2/launch_template_versions.py +48 -38
  198. cartography/models/aws/ec2/launch_templates.py +21 -17
  199. cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
  200. cartography/models/aws/ec2/load_balancers.py +47 -37
  201. cartography/models/aws/ec2/network_acl_rules.py +38 -30
  202. cartography/models/aws/ec2/network_acls.py +38 -29
  203. cartography/models/aws/ec2/networkinterface_instance.py +52 -39
  204. cartography/models/aws/ec2/networkinterfaces.py +53 -37
  205. cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
  206. cartography/models/aws/ec2/reservations.py +18 -14
  207. cartography/models/aws/ec2/route_table_associations.py +97 -0
  208. cartography/models/aws/ec2/route_tables.py +128 -0
  209. cartography/models/aws/ec2/routes.py +85 -0
  210. cartography/models/aws/ec2/securitygroup_instance.py +29 -20
  211. cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
  212. cartography/models/aws/ec2/subnet_instance.py +24 -19
  213. cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
  214. cartography/models/aws/ec2/volumes.py +47 -40
  215. cartography/models/aws/efs/__init__.py +0 -0
  216. cartography/models/aws/efs/mount_target.py +52 -0
  217. cartography/models/aws/eks/clusters.py +23 -21
  218. cartography/models/aws/emr.py +32 -30
  219. cartography/models/aws/iam/instanceprofile.py +33 -24
  220. cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
  221. cartography/models/aws/identitycenter/awspermissionset.py +37 -29
  222. cartography/models/aws/identitycenter/awsssouser.py +23 -21
  223. cartography/models/aws/inspector/findings.py +77 -65
  224. cartography/models/aws/inspector/packages.py +35 -29
  225. cartography/models/aws/s3/__init__.py +0 -0
  226. cartography/models/aws/s3/account_public_access_block.py +51 -0
  227. cartography/models/aws/sns/__init__.py +0 -0
  228. cartography/models/aws/sns/topic.py +50 -0
  229. cartography/models/aws/ssm/instance_information.py +51 -39
  230. cartography/models/aws/ssm/instance_patch.py +32 -26
  231. cartography/models/bigfix/bigfix_computer.py +42 -38
  232. cartography/models/bigfix/bigfix_root.py +3 -3
  233. cartography/models/cloudflare/__init__.py +0 -0
  234. cartography/models/cloudflare/account.py +25 -0
  235. cartography/models/cloudflare/dnsrecord.py +55 -0
  236. cartography/models/cloudflare/member.py +82 -0
  237. cartography/models/cloudflare/role.py +44 -0
  238. cartography/models/cloudflare/zone.py +59 -0
  239. cartography/models/core/common.py +12 -10
  240. cartography/models/core/nodes.py +5 -2
  241. cartography/models/core/relationships.py +14 -6
  242. cartography/models/crowdstrike/hosts.py +37 -35
  243. cartography/models/cve/cve.py +34 -32
  244. cartography/models/cve/cve_feed.py +6 -6
  245. cartography/models/digitalocean/__init__.py +0 -0
  246. cartography/models/digitalocean/account.py +21 -0
  247. cartography/models/digitalocean/droplet.py +56 -0
  248. cartography/models/digitalocean/project.py +48 -0
  249. cartography/models/duo/api_host.py +3 -3
  250. cartography/models/duo/endpoint.py +43 -41
  251. cartography/models/duo/group.py +14 -14
  252. cartography/models/duo/phone.py +27 -27
  253. cartography/models/duo/token.py +16 -16
  254. cartography/models/duo/user.py +46 -44
  255. cartography/models/duo/web_authn_credential.py +27 -19
  256. cartography/models/entra/ou.py +48 -0
  257. cartography/models/entra/tenant.py +24 -18
  258. cartography/models/entra/user.py +64 -48
  259. cartography/models/gcp/iam.py +23 -23
  260. cartography/models/github/orgs.py +5 -4
  261. cartography/models/github/teams.py +37 -31
  262. cartography/models/github/users.py +34 -23
  263. cartography/models/kandji/device.py +22 -16
  264. cartography/models/kandji/tenant.py +6 -4
  265. cartography/models/lastpass/tenant.py +3 -3
  266. cartography/models/lastpass/user.py +32 -28
  267. cartography/models/openai/__init__.py +0 -0
  268. cartography/models/openai/adminapikey.py +90 -0
  269. cartography/models/openai/apikey.py +84 -0
  270. cartography/models/openai/organization.py +17 -0
  271. cartography/models/openai/project.py +70 -0
  272. cartography/models/openai/serviceaccount.py +50 -0
  273. cartography/models/openai/user.py +49 -0
  274. cartography/models/semgrep/dependencies.py +36 -24
  275. cartography/models/semgrep/deployment.py +5 -5
  276. cartography/models/semgrep/findings.py +58 -42
  277. cartography/models/semgrep/locations.py +27 -21
  278. cartography/models/snipeit/asset.py +30 -21
  279. cartography/models/snipeit/tenant.py +6 -4
  280. cartography/models/snipeit/user.py +19 -12
  281. cartography/models/tailscale/__init__.py +0 -0
  282. cartography/models/tailscale/device.py +95 -0
  283. cartography/models/tailscale/group.py +86 -0
  284. cartography/models/tailscale/postureintegration.py +58 -0
  285. cartography/models/tailscale/tag.py +102 -0
  286. cartography/models/tailscale/tailnet.py +29 -0
  287. cartography/models/tailscale/user.py +52 -0
  288. cartography/stats.py +3 -3
  289. cartography/sync.py +113 -31
  290. cartography/util.py +84 -62
  291. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/METADATA +8 -15
  292. cartography-0.103.0.dist-info/RECORD +442 -0
  293. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/WHEEL +1 -1
  294. cartography-0.102.0rc1.dist-info/RECORD +0 -377
  295. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/entry_points.txt +0 -0
  296. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/licenses/LICENSE +0 -0
  297. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/top_level.txt +0 -0
@@ -27,9 +27,12 @@ def get_gcp_organizations(crm_v1: Resource) -> List[Resource]:
27
27
  try:
28
28
  req = crm_v1.organizations().search(body={})
29
29
  res = req.execute()
30
- return res.get('organizations', [])
30
+ return res.get("organizations", [])
31
31
  except HttpError as e:
32
- logger.warning("HttpError occurred in crm.get_gcp_organizations(), returning empty list. Details: %r", e)
32
+ logger.warning(
33
+ "HttpError occurred in crm.get_gcp_organizations(), returning empty list. Details: %r",
34
+ e,
35
+ )
33
36
  return []
34
37
 
35
38
 
@@ -45,9 +48,12 @@ def get_gcp_folders(crm_v2: Resource) -> List[Resource]:
45
48
  try:
46
49
  req = crm_v2.folders().search(body={})
47
50
  res = req.execute()
48
- return res.get('folders', [])
51
+ return res.get("folders", [])
49
52
  except HttpError as e:
50
- logger.warning("HttpError occurred in crm.get_gcp_folders(), returning empty list. Details: %r", e)
53
+ logger.warning(
54
+ "HttpError occurred in crm.get_gcp_folders(), returning empty list. Details: %r",
55
+ e,
56
+ )
51
57
  return []
52
58
 
53
59
 
@@ -65,17 +71,27 @@ def get_gcp_projects(crm_v1: Resource) -> List[Resource]:
65
71
  req = crm_v1.projects().list(filter="lifecycleState:ACTIVE")
66
72
  while req is not None:
67
73
  res = req.execute()
68
- page = res.get('projects', [])
74
+ page = res.get("projects", [])
69
75
  projects.extend(page)
70
- req = crm_v1.projects().list_next(previous_request=req, previous_response=res)
76
+ req = crm_v1.projects().list_next(
77
+ previous_request=req,
78
+ previous_response=res,
79
+ )
71
80
  return projects
72
81
  except HttpError as e:
73
- logger.warning("HttpError occurred in crm.get_gcp_projects(), returning empty list. Details: %r", e)
82
+ logger.warning(
83
+ "HttpError occurred in crm.get_gcp_projects(), returning empty list. Details: %r",
84
+ e,
85
+ )
74
86
  return []
75
87
 
76
88
 
77
89
  @timeit
78
- def load_gcp_organizations(neo4j_session: neo4j.Session, data: List[Dict], gcp_update_tag: int) -> None:
90
+ def load_gcp_organizations(
91
+ neo4j_session: neo4j.Session,
92
+ data: List[Dict],
93
+ gcp_update_tag: int,
94
+ ) -> None:
79
95
  """
80
96
  Ingest the GCP organizations to Neo4j
81
97
  :param neo4j_session: The Neo4j session
@@ -94,15 +110,19 @@ def load_gcp_organizations(neo4j_session: neo4j.Session, data: List[Dict], gcp_u
94
110
  for org_object in data:
95
111
  neo4j_session.run(
96
112
  query,
97
- OrgName=org_object['name'],
98
- DisplayName=org_object.get('displayName', None),
99
- LifecycleState=org_object.get('lifecycleState', None),
113
+ OrgName=org_object["name"],
114
+ DisplayName=org_object.get("displayName", None),
115
+ LifecycleState=org_object.get("lifecycleState", None),
100
116
  gcp_update_tag=gcp_update_tag,
101
117
  )
102
118
 
103
119
 
104
120
  @timeit
105
- def load_gcp_folders(neo4j_session: neo4j.Session, data: List[Dict], gcp_update_tag: int) -> None:
121
+ def load_gcp_folders(
122
+ neo4j_session: neo4j.Session,
123
+ data: List[Dict],
124
+ gcp_update_tag: int,
125
+ ) -> None:
106
126
  """
107
127
  Ingest the GCP folders to Neo4j
108
128
  :param neo4j_session: The Neo4j session
@@ -114,9 +134,9 @@ def load_gcp_folders(neo4j_session: neo4j.Session, data: List[Dict], gcp_update_
114
134
  # Get the correct parent type.
115
135
  # Parents of folders can only be GCPOrganizations or other folders, see
116
136
  # https://cloud.google.com/resource-manager/docs/cloud-platform-resource-hierarchy
117
- if folder['parent'].startswith("organizations"):
137
+ if folder["parent"].startswith("organizations"):
118
138
  query = "MATCH (parent:GCPOrganization{id:$ParentId})"
119
- elif folder['parent'].startswith("folders"):
139
+ elif folder["parent"].startswith("folders"):
120
140
  query = """
121
141
  MERGE (parent:GCPFolder{id:$ParentId})
122
142
  ON CREATE SET parent.firstseen = timestamp()
@@ -135,16 +155,20 @@ def load_gcp_folders(neo4j_session: neo4j.Session, data: List[Dict], gcp_update_
135
155
  """
136
156
  neo4j_session.run(
137
157
  query,
138
- ParentId=folder['parent'],
139
- FolderName=folder['name'],
140
- DisplayName=folder.get('displayName', None),
141
- LifecycleState=folder.get('lifecycleState', None),
158
+ ParentId=folder["parent"],
159
+ FolderName=folder["name"],
160
+ DisplayName=folder.get("displayName", None),
161
+ LifecycleState=folder.get("lifecycleState", None),
142
162
  gcp_update_tag=gcp_update_tag,
143
163
  )
144
164
 
145
165
 
146
166
  @timeit
147
- def load_gcp_projects(neo4j_session: neo4j.Session, data: List[Dict], gcp_update_tag: int) -> None:
167
+ def load_gcp_projects(
168
+ neo4j_session: neo4j.Session,
169
+ data: List[Dict],
170
+ gcp_update_tag: int,
171
+ ) -> None:
148
172
  """
149
173
  Ingest the GCP projects to Neo4j
150
174
  :param neo4j_session: The Neo4j session
@@ -165,35 +189,40 @@ def load_gcp_projects(neo4j_session: neo4j.Session, data: List[Dict], gcp_update
165
189
  for project in data:
166
190
  neo4j_session.run(
167
191
  query,
168
- ProjectId=project['projectId'],
169
- ProjectNumber=project['projectNumber'],
170
- DisplayName=project.get('name', None),
171
- LifecycleState=project.get('lifecycleState', None),
192
+ ProjectId=project["projectId"],
193
+ ProjectNumber=project["projectNumber"],
194
+ DisplayName=project.get("name", None),
195
+ LifecycleState=project.get("lifecycleState", None),
172
196
  gcp_update_tag=gcp_update_tag,
173
197
  )
174
- if project.get('parent'):
198
+ if project.get("parent"):
175
199
  _attach_gcp_project_parent(neo4j_session, project, gcp_update_tag)
176
200
 
177
201
 
178
202
  @timeit
179
- def _attach_gcp_project_parent(neo4j_session: neo4j.Session, project: Dict, gcp_update_tag: int) -> None:
203
+ def _attach_gcp_project_parent(
204
+ neo4j_session: neo4j.Session,
205
+ project: Dict,
206
+ gcp_update_tag: int,
207
+ ) -> None:
180
208
  """
181
209
  Attach a project to its respective parent, as in the Resource Hierarchy -
182
210
  https://cloud.google.com/resource-manager/docs/cloud-platform-resource-hierarchy
183
211
  """
184
- if project['parent']['type'] == 'organization':
185
- parent_label = 'GCPOrganization'
186
- elif project['parent']['type'] == 'folder':
187
- parent_label = 'GCPFolder'
212
+ if project["parent"]["type"] == "organization":
213
+ parent_label = "GCPOrganization"
214
+ elif project["parent"]["type"] == "folder":
215
+ parent_label = "GCPFolder"
188
216
  else:
189
217
  raise NotImplementedError(
190
218
  "Ingestion of GCP {}s as parent nodes is currently not supported. "
191
- "Please file an issue at https://github.com/lyft/cartography/issues.".format(
192
- project['parent']['type'],
219
+ "Please file an issue at https://github.com/cartography-cncf/cartography/issues.".format(
220
+ project["parent"]["type"],
193
221
  ),
194
222
  )
195
223
  parent_id = f"{project['parent']['type']}s/{project['parent']['id']}"
196
- INGEST_PARENT_TEMPLATE = Template("""
224
+ INGEST_PARENT_TEMPLATE = Template(
225
+ """
197
226
  MATCH (project:GCPProject{id:$ProjectId})
198
227
 
199
228
  MERGE (parent:$parent_label{id:$ParentId})
@@ -202,51 +231,71 @@ def _attach_gcp_project_parent(neo4j_session: neo4j.Session, project: Dict, gcp_
202
231
  MERGE (parent)-[r:RESOURCE]->(project)
203
232
  ON CREATE SET r.firstseen = timestamp()
204
233
  SET r.lastupdated = $gcp_update_tag
205
- """)
234
+ """,
235
+ )
206
236
  neo4j_session.run(
207
237
  INGEST_PARENT_TEMPLATE.safe_substitute(parent_label=parent_label),
208
238
  ParentId=parent_id,
209
- ProjectId=project['projectId'],
239
+ ProjectId=project["projectId"],
210
240
  gcp_update_tag=gcp_update_tag,
211
241
  )
212
242
 
213
243
 
214
244
  @timeit
215
- def cleanup_gcp_organizations(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
245
+ def cleanup_gcp_organizations(
246
+ neo4j_session: neo4j.Session,
247
+ common_job_parameters: Dict,
248
+ ) -> None:
216
249
  """
217
250
  Remove stale GCP organizations and their relationships
218
251
  :param neo4j_session: The Neo4j session
219
252
  :param common_job_parameters: Parameters to carry to the cleanup job
220
253
  :return: Nothing
221
254
  """
222
- run_cleanup_job('gcp_crm_organization_cleanup.json', neo4j_session, common_job_parameters)
255
+ run_cleanup_job(
256
+ "gcp_crm_organization_cleanup.json",
257
+ neo4j_session,
258
+ common_job_parameters,
259
+ )
223
260
 
224
261
 
225
262
  @timeit
226
- def cleanup_gcp_folders(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
263
+ def cleanup_gcp_folders(
264
+ neo4j_session: neo4j.Session,
265
+ common_job_parameters: Dict,
266
+ ) -> None:
227
267
  """
228
268
  Remove stale GCP folders and their relationships
229
269
  :param neo4j_session: The Neo4j session
230
270
  :param common_job_parameters: Parameters to carry to the cleanup job
231
271
  :return: Nothing
232
272
  """
233
- run_cleanup_job('gcp_crm_folder_cleanup.json', neo4j_session, common_job_parameters)
273
+ run_cleanup_job("gcp_crm_folder_cleanup.json", neo4j_session, common_job_parameters)
234
274
 
235
275
 
236
276
  @timeit
237
- def cleanup_gcp_projects(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
277
+ def cleanup_gcp_projects(
278
+ neo4j_session: neo4j.Session,
279
+ common_job_parameters: Dict,
280
+ ) -> None:
238
281
  """
239
282
  Remove stale GCP projects and their relationships
240
283
  :param neo4j_session: The Neo4j session
241
284
  :param common_job_parameters: Parameters to carry to the cleanup job
242
285
  :return: Nothing
243
286
  """
244
- run_cleanup_job('gcp_crm_project_cleanup.json', neo4j_session, common_job_parameters)
287
+ run_cleanup_job(
288
+ "gcp_crm_project_cleanup.json",
289
+ neo4j_session,
290
+ common_job_parameters,
291
+ )
245
292
 
246
293
 
247
294
  @timeit
248
295
  def sync_gcp_organizations(
249
- neo4j_session: neo4j.Session, crm_v1: Resource, gcp_update_tag: int,
296
+ neo4j_session: neo4j.Session,
297
+ crm_v1: Resource,
298
+ gcp_update_tag: int,
250
299
  common_job_parameters: Dict,
251
300
  ) -> None:
252
301
  """
@@ -266,7 +315,9 @@ def sync_gcp_organizations(
266
315
 
267
316
  @timeit
268
317
  def sync_gcp_folders(
269
- neo4j_session: neo4j.Session, crm_v2: Resource, gcp_update_tag: int,
318
+ neo4j_session: neo4j.Session,
319
+ crm_v2: Resource,
320
+ gcp_update_tag: int,
270
321
  common_job_parameters: Dict,
271
322
  ) -> None:
272
323
  """
@@ -286,7 +337,9 @@ def sync_gcp_folders(
286
337
 
287
338
  @timeit
288
339
  def sync_gcp_projects(
289
- neo4j_session: neo4j.Session, projects: List[Dict], gcp_update_tag: int,
340
+ neo4j_session: neo4j.Session,
341
+ projects: List[Dict],
342
+ gcp_update_tag: int,
290
343
  common_job_parameters: Dict,
291
344
  ) -> None:
292
345
  """
@@ -32,17 +32,26 @@ def get_dns_zones(dns: Resource, project_id: str) -> List[Resource]:
32
32
  request = dns.managedZones().list(project=project_id)
33
33
  while request is not None:
34
34
  response = request.execute()
35
- for managed_zone in response['managedZones']:
35
+ for managed_zone in response["managedZones"]:
36
36
  zones.append(managed_zone)
37
- request = dns.managedZones().list_next(previous_request=request, previous_response=response)
37
+ request = dns.managedZones().list_next(
38
+ previous_request=request,
39
+ previous_response=response,
40
+ )
38
41
  return zones
39
42
  except HttpError as e:
40
- err = json.loads(e.content.decode('utf-8'))['error']
41
- if err.get('status', '') == 'PERMISSION_DENIED' or err.get('message', '') == 'Forbidden':
43
+ err = json.loads(e.content.decode("utf-8"))["error"]
44
+ if (
45
+ err.get("status", "") == "PERMISSION_DENIED"
46
+ or err.get("message", "") == "Forbidden"
47
+ ):
42
48
  logger.warning(
43
49
  (
44
50
  "Could not retrieve DNS zones on project %s due to permissions issues. Code: %s, Message: %s"
45
- ), project_id, err['code'], err['message'],
51
+ ),
52
+ project_id,
53
+ err["code"],
54
+ err["message"],
46
55
  )
47
56
  return []
48
57
  else:
@@ -50,7 +59,11 @@ def get_dns_zones(dns: Resource, project_id: str) -> List[Resource]:
50
59
 
51
60
 
52
61
  @timeit
53
- def get_dns_rrs(dns: Resource, dns_zones: List[Dict], project_id: str) -> List[Resource]:
62
+ def get_dns_rrs(
63
+ dns: Resource,
64
+ dns_zones: List[Dict],
65
+ project_id: str,
66
+ ) -> List[Resource]:
54
67
  """
55
68
  Returns a list of DNS Resource Record Sets within the given project.
56
69
 
@@ -69,21 +82,33 @@ def get_dns_rrs(dns: Resource, dns_zones: List[Dict], project_id: str) -> List[R
69
82
  try:
70
83
  rrs: List[Resource] = []
71
84
  for zone in dns_zones:
72
- request = dns.resourceRecordSets().list(project=project_id, managedZone=zone['id'])
85
+ request = dns.resourceRecordSets().list(
86
+ project=project_id,
87
+ managedZone=zone["id"],
88
+ )
73
89
  while request is not None:
74
90
  response = request.execute()
75
- for resource_record_set in response['rrsets']:
76
- resource_record_set['zone'] = zone['id']
91
+ for resource_record_set in response["rrsets"]:
92
+ resource_record_set["zone"] = zone["id"]
77
93
  rrs.append(resource_record_set)
78
- request = dns.resourceRecordSets().list_next(previous_request=request, previous_response=response)
94
+ request = dns.resourceRecordSets().list_next(
95
+ previous_request=request,
96
+ previous_response=response,
97
+ )
79
98
  return rrs
80
99
  except HttpError as e:
81
- err = json.loads(e.content.decode('utf-8'))['error']
82
- if err.get('status', '') == 'PERMISSION_DENIED' or err.get('message', '') == 'Forbidden':
100
+ err = json.loads(e.content.decode("utf-8"))["error"]
101
+ if (
102
+ err.get("status", "") == "PERMISSION_DENIED"
103
+ or err.get("message", "") == "Forbidden"
104
+ ):
83
105
  logger.warning(
84
106
  (
85
107
  "Could not retrieve DNS RRS on project %s due to permissions issues. Code: %s, Message: %s"
86
- ), project_id, err['code'], err['message'],
108
+ ),
109
+ project_id,
110
+ err["code"],
111
+ err["message"],
87
112
  )
88
113
  return []
89
114
  else:
@@ -92,7 +117,12 @@ def get_dns_rrs(dns: Resource, dns_zones: List[Dict], project_id: str) -> List[R
92
117
 
93
118
 
94
119
  @timeit
95
- def load_dns_zones(neo4j_session: neo4j.Session, dns_zones: List[Dict], project_id: str, gcp_update_tag: int) -> None:
120
+ def load_dns_zones(
121
+ neo4j_session: neo4j.Session,
122
+ dns_zones: List[Dict],
123
+ project_id: str,
124
+ gcp_update_tag: int,
125
+ ) -> None:
96
126
  """
97
127
  Ingest GCP DNS Zones into Neo4j
98
128
 
@@ -142,7 +172,12 @@ def load_dns_zones(neo4j_session: neo4j.Session, dns_zones: List[Dict], project_
142
172
 
143
173
 
144
174
  @timeit
145
- def load_rrs(neo4j_session: neo4j.Session, dns_rrs: List[Resource], project_id: str, gcp_update_tag: int) -> None:
175
+ def load_rrs(
176
+ neo4j_session: neo4j.Session,
177
+ dns_rrs: List[Resource],
178
+ project_id: str,
179
+ gcp_update_tag: int,
180
+ ) -> None:
146
181
  """
147
182
  Ingest GCP RRS into Neo4j
148
183
 
@@ -188,7 +223,10 @@ def load_rrs(neo4j_session: neo4j.Session, dns_rrs: List[Resource], project_id:
188
223
 
189
224
 
190
225
  @timeit
191
- def cleanup_dns_records(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
226
+ def cleanup_dns_records(
227
+ neo4j_session: neo4j.Session,
228
+ common_job_parameters: Dict,
229
+ ) -> None:
192
230
  """
193
231
  Delete out-of-date GCP DNS Zones and RRS nodes and relationships
194
232
 
@@ -201,12 +239,15 @@ def cleanup_dns_records(neo4j_session: neo4j.Session, common_job_parameters: Dic
201
239
  :rtype: NoneType
202
240
  :return: Nothing
203
241
  """
204
- run_cleanup_job('gcp_dns_cleanup.json', neo4j_session, common_job_parameters)
242
+ run_cleanup_job("gcp_dns_cleanup.json", neo4j_session, common_job_parameters)
205
243
 
206
244
 
207
245
  @timeit
208
246
  def sync(
209
- neo4j_session: neo4j.Session, dns: Resource, project_id: str, gcp_update_tag: int,
247
+ neo4j_session: neo4j.Session,
248
+ dns: Resource,
249
+ project_id: str,
250
+ gcp_update_tag: int,
210
251
  common_job_parameters: Dict,
211
252
  ) -> None:
212
253
  """
@@ -237,5 +278,5 @@ def sync(
237
278
  # RECORD SETS
238
279
  dns_rrs = get_dns_rrs(dns, dns_zones, project_id)
239
280
  load_rrs(neo4j_session, dns_rrs, project_id, gcp_update_tag)
240
- # TODO scope the cleanup to the current project - https://github.com/lyft/cartography/issues/381
281
+ # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
241
282
  cleanup_dns_records(neo4j_session, common_job_parameters)
@@ -8,6 +8,7 @@ from googleapiclient.discovery import Resource
8
8
 
9
9
  from cartography.util import run_cleanup_job
10
10
  from cartography.util import timeit
11
+
11
12
  logger = logging.getLogger(__name__)
12
13
 
13
14
 
@@ -26,16 +27,21 @@ def get_gke_clusters(container: Resource, project_id: str) -> Dict:
26
27
  :return: Cluster response object
27
28
  """
28
29
  try:
29
- req = container.projects().zones().clusters().list(projectId=project_id, zone='-')
30
+ req = (
31
+ container.projects().zones().clusters().list(projectId=project_id, zone="-")
32
+ )
30
33
  res = req.execute()
31
34
  return res
32
35
  except HttpError as e:
33
- err = json.loads(e.content.decode('utf-8'))['error']
34
- if err['status'] == 'PERMISSION_DENIED':
36
+ err = json.loads(e.content.decode("utf-8"))["error"]
37
+ if err["status"] == "PERMISSION_DENIED":
35
38
  logger.warning(
36
39
  (
37
40
  "Could not retrieve GKE clusters on project %s due to permissions issue. Code: %s, Message: %s"
38
- ), project_id, err['code'], err['message'],
41
+ ),
42
+ project_id,
43
+ err["code"],
44
+ err["message"],
39
45
  )
40
46
  return {}
41
47
  else:
@@ -43,7 +49,12 @@ def get_gke_clusters(container: Resource, project_id: str) -> Dict:
43
49
 
44
50
 
45
51
  @timeit
46
- def load_gke_clusters(neo4j_session: neo4j.Session, cluster_resp: Dict, project_id: str, gcp_update_tag: int) -> None:
52
+ def load_gke_clusters(
53
+ neo4j_session: neo4j.Session,
54
+ cluster_resp: Dict,
55
+ project_id: str,
56
+ gcp_update_tag: int,
57
+ ) -> None:
47
58
  """
48
59
  Ingest GCP GKE Clusters to Neo4j
49
60
 
@@ -98,37 +109,50 @@ def load_gke_clusters(neo4j_session: neo4j.Session, cluster_resp: Dict, project_
98
109
  ON CREATE SET r.firstseen = timestamp()
99
110
  SET r.lastupdated = $gcp_update_tag
100
111
  """
101
- for cluster in cluster_resp.get('clusters', []):
112
+ for cluster in cluster_resp.get("clusters", []):
102
113
  neo4j_session.run(
103
114
  query,
104
115
  ProjectId=project_id,
105
- ClusterSelfLink=cluster['selfLink'],
106
- ClusterCreateTime=cluster['createTime'],
107
- ClusterName=cluster['name'],
108
- ClusterDescription=cluster.get('description'),
109
- ClusterLoggingService=cluster.get('loggingService'),
110
- ClusterMonitoringService=cluster.get('monitoringService'),
111
- ClusterNetwork=cluster.get('network'),
112
- ClusterSubnetwork=cluster.get('subnetwork'),
113
- ClusterIPv4Cidr=cluster.get('clusterIpv4Cidr'),
114
- ClusterZone=cluster.get('zone'),
115
- ClusterLocation=cluster.get('location'),
116
- ClusterEndpoint=cluster.get('endpoint'),
117
- ClusterInitialVersion=cluster.get('initialClusterVersion'),
118
- ClusterMasterVersion=cluster.get('currentMasterVersion'),
119
- ClusterStatus=cluster.get('status'),
120
- ClusterServicesIPv4Cidr=cluster.get('servicesIpv4Cidr'),
121
- ClusterDatabaseEncryption=cluster.get('databaseEncryption', {}).get('state'),
116
+ ClusterSelfLink=cluster["selfLink"],
117
+ ClusterCreateTime=cluster["createTime"],
118
+ ClusterName=cluster["name"],
119
+ ClusterDescription=cluster.get("description"),
120
+ ClusterLoggingService=cluster.get("loggingService"),
121
+ ClusterMonitoringService=cluster.get("monitoringService"),
122
+ ClusterNetwork=cluster.get("network"),
123
+ ClusterSubnetwork=cluster.get("subnetwork"),
124
+ ClusterIPv4Cidr=cluster.get("clusterIpv4Cidr"),
125
+ ClusterZone=cluster.get("zone"),
126
+ ClusterLocation=cluster.get("location"),
127
+ ClusterEndpoint=cluster.get("endpoint"),
128
+ ClusterInitialVersion=cluster.get("initialClusterVersion"),
129
+ ClusterMasterVersion=cluster.get("currentMasterVersion"),
130
+ ClusterStatus=cluster.get("status"),
131
+ ClusterServicesIPv4Cidr=cluster.get("servicesIpv4Cidr"),
132
+ ClusterDatabaseEncryption=cluster.get("databaseEncryption", {}).get(
133
+ "state",
134
+ ),
122
135
  ClusterNetworkPolicy=_process_network_policy(cluster),
123
- ClusterMasterAuthorizedNetworks=cluster.get('masterAuthorizedNetworksConfig', {}).get('enabled'),
124
- ClusterAbac=cluster.get('legacyAbac', {}).get('enabled'),
125
- ClusterShieldedNodes=cluster.get('shieldedNodes', {}).get('enabled'),
126
- ClusterPrivateNodes=cluster.get('privateClusterConfig', {}).get('enablePrivateNodes'),
127
- ClusterPrivateEndpointEnabled=cluster.get('privateClusterConfig', {}).get('enablePrivateEndpoint'),
128
- ClusterPrivateEndpoint=cluster.get('privateClusterConfig', {}).get('privateEndpoint'),
129
- ClusterPublicEndpoint=cluster.get('privateClusterConfig', {}).get('publicEndpoint'),
130
- ClusterMasterUsername=cluster.get('masterAuth', {}).get('username'),
131
- ClusterMasterPassword=cluster.get('masterAuth', {}).get('password'),
136
+ ClusterMasterAuthorizedNetworks=cluster.get(
137
+ "masterAuthorizedNetworksConfig",
138
+ {},
139
+ ).get("enabled"),
140
+ ClusterAbac=cluster.get("legacyAbac", {}).get("enabled"),
141
+ ClusterShieldedNodes=cluster.get("shieldedNodes", {}).get("enabled"),
142
+ ClusterPrivateNodes=cluster.get("privateClusterConfig", {}).get(
143
+ "enablePrivateNodes",
144
+ ),
145
+ ClusterPrivateEndpointEnabled=cluster.get("privateClusterConfig", {}).get(
146
+ "enablePrivateEndpoint",
147
+ ),
148
+ ClusterPrivateEndpoint=cluster.get("privateClusterConfig", {}).get(
149
+ "privateEndpoint",
150
+ ),
151
+ ClusterPublicEndpoint=cluster.get("privateClusterConfig", {}).get(
152
+ "publicEndpoint",
153
+ ),
154
+ ClusterMasterUsername=cluster.get("masterAuth", {}).get("username"),
155
+ ClusterMasterPassword=cluster.get("masterAuth", {}).get("password"),
132
156
  gcp_update_tag=gcp_update_tag,
133
157
  )
134
158
 
@@ -138,15 +162,18 @@ def _process_network_policy(cluster: Dict) -> bool:
138
162
  Parse cluster.networkPolicy to verify if
139
163
  the provider has been enabled.
140
164
  """
141
- provider = cluster.get('networkPolicy', {}).get('provider')
142
- enabled = cluster.get('networkPolicy', {}).get('enabled')
165
+ provider = cluster.get("networkPolicy", {}).get("provider")
166
+ enabled = cluster.get("networkPolicy", {}).get("enabled")
143
167
  if provider and enabled is True:
144
168
  return provider
145
169
  return False
146
170
 
147
171
 
148
172
  @timeit
149
- def cleanup_gke_clusters(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
173
+ def cleanup_gke_clusters(
174
+ neo4j_session: neo4j.Session,
175
+ common_job_parameters: Dict,
176
+ ) -> None:
150
177
  """
151
178
  Delete out-of-date GCP GKE Clusters nodes and relationships
152
179
 
@@ -159,12 +186,19 @@ def cleanup_gke_clusters(neo4j_session: neo4j.Session, common_job_parameters: Di
159
186
  :rtype: NoneType
160
187
  :return: Nothing
161
188
  """
162
- run_cleanup_job('gcp_gke_cluster_cleanup.json', neo4j_session, common_job_parameters)
189
+ run_cleanup_job(
190
+ "gcp_gke_cluster_cleanup.json",
191
+ neo4j_session,
192
+ common_job_parameters,
193
+ )
163
194
 
164
195
 
165
196
  @timeit
166
197
  def sync_gke_clusters(
167
- neo4j_session: neo4j.Session, container: Resource, project_id: str, gcp_update_tag: int,
198
+ neo4j_session: neo4j.Session,
199
+ container: Resource,
200
+ project_id: str,
201
+ gcp_update_tag: int,
168
202
  common_job_parameters: Dict,
169
203
  ) -> None:
170
204
  """
@@ -191,5 +225,5 @@ def sync_gke_clusters(
191
225
  logger.info("Syncing Compute objects for project %s.", project_id)
192
226
  gke_res = get_gke_clusters(container, project_id)
193
227
  load_gke_clusters(neo4j_session, gke_res, project_id, gcp_update_tag)
194
- # TODO scope the cleanup to the current project - https://github.com/lyft/cartography/issues/381
228
+ # TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
195
229
  cleanup_gke_clusters(neo4j_session, common_job_parameters)