cartography 0.102.0rc1__py3-none-any.whl → 0.103.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

Files changed (297) hide show
  1. cartography/__main__.py +1 -2
  2. cartography/_version.py +2 -2
  3. cartography/cli.py +376 -249
  4. cartography/client/core/tx.py +39 -18
  5. cartography/config.py +28 -0
  6. cartography/driftdetect/__main__.py +1 -2
  7. cartography/driftdetect/add_shortcut.py +10 -2
  8. cartography/driftdetect/cli.py +71 -75
  9. cartography/driftdetect/detect_deviations.py +7 -3
  10. cartography/driftdetect/get_states.py +20 -8
  11. cartography/driftdetect/model.py +5 -5
  12. cartography/driftdetect/serializers.py +8 -6
  13. cartography/driftdetect/storage.py +2 -2
  14. cartography/graph/cleanupbuilder.py +35 -15
  15. cartography/graph/job.py +46 -17
  16. cartography/graph/querybuilder.py +165 -80
  17. cartography/graph/statement.py +35 -26
  18. cartography/intel/analysis.py +4 -1
  19. cartography/intel/aws/__init__.py +114 -55
  20. cartography/intel/aws/apigateway.py +134 -63
  21. cartography/intel/aws/cloudtrail.py +127 -0
  22. cartography/intel/aws/cloudwatch.py +93 -0
  23. cartography/intel/aws/config.py +56 -20
  24. cartography/intel/aws/dynamodb.py +108 -40
  25. cartography/intel/aws/ec2/__init__.py +2 -2
  26. cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
  27. cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
  28. cartography/intel/aws/ec2/images.py +49 -20
  29. cartography/intel/aws/ec2/instances.py +234 -136
  30. cartography/intel/aws/ec2/internet_gateways.py +40 -11
  31. cartography/intel/aws/ec2/key_pairs.py +44 -20
  32. cartography/intel/aws/ec2/launch_templates.py +101 -59
  33. cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
  34. cartography/intel/aws/ec2/load_balancers.py +82 -42
  35. cartography/intel/aws/ec2/network_acls.py +89 -65
  36. cartography/intel/aws/ec2/network_interfaces.py +146 -87
  37. cartography/intel/aws/ec2/reserved_instances.py +45 -16
  38. cartography/intel/aws/ec2/route_tables.py +327 -0
  39. cartography/intel/aws/ec2/security_groups.py +71 -21
  40. cartography/intel/aws/ec2/snapshots.py +61 -22
  41. cartography/intel/aws/ec2/subnets.py +54 -18
  42. cartography/intel/aws/ec2/tgw.py +100 -34
  43. cartography/intel/aws/ec2/util.py +1 -1
  44. cartography/intel/aws/ec2/volumes.py +69 -41
  45. cartography/intel/aws/ec2/vpc.py +37 -12
  46. cartography/intel/aws/ec2/vpc_peerings.py +83 -24
  47. cartography/intel/aws/ecr.py +88 -32
  48. cartography/intel/aws/ecs.py +83 -47
  49. cartography/intel/aws/efs.py +93 -0
  50. cartography/intel/aws/eks.py +55 -29
  51. cartography/intel/aws/elasticache.py +42 -18
  52. cartography/intel/aws/elasticsearch.py +57 -20
  53. cartography/intel/aws/emr.py +61 -23
  54. cartography/intel/aws/iam.py +401 -145
  55. cartography/intel/aws/iam_instance_profiles.py +22 -22
  56. cartography/intel/aws/identitycenter.py +71 -37
  57. cartography/intel/aws/inspector.py +159 -89
  58. cartography/intel/aws/kms.py +92 -38
  59. cartography/intel/aws/lambda_function.py +103 -34
  60. cartography/intel/aws/organizations.py +30 -10
  61. cartography/intel/aws/permission_relationships.py +133 -51
  62. cartography/intel/aws/rds.py +249 -85
  63. cartography/intel/aws/redshift.py +107 -46
  64. cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
  65. cartography/intel/aws/resources.py +57 -44
  66. cartography/intel/aws/route53.py +108 -61
  67. cartography/intel/aws/s3.py +168 -83
  68. cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
  69. cartography/intel/aws/secretsmanager.py +24 -12
  70. cartography/intel/aws/securityhub.py +20 -9
  71. cartography/intel/aws/sns.py +166 -0
  72. cartography/intel/aws/sqs.py +60 -28
  73. cartography/intel/aws/ssm.py +70 -30
  74. cartography/intel/aws/util/arns.py +7 -7
  75. cartography/intel/aws/util/common.py +31 -4
  76. cartography/intel/azure/__init__.py +78 -19
  77. cartography/intel/azure/compute.py +101 -27
  78. cartography/intel/azure/cosmosdb.py +496 -170
  79. cartography/intel/azure/sql.py +296 -105
  80. cartography/intel/azure/storage.py +322 -113
  81. cartography/intel/azure/subscription.py +39 -23
  82. cartography/intel/azure/tenant.py +13 -4
  83. cartography/intel/azure/util/credentials.py +95 -55
  84. cartography/intel/bigfix/__init__.py +2 -2
  85. cartography/intel/bigfix/computers.py +93 -65
  86. cartography/intel/cloudflare/__init__.py +74 -0
  87. cartography/intel/cloudflare/accounts.py +57 -0
  88. cartography/intel/cloudflare/dnsrecords.py +64 -0
  89. cartography/intel/cloudflare/members.py +75 -0
  90. cartography/intel/cloudflare/roles.py +65 -0
  91. cartography/intel/cloudflare/zones.py +64 -0
  92. cartography/intel/create_indexes.py +3 -2
  93. cartography/intel/crowdstrike/__init__.py +11 -9
  94. cartography/intel/crowdstrike/endpoints.py +5 -1
  95. cartography/intel/crowdstrike/spotlight.py +8 -3
  96. cartography/intel/cve/__init__.py +46 -13
  97. cartography/intel/cve/feed.py +48 -12
  98. cartography/intel/digitalocean/__init__.py +22 -13
  99. cartography/intel/digitalocean/compute.py +75 -108
  100. cartography/intel/digitalocean/management.py +44 -80
  101. cartography/intel/digitalocean/platform.py +48 -43
  102. cartography/intel/dns.py +36 -10
  103. cartography/intel/duo/__init__.py +21 -16
  104. cartography/intel/duo/api_host.py +14 -9
  105. cartography/intel/duo/endpoints.py +50 -45
  106. cartography/intel/duo/groups.py +18 -14
  107. cartography/intel/duo/phones.py +37 -34
  108. cartography/intel/duo/tokens.py +26 -23
  109. cartography/intel/duo/users.py +54 -50
  110. cartography/intel/duo/web_authn_credentials.py +30 -25
  111. cartography/intel/entra/__init__.py +25 -7
  112. cartography/intel/entra/ou.py +112 -0
  113. cartography/intel/entra/users.py +69 -63
  114. cartography/intel/gcp/__init__.py +185 -49
  115. cartography/intel/gcp/compute.py +418 -231
  116. cartography/intel/gcp/crm.py +96 -43
  117. cartography/intel/gcp/dns.py +60 -19
  118. cartography/intel/gcp/gke.py +72 -38
  119. cartography/intel/gcp/iam.py +61 -41
  120. cartography/intel/gcp/storage.py +84 -55
  121. cartography/intel/github/__init__.py +13 -11
  122. cartography/intel/github/repos.py +270 -137
  123. cartography/intel/github/teams.py +170 -88
  124. cartography/intel/github/users.py +70 -39
  125. cartography/intel/github/util.py +36 -34
  126. cartography/intel/gsuite/__init__.py +47 -26
  127. cartography/intel/gsuite/api.py +73 -30
  128. cartography/intel/jamf/__init__.py +19 -1
  129. cartography/intel/jamf/computers.py +30 -7
  130. cartography/intel/jamf/util.py +7 -2
  131. cartography/intel/kandji/__init__.py +6 -3
  132. cartography/intel/kandji/devices.py +14 -8
  133. cartography/intel/kubernetes/namespaces.py +7 -4
  134. cartography/intel/kubernetes/pods.py +7 -4
  135. cartography/intel/kubernetes/services.py +8 -4
  136. cartography/intel/lastpass/__init__.py +2 -2
  137. cartography/intel/lastpass/users.py +23 -12
  138. cartography/intel/oci/__init__.py +44 -11
  139. cartography/intel/oci/iam.py +134 -38
  140. cartography/intel/oci/organizations.py +13 -6
  141. cartography/intel/oci/utils.py +43 -20
  142. cartography/intel/okta/__init__.py +66 -15
  143. cartography/intel/okta/applications.py +42 -20
  144. cartography/intel/okta/awssaml.py +93 -33
  145. cartography/intel/okta/factors.py +16 -4
  146. cartography/intel/okta/groups.py +56 -29
  147. cartography/intel/okta/organization.py +5 -1
  148. cartography/intel/okta/origins.py +6 -2
  149. cartography/intel/okta/roles.py +15 -5
  150. cartography/intel/okta/users.py +20 -8
  151. cartography/intel/okta/utils.py +6 -4
  152. cartography/intel/openai/__init__.py +86 -0
  153. cartography/intel/openai/adminapikeys.py +90 -0
  154. cartography/intel/openai/apikeys.py +96 -0
  155. cartography/intel/openai/projects.py +94 -0
  156. cartography/intel/openai/serviceaccounts.py +82 -0
  157. cartography/intel/openai/users.py +78 -0
  158. cartography/intel/openai/util.py +29 -0
  159. cartography/intel/pagerduty/__init__.py +8 -7
  160. cartography/intel/pagerduty/escalation_policies.py +18 -6
  161. cartography/intel/pagerduty/schedules.py +12 -4
  162. cartography/intel/pagerduty/services.py +11 -4
  163. cartography/intel/pagerduty/teams.py +8 -3
  164. cartography/intel/pagerduty/users.py +3 -1
  165. cartography/intel/pagerduty/vendors.py +3 -1
  166. cartography/intel/semgrep/__init__.py +24 -6
  167. cartography/intel/semgrep/dependencies.py +50 -28
  168. cartography/intel/semgrep/deployment.py +3 -1
  169. cartography/intel/semgrep/findings.py +42 -18
  170. cartography/intel/snipeit/__init__.py +17 -3
  171. cartography/intel/snipeit/asset.py +12 -6
  172. cartography/intel/snipeit/user.py +8 -5
  173. cartography/intel/snipeit/util.py +9 -4
  174. cartography/intel/tailscale/__init__.py +77 -0
  175. cartography/intel/tailscale/acls.py +146 -0
  176. cartography/intel/tailscale/devices.py +127 -0
  177. cartography/intel/tailscale/postureintegrations.py +81 -0
  178. cartography/intel/tailscale/tailnets.py +76 -0
  179. cartography/intel/tailscale/users.py +80 -0
  180. cartography/intel/tailscale/utils.py +132 -0
  181. cartography/models/aws/apigateway.py +21 -17
  182. cartography/models/aws/apigatewaycertificate.py +28 -22
  183. cartography/models/aws/apigatewayresource.py +28 -20
  184. cartography/models/aws/apigatewaystage.py +33 -25
  185. cartography/models/aws/cloudtrail/__init__.py +0 -0
  186. cartography/models/aws/cloudtrail/trail.py +61 -0
  187. cartography/models/aws/cloudwatch/__init__.py +0 -0
  188. cartography/models/aws/cloudwatch/loggroup.py +52 -0
  189. cartography/models/aws/dynamodb/gsi.py +30 -22
  190. cartography/models/aws/dynamodb/tables.py +25 -17
  191. cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
  192. cartography/models/aws/ec2/images.py +36 -34
  193. cartography/models/aws/ec2/instances.py +51 -45
  194. cartography/models/aws/ec2/keypair.py +21 -16
  195. cartography/models/aws/ec2/keypair_instance.py +28 -21
  196. cartography/models/aws/ec2/launch_configurations.py +30 -26
  197. cartography/models/aws/ec2/launch_template_versions.py +48 -38
  198. cartography/models/aws/ec2/launch_templates.py +21 -17
  199. cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
  200. cartography/models/aws/ec2/load_balancers.py +47 -37
  201. cartography/models/aws/ec2/network_acl_rules.py +38 -30
  202. cartography/models/aws/ec2/network_acls.py +38 -29
  203. cartography/models/aws/ec2/networkinterface_instance.py +52 -39
  204. cartography/models/aws/ec2/networkinterfaces.py +53 -37
  205. cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
  206. cartography/models/aws/ec2/reservations.py +18 -14
  207. cartography/models/aws/ec2/route_table_associations.py +97 -0
  208. cartography/models/aws/ec2/route_tables.py +128 -0
  209. cartography/models/aws/ec2/routes.py +85 -0
  210. cartography/models/aws/ec2/securitygroup_instance.py +29 -20
  211. cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
  212. cartography/models/aws/ec2/subnet_instance.py +24 -19
  213. cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
  214. cartography/models/aws/ec2/volumes.py +47 -40
  215. cartography/models/aws/efs/__init__.py +0 -0
  216. cartography/models/aws/efs/mount_target.py +52 -0
  217. cartography/models/aws/eks/clusters.py +23 -21
  218. cartography/models/aws/emr.py +32 -30
  219. cartography/models/aws/iam/instanceprofile.py +33 -24
  220. cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
  221. cartography/models/aws/identitycenter/awspermissionset.py +37 -29
  222. cartography/models/aws/identitycenter/awsssouser.py +23 -21
  223. cartography/models/aws/inspector/findings.py +77 -65
  224. cartography/models/aws/inspector/packages.py +35 -29
  225. cartography/models/aws/s3/__init__.py +0 -0
  226. cartography/models/aws/s3/account_public_access_block.py +51 -0
  227. cartography/models/aws/sns/__init__.py +0 -0
  228. cartography/models/aws/sns/topic.py +50 -0
  229. cartography/models/aws/ssm/instance_information.py +51 -39
  230. cartography/models/aws/ssm/instance_patch.py +32 -26
  231. cartography/models/bigfix/bigfix_computer.py +42 -38
  232. cartography/models/bigfix/bigfix_root.py +3 -3
  233. cartography/models/cloudflare/__init__.py +0 -0
  234. cartography/models/cloudflare/account.py +25 -0
  235. cartography/models/cloudflare/dnsrecord.py +55 -0
  236. cartography/models/cloudflare/member.py +82 -0
  237. cartography/models/cloudflare/role.py +44 -0
  238. cartography/models/cloudflare/zone.py +59 -0
  239. cartography/models/core/common.py +12 -10
  240. cartography/models/core/nodes.py +5 -2
  241. cartography/models/core/relationships.py +14 -6
  242. cartography/models/crowdstrike/hosts.py +37 -35
  243. cartography/models/cve/cve.py +34 -32
  244. cartography/models/cve/cve_feed.py +6 -6
  245. cartography/models/digitalocean/__init__.py +0 -0
  246. cartography/models/digitalocean/account.py +21 -0
  247. cartography/models/digitalocean/droplet.py +56 -0
  248. cartography/models/digitalocean/project.py +48 -0
  249. cartography/models/duo/api_host.py +3 -3
  250. cartography/models/duo/endpoint.py +43 -41
  251. cartography/models/duo/group.py +14 -14
  252. cartography/models/duo/phone.py +27 -27
  253. cartography/models/duo/token.py +16 -16
  254. cartography/models/duo/user.py +46 -44
  255. cartography/models/duo/web_authn_credential.py +27 -19
  256. cartography/models/entra/ou.py +48 -0
  257. cartography/models/entra/tenant.py +24 -18
  258. cartography/models/entra/user.py +64 -48
  259. cartography/models/gcp/iam.py +23 -23
  260. cartography/models/github/orgs.py +5 -4
  261. cartography/models/github/teams.py +37 -31
  262. cartography/models/github/users.py +34 -23
  263. cartography/models/kandji/device.py +22 -16
  264. cartography/models/kandji/tenant.py +6 -4
  265. cartography/models/lastpass/tenant.py +3 -3
  266. cartography/models/lastpass/user.py +32 -28
  267. cartography/models/openai/__init__.py +0 -0
  268. cartography/models/openai/adminapikey.py +90 -0
  269. cartography/models/openai/apikey.py +84 -0
  270. cartography/models/openai/organization.py +17 -0
  271. cartography/models/openai/project.py +70 -0
  272. cartography/models/openai/serviceaccount.py +50 -0
  273. cartography/models/openai/user.py +49 -0
  274. cartography/models/semgrep/dependencies.py +36 -24
  275. cartography/models/semgrep/deployment.py +5 -5
  276. cartography/models/semgrep/findings.py +58 -42
  277. cartography/models/semgrep/locations.py +27 -21
  278. cartography/models/snipeit/asset.py +30 -21
  279. cartography/models/snipeit/tenant.py +6 -4
  280. cartography/models/snipeit/user.py +19 -12
  281. cartography/models/tailscale/__init__.py +0 -0
  282. cartography/models/tailscale/device.py +95 -0
  283. cartography/models/tailscale/group.py +86 -0
  284. cartography/models/tailscale/postureintegration.py +58 -0
  285. cartography/models/tailscale/tag.py +102 -0
  286. cartography/models/tailscale/tailnet.py +29 -0
  287. cartography/models/tailscale/user.py +52 -0
  288. cartography/stats.py +3 -3
  289. cartography/sync.py +113 -31
  290. cartography/util.py +84 -62
  291. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/METADATA +8 -15
  292. cartography-0.103.0.dist-info/RECORD +442 -0
  293. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/WHEEL +1 -1
  294. cartography-0.102.0rc1.dist-info/RECORD +0 -377
  295. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/entry_points.txt +0 -0
  296. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/licenses/LICENSE +0 -0
  297. {cartography-0.102.0rc1.dist-info → cartography-0.103.0.dist-info}/top_level.txt +0 -0
@@ -13,7 +13,6 @@ from typing import Tuple
13
13
 
14
14
  import requests
15
15
 
16
-
17
16
  logger = logging.getLogger(__name__)
18
17
  # Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
19
18
  _TIMEOUT = (60, 60)
@@ -26,25 +25,28 @@ class PaginatedGraphqlData(NamedTuple):
26
25
 
27
26
 
28
27
  def handle_rate_limit_sleep(token: str) -> None:
29
- '''
28
+ """
30
29
  Check the remaining rate limit and sleep if remaining is below threshold
31
30
  :param token: The Github API token as string.
32
- '''
33
- response = requests.get('https://api.github.com/rate_limit', headers={'Authorization': f"token {token}"})
31
+ """
32
+ response = requests.get(
33
+ "https://api.github.com/rate_limit",
34
+ headers={"Authorization": f"token {token}"},
35
+ )
34
36
  response.raise_for_status()
35
37
  response_json = response.json()
36
- rate_limit_obj = response_json['resources']['graphql']
37
- remaining = rate_limit_obj['remaining']
38
+ rate_limit_obj = response_json["resources"]["graphql"]
39
+ remaining = rate_limit_obj["remaining"]
38
40
  threshold = _GRAPHQL_RATE_LIMIT_REMAINING_THRESHOLD
39
41
  if remaining > threshold:
40
42
  return
41
- reset_at = datetime.fromtimestamp(rate_limit_obj['reset'], tz=tz.utc)
43
+ reset_at = datetime.fromtimestamp(rate_limit_obj["reset"], tz=tz.utc)
42
44
  now = datetime.now(tz.utc)
43
45
  # add an extra minute for safety
44
46
  sleep_duration = reset_at - now + timedelta(minutes=1)
45
47
  logger.warning(
46
- f'Github graphql ratelimit has {remaining} remaining and is under threshold {threshold},'
47
- f' sleeping until reset at {reset_at} for {sleep_duration}',
48
+ f"Github graphql ratelimit has {remaining} remaining and is under threshold {threshold},"
49
+ f" sleeping until reset at {reset_at} for {sleep_duration}",
48
50
  )
49
51
  time.sleep(sleep_duration.seconds)
50
52
 
@@ -58,11 +60,11 @@ def call_github_api(query: str, variables: str, token: str, api_url: str) -> Dic
58
60
  :param api_url: the URL to call for the API
59
61
  :return: query results json
60
62
  """
61
- headers = {'Authorization': f"token {token}"}
63
+ headers = {"Authorization": f"token {token}"}
62
64
  try:
63
65
  response = requests.post(
64
66
  api_url,
65
- json={'query': query, 'variables': variables},
67
+ json={"query": query, "variables": variables},
66
68
  headers=headers,
67
69
  timeout=_TIMEOUT,
68
70
  )
@@ -75,7 +77,7 @@ def call_github_api(query: str, variables: str, token: str, api_url: str) -> Dic
75
77
  if "errors" in response_json:
76
78
  logger.warning(
77
79
  f'call_github_api() response has errors, please investigate. Raw response: {response_json["errors"]}; '
78
- f'continuing sync.',
80
+ f"continuing sync.",
79
81
  )
80
82
  return response_json # type: ignore
81
83
 
@@ -101,8 +103,8 @@ def fetch_page(
101
103
  """
102
104
  gql_vars = {
103
105
  **kwargs,
104
- 'login': organization,
105
- 'cursor': cursor,
106
+ "login": organization,
107
+ "cursor": cursor,
106
108
  }
107
109
  gql_vars_json = json.dumps(gql_vars)
108
110
  response = call_github_api(query, gql_vars_json, token, api_url)
@@ -110,14 +112,14 @@ def fetch_page(
110
112
 
111
113
 
112
114
  def fetch_all(
113
- token: str,
114
- api_url: str,
115
- organization: str,
116
- query: str,
117
- resource_type: str,
118
- retries: int = 5,
119
- resource_inner_type: Optional[str] = None,
120
- **kwargs: Any,
115
+ token: str,
116
+ api_url: str,
117
+ organization: str,
118
+ query: str,
119
+ resource_type: str,
120
+ retries: int = 5,
121
+ resource_inner_type: Optional[str] = None,
122
+ **kwargs: Any,
121
123
  ) -> Tuple[PaginatedGraphqlData, Dict[str, Any]]:
122
124
  """
123
125
  Fetch and return all data items of the given `resource_type` and `field_name` from Github's paginated GraphQL API as
@@ -169,32 +171,32 @@ def fetch_all(
169
171
  )
170
172
  raise exc
171
173
  elif retry > 0:
172
- time.sleep(2 ** retry)
174
+ time.sleep(2**retry)
173
175
  continue
174
176
 
175
- if 'data' not in resp:
177
+ if "data" not in resp:
176
178
  logger.warning(
177
179
  f'Got no "data" attribute in response: {resp}. '
178
- f'Stopping requests for organization: {organization} and '
179
- f'resource_type: {resource_type}',
180
+ f"Stopping requests for organization: {organization} and "
181
+ f"resource_type: {resource_type}",
180
182
  )
181
183
  has_next_page = False
182
184
  continue
183
185
 
184
- resource = resp['data']['organization'][resource_type]
186
+ resource = resp["data"]["organization"][resource_type]
185
187
  if resource_inner_type:
186
- resource = resp['data']['organization'][resource_type][resource_inner_type]
188
+ resource = resp["data"]["organization"][resource_type][resource_inner_type]
187
189
 
188
190
  # Allow for paginating both nodes and edges fields of the GitHub GQL structure.
189
- data.nodes.extend(resource.get('nodes', []))
190
- data.edges.extend(resource.get('edges', []))
191
+ data.nodes.extend(resource.get("nodes", []))
192
+ data.edges.extend(resource.get("edges", []))
191
193
 
192
- cursor = resource['pageInfo']['endCursor']
193
- has_next_page = resource['pageInfo']['hasNextPage']
194
+ cursor = resource["pageInfo"]["endCursor"]
195
+ has_next_page = resource["pageInfo"]["hasNextPage"]
194
196
  if not org_data:
195
197
  org_data = {
196
- 'url': resp['data']['organization']['url'],
197
- 'login': resp['data']['organization']['login'],
198
+ "url": resp["data"]["organization"]["url"],
199
+ "login": resp["data"]["organization"]["login"],
198
200
  }
199
201
 
200
202
  if not org_data:
@@ -20,17 +20,19 @@ from cartography.intel.gsuite import api
20
20
  from cartography.util import timeit
21
21
 
22
22
  OAUTH_SCOPES = [
23
- 'https://www.googleapis.com/auth/admin.directory.user.readonly',
24
- 'https://www.googleapis.com/auth/admin.directory.group.readonly',
25
- 'https://www.googleapis.com/auth/admin.directory.group.member',
23
+ "https://www.googleapis.com/auth/admin.directory.user.readonly",
24
+ "https://www.googleapis.com/auth/admin.directory.group.readonly",
25
+ "https://www.googleapis.com/auth/admin.directory.group.member",
26
26
  ]
27
27
 
28
28
  logger = logging.getLogger(__name__)
29
29
 
30
- Resources = namedtuple('Resources', 'admin')
30
+ Resources = namedtuple("Resources", "admin")
31
31
 
32
32
 
33
- def _get_admin_resource(credentials: OAuth2Credentials | ServiceAccountCredentials) -> Resource:
33
+ def _get_admin_resource(
34
+ credentials: OAuth2Credentials | ServiceAccountCredentials,
35
+ ) -> Resource:
34
36
  """
35
37
  Instantiates a Google API resource object to call the Google API.
36
38
  Used to pull users and groups. See https://developers.google.com/admin-sdk/directory/v1/guides/manage-users
@@ -38,10 +40,17 @@ def _get_admin_resource(credentials: OAuth2Credentials | ServiceAccountCredentia
38
40
  :param credentials: The credentials object
39
41
  :return: An admin api resource object
40
42
  """
41
- return googleapiclient.discovery.build('admin', 'directory_v1', credentials=credentials, cache_discovery=False)
43
+ return googleapiclient.discovery.build(
44
+ "admin",
45
+ "directory_v1",
46
+ credentials=credentials,
47
+ cache_discovery=False,
48
+ )
42
49
 
43
50
 
44
- def _initialize_resources(credentials: OAuth2Credentials | ServiceAccountCredentials) -> Resources:
51
+ def _initialize_resources(
52
+ credentials: OAuth2Credentials | ServiceAccountCredentials,
53
+ ) -> Resources:
45
54
  """
46
55
  Create namedtuple of all resource objects necessary for Google API data gathering.
47
56
  :param credentials: The credentials object
@@ -66,7 +75,7 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
66
75
  }
67
76
 
68
77
  creds: OAuth2Credentials | ServiceAccountCredentials
69
- if config.gsuite_auth_method == 'delegated': # Legacy delegated method
78
+ if config.gsuite_auth_method == "delegated": # Legacy delegated method
70
79
  if config.gsuite_config is None or not os.path.isfile(config.gsuite_config):
71
80
  logger.warning(
72
81
  (
@@ -75,36 +84,38 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
75
84
  ),
76
85
  )
77
86
  return
78
- logger.info('Attempting to authenticate to GSuite using legacy delegated method')
87
+ logger.info(
88
+ "Attempting to authenticate to GSuite using legacy delegated method"
89
+ )
79
90
  try:
80
91
  creds = service_account.Credentials.from_service_account_file(
81
92
  config.gsuite_config,
82
93
  scopes=OAUTH_SCOPES,
83
94
  )
84
- creds = creds.with_subject(os.environ.get('GSUITE_DELEGATED_ADMIN'))
95
+ creds = creds.with_subject(os.environ.get("GSUITE_DELEGATED_ADMIN"))
85
96
 
86
97
  except DefaultCredentialsError as e:
87
98
  logger.error(
88
99
  (
89
100
  "Unable to initialize GSuite creds. If you don't have GSuite data or don't want to load "
90
- 'Gsuite data then you can ignore this message. Otherwise, the error code is: %s '
91
- 'Make sure your GSuite credentials file (if any) is valid. '
92
- 'For more details see README'
101
+ "Gsuite data then you can ignore this message. Otherwise, the error code is: %s "
102
+ "Make sure your GSuite credentials file (if any) is valid. "
103
+ "For more details see README"
93
104
  ),
94
105
  e,
95
106
  )
96
107
  return
97
- elif config.gsuite_auth_method == 'oauth':
108
+ elif config.gsuite_auth_method == "oauth":
98
109
  auth_tokens = json.loads(str(base64.b64decode(config.gsuite_config).decode()))
99
- logger.info('Attempting to authenticate to GSuite using OAuth')
110
+ logger.info("Attempting to authenticate to GSuite using OAuth")
100
111
  try:
101
112
  creds = credentials.Credentials(
102
113
  token=None,
103
- client_id=auth_tokens['client_id'],
104
- client_secret=auth_tokens['client_secret'],
105
- refresh_token=auth_tokens['refresh_token'],
114
+ client_id=auth_tokens["client_id"],
115
+ client_secret=auth_tokens["client_secret"],
116
+ refresh_token=auth_tokens["refresh_token"],
106
117
  expiry=None,
107
- token_uri=auth_tokens['token_uri'],
118
+ token_uri=auth_tokens["token_uri"],
108
119
  scopes=OAUTH_SCOPES,
109
120
  )
110
121
  creds.refresh(Request())
@@ -113,15 +124,15 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
113
124
  logger.error(
114
125
  (
115
126
  "Unable to initialize GSuite creds. If you don't have GSuite data or don't want to load "
116
- 'Gsuite data then you can ignore this message. Otherwise, the error code is: %s '
117
- 'Make sure your GSuite credentials are configured correctly, your credentials are valid. '
118
- 'For more details see README'
127
+ "Gsuite data then you can ignore this message. Otherwise, the error code is: %s "
128
+ "Make sure your GSuite credentials are configured correctly, your credentials are valid. "
129
+ "For more details see README"
119
130
  ),
120
131
  e,
121
132
  )
122
133
  return
123
- elif config.gsuite_auth_method == 'default':
124
- logger.info('Attempting to authenticate to GSuite using default credentials')
134
+ elif config.gsuite_auth_method == "default":
135
+ logger.info("Attempting to authenticate to GSuite using default credentials")
125
136
  try:
126
137
  creds, _ = default(scopes=OAUTH_SCOPES)
127
138
  except DefaultCredentialsError as e:
@@ -137,5 +148,15 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
137
148
  return
138
149
 
139
150
  resources = _initialize_resources(creds)
140
- api.sync_gsuite_users(neo4j_session, resources.admin, config.update_tag, common_job_parameters)
141
- api.sync_gsuite_groups(neo4j_session, resources.admin, config.update_tag, common_job_parameters)
151
+ api.sync_gsuite_users(
152
+ neo4j_session,
153
+ resources.admin,
154
+ config.update_tag,
155
+ common_job_parameters,
156
+ )
157
+ api.sync_gsuite_groups(
158
+ neo4j_session,
159
+ resources.admin,
160
+ config.update_tag,
161
+ common_job_parameters,
162
+ )
@@ -9,7 +9,6 @@ from googleapiclient.errors import HttpError
9
9
  from cartography.util import run_cleanup_job
10
10
  from cartography.util import timeit
11
11
 
12
-
13
12
  logger = logging.getLogger(__name__)
14
13
 
15
14
 
@@ -28,7 +27,11 @@ def get_all_groups(admin: Resource) -> List[Dict]:
28
27
  See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
29
28
  :return: List of Google groups in domain
30
29
  """
31
- request = admin.groups().list(customer='my_customer', maxResults=20, orderBy='email')
30
+ request = admin.groups().list(
31
+ customer="my_customer",
32
+ maxResults=20,
33
+ orderBy="email",
34
+ )
32
35
  response_objects = []
33
36
  while request is not None:
34
37
  try:
@@ -36,13 +39,16 @@ def get_all_groups(admin: Resource) -> List[Dict]:
36
39
  response_objects.append(resp)
37
40
  request = admin.groups().list_next(request, resp)
38
41
  except HttpError as e:
39
- if e.resp.status == 403 and "Request had insufficient authentication scopes" in str(e):
42
+ if (
43
+ e.resp.status == 403
44
+ and "Request had insufficient authentication scopes" in str(e)
45
+ ):
40
46
  logger.error(
41
47
  "Missing required GSuite scopes. If using the gcloud CLI, ",
42
48
  "run: gcloud auth application-default login --scopes="
43
49
  '"https://www.googleapis.com/auth/admin.directory.user.readonly,'
44
- 'https://www.googleapis.com/auth/admin.directory.group.readonly,'
45
- 'https://www.googleapis.com/auth/admin.directory.group.member.readonly,'
50
+ "https://www.googleapis.com/auth/admin.directory.group.readonly,"
51
+ "https://www.googleapis.com/auth/admin.directory.group.member.readonly,"
46
52
  'https://www.googleapis.com/auth/cloud-platform"',
47
53
  )
48
54
  raise
@@ -51,34 +57,34 @@ def get_all_groups(admin: Resource) -> List[Dict]:
51
57
 
52
58
  @timeit
53
59
  def transform_groups(response_objects: List[Dict]) -> List[Dict]:
54
- """ Strips list of API response objects to return list of group objects only
60
+ """Strips list of API response objects to return list of group objects only
55
61
 
56
62
  :param response_objects:
57
63
  :return: list of dictionary objects as defined in /docs/root/modules/gsuite/schema.md
58
64
  """
59
65
  groups: List[Dict] = []
60
66
  for response_object in response_objects:
61
- for group in response_object['groups']:
67
+ for group in response_object["groups"]:
62
68
  groups.append(group)
63
69
  return groups
64
70
 
65
71
 
66
72
  @timeit
67
73
  def transform_users(response_objects: List[Dict]) -> List[Dict]:
68
- """ Strips list of API response objects to return list of group objects only
74
+ """Strips list of API response objects to return list of group objects only
69
75
  :param response_objects:
70
76
  :return: list of dictionary objects as defined in /docs/root/modules/gsuite/schema.md
71
77
  """
72
78
  users: List[Dict] = []
73
79
  for response_object in response_objects:
74
- for user in response_object['users']:
80
+ for user in response_object["users"]:
75
81
  users.append(user)
76
82
  return users
77
83
 
78
84
 
79
85
  @timeit
80
86
  def get_all_groups_for_email(admin: Resource, email: str) -> List[Dict]:
81
- """ Fetch all groups of which the given group is a member
87
+ """Fetch all groups of which the given group is a member
82
88
 
83
89
  Arguments:
84
90
  email: A string representing the email address for the group
@@ -90,14 +96,14 @@ def get_all_groups_for_email(admin: Resource, email: str) -> List[Dict]:
90
96
  groups: List[Dict] = []
91
97
  while request is not None:
92
98
  resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
93
- groups = groups + resp.get('groups', [])
99
+ groups = groups + resp.get("groups", [])
94
100
  request = admin.groups().list_next(request, resp)
95
101
  return groups
96
102
 
97
103
 
98
104
  @timeit
99
105
  def get_members_for_group(admin: Resource, group_email: str) -> List[Dict]:
100
- """ Get all members for a google group
106
+ """Get all members for a google group
101
107
 
102
108
  :param group_email: A string representing the email address for the group
103
109
 
@@ -110,7 +116,7 @@ def get_members_for_group(admin: Resource, group_email: str) -> List[Dict]:
110
116
  members: List[Dict] = []
111
117
  while request is not None:
112
118
  resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
113
- members = members + resp.get('members', [])
119
+ members = members + resp.get("members", [])
114
120
  request = admin.members().list_next(request, resp)
115
121
 
116
122
  return members
@@ -128,7 +134,11 @@ def get_all_users(admin: Resource) -> List[Dict]:
128
134
  :return: List of Google users in domain
129
135
  see https://developers.google.com/admin-sdk/directory/v1/guides/manage-users#get_all_domain_users
130
136
  """
131
- request = admin.users().list(customer='my_customer', maxResults=500, orderBy='email')
137
+ request = admin.users().list(
138
+ customer="my_customer",
139
+ maxResults=500,
140
+ orderBy="email",
141
+ )
132
142
  response_objects = []
133
143
  while request is not None:
134
144
  resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
@@ -138,7 +148,11 @@ def get_all_users(admin: Resource) -> List[Dict]:
138
148
 
139
149
 
140
150
  @timeit
141
- def load_gsuite_groups(neo4j_session: neo4j.Session, groups: List[Dict], gsuite_update_tag: int) -> None:
151
+ def load_gsuite_groups(
152
+ neo4j_session: neo4j.Session,
153
+ groups: List[Dict],
154
+ gsuite_update_tag: int,
155
+ ) -> None:
142
156
  ingestion_qry = """
143
157
  UNWIND $GroupData as group
144
158
  MERGE (g:GSuiteGroup{id: group.id})
@@ -156,12 +170,16 @@ def load_gsuite_groups(neo4j_session: neo4j.Session, groups: List[Dict], gsuite_
156
170
  g:GCPPrincipal,
157
171
  g.lastupdated = $UpdateTag
158
172
  """
159
- logger.info(f'Ingesting {len(groups)} gsuite groups')
173
+ logger.info(f"Ingesting {len(groups)} gsuite groups")
160
174
  neo4j_session.run(ingestion_qry, GroupData=groups, UpdateTag=gsuite_update_tag)
161
175
 
162
176
 
163
177
  @timeit
164
- def load_gsuite_users(neo4j_session: neo4j.Session, users: List[Dict], gsuite_update_tag: int) -> None:
178
+ def load_gsuite_users(
179
+ neo4j_session: neo4j.Session,
180
+ users: List[Dict],
181
+ gsuite_update_tag: int,
182
+ ) -> None:
165
183
  ingestion_qry = """
166
184
  UNWIND $UserData as user
167
185
  MERGE (u:GSuiteUser{id: user.id})
@@ -196,12 +214,17 @@ def load_gsuite_users(neo4j_session: neo4j.Session, users: List[Dict], gsuite_up
196
214
  u:GCPPrincipal,
197
215
  u.lastupdated = $UpdateTag
198
216
  """
199
- logger.info(f'Ingesting {len(users)} gsuite users')
217
+ logger.info(f"Ingesting {len(users)} gsuite users")
200
218
  neo4j_session.run(ingestion_qry, UserData=users, UpdateTag=gsuite_update_tag)
201
219
 
202
220
 
203
221
  @timeit
204
- def load_gsuite_members(neo4j_session: neo4j.Session, group: Dict, members: List[Dict], gsuite_update_tag: int) -> None:
222
+ def load_gsuite_members(
223
+ neo4j_session: neo4j.Session,
224
+ group: Dict,
225
+ members: List[Dict],
226
+ gsuite_update_tag: int,
227
+ ) -> None:
205
228
  ingestion_qry = """
206
229
  UNWIND $MemberData as member
207
230
  MATCH (user:GSuiteUser {id: member.id}),(group:GSuiteGroup {id: $GroupID })
@@ -226,22 +249,33 @@ def load_gsuite_members(neo4j_session: neo4j.Session, group: Dict, members: List
226
249
  SET
227
250
  r.lastupdated = $UpdateTag
228
251
  """
229
- neo4j_session.run(membership_qry, MemberData=members, GroupID=group.get("id"), UpdateTag=gsuite_update_tag)
252
+ neo4j_session.run(
253
+ membership_qry,
254
+ MemberData=members,
255
+ GroupID=group.get("id"),
256
+ UpdateTag=gsuite_update_tag,
257
+ )
230
258
 
231
259
 
232
260
  @timeit
233
- def cleanup_gsuite_users(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
261
+ def cleanup_gsuite_users(
262
+ neo4j_session: neo4j.Session,
263
+ common_job_parameters: Dict,
264
+ ) -> None:
234
265
  run_cleanup_job(
235
- 'gsuite_ingest_users_cleanup.json',
266
+ "gsuite_ingest_users_cleanup.json",
236
267
  neo4j_session,
237
268
  common_job_parameters,
238
269
  )
239
270
 
240
271
 
241
272
  @timeit
242
- def cleanup_gsuite_groups(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
273
+ def cleanup_gsuite_groups(
274
+ neo4j_session: neo4j.Session,
275
+ common_job_parameters: Dict,
276
+ ) -> None:
243
277
  run_cleanup_job(
244
- 'gsuite_ingest_groups_cleanup.json',
278
+ "gsuite_ingest_groups_cleanup.json",
245
279
  neo4j_session,
246
280
  common_job_parameters,
247
281
  )
@@ -249,7 +283,10 @@ def cleanup_gsuite_groups(neo4j_session: neo4j.Session, common_job_parameters: D
249
283
 
250
284
  @timeit
251
285
  def sync_gsuite_users(
252
- neo4j_session: neo4j.Session, admin: Resource, gsuite_update_tag: int, common_job_parameters: Dict,
286
+ neo4j_session: neo4j.Session,
287
+ admin: Resource,
288
+ gsuite_update_tag: int,
289
+ common_job_parameters: Dict,
253
290
  ) -> None:
254
291
  """
255
292
  GET GSuite user objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
@@ -261,7 +298,7 @@ def sync_gsuite_users(
261
298
  :param common_job_parameters: Parameters to carry to the Neo4j jobs
262
299
  :return: Nothing
263
300
  """
264
- logger.debug('Syncing GSuite Users')
301
+ logger.debug("Syncing GSuite Users")
265
302
  resp_objs = get_all_users(admin)
266
303
  users = transform_users(resp_objs)
267
304
  load_gsuite_users(neo4j_session, users, gsuite_update_tag)
@@ -270,7 +307,10 @@ def sync_gsuite_users(
270
307
 
271
308
  @timeit
272
309
  def sync_gsuite_groups(
273
- neo4j_session: neo4j.Session, admin: Resource, gsuite_update_tag: int, common_job_parameters: Dict,
310
+ neo4j_session: neo4j.Session,
311
+ admin: Resource,
312
+ gsuite_update_tag: int,
313
+ common_job_parameters: Dict,
274
314
  ) -> None:
275
315
  """
276
316
  GET GSuite group objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
@@ -282,7 +322,7 @@ def sync_gsuite_groups(
282
322
  :param common_job_parameters: Parameters to carry to the Neo4j jobs
283
323
  :return: Nothing
284
324
  """
285
- logger.debug('Syncing GSuite Groups')
325
+ logger.debug("Syncing GSuite Groups")
286
326
  resp_objs = get_all_groups(admin)
287
327
  groups = transform_groups(resp_objs)
288
328
  load_gsuite_groups(neo4j_session, groups, gsuite_update_tag)
@@ -292,8 +332,11 @@ def sync_gsuite_groups(
292
332
 
293
333
  @timeit
294
334
  def sync_gsuite_members(
295
- groups: List[Dict], neo4j_session: neo4j.Session, admin: Resource, gsuite_update_tag: int,
335
+ groups: List[Dict],
336
+ neo4j_session: neo4j.Session,
337
+ admin: Resource,
338
+ gsuite_update_tag: int,
296
339
  ) -> None:
297
340
  for group in groups:
298
- members = get_members_for_group(admin, group['email'])
341
+ members = get_members_for_group(admin, group["email"])
299
342
  load_gsuite_members(neo4j_session, group, members, gsuite_update_tag)
@@ -1,13 +1,31 @@
1
+ import logging
2
+
1
3
  import neo4j
2
4
 
3
5
  from cartography.config import Config
4
6
  from cartography.intel.jamf import computers
5
7
  from cartography.util import timeit
6
8
 
9
+ logger = logging.getLogger(__name__)
10
+
7
11
 
8
12
  @timeit
9
13
  def start_jamf_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
14
+
15
+ if not config.jamf_base_uri or not config.jamf_user or not config.jamf_password:
16
+ # If the config is not set, we don't want to run this module
17
+ logger.info(
18
+ "Jamf import is not configured - skipping this module. See docs to configure."
19
+ )
20
+ return
21
+
10
22
  common_job_parameters = {
11
23
  "UPDATE_TAG": config.update_tag,
12
24
  }
13
- computers.sync(neo4j_session, config.jamf_base_uri, config.jamf_user, config.jamf_password, common_job_parameters)
25
+ computers.sync(
26
+ neo4j_session,
27
+ config.jamf_base_uri,
28
+ config.jamf_user,
29
+ config.jamf_password,
30
+ common_job_parameters,
31
+ )
@@ -8,17 +8,24 @@ from cartography.intel.jamf.util import call_jamf_api
8
8
  from cartography.util import run_cleanup_job
9
9
  from cartography.util import timeit
10
10
 
11
-
12
11
  logger = logging.getLogger(__name__)
13
12
 
14
13
 
15
14
  @timeit
16
- def get_computer_groups(jamf_base_uri: str, jamf_user: str, jamf_password: str) -> List[Dict]:
15
+ def get_computer_groups(
16
+ jamf_base_uri: str,
17
+ jamf_user: str,
18
+ jamf_password: str,
19
+ ) -> List[Dict]:
17
20
  return call_jamf_api("/computergroups", jamf_base_uri, jamf_user, jamf_password)
18
21
 
19
22
 
20
23
  @timeit
21
- def load_computer_groups(data: Dict, neo4j_session: neo4j.Session, update_tag: int) -> None:
24
+ def load_computer_groups(
25
+ data: Dict,
26
+ neo4j_session: neo4j.Session,
27
+ update_tag: int,
28
+ ) -> None:
22
29
  ingest_groups = """
23
30
  UNWIND $JsonData as group
24
31
  MERGE (g:JamfComputerGroup{id: group.id})
@@ -33,12 +40,19 @@ def load_computer_groups(data: Dict, neo4j_session: neo4j.Session, update_tag: i
33
40
 
34
41
  @timeit
35
42
  def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
36
- run_cleanup_job('jamf_import_computers_cleanup.json', neo4j_session, common_job_parameters)
43
+ run_cleanup_job(
44
+ "jamf_import_computers_cleanup.json",
45
+ neo4j_session,
46
+ common_job_parameters,
47
+ )
37
48
 
38
49
 
39
50
  @timeit
40
51
  def sync_computer_groups(
41
- neo4j_session: neo4j.Session, update_tag: int, jamf_base_uri: str, jamf_user: str,
52
+ neo4j_session: neo4j.Session,
53
+ update_tag: int,
54
+ jamf_base_uri: str,
55
+ jamf_user: str,
42
56
  jamf_password: str,
43
57
  ) -> None:
44
58
  groups = get_computer_groups(jamf_base_uri, jamf_user, jamf_password)
@@ -47,7 +61,16 @@ def sync_computer_groups(
47
61
 
48
62
  @timeit
49
63
  def sync(
50
- neo4j_session: neo4j.Session, jamf_base_uri: str, jamf_user: str, jamf_password: str,
64
+ neo4j_session: neo4j.Session,
65
+ jamf_base_uri: str,
66
+ jamf_user: str,
67
+ jamf_password: str,
51
68
  common_job_parameters: Dict,
52
69
  ) -> None:
53
- sync_computer_groups(neo4j_session, common_job_parameters['UPDATE_TAG'], jamf_base_uri, jamf_user, jamf_password)
70
+ sync_computer_groups(
71
+ neo4j_session,
72
+ common_job_parameters["UPDATE_TAG"],
73
+ jamf_base_uri,
74
+ jamf_user,
75
+ jamf_password,
76
+ )