cartography 0.102.0rc2__py3-none-any.whl → 0.103.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

Files changed (297) hide show
  1. cartography/__main__.py +1 -2
  2. cartography/_version.py +2 -2
  3. cartography/cli.py +376 -249
  4. cartography/client/core/tx.py +39 -18
  5. cartography/config.py +28 -0
  6. cartography/driftdetect/__main__.py +1 -2
  7. cartography/driftdetect/add_shortcut.py +10 -2
  8. cartography/driftdetect/cli.py +71 -75
  9. cartography/driftdetect/detect_deviations.py +7 -3
  10. cartography/driftdetect/get_states.py +20 -8
  11. cartography/driftdetect/model.py +5 -5
  12. cartography/driftdetect/serializers.py +8 -6
  13. cartography/driftdetect/storage.py +2 -2
  14. cartography/graph/cleanupbuilder.py +35 -15
  15. cartography/graph/job.py +46 -17
  16. cartography/graph/querybuilder.py +165 -80
  17. cartography/graph/statement.py +35 -26
  18. cartography/intel/analysis.py +4 -1
  19. cartography/intel/aws/__init__.py +114 -55
  20. cartography/intel/aws/apigateway.py +134 -63
  21. cartography/intel/aws/cloudtrail.py +127 -0
  22. cartography/intel/aws/cloudwatch.py +93 -0
  23. cartography/intel/aws/config.py +56 -20
  24. cartography/intel/aws/dynamodb.py +108 -40
  25. cartography/intel/aws/ec2/__init__.py +2 -2
  26. cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
  27. cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
  28. cartography/intel/aws/ec2/images.py +49 -20
  29. cartography/intel/aws/ec2/instances.py +234 -136
  30. cartography/intel/aws/ec2/internet_gateways.py +40 -11
  31. cartography/intel/aws/ec2/key_pairs.py +44 -20
  32. cartography/intel/aws/ec2/launch_templates.py +101 -59
  33. cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
  34. cartography/intel/aws/ec2/load_balancers.py +82 -42
  35. cartography/intel/aws/ec2/network_acls.py +89 -65
  36. cartography/intel/aws/ec2/network_interfaces.py +146 -87
  37. cartography/intel/aws/ec2/reserved_instances.py +45 -16
  38. cartography/intel/aws/ec2/route_tables.py +138 -98
  39. cartography/intel/aws/ec2/security_groups.py +71 -21
  40. cartography/intel/aws/ec2/snapshots.py +61 -22
  41. cartography/intel/aws/ec2/subnets.py +54 -18
  42. cartography/intel/aws/ec2/tgw.py +100 -34
  43. cartography/intel/aws/ec2/util.py +1 -1
  44. cartography/intel/aws/ec2/volumes.py +69 -41
  45. cartography/intel/aws/ec2/vpc.py +37 -12
  46. cartography/intel/aws/ec2/vpc_peerings.py +83 -24
  47. cartography/intel/aws/ecr.py +88 -32
  48. cartography/intel/aws/ecs.py +83 -47
  49. cartography/intel/aws/efs.py +93 -0
  50. cartography/intel/aws/eks.py +55 -29
  51. cartography/intel/aws/elasticache.py +42 -18
  52. cartography/intel/aws/elasticsearch.py +57 -20
  53. cartography/intel/aws/emr.py +61 -23
  54. cartography/intel/aws/iam.py +401 -145
  55. cartography/intel/aws/iam_instance_profiles.py +22 -22
  56. cartography/intel/aws/identitycenter.py +71 -37
  57. cartography/intel/aws/inspector.py +159 -89
  58. cartography/intel/aws/kms.py +92 -38
  59. cartography/intel/aws/lambda_function.py +103 -34
  60. cartography/intel/aws/organizations.py +30 -10
  61. cartography/intel/aws/permission_relationships.py +133 -51
  62. cartography/intel/aws/rds.py +249 -85
  63. cartography/intel/aws/redshift.py +107 -46
  64. cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
  65. cartography/intel/aws/resources.py +57 -46
  66. cartography/intel/aws/route53.py +108 -61
  67. cartography/intel/aws/s3.py +168 -83
  68. cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
  69. cartography/intel/aws/secretsmanager.py +24 -12
  70. cartography/intel/aws/securityhub.py +20 -9
  71. cartography/intel/aws/sns.py +166 -0
  72. cartography/intel/aws/sqs.py +60 -28
  73. cartography/intel/aws/ssm.py +70 -30
  74. cartography/intel/aws/util/arns.py +7 -7
  75. cartography/intel/aws/util/common.py +31 -4
  76. cartography/intel/azure/__init__.py +78 -19
  77. cartography/intel/azure/compute.py +101 -27
  78. cartography/intel/azure/cosmosdb.py +496 -170
  79. cartography/intel/azure/sql.py +296 -105
  80. cartography/intel/azure/storage.py +322 -113
  81. cartography/intel/azure/subscription.py +39 -23
  82. cartography/intel/azure/tenant.py +13 -4
  83. cartography/intel/azure/util/credentials.py +95 -55
  84. cartography/intel/bigfix/__init__.py +2 -2
  85. cartography/intel/bigfix/computers.py +93 -65
  86. cartography/intel/cloudflare/__init__.py +74 -0
  87. cartography/intel/cloudflare/accounts.py +57 -0
  88. cartography/intel/cloudflare/dnsrecords.py +64 -0
  89. cartography/intel/cloudflare/members.py +75 -0
  90. cartography/intel/cloudflare/roles.py +65 -0
  91. cartography/intel/cloudflare/zones.py +64 -0
  92. cartography/intel/create_indexes.py +3 -2
  93. cartography/intel/crowdstrike/__init__.py +11 -9
  94. cartography/intel/crowdstrike/endpoints.py +5 -1
  95. cartography/intel/crowdstrike/spotlight.py +8 -3
  96. cartography/intel/cve/__init__.py +46 -13
  97. cartography/intel/cve/feed.py +48 -12
  98. cartography/intel/digitalocean/__init__.py +22 -13
  99. cartography/intel/digitalocean/compute.py +75 -108
  100. cartography/intel/digitalocean/management.py +44 -80
  101. cartography/intel/digitalocean/platform.py +48 -43
  102. cartography/intel/dns.py +36 -10
  103. cartography/intel/duo/__init__.py +21 -16
  104. cartography/intel/duo/api_host.py +14 -9
  105. cartography/intel/duo/endpoints.py +50 -45
  106. cartography/intel/duo/groups.py +18 -14
  107. cartography/intel/duo/phones.py +37 -34
  108. cartography/intel/duo/tokens.py +26 -23
  109. cartography/intel/duo/users.py +54 -50
  110. cartography/intel/duo/web_authn_credentials.py +30 -25
  111. cartography/intel/entra/__init__.py +25 -7
  112. cartography/intel/entra/ou.py +112 -0
  113. cartography/intel/entra/users.py +69 -63
  114. cartography/intel/gcp/__init__.py +185 -49
  115. cartography/intel/gcp/compute.py +418 -231
  116. cartography/intel/gcp/crm.py +96 -43
  117. cartography/intel/gcp/dns.py +60 -19
  118. cartography/intel/gcp/gke.py +72 -38
  119. cartography/intel/gcp/iam.py +61 -41
  120. cartography/intel/gcp/storage.py +84 -55
  121. cartography/intel/github/__init__.py +13 -11
  122. cartography/intel/github/repos.py +270 -137
  123. cartography/intel/github/teams.py +170 -88
  124. cartography/intel/github/users.py +70 -39
  125. cartography/intel/github/util.py +36 -34
  126. cartography/intel/gsuite/__init__.py +47 -26
  127. cartography/intel/gsuite/api.py +73 -30
  128. cartography/intel/jamf/__init__.py +19 -1
  129. cartography/intel/jamf/computers.py +30 -7
  130. cartography/intel/jamf/util.py +7 -2
  131. cartography/intel/kandji/__init__.py +6 -3
  132. cartography/intel/kandji/devices.py +14 -8
  133. cartography/intel/kubernetes/namespaces.py +7 -4
  134. cartography/intel/kubernetes/pods.py +7 -4
  135. cartography/intel/kubernetes/services.py +8 -4
  136. cartography/intel/lastpass/__init__.py +2 -2
  137. cartography/intel/lastpass/users.py +23 -12
  138. cartography/intel/oci/__init__.py +44 -11
  139. cartography/intel/oci/iam.py +134 -38
  140. cartography/intel/oci/organizations.py +13 -6
  141. cartography/intel/oci/utils.py +43 -20
  142. cartography/intel/okta/__init__.py +66 -15
  143. cartography/intel/okta/applications.py +42 -20
  144. cartography/intel/okta/awssaml.py +93 -33
  145. cartography/intel/okta/factors.py +16 -4
  146. cartography/intel/okta/groups.py +56 -29
  147. cartography/intel/okta/organization.py +5 -1
  148. cartography/intel/okta/origins.py +6 -2
  149. cartography/intel/okta/roles.py +15 -5
  150. cartography/intel/okta/users.py +20 -8
  151. cartography/intel/okta/utils.py +6 -4
  152. cartography/intel/openai/__init__.py +86 -0
  153. cartography/intel/openai/adminapikeys.py +90 -0
  154. cartography/intel/openai/apikeys.py +96 -0
  155. cartography/intel/openai/projects.py +94 -0
  156. cartography/intel/openai/serviceaccounts.py +82 -0
  157. cartography/intel/openai/users.py +78 -0
  158. cartography/intel/openai/util.py +29 -0
  159. cartography/intel/pagerduty/__init__.py +8 -7
  160. cartography/intel/pagerduty/escalation_policies.py +18 -6
  161. cartography/intel/pagerduty/schedules.py +12 -4
  162. cartography/intel/pagerduty/services.py +11 -4
  163. cartography/intel/pagerduty/teams.py +8 -3
  164. cartography/intel/pagerduty/users.py +3 -1
  165. cartography/intel/pagerduty/vendors.py +3 -1
  166. cartography/intel/semgrep/__init__.py +24 -6
  167. cartography/intel/semgrep/dependencies.py +50 -28
  168. cartography/intel/semgrep/deployment.py +3 -1
  169. cartography/intel/semgrep/findings.py +42 -18
  170. cartography/intel/snipeit/__init__.py +17 -3
  171. cartography/intel/snipeit/asset.py +12 -6
  172. cartography/intel/snipeit/user.py +8 -5
  173. cartography/intel/snipeit/util.py +9 -4
  174. cartography/intel/tailscale/__init__.py +77 -0
  175. cartography/intel/tailscale/acls.py +146 -0
  176. cartography/intel/tailscale/devices.py +127 -0
  177. cartography/intel/tailscale/postureintegrations.py +81 -0
  178. cartography/intel/tailscale/tailnets.py +76 -0
  179. cartography/intel/tailscale/users.py +80 -0
  180. cartography/intel/tailscale/utils.py +132 -0
  181. cartography/models/aws/apigateway.py +21 -17
  182. cartography/models/aws/apigatewaycertificate.py +28 -22
  183. cartography/models/aws/apigatewayresource.py +28 -20
  184. cartography/models/aws/apigatewaystage.py +33 -25
  185. cartography/models/aws/cloudtrail/__init__.py +0 -0
  186. cartography/models/aws/cloudtrail/trail.py +61 -0
  187. cartography/models/aws/cloudwatch/__init__.py +0 -0
  188. cartography/models/aws/cloudwatch/loggroup.py +52 -0
  189. cartography/models/aws/dynamodb/gsi.py +30 -22
  190. cartography/models/aws/dynamodb/tables.py +25 -17
  191. cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
  192. cartography/models/aws/ec2/images.py +36 -34
  193. cartography/models/aws/ec2/instances.py +51 -45
  194. cartography/models/aws/ec2/keypair.py +21 -16
  195. cartography/models/aws/ec2/keypair_instance.py +28 -21
  196. cartography/models/aws/ec2/launch_configurations.py +30 -26
  197. cartography/models/aws/ec2/launch_template_versions.py +48 -38
  198. cartography/models/aws/ec2/launch_templates.py +21 -17
  199. cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
  200. cartography/models/aws/ec2/load_balancers.py +47 -37
  201. cartography/models/aws/ec2/network_acl_rules.py +38 -30
  202. cartography/models/aws/ec2/network_acls.py +38 -29
  203. cartography/models/aws/ec2/networkinterface_instance.py +52 -39
  204. cartography/models/aws/ec2/networkinterfaces.py +53 -37
  205. cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
  206. cartography/models/aws/ec2/reservations.py +18 -14
  207. cartography/models/aws/ec2/route_table_associations.py +44 -34
  208. cartography/models/aws/ec2/route_tables.py +50 -43
  209. cartography/models/aws/ec2/routes.py +45 -37
  210. cartography/models/aws/ec2/securitygroup_instance.py +29 -20
  211. cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
  212. cartography/models/aws/ec2/subnet_instance.py +24 -19
  213. cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
  214. cartography/models/aws/ec2/volumes.py +47 -40
  215. cartography/models/aws/efs/__init__.py +0 -0
  216. cartography/models/aws/efs/mount_target.py +52 -0
  217. cartography/models/aws/eks/clusters.py +23 -21
  218. cartography/models/aws/emr.py +32 -30
  219. cartography/models/aws/iam/instanceprofile.py +33 -24
  220. cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
  221. cartography/models/aws/identitycenter/awspermissionset.py +37 -29
  222. cartography/models/aws/identitycenter/awsssouser.py +23 -21
  223. cartography/models/aws/inspector/findings.py +77 -65
  224. cartography/models/aws/inspector/packages.py +35 -29
  225. cartography/models/aws/s3/__init__.py +0 -0
  226. cartography/models/aws/s3/account_public_access_block.py +51 -0
  227. cartography/models/aws/sns/__init__.py +0 -0
  228. cartography/models/aws/sns/topic.py +50 -0
  229. cartography/models/aws/ssm/instance_information.py +51 -39
  230. cartography/models/aws/ssm/instance_patch.py +32 -26
  231. cartography/models/bigfix/bigfix_computer.py +42 -38
  232. cartography/models/bigfix/bigfix_root.py +3 -3
  233. cartography/models/cloudflare/__init__.py +0 -0
  234. cartography/models/cloudflare/account.py +25 -0
  235. cartography/models/cloudflare/dnsrecord.py +55 -0
  236. cartography/models/cloudflare/member.py +82 -0
  237. cartography/models/cloudflare/role.py +44 -0
  238. cartography/models/cloudflare/zone.py +59 -0
  239. cartography/models/core/common.py +12 -10
  240. cartography/models/core/nodes.py +5 -2
  241. cartography/models/core/relationships.py +14 -6
  242. cartography/models/crowdstrike/hosts.py +37 -35
  243. cartography/models/cve/cve.py +34 -32
  244. cartography/models/cve/cve_feed.py +6 -6
  245. cartography/models/digitalocean/__init__.py +0 -0
  246. cartography/models/digitalocean/account.py +21 -0
  247. cartography/models/digitalocean/droplet.py +56 -0
  248. cartography/models/digitalocean/project.py +48 -0
  249. cartography/models/duo/api_host.py +3 -3
  250. cartography/models/duo/endpoint.py +43 -41
  251. cartography/models/duo/group.py +14 -14
  252. cartography/models/duo/phone.py +27 -27
  253. cartography/models/duo/token.py +16 -16
  254. cartography/models/duo/user.py +46 -44
  255. cartography/models/duo/web_authn_credential.py +27 -19
  256. cartography/models/entra/ou.py +48 -0
  257. cartography/models/entra/tenant.py +24 -18
  258. cartography/models/entra/user.py +64 -48
  259. cartography/models/gcp/iam.py +23 -23
  260. cartography/models/github/orgs.py +5 -4
  261. cartography/models/github/teams.py +37 -31
  262. cartography/models/github/users.py +34 -23
  263. cartography/models/kandji/device.py +22 -16
  264. cartography/models/kandji/tenant.py +6 -4
  265. cartography/models/lastpass/tenant.py +3 -3
  266. cartography/models/lastpass/user.py +32 -28
  267. cartography/models/openai/__init__.py +0 -0
  268. cartography/models/openai/adminapikey.py +90 -0
  269. cartography/models/openai/apikey.py +84 -0
  270. cartography/models/openai/organization.py +17 -0
  271. cartography/models/openai/project.py +70 -0
  272. cartography/models/openai/serviceaccount.py +50 -0
  273. cartography/models/openai/user.py +49 -0
  274. cartography/models/semgrep/dependencies.py +36 -24
  275. cartography/models/semgrep/deployment.py +5 -5
  276. cartography/models/semgrep/findings.py +58 -42
  277. cartography/models/semgrep/locations.py +27 -21
  278. cartography/models/snipeit/asset.py +30 -21
  279. cartography/models/snipeit/tenant.py +6 -4
  280. cartography/models/snipeit/user.py +19 -12
  281. cartography/models/tailscale/__init__.py +0 -0
  282. cartography/models/tailscale/device.py +95 -0
  283. cartography/models/tailscale/group.py +86 -0
  284. cartography/models/tailscale/postureintegration.py +58 -0
  285. cartography/models/tailscale/tag.py +102 -0
  286. cartography/models/tailscale/tailnet.py +29 -0
  287. cartography/models/tailscale/user.py +52 -0
  288. cartography/stats.py +3 -3
  289. cartography/sync.py +113 -31
  290. cartography/util.py +84 -62
  291. {cartography-0.102.0rc2.dist-info → cartography-0.103.0.dist-info}/METADATA +8 -15
  292. cartography-0.103.0.dist-info/RECORD +442 -0
  293. {cartography-0.102.0rc2.dist-info → cartography-0.103.0.dist-info}/WHEEL +1 -1
  294. cartography-0.102.0rc2.dist-info/RECORD +0 -381
  295. {cartography-0.102.0rc2.dist-info → cartography-0.103.0.dist-info}/entry_points.txt +0 -0
  296. {cartography-0.102.0rc2.dist-info → cartography-0.103.0.dist-info}/licenses/LICENSE +0 -0
  297. {cartography-0.102.0rc2.dist-info → cartography-0.103.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,29 @@
1
+ from typing import Any
2
+ from typing import Generator
3
+
4
+ import requests
5
+
6
+
7
+ def paginated_get(
8
+ api_session: requests.Session,
9
+ url: str,
10
+ timeout: tuple[int, int],
11
+ after: str | None = None,
12
+ ) -> Generator[dict[str, Any], None, None]:
13
+ # DOC
14
+ params = {"after": after} if after else {}
15
+ req = api_session.get(
16
+ url,
17
+ params=params,
18
+ timeout=timeout,
19
+ )
20
+ req.raise_for_status()
21
+ result = req.json()
22
+ yield from result.get("data", [])
23
+ if result.get("has_more"):
24
+ yield from paginated_get(
25
+ api_session,
26
+ url,
27
+ timeout=timeout,
28
+ after=result.get("last_id"),
29
+ )
@@ -4,9 +4,7 @@ import neo4j
4
4
  from pdpyras import APISession
5
5
 
6
6
  from cartography.config import Config
7
- from cartography.intel.pagerduty.escalation_policies import (
8
- sync_escalation_policies,
9
- )
7
+ from cartography.intel.pagerduty.escalation_policies import sync_escalation_policies
10
8
  from cartography.intel.pagerduty.schedules import sync_schedules
11
9
  from cartography.intel.pagerduty.services import sync_services
12
10
  from cartography.intel.pagerduty.teams import sync_teams
@@ -23,7 +21,8 @@ stat_handler = get_stats_client(__name__)
23
21
 
24
22
  @timeit
25
23
  def start_pagerduty_ingestion(
26
- neo4j_session: neo4j.Session, config: Config,
24
+ neo4j_session: neo4j.Session,
25
+ config: Config,
27
26
  ) -> None:
28
27
  """
29
28
  Perform ingestion of pagerduty data.
@@ -35,7 +34,9 @@ def start_pagerduty_ingestion(
35
34
  "UPDATE_TAG": config.update_tag,
36
35
  }
37
36
  if not config.pagerduty_api_key:
38
- logger.info('PagerDuty import is not configured - skipping this module. See docs to configure.')
37
+ logger.info(
38
+ "PagerDuty import is not configured - skipping this module. See docs to configure.",
39
+ )
39
40
  return
40
41
  session = APISession(config.pagerduty_api_key)
41
42
  if config.pagerduty_request_timeout is not None:
@@ -54,8 +55,8 @@ def start_pagerduty_ingestion(
54
55
 
55
56
  merge_module_sync_metadata(
56
57
  neo4j_session,
57
- group_type='pagerduty',
58
- group_id='module',
58
+ group_type="pagerduty",
59
+ group_id="module",
59
60
  synced_type="pagerduty",
60
61
  update_tag=config.update_tag,
61
62
  stat_handler=stat_handler,
@@ -31,7 +31,9 @@ def get_escalation_policies(pd_session: APISession) -> List[Dict[str, Any]]:
31
31
 
32
32
 
33
33
  def load_escalation_policy_data(
34
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
34
+ neo4j_session: neo4j.Session,
35
+ data: List[Dict],
36
+ update_tag: int,
35
37
  ) -> None:
36
38
  """
37
39
  Transform and load escalation_policy information
@@ -82,7 +84,9 @@ def load_escalation_policy_data(
82
84
 
83
85
 
84
86
  def _attach_rules(
85
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
87
+ neo4j_session: neo4j.Session,
88
+ data: List[Dict],
89
+ update_tag: int,
86
90
  ) -> None:
87
91
  """
88
92
  Add escalation policy rules, and attach them to targets.
@@ -122,7 +126,9 @@ def _attach_rules(
122
126
 
123
127
 
124
128
  def _attach_user_targets(
125
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
129
+ neo4j_session: neo4j.Session,
130
+ data: List[Dict],
131
+ update_tag: int,
126
132
  ) -> None:
127
133
  """
128
134
  Add relationship between escalation policy and services.
@@ -142,7 +148,9 @@ def _attach_user_targets(
142
148
 
143
149
 
144
150
  def _attach_schedule_targets(
145
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
151
+ neo4j_session: neo4j.Session,
152
+ data: List[Dict],
153
+ update_tag: int,
146
154
  ) -> None:
147
155
  """
148
156
  Add relationship between escalation policy and services.
@@ -162,7 +170,9 @@ def _attach_schedule_targets(
162
170
 
163
171
 
164
172
  def _attach_services(
165
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
173
+ neo4j_session: neo4j.Session,
174
+ data: List[Dict],
175
+ update_tag: int,
166
176
  ) -> None:
167
177
  """
168
178
  Add relationship between escalation policy and services.
@@ -182,7 +192,9 @@ def _attach_services(
182
192
 
183
193
 
184
194
  def _attach_teams(
185
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
195
+ neo4j_session: neo4j.Session,
196
+ data: List[Dict],
197
+ update_tag: int,
186
198
  ) -> None:
187
199
  """
188
200
  Add relationship between escalation policy and teams.
@@ -32,7 +32,9 @@ def get_schedules(pd_session: APISession) -> List[Dict[str, Any]]:
32
32
 
33
33
 
34
34
  def load_schedule_data(
35
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
35
+ neo4j_session: neo4j.Session,
36
+ data: List[Dict],
37
+ update_tag: int,
36
38
  ) -> None:
37
39
  """
38
40
  Transform and load schedule information
@@ -72,7 +74,9 @@ def load_schedule_data(
72
74
 
73
75
 
74
76
  def _attach_users(
75
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
77
+ neo4j_session: neo4j.Session,
78
+ data: List[Dict],
79
+ update_tag: int,
76
80
  ) -> None:
77
81
  """
78
82
  Add relationship between schedule and users.
@@ -91,7 +95,9 @@ def _attach_users(
91
95
 
92
96
 
93
97
  def _attach_layers(
94
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
98
+ neo4j_session: neo4j.Session,
99
+ data: List[Dict],
100
+ update_tag: int,
95
101
  ) -> None:
96
102
  """
97
103
  Create layers for a schedule and attach them together
@@ -133,7 +139,9 @@ def _attach_layers(
133
139
 
134
140
 
135
141
  def _attach_layer_users(
136
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
142
+ neo4j_session: neo4j.Session,
143
+ data: List[Dict],
144
+ update_tag: int,
137
145
  ) -> None:
138
146
  """
139
147
  Add relationship between schedule layers and users.
@@ -34,7 +34,8 @@ def get_services(pd_session: APISession) -> List[Dict[str, Any]]:
34
34
 
35
35
  @timeit
36
36
  def get_integrations(
37
- pd_session: APISession, services: List[Dict[str, Any]],
37
+ pd_session: APISession,
38
+ services: List[Dict[str, Any]],
38
39
  ) -> List[Dict[str, Any]]:
39
40
  """
40
41
  Get integrations from services.
@@ -51,7 +52,9 @@ def get_integrations(
51
52
 
52
53
 
53
54
  def load_service_data(
54
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
55
+ neo4j_session: neo4j.Session,
56
+ data: List[Dict],
57
+ update_tag: int,
55
58
  ) -> None:
56
59
  """
57
60
  Transform and load service information
@@ -104,7 +107,9 @@ def load_service_data(
104
107
 
105
108
 
106
109
  def _attach_teams(
107
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
110
+ neo4j_session: neo4j.Session,
111
+ data: List[Dict],
112
+ update_tag: int,
108
113
  ) -> None:
109
114
  """
110
115
  Add relationship between teams and services.
@@ -123,7 +128,9 @@ def _attach_teams(
123
128
 
124
129
 
125
130
  def load_integration_data(
126
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
131
+ neo4j_session: neo4j.Session,
132
+ data: List[Dict],
133
+ update_tag: int,
127
134
  ) -> None:
128
135
  """
129
136
  Transform and load integration information
@@ -33,7 +33,8 @@ def get_teams(pd_session: APISession) -> List[Dict[str, Any]]:
33
33
 
34
34
  @timeit
35
35
  def get_team_members(
36
- pd_session: APISession, teams: List[Dict[str, Any]],
36
+ pd_session: APISession,
37
+ teams: List[Dict[str, Any]],
37
38
  ) -> List[Dict[str, str]]:
38
39
  relations: List[Dict[str, str]] = []
39
40
  for team in teams:
@@ -46,7 +47,9 @@ def get_team_members(
46
47
 
47
48
 
48
49
  def load_team_data(
49
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
50
+ neo4j_session: neo4j.Session,
51
+ data: List[Dict],
52
+ update_tag: int,
50
53
  ) -> None:
51
54
  """
52
55
  Transform and load teamuser information
@@ -73,7 +76,9 @@ def load_team_data(
73
76
 
74
77
 
75
78
  def load_team_relations(
76
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
79
+ neo4j_session: neo4j.Session,
80
+ data: List[Dict],
81
+ update_tag: int,
77
82
  ) -> None:
78
83
  """
79
84
  Attach users to their teams
@@ -30,7 +30,9 @@ def get_users(pd_session: APISession) -> List[Dict[str, Any]]:
30
30
 
31
31
 
32
32
  def load_user_data(
33
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
33
+ neo4j_session: neo4j.Session,
34
+ data: List[Dict],
35
+ update_tag: int,
34
36
  ) -> None:
35
37
  """
36
38
  Transform and load user information
@@ -30,7 +30,9 @@ def get_vendors(pd_session: APISession) -> List[Dict[str, Any]]:
30
30
 
31
31
 
32
32
  def load_vendor_data(
33
- neo4j_session: neo4j.Session, data: List[Dict], update_tag: int,
33
+ neo4j_session: neo4j.Session,
34
+ data: List[Dict],
35
+ update_tag: int,
34
36
  ) -> None:
35
37
  """
36
38
  Transform and load vendor information
@@ -8,23 +8,41 @@ from cartography.intel.semgrep.deployment import sync_deployment
8
8
  from cartography.intel.semgrep.findings import sync_findings
9
9
  from cartography.util import timeit
10
10
 
11
-
12
11
  logger = logging.getLogger(__name__)
13
12
 
14
13
 
15
14
  @timeit
16
15
  def start_semgrep_ingestion(
17
- neo4j_session: neo4j.Session, config: Config,
16
+ neo4j_session: neo4j.Session,
17
+ config: Config,
18
18
  ) -> None:
19
19
  common_job_parameters = {
20
20
  "UPDATE_TAG": config.update_tag,
21
21
  }
22
22
  if not config.semgrep_app_token:
23
- logger.info('Semgrep import is not configured - skipping this module. See docs to configure.')
23
+ logger.info(
24
+ "Semgrep import is not configured - skipping this module. See docs to configure.",
25
+ )
24
26
  return
25
27
 
26
28
  # sync_deployment must be called first since it populates common_job_parameters
27
29
  # with the deployment ID and slug, which are required by the other sync functions
28
- sync_deployment(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
29
- sync_dependencies(neo4j_session, config.semgrep_app_token, config.semgrep_dependency_ecosystems, config.update_tag, common_job_parameters) # noqa: E501
30
- sync_findings(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
30
+ sync_deployment(
31
+ neo4j_session,
32
+ config.semgrep_app_token,
33
+ config.update_tag,
34
+ common_job_parameters,
35
+ )
36
+ sync_dependencies(
37
+ neo4j_session,
38
+ config.semgrep_app_token,
39
+ config.semgrep_dependency_ecosystems,
40
+ config.update_tag,
41
+ common_job_parameters,
42
+ ) # noqa: E501
43
+ sync_findings(
44
+ neo4j_session,
45
+ config.semgrep_app_token,
46
+ config.update_tag,
47
+ common_job_parameters,
48
+ )
@@ -26,30 +26,34 @@ _MAX_RETRIES = 3
26
26
  # The keys in this dictionary must be in Semgrep's list of supported ecosystems, defined here:
27
27
  # https://semgrep.dev/api/v1/docs/#tag/SupplyChainService/operation/semgrep_app.products.sca.handlers.dependency.list_dependencies_conexxion
28
28
  ECOSYSTEM_TO_SCHEMA: Dict = {
29
- 'gomod': SemgrepGoLibrarySchema,
30
- 'npm': SemgrepNpmLibrarySchema,
29
+ "gomod": SemgrepGoLibrarySchema,
30
+ "npm": SemgrepNpmLibrarySchema,
31
31
  }
32
32
 
33
33
 
34
34
  def parse_and_validate_semgrep_ecosystems(ecosystems: str) -> List[str]:
35
35
  validated_ecosystems: List[str] = []
36
- for ecosystem in ecosystems.split(','):
36
+ for ecosystem in ecosystems.split(","):
37
37
  ecosystem = ecosystem.strip().lower()
38
38
 
39
39
  if ecosystem in ECOSYSTEM_TO_SCHEMA:
40
40
  validated_ecosystems.append(ecosystem)
41
41
  else:
42
- valid_ecosystems: str = ','.join(ECOSYSTEM_TO_SCHEMA.keys())
42
+ valid_ecosystems: str = ",".join(ECOSYSTEM_TO_SCHEMA.keys())
43
43
  raise ValueError(
44
44
  f'Error parsing `semgrep-dependency-ecosystems`. You specified "{ecosystems}". '
45
45
  f'Please check that your input is formatted as comma-separated values, e.g. "gomod,npm". '
46
- f'Full list of supported ecosystems: {valid_ecosystems}.',
46
+ f"Full list of supported ecosystems: {valid_ecosystems}.",
47
47
  )
48
48
  return validated_ecosystems
49
49
 
50
50
 
51
51
  @timeit
52
- def get_dependencies(semgrep_app_token: str, deployment_id: str, ecosystem: str) -> List[Dict[str, Any]]:
52
+ def get_dependencies(
53
+ semgrep_app_token: str,
54
+ deployment_id: str,
55
+ ecosystem: str,
56
+ ) -> List[Dict[str, Any]]:
53
57
  """
54
58
  Gets all dependencies for the given ecosystem within the given Semgrep deployment ID.
55
59
  param: semgrep_app_token: The Semgrep App token to use for authentication.
@@ -73,14 +77,23 @@ def get_dependencies(semgrep_app_token: str, deployment_id: str, ecosystem: str)
73
77
  },
74
78
  }
75
79
 
76
- logger.info(f"Retrieving Semgrep {ecosystem} dependencies for deployment '{deployment_id}'.")
80
+ logger.info(
81
+ f"Retrieving Semgrep {ecosystem} dependencies for deployment '{deployment_id}'.",
82
+ )
77
83
  while has_more:
78
84
  try:
79
- response = requests.post(deps_url, json=request_data, headers=headers, timeout=_TIMEOUT)
85
+ response = requests.post(
86
+ deps_url,
87
+ json=request_data,
88
+ headers=headers,
89
+ timeout=_TIMEOUT,
90
+ )
80
91
  response.raise_for_status()
81
92
  data = response.json()
82
93
  except (ReadTimeout, HTTPError):
83
- logger.warning(f"Failed to retrieve Semgrep {ecosystem} dependencies for page {page}. Retrying...")
94
+ logger.warning(
95
+ f"Failed to retrieve Semgrep {ecosystem} dependencies for page {page}. Retrying...",
96
+ )
84
97
  retries += 1
85
98
  if retries >= _MAX_RETRIES:
86
99
  raise
@@ -93,7 +106,9 @@ def get_dependencies(semgrep_app_token: str, deployment_id: str, ecosystem: str)
93
106
  page += 1
94
107
  request_data["cursor"] = data.get("cursor")
95
108
 
96
- logger.info(f"Retrieved {len(all_deps)} Semgrep {ecosystem} dependencies in {page} pages.")
109
+ logger.info(
110
+ f"Retrieved {len(all_deps)} Semgrep {ecosystem} dependencies in {page} pages.",
111
+ )
97
112
  return all_deps
98
113
 
99
114
 
@@ -142,19 +157,20 @@ def transform_dependencies(raw_deps: List[Dict[str, Any]]) -> List[Dict[str, Any
142
157
  # If Semgrep eventually supports version specifiers, update this line accordingly.
143
158
  specifier = f"=={version}"
144
159
 
145
- deps.append({
146
- # existing dependency properties:
147
- "id": id,
148
- "name": name,
149
- "specifier": specifier,
150
- "version": version,
151
- "repo_url": repo_url,
152
-
153
- # Semgrep-specific properties:
154
- "ecosystem": raw_dep["ecosystem"],
155
- "transitivity": raw_dep["transitivity"].lower(),
156
- "url": raw_dep["definedAt"]["url"],
157
- })
160
+ deps.append(
161
+ {
162
+ # existing dependency properties:
163
+ "id": id,
164
+ "name": name,
165
+ "specifier": specifier,
166
+ "version": version,
167
+ "repo_url": repo_url,
168
+ # Semgrep-specific properties:
169
+ "ecosystem": raw_dep["ecosystem"],
170
+ "transitivity": raw_dep["transitivity"].lower(),
171
+ "url": raw_dep["definedAt"]["url"],
172
+ },
173
+ )
158
174
 
159
175
  return deps
160
176
 
@@ -167,7 +183,9 @@ def load_dependencies(
167
183
  deployment_id: str,
168
184
  update_tag: int,
169
185
  ) -> None:
170
- logger.info(f"Loading {len(dependencies)} {dependency_schema().label} objects into the graph.")
186
+ logger.info(
187
+ f"Loading {len(dependencies)} {dependency_schema().label} objects into the graph.",
188
+ )
171
189
  load(
172
190
  neo4j_session,
173
191
  dependency_schema(),
@@ -183,8 +201,12 @@ def cleanup(
183
201
  dependency_schema: Callable,
184
202
  common_job_parameters: Dict[str, Any],
185
203
  ) -> None:
186
- logger.info(f"Running Semgrep Dependencies cleanup job for {dependency_schema().label}.")
187
- GraphJob.from_node_schema(dependency_schema(), common_job_parameters).run(neo4j_session)
204
+ logger.info(
205
+ f"Running Semgrep Dependencies cleanup job for {dependency_schema().label}.",
206
+ )
207
+ GraphJob.from_node_schema(dependency_schema(), common_job_parameters).run(
208
+ neo4j_session,
209
+ )
188
210
 
189
211
 
190
212
  @timeit
@@ -225,9 +247,9 @@ def sync_dependencies(
225
247
 
226
248
  merge_module_sync_metadata(
227
249
  neo4j_session=neo4j_session,
228
- group_type='Semgrep',
250
+ group_type="Semgrep",
229
251
  group_id=deployment_id,
230
- synced_type='SemgrepDependency',
252
+ synced_type="SemgrepDependency",
231
253
  update_tag=update_tag,
232
254
  stat_handler=stat_handler,
233
255
  )
@@ -40,7 +40,9 @@ def get_deployment(semgrep_app_token: str) -> Dict[str, Any]:
40
40
 
41
41
  @timeit
42
42
  def load_semgrep_deployment(
43
- neo4j_session: neo4j.Session, deployment: Dict[str, Any], update_tag: int,
43
+ neo4j_session: neo4j.Session,
44
+ deployment: Dict[str, Any],
45
+ update_tag: int,
44
46
  ) -> None:
45
47
  logger.info(f"Loading SemgrepDeployment {deployment} into the graph.")
46
48
  load(
@@ -54,11 +54,18 @@ def get_sca_vulns(semgrep_app_token: str, deployment_slug: str) -> List[Dict[str
54
54
  while has_more:
55
55
 
56
56
  try:
57
- response = requests.get(sca_url, params=request_data, headers=headers, timeout=_TIMEOUT)
57
+ response = requests.get(
58
+ sca_url,
59
+ params=request_data,
60
+ headers=headers,
61
+ timeout=_TIMEOUT,
62
+ )
58
63
  response.raise_for_status()
59
64
  data = response.json()
60
65
  except (ReadTimeout, HTTPError):
61
- logger.warning(f"Failed to retrieve Semgrep SCA vulns for page {page}. Retrying...")
66
+ logger.warning(
67
+ f"Failed to retrieve Semgrep SCA vulns for page {page}. Retrying...",
68
+ )
62
69
  retries += 1
63
70
  if retries >= _MAX_RETRIES:
64
71
  raise
@@ -104,14 +111,16 @@ def _determine_exposure(vuln: Dict[str, Any]) -> str | None:
104
111
 
105
112
 
106
113
  def _build_vuln_url(vuln: str) -> str | None:
107
- if 'CVE' in vuln:
114
+ if "CVE" in vuln:
108
115
  return f"https://nvd.nist.gov/vuln/detail/{vuln}"
109
- if 'GHSA' in vuln:
116
+ if "GHSA" in vuln:
110
117
  return f"https://github.com/advisories/{vuln}"
111
118
  return None
112
119
 
113
120
 
114
- def transform_sca_vulns(raw_vulns: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List[Dict[str, str]]]:
121
+ def transform_sca_vulns(
122
+ raw_vulns: List[Dict[str, Any]],
123
+ ) -> Tuple[List[Dict[str, Any]], List[Dict[str, str]]]:
115
124
  """
116
125
  Transforms the raw SCA vulns response from Semgrep API into a list of dicts
117
126
  that can be used to create the SemgrepSCAFinding nodes.
@@ -124,7 +133,7 @@ def transform_sca_vulns(raw_vulns: List[Dict[str, Any]]) -> Tuple[List[Dict[str,
124
133
  repository_name = vuln["repository"]["name"]
125
134
  rule_id = vuln["rule"]["name"]
126
135
  vulnerability_class = _get_vuln_class(vuln)
127
- package = vuln['found_dependency']['package']
136
+ package = vuln["found_dependency"]["package"]
128
137
  sca_vuln["id"] = vuln["id"]
129
138
  sca_vuln["repositoryName"] = repository_name
130
139
  sca_vuln["branch"] = vuln["ref"]
@@ -133,9 +142,15 @@ def transform_sca_vulns(raw_vulns: List[Dict[str, Any]]) -> Tuple[List[Dict[str,
133
142
  sca_vuln["description"] = vuln["rule"]["message"]
134
143
  sca_vuln["ecosystem"] = vuln["found_dependency"]["ecosystem"]
135
144
  sca_vuln["severity"] = vuln["severity"].upper()
136
- sca_vuln["reachability"] = vuln["reachability"].upper() # Check done to determine rechabilitity
137
- sca_vuln["reachableIf"] = vuln["reachable_condition"].upper() if vuln["reachable_condition"] else None
138
- sca_vuln["exposureType"] = _determine_exposure(vuln) # Determintes if reachable or unreachable
145
+ sca_vuln["reachability"] = vuln[
146
+ "reachability"
147
+ ].upper() # Check done to determine rechabilitity
148
+ sca_vuln["reachableIf"] = (
149
+ vuln["reachable_condition"].upper() if vuln["reachable_condition"] else None
150
+ )
151
+ sca_vuln["exposureType"] = _determine_exposure(
152
+ vuln,
153
+ ) # Determintes if reachable or unreachable
139
154
  dependency = f"{package}|{vuln['found_dependency']['version']}"
140
155
  sca_vuln["matchedDependency"] = dependency
141
156
  dep_url = vuln["found_dependency"]["lockfile_line_url"]
@@ -145,14 +160,16 @@ def transform_sca_vulns(raw_vulns: List[Dict[str, Any]]) -> Tuple[List[Dict[str,
145
160
  sca_vuln["dependencyFileLocation_url"] = dep_url
146
161
  else:
147
162
  if sca_vuln.get("location"):
148
- sca_vuln["dependencyFileLocation_path"] = sca_vuln["location"]["file_path"]
163
+ sca_vuln["dependencyFileLocation_path"] = sca_vuln["location"][
164
+ "file_path"
165
+ ]
149
166
  sca_vuln["transitivity"] = vuln["found_dependency"]["transitivity"].upper()
150
167
  if vuln.get("vulnerability_identifier"):
151
168
  vuln_id = vuln["vulnerability_identifier"].upper()
152
169
  sca_vuln["cveId"] = vuln_id
153
170
  sca_vuln["ref_urls"] = [_build_vuln_url(vuln_id)]
154
- if vuln.get('fix_recommendations') and len(vuln['fix_recommendations']) > 0:
155
- fix = vuln['fix_recommendations'][0]
171
+ if vuln.get("fix_recommendations") and len(vuln["fix_recommendations"]) > 0:
172
+ fix = vuln["fix_recommendations"][0]
156
173
  dep_fix = f"{fix['package']}|{fix['version']}"
157
174
  sca_vuln["closestSafeDependency"] = dep_fix
158
175
  sca_vuln["openedAt"] = vuln["created_at"]
@@ -213,16 +230,19 @@ def load_semgrep_sca_usages(
213
230
 
214
231
  @timeit
215
232
  def cleanup(
216
- neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any],
233
+ neo4j_session: neo4j.Session,
234
+ common_job_parameters: Dict[str, Any],
217
235
  ) -> None:
218
236
  logger.info("Running Semgrep SCA findings cleanup job.")
219
237
  findings_cleanup_job = GraphJob.from_node_schema(
220
- SemgrepSCAFindingSchema(), common_job_parameters,
238
+ SemgrepSCAFindingSchema(),
239
+ common_job_parameters,
221
240
  )
222
241
  findings_cleanup_job.run(neo4j_session)
223
242
  logger.info("Running Semgrep SCA Locations cleanup job.")
224
243
  locations_cleanup_job = GraphJob.from_node_schema(
225
- SemgrepSCALocationSchema(), common_job_parameters,
244
+ SemgrepSCALocationSchema(),
245
+ common_job_parameters,
226
246
  )
227
247
  locations_cleanup_job.run(neo4j_session)
228
248
 
@@ -249,14 +269,18 @@ def sync_findings(
249
269
  vulns, usages = transform_sca_vulns(raw_vulns)
250
270
  load_semgrep_sca_vulns(neo4j_session, vulns, deployment_id, update_tag)
251
271
  load_semgrep_sca_usages(neo4j_session, usages, deployment_id, update_tag)
252
- run_scoped_analysis_job('semgrep_sca_risk_analysis.json', neo4j_session, common_job_parameters)
272
+ run_scoped_analysis_job(
273
+ "semgrep_sca_risk_analysis.json",
274
+ neo4j_session,
275
+ common_job_parameters,
276
+ )
253
277
 
254
278
  cleanup(neo4j_session, common_job_parameters)
255
279
  merge_module_sync_metadata(
256
280
  neo4j_session=neo4j_session,
257
- group_type='Semgrep',
281
+ group_type="Semgrep",
258
282
  group_id=deployment_id,
259
- synced_type='SCA',
283
+ synced_type="SCA",
260
284
  update_tag=update_tag,
261
285
  stat_handler=stat_handler,
262
286
  )
@@ -14,7 +14,11 @@ stat_handler = get_stats_client(__name__)
14
14
 
15
15
  @timeit
16
16
  def start_snipeit_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
17
- if config.snipeit_base_uri is None or config.snipeit_token is None or config.snipeit_tenant_id is None:
17
+ if (
18
+ config.snipeit_base_uri is None
19
+ or config.snipeit_token is None
20
+ or config.snipeit_tenant_id is None
21
+ ):
18
22
  logger.warning(
19
23
  "Required parameter(s) missing. Skipping sync.",
20
24
  )
@@ -26,5 +30,15 @@ def start_snipeit_ingestion(neo4j_session: neo4j.Session, config: Config) -> Non
26
30
  }
27
31
 
28
32
  # Ingest SnipeIT users and assets
29
- user.sync(neo4j_session, common_job_parameters, config.snipeit_base_uri, config.snipeit_token)
30
- asset.sync(neo4j_session, common_job_parameters, config.snipeit_base_uri, config.snipeit_token)
33
+ user.sync(
34
+ neo4j_session,
35
+ common_job_parameters,
36
+ config.snipeit_base_uri,
37
+ config.snipeit_token,
38
+ )
39
+ asset.sync(
40
+ neo4j_session,
41
+ common_job_parameters,
42
+ config.snipeit_base_uri,
43
+ config.snipeit_token,
44
+ )