cartography 0.102.0rc1__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

Files changed (251) hide show
  1. cartography/__main__.py +1 -2
  2. cartography/_version.py +2 -2
  3. cartography/cli.py +302 -253
  4. cartography/client/core/tx.py +39 -18
  5. cartography/config.py +4 -0
  6. cartography/driftdetect/__main__.py +1 -2
  7. cartography/driftdetect/add_shortcut.py +10 -2
  8. cartography/driftdetect/cli.py +71 -75
  9. cartography/driftdetect/detect_deviations.py +7 -3
  10. cartography/driftdetect/get_states.py +20 -8
  11. cartography/driftdetect/model.py +5 -5
  12. cartography/driftdetect/serializers.py +8 -6
  13. cartography/driftdetect/storage.py +2 -2
  14. cartography/graph/cleanupbuilder.py +35 -15
  15. cartography/graph/job.py +46 -17
  16. cartography/graph/querybuilder.py +165 -80
  17. cartography/graph/statement.py +35 -26
  18. cartography/intel/analysis.py +4 -1
  19. cartography/intel/aws/__init__.py +114 -55
  20. cartography/intel/aws/apigateway.py +134 -63
  21. cartography/intel/aws/cloudtrail.py +127 -0
  22. cartography/intel/aws/config.py +56 -20
  23. cartography/intel/aws/dynamodb.py +108 -40
  24. cartography/intel/aws/ec2/__init__.py +2 -2
  25. cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
  26. cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
  27. cartography/intel/aws/ec2/images.py +49 -20
  28. cartography/intel/aws/ec2/instances.py +234 -136
  29. cartography/intel/aws/ec2/internet_gateways.py +40 -11
  30. cartography/intel/aws/ec2/key_pairs.py +44 -20
  31. cartography/intel/aws/ec2/launch_templates.py +101 -59
  32. cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
  33. cartography/intel/aws/ec2/load_balancers.py +82 -42
  34. cartography/intel/aws/ec2/network_acls.py +89 -65
  35. cartography/intel/aws/ec2/network_interfaces.py +146 -87
  36. cartography/intel/aws/ec2/reserved_instances.py +45 -16
  37. cartography/intel/aws/ec2/route_tables.py +327 -0
  38. cartography/intel/aws/ec2/security_groups.py +71 -21
  39. cartography/intel/aws/ec2/snapshots.py +61 -22
  40. cartography/intel/aws/ec2/subnets.py +54 -18
  41. cartography/intel/aws/ec2/tgw.py +100 -34
  42. cartography/intel/aws/ec2/util.py +1 -1
  43. cartography/intel/aws/ec2/volumes.py +69 -41
  44. cartography/intel/aws/ec2/vpc.py +37 -12
  45. cartography/intel/aws/ec2/vpc_peerings.py +83 -24
  46. cartography/intel/aws/ecr.py +88 -32
  47. cartography/intel/aws/ecs.py +83 -47
  48. cartography/intel/aws/eks.py +55 -29
  49. cartography/intel/aws/elasticache.py +42 -18
  50. cartography/intel/aws/elasticsearch.py +57 -20
  51. cartography/intel/aws/emr.py +61 -23
  52. cartography/intel/aws/iam.py +401 -145
  53. cartography/intel/aws/iam_instance_profiles.py +22 -22
  54. cartography/intel/aws/identitycenter.py +71 -37
  55. cartography/intel/aws/inspector.py +159 -89
  56. cartography/intel/aws/kms.py +92 -38
  57. cartography/intel/aws/lambda_function.py +103 -34
  58. cartography/intel/aws/organizations.py +30 -10
  59. cartography/intel/aws/permission_relationships.py +133 -51
  60. cartography/intel/aws/rds.py +249 -85
  61. cartography/intel/aws/redshift.py +107 -46
  62. cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
  63. cartography/intel/aws/resources.py +53 -44
  64. cartography/intel/aws/route53.py +108 -61
  65. cartography/intel/aws/s3.py +168 -83
  66. cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
  67. cartography/intel/aws/secretsmanager.py +24 -12
  68. cartography/intel/aws/securityhub.py +20 -9
  69. cartography/intel/aws/sns.py +166 -0
  70. cartography/intel/aws/sqs.py +60 -28
  71. cartography/intel/aws/ssm.py +70 -30
  72. cartography/intel/aws/util/arns.py +7 -7
  73. cartography/intel/aws/util/common.py +31 -4
  74. cartography/intel/azure/__init__.py +78 -19
  75. cartography/intel/azure/compute.py +101 -27
  76. cartography/intel/azure/cosmosdb.py +496 -170
  77. cartography/intel/azure/sql.py +296 -105
  78. cartography/intel/azure/storage.py +322 -113
  79. cartography/intel/azure/subscription.py +39 -23
  80. cartography/intel/azure/tenant.py +13 -4
  81. cartography/intel/azure/util/credentials.py +95 -55
  82. cartography/intel/bigfix/__init__.py +2 -2
  83. cartography/intel/bigfix/computers.py +93 -65
  84. cartography/intel/create_indexes.py +3 -2
  85. cartography/intel/crowdstrike/__init__.py +11 -9
  86. cartography/intel/crowdstrike/endpoints.py +5 -1
  87. cartography/intel/crowdstrike/spotlight.py +8 -3
  88. cartography/intel/cve/__init__.py +46 -13
  89. cartography/intel/cve/feed.py +48 -12
  90. cartography/intel/digitalocean/__init__.py +22 -13
  91. cartography/intel/digitalocean/compute.py +75 -108
  92. cartography/intel/digitalocean/management.py +44 -80
  93. cartography/intel/digitalocean/platform.py +48 -43
  94. cartography/intel/dns.py +36 -10
  95. cartography/intel/duo/__init__.py +21 -16
  96. cartography/intel/duo/api_host.py +14 -9
  97. cartography/intel/duo/endpoints.py +50 -45
  98. cartography/intel/duo/groups.py +18 -14
  99. cartography/intel/duo/phones.py +37 -34
  100. cartography/intel/duo/tokens.py +26 -23
  101. cartography/intel/duo/users.py +54 -50
  102. cartography/intel/duo/web_authn_credentials.py +30 -25
  103. cartography/intel/entra/__init__.py +25 -7
  104. cartography/intel/entra/ou.py +112 -0
  105. cartography/intel/entra/users.py +69 -63
  106. cartography/intel/gcp/__init__.py +185 -49
  107. cartography/intel/gcp/compute.py +418 -231
  108. cartography/intel/gcp/crm.py +96 -43
  109. cartography/intel/gcp/dns.py +60 -19
  110. cartography/intel/gcp/gke.py +72 -38
  111. cartography/intel/gcp/iam.py +61 -41
  112. cartography/intel/gcp/storage.py +84 -55
  113. cartography/intel/github/__init__.py +13 -11
  114. cartography/intel/github/repos.py +270 -137
  115. cartography/intel/github/teams.py +170 -88
  116. cartography/intel/github/users.py +70 -39
  117. cartography/intel/github/util.py +36 -34
  118. cartography/intel/gsuite/__init__.py +47 -26
  119. cartography/intel/gsuite/api.py +73 -30
  120. cartography/intel/jamf/__init__.py +19 -1
  121. cartography/intel/jamf/computers.py +30 -7
  122. cartography/intel/jamf/util.py +7 -2
  123. cartography/intel/kandji/__init__.py +6 -3
  124. cartography/intel/kandji/devices.py +14 -8
  125. cartography/intel/kubernetes/namespaces.py +7 -4
  126. cartography/intel/kubernetes/pods.py +7 -4
  127. cartography/intel/kubernetes/services.py +8 -4
  128. cartography/intel/lastpass/__init__.py +2 -2
  129. cartography/intel/lastpass/users.py +23 -12
  130. cartography/intel/oci/__init__.py +44 -11
  131. cartography/intel/oci/iam.py +134 -38
  132. cartography/intel/oci/organizations.py +13 -6
  133. cartography/intel/oci/utils.py +43 -20
  134. cartography/intel/okta/__init__.py +66 -15
  135. cartography/intel/okta/applications.py +42 -20
  136. cartography/intel/okta/awssaml.py +93 -33
  137. cartography/intel/okta/factors.py +16 -4
  138. cartography/intel/okta/groups.py +56 -29
  139. cartography/intel/okta/organization.py +5 -1
  140. cartography/intel/okta/origins.py +6 -2
  141. cartography/intel/okta/roles.py +15 -5
  142. cartography/intel/okta/users.py +20 -8
  143. cartography/intel/okta/utils.py +6 -4
  144. cartography/intel/pagerduty/__init__.py +8 -7
  145. cartography/intel/pagerduty/escalation_policies.py +18 -6
  146. cartography/intel/pagerduty/schedules.py +12 -4
  147. cartography/intel/pagerduty/services.py +11 -4
  148. cartography/intel/pagerduty/teams.py +8 -3
  149. cartography/intel/pagerduty/users.py +3 -1
  150. cartography/intel/pagerduty/vendors.py +3 -1
  151. cartography/intel/semgrep/__init__.py +24 -6
  152. cartography/intel/semgrep/dependencies.py +50 -28
  153. cartography/intel/semgrep/deployment.py +3 -1
  154. cartography/intel/semgrep/findings.py +42 -18
  155. cartography/intel/snipeit/__init__.py +17 -3
  156. cartography/intel/snipeit/asset.py +12 -6
  157. cartography/intel/snipeit/user.py +8 -5
  158. cartography/intel/snipeit/util.py +9 -4
  159. cartography/models/aws/apigateway.py +21 -17
  160. cartography/models/aws/apigatewaycertificate.py +28 -22
  161. cartography/models/aws/apigatewayresource.py +28 -20
  162. cartography/models/aws/apigatewaystage.py +33 -25
  163. cartography/models/aws/cloudtrail/__init__.py +0 -0
  164. cartography/models/aws/cloudtrail/trail.py +61 -0
  165. cartography/models/aws/dynamodb/gsi.py +30 -22
  166. cartography/models/aws/dynamodb/tables.py +25 -17
  167. cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
  168. cartography/models/aws/ec2/images.py +36 -34
  169. cartography/models/aws/ec2/instances.py +51 -45
  170. cartography/models/aws/ec2/keypair.py +21 -16
  171. cartography/models/aws/ec2/keypair_instance.py +28 -21
  172. cartography/models/aws/ec2/launch_configurations.py +30 -26
  173. cartography/models/aws/ec2/launch_template_versions.py +48 -38
  174. cartography/models/aws/ec2/launch_templates.py +21 -17
  175. cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
  176. cartography/models/aws/ec2/load_balancers.py +47 -37
  177. cartography/models/aws/ec2/network_acl_rules.py +38 -30
  178. cartography/models/aws/ec2/network_acls.py +38 -29
  179. cartography/models/aws/ec2/networkinterface_instance.py +52 -39
  180. cartography/models/aws/ec2/networkinterfaces.py +53 -37
  181. cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
  182. cartography/models/aws/ec2/reservations.py +18 -14
  183. cartography/models/aws/ec2/route_table_associations.py +97 -0
  184. cartography/models/aws/ec2/route_tables.py +128 -0
  185. cartography/models/aws/ec2/routes.py +85 -0
  186. cartography/models/aws/ec2/securitygroup_instance.py +29 -20
  187. cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
  188. cartography/models/aws/ec2/subnet_instance.py +24 -19
  189. cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
  190. cartography/models/aws/ec2/volumes.py +47 -40
  191. cartography/models/aws/eks/clusters.py +23 -21
  192. cartography/models/aws/emr.py +32 -30
  193. cartography/models/aws/iam/instanceprofile.py +33 -24
  194. cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
  195. cartography/models/aws/identitycenter/awspermissionset.py +37 -29
  196. cartography/models/aws/identitycenter/awsssouser.py +23 -21
  197. cartography/models/aws/inspector/findings.py +77 -65
  198. cartography/models/aws/inspector/packages.py +35 -29
  199. cartography/models/aws/s3/__init__.py +0 -0
  200. cartography/models/aws/s3/account_public_access_block.py +51 -0
  201. cartography/models/aws/sns/__init__.py +0 -0
  202. cartography/models/aws/sns/topic.py +50 -0
  203. cartography/models/aws/ssm/instance_information.py +51 -39
  204. cartography/models/aws/ssm/instance_patch.py +32 -26
  205. cartography/models/bigfix/bigfix_computer.py +42 -38
  206. cartography/models/bigfix/bigfix_root.py +3 -3
  207. cartography/models/core/common.py +12 -10
  208. cartography/models/core/nodes.py +5 -2
  209. cartography/models/core/relationships.py +14 -6
  210. cartography/models/crowdstrike/hosts.py +37 -35
  211. cartography/models/cve/cve.py +34 -32
  212. cartography/models/cve/cve_feed.py +6 -6
  213. cartography/models/digitalocean/__init__.py +0 -0
  214. cartography/models/digitalocean/account.py +21 -0
  215. cartography/models/digitalocean/droplet.py +56 -0
  216. cartography/models/digitalocean/project.py +48 -0
  217. cartography/models/duo/api_host.py +3 -3
  218. cartography/models/duo/endpoint.py +43 -41
  219. cartography/models/duo/group.py +14 -14
  220. cartography/models/duo/phone.py +27 -27
  221. cartography/models/duo/token.py +16 -16
  222. cartography/models/duo/user.py +46 -44
  223. cartography/models/duo/web_authn_credential.py +27 -19
  224. cartography/models/entra/ou.py +48 -0
  225. cartography/models/entra/tenant.py +24 -18
  226. cartography/models/entra/user.py +64 -48
  227. cartography/models/gcp/iam.py +23 -23
  228. cartography/models/github/orgs.py +5 -4
  229. cartography/models/github/teams.py +37 -31
  230. cartography/models/github/users.py +34 -23
  231. cartography/models/kandji/device.py +22 -16
  232. cartography/models/kandji/tenant.py +6 -4
  233. cartography/models/lastpass/tenant.py +3 -3
  234. cartography/models/lastpass/user.py +32 -28
  235. cartography/models/semgrep/dependencies.py +36 -24
  236. cartography/models/semgrep/deployment.py +5 -5
  237. cartography/models/semgrep/findings.py +58 -42
  238. cartography/models/semgrep/locations.py +27 -21
  239. cartography/models/snipeit/asset.py +30 -21
  240. cartography/models/snipeit/tenant.py +6 -4
  241. cartography/models/snipeit/user.py +19 -12
  242. cartography/stats.py +3 -3
  243. cartography/sync.py +107 -31
  244. cartography/util.py +84 -62
  245. {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/METADATA +3 -14
  246. cartography-0.103.0rc1.dist-info/RECORD +396 -0
  247. {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/WHEEL +1 -1
  248. cartography-0.102.0rc1.dist-info/RECORD +0 -377
  249. {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/entry_points.txt +0 -0
  250. {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/licenses/LICENSE +0 -0
  251. {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/top_level.txt +0 -0
@@ -15,35 +15,42 @@ class SnipeitUserNodeProperties(CartographyNodeProperties):
15
15
  """
16
16
  Ref: https://snipe-it.readme.io/reference/users
17
17
  """
18
+
18
19
  # Common properties
19
- id: PropertyRef = PropertyRef('id')
20
- lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
20
+ id: PropertyRef = PropertyRef("id")
21
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
21
22
 
22
23
  # SnipeIT specific properties
23
- company: PropertyRef = PropertyRef('company_id.name', extra_index=True)
24
- email: PropertyRef = PropertyRef('email', extra_index=True)
25
- username: PropertyRef = PropertyRef('username')
24
+ company: PropertyRef = PropertyRef("company_id.name", extra_index=True)
25
+ email: PropertyRef = PropertyRef("email", extra_index=True)
26
+ username: PropertyRef = PropertyRef("username")
26
27
 
27
28
 
28
29
  @dataclass(frozen=True)
29
30
  class SnipeitTenantToSnipeitUserRelProperties(CartographyRelProperties):
30
- lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
31
+ lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
31
32
 
32
33
 
33
34
  @dataclass(frozen=True)
34
35
  # (:SnipeitTenant)-[:HAS_USER]->(:SnipeitUser)
35
36
  class SnipeitTenantToSnipeitUserRel(CartographyRelSchema):
36
- target_node_label: str = 'SnipeitTenant'
37
+ target_node_label: str = "SnipeitTenant"
37
38
  target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
38
- {'id': PropertyRef('TENANT_ID', set_in_kwargs=True)},
39
+ {"id": PropertyRef("TENANT_ID", set_in_kwargs=True)},
39
40
  )
40
41
  direction: LinkDirection = LinkDirection.INWARD
41
42
  rel_label: str = "HAS_USER"
42
- properties: SnipeitTenantToSnipeitUserRelProperties = SnipeitTenantToSnipeitUserRelProperties()
43
+ properties: SnipeitTenantToSnipeitUserRelProperties = (
44
+ SnipeitTenantToSnipeitUserRelProperties()
45
+ )
43
46
 
44
47
 
45
48
  @dataclass(frozen=True)
46
49
  class SnipeitUserSchema(CartographyNodeSchema):
47
- label: str = 'SnipeitUser' # The label of the node
48
- properties: SnipeitUserNodeProperties = SnipeitUserNodeProperties() # An object representing all properties
49
- sub_resource_relationship: SnipeitTenantToSnipeitUserRel = SnipeitTenantToSnipeitUserRel()
50
+ label: str = "SnipeitUser" # The label of the node
51
+ properties: SnipeitUserNodeProperties = (
52
+ SnipeitUserNodeProperties()
53
+ ) # An object representing all properties
54
+ sub_resource_relationship: SnipeitTenantToSnipeitUserRel = (
55
+ SnipeitTenantToSnipeitUserRel()
56
+ )
cartography/stats.py CHANGED
@@ -17,11 +17,11 @@ class ScopedStatsClient:
17
17
 
18
18
  _client: StatsClient = None
19
19
 
20
- def __init__(self, prefix: Optional[str], root: 'ScopedStatsClient'):
20
+ def __init__(self, prefix: Optional[str], root: "ScopedStatsClient"):
21
21
  self._scope_prefix = prefix
22
22
  self._root = root
23
23
 
24
- def get_stats_client(self, scope: str) -> 'ScopedStatsClient':
24
+ def get_stats_client(self, scope: str) -> "ScopedStatsClient":
25
25
  """
26
26
  This method returns a new proxy to the same client
27
27
  which will prefix all calls to underlying methods with the scoped prefix
@@ -35,7 +35,7 @@ class ScopedStatsClient:
35
35
  return scoped_stats_client
36
36
 
37
37
  @staticmethod
38
- def get_root_client() -> 'ScopedStatsClient':
38
+ def get_root_client() -> "ScopedStatsClient":
39
39
  client = ScopedStatsClient(prefix=None, root=None) # type: ignore
40
40
  client._root = client
41
41
  return client
cartography/sync.py CHANGED
@@ -1,7 +1,9 @@
1
1
  import argparse
2
2
  import logging
3
+ import re
3
4
  import time
4
5
  from collections import OrderedDict
6
+ from pkgutil import iter_modules
5
7
  from typing import Callable
6
8
  from typing import List
7
9
  from typing import Tuple
@@ -39,28 +41,30 @@ from cartography.util import STATUS_SUCCESS
39
41
  logger = logging.getLogger(__name__)
40
42
 
41
43
 
42
- TOP_LEVEL_MODULES = OrderedDict({ # preserve order so that the default sync always runs `analysis` at the very end
43
- 'create-indexes': cartography.intel.create_indexes.run,
44
- 'aws': cartography.intel.aws.start_aws_ingestion,
45
- 'azure': cartography.intel.azure.start_azure_ingestion,
46
- 'entra': cartography.intel.entra.start_entra_ingestion,
47
- 'crowdstrike': cartography.intel.crowdstrike.start_crowdstrike_ingestion,
48
- 'gcp': cartography.intel.gcp.start_gcp_ingestion,
49
- 'gsuite': cartography.intel.gsuite.start_gsuite_ingestion,
50
- 'cve': cartography.intel.cve.start_cve_ingestion,
51
- 'oci': cartography.intel.oci.start_oci_ingestion,
52
- 'okta': cartography.intel.okta.start_okta_ingestion,
53
- 'github': cartography.intel.github.start_github_ingestion,
54
- 'digitalocean': cartography.intel.digitalocean.start_digitalocean_ingestion,
55
- 'kandji': cartography.intel.kandji.start_kandji_ingestion,
56
- 'kubernetes': cartography.intel.kubernetes.start_k8s_ingestion,
57
- 'lastpass': cartography.intel.lastpass.start_lastpass_ingestion,
58
- 'bigfix': cartography.intel.bigfix.start_bigfix_ingestion,
59
- 'duo': cartography.intel.duo.start_duo_ingestion,
60
- 'semgrep': cartography.intel.semgrep.start_semgrep_ingestion,
61
- 'snipeit': cartography.intel.snipeit.start_snipeit_ingestion,
62
- 'analysis': cartography.intel.analysis.run,
63
- })
44
+ TOP_LEVEL_MODULES = OrderedDict(
45
+ { # preserve order so that the default sync always runs `analysis` at the very end
46
+ "create-indexes": cartography.intel.create_indexes.run,
47
+ "aws": cartography.intel.aws.start_aws_ingestion,
48
+ "azure": cartography.intel.azure.start_azure_ingestion,
49
+ "entra": cartography.intel.entra.start_entra_ingestion,
50
+ "crowdstrike": cartography.intel.crowdstrike.start_crowdstrike_ingestion,
51
+ "gcp": cartography.intel.gcp.start_gcp_ingestion,
52
+ "gsuite": cartography.intel.gsuite.start_gsuite_ingestion,
53
+ "cve": cartography.intel.cve.start_cve_ingestion,
54
+ "oci": cartography.intel.oci.start_oci_ingestion,
55
+ "okta": cartography.intel.okta.start_okta_ingestion,
56
+ "github": cartography.intel.github.start_github_ingestion,
57
+ "digitalocean": cartography.intel.digitalocean.start_digitalocean_ingestion,
58
+ "kandji": cartography.intel.kandji.start_kandji_ingestion,
59
+ "kubernetes": cartography.intel.kubernetes.start_k8s_ingestion,
60
+ "lastpass": cartography.intel.lastpass.start_lastpass_ingestion,
61
+ "bigfix": cartography.intel.bigfix.start_bigfix_ingestion,
62
+ "duo": cartography.intel.duo.start_duo_ingestion,
63
+ "semgrep": cartography.intel.semgrep.start_semgrep_ingestion,
64
+ "snipeit": cartography.intel.snipeit.start_snipeit_ingestion,
65
+ "analysis": cartography.intel.analysis.run,
66
+ }
67
+ )
64
68
 
65
69
 
66
70
  class Sync:
@@ -98,7 +102,11 @@ class Sync:
98
102
  for name, func in stages:
99
103
  self.add_stage(name, func)
100
104
 
101
- def run(self, neo4j_driver: neo4j.Driver, config: Union[Config, argparse.Namespace]) -> int:
105
+ def run(
106
+ self,
107
+ neo4j_driver: neo4j.Driver,
108
+ config: Union[Config, argparse.Namespace],
109
+ ) -> int:
102
110
  """
103
111
  Execute all stages in the sync task in sequence.
104
112
 
@@ -117,12 +125,77 @@ class Sync:
117
125
  logger.warning("Sync interrupted during stage '%s'.", stage_name)
118
126
  raise
119
127
  except Exception:
120
- logger.exception("Unhandled exception during sync stage '%s'", stage_name)
128
+ logger.exception(
129
+ "Unhandled exception during sync stage '%s'",
130
+ stage_name,
131
+ )
121
132
  raise # TODO this should be configurable
122
133
  logger.info("Finishing sync stage '%s'", stage_name)
123
134
  logger.info("Finishing sync with update tag '%d'", config.update_tag)
124
135
  return STATUS_SUCCESS
125
136
 
137
+ @classmethod
138
+ def list_intel_modules(cls) -> OrderedDict:
139
+ """
140
+ List all available intel modules.
141
+
142
+ This method will load all modules in the cartography.intel package and return a dictionary of their names and
143
+ their callable functions. The keys of the dictionary are the module names, and the values are the callable
144
+ functions (with `start_{module}_ingestion` pattern) that should be executed during the sync process.
145
+ analysis and create_indexes are loaded separately to ensure they are always available and run first
146
+ (for create-index) and last (for analysis).
147
+
148
+ :rtype: OrderedDict
149
+ :return: A dictionary of available intel modules.
150
+ """
151
+ available_modules = OrderedDict({})
152
+ available_modules["create-indexes"] = cartography.intel.create_indexes.run
153
+ callable_regex = re.compile(r"^start_(.+)_ingestion$")
154
+ # Load built-in modules
155
+ for intel_module_info in iter_modules(cartography.intel.__path__):
156
+ if intel_module_info.name in ("analysis", "create_indexes"):
157
+ continue
158
+ try:
159
+ logger.debug("Loading module: %s", intel_module_info.name)
160
+ intel_module = __import__(
161
+ f"cartography.intel.{intel_module_info.name}",
162
+ fromlist=[""],
163
+ )
164
+ except ImportError as e:
165
+ logger.error(
166
+ "Failed to import module '%s'. Error: %s",
167
+ intel_module_info.name,
168
+ e,
169
+ )
170
+ continue
171
+ logger.debug("Loading module: %s", intel_module_info.name)
172
+ intel_module = __import__(
173
+ f"cartography.intel.{intel_module_info.name}",
174
+ fromlist=[""],
175
+ )
176
+ for k, v in intel_module.__dict__.items():
177
+ if not callable(v):
178
+ continue
179
+ match_callable_name = callable_regex.match(k)
180
+ if not match_callable_name:
181
+ continue
182
+ callable_module_name = (
183
+ match_callable_name.group(1) if match_callable_name else None
184
+ )
185
+ if callable_module_name != intel_module_info.name:
186
+ logger.debug(
187
+ "Module name '%s' does not match intel module name '%s'.",
188
+ callable_module_name,
189
+ intel_module_info.name,
190
+ )
191
+ available_modules[intel_module_info.name] = v
192
+ available_modules["analysis"] = cartography.intel.analysis.run
193
+ return available_modules
194
+
195
+
196
+ # Used to avoid repeatedly calling Sync.list_intel_modules()
197
+ TOP_LEVEL_MODULES = Sync.list_intel_modules()
198
+
126
199
 
127
200
  def run_with_config(sync: Sync, config: Union[Config, argparse.Namespace]) -> int:
128
201
  """
@@ -201,9 +274,12 @@ def build_default_sync() -> Sync:
201
274
  :return: The default cartography sync object.
202
275
  """
203
276
  sync = Sync()
204
- sync.add_stages([
205
- (stage_name, stage_func) for stage_name, stage_func in TOP_LEVEL_MODULES.items()
206
- ])
277
+ sync.add_stages(
278
+ [
279
+ (stage_name, stage_func)
280
+ for stage_name, stage_func in TOP_LEVEL_MODULES.items()
281
+ ],
282
+ )
207
283
  return sync
208
284
 
209
285
 
@@ -214,18 +290,18 @@ def parse_and_validate_selected_modules(selected_modules: str) -> List[str]:
214
290
  :return: A validated list of module names that we will run
215
291
  """
216
292
  validated_modules: List[str] = []
217
- for module in selected_modules.split(','):
293
+ for module in selected_modules.split(","):
218
294
  module = module.strip()
219
295
 
220
296
  if module in TOP_LEVEL_MODULES.keys():
221
297
  validated_modules.append(module)
222
298
  else:
223
- valid_modules = ', '.join(TOP_LEVEL_MODULES.keys())
299
+ valid_modules = ", ".join(TOP_LEVEL_MODULES.keys())
224
300
  raise ValueError(
225
301
  f'Error parsing `selected_modules`. You specified "{selected_modules}". '
226
- f'Please check that your string is formatted properly. '
302
+ f"Please check that your string is formatted properly. "
227
303
  f'Example valid input looks like "aws,gcp,analysis" or "azure, oci, crowdstrike". '
228
- f'Our full list of valid values is: {valid_modules}.',
304
+ f"Our full list of valid values is: {valid_modules}.",
229
305
  )
230
306
  return validated_modules
231
307
 
cartography/util.py CHANGED
@@ -30,7 +30,6 @@ from cartography.graph.statement import get_job_shortname
30
30
  from cartography.stats import get_stats_client
31
31
  from cartography.stats import ScopedStatsClient
32
32
 
33
-
34
33
  logger = logging.getLogger(__name__)
35
34
 
36
35
 
@@ -44,7 +43,7 @@ def run_analysis_job(
44
43
  filename: str,
45
44
  neo4j_session: neo4j.Session,
46
45
  common_job_parameters: Dict,
47
- package: str = 'cartography.data.jobs.analysis',
46
+ package: str = "cartography.data.jobs.analysis",
48
47
  ) -> None:
49
48
  """
50
49
  Enriches existing graph data with analysis jobs. This is designed for use with the sync stage
@@ -66,11 +65,11 @@ def run_analysis_job(
66
65
 
67
66
 
68
67
  def run_analysis_and_ensure_deps(
69
- analysis_job_name: str,
70
- resource_dependencies: Set[str],
71
- requested_syncs: Set[str],
72
- common_job_parameters: Dict[str, Any],
73
- neo4j_session: neo4j.Session,
68
+ analysis_job_name: str,
69
+ resource_dependencies: Set[str],
70
+ requested_syncs: Set[str],
71
+ common_job_parameters: Dict[str, Any],
72
+ neo4j_session: neo4j.Session,
74
73
  ) -> None:
75
74
  """
76
75
  Runs analysis job only if the given set of resource dependencies was included in the requested_syncs.
@@ -100,7 +99,7 @@ def run_scoped_analysis_job(
100
99
  filename: str,
101
100
  neo4j_session: neo4j.Session,
102
101
  common_job_parameters: Dict,
103
- package: str = 'cartography.data.jobs.scoped_analysis',
102
+ package: str = "cartography.data.jobs.scoped_analysis",
104
103
  ) -> None:
105
104
  """
106
105
  Enriches existing graph data scoped to a given sub resource - e.g. the current AWS account.
@@ -116,8 +115,10 @@ def run_scoped_analysis_job(
116
115
 
117
116
 
118
117
  def run_cleanup_job(
119
- filename: str, neo4j_session: neo4j.Session, common_job_parameters: Dict,
120
- package: str = 'cartography.data.jobs.cleanup',
118
+ filename: str,
119
+ neo4j_session: neo4j.Session,
120
+ common_job_parameters: Dict,
121
+ package: str = "cartography.data.jobs.cleanup",
121
122
  ) -> None:
122
123
  GraphJob.run_from_json(
123
124
  neo4j_session,
@@ -138,7 +139,7 @@ def merge_module_sync_metadata(
138
139
  update_tag: int,
139
140
  stat_handler: ScopedStatsClient,
140
141
  ) -> None:
141
- '''
142
+ """
142
143
  This creates `ModuleSyncMetadata` nodes when called from each of the individual modules or sub-modules.
143
144
  The 'types' used here should be actual node labels. For example, if we did sync a particular AWSAccount's S3Buckets,
144
145
  the `grouptype` is 'AWSAccount', the `groupid` is the particular account's `id`, and the `syncedtype` is 'S3Bucket'.
@@ -148,8 +149,9 @@ def merge_module_sync_metadata(
148
149
  :param group_id: The parent module's id
149
150
  :param synced_type: The sub-module's type
150
151
  :param update_tag: Timestamp used to determine data freshness
151
- '''
152
- template = Template("""
152
+ """
153
+ template = Template(
154
+ """
153
155
  MERGE (n:ModuleSyncMetadata{id:'${group_type}_${group_id}_${synced_type}'})
154
156
  ON CREATE SET
155
157
  n:SyncMetadata, n.firstseen=timestamp()
@@ -157,20 +159,25 @@ def merge_module_sync_metadata(
157
159
  n.grouptype='${group_type}',
158
160
  n.groupid='${group_id}',
159
161
  n.lastupdated=$UPDATE_TAG
160
- """)
162
+ """,
163
+ )
161
164
  neo4j_session.run(
162
- template.safe_substitute(group_type=group_type, group_id=group_id, synced_type=synced_type),
165
+ template.safe_substitute(
166
+ group_type=group_type,
167
+ group_id=group_id,
168
+ synced_type=synced_type,
169
+ ),
163
170
  UPDATE_TAG=update_tag,
164
171
  )
165
- stat_handler.incr(f'{group_type}_{group_id}_{synced_type}_lastupdated', update_tag)
172
+ stat_handler.incr(f"{group_type}_{group_id}_{synced_type}_lastupdated", update_tag)
166
173
 
167
174
 
168
175
  def load_resource_binary(package: str, resource_name: str) -> BinaryIO:
169
176
  return open_binary(package, resource_name)
170
177
 
171
178
 
172
- R = TypeVar('R')
173
- F = TypeVar('F', bound=Callable[..., Any])
179
+ R = TypeVar("R")
180
+ F = TypeVar("F", bound=Callable[..., Any])
174
181
 
175
182
 
176
183
  def timeit(method: F) -> F:
@@ -179,6 +186,7 @@ def timeit(method: F) -> F:
179
186
  This is only active if config.statsd_enabled is True.
180
187
  :param method: The function to measure execution
181
188
  """
189
+
182
190
  # Allow access via `inspect` to the wrapped function. This is used in integration tests to standardize param names.
183
191
  @wraps(method)
184
192
  def timed(*args, **kwargs): # type: ignore
@@ -202,40 +210,44 @@ def aws_paginate(
202
210
  object_name: str,
203
211
  **kwargs: Any,
204
212
  ) -> List[Dict]:
205
- '''
213
+ """
206
214
  Helper method for boilerplate boto3 pagination
207
215
  The **kwargs will be forwarded to the paginator
208
- '''
216
+ """
209
217
  paginator = client.get_paginator(method_name)
210
218
  items = []
211
219
  i = 0
212
220
  for i, page in enumerate(paginator.paginate(**kwargs), start=1):
213
221
  if i % 100 == 0:
214
- logger.info(f'fetching page number {i}')
222
+ logger.info(f"fetching page number {i}")
215
223
  if object_name in page:
216
224
  items.extend(page[object_name])
217
225
  else:
218
226
  logger.warning(
219
- f'''aws_paginate: Key "{object_name}" is not present, check if this is a typo.
220
- If not, then the AWS datatype somehow does not have this key.''',
227
+ f"""aws_paginate: Key "{object_name}" is not present, check if this is a typo.
228
+ If not, then the AWS datatype somehow does not have this key.""",
221
229
  )
222
230
  return items
223
231
 
224
232
 
225
- AWSGetFunc = TypeVar('AWSGetFunc', bound=Callable[..., Iterable])
233
+ AWSGetFunc = TypeVar("AWSGetFunc", bound=Callable[..., Iterable])
226
234
 
227
235
  # fix for AWS TooManyRequestsException
228
- # https://github.com/lyft/cartography/issues/297
229
- # https://github.com/lyft/cartography/issues/243
230
- # https://github.com/lyft/cartography/issues/65
231
- # https://github.com/lyft/cartography/issues/25
236
+ # https://github.com/cartography-cncf/cartography/issues/297
237
+ # https://github.com/cartography-cncf/cartography/issues/243
238
+ # https://github.com/cartography-cncf/cartography/issues/65
239
+ # https://github.com/cartography-cncf/cartography/issues/25
232
240
 
233
241
 
234
242
  def backoff_handler(details: Dict) -> None:
235
243
  """
236
244
  Handler that will be executed on exception by backoff mechanism
237
245
  """
238
- logger.warning("Backing off {wait:0.1f} seconds after {tries} tries. Calling function {target}".format(**details))
246
+ logger.warning(
247
+ "Backing off {wait:0.1f} seconds after {tries} tries. Calling function {target}".format(
248
+ **details,
249
+ ),
250
+ )
239
251
 
240
252
 
241
253
  # TODO Move this to cartography.intel.aws.util.common
@@ -250,21 +262,21 @@ def aws_handle_regions(func: AWSGetFunc) -> AWSGetFunc:
250
262
  This should be used on `get_` functions that normally return a list of items.
251
263
  """
252
264
  ERROR_CODES = [
253
- 'AccessDenied',
254
- 'AccessDeniedException',
255
- 'AuthFailure',
256
- 'InvalidClientTokenId',
257
- 'UnauthorizedOperation',
258
- 'UnrecognizedClientException',
259
- 'InternalServerErrorException',
265
+ "AccessDenied",
266
+ "AccessDeniedException",
267
+ "AuthFailure",
268
+ "InvalidClientTokenId",
269
+ "UnauthorizedOperation",
270
+ "UnrecognizedClientException",
271
+ "InternalServerErrorException",
260
272
  ]
261
273
 
262
274
  @wraps(func)
263
275
  # fix for AWS TooManyRequestsException
264
- # https://github.com/lyft/cartography/issues/297
265
- # https://github.com/lyft/cartography/issues/243
266
- # https://github.com/lyft/cartography/issues/65
267
- # https://github.com/lyft/cartography/issues/25
276
+ # https://github.com/cartography-cncf/cartography/issues/297
277
+ # https://github.com/cartography-cncf/cartography/issues/243
278
+ # https://github.com/cartography-cncf/cartography/issues/65
279
+ # https://github.com/cartography-cncf/cartography/issues/25
268
280
  @backoff.on_exception(
269
281
  backoff.expo,
270
282
  botocore.exceptions.ClientError,
@@ -277,23 +289,29 @@ def aws_handle_regions(func: AWSGetFunc) -> AWSGetFunc:
277
289
  except botocore.exceptions.ClientError as e:
278
290
  # The account is not authorized to use this service in this region
279
291
  # so we can continue without raising an exception
280
- if e.response['Error']['Code'] in ERROR_CODES:
281
- logger.warning("{} in this region. Skipping...".format(e.response['Error']['Message']))
292
+ if e.response["Error"]["Code"] in ERROR_CODES:
293
+ logger.warning(
294
+ "{} in this region. Skipping...".format(
295
+ e.response["Error"]["Message"],
296
+ ),
297
+ )
282
298
  return []
283
299
  else:
284
300
  raise
301
+
285
302
  return cast(AWSGetFunc, inner_function)
286
303
 
287
304
 
288
305
  def retries_with_backoff(
289
- func: Callable,
290
- exception_type: Type[Exception],
291
- max_tries: int,
292
- on_backoff: Callable,
306
+ func: Callable,
307
+ exception_type: Type[Exception],
308
+ max_tries: int,
309
+ on_backoff: Callable,
293
310
  ) -> Callable:
294
311
  """
295
312
  Adds retry with backoff to the given function. (Could expand the possible input parameters as needed.)
296
313
  """
314
+
297
315
  @wraps(func)
298
316
  @backoff.on_exception(
299
317
  backoff.expo,
@@ -303,6 +321,7 @@ def retries_with_backoff(
303
321
  )
304
322
  def inner_function(*args, **kwargs): # type: ignore
305
323
  return func(*args, **kwargs)
324
+
306
325
  return cast(Callable, inner_function)
307
326
 
308
327
 
@@ -331,32 +350,29 @@ def dict_date_to_epoch(obj: Dict, key: str) -> Optional[int]:
331
350
 
332
351
 
333
352
  def camel_to_snake(name: str) -> str:
334
- return re.sub(r'(?<!^)(?=[A-Z])', '_', name).lower()
353
+ return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
335
354
 
336
355
 
337
356
  def batch(items: Iterable, size: int = DEFAULT_BATCH_SIZE) -> List[List]:
338
- '''
357
+ """
339
358
  Takes an Iterable of items and returns a list of lists of the same items,
340
359
  batched into chunks of the provided `size`.
341
360
 
342
361
  Use:
343
362
  x = [1,2,3,4,5,6,7,8]
344
363
  batch(x, size=3) -> [[1, 2, 3], [4, 5, 6], [7, 8]]
345
- '''
364
+ """
346
365
  items = list(items)
347
- return [
348
- items[i: i + size]
349
- for i in range(0, len(items), size)
350
- ]
366
+ return [items[i : i + size] for i in range(0, len(items), size)]
351
367
 
352
368
 
353
369
  def is_throttling_exception(exc: Exception) -> bool:
354
- '''
370
+ """
355
371
  Returns True if the exception is caused by a client libraries throttling mechanism
356
- '''
372
+ """
357
373
  # https://boto3.amazonaws.com/v1/documentation/api/1.19.9/guide/error-handling.html
358
374
  if isinstance(exc, botocore.exceptions.ClientError):
359
- if exc.response['Error']['Code'] in ['LimitExceededException', 'Throttling']:
375
+ if exc.response["Error"]["Code"] in ["LimitExceededException", "Throttling"]:
360
376
  return True
361
377
  # add other exceptions here, if needed, like:
362
378
  # https://cloud.google.com/python/docs/reference/storage/1.39.0/retry_timeout#configuring-retries
@@ -366,7 +382,7 @@ def is_throttling_exception(exc: Exception) -> bool:
366
382
 
367
383
 
368
384
  def to_asynchronous(func: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
369
- '''
385
+ """
370
386
  Returns a Future that will run a function and its arguments in the default threadpool.
371
387
  Helper until we start using python 3.9's asyncio.to_thread
372
388
 
@@ -396,8 +412,12 @@ def to_asynchronous(func: Callable[..., R], *args: Any, **kwargs: Any) -> Awaita
396
412
  NOTE: to use this in a Jupyter notebook, you need to do:
397
413
  # import nest_asyncio
398
414
  # nest_asyncio.apply()
399
- '''
400
- CartographyThrottlingException = type('CartographyThrottlingException', (Exception,), {})
415
+ """
416
+ CartographyThrottlingException = type(
417
+ "CartographyThrottlingException",
418
+ (Exception,),
419
+ {},
420
+ )
401
421
 
402
422
  @wraps(func)
403
423
  def wrapper(*args: Any, **kwargs: Any) -> R:
@@ -409,13 +429,15 @@ def to_asynchronous(func: Callable[..., R], *args: Any, **kwargs: Any) -> Awaita
409
429
  raise
410
430
 
411
431
  # don't use @backoff as decorator, to preserve typing
412
- wrapped = backoff.on_exception(backoff.expo, CartographyThrottlingException)(wrapper)
432
+ wrapped = backoff.on_exception(backoff.expo, CartographyThrottlingException)(
433
+ wrapper,
434
+ )
413
435
  call = partial(wrapped, *args, **kwargs)
414
436
  return asyncio.get_event_loop().run_in_executor(None, call)
415
437
 
416
438
 
417
439
  def to_synchronous(*awaitables: Awaitable[Any]) -> List[Any]:
418
- '''
440
+ """
419
441
  Synchronously waits for the Awaitable(s) to complete and returns their result(s).
420
442
  See https://docs.python.org/3.8/library/asyncio-task.html#asyncio-awaitables
421
443
 
@@ -437,5 +459,5 @@ def to_synchronous(*awaitables: Awaitable[Any]) -> List[Any]:
437
459
  future_2 = another_async_func(2)
438
460
 
439
461
  results = to_synchronous(future_1, future_2)
440
- '''
462
+ """
441
463
  return asyncio.get_event_loop().run_until_complete(asyncio.gather(*awaitables))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cartography
3
- Version: 0.102.0rc1
3
+ Version: 0.103.0rc1
4
4
  Summary: Explore assets and their relationships across your technical infrastructure.
5
5
  Maintainer: Cartography Contributors
6
6
  License: apache2
@@ -54,17 +54,6 @@ Requires-Dist: crowdstrike-falconpy>=0.5.1
54
54
  Requires-Dist: python-dateutil
55
55
  Requires-Dist: xmltodict
56
56
  Requires-Dist: duo-client
57
- Provides-Extra: dev
58
- Requires-Dist: backoff>=2.1.2; extra == "dev"
59
- Requires-Dist: moto; extra == "dev"
60
- Requires-Dist: pre-commit; extra == "dev"
61
- Requires-Dist: pytest>=6.2.4; extra == "dev"
62
- Requires-Dist: pytest-mock; extra == "dev"
63
- Requires-Dist: pytest-cov==6.1.1; extra == "dev"
64
- Requires-Dist: pytest-rerunfailures; extra == "dev"
65
- Requires-Dist: pytest-asyncio; extra == "dev"
66
- Requires-Dist: types-PyYAML; extra == "dev"
67
- Requires-Dist: types-requests<2.32.0.20250329; extra == "dev"
68
57
  Dynamic: license-file
69
58
 
70
59
  ![Cartography](docs/root/images/logo-horizontal.png)
@@ -91,7 +80,7 @@ You can learn more about the story behind Cartography in our [presentation at BS
91
80
 
92
81
  ## Supported platforms
93
82
 
94
- - [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - API Gateway, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager, Security Hub, SQS, SSM, STS, Tags
83
+ - [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - API Gateway, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, S3AccountPublicAccessBlock, Secrets Manager, Security Hub, SNS, SQS, SSM, STS, Tags
95
84
  - [Google Cloud Platform](https://cartography-cncf.github.io/cartography/modules/gcp/index.html) - Cloud Resource Manager, Compute, DNS, Storage, Google Kubernetes Engine
96
85
  - [Google GSuite](https://cartography-cncf.github.io/cartography/modules/gsuite/index.html) - users, groups
97
86
  - [Oracle Cloud Infrastructure](docs/setup/config/oci.md) - IAM
@@ -172,7 +161,7 @@ Thank you for considering contributing to Cartography!
172
161
  All contributors and participants of this project must follow the [CNCF code of conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md).
173
162
 
174
163
  ### Bug reports and feature requests and discussions
175
- Submit a GitHub issue to report a bug or request a new feature. If we decide that the issue needs more discussion - usually because the scope is too large or we need to make careful decision - we will convert the issue to a [GitHub Discussion](https://github.com/lyft/cartography/discussions).
164
+ Submit a GitHub issue to report a bug or request a new feature. If we decide that the issue needs more discussion - usually because the scope is too large or we need to make careful decision - we will convert the issue to a [GitHub Discussion](https://github.com/cartography-cncf/cartography/discussions).
176
165
 
177
166
  ### Developing Cartography
178
167