cartography 0.102.0rc2__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

Files changed (251) hide show
  1. cartography/__main__.py +1 -2
  2. cartography/_version.py +2 -2
  3. cartography/cli.py +302 -253
  4. cartography/client/core/tx.py +39 -18
  5. cartography/config.py +4 -0
  6. cartography/driftdetect/__main__.py +1 -2
  7. cartography/driftdetect/add_shortcut.py +10 -2
  8. cartography/driftdetect/cli.py +71 -75
  9. cartography/driftdetect/detect_deviations.py +7 -3
  10. cartography/driftdetect/get_states.py +20 -8
  11. cartography/driftdetect/model.py +5 -5
  12. cartography/driftdetect/serializers.py +8 -6
  13. cartography/driftdetect/storage.py +2 -2
  14. cartography/graph/cleanupbuilder.py +35 -15
  15. cartography/graph/job.py +46 -17
  16. cartography/graph/querybuilder.py +165 -80
  17. cartography/graph/statement.py +35 -26
  18. cartography/intel/analysis.py +4 -1
  19. cartography/intel/aws/__init__.py +114 -55
  20. cartography/intel/aws/apigateway.py +134 -63
  21. cartography/intel/aws/cloudtrail.py +127 -0
  22. cartography/intel/aws/config.py +56 -20
  23. cartography/intel/aws/dynamodb.py +108 -40
  24. cartography/intel/aws/ec2/__init__.py +2 -2
  25. cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
  26. cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
  27. cartography/intel/aws/ec2/images.py +49 -20
  28. cartography/intel/aws/ec2/instances.py +234 -136
  29. cartography/intel/aws/ec2/internet_gateways.py +40 -11
  30. cartography/intel/aws/ec2/key_pairs.py +44 -20
  31. cartography/intel/aws/ec2/launch_templates.py +101 -59
  32. cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
  33. cartography/intel/aws/ec2/load_balancers.py +82 -42
  34. cartography/intel/aws/ec2/network_acls.py +89 -65
  35. cartography/intel/aws/ec2/network_interfaces.py +146 -87
  36. cartography/intel/aws/ec2/reserved_instances.py +45 -16
  37. cartography/intel/aws/ec2/route_tables.py +138 -98
  38. cartography/intel/aws/ec2/security_groups.py +71 -21
  39. cartography/intel/aws/ec2/snapshots.py +61 -22
  40. cartography/intel/aws/ec2/subnets.py +54 -18
  41. cartography/intel/aws/ec2/tgw.py +100 -34
  42. cartography/intel/aws/ec2/util.py +1 -1
  43. cartography/intel/aws/ec2/volumes.py +69 -41
  44. cartography/intel/aws/ec2/vpc.py +37 -12
  45. cartography/intel/aws/ec2/vpc_peerings.py +83 -24
  46. cartography/intel/aws/ecr.py +88 -32
  47. cartography/intel/aws/ecs.py +83 -47
  48. cartography/intel/aws/eks.py +55 -29
  49. cartography/intel/aws/elasticache.py +42 -18
  50. cartography/intel/aws/elasticsearch.py +57 -20
  51. cartography/intel/aws/emr.py +61 -23
  52. cartography/intel/aws/iam.py +401 -145
  53. cartography/intel/aws/iam_instance_profiles.py +22 -22
  54. cartography/intel/aws/identitycenter.py +71 -37
  55. cartography/intel/aws/inspector.py +159 -89
  56. cartography/intel/aws/kms.py +92 -38
  57. cartography/intel/aws/lambda_function.py +103 -34
  58. cartography/intel/aws/organizations.py +30 -10
  59. cartography/intel/aws/permission_relationships.py +133 -51
  60. cartography/intel/aws/rds.py +249 -85
  61. cartography/intel/aws/redshift.py +107 -46
  62. cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
  63. cartography/intel/aws/resources.py +53 -46
  64. cartography/intel/aws/route53.py +108 -61
  65. cartography/intel/aws/s3.py +168 -83
  66. cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
  67. cartography/intel/aws/secretsmanager.py +24 -12
  68. cartography/intel/aws/securityhub.py +20 -9
  69. cartography/intel/aws/sns.py +166 -0
  70. cartography/intel/aws/sqs.py +60 -28
  71. cartography/intel/aws/ssm.py +70 -30
  72. cartography/intel/aws/util/arns.py +7 -7
  73. cartography/intel/aws/util/common.py +31 -4
  74. cartography/intel/azure/__init__.py +78 -19
  75. cartography/intel/azure/compute.py +101 -27
  76. cartography/intel/azure/cosmosdb.py +496 -170
  77. cartography/intel/azure/sql.py +296 -105
  78. cartography/intel/azure/storage.py +322 -113
  79. cartography/intel/azure/subscription.py +39 -23
  80. cartography/intel/azure/tenant.py +13 -4
  81. cartography/intel/azure/util/credentials.py +95 -55
  82. cartography/intel/bigfix/__init__.py +2 -2
  83. cartography/intel/bigfix/computers.py +93 -65
  84. cartography/intel/create_indexes.py +3 -2
  85. cartography/intel/crowdstrike/__init__.py +11 -9
  86. cartography/intel/crowdstrike/endpoints.py +5 -1
  87. cartography/intel/crowdstrike/spotlight.py +8 -3
  88. cartography/intel/cve/__init__.py +46 -13
  89. cartography/intel/cve/feed.py +48 -12
  90. cartography/intel/digitalocean/__init__.py +22 -13
  91. cartography/intel/digitalocean/compute.py +75 -108
  92. cartography/intel/digitalocean/management.py +44 -80
  93. cartography/intel/digitalocean/platform.py +48 -43
  94. cartography/intel/dns.py +36 -10
  95. cartography/intel/duo/__init__.py +21 -16
  96. cartography/intel/duo/api_host.py +14 -9
  97. cartography/intel/duo/endpoints.py +50 -45
  98. cartography/intel/duo/groups.py +18 -14
  99. cartography/intel/duo/phones.py +37 -34
  100. cartography/intel/duo/tokens.py +26 -23
  101. cartography/intel/duo/users.py +54 -50
  102. cartography/intel/duo/web_authn_credentials.py +30 -25
  103. cartography/intel/entra/__init__.py +25 -7
  104. cartography/intel/entra/ou.py +112 -0
  105. cartography/intel/entra/users.py +69 -63
  106. cartography/intel/gcp/__init__.py +185 -49
  107. cartography/intel/gcp/compute.py +418 -231
  108. cartography/intel/gcp/crm.py +96 -43
  109. cartography/intel/gcp/dns.py +60 -19
  110. cartography/intel/gcp/gke.py +72 -38
  111. cartography/intel/gcp/iam.py +61 -41
  112. cartography/intel/gcp/storage.py +84 -55
  113. cartography/intel/github/__init__.py +13 -11
  114. cartography/intel/github/repos.py +270 -137
  115. cartography/intel/github/teams.py +170 -88
  116. cartography/intel/github/users.py +70 -39
  117. cartography/intel/github/util.py +36 -34
  118. cartography/intel/gsuite/__init__.py +47 -26
  119. cartography/intel/gsuite/api.py +73 -30
  120. cartography/intel/jamf/__init__.py +19 -1
  121. cartography/intel/jamf/computers.py +30 -7
  122. cartography/intel/jamf/util.py +7 -2
  123. cartography/intel/kandji/__init__.py +6 -3
  124. cartography/intel/kandji/devices.py +14 -8
  125. cartography/intel/kubernetes/namespaces.py +7 -4
  126. cartography/intel/kubernetes/pods.py +7 -4
  127. cartography/intel/kubernetes/services.py +8 -4
  128. cartography/intel/lastpass/__init__.py +2 -2
  129. cartography/intel/lastpass/users.py +23 -12
  130. cartography/intel/oci/__init__.py +44 -11
  131. cartography/intel/oci/iam.py +134 -38
  132. cartography/intel/oci/organizations.py +13 -6
  133. cartography/intel/oci/utils.py +43 -20
  134. cartography/intel/okta/__init__.py +66 -15
  135. cartography/intel/okta/applications.py +42 -20
  136. cartography/intel/okta/awssaml.py +93 -33
  137. cartography/intel/okta/factors.py +16 -4
  138. cartography/intel/okta/groups.py +56 -29
  139. cartography/intel/okta/organization.py +5 -1
  140. cartography/intel/okta/origins.py +6 -2
  141. cartography/intel/okta/roles.py +15 -5
  142. cartography/intel/okta/users.py +20 -8
  143. cartography/intel/okta/utils.py +6 -4
  144. cartography/intel/pagerduty/__init__.py +8 -7
  145. cartography/intel/pagerduty/escalation_policies.py +18 -6
  146. cartography/intel/pagerduty/schedules.py +12 -4
  147. cartography/intel/pagerduty/services.py +11 -4
  148. cartography/intel/pagerduty/teams.py +8 -3
  149. cartography/intel/pagerduty/users.py +3 -1
  150. cartography/intel/pagerduty/vendors.py +3 -1
  151. cartography/intel/semgrep/__init__.py +24 -6
  152. cartography/intel/semgrep/dependencies.py +50 -28
  153. cartography/intel/semgrep/deployment.py +3 -1
  154. cartography/intel/semgrep/findings.py +42 -18
  155. cartography/intel/snipeit/__init__.py +17 -3
  156. cartography/intel/snipeit/asset.py +12 -6
  157. cartography/intel/snipeit/user.py +8 -5
  158. cartography/intel/snipeit/util.py +9 -4
  159. cartography/models/aws/apigateway.py +21 -17
  160. cartography/models/aws/apigatewaycertificate.py +28 -22
  161. cartography/models/aws/apigatewayresource.py +28 -20
  162. cartography/models/aws/apigatewaystage.py +33 -25
  163. cartography/models/aws/cloudtrail/__init__.py +0 -0
  164. cartography/models/aws/cloudtrail/trail.py +61 -0
  165. cartography/models/aws/dynamodb/gsi.py +30 -22
  166. cartography/models/aws/dynamodb/tables.py +25 -17
  167. cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
  168. cartography/models/aws/ec2/images.py +36 -34
  169. cartography/models/aws/ec2/instances.py +51 -45
  170. cartography/models/aws/ec2/keypair.py +21 -16
  171. cartography/models/aws/ec2/keypair_instance.py +28 -21
  172. cartography/models/aws/ec2/launch_configurations.py +30 -26
  173. cartography/models/aws/ec2/launch_template_versions.py +48 -38
  174. cartography/models/aws/ec2/launch_templates.py +21 -17
  175. cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
  176. cartography/models/aws/ec2/load_balancers.py +47 -37
  177. cartography/models/aws/ec2/network_acl_rules.py +38 -30
  178. cartography/models/aws/ec2/network_acls.py +38 -29
  179. cartography/models/aws/ec2/networkinterface_instance.py +52 -39
  180. cartography/models/aws/ec2/networkinterfaces.py +53 -37
  181. cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
  182. cartography/models/aws/ec2/reservations.py +18 -14
  183. cartography/models/aws/ec2/route_table_associations.py +44 -34
  184. cartography/models/aws/ec2/route_tables.py +50 -43
  185. cartography/models/aws/ec2/routes.py +45 -37
  186. cartography/models/aws/ec2/securitygroup_instance.py +29 -20
  187. cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
  188. cartography/models/aws/ec2/subnet_instance.py +24 -19
  189. cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
  190. cartography/models/aws/ec2/volumes.py +47 -40
  191. cartography/models/aws/eks/clusters.py +23 -21
  192. cartography/models/aws/emr.py +32 -30
  193. cartography/models/aws/iam/instanceprofile.py +33 -24
  194. cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
  195. cartography/models/aws/identitycenter/awspermissionset.py +37 -29
  196. cartography/models/aws/identitycenter/awsssouser.py +23 -21
  197. cartography/models/aws/inspector/findings.py +77 -65
  198. cartography/models/aws/inspector/packages.py +35 -29
  199. cartography/models/aws/s3/__init__.py +0 -0
  200. cartography/models/aws/s3/account_public_access_block.py +51 -0
  201. cartography/models/aws/sns/__init__.py +0 -0
  202. cartography/models/aws/sns/topic.py +50 -0
  203. cartography/models/aws/ssm/instance_information.py +51 -39
  204. cartography/models/aws/ssm/instance_patch.py +32 -26
  205. cartography/models/bigfix/bigfix_computer.py +42 -38
  206. cartography/models/bigfix/bigfix_root.py +3 -3
  207. cartography/models/core/common.py +12 -10
  208. cartography/models/core/nodes.py +5 -2
  209. cartography/models/core/relationships.py +14 -6
  210. cartography/models/crowdstrike/hosts.py +37 -35
  211. cartography/models/cve/cve.py +34 -32
  212. cartography/models/cve/cve_feed.py +6 -6
  213. cartography/models/digitalocean/__init__.py +0 -0
  214. cartography/models/digitalocean/account.py +21 -0
  215. cartography/models/digitalocean/droplet.py +56 -0
  216. cartography/models/digitalocean/project.py +48 -0
  217. cartography/models/duo/api_host.py +3 -3
  218. cartography/models/duo/endpoint.py +43 -41
  219. cartography/models/duo/group.py +14 -14
  220. cartography/models/duo/phone.py +27 -27
  221. cartography/models/duo/token.py +16 -16
  222. cartography/models/duo/user.py +46 -44
  223. cartography/models/duo/web_authn_credential.py +27 -19
  224. cartography/models/entra/ou.py +48 -0
  225. cartography/models/entra/tenant.py +24 -18
  226. cartography/models/entra/user.py +64 -48
  227. cartography/models/gcp/iam.py +23 -23
  228. cartography/models/github/orgs.py +5 -4
  229. cartography/models/github/teams.py +37 -31
  230. cartography/models/github/users.py +34 -23
  231. cartography/models/kandji/device.py +22 -16
  232. cartography/models/kandji/tenant.py +6 -4
  233. cartography/models/lastpass/tenant.py +3 -3
  234. cartography/models/lastpass/user.py +32 -28
  235. cartography/models/semgrep/dependencies.py +36 -24
  236. cartography/models/semgrep/deployment.py +5 -5
  237. cartography/models/semgrep/findings.py +58 -42
  238. cartography/models/semgrep/locations.py +27 -21
  239. cartography/models/snipeit/asset.py +30 -21
  240. cartography/models/snipeit/tenant.py +6 -4
  241. cartography/models/snipeit/user.py +19 -12
  242. cartography/stats.py +3 -3
  243. cartography/sync.py +107 -31
  244. cartography/util.py +84 -62
  245. {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/METADATA +3 -14
  246. cartography-0.103.0rc1.dist-info/RECORD +396 -0
  247. {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/WHEEL +1 -1
  248. cartography-0.102.0rc2.dist-info/RECORD +0 -381
  249. {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/entry_points.txt +0 -0
  250. {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/licenses/LICENSE +0 -0
  251. {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/top_level.txt +0 -0
@@ -16,11 +16,11 @@ logger = logging.getLogger(__name__)
16
16
  @timeit
17
17
  @aws_handle_regions
18
18
  def get_secret_list(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
19
- client = boto3_session.client('secretsmanager', region_name=region)
20
- paginator = client.get_paginator('list_secrets')
19
+ client = boto3_session.client("secretsmanager", region_name=region)
20
+ paginator = client.get_paginator("list_secrets")
21
21
  secrets: List[Dict] = []
22
22
  for page in paginator.paginate():
23
- secrets.extend(page['SecretList'])
23
+ secrets.extend(page["SecretList"])
24
24
  return secrets
25
25
 
26
26
 
@@ -52,11 +52,11 @@ def load_secrets(
52
52
  SET r.lastupdated = $aws_update_tag
53
53
  """
54
54
  for secret in data:
55
- secret['LastRotatedDate'] = dict_date_to_epoch(secret, 'LastRotatedDate')
56
- secret['LastChangedDate'] = dict_date_to_epoch(secret, 'LastChangedDate')
57
- secret['LastAccessedDate'] = dict_date_to_epoch(secret, 'LastAccessedDate')
58
- secret['DeletedDate'] = dict_date_to_epoch(secret, 'DeletedDate')
59
- secret['CreatedDate'] = dict_date_to_epoch(secret, 'CreatedDate')
55
+ secret["LastRotatedDate"] = dict_date_to_epoch(secret, "LastRotatedDate")
56
+ secret["LastChangedDate"] = dict_date_to_epoch(secret, "LastChangedDate")
57
+ secret["LastAccessedDate"] = dict_date_to_epoch(secret, "LastAccessedDate")
58
+ secret["DeletedDate"] = dict_date_to_epoch(secret, "DeletedDate")
59
+ secret["CreatedDate"] = dict_date_to_epoch(secret, "CreatedDate")
60
60
 
61
61
  neo4j_session.run(
62
62
  ingest_secrets,
@@ -69,16 +69,28 @@ def load_secrets(
69
69
 
70
70
  @timeit
71
71
  def cleanup_secrets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
72
- run_cleanup_job('aws_import_secrets_cleanup.json', neo4j_session, common_job_parameters)
72
+ run_cleanup_job(
73
+ "aws_import_secrets_cleanup.json",
74
+ neo4j_session,
75
+ common_job_parameters,
76
+ )
73
77
 
74
78
 
75
79
  @timeit
76
80
  def sync(
77
- neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
78
- update_tag: int, common_job_parameters: Dict,
81
+ neo4j_session: neo4j.Session,
82
+ boto3_session: boto3.session.Session,
83
+ regions: List[str],
84
+ current_aws_account_id: str,
85
+ update_tag: int,
86
+ common_job_parameters: Dict,
79
87
  ) -> None:
80
88
  for region in regions:
81
- logger.info("Syncing Secrets Manager for region '%s' in account '%s'.", region, current_aws_account_id)
89
+ logger.info(
90
+ "Syncing Secrets Manager for region '%s' in account '%s'.",
91
+ region,
92
+ current_aws_account_id,
93
+ )
82
94
  secrets = get_secret_list(boto3_session, region)
83
95
  load_secrets(neo4j_session, secrets, region, current_aws_account_id, update_tag)
84
96
  cleanup_secrets(neo4j_session, common_job_parameters)
@@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
14
14
 
15
15
  @timeit
16
16
  def get_hub(boto3_session: boto3.session.Session) -> Dict:
17
- client = boto3_session.client('securityhub')
17
+ client = boto3_session.client("securityhub")
18
18
  try:
19
19
  return client.describe_hub()
20
20
  except client.exceptions.ResourceNotFoundException:
@@ -24,11 +24,11 @@ def get_hub(boto3_session: boto3.session.Session) -> Dict:
24
24
 
25
25
 
26
26
  def transform_hub(hub_data: Dict) -> None:
27
- if 'SubscribedAt' in hub_data and hub_data['SubscribedAt']:
28
- subbed_at = parser.parse(hub_data['SubscribedAt'])
29
- hub_data['SubscribedAt'] = int(subbed_at.timestamp())
27
+ if "SubscribedAt" in hub_data and hub_data["SubscribedAt"]:
28
+ subbed_at = parser.parse(hub_data["SubscribedAt"])
29
+ hub_data["SubscribedAt"] = int(subbed_at.timestamp())
30
30
  else:
31
- hub_data['SubscribedAt'] = None
31
+ hub_data["SubscribedAt"] = None
32
32
 
33
33
 
34
34
  @timeit
@@ -59,14 +59,25 @@ def load_hub(
59
59
 
60
60
 
61
61
  @timeit
62
- def cleanup_securityhub(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
63
- run_cleanup_job('aws_import_securityhub_cleanup.json', neo4j_session, common_job_parameters)
62
+ def cleanup_securityhub(
63
+ neo4j_session: neo4j.Session,
64
+ common_job_parameters: Dict,
65
+ ) -> None:
66
+ run_cleanup_job(
67
+ "aws_import_securityhub_cleanup.json",
68
+ neo4j_session,
69
+ common_job_parameters,
70
+ )
64
71
 
65
72
 
66
73
  @timeit
67
74
  def sync(
68
- neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
69
- update_tag: int, common_job_parameters: Dict,
75
+ neo4j_session: neo4j.Session,
76
+ boto3_session: boto3.session.Session,
77
+ regions: List[str],
78
+ current_aws_account_id: str,
79
+ update_tag: int,
80
+ common_job_parameters: Dict,
70
81
  ) -> None:
71
82
  logger.info("Syncing Security Hub in account '%s'.", current_aws_account_id)
72
83
  hub = get_hub(boto3_session)
@@ -0,0 +1,166 @@
1
+ import logging
2
+ from typing import Dict
3
+ from typing import List
4
+ from typing import Optional
5
+
6
+ import boto3
7
+ import neo4j
8
+
9
+ from cartography.client.core.tx import load
10
+ from cartography.graph.job import GraphJob
11
+ from cartography.models.aws.sns.topic import SNSTopicSchema
12
+ from cartography.stats import get_stats_client
13
+ from cartography.util import aws_handle_regions
14
+ from cartography.util import merge_module_sync_metadata
15
+ from cartography.util import timeit
16
+
17
+ logger = logging.getLogger(__name__)
18
+ stat_handler = get_stats_client(__name__)
19
+
20
+
21
+ @timeit
22
+ @aws_handle_regions
23
+ def get_sns_topics(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
24
+ """
25
+ Get all SNS Topics for a region.
26
+ """
27
+ client = boto3_session.client("sns", region_name=region)
28
+ paginator = client.get_paginator("list_topics")
29
+ topics = []
30
+ for page in paginator.paginate():
31
+ topics.extend(page.get("Topics", []))
32
+
33
+ return topics
34
+
35
+
36
+ @timeit
37
+ def get_topic_attributes(
38
+ boto3_session: boto3.session.Session, topic_arn: str, region: str
39
+ ) -> Optional[Dict]:
40
+ """
41
+ Get attributes for an SNS Topic.
42
+ """
43
+ client = boto3_session.client("sns", region_name=region)
44
+ try:
45
+ return client.get_topic_attributes(TopicArn=topic_arn)
46
+ except Exception as e:
47
+ logger.warning(f"Error getting attributes for SNS topic {topic_arn}: {e}")
48
+ return None
49
+
50
+
51
+ def transform_sns_topics(
52
+ topics: List[Dict], attributes: Dict[str, Dict], region: str
53
+ ) -> List[Dict]:
54
+ """
55
+ Transform SNS topic data for ingestion
56
+ """
57
+ transformed_topics = []
58
+ for topic in topics:
59
+ topic_arn = topic["TopicArn"]
60
+
61
+ # Extract topic name from ARN
62
+ # Format: arn:aws:sns:region:account-id:topic-name
63
+ topic_name = topic_arn.split(":")[-1]
64
+
65
+ # Get attributes
66
+ topic_attrs = attributes.get(topic_arn, {}).get("Attributes", {})
67
+
68
+ transformed_topic = {
69
+ "TopicArn": topic_arn,
70
+ "TopicName": topic_name,
71
+ "DisplayName": topic_attrs.get("DisplayName", ""),
72
+ "Owner": topic_attrs.get("Owner", ""),
73
+ "SubscriptionsPending": int(topic_attrs.get("SubscriptionsPending", "0")),
74
+ "SubscriptionsConfirmed": int(
75
+ topic_attrs.get("SubscriptionsConfirmed", "0")
76
+ ),
77
+ "SubscriptionsDeleted": int(topic_attrs.get("SubscriptionsDeleted", "0")),
78
+ "DeliveryPolicy": topic_attrs.get("DeliveryPolicy", ""),
79
+ "EffectiveDeliveryPolicy": topic_attrs.get("EffectiveDeliveryPolicy", ""),
80
+ "KmsMasterKeyId": topic_attrs.get("KmsMasterKeyId", ""),
81
+ }
82
+
83
+ transformed_topics.append(transformed_topic)
84
+
85
+ return transformed_topics
86
+
87
+
88
+ @timeit
89
+ def load_sns_topics(
90
+ neo4j_session: neo4j.Session,
91
+ data: List[Dict],
92
+ region: str,
93
+ aws_account_id: str,
94
+ update_tag: int,
95
+ ) -> None:
96
+ """
97
+ Load SNS Topics information into the graph
98
+ """
99
+ logger.info(f"Loading {len(data)} SNS topics for region {region} into graph.")
100
+
101
+ load(
102
+ neo4j_session,
103
+ SNSTopicSchema(),
104
+ data,
105
+ lastupdated=update_tag,
106
+ Region=region,
107
+ AWS_ID=aws_account_id,
108
+ )
109
+
110
+
111
+ @timeit
112
+ def cleanup_sns(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
113
+ """
114
+ Run SNS cleanup job
115
+ """
116
+ logger.debug("Running SNS cleanup job.")
117
+ cleanup_job = GraphJob.from_node_schema(SNSTopicSchema(), common_job_parameters)
118
+ cleanup_job.run(neo4j_session)
119
+
120
+
121
+ @timeit
122
+ def sync(
123
+ neo4j_session: neo4j.Session,
124
+ boto3_session: boto3.session.Session,
125
+ regions: List[str],
126
+ current_aws_account_id: str,
127
+ update_tag: int,
128
+ common_job_parameters: Dict,
129
+ ) -> None:
130
+ """
131
+ Sync SNS Topics for all regions
132
+ """
133
+ for region in regions:
134
+ logger.info(
135
+ f"Syncing SNS Topics for {region} in account {current_aws_account_id}"
136
+ )
137
+ topics = get_sns_topics(boto3_session, region)
138
+
139
+ topic_attributes = {}
140
+ for topic in topics:
141
+ topic_arn = topic["TopicArn"]
142
+ attrs = get_topic_attributes(boto3_session, topic_arn, region)
143
+ if attrs:
144
+ topic_attributes[topic_arn] = attrs
145
+
146
+ transformed_topics = transform_sns_topics(topics, topic_attributes, region)
147
+
148
+ load_sns_topics(
149
+ neo4j_session,
150
+ transformed_topics,
151
+ region,
152
+ current_aws_account_id,
153
+ update_tag,
154
+ )
155
+
156
+ cleanup_sns(neo4j_session, common_job_parameters)
157
+
158
+ # Record that we've synced this module
159
+ merge_module_sync_metadata(
160
+ neo4j_session,
161
+ group_type="AWSAccount",
162
+ group_id=current_aws_account_id,
163
+ synced_type="SNSTopic",
164
+ update_tag=update_tag,
165
+ stat_handler=stat_handler,
166
+ )
@@ -19,36 +19,41 @@ logger = logging.getLogger(__name__)
19
19
  @timeit
20
20
  @aws_handle_regions
21
21
  def get_sqs_queue_list(boto3_session: boto3.session.Session, region: str) -> List[str]:
22
- client = boto3_session.client('sqs', region_name=region)
23
- paginator = client.get_paginator('list_queues')
22
+ client = boto3_session.client("sqs", region_name=region)
23
+ paginator = client.get_paginator("list_queues")
24
24
  queues: List[Any] = []
25
25
  for page in paginator.paginate():
26
- queues.extend(page.get('QueueUrls', []))
26
+ queues.extend(page.get("QueueUrls", []))
27
27
  return queues
28
28
 
29
29
 
30
30
  @timeit
31
31
  @aws_handle_regions
32
32
  def get_sqs_queue_attributes(
33
- boto3_session: boto3.session.Session,
34
- queue_urls: List[str],
33
+ boto3_session: boto3.session.Session,
34
+ queue_urls: List[str],
35
35
  ) -> List[Tuple[str, Any]]:
36
36
  """
37
37
  Iterates over all SQS queues. Returns a dict with url as key, and attributes as value.
38
38
  """
39
- client = boto3_session.client('sqs')
39
+ client = boto3_session.client("sqs")
40
40
 
41
41
  queue_attributes = []
42
42
  for queue_url in queue_urls:
43
43
  try:
44
- response = client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=['All'])
44
+ response = client.get_queue_attributes(
45
+ QueueUrl=queue_url,
46
+ AttributeNames=["All"],
47
+ )
45
48
  except ClientError as e:
46
- if e.response['Error']['Code'] == 'AWS.SimpleQueueService.NonExistentQueue':
47
- logger.warning(f"Failed to retrieve SQS queue {queue_url} - Queue does not exist error")
49
+ if e.response["Error"]["Code"] == "AWS.SimpleQueueService.NonExistentQueue":
50
+ logger.warning(
51
+ f"Failed to retrieve SQS queue {queue_url} - Queue does not exist error",
52
+ )
48
53
  continue
49
54
  else:
50
55
  raise
51
- queue_attributes.append((queue_url, response['Attributes']))
56
+ queue_attributes.append((queue_url, response["Attributes"]))
52
57
 
53
58
  return queue_attributes
54
59
 
@@ -91,23 +96,25 @@ def load_sqs_queues(
91
96
  dead_letter_queues: List[Dict] = []
92
97
  queues: List[Dict] = []
93
98
  for url, queue in data:
94
- queue['url'] = url
95
- queue['name'] = queue['QueueArn'].split(':')[-1]
96
- queue['CreatedTimestamp'] = int(queue['CreatedTimestamp'])
97
- queue['LastModifiedTimestamp'] = int(queue['LastModifiedTimestamp'])
98
- redrive_policy = queue.get('RedrivePolicy')
99
+ queue["url"] = url
100
+ queue["name"] = queue["QueueArn"].split(":")[-1]
101
+ queue["CreatedTimestamp"] = int(queue["CreatedTimestamp"])
102
+ queue["LastModifiedTimestamp"] = int(queue["LastModifiedTimestamp"])
103
+ redrive_policy = queue.get("RedrivePolicy")
99
104
  if redrive_policy:
100
105
  try:
101
106
  rp = json.loads(redrive_policy)
102
107
  except TypeError:
103
108
  rp = {}
104
- queue['RedrivePolicy'] = rp
105
- dead_letter_arn = rp.get('deadLetterTargetArn')
109
+ queue["RedrivePolicy"] = rp
110
+ dead_letter_arn = rp.get("deadLetterTargetArn")
106
111
  if dead_letter_arn:
107
- dead_letter_queues.append({
108
- 'arn': queue['QueueArn'],
109
- 'dead_letter_arn': dead_letter_arn,
110
- })
112
+ dead_letter_queues.append(
113
+ {
114
+ "arn": queue["QueueArn"],
115
+ "dead_letter_arn": dead_letter_arn,
116
+ },
117
+ )
111
118
  queues.append(queue)
112
119
 
113
120
  neo4j_session.run(
@@ -122,7 +129,11 @@ def load_sqs_queues(
122
129
 
123
130
 
124
131
  @timeit
125
- def _attach_dead_letter_queues(neo4j_session: neo4j.Session, data: List[Dict[str, str]], aws_update_tag: int) -> None:
132
+ def _attach_dead_letter_queues(
133
+ neo4j_session: neo4j.Session,
134
+ data: List[Dict[str, str]],
135
+ aws_update_tag: int,
136
+ ) -> None:
126
137
  """
127
138
  Attach deadletter queues to their queues.
128
139
  """
@@ -141,20 +152,41 @@ def _attach_dead_letter_queues(neo4j_session: neo4j.Session, data: List[Dict[str
141
152
 
142
153
 
143
154
  @timeit
144
- def cleanup_sqs_queues(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
145
- run_cleanup_job('aws_import_sqs_queues_cleanup.json', neo4j_session, common_job_parameters)
155
+ def cleanup_sqs_queues(
156
+ neo4j_session: neo4j.Session,
157
+ common_job_parameters: Dict,
158
+ ) -> None:
159
+ run_cleanup_job(
160
+ "aws_import_sqs_queues_cleanup.json",
161
+ neo4j_session,
162
+ common_job_parameters,
163
+ )
146
164
 
147
165
 
148
166
  @timeit
149
167
  def sync(
150
- neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str,
151
- update_tag: int, common_job_parameters: Dict,
168
+ neo4j_session: neo4j.Session,
169
+ boto3_session: boto3.session.Session,
170
+ regions: List[str],
171
+ current_aws_account_id: str,
172
+ update_tag: int,
173
+ common_job_parameters: Dict,
152
174
  ) -> None:
153
175
  for region in regions:
154
- logger.info("Syncing SQS for region '%s' in account '%s'.", region, current_aws_account_id)
176
+ logger.info(
177
+ "Syncing SQS for region '%s' in account '%s'.",
178
+ region,
179
+ current_aws_account_id,
180
+ )
155
181
  queue_urls = get_sqs_queue_list(boto3_session, region)
156
182
  if len(queue_urls) == 0:
157
183
  continue
158
184
  queue_attributes = get_sqs_queue_attributes(boto3_session, queue_urls)
159
- load_sqs_queues(neo4j_session, queue_attributes, region, current_aws_account_id, update_tag)
185
+ load_sqs_queues(
186
+ neo4j_session,
187
+ queue_attributes,
188
+ region,
189
+ current_aws_account_id,
190
+ update_tag,
191
+ )
160
192
  cleanup_sqs_queues(neo4j_session, common_job_parameters)
@@ -18,58 +18,74 @@ logger = logging.getLogger(__name__)
18
18
 
19
19
 
20
20
  @timeit
21
- def get_instance_ids(neo4j_session: neo4j.Session, region: str, current_aws_account_id: str) -> List[str]:
21
+ def get_instance_ids(
22
+ neo4j_session: neo4j.Session,
23
+ region: str,
24
+ current_aws_account_id: str,
25
+ ) -> List[str]:
22
26
  get_instances_query = """
23
27
  MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(i:EC2Instance)
24
28
  WHERE i.region = $Region
25
29
  RETURN i.id
26
30
  """
27
- results = neo4j_session.run(get_instances_query, AWS_ACCOUNT_ID=current_aws_account_id, Region=region)
31
+ results = neo4j_session.run(
32
+ get_instances_query,
33
+ AWS_ACCOUNT_ID=current_aws_account_id,
34
+ Region=region,
35
+ )
28
36
  instance_ids = []
29
37
  for r in results:
30
- instance_ids.append(r['i.id'])
38
+ instance_ids.append(r["i.id"])
31
39
  return instance_ids
32
40
 
33
41
 
34
42
  @timeit
35
43
  @aws_handle_regions
36
44
  def get_instance_information(
37
- boto3_session: boto3.session.Session,
38
- region: str,
39
- instance_ids: List[str],
45
+ boto3_session: boto3.session.Session,
46
+ region: str,
47
+ instance_ids: List[str],
40
48
  ) -> List[Dict[str, Any]]:
41
- client = boto3_session.client('ssm', region_name=region)
49
+ client = boto3_session.client("ssm", region_name=region)
42
50
  instance_information: List[Dict[str, Any]] = []
43
- paginator = client.get_paginator('describe_instance_information')
51
+ paginator = client.get_paginator("describe_instance_information")
44
52
  for i in range(0, len(instance_ids), 50):
45
- instance_ids_chunk = instance_ids[i:i + 50]
53
+ instance_ids_chunk = instance_ids[i : i + 50]
46
54
  for info_chunk in paginator.paginate(
47
55
  Filters=[{"Key": "InstanceIds", "Values": instance_ids_chunk}],
48
56
  MaxResults=50,
49
57
  ):
50
- instance_information.extend(info_chunk.get('InstanceInformationList', []))
58
+ instance_information.extend(info_chunk.get("InstanceInformationList", []))
51
59
  return instance_information
52
60
 
53
61
 
54
- def transform_instance_information(data_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
62
+ def transform_instance_information(
63
+ data_list: List[Dict[str, Any]],
64
+ ) -> List[Dict[str, Any]]:
55
65
  for ii in data_list:
56
66
  ii["LastPingDateTime"] = dict_date_to_epoch(ii, "LastPingDateTime")
57
67
  ii["RegistrationDate"] = dict_date_to_epoch(ii, "RegistrationDate")
58
- ii["LastAssociationExecutionDate"] = dict_date_to_epoch(ii, "LastAssociationExecutionDate")
59
- ii["LastSuccessfulAssociationExecutionDate"] = dict_date_to_epoch(ii, "LastSuccessfulAssociationExecutionDate")
68
+ ii["LastAssociationExecutionDate"] = dict_date_to_epoch(
69
+ ii,
70
+ "LastAssociationExecutionDate",
71
+ )
72
+ ii["LastSuccessfulAssociationExecutionDate"] = dict_date_to_epoch(
73
+ ii,
74
+ "LastSuccessfulAssociationExecutionDate",
75
+ )
60
76
  return data_list
61
77
 
62
78
 
63
79
  @timeit
64
80
  @aws_handle_regions
65
81
  def get_instance_patches(
66
- boto3_session: boto3.session.Session,
67
- region: str,
68
- instance_ids: List[str],
82
+ boto3_session: boto3.session.Session,
83
+ region: str,
84
+ instance_ids: List[str],
69
85
  ) -> List[Dict[str, Any]]:
70
- client = boto3_session.client('ssm', region_name=region)
86
+ client = boto3_session.client("ssm", region_name=region)
71
87
  instance_patches: List[Dict[str, Any]] = []
72
- paginator = client.get_paginator('describe_instance_patches')
88
+ paginator = client.get_paginator("describe_instance_patches")
73
89
  for instance_id in instance_ids:
74
90
  patches = []
75
91
  for page in paginator.paginate(InstanceId=instance_id):
@@ -128,29 +144,53 @@ def load_instance_patches(
128
144
 
129
145
 
130
146
  @timeit
131
- def cleanup_ssm(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
147
+ def cleanup_ssm(
148
+ neo4j_session: neo4j.Session,
149
+ common_job_parameters: Dict[str, Any],
150
+ ) -> None:
132
151
  logger.info("Running SSM cleanup")
133
- GraphJob.from_node_schema(SSMInstanceInformationSchema(), common_job_parameters).run(neo4j_session)
134
- GraphJob.from_node_schema(SSMInstancePatchSchema(), common_job_parameters).run(neo4j_session)
152
+ GraphJob.from_node_schema(
153
+ SSMInstanceInformationSchema(),
154
+ common_job_parameters,
155
+ ).run(neo4j_session)
156
+ GraphJob.from_node_schema(SSMInstancePatchSchema(), common_job_parameters).run(
157
+ neo4j_session,
158
+ )
135
159
 
136
160
 
137
161
  @timeit
138
162
  def sync(
139
- neo4j_session: neo4j.Session,
140
- boto3_session: boto3.session.Session,
141
- regions: List[str],
142
- current_aws_account_id: str,
143
- update_tag: int,
144
- common_job_parameters: Dict[str, Any],
163
+ neo4j_session: neo4j.Session,
164
+ boto3_session: boto3.session.Session,
165
+ regions: List[str],
166
+ current_aws_account_id: str,
167
+ update_tag: int,
168
+ common_job_parameters: Dict[str, Any],
145
169
  ) -> None:
146
170
  for region in regions:
147
- logger.info("Syncing SSM for region '%s' in account '%s'.", region, current_aws_account_id)
171
+ logger.info(
172
+ "Syncing SSM for region '%s' in account '%s'.",
173
+ region,
174
+ current_aws_account_id,
175
+ )
148
176
  instance_ids = get_instance_ids(neo4j_session, region, current_aws_account_id)
149
177
  data = get_instance_information(boto3_session, region, instance_ids)
150
178
  data = transform_instance_information(data)
151
- load_instance_information(neo4j_session, data, region, current_aws_account_id, update_tag)
179
+ load_instance_information(
180
+ neo4j_session,
181
+ data,
182
+ region,
183
+ current_aws_account_id,
184
+ update_tag,
185
+ )
152
186
 
153
187
  data = get_instance_patches(boto3_session, region, instance_ids)
154
188
  data = transform_instance_patches(data)
155
- load_instance_patches(neo4j_session, data, region, current_aws_account_id, update_tag)
189
+ load_instance_patches(
190
+ neo4j_session,
191
+ data,
192
+ region,
193
+ current_aws_account_id,
194
+ update_tag,
195
+ )
156
196
  cleanup_ssm(neo4j_session, common_job_parameters)
@@ -2,16 +2,16 @@ from typing import Optional
2
2
 
3
3
 
4
4
  def build_arn(
5
- resource: str,
6
- account: str,
7
- typename: str,
8
- name: str,
9
- region: Optional[str] = None,
10
- partition: Optional[str] = None,
5
+ resource: str,
6
+ account: str,
7
+ typename: str,
8
+ name: str,
9
+ region: Optional[str] = None,
10
+ partition: Optional[str] = None,
11
11
  ) -> str:
12
12
  if not partition:
13
13
  # TODO: support partitions from others. Please file an issue on this if needed, would love to hear from you
14
- partition = 'aws'
14
+ partition = "aws"
15
15
  if not region:
16
16
  # Some resources are present in all regions, e.g. IAM policies
17
17
  region = ""
@@ -1,21 +1,48 @@
1
+ import logging
1
2
  from typing import List
2
3
 
3
4
  from cartography.intel.aws.resources import RESOURCE_FUNCTIONS
4
5
 
6
+ logger = logging.getLogger(__name__)
7
+
5
8
 
6
9
  def parse_and_validate_aws_requested_syncs(aws_requested_syncs: str) -> List[str]:
7
10
  validated_resources: List[str] = []
8
- for resource in aws_requested_syncs.split(','):
11
+ for resource in aws_requested_syncs.split(","):
9
12
  resource = resource.strip()
10
13
 
11
14
  if resource in RESOURCE_FUNCTIONS:
12
15
  validated_resources.append(resource)
13
16
  else:
14
- valid_syncs: str = ', '.join(RESOURCE_FUNCTIONS.keys())
17
+ valid_syncs: str = ", ".join(RESOURCE_FUNCTIONS.keys())
15
18
  raise ValueError(
16
19
  f'Error parsing `aws-requested-syncs`. You specified "{aws_requested_syncs}". '
17
- f'Please check that your string is formatted properly. '
20
+ f"Please check that your string is formatted properly. "
18
21
  f'Example valid input looks like "s3,iam,rds" or "s3, ec2:instance, dynamodb". '
19
- f'Our full list of valid values is: {valid_syncs}.',
22
+ f"Our full list of valid values is: {valid_syncs}.",
20
23
  )
21
24
  return validated_resources
25
+
26
+
27
+ def parse_and_validate_aws_regions(aws_regions: str) -> list[str]:
28
+ """
29
+ Parse and validate a comma-separated string of AWS regions.
30
+ :param aws_regions: Comma-separated string of AWS regions
31
+ :return: A validated list of AWS regions
32
+ """
33
+ validated_regions: List[str] = []
34
+ for region in aws_regions.split(","):
35
+ region = region.strip()
36
+ if region:
37
+ validated_regions.append(region)
38
+ else:
39
+ logger.warning(
40
+ f'Unable to parse string "{region}". Please check the value you passed to `aws-regions`. '
41
+ f'You specified "{aws_regions}". Continuing on with sync.',
42
+ )
43
+
44
+ if not validated_regions:
45
+ raise ValueError(
46
+ f'`aws-regions` was set but no regions were specified. You provided this string: "{aws_regions}"',
47
+ )
48
+ return validated_regions