cartography 0.102.0rc2__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/__main__.py +1 -2
- cartography/_version.py +2 -2
- cartography/cli.py +302 -253
- cartography/client/core/tx.py +39 -18
- cartography/config.py +4 -0
- cartography/driftdetect/__main__.py +1 -2
- cartography/driftdetect/add_shortcut.py +10 -2
- cartography/driftdetect/cli.py +71 -75
- cartography/driftdetect/detect_deviations.py +7 -3
- cartography/driftdetect/get_states.py +20 -8
- cartography/driftdetect/model.py +5 -5
- cartography/driftdetect/serializers.py +8 -6
- cartography/driftdetect/storage.py +2 -2
- cartography/graph/cleanupbuilder.py +35 -15
- cartography/graph/job.py +46 -17
- cartography/graph/querybuilder.py +165 -80
- cartography/graph/statement.py +35 -26
- cartography/intel/analysis.py +4 -1
- cartography/intel/aws/__init__.py +114 -55
- cartography/intel/aws/apigateway.py +134 -63
- cartography/intel/aws/cloudtrail.py +127 -0
- cartography/intel/aws/config.py +56 -20
- cartography/intel/aws/dynamodb.py +108 -40
- cartography/intel/aws/ec2/__init__.py +2 -2
- cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
- cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
- cartography/intel/aws/ec2/images.py +49 -20
- cartography/intel/aws/ec2/instances.py +234 -136
- cartography/intel/aws/ec2/internet_gateways.py +40 -11
- cartography/intel/aws/ec2/key_pairs.py +44 -20
- cartography/intel/aws/ec2/launch_templates.py +101 -59
- cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
- cartography/intel/aws/ec2/load_balancers.py +82 -42
- cartography/intel/aws/ec2/network_acls.py +89 -65
- cartography/intel/aws/ec2/network_interfaces.py +146 -87
- cartography/intel/aws/ec2/reserved_instances.py +45 -16
- cartography/intel/aws/ec2/route_tables.py +138 -98
- cartography/intel/aws/ec2/security_groups.py +71 -21
- cartography/intel/aws/ec2/snapshots.py +61 -22
- cartography/intel/aws/ec2/subnets.py +54 -18
- cartography/intel/aws/ec2/tgw.py +100 -34
- cartography/intel/aws/ec2/util.py +1 -1
- cartography/intel/aws/ec2/volumes.py +69 -41
- cartography/intel/aws/ec2/vpc.py +37 -12
- cartography/intel/aws/ec2/vpc_peerings.py +83 -24
- cartography/intel/aws/ecr.py +88 -32
- cartography/intel/aws/ecs.py +83 -47
- cartography/intel/aws/eks.py +55 -29
- cartography/intel/aws/elasticache.py +42 -18
- cartography/intel/aws/elasticsearch.py +57 -20
- cartography/intel/aws/emr.py +61 -23
- cartography/intel/aws/iam.py +401 -145
- cartography/intel/aws/iam_instance_profiles.py +22 -22
- cartography/intel/aws/identitycenter.py +71 -37
- cartography/intel/aws/inspector.py +159 -89
- cartography/intel/aws/kms.py +92 -38
- cartography/intel/aws/lambda_function.py +103 -34
- cartography/intel/aws/organizations.py +30 -10
- cartography/intel/aws/permission_relationships.py +133 -51
- cartography/intel/aws/rds.py +249 -85
- cartography/intel/aws/redshift.py +107 -46
- cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
- cartography/intel/aws/resources.py +53 -46
- cartography/intel/aws/route53.py +108 -61
- cartography/intel/aws/s3.py +168 -83
- cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
- cartography/intel/aws/secretsmanager.py +24 -12
- cartography/intel/aws/securityhub.py +20 -9
- cartography/intel/aws/sns.py +166 -0
- cartography/intel/aws/sqs.py +60 -28
- cartography/intel/aws/ssm.py +70 -30
- cartography/intel/aws/util/arns.py +7 -7
- cartography/intel/aws/util/common.py +31 -4
- cartography/intel/azure/__init__.py +78 -19
- cartography/intel/azure/compute.py +101 -27
- cartography/intel/azure/cosmosdb.py +496 -170
- cartography/intel/azure/sql.py +296 -105
- cartography/intel/azure/storage.py +322 -113
- cartography/intel/azure/subscription.py +39 -23
- cartography/intel/azure/tenant.py +13 -4
- cartography/intel/azure/util/credentials.py +95 -55
- cartography/intel/bigfix/__init__.py +2 -2
- cartography/intel/bigfix/computers.py +93 -65
- cartography/intel/create_indexes.py +3 -2
- cartography/intel/crowdstrike/__init__.py +11 -9
- cartography/intel/crowdstrike/endpoints.py +5 -1
- cartography/intel/crowdstrike/spotlight.py +8 -3
- cartography/intel/cve/__init__.py +46 -13
- cartography/intel/cve/feed.py +48 -12
- cartography/intel/digitalocean/__init__.py +22 -13
- cartography/intel/digitalocean/compute.py +75 -108
- cartography/intel/digitalocean/management.py +44 -80
- cartography/intel/digitalocean/platform.py +48 -43
- cartography/intel/dns.py +36 -10
- cartography/intel/duo/__init__.py +21 -16
- cartography/intel/duo/api_host.py +14 -9
- cartography/intel/duo/endpoints.py +50 -45
- cartography/intel/duo/groups.py +18 -14
- cartography/intel/duo/phones.py +37 -34
- cartography/intel/duo/tokens.py +26 -23
- cartography/intel/duo/users.py +54 -50
- cartography/intel/duo/web_authn_credentials.py +30 -25
- cartography/intel/entra/__init__.py +25 -7
- cartography/intel/entra/ou.py +112 -0
- cartography/intel/entra/users.py +69 -63
- cartography/intel/gcp/__init__.py +185 -49
- cartography/intel/gcp/compute.py +418 -231
- cartography/intel/gcp/crm.py +96 -43
- cartography/intel/gcp/dns.py +60 -19
- cartography/intel/gcp/gke.py +72 -38
- cartography/intel/gcp/iam.py +61 -41
- cartography/intel/gcp/storage.py +84 -55
- cartography/intel/github/__init__.py +13 -11
- cartography/intel/github/repos.py +270 -137
- cartography/intel/github/teams.py +170 -88
- cartography/intel/github/users.py +70 -39
- cartography/intel/github/util.py +36 -34
- cartography/intel/gsuite/__init__.py +47 -26
- cartography/intel/gsuite/api.py +73 -30
- cartography/intel/jamf/__init__.py +19 -1
- cartography/intel/jamf/computers.py +30 -7
- cartography/intel/jamf/util.py +7 -2
- cartography/intel/kandji/__init__.py +6 -3
- cartography/intel/kandji/devices.py +14 -8
- cartography/intel/kubernetes/namespaces.py +7 -4
- cartography/intel/kubernetes/pods.py +7 -4
- cartography/intel/kubernetes/services.py +8 -4
- cartography/intel/lastpass/__init__.py +2 -2
- cartography/intel/lastpass/users.py +23 -12
- cartography/intel/oci/__init__.py +44 -11
- cartography/intel/oci/iam.py +134 -38
- cartography/intel/oci/organizations.py +13 -6
- cartography/intel/oci/utils.py +43 -20
- cartography/intel/okta/__init__.py +66 -15
- cartography/intel/okta/applications.py +42 -20
- cartography/intel/okta/awssaml.py +93 -33
- cartography/intel/okta/factors.py +16 -4
- cartography/intel/okta/groups.py +56 -29
- cartography/intel/okta/organization.py +5 -1
- cartography/intel/okta/origins.py +6 -2
- cartography/intel/okta/roles.py +15 -5
- cartography/intel/okta/users.py +20 -8
- cartography/intel/okta/utils.py +6 -4
- cartography/intel/pagerduty/__init__.py +8 -7
- cartography/intel/pagerduty/escalation_policies.py +18 -6
- cartography/intel/pagerduty/schedules.py +12 -4
- cartography/intel/pagerduty/services.py +11 -4
- cartography/intel/pagerduty/teams.py +8 -3
- cartography/intel/pagerduty/users.py +3 -1
- cartography/intel/pagerduty/vendors.py +3 -1
- cartography/intel/semgrep/__init__.py +24 -6
- cartography/intel/semgrep/dependencies.py +50 -28
- cartography/intel/semgrep/deployment.py +3 -1
- cartography/intel/semgrep/findings.py +42 -18
- cartography/intel/snipeit/__init__.py +17 -3
- cartography/intel/snipeit/asset.py +12 -6
- cartography/intel/snipeit/user.py +8 -5
- cartography/intel/snipeit/util.py +9 -4
- cartography/models/aws/apigateway.py +21 -17
- cartography/models/aws/apigatewaycertificate.py +28 -22
- cartography/models/aws/apigatewayresource.py +28 -20
- cartography/models/aws/apigatewaystage.py +33 -25
- cartography/models/aws/cloudtrail/__init__.py +0 -0
- cartography/models/aws/cloudtrail/trail.py +61 -0
- cartography/models/aws/dynamodb/gsi.py +30 -22
- cartography/models/aws/dynamodb/tables.py +25 -17
- cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
- cartography/models/aws/ec2/images.py +36 -34
- cartography/models/aws/ec2/instances.py +51 -45
- cartography/models/aws/ec2/keypair.py +21 -16
- cartography/models/aws/ec2/keypair_instance.py +28 -21
- cartography/models/aws/ec2/launch_configurations.py +30 -26
- cartography/models/aws/ec2/launch_template_versions.py +48 -38
- cartography/models/aws/ec2/launch_templates.py +21 -17
- cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
- cartography/models/aws/ec2/load_balancers.py +47 -37
- cartography/models/aws/ec2/network_acl_rules.py +38 -30
- cartography/models/aws/ec2/network_acls.py +38 -29
- cartography/models/aws/ec2/networkinterface_instance.py +52 -39
- cartography/models/aws/ec2/networkinterfaces.py +53 -37
- cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
- cartography/models/aws/ec2/reservations.py +18 -14
- cartography/models/aws/ec2/route_table_associations.py +44 -34
- cartography/models/aws/ec2/route_tables.py +50 -43
- cartography/models/aws/ec2/routes.py +45 -37
- cartography/models/aws/ec2/securitygroup_instance.py +29 -20
- cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
- cartography/models/aws/ec2/subnet_instance.py +24 -19
- cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
- cartography/models/aws/ec2/volumes.py +47 -40
- cartography/models/aws/eks/clusters.py +23 -21
- cartography/models/aws/emr.py +32 -30
- cartography/models/aws/iam/instanceprofile.py +33 -24
- cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
- cartography/models/aws/identitycenter/awspermissionset.py +37 -29
- cartography/models/aws/identitycenter/awsssouser.py +23 -21
- cartography/models/aws/inspector/findings.py +77 -65
- cartography/models/aws/inspector/packages.py +35 -29
- cartography/models/aws/s3/__init__.py +0 -0
- cartography/models/aws/s3/account_public_access_block.py +51 -0
- cartography/models/aws/sns/__init__.py +0 -0
- cartography/models/aws/sns/topic.py +50 -0
- cartography/models/aws/ssm/instance_information.py +51 -39
- cartography/models/aws/ssm/instance_patch.py +32 -26
- cartography/models/bigfix/bigfix_computer.py +42 -38
- cartography/models/bigfix/bigfix_root.py +3 -3
- cartography/models/core/common.py +12 -10
- cartography/models/core/nodes.py +5 -2
- cartography/models/core/relationships.py +14 -6
- cartography/models/crowdstrike/hosts.py +37 -35
- cartography/models/cve/cve.py +34 -32
- cartography/models/cve/cve_feed.py +6 -6
- cartography/models/digitalocean/__init__.py +0 -0
- cartography/models/digitalocean/account.py +21 -0
- cartography/models/digitalocean/droplet.py +56 -0
- cartography/models/digitalocean/project.py +48 -0
- cartography/models/duo/api_host.py +3 -3
- cartography/models/duo/endpoint.py +43 -41
- cartography/models/duo/group.py +14 -14
- cartography/models/duo/phone.py +27 -27
- cartography/models/duo/token.py +16 -16
- cartography/models/duo/user.py +46 -44
- cartography/models/duo/web_authn_credential.py +27 -19
- cartography/models/entra/ou.py +48 -0
- cartography/models/entra/tenant.py +24 -18
- cartography/models/entra/user.py +64 -48
- cartography/models/gcp/iam.py +23 -23
- cartography/models/github/orgs.py +5 -4
- cartography/models/github/teams.py +37 -31
- cartography/models/github/users.py +34 -23
- cartography/models/kandji/device.py +22 -16
- cartography/models/kandji/tenant.py +6 -4
- cartography/models/lastpass/tenant.py +3 -3
- cartography/models/lastpass/user.py +32 -28
- cartography/models/semgrep/dependencies.py +36 -24
- cartography/models/semgrep/deployment.py +5 -5
- cartography/models/semgrep/findings.py +58 -42
- cartography/models/semgrep/locations.py +27 -21
- cartography/models/snipeit/asset.py +30 -21
- cartography/models/snipeit/tenant.py +6 -4
- cartography/models/snipeit/user.py +19 -12
- cartography/stats.py +3 -3
- cartography/sync.py +107 -31
- cartography/util.py +84 -62
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/METADATA +3 -14
- cartography-0.103.0rc1.dist-info/RECORD +396 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/WHEEL +1 -1
- cartography-0.102.0rc2.dist-info/RECORD +0 -381
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/entry_points.txt +0 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/licenses/LICENSE +0 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -16,11 +16,11 @@ logger = logging.getLogger(__name__)
|
|
|
16
16
|
@timeit
|
|
17
17
|
@aws_handle_regions
|
|
18
18
|
def get_secret_list(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
|
|
19
|
-
client = boto3_session.client(
|
|
20
|
-
paginator = client.get_paginator(
|
|
19
|
+
client = boto3_session.client("secretsmanager", region_name=region)
|
|
20
|
+
paginator = client.get_paginator("list_secrets")
|
|
21
21
|
secrets: List[Dict] = []
|
|
22
22
|
for page in paginator.paginate():
|
|
23
|
-
secrets.extend(page[
|
|
23
|
+
secrets.extend(page["SecretList"])
|
|
24
24
|
return secrets
|
|
25
25
|
|
|
26
26
|
|
|
@@ -52,11 +52,11 @@ def load_secrets(
|
|
|
52
52
|
SET r.lastupdated = $aws_update_tag
|
|
53
53
|
"""
|
|
54
54
|
for secret in data:
|
|
55
|
-
secret[
|
|
56
|
-
secret[
|
|
57
|
-
secret[
|
|
58
|
-
secret[
|
|
59
|
-
secret[
|
|
55
|
+
secret["LastRotatedDate"] = dict_date_to_epoch(secret, "LastRotatedDate")
|
|
56
|
+
secret["LastChangedDate"] = dict_date_to_epoch(secret, "LastChangedDate")
|
|
57
|
+
secret["LastAccessedDate"] = dict_date_to_epoch(secret, "LastAccessedDate")
|
|
58
|
+
secret["DeletedDate"] = dict_date_to_epoch(secret, "DeletedDate")
|
|
59
|
+
secret["CreatedDate"] = dict_date_to_epoch(secret, "CreatedDate")
|
|
60
60
|
|
|
61
61
|
neo4j_session.run(
|
|
62
62
|
ingest_secrets,
|
|
@@ -69,16 +69,28 @@ def load_secrets(
|
|
|
69
69
|
|
|
70
70
|
@timeit
|
|
71
71
|
def cleanup_secrets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
|
|
72
|
-
run_cleanup_job(
|
|
72
|
+
run_cleanup_job(
|
|
73
|
+
"aws_import_secrets_cleanup.json",
|
|
74
|
+
neo4j_session,
|
|
75
|
+
common_job_parameters,
|
|
76
|
+
)
|
|
73
77
|
|
|
74
78
|
|
|
75
79
|
@timeit
|
|
76
80
|
def sync(
|
|
77
|
-
neo4j_session: neo4j.Session,
|
|
78
|
-
|
|
81
|
+
neo4j_session: neo4j.Session,
|
|
82
|
+
boto3_session: boto3.session.Session,
|
|
83
|
+
regions: List[str],
|
|
84
|
+
current_aws_account_id: str,
|
|
85
|
+
update_tag: int,
|
|
86
|
+
common_job_parameters: Dict,
|
|
79
87
|
) -> None:
|
|
80
88
|
for region in regions:
|
|
81
|
-
logger.info(
|
|
89
|
+
logger.info(
|
|
90
|
+
"Syncing Secrets Manager for region '%s' in account '%s'.",
|
|
91
|
+
region,
|
|
92
|
+
current_aws_account_id,
|
|
93
|
+
)
|
|
82
94
|
secrets = get_secret_list(boto3_session, region)
|
|
83
95
|
load_secrets(neo4j_session, secrets, region, current_aws_account_id, update_tag)
|
|
84
96
|
cleanup_secrets(neo4j_session, common_job_parameters)
|
|
@@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
|
|
|
14
14
|
|
|
15
15
|
@timeit
|
|
16
16
|
def get_hub(boto3_session: boto3.session.Session) -> Dict:
|
|
17
|
-
client = boto3_session.client(
|
|
17
|
+
client = boto3_session.client("securityhub")
|
|
18
18
|
try:
|
|
19
19
|
return client.describe_hub()
|
|
20
20
|
except client.exceptions.ResourceNotFoundException:
|
|
@@ -24,11 +24,11 @@ def get_hub(boto3_session: boto3.session.Session) -> Dict:
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
def transform_hub(hub_data: Dict) -> None:
|
|
27
|
-
if
|
|
28
|
-
subbed_at = parser.parse(hub_data[
|
|
29
|
-
hub_data[
|
|
27
|
+
if "SubscribedAt" in hub_data and hub_data["SubscribedAt"]:
|
|
28
|
+
subbed_at = parser.parse(hub_data["SubscribedAt"])
|
|
29
|
+
hub_data["SubscribedAt"] = int(subbed_at.timestamp())
|
|
30
30
|
else:
|
|
31
|
-
hub_data[
|
|
31
|
+
hub_data["SubscribedAt"] = None
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
@timeit
|
|
@@ -59,14 +59,25 @@ def load_hub(
|
|
|
59
59
|
|
|
60
60
|
|
|
61
61
|
@timeit
|
|
62
|
-
def cleanup_securityhub(
|
|
63
|
-
|
|
62
|
+
def cleanup_securityhub(
|
|
63
|
+
neo4j_session: neo4j.Session,
|
|
64
|
+
common_job_parameters: Dict,
|
|
65
|
+
) -> None:
|
|
66
|
+
run_cleanup_job(
|
|
67
|
+
"aws_import_securityhub_cleanup.json",
|
|
68
|
+
neo4j_session,
|
|
69
|
+
common_job_parameters,
|
|
70
|
+
)
|
|
64
71
|
|
|
65
72
|
|
|
66
73
|
@timeit
|
|
67
74
|
def sync(
|
|
68
|
-
neo4j_session: neo4j.Session,
|
|
69
|
-
|
|
75
|
+
neo4j_session: neo4j.Session,
|
|
76
|
+
boto3_session: boto3.session.Session,
|
|
77
|
+
regions: List[str],
|
|
78
|
+
current_aws_account_id: str,
|
|
79
|
+
update_tag: int,
|
|
80
|
+
common_job_parameters: Dict,
|
|
70
81
|
) -> None:
|
|
71
82
|
logger.info("Syncing Security Hub in account '%s'.", current_aws_account_id)
|
|
72
83
|
hub = get_hub(boto3_session)
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Dict
|
|
3
|
+
from typing import List
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
import boto3
|
|
7
|
+
import neo4j
|
|
8
|
+
|
|
9
|
+
from cartography.client.core.tx import load
|
|
10
|
+
from cartography.graph.job import GraphJob
|
|
11
|
+
from cartography.models.aws.sns.topic import SNSTopicSchema
|
|
12
|
+
from cartography.stats import get_stats_client
|
|
13
|
+
from cartography.util import aws_handle_regions
|
|
14
|
+
from cartography.util import merge_module_sync_metadata
|
|
15
|
+
from cartography.util import timeit
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
stat_handler = get_stats_client(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@timeit
|
|
22
|
+
@aws_handle_regions
|
|
23
|
+
def get_sns_topics(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
|
|
24
|
+
"""
|
|
25
|
+
Get all SNS Topics for a region.
|
|
26
|
+
"""
|
|
27
|
+
client = boto3_session.client("sns", region_name=region)
|
|
28
|
+
paginator = client.get_paginator("list_topics")
|
|
29
|
+
topics = []
|
|
30
|
+
for page in paginator.paginate():
|
|
31
|
+
topics.extend(page.get("Topics", []))
|
|
32
|
+
|
|
33
|
+
return topics
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@timeit
|
|
37
|
+
def get_topic_attributes(
|
|
38
|
+
boto3_session: boto3.session.Session, topic_arn: str, region: str
|
|
39
|
+
) -> Optional[Dict]:
|
|
40
|
+
"""
|
|
41
|
+
Get attributes for an SNS Topic.
|
|
42
|
+
"""
|
|
43
|
+
client = boto3_session.client("sns", region_name=region)
|
|
44
|
+
try:
|
|
45
|
+
return client.get_topic_attributes(TopicArn=topic_arn)
|
|
46
|
+
except Exception as e:
|
|
47
|
+
logger.warning(f"Error getting attributes for SNS topic {topic_arn}: {e}")
|
|
48
|
+
return None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def transform_sns_topics(
|
|
52
|
+
topics: List[Dict], attributes: Dict[str, Dict], region: str
|
|
53
|
+
) -> List[Dict]:
|
|
54
|
+
"""
|
|
55
|
+
Transform SNS topic data for ingestion
|
|
56
|
+
"""
|
|
57
|
+
transformed_topics = []
|
|
58
|
+
for topic in topics:
|
|
59
|
+
topic_arn = topic["TopicArn"]
|
|
60
|
+
|
|
61
|
+
# Extract topic name from ARN
|
|
62
|
+
# Format: arn:aws:sns:region:account-id:topic-name
|
|
63
|
+
topic_name = topic_arn.split(":")[-1]
|
|
64
|
+
|
|
65
|
+
# Get attributes
|
|
66
|
+
topic_attrs = attributes.get(topic_arn, {}).get("Attributes", {})
|
|
67
|
+
|
|
68
|
+
transformed_topic = {
|
|
69
|
+
"TopicArn": topic_arn,
|
|
70
|
+
"TopicName": topic_name,
|
|
71
|
+
"DisplayName": topic_attrs.get("DisplayName", ""),
|
|
72
|
+
"Owner": topic_attrs.get("Owner", ""),
|
|
73
|
+
"SubscriptionsPending": int(topic_attrs.get("SubscriptionsPending", "0")),
|
|
74
|
+
"SubscriptionsConfirmed": int(
|
|
75
|
+
topic_attrs.get("SubscriptionsConfirmed", "0")
|
|
76
|
+
),
|
|
77
|
+
"SubscriptionsDeleted": int(topic_attrs.get("SubscriptionsDeleted", "0")),
|
|
78
|
+
"DeliveryPolicy": topic_attrs.get("DeliveryPolicy", ""),
|
|
79
|
+
"EffectiveDeliveryPolicy": topic_attrs.get("EffectiveDeliveryPolicy", ""),
|
|
80
|
+
"KmsMasterKeyId": topic_attrs.get("KmsMasterKeyId", ""),
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
transformed_topics.append(transformed_topic)
|
|
84
|
+
|
|
85
|
+
return transformed_topics
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@timeit
|
|
89
|
+
def load_sns_topics(
|
|
90
|
+
neo4j_session: neo4j.Session,
|
|
91
|
+
data: List[Dict],
|
|
92
|
+
region: str,
|
|
93
|
+
aws_account_id: str,
|
|
94
|
+
update_tag: int,
|
|
95
|
+
) -> None:
|
|
96
|
+
"""
|
|
97
|
+
Load SNS Topics information into the graph
|
|
98
|
+
"""
|
|
99
|
+
logger.info(f"Loading {len(data)} SNS topics for region {region} into graph.")
|
|
100
|
+
|
|
101
|
+
load(
|
|
102
|
+
neo4j_session,
|
|
103
|
+
SNSTopicSchema(),
|
|
104
|
+
data,
|
|
105
|
+
lastupdated=update_tag,
|
|
106
|
+
Region=region,
|
|
107
|
+
AWS_ID=aws_account_id,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@timeit
|
|
112
|
+
def cleanup_sns(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
|
|
113
|
+
"""
|
|
114
|
+
Run SNS cleanup job
|
|
115
|
+
"""
|
|
116
|
+
logger.debug("Running SNS cleanup job.")
|
|
117
|
+
cleanup_job = GraphJob.from_node_schema(SNSTopicSchema(), common_job_parameters)
|
|
118
|
+
cleanup_job.run(neo4j_session)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@timeit
|
|
122
|
+
def sync(
|
|
123
|
+
neo4j_session: neo4j.Session,
|
|
124
|
+
boto3_session: boto3.session.Session,
|
|
125
|
+
regions: List[str],
|
|
126
|
+
current_aws_account_id: str,
|
|
127
|
+
update_tag: int,
|
|
128
|
+
common_job_parameters: Dict,
|
|
129
|
+
) -> None:
|
|
130
|
+
"""
|
|
131
|
+
Sync SNS Topics for all regions
|
|
132
|
+
"""
|
|
133
|
+
for region in regions:
|
|
134
|
+
logger.info(
|
|
135
|
+
f"Syncing SNS Topics for {region} in account {current_aws_account_id}"
|
|
136
|
+
)
|
|
137
|
+
topics = get_sns_topics(boto3_session, region)
|
|
138
|
+
|
|
139
|
+
topic_attributes = {}
|
|
140
|
+
for topic in topics:
|
|
141
|
+
topic_arn = topic["TopicArn"]
|
|
142
|
+
attrs = get_topic_attributes(boto3_session, topic_arn, region)
|
|
143
|
+
if attrs:
|
|
144
|
+
topic_attributes[topic_arn] = attrs
|
|
145
|
+
|
|
146
|
+
transformed_topics = transform_sns_topics(topics, topic_attributes, region)
|
|
147
|
+
|
|
148
|
+
load_sns_topics(
|
|
149
|
+
neo4j_session,
|
|
150
|
+
transformed_topics,
|
|
151
|
+
region,
|
|
152
|
+
current_aws_account_id,
|
|
153
|
+
update_tag,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
cleanup_sns(neo4j_session, common_job_parameters)
|
|
157
|
+
|
|
158
|
+
# Record that we've synced this module
|
|
159
|
+
merge_module_sync_metadata(
|
|
160
|
+
neo4j_session,
|
|
161
|
+
group_type="AWSAccount",
|
|
162
|
+
group_id=current_aws_account_id,
|
|
163
|
+
synced_type="SNSTopic",
|
|
164
|
+
update_tag=update_tag,
|
|
165
|
+
stat_handler=stat_handler,
|
|
166
|
+
)
|
cartography/intel/aws/sqs.py
CHANGED
|
@@ -19,36 +19,41 @@ logger = logging.getLogger(__name__)
|
|
|
19
19
|
@timeit
|
|
20
20
|
@aws_handle_regions
|
|
21
21
|
def get_sqs_queue_list(boto3_session: boto3.session.Session, region: str) -> List[str]:
|
|
22
|
-
client = boto3_session.client(
|
|
23
|
-
paginator = client.get_paginator(
|
|
22
|
+
client = boto3_session.client("sqs", region_name=region)
|
|
23
|
+
paginator = client.get_paginator("list_queues")
|
|
24
24
|
queues: List[Any] = []
|
|
25
25
|
for page in paginator.paginate():
|
|
26
|
-
queues.extend(page.get(
|
|
26
|
+
queues.extend(page.get("QueueUrls", []))
|
|
27
27
|
return queues
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
@timeit
|
|
31
31
|
@aws_handle_regions
|
|
32
32
|
def get_sqs_queue_attributes(
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
boto3_session: boto3.session.Session,
|
|
34
|
+
queue_urls: List[str],
|
|
35
35
|
) -> List[Tuple[str, Any]]:
|
|
36
36
|
"""
|
|
37
37
|
Iterates over all SQS queues. Returns a dict with url as key, and attributes as value.
|
|
38
38
|
"""
|
|
39
|
-
client = boto3_session.client(
|
|
39
|
+
client = boto3_session.client("sqs")
|
|
40
40
|
|
|
41
41
|
queue_attributes = []
|
|
42
42
|
for queue_url in queue_urls:
|
|
43
43
|
try:
|
|
44
|
-
response = client.get_queue_attributes(
|
|
44
|
+
response = client.get_queue_attributes(
|
|
45
|
+
QueueUrl=queue_url,
|
|
46
|
+
AttributeNames=["All"],
|
|
47
|
+
)
|
|
45
48
|
except ClientError as e:
|
|
46
|
-
if e.response[
|
|
47
|
-
logger.warning(
|
|
49
|
+
if e.response["Error"]["Code"] == "AWS.SimpleQueueService.NonExistentQueue":
|
|
50
|
+
logger.warning(
|
|
51
|
+
f"Failed to retrieve SQS queue {queue_url} - Queue does not exist error",
|
|
52
|
+
)
|
|
48
53
|
continue
|
|
49
54
|
else:
|
|
50
55
|
raise
|
|
51
|
-
queue_attributes.append((queue_url, response[
|
|
56
|
+
queue_attributes.append((queue_url, response["Attributes"]))
|
|
52
57
|
|
|
53
58
|
return queue_attributes
|
|
54
59
|
|
|
@@ -91,23 +96,25 @@ def load_sqs_queues(
|
|
|
91
96
|
dead_letter_queues: List[Dict] = []
|
|
92
97
|
queues: List[Dict] = []
|
|
93
98
|
for url, queue in data:
|
|
94
|
-
queue[
|
|
95
|
-
queue[
|
|
96
|
-
queue[
|
|
97
|
-
queue[
|
|
98
|
-
redrive_policy = queue.get(
|
|
99
|
+
queue["url"] = url
|
|
100
|
+
queue["name"] = queue["QueueArn"].split(":")[-1]
|
|
101
|
+
queue["CreatedTimestamp"] = int(queue["CreatedTimestamp"])
|
|
102
|
+
queue["LastModifiedTimestamp"] = int(queue["LastModifiedTimestamp"])
|
|
103
|
+
redrive_policy = queue.get("RedrivePolicy")
|
|
99
104
|
if redrive_policy:
|
|
100
105
|
try:
|
|
101
106
|
rp = json.loads(redrive_policy)
|
|
102
107
|
except TypeError:
|
|
103
108
|
rp = {}
|
|
104
|
-
queue[
|
|
105
|
-
dead_letter_arn = rp.get(
|
|
109
|
+
queue["RedrivePolicy"] = rp
|
|
110
|
+
dead_letter_arn = rp.get("deadLetterTargetArn")
|
|
106
111
|
if dead_letter_arn:
|
|
107
|
-
dead_letter_queues.append(
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
112
|
+
dead_letter_queues.append(
|
|
113
|
+
{
|
|
114
|
+
"arn": queue["QueueArn"],
|
|
115
|
+
"dead_letter_arn": dead_letter_arn,
|
|
116
|
+
},
|
|
117
|
+
)
|
|
111
118
|
queues.append(queue)
|
|
112
119
|
|
|
113
120
|
neo4j_session.run(
|
|
@@ -122,7 +129,11 @@ def load_sqs_queues(
|
|
|
122
129
|
|
|
123
130
|
|
|
124
131
|
@timeit
|
|
125
|
-
def _attach_dead_letter_queues(
|
|
132
|
+
def _attach_dead_letter_queues(
|
|
133
|
+
neo4j_session: neo4j.Session,
|
|
134
|
+
data: List[Dict[str, str]],
|
|
135
|
+
aws_update_tag: int,
|
|
136
|
+
) -> None:
|
|
126
137
|
"""
|
|
127
138
|
Attach deadletter queues to their queues.
|
|
128
139
|
"""
|
|
@@ -141,20 +152,41 @@ def _attach_dead_letter_queues(neo4j_session: neo4j.Session, data: List[Dict[str
|
|
|
141
152
|
|
|
142
153
|
|
|
143
154
|
@timeit
|
|
144
|
-
def cleanup_sqs_queues(
|
|
145
|
-
|
|
155
|
+
def cleanup_sqs_queues(
|
|
156
|
+
neo4j_session: neo4j.Session,
|
|
157
|
+
common_job_parameters: Dict,
|
|
158
|
+
) -> None:
|
|
159
|
+
run_cleanup_job(
|
|
160
|
+
"aws_import_sqs_queues_cleanup.json",
|
|
161
|
+
neo4j_session,
|
|
162
|
+
common_job_parameters,
|
|
163
|
+
)
|
|
146
164
|
|
|
147
165
|
|
|
148
166
|
@timeit
|
|
149
167
|
def sync(
|
|
150
|
-
neo4j_session: neo4j.Session,
|
|
151
|
-
|
|
168
|
+
neo4j_session: neo4j.Session,
|
|
169
|
+
boto3_session: boto3.session.Session,
|
|
170
|
+
regions: List[str],
|
|
171
|
+
current_aws_account_id: str,
|
|
172
|
+
update_tag: int,
|
|
173
|
+
common_job_parameters: Dict,
|
|
152
174
|
) -> None:
|
|
153
175
|
for region in regions:
|
|
154
|
-
logger.info(
|
|
176
|
+
logger.info(
|
|
177
|
+
"Syncing SQS for region '%s' in account '%s'.",
|
|
178
|
+
region,
|
|
179
|
+
current_aws_account_id,
|
|
180
|
+
)
|
|
155
181
|
queue_urls = get_sqs_queue_list(boto3_session, region)
|
|
156
182
|
if len(queue_urls) == 0:
|
|
157
183
|
continue
|
|
158
184
|
queue_attributes = get_sqs_queue_attributes(boto3_session, queue_urls)
|
|
159
|
-
load_sqs_queues(
|
|
185
|
+
load_sqs_queues(
|
|
186
|
+
neo4j_session,
|
|
187
|
+
queue_attributes,
|
|
188
|
+
region,
|
|
189
|
+
current_aws_account_id,
|
|
190
|
+
update_tag,
|
|
191
|
+
)
|
|
160
192
|
cleanup_sqs_queues(neo4j_session, common_job_parameters)
|
cartography/intel/aws/ssm.py
CHANGED
|
@@ -18,58 +18,74 @@ logger = logging.getLogger(__name__)
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
@timeit
|
|
21
|
-
def get_instance_ids(
|
|
21
|
+
def get_instance_ids(
|
|
22
|
+
neo4j_session: neo4j.Session,
|
|
23
|
+
region: str,
|
|
24
|
+
current_aws_account_id: str,
|
|
25
|
+
) -> List[str]:
|
|
22
26
|
get_instances_query = """
|
|
23
27
|
MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(i:EC2Instance)
|
|
24
28
|
WHERE i.region = $Region
|
|
25
29
|
RETURN i.id
|
|
26
30
|
"""
|
|
27
|
-
results = neo4j_session.run(
|
|
31
|
+
results = neo4j_session.run(
|
|
32
|
+
get_instances_query,
|
|
33
|
+
AWS_ACCOUNT_ID=current_aws_account_id,
|
|
34
|
+
Region=region,
|
|
35
|
+
)
|
|
28
36
|
instance_ids = []
|
|
29
37
|
for r in results:
|
|
30
|
-
instance_ids.append(r[
|
|
38
|
+
instance_ids.append(r["i.id"])
|
|
31
39
|
return instance_ids
|
|
32
40
|
|
|
33
41
|
|
|
34
42
|
@timeit
|
|
35
43
|
@aws_handle_regions
|
|
36
44
|
def get_instance_information(
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
45
|
+
boto3_session: boto3.session.Session,
|
|
46
|
+
region: str,
|
|
47
|
+
instance_ids: List[str],
|
|
40
48
|
) -> List[Dict[str, Any]]:
|
|
41
|
-
client = boto3_session.client(
|
|
49
|
+
client = boto3_session.client("ssm", region_name=region)
|
|
42
50
|
instance_information: List[Dict[str, Any]] = []
|
|
43
|
-
paginator = client.get_paginator(
|
|
51
|
+
paginator = client.get_paginator("describe_instance_information")
|
|
44
52
|
for i in range(0, len(instance_ids), 50):
|
|
45
|
-
instance_ids_chunk = instance_ids[i:i + 50]
|
|
53
|
+
instance_ids_chunk = instance_ids[i : i + 50]
|
|
46
54
|
for info_chunk in paginator.paginate(
|
|
47
55
|
Filters=[{"Key": "InstanceIds", "Values": instance_ids_chunk}],
|
|
48
56
|
MaxResults=50,
|
|
49
57
|
):
|
|
50
|
-
instance_information.extend(info_chunk.get(
|
|
58
|
+
instance_information.extend(info_chunk.get("InstanceInformationList", []))
|
|
51
59
|
return instance_information
|
|
52
60
|
|
|
53
61
|
|
|
54
|
-
def transform_instance_information(
|
|
62
|
+
def transform_instance_information(
|
|
63
|
+
data_list: List[Dict[str, Any]],
|
|
64
|
+
) -> List[Dict[str, Any]]:
|
|
55
65
|
for ii in data_list:
|
|
56
66
|
ii["LastPingDateTime"] = dict_date_to_epoch(ii, "LastPingDateTime")
|
|
57
67
|
ii["RegistrationDate"] = dict_date_to_epoch(ii, "RegistrationDate")
|
|
58
|
-
ii["LastAssociationExecutionDate"] = dict_date_to_epoch(
|
|
59
|
-
|
|
68
|
+
ii["LastAssociationExecutionDate"] = dict_date_to_epoch(
|
|
69
|
+
ii,
|
|
70
|
+
"LastAssociationExecutionDate",
|
|
71
|
+
)
|
|
72
|
+
ii["LastSuccessfulAssociationExecutionDate"] = dict_date_to_epoch(
|
|
73
|
+
ii,
|
|
74
|
+
"LastSuccessfulAssociationExecutionDate",
|
|
75
|
+
)
|
|
60
76
|
return data_list
|
|
61
77
|
|
|
62
78
|
|
|
63
79
|
@timeit
|
|
64
80
|
@aws_handle_regions
|
|
65
81
|
def get_instance_patches(
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
82
|
+
boto3_session: boto3.session.Session,
|
|
83
|
+
region: str,
|
|
84
|
+
instance_ids: List[str],
|
|
69
85
|
) -> List[Dict[str, Any]]:
|
|
70
|
-
client = boto3_session.client(
|
|
86
|
+
client = boto3_session.client("ssm", region_name=region)
|
|
71
87
|
instance_patches: List[Dict[str, Any]] = []
|
|
72
|
-
paginator = client.get_paginator(
|
|
88
|
+
paginator = client.get_paginator("describe_instance_patches")
|
|
73
89
|
for instance_id in instance_ids:
|
|
74
90
|
patches = []
|
|
75
91
|
for page in paginator.paginate(InstanceId=instance_id):
|
|
@@ -128,29 +144,53 @@ def load_instance_patches(
|
|
|
128
144
|
|
|
129
145
|
|
|
130
146
|
@timeit
|
|
131
|
-
def cleanup_ssm(
|
|
147
|
+
def cleanup_ssm(
|
|
148
|
+
neo4j_session: neo4j.Session,
|
|
149
|
+
common_job_parameters: Dict[str, Any],
|
|
150
|
+
) -> None:
|
|
132
151
|
logger.info("Running SSM cleanup")
|
|
133
|
-
GraphJob.from_node_schema(
|
|
134
|
-
|
|
152
|
+
GraphJob.from_node_schema(
|
|
153
|
+
SSMInstanceInformationSchema(),
|
|
154
|
+
common_job_parameters,
|
|
155
|
+
).run(neo4j_session)
|
|
156
|
+
GraphJob.from_node_schema(SSMInstancePatchSchema(), common_job_parameters).run(
|
|
157
|
+
neo4j_session,
|
|
158
|
+
)
|
|
135
159
|
|
|
136
160
|
|
|
137
161
|
@timeit
|
|
138
162
|
def sync(
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
163
|
+
neo4j_session: neo4j.Session,
|
|
164
|
+
boto3_session: boto3.session.Session,
|
|
165
|
+
regions: List[str],
|
|
166
|
+
current_aws_account_id: str,
|
|
167
|
+
update_tag: int,
|
|
168
|
+
common_job_parameters: Dict[str, Any],
|
|
145
169
|
) -> None:
|
|
146
170
|
for region in regions:
|
|
147
|
-
logger.info(
|
|
171
|
+
logger.info(
|
|
172
|
+
"Syncing SSM for region '%s' in account '%s'.",
|
|
173
|
+
region,
|
|
174
|
+
current_aws_account_id,
|
|
175
|
+
)
|
|
148
176
|
instance_ids = get_instance_ids(neo4j_session, region, current_aws_account_id)
|
|
149
177
|
data = get_instance_information(boto3_session, region, instance_ids)
|
|
150
178
|
data = transform_instance_information(data)
|
|
151
|
-
load_instance_information(
|
|
179
|
+
load_instance_information(
|
|
180
|
+
neo4j_session,
|
|
181
|
+
data,
|
|
182
|
+
region,
|
|
183
|
+
current_aws_account_id,
|
|
184
|
+
update_tag,
|
|
185
|
+
)
|
|
152
186
|
|
|
153
187
|
data = get_instance_patches(boto3_session, region, instance_ids)
|
|
154
188
|
data = transform_instance_patches(data)
|
|
155
|
-
load_instance_patches(
|
|
189
|
+
load_instance_patches(
|
|
190
|
+
neo4j_session,
|
|
191
|
+
data,
|
|
192
|
+
region,
|
|
193
|
+
current_aws_account_id,
|
|
194
|
+
update_tag,
|
|
195
|
+
)
|
|
156
196
|
cleanup_ssm(neo4j_session, common_job_parameters)
|
|
@@ -2,16 +2,16 @@ from typing import Optional
|
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
def build_arn(
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
resource: str,
|
|
6
|
+
account: str,
|
|
7
|
+
typename: str,
|
|
8
|
+
name: str,
|
|
9
|
+
region: Optional[str] = None,
|
|
10
|
+
partition: Optional[str] = None,
|
|
11
11
|
) -> str:
|
|
12
12
|
if not partition:
|
|
13
13
|
# TODO: support partitions from others. Please file an issue on this if needed, would love to hear from you
|
|
14
|
-
partition =
|
|
14
|
+
partition = "aws"
|
|
15
15
|
if not region:
|
|
16
16
|
# Some resources are present in all regions, e.g. IAM policies
|
|
17
17
|
region = ""
|
|
@@ -1,21 +1,48 @@
|
|
|
1
|
+
import logging
|
|
1
2
|
from typing import List
|
|
2
3
|
|
|
3
4
|
from cartography.intel.aws.resources import RESOURCE_FUNCTIONS
|
|
4
5
|
|
|
6
|
+
logger = logging.getLogger(__name__)
|
|
7
|
+
|
|
5
8
|
|
|
6
9
|
def parse_and_validate_aws_requested_syncs(aws_requested_syncs: str) -> List[str]:
|
|
7
10
|
validated_resources: List[str] = []
|
|
8
|
-
for resource in aws_requested_syncs.split(
|
|
11
|
+
for resource in aws_requested_syncs.split(","):
|
|
9
12
|
resource = resource.strip()
|
|
10
13
|
|
|
11
14
|
if resource in RESOURCE_FUNCTIONS:
|
|
12
15
|
validated_resources.append(resource)
|
|
13
16
|
else:
|
|
14
|
-
valid_syncs: str =
|
|
17
|
+
valid_syncs: str = ", ".join(RESOURCE_FUNCTIONS.keys())
|
|
15
18
|
raise ValueError(
|
|
16
19
|
f'Error parsing `aws-requested-syncs`. You specified "{aws_requested_syncs}". '
|
|
17
|
-
f
|
|
20
|
+
f"Please check that your string is formatted properly. "
|
|
18
21
|
f'Example valid input looks like "s3,iam,rds" or "s3, ec2:instance, dynamodb". '
|
|
19
|
-
f
|
|
22
|
+
f"Our full list of valid values is: {valid_syncs}.",
|
|
20
23
|
)
|
|
21
24
|
return validated_resources
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def parse_and_validate_aws_regions(aws_regions: str) -> list[str]:
|
|
28
|
+
"""
|
|
29
|
+
Parse and validate a comma-separated string of AWS regions.
|
|
30
|
+
:param aws_regions: Comma-separated string of AWS regions
|
|
31
|
+
:return: A validated list of AWS regions
|
|
32
|
+
"""
|
|
33
|
+
validated_regions: List[str] = []
|
|
34
|
+
for region in aws_regions.split(","):
|
|
35
|
+
region = region.strip()
|
|
36
|
+
if region:
|
|
37
|
+
validated_regions.append(region)
|
|
38
|
+
else:
|
|
39
|
+
logger.warning(
|
|
40
|
+
f'Unable to parse string "{region}". Please check the value you passed to `aws-regions`. '
|
|
41
|
+
f'You specified "{aws_regions}". Continuing on with sync.',
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
if not validated_regions:
|
|
45
|
+
raise ValueError(
|
|
46
|
+
f'`aws-regions` was set but no regions were specified. You provided this string: "{aws_regions}"',
|
|
47
|
+
)
|
|
48
|
+
return validated_regions
|