cartography 0.102.0rc1__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/__main__.py +1 -2
- cartography/_version.py +2 -2
- cartography/cli.py +302 -253
- cartography/client/core/tx.py +39 -18
- cartography/config.py +4 -0
- cartography/driftdetect/__main__.py +1 -2
- cartography/driftdetect/add_shortcut.py +10 -2
- cartography/driftdetect/cli.py +71 -75
- cartography/driftdetect/detect_deviations.py +7 -3
- cartography/driftdetect/get_states.py +20 -8
- cartography/driftdetect/model.py +5 -5
- cartography/driftdetect/serializers.py +8 -6
- cartography/driftdetect/storage.py +2 -2
- cartography/graph/cleanupbuilder.py +35 -15
- cartography/graph/job.py +46 -17
- cartography/graph/querybuilder.py +165 -80
- cartography/graph/statement.py +35 -26
- cartography/intel/analysis.py +4 -1
- cartography/intel/aws/__init__.py +114 -55
- cartography/intel/aws/apigateway.py +134 -63
- cartography/intel/aws/cloudtrail.py +127 -0
- cartography/intel/aws/config.py +56 -20
- cartography/intel/aws/dynamodb.py +108 -40
- cartography/intel/aws/ec2/__init__.py +2 -2
- cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
- cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
- cartography/intel/aws/ec2/images.py +49 -20
- cartography/intel/aws/ec2/instances.py +234 -136
- cartography/intel/aws/ec2/internet_gateways.py +40 -11
- cartography/intel/aws/ec2/key_pairs.py +44 -20
- cartography/intel/aws/ec2/launch_templates.py +101 -59
- cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
- cartography/intel/aws/ec2/load_balancers.py +82 -42
- cartography/intel/aws/ec2/network_acls.py +89 -65
- cartography/intel/aws/ec2/network_interfaces.py +146 -87
- cartography/intel/aws/ec2/reserved_instances.py +45 -16
- cartography/intel/aws/ec2/route_tables.py +327 -0
- cartography/intel/aws/ec2/security_groups.py +71 -21
- cartography/intel/aws/ec2/snapshots.py +61 -22
- cartography/intel/aws/ec2/subnets.py +54 -18
- cartography/intel/aws/ec2/tgw.py +100 -34
- cartography/intel/aws/ec2/util.py +1 -1
- cartography/intel/aws/ec2/volumes.py +69 -41
- cartography/intel/aws/ec2/vpc.py +37 -12
- cartography/intel/aws/ec2/vpc_peerings.py +83 -24
- cartography/intel/aws/ecr.py +88 -32
- cartography/intel/aws/ecs.py +83 -47
- cartography/intel/aws/eks.py +55 -29
- cartography/intel/aws/elasticache.py +42 -18
- cartography/intel/aws/elasticsearch.py +57 -20
- cartography/intel/aws/emr.py +61 -23
- cartography/intel/aws/iam.py +401 -145
- cartography/intel/aws/iam_instance_profiles.py +22 -22
- cartography/intel/aws/identitycenter.py +71 -37
- cartography/intel/aws/inspector.py +159 -89
- cartography/intel/aws/kms.py +92 -38
- cartography/intel/aws/lambda_function.py +103 -34
- cartography/intel/aws/organizations.py +30 -10
- cartography/intel/aws/permission_relationships.py +133 -51
- cartography/intel/aws/rds.py +249 -85
- cartography/intel/aws/redshift.py +107 -46
- cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
- cartography/intel/aws/resources.py +53 -44
- cartography/intel/aws/route53.py +108 -61
- cartography/intel/aws/s3.py +168 -83
- cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
- cartography/intel/aws/secretsmanager.py +24 -12
- cartography/intel/aws/securityhub.py +20 -9
- cartography/intel/aws/sns.py +166 -0
- cartography/intel/aws/sqs.py +60 -28
- cartography/intel/aws/ssm.py +70 -30
- cartography/intel/aws/util/arns.py +7 -7
- cartography/intel/aws/util/common.py +31 -4
- cartography/intel/azure/__init__.py +78 -19
- cartography/intel/azure/compute.py +101 -27
- cartography/intel/azure/cosmosdb.py +496 -170
- cartography/intel/azure/sql.py +296 -105
- cartography/intel/azure/storage.py +322 -113
- cartography/intel/azure/subscription.py +39 -23
- cartography/intel/azure/tenant.py +13 -4
- cartography/intel/azure/util/credentials.py +95 -55
- cartography/intel/bigfix/__init__.py +2 -2
- cartography/intel/bigfix/computers.py +93 -65
- cartography/intel/create_indexes.py +3 -2
- cartography/intel/crowdstrike/__init__.py +11 -9
- cartography/intel/crowdstrike/endpoints.py +5 -1
- cartography/intel/crowdstrike/spotlight.py +8 -3
- cartography/intel/cve/__init__.py +46 -13
- cartography/intel/cve/feed.py +48 -12
- cartography/intel/digitalocean/__init__.py +22 -13
- cartography/intel/digitalocean/compute.py +75 -108
- cartography/intel/digitalocean/management.py +44 -80
- cartography/intel/digitalocean/platform.py +48 -43
- cartography/intel/dns.py +36 -10
- cartography/intel/duo/__init__.py +21 -16
- cartography/intel/duo/api_host.py +14 -9
- cartography/intel/duo/endpoints.py +50 -45
- cartography/intel/duo/groups.py +18 -14
- cartography/intel/duo/phones.py +37 -34
- cartography/intel/duo/tokens.py +26 -23
- cartography/intel/duo/users.py +54 -50
- cartography/intel/duo/web_authn_credentials.py +30 -25
- cartography/intel/entra/__init__.py +25 -7
- cartography/intel/entra/ou.py +112 -0
- cartography/intel/entra/users.py +69 -63
- cartography/intel/gcp/__init__.py +185 -49
- cartography/intel/gcp/compute.py +418 -231
- cartography/intel/gcp/crm.py +96 -43
- cartography/intel/gcp/dns.py +60 -19
- cartography/intel/gcp/gke.py +72 -38
- cartography/intel/gcp/iam.py +61 -41
- cartography/intel/gcp/storage.py +84 -55
- cartography/intel/github/__init__.py +13 -11
- cartography/intel/github/repos.py +270 -137
- cartography/intel/github/teams.py +170 -88
- cartography/intel/github/users.py +70 -39
- cartography/intel/github/util.py +36 -34
- cartography/intel/gsuite/__init__.py +47 -26
- cartography/intel/gsuite/api.py +73 -30
- cartography/intel/jamf/__init__.py +19 -1
- cartography/intel/jamf/computers.py +30 -7
- cartography/intel/jamf/util.py +7 -2
- cartography/intel/kandji/__init__.py +6 -3
- cartography/intel/kandji/devices.py +14 -8
- cartography/intel/kubernetes/namespaces.py +7 -4
- cartography/intel/kubernetes/pods.py +7 -4
- cartography/intel/kubernetes/services.py +8 -4
- cartography/intel/lastpass/__init__.py +2 -2
- cartography/intel/lastpass/users.py +23 -12
- cartography/intel/oci/__init__.py +44 -11
- cartography/intel/oci/iam.py +134 -38
- cartography/intel/oci/organizations.py +13 -6
- cartography/intel/oci/utils.py +43 -20
- cartography/intel/okta/__init__.py +66 -15
- cartography/intel/okta/applications.py +42 -20
- cartography/intel/okta/awssaml.py +93 -33
- cartography/intel/okta/factors.py +16 -4
- cartography/intel/okta/groups.py +56 -29
- cartography/intel/okta/organization.py +5 -1
- cartography/intel/okta/origins.py +6 -2
- cartography/intel/okta/roles.py +15 -5
- cartography/intel/okta/users.py +20 -8
- cartography/intel/okta/utils.py +6 -4
- cartography/intel/pagerduty/__init__.py +8 -7
- cartography/intel/pagerduty/escalation_policies.py +18 -6
- cartography/intel/pagerduty/schedules.py +12 -4
- cartography/intel/pagerduty/services.py +11 -4
- cartography/intel/pagerduty/teams.py +8 -3
- cartography/intel/pagerduty/users.py +3 -1
- cartography/intel/pagerduty/vendors.py +3 -1
- cartography/intel/semgrep/__init__.py +24 -6
- cartography/intel/semgrep/dependencies.py +50 -28
- cartography/intel/semgrep/deployment.py +3 -1
- cartography/intel/semgrep/findings.py +42 -18
- cartography/intel/snipeit/__init__.py +17 -3
- cartography/intel/snipeit/asset.py +12 -6
- cartography/intel/snipeit/user.py +8 -5
- cartography/intel/snipeit/util.py +9 -4
- cartography/models/aws/apigateway.py +21 -17
- cartography/models/aws/apigatewaycertificate.py +28 -22
- cartography/models/aws/apigatewayresource.py +28 -20
- cartography/models/aws/apigatewaystage.py +33 -25
- cartography/models/aws/cloudtrail/__init__.py +0 -0
- cartography/models/aws/cloudtrail/trail.py +61 -0
- cartography/models/aws/dynamodb/gsi.py +30 -22
- cartography/models/aws/dynamodb/tables.py +25 -17
- cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
- cartography/models/aws/ec2/images.py +36 -34
- cartography/models/aws/ec2/instances.py +51 -45
- cartography/models/aws/ec2/keypair.py +21 -16
- cartography/models/aws/ec2/keypair_instance.py +28 -21
- cartography/models/aws/ec2/launch_configurations.py +30 -26
- cartography/models/aws/ec2/launch_template_versions.py +48 -38
- cartography/models/aws/ec2/launch_templates.py +21 -17
- cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
- cartography/models/aws/ec2/load_balancers.py +47 -37
- cartography/models/aws/ec2/network_acl_rules.py +38 -30
- cartography/models/aws/ec2/network_acls.py +38 -29
- cartography/models/aws/ec2/networkinterface_instance.py +52 -39
- cartography/models/aws/ec2/networkinterfaces.py +53 -37
- cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
- cartography/models/aws/ec2/reservations.py +18 -14
- cartography/models/aws/ec2/route_table_associations.py +97 -0
- cartography/models/aws/ec2/route_tables.py +128 -0
- cartography/models/aws/ec2/routes.py +85 -0
- cartography/models/aws/ec2/securitygroup_instance.py +29 -20
- cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
- cartography/models/aws/ec2/subnet_instance.py +24 -19
- cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
- cartography/models/aws/ec2/volumes.py +47 -40
- cartography/models/aws/eks/clusters.py +23 -21
- cartography/models/aws/emr.py +32 -30
- cartography/models/aws/iam/instanceprofile.py +33 -24
- cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
- cartography/models/aws/identitycenter/awspermissionset.py +37 -29
- cartography/models/aws/identitycenter/awsssouser.py +23 -21
- cartography/models/aws/inspector/findings.py +77 -65
- cartography/models/aws/inspector/packages.py +35 -29
- cartography/models/aws/s3/__init__.py +0 -0
- cartography/models/aws/s3/account_public_access_block.py +51 -0
- cartography/models/aws/sns/__init__.py +0 -0
- cartography/models/aws/sns/topic.py +50 -0
- cartography/models/aws/ssm/instance_information.py +51 -39
- cartography/models/aws/ssm/instance_patch.py +32 -26
- cartography/models/bigfix/bigfix_computer.py +42 -38
- cartography/models/bigfix/bigfix_root.py +3 -3
- cartography/models/core/common.py +12 -10
- cartography/models/core/nodes.py +5 -2
- cartography/models/core/relationships.py +14 -6
- cartography/models/crowdstrike/hosts.py +37 -35
- cartography/models/cve/cve.py +34 -32
- cartography/models/cve/cve_feed.py +6 -6
- cartography/models/digitalocean/__init__.py +0 -0
- cartography/models/digitalocean/account.py +21 -0
- cartography/models/digitalocean/droplet.py +56 -0
- cartography/models/digitalocean/project.py +48 -0
- cartography/models/duo/api_host.py +3 -3
- cartography/models/duo/endpoint.py +43 -41
- cartography/models/duo/group.py +14 -14
- cartography/models/duo/phone.py +27 -27
- cartography/models/duo/token.py +16 -16
- cartography/models/duo/user.py +46 -44
- cartography/models/duo/web_authn_credential.py +27 -19
- cartography/models/entra/ou.py +48 -0
- cartography/models/entra/tenant.py +24 -18
- cartography/models/entra/user.py +64 -48
- cartography/models/gcp/iam.py +23 -23
- cartography/models/github/orgs.py +5 -4
- cartography/models/github/teams.py +37 -31
- cartography/models/github/users.py +34 -23
- cartography/models/kandji/device.py +22 -16
- cartography/models/kandji/tenant.py +6 -4
- cartography/models/lastpass/tenant.py +3 -3
- cartography/models/lastpass/user.py +32 -28
- cartography/models/semgrep/dependencies.py +36 -24
- cartography/models/semgrep/deployment.py +5 -5
- cartography/models/semgrep/findings.py +58 -42
- cartography/models/semgrep/locations.py +27 -21
- cartography/models/snipeit/asset.py +30 -21
- cartography/models/snipeit/tenant.py +6 -4
- cartography/models/snipeit/user.py +19 -12
- cartography/stats.py +3 -3
- cartography/sync.py +107 -31
- cartography/util.py +84 -62
- {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/METADATA +3 -14
- cartography-0.103.0rc1.dist-info/RECORD +396 -0
- {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/WHEEL +1 -1
- cartography-0.102.0rc1.dist-info/RECORD +0 -377
- {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/entry_points.txt +0 -0
- {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/licenses/LICENSE +0 -0
- {cartography-0.102.0rc1.dist-info → cartography-0.103.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -14,34 +14,46 @@ logger = logging.getLogger(__name__)
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
@timeit
|
|
17
|
-
def get_snapshots_in_use(
|
|
17
|
+
def get_snapshots_in_use(
|
|
18
|
+
neo4j_session: neo4j.Session,
|
|
19
|
+
region: str,
|
|
20
|
+
current_aws_account_id: str,
|
|
21
|
+
) -> List[str]:
|
|
18
22
|
query = """
|
|
19
23
|
MATCH (:AWSAccount{id: $AWS_ACCOUNT_ID})-[:RESOURCE]->(v:EBSVolume)
|
|
20
24
|
WHERE v.region = $Region
|
|
21
25
|
RETURN v.snapshotid as snapshot
|
|
22
26
|
"""
|
|
23
|
-
results = neo4j_session.run(
|
|
24
|
-
|
|
27
|
+
results = neo4j_session.run(
|
|
28
|
+
query,
|
|
29
|
+
AWS_ACCOUNT_ID=current_aws_account_id,
|
|
30
|
+
Region=region,
|
|
31
|
+
)
|
|
32
|
+
return [r["snapshot"] for r in results if r["snapshot"]]
|
|
25
33
|
|
|
26
34
|
|
|
27
35
|
@timeit
|
|
28
36
|
@aws_handle_regions
|
|
29
|
-
def get_snapshots(
|
|
30
|
-
|
|
31
|
-
|
|
37
|
+
def get_snapshots(
|
|
38
|
+
boto3_session: boto3.session.Session,
|
|
39
|
+
region: str,
|
|
40
|
+
in_use_snapshot_ids: List[str],
|
|
41
|
+
) -> List[Dict]:
|
|
42
|
+
client = boto3_session.client("ec2", region_name=region)
|
|
43
|
+
paginator = client.get_paginator("describe_snapshots")
|
|
32
44
|
snapshots: List[Dict] = []
|
|
33
|
-
for page in paginator.paginate(OwnerIds=[
|
|
34
|
-
snapshots.extend(page[
|
|
45
|
+
for page in paginator.paginate(OwnerIds=["self"]):
|
|
46
|
+
snapshots.extend(page["Snapshots"])
|
|
35
47
|
|
|
36
48
|
# fetch in-use snapshots not in self_owned snapshots
|
|
37
|
-
self_owned_snapshot_ids = {s[
|
|
49
|
+
self_owned_snapshot_ids = {s["SnapshotId"] for s in snapshots}
|
|
38
50
|
other_snapshot_ids = set(in_use_snapshot_ids) - self_owned_snapshot_ids
|
|
39
51
|
if other_snapshot_ids:
|
|
40
52
|
try:
|
|
41
53
|
for page in paginator.paginate(SnapshotIds=list(other_snapshot_ids)):
|
|
42
|
-
snapshots.extend(page[
|
|
54
|
+
snapshots.extend(page["Snapshots"])
|
|
43
55
|
except ClientError as e:
|
|
44
|
-
if e.response[
|
|
56
|
+
if e.response["Error"]["Code"] == "InvalidSnapshot.NotFound":
|
|
45
57
|
logger.warning(
|
|
46
58
|
f"Failed to retrieve page of in-use, \
|
|
47
59
|
not owned snapshots. Continuing anyway. Error - {e}",
|
|
@@ -54,7 +66,11 @@ def get_snapshots(boto3_session: boto3.session.Session, region: str, in_use_snap
|
|
|
54
66
|
|
|
55
67
|
@timeit
|
|
56
68
|
def load_snapshots(
|
|
57
|
-
|
|
69
|
+
neo4j_session: neo4j.Session,
|
|
70
|
+
data: List[Dict],
|
|
71
|
+
region: str,
|
|
72
|
+
current_aws_account_id: str,
|
|
73
|
+
update_tag: int,
|
|
58
74
|
) -> None:
|
|
59
75
|
ingest_snapshots = """
|
|
60
76
|
UNWIND $snapshots_list as snapshot
|
|
@@ -73,7 +89,7 @@ def load_snapshots(
|
|
|
73
89
|
"""
|
|
74
90
|
|
|
75
91
|
for snapshot in data:
|
|
76
|
-
snapshot[
|
|
92
|
+
snapshot["StartTime"] = str(snapshot["StartTime"])
|
|
77
93
|
|
|
78
94
|
neo4j_session.run(
|
|
79
95
|
ingest_snapshots,
|
|
@@ -88,7 +104,7 @@ def load_snapshots(
|
|
|
88
104
|
def get_snapshot_volumes(snapshots: List[Dict]) -> List[Dict]:
|
|
89
105
|
snapshot_volumes: List[Dict] = []
|
|
90
106
|
for snapshot in snapshots:
|
|
91
|
-
if snapshot.get(
|
|
107
|
+
if snapshot.get("VolumeId"):
|
|
92
108
|
snapshot_volumes.append(snapshot)
|
|
93
109
|
|
|
94
110
|
return snapshot_volumes
|
|
@@ -96,7 +112,10 @@ def get_snapshot_volumes(snapshots: List[Dict]) -> List[Dict]:
|
|
|
96
112
|
|
|
97
113
|
@timeit
|
|
98
114
|
def load_snapshot_volume_relations(
|
|
99
|
-
|
|
115
|
+
neo4j_session: neo4j.Session,
|
|
116
|
+
data: List[Dict],
|
|
117
|
+
current_aws_account_id: str,
|
|
118
|
+
update_tag: int,
|
|
100
119
|
) -> None:
|
|
101
120
|
ingest_volumes = """
|
|
102
121
|
UNWIND $snapshot_volumes_list as volume
|
|
@@ -124,9 +143,12 @@ def load_snapshot_volume_relations(
|
|
|
124
143
|
|
|
125
144
|
|
|
126
145
|
@timeit
|
|
127
|
-
def cleanup_snapshots(
|
|
146
|
+
def cleanup_snapshots(
|
|
147
|
+
neo4j_session: neo4j.Session,
|
|
148
|
+
common_job_parameters: Dict,
|
|
149
|
+
) -> None:
|
|
128
150
|
run_cleanup_job(
|
|
129
|
-
|
|
151
|
+
"aws_import_snapshots_cleanup.json",
|
|
130
152
|
neo4j_session,
|
|
131
153
|
common_job_parameters,
|
|
132
154
|
)
|
|
@@ -134,14 +156,31 @@ def cleanup_snapshots(neo4j_session: neo4j.Session, common_job_parameters: Dict)
|
|
|
134
156
|
|
|
135
157
|
@timeit
|
|
136
158
|
def sync_ebs_snapshots(
|
|
137
|
-
|
|
138
|
-
|
|
159
|
+
neo4j_session: neo4j.Session,
|
|
160
|
+
boto3_session: boto3.session.Session,
|
|
161
|
+
regions: List[str],
|
|
162
|
+
current_aws_account_id: str,
|
|
163
|
+
update_tag: int,
|
|
164
|
+
common_job_parameters: Dict,
|
|
139
165
|
) -> None:
|
|
140
166
|
for region in regions:
|
|
141
|
-
logger.debug(
|
|
142
|
-
|
|
167
|
+
logger.debug(
|
|
168
|
+
"Syncing snapshots for region '%s' in account '%s'.",
|
|
169
|
+
region,
|
|
170
|
+
current_aws_account_id,
|
|
171
|
+
)
|
|
172
|
+
snapshots_in_use = get_snapshots_in_use(
|
|
173
|
+
neo4j_session,
|
|
174
|
+
region,
|
|
175
|
+
current_aws_account_id,
|
|
176
|
+
)
|
|
143
177
|
data = get_snapshots(boto3_session, region, snapshots_in_use)
|
|
144
178
|
load_snapshots(neo4j_session, data, region, current_aws_account_id, update_tag)
|
|
145
179
|
snapshot_volumes = get_snapshot_volumes(data)
|
|
146
|
-
load_snapshot_volume_relations(
|
|
180
|
+
load_snapshot_volume_relations(
|
|
181
|
+
neo4j_session,
|
|
182
|
+
snapshot_volumes,
|
|
183
|
+
current_aws_account_id,
|
|
184
|
+
update_tag,
|
|
185
|
+
)
|
|
147
186
|
cleanup_snapshots(neo4j_session, common_job_parameters)
|
|
@@ -5,31 +5,41 @@ from typing import List
|
|
|
5
5
|
import boto3
|
|
6
6
|
import neo4j
|
|
7
7
|
|
|
8
|
-
from .util import get_botocore_config
|
|
9
8
|
from cartography.graph.job import GraphJob
|
|
10
|
-
from cartography.models.aws.ec2.auto_scaling_groups import
|
|
9
|
+
from cartography.models.aws.ec2.auto_scaling_groups import (
|
|
10
|
+
EC2SubnetAutoScalingGroupSchema,
|
|
11
|
+
)
|
|
11
12
|
from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema
|
|
12
13
|
from cartography.util import aws_handle_regions
|
|
13
14
|
from cartography.util import run_cleanup_job
|
|
14
15
|
from cartography.util import timeit
|
|
15
16
|
|
|
17
|
+
from .util import get_botocore_config
|
|
18
|
+
|
|
16
19
|
logger = logging.getLogger(__name__)
|
|
17
20
|
|
|
18
21
|
|
|
19
22
|
@timeit
|
|
20
23
|
@aws_handle_regions
|
|
21
24
|
def get_subnet_data(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
|
|
22
|
-
client = boto3_session.client(
|
|
23
|
-
|
|
25
|
+
client = boto3_session.client(
|
|
26
|
+
"ec2",
|
|
27
|
+
region_name=region,
|
|
28
|
+
config=get_botocore_config(),
|
|
29
|
+
)
|
|
30
|
+
paginator = client.get_paginator("describe_subnets")
|
|
24
31
|
subnets: List[Dict] = []
|
|
25
32
|
for page in paginator.paginate():
|
|
26
|
-
subnets.extend(page[
|
|
33
|
+
subnets.extend(page["Subnets"])
|
|
27
34
|
return subnets
|
|
28
35
|
|
|
29
36
|
|
|
30
37
|
@timeit
|
|
31
38
|
def load_subnets(
|
|
32
|
-
neo4j_session: neo4j.Session,
|
|
39
|
+
neo4j_session: neo4j.Session,
|
|
40
|
+
data: List[Dict],
|
|
41
|
+
region: str,
|
|
42
|
+
aws_account_id: str,
|
|
33
43
|
aws_update_tag: int,
|
|
34
44
|
) -> None:
|
|
35
45
|
|
|
@@ -63,33 +73,59 @@ def load_subnets(
|
|
|
63
73
|
"""
|
|
64
74
|
|
|
65
75
|
neo4j_session.run(
|
|
66
|
-
ingest_subnets,
|
|
67
|
-
|
|
76
|
+
ingest_subnets,
|
|
77
|
+
subnets=data,
|
|
78
|
+
aws_update_tag=aws_update_tag,
|
|
79
|
+
region=region,
|
|
80
|
+
aws_account_id=aws_account_id,
|
|
68
81
|
)
|
|
69
82
|
neo4j_session.run(
|
|
70
|
-
ingest_subnet_vpc_relations,
|
|
71
|
-
|
|
83
|
+
ingest_subnet_vpc_relations,
|
|
84
|
+
subnets=data,
|
|
85
|
+
aws_update_tag=aws_update_tag,
|
|
86
|
+
region=region,
|
|
87
|
+
aws_account_id=aws_account_id,
|
|
72
88
|
)
|
|
73
89
|
neo4j_session.run(
|
|
74
|
-
ingest_subnet_aws_account_relations,
|
|
75
|
-
|
|
90
|
+
ingest_subnet_aws_account_relations,
|
|
91
|
+
subnets=data,
|
|
92
|
+
aws_update_tag=aws_update_tag,
|
|
93
|
+
region=region,
|
|
94
|
+
aws_account_id=aws_account_id,
|
|
76
95
|
)
|
|
77
96
|
|
|
78
97
|
|
|
79
98
|
@timeit
|
|
80
99
|
def cleanup_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
|
|
81
|
-
run_cleanup_job(
|
|
82
|
-
|
|
83
|
-
|
|
100
|
+
run_cleanup_job(
|
|
101
|
+
"aws_ingest_subnets_cleanup.json",
|
|
102
|
+
neo4j_session,
|
|
103
|
+
common_job_parameters,
|
|
104
|
+
)
|
|
105
|
+
GraphJob.from_node_schema(EC2SubnetInstanceSchema(), common_job_parameters).run(
|
|
106
|
+
neo4j_session,
|
|
107
|
+
)
|
|
108
|
+
GraphJob.from_node_schema(
|
|
109
|
+
EC2SubnetAutoScalingGroupSchema(),
|
|
110
|
+
common_job_parameters,
|
|
111
|
+
).run(neo4j_session)
|
|
84
112
|
|
|
85
113
|
|
|
86
114
|
@timeit
|
|
87
115
|
def sync_subnets(
|
|
88
|
-
|
|
89
|
-
|
|
116
|
+
neo4j_session: neo4j.Session,
|
|
117
|
+
boto3_session: boto3.session.Session,
|
|
118
|
+
regions: List[str],
|
|
119
|
+
current_aws_account_id: str,
|
|
120
|
+
update_tag: int,
|
|
121
|
+
common_job_parameters: Dict,
|
|
90
122
|
) -> None:
|
|
91
123
|
for region in regions:
|
|
92
|
-
logger.info(
|
|
124
|
+
logger.info(
|
|
125
|
+
"Syncing EC2 subnets for region '%s' in account '%s'.",
|
|
126
|
+
region,
|
|
127
|
+
current_aws_account_id,
|
|
128
|
+
)
|
|
93
129
|
data = get_subnet_data(boto3_session, region)
|
|
94
130
|
load_subnets(neo4j_session, data, region, current_aws_account_id, update_tag)
|
|
95
131
|
cleanup_subnets(neo4j_session, common_job_parameters)
|
cartography/intel/aws/ec2/tgw.py
CHANGED
|
@@ -6,18 +6,26 @@ import boto3
|
|
|
6
6
|
import botocore.exceptions
|
|
7
7
|
import neo4j
|
|
8
8
|
|
|
9
|
-
from .util import get_botocore_config
|
|
10
9
|
from cartography.util import aws_handle_regions
|
|
11
10
|
from cartography.util import run_cleanup_job
|
|
12
11
|
from cartography.util import timeit
|
|
13
12
|
|
|
13
|
+
from .util import get_botocore_config
|
|
14
|
+
|
|
14
15
|
logger = logging.getLogger(__name__)
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
@timeit
|
|
18
19
|
@aws_handle_regions
|
|
19
|
-
def get_transit_gateways(
|
|
20
|
-
|
|
20
|
+
def get_transit_gateways(
|
|
21
|
+
boto3_session: boto3.session.Session,
|
|
22
|
+
region: str,
|
|
23
|
+
) -> List[Dict]:
|
|
24
|
+
client = boto3_session.client(
|
|
25
|
+
"ec2",
|
|
26
|
+
region_name=region,
|
|
27
|
+
config=get_botocore_config(),
|
|
28
|
+
)
|
|
21
29
|
data: List[Dict] = []
|
|
22
30
|
try:
|
|
23
31
|
data = client.describe_transit_gateways()["TransitGateways"]
|
|
@@ -25,51 +33,68 @@ def get_transit_gateways(boto3_session: boto3.session.Session, region: str) -> L
|
|
|
25
33
|
# https://boto3.amazonaws.com/v1/documentation/api/latest/guide/error-handling.html#parsing-error-responses-and-catching-exceptions-from-aws-services
|
|
26
34
|
logger.warning(
|
|
27
35
|
"Could not retrieve Transit Gateways due to boto3 error %s: %s. Skipping.",
|
|
28
|
-
e.response[
|
|
29
|
-
e.response[
|
|
36
|
+
e.response["Error"]["Code"],
|
|
37
|
+
e.response["Error"]["Message"],
|
|
30
38
|
)
|
|
31
39
|
return data
|
|
32
40
|
|
|
33
41
|
|
|
34
42
|
@timeit
|
|
35
43
|
@aws_handle_regions
|
|
36
|
-
def get_tgw_attachments(
|
|
37
|
-
|
|
44
|
+
def get_tgw_attachments(
|
|
45
|
+
boto3_session: boto3.session.Session,
|
|
46
|
+
region: str,
|
|
47
|
+
) -> List[Dict]:
|
|
48
|
+
client = boto3_session.client(
|
|
49
|
+
"ec2",
|
|
50
|
+
region_name=region,
|
|
51
|
+
config=get_botocore_config(),
|
|
52
|
+
)
|
|
38
53
|
tgw_attachments: List[Dict] = []
|
|
39
54
|
try:
|
|
40
|
-
paginator = client.get_paginator(
|
|
55
|
+
paginator = client.get_paginator("describe_transit_gateway_attachments")
|
|
41
56
|
for page in paginator.paginate():
|
|
42
|
-
tgw_attachments.extend(page[
|
|
57
|
+
tgw_attachments.extend(page["TransitGatewayAttachments"])
|
|
43
58
|
except botocore.exceptions.ClientError as e:
|
|
44
59
|
logger.warning(
|
|
45
60
|
"Could not retrieve Transit Gateway Attachments due to boto3 error %s: %s. Skipping.",
|
|
46
|
-
e.response[
|
|
47
|
-
e.response[
|
|
61
|
+
e.response["Error"]["Code"],
|
|
62
|
+
e.response["Error"]["Message"],
|
|
48
63
|
)
|
|
49
64
|
return tgw_attachments
|
|
50
65
|
|
|
51
66
|
|
|
52
67
|
@timeit
|
|
53
68
|
@aws_handle_regions
|
|
54
|
-
def get_tgw_vpc_attachments(
|
|
55
|
-
|
|
69
|
+
def get_tgw_vpc_attachments(
|
|
70
|
+
boto3_session: boto3.session.Session,
|
|
71
|
+
region: str,
|
|
72
|
+
) -> List[Dict]:
|
|
73
|
+
client = boto3_session.client(
|
|
74
|
+
"ec2",
|
|
75
|
+
region_name=region,
|
|
76
|
+
config=get_botocore_config(),
|
|
77
|
+
)
|
|
56
78
|
tgw_vpc_attachments: List[Dict] = []
|
|
57
79
|
try:
|
|
58
|
-
paginator = client.get_paginator(
|
|
80
|
+
paginator = client.get_paginator("describe_transit_gateway_vpc_attachments")
|
|
59
81
|
for page in paginator.paginate():
|
|
60
|
-
tgw_vpc_attachments.extend(page[
|
|
82
|
+
tgw_vpc_attachments.extend(page["TransitGatewayVpcAttachments"])
|
|
61
83
|
except botocore.exceptions.ClientError as e:
|
|
62
84
|
logger.warning(
|
|
63
85
|
"Could not retrieve Transit Gateway VPC Attachments due to boto3 error %s: %s. Skipping.",
|
|
64
|
-
e.response[
|
|
65
|
-
e.response[
|
|
86
|
+
e.response["Error"]["Code"],
|
|
87
|
+
e.response["Error"]["Message"],
|
|
66
88
|
)
|
|
67
89
|
return tgw_vpc_attachments
|
|
68
90
|
|
|
69
91
|
|
|
70
92
|
@timeit
|
|
71
93
|
def load_transit_gateways(
|
|
72
|
-
neo4j_session: neo4j.Session,
|
|
94
|
+
neo4j_session: neo4j.Session,
|
|
95
|
+
data: List[Dict],
|
|
96
|
+
region: str,
|
|
97
|
+
current_aws_account_id: str,
|
|
73
98
|
update_tag: int,
|
|
74
99
|
) -> None:
|
|
75
100
|
ingest_transit_gateway = """
|
|
@@ -107,13 +132,21 @@ def load_transit_gateways(
|
|
|
107
132
|
update_tag=update_tag,
|
|
108
133
|
)
|
|
109
134
|
_attach_shared_transit_gateway(
|
|
110
|
-
neo4j_session,
|
|
135
|
+
neo4j_session,
|
|
136
|
+
tgw,
|
|
137
|
+
region,
|
|
138
|
+
current_aws_account_id,
|
|
139
|
+
update_tag,
|
|
111
140
|
)
|
|
112
141
|
|
|
113
142
|
|
|
114
143
|
@timeit
|
|
115
144
|
def _attach_shared_transit_gateway(
|
|
116
|
-
neo4j_session: neo4j.Session,
|
|
145
|
+
neo4j_session: neo4j.Session,
|
|
146
|
+
tgw: Dict,
|
|
147
|
+
region: str,
|
|
148
|
+
current_aws_account_id: str,
|
|
149
|
+
update_tag: int,
|
|
117
150
|
) -> None:
|
|
118
151
|
attach_tgw = """
|
|
119
152
|
MERGE (tgw:AWSTransitGateway {id: $ARN})
|
|
@@ -139,7 +172,10 @@ def _attach_shared_transit_gateway(
|
|
|
139
172
|
|
|
140
173
|
@timeit
|
|
141
174
|
def load_tgw_attachments(
|
|
142
|
-
neo4j_session: neo4j.Session,
|
|
175
|
+
neo4j_session: neo4j.Session,
|
|
176
|
+
data: List[Dict],
|
|
177
|
+
region: str,
|
|
178
|
+
current_aws_account_id: str,
|
|
143
179
|
update_tag: int,
|
|
144
180
|
) -> None:
|
|
145
181
|
ingest_transit_gateway = """
|
|
@@ -178,16 +214,25 @@ def load_tgw_attachments(
|
|
|
178
214
|
update_tag=update_tag,
|
|
179
215
|
)
|
|
180
216
|
|
|
181
|
-
if tgwa.get(
|
|
217
|
+
if tgwa.get(
|
|
218
|
+
"VpcId",
|
|
219
|
+
): # only attach if the TGW attachment is a VPC TGW attachment
|
|
182
220
|
_attach_tgw_vpc_attachment_to_vpc_subnets(
|
|
183
|
-
neo4j_session,
|
|
221
|
+
neo4j_session,
|
|
222
|
+
tgwa,
|
|
223
|
+
region,
|
|
224
|
+
current_aws_account_id,
|
|
225
|
+
update_tag,
|
|
184
226
|
)
|
|
185
227
|
|
|
186
228
|
|
|
187
229
|
@timeit
|
|
188
230
|
def _attach_tgw_vpc_attachment_to_vpc_subnets(
|
|
189
|
-
neo4j_session: neo4j.Session,
|
|
190
|
-
|
|
231
|
+
neo4j_session: neo4j.Session,
|
|
232
|
+
tgw_vpc_attachment: Dict,
|
|
233
|
+
region: str,
|
|
234
|
+
current_aws_account_id: str,
|
|
235
|
+
update_tag: int,
|
|
191
236
|
) -> None:
|
|
192
237
|
"""
|
|
193
238
|
Attach a VPC Transit Gateway Attachment to the VPC and and subnets
|
|
@@ -233,27 +278,48 @@ def _attach_tgw_vpc_attachment_to_vpc_subnets(
|
|
|
233
278
|
|
|
234
279
|
|
|
235
280
|
@timeit
|
|
236
|
-
def cleanup_transit_gateways(
|
|
237
|
-
|
|
281
|
+
def cleanup_transit_gateways(
|
|
282
|
+
neo4j_session: neo4j.Session,
|
|
283
|
+
common_job_parameters: Dict,
|
|
284
|
+
) -> None:
|
|
285
|
+
run_cleanup_job("aws_import_tgw_cleanup.json", neo4j_session, common_job_parameters)
|
|
238
286
|
|
|
239
287
|
|
|
240
288
|
def sync_transit_gateways(
|
|
241
|
-
neo4j_session: neo4j.Session,
|
|
242
|
-
|
|
289
|
+
neo4j_session: neo4j.Session,
|
|
290
|
+
boto3_session: boto3.session.Session,
|
|
291
|
+
regions: List[str],
|
|
292
|
+
current_aws_account_id: str,
|
|
293
|
+
update_tag: int,
|
|
294
|
+
common_job_parameters: Dict,
|
|
243
295
|
) -> None:
|
|
244
296
|
for region in regions:
|
|
245
|
-
logger.info(
|
|
297
|
+
logger.info(
|
|
298
|
+
"Syncing AWS Transit Gateways for region '%s' in account '%s'.",
|
|
299
|
+
region,
|
|
300
|
+
current_aws_account_id,
|
|
301
|
+
)
|
|
246
302
|
tgws = get_transit_gateways(boto3_session, region)
|
|
247
|
-
load_transit_gateways(
|
|
303
|
+
load_transit_gateways(
|
|
304
|
+
neo4j_session,
|
|
305
|
+
tgws,
|
|
306
|
+
region,
|
|
307
|
+
current_aws_account_id,
|
|
308
|
+
update_tag,
|
|
309
|
+
)
|
|
248
310
|
|
|
249
311
|
logger.debug(
|
|
250
312
|
"Syncing AWS Transit Gateway Attachments for region '%s' in account '%s'.",
|
|
251
|
-
region,
|
|
313
|
+
region,
|
|
314
|
+
current_aws_account_id,
|
|
252
315
|
)
|
|
253
316
|
tgw_attachments = get_tgw_attachments(boto3_session, region)
|
|
254
317
|
tgw_vpc_attachments = get_tgw_vpc_attachments(boto3_session, region)
|
|
255
318
|
load_tgw_attachments(
|
|
256
|
-
neo4j_session,
|
|
257
|
-
|
|
319
|
+
neo4j_session,
|
|
320
|
+
tgw_attachments + tgw_vpc_attachments,
|
|
321
|
+
region,
|
|
322
|
+
current_aws_account_id,
|
|
323
|
+
update_tag,
|
|
258
324
|
)
|
|
259
325
|
cleanup_transit_gateways(neo4j_session, common_job_parameters)
|
|
@@ -18,38 +18,51 @@ logger = logging.getLogger(__name__)
|
|
|
18
18
|
|
|
19
19
|
@timeit
|
|
20
20
|
@aws_handle_regions
|
|
21
|
-
def get_volumes(
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
def get_volumes(
|
|
22
|
+
boto3_session: boto3.session.Session,
|
|
23
|
+
region: str,
|
|
24
|
+
) -> List[Dict[str, Any]]:
|
|
25
|
+
client = boto3_session.client("ec2", region_name=region)
|
|
26
|
+
paginator = client.get_paginator("describe_volumes")
|
|
24
27
|
volumes: List[Dict] = []
|
|
25
28
|
for page in paginator.paginate():
|
|
26
|
-
volumes.extend(page[
|
|
29
|
+
volumes.extend(page["Volumes"])
|
|
27
30
|
return volumes
|
|
28
31
|
|
|
29
32
|
|
|
30
|
-
def transform_volumes(
|
|
33
|
+
def transform_volumes(
|
|
34
|
+
volumes: List[Dict[str, Any]],
|
|
35
|
+
region: str,
|
|
36
|
+
current_aws_account_id: str,
|
|
37
|
+
) -> List[Dict[str, Any]]:
|
|
31
38
|
result = []
|
|
32
39
|
for volume in volumes:
|
|
33
|
-
attachments = volume.get(
|
|
34
|
-
active_attachments = [a for a in attachments if a[
|
|
35
|
-
|
|
36
|
-
volume_id = volume[
|
|
37
|
-
raw_vol =
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
40
|
+
attachments = volume.get("Attachments", [])
|
|
41
|
+
active_attachments = [a for a in attachments if a["State"] == "attached"]
|
|
42
|
+
|
|
43
|
+
volume_id = volume["VolumeId"]
|
|
44
|
+
raw_vol = {
|
|
45
|
+
"Arn": build_arn(
|
|
46
|
+
"ec2",
|
|
47
|
+
current_aws_account_id,
|
|
48
|
+
"volume",
|
|
49
|
+
volume_id,
|
|
50
|
+
region,
|
|
51
|
+
),
|
|
52
|
+
"AvailabilityZone": volume.get("AvailabilityZone"),
|
|
53
|
+
"CreateTime": volume.get("CreateTime"),
|
|
54
|
+
"Encrypted": volume.get("Encrypted"),
|
|
55
|
+
"Size": volume.get("Size"),
|
|
56
|
+
"State": volume.get("State"),
|
|
57
|
+
"OutpostArn": volume.get("OutpostArn"),
|
|
58
|
+
"SnapshotId": volume.get("SnapshotId"),
|
|
59
|
+
"Iops": volume.get("Iops"),
|
|
60
|
+
"FastRestored": volume.get("FastRestored"),
|
|
61
|
+
"MultiAttachEnabled": volume.get("MultiAttachEnabled"),
|
|
62
|
+
"VolumeType": volume.get("VolumeType"),
|
|
63
|
+
"VolumeId": volume_id,
|
|
64
|
+
"KmsKeyId": volume.get("KmsKeyId"),
|
|
65
|
+
}
|
|
53
66
|
|
|
54
67
|
if not active_attachments:
|
|
55
68
|
result.append(raw_vol)
|
|
@@ -57,7 +70,7 @@ def transform_volumes(volumes: List[Dict[str, Any]], region: str, current_aws_ac
|
|
|
57
70
|
|
|
58
71
|
for attachment in active_attachments:
|
|
59
72
|
vol_with_attachment = raw_vol.copy()
|
|
60
|
-
vol_with_attachment[
|
|
73
|
+
vol_with_attachment["InstanceId"] = attachment["InstanceId"]
|
|
61
74
|
result.append(vol_with_attachment)
|
|
62
75
|
|
|
63
76
|
return result
|
|
@@ -65,11 +78,11 @@ def transform_volumes(volumes: List[Dict[str, Any]], region: str, current_aws_ac
|
|
|
65
78
|
|
|
66
79
|
@timeit
|
|
67
80
|
def load_volumes(
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
81
|
+
neo4j_session: neo4j.Session,
|
|
82
|
+
ebs_data: List[Dict[str, Any]],
|
|
83
|
+
region: str,
|
|
84
|
+
current_aws_account_id: str,
|
|
85
|
+
update_tag: int,
|
|
73
86
|
) -> None:
|
|
74
87
|
load(
|
|
75
88
|
neo4j_session,
|
|
@@ -82,22 +95,37 @@ def load_volumes(
|
|
|
82
95
|
|
|
83
96
|
|
|
84
97
|
@timeit
|
|
85
|
-
def cleanup_volumes(
|
|
86
|
-
|
|
98
|
+
def cleanup_volumes(
|
|
99
|
+
neo4j_session: neo4j.Session,
|
|
100
|
+
common_job_parameters: Dict[str, Any],
|
|
101
|
+
) -> None:
|
|
102
|
+
GraphJob.from_node_schema(EBSVolumeSchema(), common_job_parameters).run(
|
|
103
|
+
neo4j_session,
|
|
104
|
+
)
|
|
87
105
|
|
|
88
106
|
|
|
89
107
|
@timeit
|
|
90
108
|
def sync_ebs_volumes(
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
109
|
+
neo4j_session: neo4j.Session,
|
|
110
|
+
boto3_session: boto3.session.Session,
|
|
111
|
+
regions: List[str],
|
|
112
|
+
current_aws_account_id: str,
|
|
113
|
+
update_tag: int,
|
|
114
|
+
common_job_parameters: Dict[str, Any],
|
|
97
115
|
) -> None:
|
|
98
116
|
for region in regions:
|
|
99
|
-
logger.debug(
|
|
117
|
+
logger.debug(
|
|
118
|
+
"Syncing volumes for region '%s' in account '%s'.",
|
|
119
|
+
region,
|
|
120
|
+
current_aws_account_id,
|
|
121
|
+
)
|
|
100
122
|
data = get_volumes(boto3_session, region)
|
|
101
123
|
transformed_data = transform_volumes(data, region, current_aws_account_id)
|
|
102
|
-
load_volumes(
|
|
124
|
+
load_volumes(
|
|
125
|
+
neo4j_session,
|
|
126
|
+
transformed_data,
|
|
127
|
+
region,
|
|
128
|
+
current_aws_account_id,
|
|
129
|
+
update_tag,
|
|
130
|
+
)
|
|
103
131
|
cleanup_volumes(neo4j_session, common_job_parameters)
|