cartography 0.108.0rc1__py3-none-any.whl → 0.108.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/_version.py +2 -2
- cartography/intel/aws/cloudtrail.py +17 -4
- cartography/intel/aws/cloudtrail_management_events.py +560 -16
- cartography/intel/aws/cloudwatch.py +73 -4
- cartography/intel/aws/ec2/subnets.py +37 -63
- cartography/intel/aws/elasticache.py +102 -79
- cartography/models/aws/cloudtrail/management_events.py +95 -6
- cartography/models/aws/cloudtrail/trail.py +21 -0
- cartography/models/aws/cloudwatch/metric_alarm.py +53 -0
- cartography/models/aws/ec2/subnets.py +65 -0
- cartography/models/aws/elasticache/__init__.py +0 -0
- cartography/models/aws/elasticache/cluster.py +65 -0
- cartography/models/aws/elasticache/topic.py +67 -0
- {cartography-0.108.0rc1.dist-info → cartography-0.108.0rc2.dist-info}/METADATA +1 -1
- {cartography-0.108.0rc1.dist-info → cartography-0.108.0rc2.dist-info}/RECORD +19 -14
- {cartography-0.108.0rc1.dist-info → cartography-0.108.0rc2.dist-info}/WHEEL +0 -0
- {cartography-0.108.0rc1.dist-info → cartography-0.108.0rc2.dist-info}/entry_points.txt +0 -0
- {cartography-0.108.0rc1.dist-info → cartography-0.108.0rc2.dist-info}/licenses/LICENSE +0 -0
- {cartography-0.108.0rc1.dist-info → cartography-0.108.0rc2.dist-info}/top_level.txt +0 -0
|
@@ -13,6 +13,7 @@ from cartography.models.aws.cloudwatch.log_metric_filter import (
|
|
|
13
13
|
CloudWatchLogMetricFilterSchema,
|
|
14
14
|
)
|
|
15
15
|
from cartography.models.aws.cloudwatch.loggroup import CloudWatchLogGroupSchema
|
|
16
|
+
from cartography.models.aws.cloudwatch.metric_alarm import CloudWatchMetricAlarmSchema
|
|
16
17
|
from cartography.util import aws_handle_regions
|
|
17
18
|
from cartography.util import timeit
|
|
18
19
|
|
|
@@ -75,6 +76,43 @@ def transform_metric_filters(
|
|
|
75
76
|
return transformed_filters
|
|
76
77
|
|
|
77
78
|
|
|
79
|
+
@timeit
|
|
80
|
+
@aws_handle_regions
|
|
81
|
+
def get_cloudwatch_metric_alarms(
|
|
82
|
+
boto3_session: boto3.Session, region: str
|
|
83
|
+
) -> List[Dict[str, Any]]:
|
|
84
|
+
client = boto3_session.client(
|
|
85
|
+
"cloudwatch", region_name=region, config=get_botocore_config()
|
|
86
|
+
)
|
|
87
|
+
paginator = client.get_paginator("describe_alarms")
|
|
88
|
+
alarms = []
|
|
89
|
+
for page in paginator.paginate():
|
|
90
|
+
alarms.extend(page["MetricAlarms"])
|
|
91
|
+
return alarms
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def transform_metric_alarms(
|
|
95
|
+
metric_alarms: List[Dict[str, Any]], region: str
|
|
96
|
+
) -> List[Dict[str, Any]]:
|
|
97
|
+
"""
|
|
98
|
+
Transform CloudWatch metric alarm data for ingestion into Neo4j.
|
|
99
|
+
"""
|
|
100
|
+
transformed_alarms = []
|
|
101
|
+
for alarm in metric_alarms:
|
|
102
|
+
transformed_alarm = {
|
|
103
|
+
"AlarmArn": alarm["AlarmArn"],
|
|
104
|
+
"AlarmName": alarm.get("AlarmName"),
|
|
105
|
+
"AlarmDescription": alarm.get("AlarmDescription"),
|
|
106
|
+
"StateValue": alarm.get("StateValue"),
|
|
107
|
+
"StateReason": alarm.get("StateReason"),
|
|
108
|
+
"ActionsEnabled": alarm.get("ActionsEnabled"),
|
|
109
|
+
"ComparisonOperator": alarm.get("ComparisonOperator"),
|
|
110
|
+
"Region": region,
|
|
111
|
+
}
|
|
112
|
+
transformed_alarms.append(transformed_alarm)
|
|
113
|
+
return transformed_alarms
|
|
114
|
+
|
|
115
|
+
|
|
78
116
|
@timeit
|
|
79
117
|
def load_cloudwatch_log_groups(
|
|
80
118
|
neo4j_session: neo4j.Session,
|
|
@@ -117,6 +155,27 @@ def load_cloudwatch_log_metric_filters(
|
|
|
117
155
|
)
|
|
118
156
|
|
|
119
157
|
|
|
158
|
+
@timeit
|
|
159
|
+
def load_cloudwatch_metric_alarms(
|
|
160
|
+
neo4j_session: neo4j.Session,
|
|
161
|
+
data: List[Dict[str, Any]],
|
|
162
|
+
region: str,
|
|
163
|
+
current_aws_account_id: str,
|
|
164
|
+
aws_update_tag: int,
|
|
165
|
+
) -> None:
|
|
166
|
+
logger.info(
|
|
167
|
+
f"Loading CloudWatch {len(data)} metric alarms for region '{region}' into graph.",
|
|
168
|
+
)
|
|
169
|
+
load(
|
|
170
|
+
neo4j_session,
|
|
171
|
+
CloudWatchMetricAlarmSchema(),
|
|
172
|
+
data,
|
|
173
|
+
lastupdated=aws_update_tag,
|
|
174
|
+
Region=region,
|
|
175
|
+
AWS_ID=current_aws_account_id,
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
120
179
|
@timeit
|
|
121
180
|
def cleanup(
|
|
122
181
|
neo4j_session: neo4j.Session,
|
|
@@ -130,6 +189,9 @@ def cleanup(
|
|
|
130
189
|
GraphJob.from_node_schema(
|
|
131
190
|
CloudWatchLogMetricFilterSchema(), common_job_parameters
|
|
132
191
|
).run(neo4j_session)
|
|
192
|
+
GraphJob.from_node_schema(CloudWatchMetricAlarmSchema(), common_job_parameters).run(
|
|
193
|
+
neo4j_session
|
|
194
|
+
)
|
|
133
195
|
|
|
134
196
|
|
|
135
197
|
@timeit
|
|
@@ -146,13 +208,10 @@ def sync(
|
|
|
146
208
|
f"Syncing CloudWatch for region '{region}' in account '{current_aws_account_id}'.",
|
|
147
209
|
)
|
|
148
210
|
logGroups = get_cloudwatch_log_groups(boto3_session, region)
|
|
149
|
-
group_data: List[Dict[str, Any]] = []
|
|
150
|
-
for logGroup in logGroups:
|
|
151
|
-
group_data.append(logGroup)
|
|
152
211
|
|
|
153
212
|
load_cloudwatch_log_groups(
|
|
154
213
|
neo4j_session,
|
|
155
|
-
|
|
214
|
+
logGroups,
|
|
156
215
|
region,
|
|
157
216
|
current_aws_account_id,
|
|
158
217
|
update_tag,
|
|
@@ -167,4 +226,14 @@ def sync(
|
|
|
167
226
|
current_aws_account_id,
|
|
168
227
|
update_tag,
|
|
169
228
|
)
|
|
229
|
+
|
|
230
|
+
metric_alarms = get_cloudwatch_metric_alarms(boto3_session, region)
|
|
231
|
+
transformed_alarms = transform_metric_alarms(metric_alarms, region)
|
|
232
|
+
load_cloudwatch_metric_alarms(
|
|
233
|
+
neo4j_session,
|
|
234
|
+
transformed_alarms,
|
|
235
|
+
region,
|
|
236
|
+
current_aws_account_id,
|
|
237
|
+
update_tag,
|
|
238
|
+
)
|
|
170
239
|
cleanup(neo4j_session, common_job_parameters)
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import
|
|
3
|
-
from typing import List
|
|
2
|
+
from typing import Any
|
|
4
3
|
|
|
5
4
|
import boto3
|
|
6
5
|
import neo4j
|
|
7
6
|
|
|
7
|
+
from cartography.client.core.tx import load
|
|
8
8
|
from cartography.graph.job import GraphJob
|
|
9
9
|
from cartography.models.aws.ec2.auto_scaling_groups import (
|
|
10
10
|
EC2SubnetAutoScalingGroupSchema,
|
|
11
11
|
)
|
|
12
12
|
from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema
|
|
13
|
+
from cartography.models.aws.ec2.subnets import EC2SubnetSchema
|
|
13
14
|
from cartography.util import aws_handle_regions
|
|
14
|
-
from cartography.util import run_cleanup_job
|
|
15
15
|
from cartography.util import timeit
|
|
16
16
|
|
|
17
17
|
from .util import get_botocore_config
|
|
@@ -21,86 +21,53 @@ logger = logging.getLogger(__name__)
|
|
|
21
21
|
|
|
22
22
|
@timeit
|
|
23
23
|
@aws_handle_regions
|
|
24
|
-
def get_subnet_data(
|
|
24
|
+
def get_subnet_data(
|
|
25
|
+
boto3_session: boto3.session.Session, region: str
|
|
26
|
+
) -> list[dict[str, Any]]:
|
|
25
27
|
client = boto3_session.client(
|
|
26
28
|
"ec2",
|
|
27
29
|
region_name=region,
|
|
28
30
|
config=get_botocore_config(),
|
|
29
31
|
)
|
|
30
32
|
paginator = client.get_paginator("describe_subnets")
|
|
31
|
-
subnets:
|
|
33
|
+
subnets: list[dict[str, Any]] = []
|
|
32
34
|
for page in paginator.paginate():
|
|
33
35
|
subnets.extend(page["Subnets"])
|
|
34
36
|
return subnets
|
|
35
37
|
|
|
36
38
|
|
|
39
|
+
def transform_subnet_data(subnets: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
40
|
+
"""Transform subnet data into a loadable format."""
|
|
41
|
+
transformed: list[dict[str, Any]] = []
|
|
42
|
+
for subnet in subnets:
|
|
43
|
+
transformed.append(subnet.copy())
|
|
44
|
+
return transformed
|
|
45
|
+
|
|
46
|
+
|
|
37
47
|
@timeit
|
|
38
48
|
def load_subnets(
|
|
39
49
|
neo4j_session: neo4j.Session,
|
|
40
|
-
data:
|
|
50
|
+
data: list[dict[str, Any]],
|
|
41
51
|
region: str,
|
|
42
52
|
aws_account_id: str,
|
|
43
53
|
aws_update_tag: int,
|
|
44
54
|
) -> None:
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
snet.map_customer_owned_ip_on_launch = subnet.MapCustomerOwnedIpOnLaunch,
|
|
53
|
-
snet.state = subnet.State, snet.assignipv6addressoncreation = subnet.AssignIpv6AddressOnCreation,
|
|
54
|
-
snet.map_public_ip_on_launch = subnet.MapPublicIpOnLaunch, snet.subnet_arn = subnet.SubnetArn,
|
|
55
|
-
snet.availability_zone = subnet.AvailabilityZone, snet.availability_zone_id = subnet.AvailabilityZoneId,
|
|
56
|
-
snet.subnet_id = subnet.SubnetId
|
|
57
|
-
"""
|
|
58
|
-
|
|
59
|
-
ingest_subnet_vpc_relations = """
|
|
60
|
-
UNWIND $subnets as subnet
|
|
61
|
-
MATCH (snet:EC2Subnet{subnetid: subnet.SubnetId}), (vpc:AWSVpc{id: subnet.VpcId})
|
|
62
|
-
MERGE (snet)-[r:MEMBER_OF_AWS_VPC]->(vpc)
|
|
63
|
-
ON CREATE SET r.firstseen = timestamp()
|
|
64
|
-
SET r.lastupdated = $aws_update_tag
|
|
65
|
-
"""
|
|
66
|
-
|
|
67
|
-
ingest_subnet_aws_account_relations = """
|
|
68
|
-
UNWIND $subnets as subnet
|
|
69
|
-
MATCH (snet:EC2Subnet{subnetid: subnet.SubnetId}), (aws:AWSAccount{id: $aws_account_id})
|
|
70
|
-
MERGE (aws)-[r:RESOURCE]->(snet)
|
|
71
|
-
ON CREATE SET r.firstseen = timestamp()
|
|
72
|
-
SET r.lastupdated = $aws_update_tag
|
|
73
|
-
"""
|
|
74
|
-
|
|
75
|
-
neo4j_session.run(
|
|
76
|
-
ingest_subnets,
|
|
77
|
-
subnets=data,
|
|
78
|
-
aws_update_tag=aws_update_tag,
|
|
79
|
-
region=region,
|
|
80
|
-
aws_account_id=aws_account_id,
|
|
81
|
-
)
|
|
82
|
-
neo4j_session.run(
|
|
83
|
-
ingest_subnet_vpc_relations,
|
|
84
|
-
subnets=data,
|
|
85
|
-
aws_update_tag=aws_update_tag,
|
|
86
|
-
region=region,
|
|
87
|
-
aws_account_id=aws_account_id,
|
|
88
|
-
)
|
|
89
|
-
neo4j_session.run(
|
|
90
|
-
ingest_subnet_aws_account_relations,
|
|
91
|
-
subnets=data,
|
|
92
|
-
aws_update_tag=aws_update_tag,
|
|
93
|
-
region=region,
|
|
94
|
-
aws_account_id=aws_account_id,
|
|
55
|
+
load(
|
|
56
|
+
neo4j_session,
|
|
57
|
+
EC2SubnetSchema(),
|
|
58
|
+
data,
|
|
59
|
+
Region=region,
|
|
60
|
+
AWS_ID=aws_account_id,
|
|
61
|
+
lastupdated=aws_update_tag,
|
|
95
62
|
)
|
|
96
63
|
|
|
97
64
|
|
|
98
65
|
@timeit
|
|
99
|
-
def cleanup_subnets(
|
|
100
|
-
|
|
101
|
-
|
|
66
|
+
def cleanup_subnets(
|
|
67
|
+
neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]
|
|
68
|
+
) -> None:
|
|
69
|
+
GraphJob.from_node_schema(EC2SubnetSchema(), common_job_parameters).run(
|
|
102
70
|
neo4j_session,
|
|
103
|
-
common_job_parameters,
|
|
104
71
|
)
|
|
105
72
|
GraphJob.from_node_schema(EC2SubnetInstanceSchema(), common_job_parameters).run(
|
|
106
73
|
neo4j_session,
|
|
@@ -115,10 +82,10 @@ def cleanup_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -
|
|
|
115
82
|
def sync_subnets(
|
|
116
83
|
neo4j_session: neo4j.Session,
|
|
117
84
|
boto3_session: boto3.session.Session,
|
|
118
|
-
regions:
|
|
85
|
+
regions: list[str],
|
|
119
86
|
current_aws_account_id: str,
|
|
120
87
|
update_tag: int,
|
|
121
|
-
common_job_parameters:
|
|
88
|
+
common_job_parameters: dict[str, Any],
|
|
122
89
|
) -> None:
|
|
123
90
|
for region in regions:
|
|
124
91
|
logger.info(
|
|
@@ -127,5 +94,12 @@ def sync_subnets(
|
|
|
127
94
|
current_aws_account_id,
|
|
128
95
|
)
|
|
129
96
|
data = get_subnet_data(boto3_session, region)
|
|
130
|
-
|
|
97
|
+
transformed = transform_subnet_data(data)
|
|
98
|
+
load_subnets(
|
|
99
|
+
neo4j_session,
|
|
100
|
+
transformed,
|
|
101
|
+
region,
|
|
102
|
+
current_aws_account_id,
|
|
103
|
+
update_tag,
|
|
104
|
+
)
|
|
131
105
|
cleanup_subnets(neo4j_session, common_job_parameters)
|
|
@@ -1,118 +1,132 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import
|
|
3
|
-
from typing import List
|
|
4
|
-
from typing import Set
|
|
2
|
+
from typing import Any
|
|
5
3
|
|
|
6
4
|
import boto3
|
|
7
5
|
import neo4j
|
|
8
6
|
|
|
7
|
+
from cartography.client.core.tx import load
|
|
8
|
+
from cartography.graph.job import GraphJob
|
|
9
|
+
from cartography.models.aws.elasticache.cluster import ElasticacheClusterSchema
|
|
10
|
+
from cartography.models.aws.elasticache.topic import ElasticacheTopicSchema
|
|
9
11
|
from cartography.stats import get_stats_client
|
|
10
12
|
from cartography.util import aws_handle_regions
|
|
11
13
|
from cartography.util import merge_module_sync_metadata
|
|
12
|
-
from cartography.util import run_cleanup_job
|
|
13
14
|
from cartography.util import timeit
|
|
14
15
|
|
|
15
16
|
logger = logging.getLogger(__name__)
|
|
16
17
|
stat_handler = get_stats_client(__name__)
|
|
17
18
|
|
|
18
19
|
|
|
19
|
-
def _get_topic(cluster: Dict) -> Dict:
|
|
20
|
-
return cluster["NotificationConfiguration"]
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def transform_elasticache_topics(cluster_data: List[Dict]) -> List[Dict]:
|
|
24
|
-
"""
|
|
25
|
-
Collect unique TopicArns from the cluster data
|
|
26
|
-
"""
|
|
27
|
-
seen: Set[str] = set()
|
|
28
|
-
topics: List[Dict] = []
|
|
29
|
-
for cluster in cluster_data:
|
|
30
|
-
topic = _get_topic(cluster)
|
|
31
|
-
topic_arn = topic["TopicArn"]
|
|
32
|
-
if topic_arn not in seen:
|
|
33
|
-
seen.add(topic_arn)
|
|
34
|
-
topics.append(topic)
|
|
35
|
-
return topics
|
|
36
|
-
|
|
37
|
-
|
|
38
20
|
@timeit
|
|
39
21
|
@aws_handle_regions
|
|
40
22
|
def get_elasticache_clusters(
|
|
41
23
|
boto3_session: boto3.session.Session,
|
|
42
24
|
region: str,
|
|
43
|
-
) ->
|
|
44
|
-
logger.debug(f"Getting ElastiCache Clusters in region '{region}'.")
|
|
25
|
+
) -> list[dict[str, Any]]:
|
|
45
26
|
client = boto3_session.client("elasticache", region_name=region)
|
|
46
27
|
paginator = client.get_paginator("describe_cache_clusters")
|
|
47
|
-
clusters:
|
|
28
|
+
clusters: list[dict[str, Any]] = []
|
|
48
29
|
for page in paginator.paginate():
|
|
49
|
-
clusters.extend(page
|
|
30
|
+
clusters.extend(page.get("CacheClusters", []))
|
|
50
31
|
return clusters
|
|
51
32
|
|
|
52
33
|
|
|
34
|
+
def transform_elasticache_clusters(
|
|
35
|
+
clusters: list[dict[str, Any]], region: str
|
|
36
|
+
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
37
|
+
cluster_data: list[dict[str, Any]] = []
|
|
38
|
+
topics: dict[str, dict[str, Any]] = {}
|
|
39
|
+
|
|
40
|
+
for cluster in clusters:
|
|
41
|
+
notification = cluster.get("NotificationConfiguration", {})
|
|
42
|
+
topic_arn = notification.get("TopicArn")
|
|
43
|
+
cluster_record = {
|
|
44
|
+
"ARN": cluster["ARN"],
|
|
45
|
+
"CacheClusterId": cluster["CacheClusterId"],
|
|
46
|
+
"CacheNodeType": cluster.get("CacheNodeType"),
|
|
47
|
+
"Engine": cluster.get("Engine"),
|
|
48
|
+
"EngineVersion": cluster.get("EngineVersion"),
|
|
49
|
+
"CacheClusterStatus": cluster.get("CacheClusterStatus"),
|
|
50
|
+
"NumCacheNodes": cluster.get("NumCacheNodes"),
|
|
51
|
+
"PreferredAvailabilityZone": cluster.get("PreferredAvailabilityZone"),
|
|
52
|
+
"PreferredMaintenanceWindow": cluster.get("PreferredMaintenanceWindow"),
|
|
53
|
+
"CacheClusterCreateTime": cluster.get("CacheClusterCreateTime"),
|
|
54
|
+
"CacheSubnetGroupName": cluster.get("CacheSubnetGroupName"),
|
|
55
|
+
"AutoMinorVersionUpgrade": cluster.get("AutoMinorVersionUpgrade"),
|
|
56
|
+
"ReplicationGroupId": cluster.get("ReplicationGroupId"),
|
|
57
|
+
"SnapshotRetentionLimit": cluster.get("SnapshotRetentionLimit"),
|
|
58
|
+
"SnapshotWindow": cluster.get("SnapshotWindow"),
|
|
59
|
+
"AuthTokenEnabled": cluster.get("AuthTokenEnabled"),
|
|
60
|
+
"TransitEncryptionEnabled": cluster.get("TransitEncryptionEnabled"),
|
|
61
|
+
"AtRestEncryptionEnabled": cluster.get("AtRestEncryptionEnabled"),
|
|
62
|
+
"TopicArn": topic_arn,
|
|
63
|
+
"Region": region,
|
|
64
|
+
}
|
|
65
|
+
cluster_data.append(cluster_record)
|
|
66
|
+
|
|
67
|
+
if topic_arn:
|
|
68
|
+
topics.setdefault(
|
|
69
|
+
topic_arn,
|
|
70
|
+
{
|
|
71
|
+
"TopicArn": topic_arn,
|
|
72
|
+
"TopicStatus": notification.get("TopicStatus"),
|
|
73
|
+
"cluster_arns": [],
|
|
74
|
+
},
|
|
75
|
+
)["cluster_arns"].append(cluster["ARN"])
|
|
76
|
+
|
|
77
|
+
return cluster_data, list(topics.values())
|
|
78
|
+
|
|
79
|
+
|
|
53
80
|
@timeit
|
|
54
81
|
def load_elasticache_clusters(
|
|
55
82
|
neo4j_session: neo4j.Session,
|
|
56
|
-
clusters:
|
|
83
|
+
clusters: list[dict[str, Any]],
|
|
57
84
|
region: str,
|
|
58
85
|
aws_account_id: str,
|
|
59
86
|
update_tag: int,
|
|
60
87
|
) -> None:
|
|
61
|
-
query = """
|
|
62
|
-
UNWIND $clusters as elasticache_cluster
|
|
63
|
-
MERGE (cluster:ElasticacheCluster{id:elasticache_cluster.ARN})
|
|
64
|
-
ON CREATE SET cluster.firstseen = timestamp(),
|
|
65
|
-
cluster.arn = elasticache_cluster.ARN,
|
|
66
|
-
cluster.topic_arn = elasticache_cluster.NotificationConfiguration.TopicArn,
|
|
67
|
-
cluster.id = elasticache_cluster.CacheClusterId,
|
|
68
|
-
cluster.region = $region
|
|
69
|
-
SET cluster.lastupdated = $aws_update_tag
|
|
70
|
-
|
|
71
|
-
WITH cluster, elasticache_cluster
|
|
72
|
-
MATCH (owner:AWSAccount{id: $aws_account_id})
|
|
73
|
-
MERGE (owner)-[r3:RESOURCE]->(cluster)
|
|
74
|
-
ON CREATE SET r3.firstseen = timestamp()
|
|
75
|
-
SET r3.lastupdated = $aws_update_tag
|
|
76
|
-
|
|
77
|
-
WITH elasticache_cluster, owner
|
|
78
|
-
WHERE NOT elasticache_cluster.NotificationConfiguration IS NULL
|
|
79
|
-
MERGE (topic:ElasticacheTopic{id: elasticache_cluster.NotificationConfiguration.TopicArn})
|
|
80
|
-
ON CREATE SET topic.firstseen = timestamp(),
|
|
81
|
-
topic.arn = elasticache_cluster.NotificationConfiguration.TopicArn
|
|
82
|
-
SET topic.lastupdated = $aws_update_tag,
|
|
83
|
-
topic.status = elasticache_cluster.NotificationConfiguration.Status
|
|
84
|
-
|
|
85
|
-
MERGE (topic)-[r:CACHE_CLUSTER]->(cluster)
|
|
86
|
-
ON CREATE SET r.firstseen = timestamp()
|
|
87
|
-
SET r.lastupdated = $aws_update_tag
|
|
88
|
-
WITH cluster, topic
|
|
89
|
-
|
|
90
|
-
MERGE (owner)-[r2:RESOURCE]->(topic)
|
|
91
|
-
ON CREATE SET r2.firstseen = timestamp()
|
|
92
|
-
SET r2.lastupdated = $aws_update_tag
|
|
93
|
-
"""
|
|
94
88
|
logger.info(
|
|
95
|
-
f"Loading
|
|
89
|
+
f"Loading {len(clusters)} ElastiCache clusters for region '{region}' into graph."
|
|
96
90
|
)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
91
|
+
load(
|
|
92
|
+
neo4j_session,
|
|
93
|
+
ElasticacheClusterSchema(),
|
|
94
|
+
clusters,
|
|
95
|
+
lastupdated=update_tag,
|
|
96
|
+
Region=region,
|
|
97
|
+
AWS_ID=aws_account_id,
|
|
103
98
|
)
|
|
104
99
|
|
|
105
100
|
|
|
106
101
|
@timeit
|
|
107
|
-
def
|
|
102
|
+
def load_elasticache_topics(
|
|
108
103
|
neo4j_session: neo4j.Session,
|
|
109
|
-
|
|
104
|
+
topics: list[dict[str, Any]],
|
|
105
|
+
aws_account_id: str,
|
|
110
106
|
update_tag: int,
|
|
111
107
|
) -> None:
|
|
112
|
-
|
|
113
|
-
|
|
108
|
+
if not topics:
|
|
109
|
+
return
|
|
110
|
+
logger.info(f"Loading {len(topics)} ElastiCache topics into graph.")
|
|
111
|
+
load(
|
|
114
112
|
neo4j_session,
|
|
115
|
-
|
|
113
|
+
ElasticacheTopicSchema(),
|
|
114
|
+
topics,
|
|
115
|
+
lastupdated=update_tag,
|
|
116
|
+
AWS_ID=aws_account_id,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@timeit
|
|
121
|
+
def cleanup(
|
|
122
|
+
neo4j_session: neo4j.Session,
|
|
123
|
+
common_job_parameters: dict[str, Any],
|
|
124
|
+
) -> None:
|
|
125
|
+
GraphJob.from_node_schema(ElasticacheClusterSchema(), common_job_parameters).run(
|
|
126
|
+
neo4j_session
|
|
127
|
+
)
|
|
128
|
+
GraphJob.from_node_schema(ElasticacheTopicSchema(), common_job_parameters).run(
|
|
129
|
+
neo4j_session
|
|
116
130
|
)
|
|
117
131
|
|
|
118
132
|
|
|
@@ -120,24 +134,33 @@ def cleanup(
|
|
|
120
134
|
def sync(
|
|
121
135
|
neo4j_session: neo4j.Session,
|
|
122
136
|
boto3_session: boto3.session.Session,
|
|
123
|
-
regions:
|
|
137
|
+
regions: list[str],
|
|
124
138
|
current_aws_account_id: str,
|
|
125
139
|
update_tag: int,
|
|
126
|
-
common_job_parameters:
|
|
140
|
+
common_job_parameters: dict[str, Any],
|
|
127
141
|
) -> None:
|
|
128
142
|
for region in regions:
|
|
129
143
|
logger.info(
|
|
130
|
-
|
|
144
|
+
"Syncing ElastiCache clusters for region '%s' in account '%s'.",
|
|
145
|
+
region,
|
|
146
|
+
current_aws_account_id,
|
|
131
147
|
)
|
|
132
|
-
|
|
148
|
+
raw_clusters = get_elasticache_clusters(boto3_session, region)
|
|
149
|
+
cluster_data, topic_data = transform_elasticache_clusters(raw_clusters, region)
|
|
133
150
|
load_elasticache_clusters(
|
|
134
151
|
neo4j_session,
|
|
135
|
-
|
|
152
|
+
cluster_data,
|
|
136
153
|
region,
|
|
137
154
|
current_aws_account_id,
|
|
138
155
|
update_tag,
|
|
139
156
|
)
|
|
140
|
-
|
|
157
|
+
load_elasticache_topics(
|
|
158
|
+
neo4j_session,
|
|
159
|
+
topic_data,
|
|
160
|
+
current_aws_account_id,
|
|
161
|
+
update_tag,
|
|
162
|
+
)
|
|
163
|
+
cleanup(neo4j_session, common_job_parameters)
|
|
141
164
|
merge_module_sync_metadata(
|
|
142
165
|
neo4j_session,
|
|
143
166
|
group_type="AWSAccount",
|
|
@@ -29,12 +29,6 @@ class AssumedRoleRelProperties(CartographyRelProperties):
|
|
|
29
29
|
times_used: PropertyRef = PropertyRef("times_used")
|
|
30
30
|
first_seen_in_time_window: PropertyRef = PropertyRef("first_seen_in_time_window")
|
|
31
31
|
|
|
32
|
-
# Event type tracking properties
|
|
33
|
-
event_types: PropertyRef = PropertyRef("event_types")
|
|
34
|
-
assume_role_count: PropertyRef = PropertyRef("assume_role_count")
|
|
35
|
-
saml_count: PropertyRef = PropertyRef("saml_count")
|
|
36
|
-
web_identity_count: PropertyRef = PropertyRef("web_identity_count")
|
|
37
|
-
|
|
38
32
|
|
|
39
33
|
@dataclass(frozen=True)
|
|
40
34
|
class AssumedRoleMatchLink(CartographyRelSchema):
|
|
@@ -62,3 +56,98 @@ class AssumedRoleMatchLink(CartographyRelSchema):
|
|
|
62
56
|
direction: LinkDirection = LinkDirection.OUTWARD
|
|
63
57
|
rel_label: str = "ASSUMED_ROLE"
|
|
64
58
|
properties: AssumedRoleRelProperties = AssumedRoleRelProperties()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True)
|
|
62
|
+
class AssumedRoleWithSAMLRelProperties(CartographyRelProperties):
|
|
63
|
+
"""
|
|
64
|
+
Properties for the ASSUMED_ROLE_WITH_SAML relationship representing SAML-based role assumption events.
|
|
65
|
+
Focuses specifically on SAML federated identity role assumptions.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
# Mandatory fields for MatchLinks
|
|
69
|
+
lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
|
|
70
|
+
_sub_resource_label: PropertyRef = PropertyRef(
|
|
71
|
+
"_sub_resource_label", set_in_kwargs=True
|
|
72
|
+
)
|
|
73
|
+
_sub_resource_id: PropertyRef = PropertyRef("_sub_resource_id", set_in_kwargs=True)
|
|
74
|
+
|
|
75
|
+
# CloudTrail-specific relationship properties
|
|
76
|
+
last_used: PropertyRef = PropertyRef("last_used")
|
|
77
|
+
times_used: PropertyRef = PropertyRef("times_used")
|
|
78
|
+
first_seen_in_time_window: PropertyRef = PropertyRef("first_seen_in_time_window")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass(frozen=True)
|
|
82
|
+
class AssumedRoleWithSAMLMatchLink(CartographyRelSchema):
|
|
83
|
+
"""
|
|
84
|
+
MatchLink schema for ASSUMED_ROLE_WITH_SAML relationships from CloudTrail SAML events.
|
|
85
|
+
Creates relationships like: (AWSRole)-[:ASSUMED_ROLE_WITH_SAML]->(AWSRole)
|
|
86
|
+
|
|
87
|
+
This MatchLink handles SAML-based role assumption relationships discovered via CloudTrail
|
|
88
|
+
AssumeRoleWithSAML events. It creates separate relationships from regular AssumeRole events
|
|
89
|
+
to preserve visibility into authentication methods used.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
# MatchLink-specific fields
|
|
93
|
+
source_node_label: str = "AWSSSOUser" # Match against AWS SSO User nodes
|
|
94
|
+
source_node_matcher: SourceNodeMatcher = make_source_node_matcher(
|
|
95
|
+
{"user_name": PropertyRef("source_principal_arn")},
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
# Standard CartographyRelSchema fields
|
|
99
|
+
target_node_label: str = "AWSRole"
|
|
100
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
101
|
+
{"arn": PropertyRef("destination_principal_arn")},
|
|
102
|
+
)
|
|
103
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
104
|
+
rel_label: str = "ASSUMED_ROLE_WITH_SAML"
|
|
105
|
+
properties: AssumedRoleWithSAMLRelProperties = AssumedRoleWithSAMLRelProperties()
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@dataclass(frozen=True)
|
|
109
|
+
class AssumeRoleWithWebIdentityRelProperties(CartographyRelProperties):
|
|
110
|
+
"""
|
|
111
|
+
Properties for the ASSUMED_ROLE_WITH_WEB_IDENTITY relationship representing web identity-based role assumption events.
|
|
112
|
+
Focuses specifically on web identity federation role assumptions (Google, Amazon, Facebook, etc.).
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
# Mandatory fields for MatchLinks
|
|
116
|
+
lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
|
|
117
|
+
_sub_resource_label: PropertyRef = PropertyRef(
|
|
118
|
+
"_sub_resource_label", set_in_kwargs=True
|
|
119
|
+
)
|
|
120
|
+
_sub_resource_id: PropertyRef = PropertyRef("_sub_resource_id", set_in_kwargs=True)
|
|
121
|
+
|
|
122
|
+
# CloudTrail-specific relationship properties
|
|
123
|
+
last_used: PropertyRef = PropertyRef("last_used")
|
|
124
|
+
times_used: PropertyRef = PropertyRef("times_used")
|
|
125
|
+
first_seen_in_time_window: PropertyRef = PropertyRef("first_seen_in_time_window")
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@dataclass(frozen=True)
|
|
129
|
+
class GitHubRepoAssumeRoleWithWebIdentityMatchLink(CartographyRelSchema):
|
|
130
|
+
"""
|
|
131
|
+
MatchLink schema for ASSUMED_ROLE_WITH_WEB_IDENTITY relationships from GitHub Actions to AWS roles.
|
|
132
|
+
Creates relationships like: (GitHubRepository)-[:ASSUMED_ROLE_WITH_WEB_IDENTITY]->(AWSRole)
|
|
133
|
+
|
|
134
|
+
This MatchLink provides granular visibility into which specific GitHub repositories are assuming
|
|
135
|
+
AWS roles via GitHub Actions OIDC, rather than just showing provider-level relationships.
|
|
136
|
+
"""
|
|
137
|
+
|
|
138
|
+
# MatchLink-specific fields for GitHub repositories
|
|
139
|
+
source_node_label: str = "GitHubRepository"
|
|
140
|
+
source_node_matcher: SourceNodeMatcher = make_source_node_matcher(
|
|
141
|
+
{"fullname": PropertyRef("source_repo_fullname")},
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
# Standard CartographyRelSchema fields
|
|
145
|
+
target_node_label: str = "AWSRole"
|
|
146
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
147
|
+
{"arn": PropertyRef("destination_principal_arn")},
|
|
148
|
+
)
|
|
149
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
150
|
+
rel_label: str = "ASSUMED_ROLE_WITH_WEB_IDENTITY"
|
|
151
|
+
properties: AssumeRoleWithWebIdentityRelProperties = (
|
|
152
|
+
AssumeRoleWithWebIdentityRelProperties()
|
|
153
|
+
)
|
|
@@ -73,6 +73,26 @@ class CloudTrailTrailToS3BucketRel(CartographyRelSchema):
|
|
|
73
73
|
)
|
|
74
74
|
|
|
75
75
|
|
|
76
|
+
@dataclass(frozen=True)
|
|
77
|
+
class CloudTrailTrailToCloudWatchLogGroupRelProperties(CartographyRelProperties):
|
|
78
|
+
lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass(frozen=True)
|
|
82
|
+
class CloudTrailTrailToCloudWatchLogGroupRel(CartographyRelSchema):
|
|
83
|
+
target_node_label: str = "CloudWatchLogGroup"
|
|
84
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
85
|
+
{
|
|
86
|
+
"id": PropertyRef("CloudWatchLogsLogGroupArn"),
|
|
87
|
+
}
|
|
88
|
+
)
|
|
89
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
90
|
+
rel_label: str = "SENDS_LOGS_TO_CLOUDWATCH"
|
|
91
|
+
properties: CloudTrailTrailToCloudWatchLogGroupRelProperties = (
|
|
92
|
+
CloudTrailTrailToCloudWatchLogGroupRelProperties()
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
76
96
|
@dataclass(frozen=True)
|
|
77
97
|
class CloudTrailTrailSchema(CartographyNodeSchema):
|
|
78
98
|
label: str = "CloudTrailTrail"
|
|
@@ -81,5 +101,6 @@ class CloudTrailTrailSchema(CartographyNodeSchema):
|
|
|
81
101
|
other_relationships: OtherRelationships = OtherRelationships(
|
|
82
102
|
[
|
|
83
103
|
CloudTrailTrailToS3BucketRel(),
|
|
104
|
+
CloudTrailTrailToCloudWatchLogGroupRel(),
|
|
84
105
|
]
|
|
85
106
|
)
|