cartography 0.84.0__py3-none-any.whl → 0.85.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/data/indexes.cypher +0 -8
- cartography/data/jobs/analysis/aws_s3acl_analysis.json +7 -2
- cartography/intel/aws/ec2/instances.py +6 -6
- cartography/intel/aws/ec2/network_interfaces.py +186 -213
- cartography/intel/aws/ec2/security_groups.py +2 -2
- cartography/intel/aws/ec2/subnets.py +2 -2
- cartography/intel/aws/eks.py +46 -54
- cartography/intel/aws/inspector.py +44 -142
- cartography/intel/aws/s3.py +6 -1
- cartography/models/aws/ec2/loadbalancerv2.py +0 -0
- cartography/models/aws/ec2/networkinterface_instance.py +109 -0
- cartography/models/aws/ec2/networkinterfaces.py +36 -49
- cartography/models/aws/ec2/privateip_networkinterface.py +72 -0
- cartography/models/aws/ec2/{securitygroups.py → securitygroup_instance.py} +9 -6
- cartography/models/aws/ec2/securitygroup_networkinterface.py +52 -0
- cartography/models/aws/ec2/{subnets.py → subnet_instance.py} +7 -4
- cartography/models/aws/ec2/subnet_networkinterface.py +87 -0
- cartography/models/aws/eks/__init__.py +0 -0
- cartography/models/aws/eks/clusters.py +50 -0
- cartography/models/aws/inspector/__init__.py +0 -0
- cartography/models/aws/inspector/findings.py +124 -0
- cartography/models/aws/inspector/packages.py +73 -0
- cartography/util.py +8 -0
- {cartography-0.84.0.dist-info → cartography-0.85.1.dist-info}/METADATA +1 -1
- {cartography-0.84.0.dist-info → cartography-0.85.1.dist-info}/RECORD +30 -23
- cartography/data/jobs/cleanup/aws_import_eks_cleanup.json +0 -15
- cartography/data/jobs/cleanup/aws_import_inspector_cleanup.json +0 -35
- cartography/data/jobs/cleanup/aws_ingest_network_interfaces_cleanup.json +0 -30
- {cartography-0.84.0.dist-info → cartography-0.85.1.dist-info}/LICENSE +0 -0
- {cartography-0.84.0.dist-info → cartography-0.85.1.dist-info}/NOTICE +0 -0
- {cartography-0.84.0.dist-info → cartography-0.85.1.dist-info}/WHEEL +0 -0
- {cartography-0.84.0.dist-info → cartography-0.85.1.dist-info}/entry_points.txt +0 -0
- {cartography-0.84.0.dist-info → cartography-0.85.1.dist-info}/top_level.txt +0 -0
cartography/intel/aws/eks.py
CHANGED
|
@@ -6,8 +6,10 @@ from typing import List
|
|
|
6
6
|
import boto3
|
|
7
7
|
import neo4j
|
|
8
8
|
|
|
9
|
+
from cartography.client.core.tx import load
|
|
10
|
+
from cartography.graph.job import GraphJob
|
|
11
|
+
from cartography.models.aws.eks.clusters import EKSClusterSchema
|
|
9
12
|
from cartography.util import aws_handle_regions
|
|
10
|
-
from cartography.util import run_cleanup_job
|
|
11
13
|
from cartography.util import timeit
|
|
12
14
|
|
|
13
15
|
logger = logging.getLogger(__name__)
|
|
@@ -15,9 +17,9 @@ logger = logging.getLogger(__name__)
|
|
|
15
17
|
|
|
16
18
|
@timeit
|
|
17
19
|
@aws_handle_regions
|
|
18
|
-
def get_eks_clusters(boto3_session: boto3.session.Session, region: str) -> List[
|
|
20
|
+
def get_eks_clusters(boto3_session: boto3.session.Session, region: str) -> List[str]:
|
|
19
21
|
client = boto3_session.client('eks', region_name=region)
|
|
20
|
-
clusters: List[
|
|
22
|
+
clusters: List[str] = []
|
|
21
23
|
paginator = client.get_paginator('list_clusters')
|
|
22
24
|
for page in paginator.paginate():
|
|
23
25
|
clusters.extend(page['clusters'])
|
|
@@ -33,49 +35,20 @@ def get_eks_describe_cluster(boto3_session: boto3.session.Session, region: str,
|
|
|
33
35
|
|
|
34
36
|
@timeit
|
|
35
37
|
def load_eks_clusters(
|
|
36
|
-
|
|
37
|
-
|
|
38
|
+
neo4j_session: neo4j.Session,
|
|
39
|
+
cluster_data: List[Dict[str, Any]],
|
|
40
|
+
region: str,
|
|
41
|
+
current_aws_account_id: str,
|
|
42
|
+
aws_update_tag: int,
|
|
38
43
|
) -> None:
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
cluster.endpoint = $ClusterEndpoint,
|
|
48
|
-
cluster.endpoint_public_access = $ClusterEndointPublic,
|
|
49
|
-
cluster.rolearn = $ClusterRoleArn,
|
|
50
|
-
cluster.version = $ClusterVersion,
|
|
51
|
-
cluster.platform_version = $ClusterPlatformVersion,
|
|
52
|
-
cluster.status = $ClusterStatus,
|
|
53
|
-
cluster.audit_logging = $ClusterLogging
|
|
54
|
-
WITH cluster
|
|
55
|
-
MATCH (owner:AWSAccount{id: $AWS_ACCOUNT_ID})
|
|
56
|
-
MERGE (owner)-[r:RESOURCE]->(cluster)
|
|
57
|
-
ON CREATE SET r.firstseen = timestamp()
|
|
58
|
-
SET r.lastupdated = $aws_update_tag
|
|
59
|
-
"""
|
|
60
|
-
|
|
61
|
-
for cd in cluster_data:
|
|
62
|
-
cluster = cluster_data[cd]
|
|
63
|
-
neo4j_session.run(
|
|
64
|
-
query,
|
|
65
|
-
ClusterArn=cluster['arn'],
|
|
66
|
-
ClusterName=cluster['name'],
|
|
67
|
-
ClusterEndpoint=cluster.get('endpoint'),
|
|
68
|
-
ClusterEndointPublic=cluster.get('resourcesVpcConfig', {}).get('endpointPublicAccess'),
|
|
69
|
-
ClusterRoleArn=cluster.get('roleArn'),
|
|
70
|
-
ClusterVersion=cluster.get('version'),
|
|
71
|
-
ClusterPlatformVersion=cluster.get('platformVersion'),
|
|
72
|
-
ClusterStatus=cluster.get('status'),
|
|
73
|
-
CreatedAt=str(cluster.get('createdAt')),
|
|
74
|
-
ClusterLogging=_process_logging(cluster),
|
|
75
|
-
Region=region,
|
|
76
|
-
aws_update_tag=aws_update_tag,
|
|
77
|
-
AWS_ACCOUNT_ID=current_aws_account_id,
|
|
78
|
-
)
|
|
44
|
+
load(
|
|
45
|
+
neo4j_session,
|
|
46
|
+
EKSClusterSchema(),
|
|
47
|
+
cluster_data,
|
|
48
|
+
Region=region,
|
|
49
|
+
AWS_ID=current_aws_account_id,
|
|
50
|
+
lastupdated=aws_update_tag,
|
|
51
|
+
)
|
|
79
52
|
|
|
80
53
|
|
|
81
54
|
def _process_logging(cluster: Dict) -> bool:
|
|
@@ -91,24 +64,43 @@ def _process_logging(cluster: Dict) -> bool:
|
|
|
91
64
|
|
|
92
65
|
|
|
93
66
|
@timeit
|
|
94
|
-
def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
|
|
95
|
-
|
|
67
|
+
def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
|
|
68
|
+
logger.info("Running EKS cluster cleanup")
|
|
69
|
+
GraphJob.from_node_schema(EKSClusterSchema(), common_job_parameters).run(neo4j_session)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def transform(cluster_data: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
73
|
+
transformed_list = []
|
|
74
|
+
for cluster_name, cluster_dict in cluster_data.items():
|
|
75
|
+
transformed_dict = cluster_dict.copy()
|
|
76
|
+
transformed_dict['ClusterLogging'] = _process_logging(transformed_dict)
|
|
77
|
+
transformed_dict['ClusterEndpointPublic'] = transformed_dict.get('resourcesVpcConfig', {}).get(
|
|
78
|
+
'endpointPublicAccess',
|
|
79
|
+
)
|
|
80
|
+
if 'createdAt' in transformed_dict:
|
|
81
|
+
transformed_dict['created_at'] = str(transformed_dict['createdAt'])
|
|
82
|
+
transformed_list.append(transformed_dict)
|
|
83
|
+
return transformed_list
|
|
96
84
|
|
|
97
85
|
|
|
98
86
|
@timeit
|
|
99
87
|
def sync(
|
|
100
|
-
|
|
101
|
-
|
|
88
|
+
neo4j_session: neo4j.Session,
|
|
89
|
+
boto3_session: boto3.session.Session,
|
|
90
|
+
regions: List[str],
|
|
91
|
+
current_aws_account_id: str,
|
|
92
|
+
update_tag: int,
|
|
93
|
+
common_job_parameters: Dict[str, Any],
|
|
102
94
|
) -> None:
|
|
103
95
|
for region in regions:
|
|
104
96
|
logger.info("Syncing EKS for region '%s' in account '%s'.", region, current_aws_account_id)
|
|
105
97
|
|
|
106
|
-
clusters: List[
|
|
107
|
-
|
|
108
|
-
cluster_data: Dict = {}
|
|
98
|
+
clusters: List[str] = get_eks_clusters(boto3_session, region)
|
|
99
|
+
cluster_data = {}
|
|
109
100
|
for cluster_name in clusters:
|
|
110
|
-
cluster_data[cluster_name] = get_eks_describe_cluster(boto3_session, region, cluster_name)
|
|
101
|
+
cluster_data[cluster_name] = get_eks_describe_cluster(boto3_session, region, cluster_name)
|
|
102
|
+
transformed_list = transform(cluster_data)
|
|
111
103
|
|
|
112
|
-
load_eks_clusters(neo4j_session,
|
|
104
|
+
load_eks_clusters(neo4j_session, transformed_list, region, current_aws_account_id, update_tag)
|
|
113
105
|
|
|
114
106
|
cleanup(neo4j_session, common_job_parameters)
|
|
@@ -7,10 +7,12 @@ from typing import Tuple
|
|
|
7
7
|
import boto3
|
|
8
8
|
import neo4j
|
|
9
9
|
|
|
10
|
+
from cartography.client.core.tx import load
|
|
11
|
+
from cartography.graph.job import GraphJob
|
|
12
|
+
from cartography.models.aws.inspector.findings import AWSInspectorFindingSchema
|
|
13
|
+
from cartography.models.aws.inspector.packages import AWSInspectorPackageSchema
|
|
10
14
|
from cartography.util import aws_handle_regions
|
|
11
15
|
from cartography.util import aws_paginate
|
|
12
|
-
from cartography.util import batch
|
|
13
|
-
from cartography.util import run_cleanup_job
|
|
14
16
|
from cartography.util import timeit
|
|
15
17
|
|
|
16
18
|
|
|
@@ -20,17 +22,17 @@ logger = logging.getLogger(__name__)
|
|
|
20
22
|
@timeit
|
|
21
23
|
@aws_handle_regions
|
|
22
24
|
def get_inspector_findings(
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
) -> List[Dict]:
|
|
27
|
-
|
|
25
|
+
session: boto3.session.Session,
|
|
26
|
+
region: str,
|
|
27
|
+
current_aws_account_id: str,
|
|
28
|
+
) -> List[Dict[str, Any]]:
|
|
29
|
+
"""
|
|
28
30
|
We must list_findings by filtering the request, otherwise the request could tiemout.
|
|
29
31
|
First, we filter by account_id. And since there may be millions of CLOSED findings that may never go away,
|
|
30
32
|
we will only fetch those in ACTIVE or SUPPRESSED statuses.
|
|
31
33
|
list_members will get us all the accounts that
|
|
32
34
|
have delegated access to the account specified by current_aws_account_id.
|
|
33
|
-
|
|
35
|
+
"""
|
|
34
36
|
client = session.client('inspector2', region_name=region)
|
|
35
37
|
|
|
36
38
|
members = aws_paginate(client, 'list_members', 'members')
|
|
@@ -61,7 +63,7 @@ def get_inspector_findings(
|
|
|
61
63
|
return findings
|
|
62
64
|
|
|
63
65
|
|
|
64
|
-
def transform_inspector_findings(results: List[Dict]) -> Tuple[List, List]:
|
|
66
|
+
def transform_inspector_findings(results: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
|
65
67
|
findings_list: List[Dict] = []
|
|
66
68
|
packages: Dict[str, Any] = {}
|
|
67
69
|
|
|
@@ -110,7 +112,7 @@ def transform_inspector_findings(results: List[Dict]) -> Tuple[List, List]:
|
|
|
110
112
|
return findings_list, packages_list
|
|
111
113
|
|
|
112
114
|
|
|
113
|
-
def transform_inspector_packages(packages: Dict[str, Any]) -> List[Dict]:
|
|
115
|
+
def transform_inspector_packages(packages: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
114
116
|
packages_list: List[Dict] = []
|
|
115
117
|
for package_id in packages.keys():
|
|
116
118
|
packages_list.append(packages[package_id])
|
|
@@ -146,153 +148,53 @@ def _process_packages(package_details: Dict[str, Any], aws_account_id: str, find
|
|
|
146
148
|
return packages
|
|
147
149
|
|
|
148
150
|
|
|
149
|
-
def _port_range_string(details: Dict) -> str:
|
|
151
|
+
def _port_range_string(details: Dict[str, Any]) -> str:
|
|
150
152
|
begin = details['openPortRange']['begin']
|
|
151
153
|
end = details['openPortRange']['end']
|
|
152
154
|
return f"{begin}-{end}"
|
|
153
155
|
|
|
154
156
|
|
|
155
|
-
def _load_findings_tx(
|
|
156
|
-
tx: neo4j.Transaction,
|
|
157
|
-
findings: List[Dict],
|
|
158
|
-
region: str,
|
|
159
|
-
aws_update_tag: int,
|
|
160
|
-
) -> None:
|
|
161
|
-
query = """
|
|
162
|
-
UNWIND $Findings as new_finding
|
|
163
|
-
MERGE (finding:AWSInspectorFinding{id: new_finding.id})
|
|
164
|
-
ON CREATE SET finding.firstseen = timestamp(),
|
|
165
|
-
finding.arn = new_finding.arn,
|
|
166
|
-
finding.region = $Region,
|
|
167
|
-
finding.awsaccount = new_finding.awsaccount
|
|
168
|
-
SET finding.lastupdated = $UpdateTag,
|
|
169
|
-
finding.name = new_finding.title,
|
|
170
|
-
finding.instanceid = new_finding.instanceid,
|
|
171
|
-
finding.ecrimageid = new_finding.ecrimageid,
|
|
172
|
-
finding.ecrrepositoryid = new_finding.ecrrepositoryid,
|
|
173
|
-
finding.severity = new_finding.severity,
|
|
174
|
-
finding.firstobservedat = new_finding.firstobservedat,
|
|
175
|
-
finding.updatedat = new_finding.updatedat,
|
|
176
|
-
finding.description = new_finding.description,
|
|
177
|
-
finding.type = new_finding.type,
|
|
178
|
-
finding.cvssscore = new_finding.cvssscore,
|
|
179
|
-
finding.protocol = new_finding.protocol,
|
|
180
|
-
finding.portrange = new_finding.portrange,
|
|
181
|
-
finding.portrangebegin = new_finding.portrangebegin,
|
|
182
|
-
finding.portrangeend = new_finding.portrangeend,
|
|
183
|
-
finding.vulnerabilityid = new_finding.vulnerabilityid,
|
|
184
|
-
finding.referenceurls = new_finding.referenceurls,
|
|
185
|
-
finding.relatedvulnerabilities = new_finding.relatedvulnerabilities,
|
|
186
|
-
finding.source = new_finding.source,
|
|
187
|
-
finding.sourceurl = new_finding.sourceurl,
|
|
188
|
-
finding.status = new_finding.status,
|
|
189
|
-
finding.vendorcreatedat = new_finding.vendorcreatedat,
|
|
190
|
-
finding.vendorseverity = new_finding.vendorseverity,
|
|
191
|
-
finding.vendorupdatedat = new_finding.vendorupdatedat,
|
|
192
|
-
finding.vulnerablepackageids = new_finding.vulnerablepackageids,
|
|
193
|
-
finding:Risk
|
|
194
|
-
WITH finding
|
|
195
|
-
MATCH (account:AWSAccount{id: finding.awsaccount})
|
|
196
|
-
MERGE (account)-[r:RESOURCE]->(finding)
|
|
197
|
-
ON CREATE SET r.firstseen = timestamp()
|
|
198
|
-
SET r.lastupdated = $UpdateTag
|
|
199
|
-
WITH finding
|
|
200
|
-
MATCH (instance:EC2Instance{id: finding.instanceid})
|
|
201
|
-
MERGE (instance)<-[r2:AFFECTS]-(finding)
|
|
202
|
-
ON CREATE SET r2.firstseen = timestamp()
|
|
203
|
-
SET r2.lastupdated = $UpdateTag
|
|
204
|
-
WITH finding
|
|
205
|
-
MATCH (repo:ECRRepository{id: finding.ecrrepositoryid})
|
|
206
|
-
MERGE (repo)<-[r3:AFFECTS]-(finding)
|
|
207
|
-
ON CREATE SET r3.firstseen = timestamp()
|
|
208
|
-
SET r3.lastupdated = $UpdateTag
|
|
209
|
-
WITH finding
|
|
210
|
-
MATCH (image:ECRImage{id: finding.ecrimageid})
|
|
211
|
-
MERGE (image)<-[r4:AFFECTS]-(finding)
|
|
212
|
-
ON CREATE SET r4.firstseen = timestamp()
|
|
213
|
-
SET r4.lastupdated = $UpdateTag
|
|
214
|
-
"""
|
|
215
|
-
|
|
216
|
-
tx.run(
|
|
217
|
-
query,
|
|
218
|
-
Findings=findings,
|
|
219
|
-
UpdateTag=aws_update_tag,
|
|
220
|
-
Region=region,
|
|
221
|
-
)
|
|
222
|
-
|
|
223
|
-
|
|
224
157
|
@timeit
|
|
225
158
|
def load_inspector_findings(
|
|
226
|
-
|
|
227
|
-
|
|
159
|
+
neo4j_session: neo4j.Session,
|
|
160
|
+
findings: List[Dict[str, Any]],
|
|
161
|
+
region: str,
|
|
162
|
+
aws_update_tag: int,
|
|
163
|
+
current_aws_account_id: str,
|
|
228
164
|
) -> None:
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
findings=findings_batch,
|
|
234
|
-
region=region,
|
|
235
|
-
aws_update_tag=aws_update_tag,
|
|
236
|
-
)
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
def _load_packages_tx(
|
|
240
|
-
tx: neo4j.Transaction,
|
|
241
|
-
packages: List[Dict],
|
|
242
|
-
region: str,
|
|
243
|
-
aws_update_tag: int,
|
|
244
|
-
) -> None:
|
|
245
|
-
query = """
|
|
246
|
-
UNWIND $Packages as new_package
|
|
247
|
-
MERGE (package:AWSInspectorPackage{id: new_package.id})
|
|
248
|
-
ON CREATE SET package.firstseen = timestamp(),
|
|
249
|
-
package.region = $Region,
|
|
250
|
-
package.awsaccount = new_package.awsaccount,
|
|
251
|
-
package.findingarn = new_package.findingarn
|
|
252
|
-
SET package.lastupdated = $UpdateTag,
|
|
253
|
-
package.name = new_package.name,
|
|
254
|
-
package.arch = new_package.arch,
|
|
255
|
-
package.version = new_package.version,
|
|
256
|
-
package.release = new_package.release,
|
|
257
|
-
package.epoch = new_package.epoch,
|
|
258
|
-
package.manager = new_package.packageManager,
|
|
259
|
-
package.filepath = new_package.filePath,
|
|
260
|
-
package.fixedinversion = new_package.fixedInVersion,
|
|
261
|
-
package.sourcelayerhash = new_package.sourceLayerHash
|
|
262
|
-
WITH package
|
|
263
|
-
MATCH (finding:AWSInspectorFinding{id: package.findingarn})
|
|
264
|
-
MERGE (finding)-[r:HAS]->(package)
|
|
265
|
-
WITH package
|
|
266
|
-
MATCH (account:AWSAccount{id: package.awsaccount})
|
|
267
|
-
MERGE (account)-[r:RESOURCE]->(package)
|
|
268
|
-
ON CREATE SET r.firstseen = timestamp()
|
|
269
|
-
SET r.lastupdated = $UpdateTag
|
|
270
|
-
"""
|
|
271
|
-
|
|
272
|
-
tx.run(
|
|
273
|
-
query,
|
|
274
|
-
Packages=packages,
|
|
275
|
-
UpdateTag=aws_update_tag,
|
|
165
|
+
load(
|
|
166
|
+
neo4j_session,
|
|
167
|
+
AWSInspectorFindingSchema(),
|
|
168
|
+
findings,
|
|
276
169
|
Region=region,
|
|
170
|
+
AWS_ID=current_aws_account_id,
|
|
171
|
+
lastupdated=aws_update_tag,
|
|
277
172
|
)
|
|
278
173
|
|
|
279
174
|
|
|
280
175
|
@timeit
|
|
281
176
|
def load_inspector_packages(
|
|
282
|
-
|
|
283
|
-
|
|
177
|
+
neo4j_session: neo4j.Session,
|
|
178
|
+
packages: List[Dict[str, Any]],
|
|
179
|
+
region: str,
|
|
180
|
+
aws_update_tag: int,
|
|
181
|
+
current_aws_account_id: str,
|
|
284
182
|
) -> None:
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
183
|
+
load(
|
|
184
|
+
neo4j_session,
|
|
185
|
+
AWSInspectorPackageSchema(),
|
|
186
|
+
packages,
|
|
187
|
+
Region=region,
|
|
188
|
+
AWS_ID=current_aws_account_id,
|
|
189
|
+
lastupdated=aws_update_tag,
|
|
290
190
|
)
|
|
291
191
|
|
|
292
192
|
|
|
293
193
|
@timeit
|
|
294
|
-
def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
|
|
295
|
-
|
|
194
|
+
def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
|
|
195
|
+
logger.info("Running AWS Inspector cleanup")
|
|
196
|
+
GraphJob.from_node_schema(AWSInspectorFindingSchema(), common_job_parameters).run(neo4j_session)
|
|
197
|
+
GraphJob.from_node_schema(AWSInspectorPackageSchema(), common_job_parameters).run(neo4j_session)
|
|
296
198
|
|
|
297
199
|
|
|
298
200
|
@timeit
|
|
@@ -302,14 +204,14 @@ def sync(
|
|
|
302
204
|
regions: List[str],
|
|
303
205
|
current_aws_account_id: str,
|
|
304
206
|
update_tag: int,
|
|
305
|
-
common_job_parameters: Dict,
|
|
207
|
+
common_job_parameters: Dict[str, Any],
|
|
306
208
|
) -> None:
|
|
307
209
|
for region in regions:
|
|
308
210
|
logger.info(f"Syncing AWS Inspector findings for account {current_aws_account_id} and region {region}")
|
|
309
211
|
findings = get_inspector_findings(boto3_session, region, current_aws_account_id)
|
|
310
212
|
finding_data, package_data = transform_inspector_findings(findings)
|
|
311
|
-
logger.info(f"Loading {len(package_data)} packages")
|
|
312
|
-
load_inspector_packages(neo4j_session, package_data, region, update_tag)
|
|
313
213
|
logger.info(f"Loading {len(finding_data)} findings")
|
|
314
|
-
load_inspector_findings(neo4j_session, finding_data, region, update_tag)
|
|
214
|
+
load_inspector_findings(neo4j_session, finding_data, region, update_tag, current_aws_account_id)
|
|
215
|
+
logger.info(f"Loading {len(package_data)} packages")
|
|
216
|
+
load_inspector_packages(neo4j_session, package_data, region, update_tag, current_aws_account_id)
|
|
315
217
|
cleanup(neo4j_session, common_job_parameters)
|
cartography/intel/aws/s3.py
CHANGED
|
@@ -222,7 +222,12 @@ def _is_common_exception(e: Exception, bucket: Dict) -> bool:
|
|
|
222
222
|
|
|
223
223
|
|
|
224
224
|
@timeit
|
|
225
|
-
def _load_s3_acls(
|
|
225
|
+
def _load_s3_acls(
|
|
226
|
+
neo4j_session: neo4j.Session,
|
|
227
|
+
acls: List[Dict[str, Any]],
|
|
228
|
+
aws_account_id: str,
|
|
229
|
+
update_tag: int,
|
|
230
|
+
) -> None:
|
|
226
231
|
"""
|
|
227
232
|
Ingest S3 ACL into neo4j.
|
|
228
233
|
"""
|
|
File without changes
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from cartography.models.core.common import PropertyRef
|
|
4
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
7
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
8
|
+
from cartography.models.core.relationships import LinkDirection
|
|
9
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
10
|
+
from cartography.models.core.relationships import OtherRelationships
|
|
11
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class EC2NetworkInterfaceInstanceNodeProperties(CartographyNodeProperties):
|
|
16
|
+
"""
|
|
17
|
+
Selection of properties of a network interface as known by an EC2 instance
|
|
18
|
+
"""
|
|
19
|
+
# arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO use arn; issue #1024
|
|
20
|
+
id: PropertyRef = PropertyRef('NetworkInterfaceId')
|
|
21
|
+
status: PropertyRef = PropertyRef('Status')
|
|
22
|
+
mac_address: PropertyRef = PropertyRef('MacAddress', extra_index=True)
|
|
23
|
+
description: PropertyRef = PropertyRef('Description')
|
|
24
|
+
private_dns_name: PropertyRef = PropertyRef('PrivateDnsName', extra_index=True)
|
|
25
|
+
private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress', extra_index=True)
|
|
26
|
+
region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
|
|
27
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class EC2NetworkInterfaceToAwsAccountRelProperties(CartographyRelProperties):
|
|
32
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class EC2NetworkInterfaceToAWSAccount(CartographyRelSchema):
|
|
37
|
+
target_node_label: str = 'AWSAccount'
|
|
38
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
39
|
+
{'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
|
|
40
|
+
)
|
|
41
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
42
|
+
rel_label: str = "RESOURCE"
|
|
43
|
+
properties: EC2NetworkInterfaceToAwsAccountRelProperties = EC2NetworkInterfaceToAwsAccountRelProperties()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class EC2NetworkInterfaceToEC2InstanceRelProperties(CartographyRelProperties):
|
|
48
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True)
|
|
52
|
+
class EC2NetworkInterfaceToEC2Instance(CartographyRelSchema):
|
|
53
|
+
target_node_label: str = 'EC2Instance'
|
|
54
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
55
|
+
{'id': PropertyRef('InstanceId')},
|
|
56
|
+
)
|
|
57
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
58
|
+
rel_label: str = "NETWORK_INTERFACE"
|
|
59
|
+
properties: EC2NetworkInterfaceToEC2InstanceRelProperties = EC2NetworkInterfaceToEC2InstanceRelProperties()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class EC2NetworkInterfaceToEC2SubnetRelProperties(CartographyRelProperties):
|
|
64
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class EC2NetworkInterfaceToEC2Subnet(CartographyRelSchema):
|
|
69
|
+
target_node_label: str = 'EC2Subnet'
|
|
70
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
71
|
+
{'id': PropertyRef('SubnetId')},
|
|
72
|
+
)
|
|
73
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
74
|
+
rel_label: str = "PART_OF_SUBNET"
|
|
75
|
+
properties: EC2NetworkInterfaceToEC2SubnetRelProperties = EC2NetworkInterfaceToEC2SubnetRelProperties()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class EC2NetworkInterfaceToEC2SecurityGroupRelProperties(CartographyRelProperties):
|
|
80
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True)
|
|
84
|
+
class EC2NetworkInterfaceToEC2SecurityGroup(CartographyRelSchema):
|
|
85
|
+
target_node_label: str = 'EC2SecurityGroup'
|
|
86
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
87
|
+
{'id': PropertyRef('GroupId')},
|
|
88
|
+
)
|
|
89
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
90
|
+
rel_label: str = "MEMBER_OF_EC2_SECURITY_GROUP"
|
|
91
|
+
properties: EC2NetworkInterfaceToEC2SecurityGroupRelProperties = \
|
|
92
|
+
EC2NetworkInterfaceToEC2SecurityGroupRelProperties()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass(frozen=True)
|
|
96
|
+
class EC2NetworkInterfaceInstanceSchema(CartographyNodeSchema):
|
|
97
|
+
"""
|
|
98
|
+
Network interface as known by an EC2 instance
|
|
99
|
+
"""
|
|
100
|
+
label: str = 'NetworkInterface'
|
|
101
|
+
properties: EC2NetworkInterfaceInstanceNodeProperties = EC2NetworkInterfaceInstanceNodeProperties()
|
|
102
|
+
sub_resource_relationship: EC2NetworkInterfaceToAWSAccount = EC2NetworkInterfaceToAWSAccount()
|
|
103
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
104
|
+
[
|
|
105
|
+
EC2NetworkInterfaceToEC2Instance(),
|
|
106
|
+
EC2NetworkInterfaceToEC2Subnet(),
|
|
107
|
+
EC2NetworkInterfaceToEC2SecurityGroup(),
|
|
108
|
+
],
|
|
109
|
+
)
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
2
|
|
|
3
|
+
from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToAWSAccount
|
|
4
|
+
from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToEC2Instance
|
|
5
|
+
from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToEC2SecurityGroup
|
|
6
|
+
from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToEC2Subnet
|
|
3
7
|
from cartography.models.core.common import PropertyRef
|
|
4
8
|
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
9
|
from cartography.models.core.nodes import CartographyNodeSchema
|
|
@@ -13,90 +17,73 @@ from cartography.models.core.relationships import TargetNodeMatcher
|
|
|
13
17
|
|
|
14
18
|
@dataclass(frozen=True)
|
|
15
19
|
class EC2NetworkInterfaceNodeProperties(CartographyNodeProperties):
|
|
16
|
-
|
|
20
|
+
"""
|
|
21
|
+
Network interface properties
|
|
22
|
+
"""
|
|
17
23
|
id: PropertyRef = PropertyRef('NetworkInterfaceId')
|
|
18
|
-
|
|
19
|
-
mac_address: PropertyRef = PropertyRef('MacAddress')
|
|
24
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
20
25
|
description: PropertyRef = PropertyRef('Description')
|
|
26
|
+
mac_address: PropertyRef = PropertyRef('MacAddress', extra_index=True)
|
|
21
27
|
private_dns_name: PropertyRef = PropertyRef('PrivateDnsName')
|
|
22
|
-
private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress')
|
|
28
|
+
private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress', extra_index=True)
|
|
23
29
|
region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
@dataclass(frozen=True)
|
|
28
|
-
class EC2NetworkInterfaceToAwsAccountRelProperties(CartographyRelProperties):
|
|
29
|
-
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
30
|
-
|
|
30
|
+
status: PropertyRef = PropertyRef('Status')
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
)
|
|
38
|
-
|
|
39
|
-
rel_label: str = "RESOURCE"
|
|
40
|
-
properties: EC2NetworkInterfaceToAwsAccountRelProperties = EC2NetworkInterfaceToAwsAccountRelProperties()
|
|
32
|
+
# Properties only returned by describe-network-interfaces
|
|
33
|
+
interface_type: PropertyRef = PropertyRef('InterfaceType')
|
|
34
|
+
public_ip: PropertyRef = PropertyRef('PublicIp', extra_index=True)
|
|
35
|
+
requester_id: PropertyRef = PropertyRef('RequesterId', extra_index=True)
|
|
36
|
+
requester_managed: PropertyRef = PropertyRef('RequesterManaged')
|
|
37
|
+
source_dest_check: PropertyRef = PropertyRef('SourceDestCheck')
|
|
38
|
+
subnetid: PropertyRef = PropertyRef('SubnetId', extra_index=True)
|
|
41
39
|
|
|
42
40
|
|
|
43
41
|
@dataclass(frozen=True)
|
|
44
|
-
class
|
|
42
|
+
class EC2NetworkInterfaceToElbRelProperties(CartographyRelProperties):
|
|
45
43
|
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
46
44
|
|
|
47
45
|
|
|
48
46
|
@dataclass(frozen=True)
|
|
49
|
-
class
|
|
50
|
-
target_node_label: str = '
|
|
47
|
+
class EC2NetworkInterfaceToElb(CartographyRelSchema):
|
|
48
|
+
target_node_label: str = 'LoadBalancer'
|
|
51
49
|
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
52
|
-
{'
|
|
50
|
+
{'name': PropertyRef('ElbV1Id')},
|
|
53
51
|
)
|
|
54
52
|
direction: LinkDirection = LinkDirection.INWARD
|
|
55
53
|
rel_label: str = "NETWORK_INTERFACE"
|
|
56
|
-
properties:
|
|
54
|
+
properties: EC2NetworkInterfaceToElbRelProperties = EC2NetworkInterfaceToElbRelProperties()
|
|
57
55
|
|
|
58
56
|
|
|
59
57
|
@dataclass(frozen=True)
|
|
60
|
-
class
|
|
58
|
+
class EC2NetworkInterfaceToElbV2RelProperties(CartographyRelProperties):
|
|
61
59
|
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
62
60
|
|
|
63
61
|
|
|
64
62
|
@dataclass(frozen=True)
|
|
65
|
-
class
|
|
66
|
-
target_node_label: str = '
|
|
63
|
+
class EC2NetworkInterfaceToElbV2(CartographyRelSchema):
|
|
64
|
+
target_node_label: str = 'LoadBalancerV2'
|
|
67
65
|
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
68
|
-
{'id': PropertyRef('
|
|
66
|
+
{'id': PropertyRef('ElbV2Id')},
|
|
69
67
|
)
|
|
70
|
-
direction: LinkDirection = LinkDirection.
|
|
71
|
-
rel_label: str = "
|
|
72
|
-
properties:
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
@dataclass(frozen=True)
|
|
76
|
-
class EC2NetworkInterfaceToEC2SecurityGroupRelProperties(CartographyRelProperties):
|
|
77
|
-
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
@dataclass(frozen=True)
|
|
81
|
-
class EC2NetworkInterfaceToEC2SecurityGroup(CartographyRelSchema):
|
|
82
|
-
target_node_label: str = 'EC2SecurityGroup'
|
|
83
|
-
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
84
|
-
{'id': PropertyRef('GroupId')},
|
|
85
|
-
)
|
|
86
|
-
direction: LinkDirection = LinkDirection.OUTWARD
|
|
87
|
-
rel_label: str = "MEMBER_OF_EC2_SECURITY_GROUP"
|
|
88
|
-
properties: EC2NetworkInterfaceToEC2SubnetRelProperties = EC2NetworkInterfaceToEC2SubnetRelProperties()
|
|
68
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
69
|
+
rel_label: str = "NETWORK_INTERFACE"
|
|
70
|
+
properties: EC2NetworkInterfaceToElbV2RelProperties = EC2NetworkInterfaceToElbV2RelProperties()
|
|
89
71
|
|
|
90
72
|
|
|
91
73
|
@dataclass(frozen=True)
|
|
92
74
|
class EC2NetworkInterfaceSchema(CartographyNodeSchema):
|
|
75
|
+
"""
|
|
76
|
+
Network interface as known by describe-network-interfaces.
|
|
77
|
+
"""
|
|
93
78
|
label: str = 'NetworkInterface'
|
|
94
79
|
properties: EC2NetworkInterfaceNodeProperties = EC2NetworkInterfaceNodeProperties()
|
|
95
80
|
sub_resource_relationship: EC2NetworkInterfaceToAWSAccount = EC2NetworkInterfaceToAWSAccount()
|
|
96
81
|
other_relationships: OtherRelationships = OtherRelationships(
|
|
97
82
|
[
|
|
98
|
-
EC2NetworkInterfaceToEC2Instance(),
|
|
99
83
|
EC2NetworkInterfaceToEC2Subnet(),
|
|
100
84
|
EC2NetworkInterfaceToEC2SecurityGroup(),
|
|
85
|
+
EC2NetworkInterfaceToElb(),
|
|
86
|
+
EC2NetworkInterfaceToElbV2(),
|
|
87
|
+
EC2NetworkInterfaceToEC2Instance(),
|
|
101
88
|
],
|
|
102
89
|
)
|