cartography 0.95.0rc1__py3-none-any.whl → 0.96.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/cli.py +15 -0
- cartography/config.py +4 -0
- cartography/data/indexes.cypher +1 -2
- cartography/data/jobs/cleanup/aws_import_identity_center_cleanup.json +16 -0
- cartography/data/jobs/cleanup/{github_users_cleanup.json → github_org_and_users_cleanup.json} +5 -0
- cartography/graph/querybuilder.py +4 -0
- cartography/intel/aws/ec2/network_acls.py +208 -0
- cartography/intel/aws/identitycenter.py +307 -0
- cartography/intel/aws/resources.py +4 -0
- cartography/intel/github/users.py +156 -39
- cartography/intel/okta/users.py +2 -1
- cartography/intel/semgrep/__init__.py +9 -2
- cartography/intel/semgrep/dependencies.py +233 -0
- cartography/intel/semgrep/deployment.py +67 -0
- cartography/intel/semgrep/findings.py +22 -53
- cartography/models/aws/ec2/network_acl_rules.py +97 -0
- cartography/models/aws/ec2/network_acls.py +86 -0
- cartography/models/core/common.py +18 -1
- cartography/models/github/orgs.py +26 -0
- cartography/models/github/users.py +119 -0
- cartography/models/semgrep/dependencies.py +90 -0
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/METADATA +1 -1
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/RECORD +27 -17
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/WHEEL +1 -1
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/LICENSE +0 -0
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/entry_points.txt +0 -0
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/top_level.txt +0 -0
cartography/cli.py
CHANGED
|
@@ -9,6 +9,7 @@ import cartography.config
|
|
|
9
9
|
import cartography.sync
|
|
10
10
|
import cartography.util
|
|
11
11
|
from cartography.intel.aws.util.common import parse_and_validate_aws_requested_syncs
|
|
12
|
+
from cartography.intel.semgrep.dependencies import parse_and_validate_semgrep_ecosystems
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
logger = logging.getLogger(__name__)
|
|
@@ -524,6 +525,17 @@ class CLI:
|
|
|
524
525
|
'Required if you are using the Semgrep intel module. Ignored otherwise.'
|
|
525
526
|
),
|
|
526
527
|
)
|
|
528
|
+
parser.add_argument(
|
|
529
|
+
'--semgrep-dependency-ecosystems',
|
|
530
|
+
type=str,
|
|
531
|
+
default=None,
|
|
532
|
+
help=(
|
|
533
|
+
'Comma-separated list of language ecosystems for which dependencies will be retrieved from Semgrep. '
|
|
534
|
+
'For example, a value of "gomod,npm" will retrieve Go and NPM dependencies. '
|
|
535
|
+
'See the full list of supported ecosystems in source code at cartography.intel.semgrep.dependencies. '
|
|
536
|
+
'Required if you are using the Semgrep dependencies intel module. Ignored otherwise.'
|
|
537
|
+
),
|
|
538
|
+
)
|
|
527
539
|
parser.add_argument(
|
|
528
540
|
'--snipeit-base-uri',
|
|
529
541
|
type=str,
|
|
@@ -734,6 +746,9 @@ class CLI:
|
|
|
734
746
|
config.semgrep_app_token = os.environ.get(config.semgrep_app_token_env_var)
|
|
735
747
|
else:
|
|
736
748
|
config.semgrep_app_token = None
|
|
749
|
+
if config.semgrep_dependency_ecosystems:
|
|
750
|
+
# No need to store the returned value; we're using this for input validation.
|
|
751
|
+
parse_and_validate_semgrep_ecosystems(config.semgrep_dependency_ecosystems)
|
|
737
752
|
|
|
738
753
|
# CVE feed config
|
|
739
754
|
if config.cve_api_key_env_var:
|
cartography/config.py
CHANGED
|
@@ -107,6 +107,8 @@ class Config:
|
|
|
107
107
|
:param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional.
|
|
108
108
|
:param semgrep_app_token: The Semgrep api token. Optional.
|
|
109
109
|
:type semgrep_app_token: str
|
|
110
|
+
:param semgrep_dependency_ecosystems: Comma-separated list of Semgrep dependency ecosystems to fetch. Optional.
|
|
111
|
+
:type semgrep_dependency_ecosystems: str
|
|
110
112
|
:type snipeit_base_uri: string
|
|
111
113
|
:param snipeit_base_uri: SnipeIT data provider base URI. Optional.
|
|
112
114
|
:type snipeit_token: string
|
|
@@ -170,6 +172,7 @@ class Config:
|
|
|
170
172
|
duo_api_secret=None,
|
|
171
173
|
duo_api_hostname=None,
|
|
172
174
|
semgrep_app_token=None,
|
|
175
|
+
semgrep_dependency_ecosystems=None,
|
|
173
176
|
snipeit_base_uri=None,
|
|
174
177
|
snipeit_token=None,
|
|
175
178
|
snipeit_tenant_id=None,
|
|
@@ -227,6 +230,7 @@ class Config:
|
|
|
227
230
|
self.duo_api_secret = duo_api_secret
|
|
228
231
|
self.duo_api_hostname = duo_api_hostname
|
|
229
232
|
self.semgrep_app_token = semgrep_app_token
|
|
233
|
+
self.semgrep_dependency_ecosystems = semgrep_dependency_ecosystems
|
|
230
234
|
self.snipeit_base_uri = snipeit_base_uri
|
|
231
235
|
self.snipeit_token = snipeit_token
|
|
232
236
|
self.snipeit_tenant_id = snipeit_tenant_id
|
cartography/data/indexes.cypher
CHANGED
|
@@ -305,8 +305,7 @@ CREATE INDEX IF NOT EXISTS FOR (n:SpotlightVulnerability) ON (n.host_info_local_
|
|
|
305
305
|
CREATE INDEX IF NOT EXISTS FOR (n:SpotlightVulnerability) ON (n.lastupdated);
|
|
306
306
|
CREATE INDEX IF NOT EXISTS FOR (n:SQSQueue) ON (n.id);
|
|
307
307
|
CREATE INDEX IF NOT EXISTS FOR (n:SQSQueue) ON (n.lastupdated);
|
|
308
|
-
CREATE INDEX IF NOT EXISTS FOR (n:
|
|
309
|
-
CREATE INDEX IF NOT EXISTS FOR (n:User) ON (n.lastupdated);
|
|
308
|
+
CREATE INDEX IF NOT EXISTS FOR (n:UserAccount) ON (n.id);
|
|
310
309
|
CREATE INDEX IF NOT EXISTS FOR (n:AzureTenant) ON (n.id);
|
|
311
310
|
CREATE INDEX IF NOT EXISTS FOR (n:AzureTenant) ON (n.lastupdated);
|
|
312
311
|
CREATE INDEX IF NOT EXISTS FOR (n:AzurePrincipal) ON (n.email);
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"statements": [
|
|
3
|
+
|
|
4
|
+
{
|
|
5
|
+
"query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSSSOUser)<-[r:CAN_ASSUME_IDENTITY]-(:OktaUser) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r) RETURN COUNT(*) as TotalDeleted",
|
|
6
|
+
"iterative": true,
|
|
7
|
+
"iterationsize": 100
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSRole)-[r:ALLOWED_BY]->(:AWSSSOUser) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r) RETURN COUNT(*) as TotalDeleted",
|
|
11
|
+
"iterative": true,
|
|
12
|
+
"iterationsize": 100
|
|
13
|
+
}
|
|
14
|
+
],
|
|
15
|
+
"name": "cleanup AWS Identity Center Instances and Related Data"
|
|
16
|
+
}
|
cartography/data/jobs/cleanup/{github_users_cleanup.json → github_org_and_users_cleanup.json}
RENAMED
|
@@ -18,6 +18,11 @@
|
|
|
18
18
|
"query": "MATCH (:GitHubUser)-[r:MEMBER_OF]->(:GitHubOrganization) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
|
|
19
19
|
"iterative": true,
|
|
20
20
|
"iterationsize": 100
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"query": "MATCH (:GitHubUser)-[r:UNAFFILIATED]->(:GitHubOrganization) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
|
|
24
|
+
"iterative": true,
|
|
25
|
+
"iterationsize": 100
|
|
21
26
|
}],
|
|
22
27
|
"name": "cleanup GitHub users data"
|
|
23
28
|
}
|
|
@@ -118,6 +118,7 @@ def _build_where_clause_for_rel_match(node_var: str, matcher: TargetNodeMatcher)
|
|
|
118
118
|
"""
|
|
119
119
|
match = Template("$node_var.$key = $prop_ref")
|
|
120
120
|
case_insensitive_match = Template("toLower($node_var.$key) = toLower($prop_ref)")
|
|
121
|
+
fuzzy_and_ignorecase_match = Template("toLower($node_var.$key) CONTAINS toLower($prop_ref)")
|
|
121
122
|
|
|
122
123
|
matcher_asdict = asdict(matcher)
|
|
123
124
|
|
|
@@ -125,7 +126,10 @@ def _build_where_clause_for_rel_match(node_var: str, matcher: TargetNodeMatcher)
|
|
|
125
126
|
for key, prop_ref in matcher_asdict.items():
|
|
126
127
|
if prop_ref.ignore_case:
|
|
127
128
|
prop_line = case_insensitive_match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
|
|
129
|
+
elif prop_ref.fuzzy_and_ignore_case:
|
|
130
|
+
prop_line = fuzzy_and_ignorecase_match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
|
|
128
131
|
else:
|
|
132
|
+
# Exact match (default; most efficient)
|
|
129
133
|
prop_line = match.safe_substitute(node_var=node_var, key=key, prop_ref=prop_ref)
|
|
130
134
|
result.append(prop_line)
|
|
131
135
|
return ' AND\n'.join(result)
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from collections import namedtuple
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import boto3
|
|
6
|
+
import neo4j
|
|
7
|
+
|
|
8
|
+
from .util import get_botocore_config
|
|
9
|
+
from cartography.client.core.tx import load
|
|
10
|
+
from cartography.graph.job import GraphJob
|
|
11
|
+
from cartography.models.aws.ec2.network_acl_rules import EC2NetworkAclEgressRuleSchema
|
|
12
|
+
from cartography.models.aws.ec2.network_acl_rules import EC2NetworkAclInboundRuleSchema
|
|
13
|
+
from cartography.models.aws.ec2.network_acls import EC2NetworkAclSchema
|
|
14
|
+
from cartography.util import aws_handle_regions
|
|
15
|
+
from cartography.util import timeit
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
Ec2AclObjects = namedtuple(
|
|
20
|
+
"Ec2AclObjects", [
|
|
21
|
+
'network_acls',
|
|
22
|
+
'inbound_rules',
|
|
23
|
+
'outbound_rules',
|
|
24
|
+
],
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@timeit
|
|
29
|
+
@aws_handle_regions
|
|
30
|
+
def get_network_acl_data(boto3_session: boto3.session.Session, region: str) -> list[dict[str, Any]]:
|
|
31
|
+
client = boto3_session.client('ec2', region_name=region, config=get_botocore_config())
|
|
32
|
+
paginator = client.get_paginator('describe_network_acls')
|
|
33
|
+
acls = []
|
|
34
|
+
for page in paginator.paginate():
|
|
35
|
+
acls.extend(page['NetworkAcls'])
|
|
36
|
+
return acls
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def transform_network_acl_data(
|
|
40
|
+
data_list: list[dict[str, Any]],
|
|
41
|
+
region: str,
|
|
42
|
+
current_aws_account_id: str,
|
|
43
|
+
) -> Ec2AclObjects:
|
|
44
|
+
network_acls = []
|
|
45
|
+
inbound_rules = []
|
|
46
|
+
outbound_rules = []
|
|
47
|
+
|
|
48
|
+
for network_acl in data_list:
|
|
49
|
+
network_acl_id = network_acl['NetworkAclId']
|
|
50
|
+
base_network_acl = {
|
|
51
|
+
'Id': network_acl_id,
|
|
52
|
+
'Arn': f'arn:aws:ec2:{region}:{current_aws_account_id}:network-acl/{network_acl_id}',
|
|
53
|
+
'IsDefault': network_acl['IsDefault'],
|
|
54
|
+
'VpcId': network_acl['VpcId'],
|
|
55
|
+
'OwnerId': network_acl['OwnerId'],
|
|
56
|
+
}
|
|
57
|
+
if network_acl.get('Associations') and network_acl['Associations']:
|
|
58
|
+
# Include subnet associations in the data object if they exist
|
|
59
|
+
for association in network_acl['Associations']:
|
|
60
|
+
base_network_acl['NetworkAclAssociationId'] = association['NetworkAclAssociationId']
|
|
61
|
+
base_network_acl['SubnetId'] = association['SubnetId']
|
|
62
|
+
network_acls.append(base_network_acl)
|
|
63
|
+
else:
|
|
64
|
+
# Otherwise if there's no associations then don't include that in the data object
|
|
65
|
+
network_acls.append(base_network_acl)
|
|
66
|
+
|
|
67
|
+
if network_acl.get("Entries"):
|
|
68
|
+
for rule in network_acl["Entries"]:
|
|
69
|
+
direction = 'egress' if rule['Egress'] else 'inbound'
|
|
70
|
+
transformed_rule = {
|
|
71
|
+
'Id': f"{network_acl['NetworkAclId']}/{direction}/{rule['RuleNumber']}",
|
|
72
|
+
'CidrBlock': rule['CidrBlock'],
|
|
73
|
+
'Egress': rule['Egress'],
|
|
74
|
+
'Protocol': rule['Protocol'],
|
|
75
|
+
'RuleAction': rule['RuleAction'],
|
|
76
|
+
'RuleNumber': rule['RuleNumber'],
|
|
77
|
+
# Add pointer back to the nacl to create an edge
|
|
78
|
+
'NetworkAclId': network_acl_id,
|
|
79
|
+
'FromPort': rule.get('PortRange', {}).get('FromPort'),
|
|
80
|
+
'ToPort': rule.get('PortRange', {}).get('ToPort'),
|
|
81
|
+
}
|
|
82
|
+
if transformed_rule['Egress']:
|
|
83
|
+
outbound_rules.append(transformed_rule)
|
|
84
|
+
else:
|
|
85
|
+
inbound_rules.append(transformed_rule)
|
|
86
|
+
return Ec2AclObjects(
|
|
87
|
+
network_acls=network_acls,
|
|
88
|
+
inbound_rules=inbound_rules,
|
|
89
|
+
outbound_rules=outbound_rules,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@timeit
|
|
94
|
+
def load_all_nacl_data(
|
|
95
|
+
neo4j_session: neo4j.Session,
|
|
96
|
+
ec2_acl_objects: Ec2AclObjects,
|
|
97
|
+
region: str,
|
|
98
|
+
aws_account_id: str,
|
|
99
|
+
update_tag: int,
|
|
100
|
+
) -> None:
|
|
101
|
+
load_network_acls(
|
|
102
|
+
neo4j_session,
|
|
103
|
+
ec2_acl_objects.network_acls,
|
|
104
|
+
region,
|
|
105
|
+
aws_account_id,
|
|
106
|
+
update_tag,
|
|
107
|
+
)
|
|
108
|
+
load_network_acl_inbound_rules(
|
|
109
|
+
neo4j_session,
|
|
110
|
+
ec2_acl_objects.inbound_rules,
|
|
111
|
+
region,
|
|
112
|
+
aws_account_id,
|
|
113
|
+
update_tag,
|
|
114
|
+
)
|
|
115
|
+
load_network_acl_egress_rules(
|
|
116
|
+
neo4j_session,
|
|
117
|
+
ec2_acl_objects.outbound_rules,
|
|
118
|
+
region,
|
|
119
|
+
aws_account_id,
|
|
120
|
+
update_tag,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@timeit
|
|
125
|
+
def load_network_acls(
|
|
126
|
+
neo4j_session: neo4j.Session,
|
|
127
|
+
data: list[dict[str, Any]],
|
|
128
|
+
region: str,
|
|
129
|
+
aws_account_id: str,
|
|
130
|
+
update_tag: int,
|
|
131
|
+
) -> None:
|
|
132
|
+
logger.info(f"Loading {len(data)} network acls in {region}.")
|
|
133
|
+
load(
|
|
134
|
+
neo4j_session,
|
|
135
|
+
EC2NetworkAclSchema(),
|
|
136
|
+
data,
|
|
137
|
+
Region=region,
|
|
138
|
+
AWS_ID=aws_account_id,
|
|
139
|
+
lastupdated=update_tag,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@timeit
|
|
144
|
+
def load_network_acl_inbound_rules(
|
|
145
|
+
neo4j_session: neo4j.Session,
|
|
146
|
+
data: list[dict[str, Any]],
|
|
147
|
+
region: str,
|
|
148
|
+
aws_account_id: str,
|
|
149
|
+
update_tag: int,
|
|
150
|
+
) -> None:
|
|
151
|
+
logger.info(f"Loading {len(data)} network acl inbound rules in {region}.")
|
|
152
|
+
load(
|
|
153
|
+
neo4j_session,
|
|
154
|
+
EC2NetworkAclInboundRuleSchema(),
|
|
155
|
+
data,
|
|
156
|
+
Region=region,
|
|
157
|
+
AWS_ID=aws_account_id,
|
|
158
|
+
lastupdated=update_tag,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@timeit
|
|
163
|
+
def load_network_acl_egress_rules(
|
|
164
|
+
neo4j_session: neo4j.Session,
|
|
165
|
+
data: list[dict[str, Any]],
|
|
166
|
+
region: str,
|
|
167
|
+
aws_account_id: str,
|
|
168
|
+
update_tag: int,
|
|
169
|
+
) -> None:
|
|
170
|
+
logger.info(f"Loading {len(data)} network acl egress rules in {region}.")
|
|
171
|
+
load(
|
|
172
|
+
neo4j_session,
|
|
173
|
+
EC2NetworkAclEgressRuleSchema(),
|
|
174
|
+
data,
|
|
175
|
+
Region=region,
|
|
176
|
+
AWS_ID=aws_account_id,
|
|
177
|
+
lastupdated=update_tag,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@timeit
|
|
182
|
+
def cleanup_network_acls(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None:
|
|
183
|
+
GraphJob.from_node_schema(EC2NetworkAclSchema(), common_job_parameters).run(neo4j_session)
|
|
184
|
+
GraphJob.from_node_schema(EC2NetworkAclInboundRuleSchema(), common_job_parameters).run(neo4j_session)
|
|
185
|
+
GraphJob.from_node_schema(EC2NetworkAclEgressRuleSchema(), common_job_parameters).run(neo4j_session)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@timeit
|
|
189
|
+
def sync_network_acls(
|
|
190
|
+
neo4j_session: neo4j.Session,
|
|
191
|
+
boto3_session: boto3.session.Session,
|
|
192
|
+
regions: list[str],
|
|
193
|
+
current_aws_account_id: str,
|
|
194
|
+
update_tag: int,
|
|
195
|
+
common_job_parameters: dict[str, Any],
|
|
196
|
+
) -> None:
|
|
197
|
+
for region in regions:
|
|
198
|
+
logger.info(f"Syncing EC2 network ACLs for region '{region}' in account '{current_aws_account_id}'.")
|
|
199
|
+
data = get_network_acl_data(boto3_session, region)
|
|
200
|
+
ec2_acl_data = transform_network_acl_data(data, region, current_aws_account_id)
|
|
201
|
+
load_all_nacl_data(
|
|
202
|
+
neo4j_session,
|
|
203
|
+
ec2_acl_data,
|
|
204
|
+
region,
|
|
205
|
+
current_aws_account_id,
|
|
206
|
+
update_tag,
|
|
207
|
+
)
|
|
208
|
+
cleanup_network_acls(neo4j_session, common_job_parameters)
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
from typing import Dict
|
|
4
|
+
from typing import List
|
|
5
|
+
|
|
6
|
+
import boto3
|
|
7
|
+
import neo4j
|
|
8
|
+
|
|
9
|
+
from cartography.client.core.tx import load
|
|
10
|
+
from cartography.graph.job import GraphJob
|
|
11
|
+
from cartography.models.aws.identitycenter.awsidentitycenter import AWSIdentityCenterInstanceSchema
|
|
12
|
+
from cartography.models.aws.identitycenter.awspermissionset import AWSPermissionSetSchema
|
|
13
|
+
from cartography.models.aws.identitycenter.awsssouser import AWSSSOUserSchema
|
|
14
|
+
from cartography.util import aws_handle_regions
|
|
15
|
+
from cartography.util import run_cleanup_job
|
|
16
|
+
from cartography.util import timeit
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@timeit
|
|
21
|
+
@aws_handle_regions
|
|
22
|
+
def get_identity_center_instances(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
|
|
23
|
+
"""
|
|
24
|
+
Get all AWS IAM Identity Center instances in the current region
|
|
25
|
+
"""
|
|
26
|
+
client = boto3_session.client('sso-admin', region_name=region)
|
|
27
|
+
instances = []
|
|
28
|
+
|
|
29
|
+
paginator = client.get_paginator('list_instances')
|
|
30
|
+
for page in paginator.paginate():
|
|
31
|
+
instances.extend(page.get('Instances', []))
|
|
32
|
+
|
|
33
|
+
return instances
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@timeit
|
|
37
|
+
def load_identity_center_instances(
|
|
38
|
+
neo4j_session: neo4j.Session,
|
|
39
|
+
instance_data: List[Dict],
|
|
40
|
+
region: str,
|
|
41
|
+
current_aws_account_id: str,
|
|
42
|
+
aws_update_tag: int,
|
|
43
|
+
) -> None:
|
|
44
|
+
"""
|
|
45
|
+
Load Identity Center instances into the graph
|
|
46
|
+
"""
|
|
47
|
+
logger.info(f"Loading {len(instance_data)} Identity Center instances for region {region}")
|
|
48
|
+
load(
|
|
49
|
+
neo4j_session,
|
|
50
|
+
AWSIdentityCenterInstanceSchema(),
|
|
51
|
+
instance_data,
|
|
52
|
+
lastupdated=aws_update_tag,
|
|
53
|
+
Region=region,
|
|
54
|
+
AWS_ID=current_aws_account_id,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@timeit
|
|
59
|
+
def get_permission_sets(boto3_session: boto3.session.Session, instance_arn: str, region: str) -> List[Dict]:
|
|
60
|
+
"""
|
|
61
|
+
Get all permission sets for a given Identity Center instance
|
|
62
|
+
"""
|
|
63
|
+
client = boto3_session.client('sso-admin', region_name=region)
|
|
64
|
+
permission_sets = []
|
|
65
|
+
|
|
66
|
+
paginator = client.get_paginator('list_permission_sets')
|
|
67
|
+
for page in paginator.paginate(InstanceArn=instance_arn):
|
|
68
|
+
# Get detailed info for each permission set
|
|
69
|
+
for arn in page.get('PermissionSets', []):
|
|
70
|
+
details = client.describe_permission_set(
|
|
71
|
+
InstanceArn=instance_arn,
|
|
72
|
+
PermissionSetArn=arn,
|
|
73
|
+
)
|
|
74
|
+
permission_set = details.get('PermissionSet', {})
|
|
75
|
+
if permission_set:
|
|
76
|
+
permission_set['RoleHint'] = (
|
|
77
|
+
f":role/aws-reserved/sso.amazonaws.com/AWSReservedSSO_{permission_set.get('Name')}"
|
|
78
|
+
)
|
|
79
|
+
permission_sets.append(permission_set)
|
|
80
|
+
|
|
81
|
+
return permission_sets
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@timeit
|
|
85
|
+
def get_permission_set_roles(
|
|
86
|
+
boto3_session: boto3.session.Session,
|
|
87
|
+
instance_arn: str,
|
|
88
|
+
permission_set_arn: str,
|
|
89
|
+
region: str,
|
|
90
|
+
) -> List[Dict]:
|
|
91
|
+
"""
|
|
92
|
+
Get all accounts associated with a given permission set
|
|
93
|
+
"""
|
|
94
|
+
client = boto3_session.client('sso-admin', region_name=region)
|
|
95
|
+
accounts = []
|
|
96
|
+
|
|
97
|
+
paginator = client.get_paginator('list_accounts_for_provisioned_permission_set')
|
|
98
|
+
for page in paginator.paginate(InstanceArn=instance_arn, PermissionSetArn=permission_set_arn):
|
|
99
|
+
accounts.extend(page.get('AccountIds', []))
|
|
100
|
+
|
|
101
|
+
return accounts
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@timeit
|
|
105
|
+
def load_permission_sets(
|
|
106
|
+
neo4j_session: neo4j.Session,
|
|
107
|
+
permission_sets: List[Dict],
|
|
108
|
+
instance_arn: str,
|
|
109
|
+
region: str,
|
|
110
|
+
aws_account_id: str,
|
|
111
|
+
aws_update_tag: int,
|
|
112
|
+
) -> None:
|
|
113
|
+
"""
|
|
114
|
+
Load Identity Center permission sets into the graph
|
|
115
|
+
"""
|
|
116
|
+
logger.info(f"Loading {len(permission_sets)} permission sets for instance {instance_arn} in region {region}")
|
|
117
|
+
|
|
118
|
+
load(
|
|
119
|
+
neo4j_session,
|
|
120
|
+
AWSPermissionSetSchema(),
|
|
121
|
+
permission_sets,
|
|
122
|
+
lastupdated=aws_update_tag,
|
|
123
|
+
InstanceArn=instance_arn,
|
|
124
|
+
Region=region,
|
|
125
|
+
AWS_ID=aws_account_id,
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@timeit
|
|
130
|
+
def get_sso_users(
|
|
131
|
+
boto3_session: boto3.session.Session,
|
|
132
|
+
identity_store_id: str,
|
|
133
|
+
region: str,
|
|
134
|
+
) -> List[Dict]:
|
|
135
|
+
"""
|
|
136
|
+
Get all SSO users for a given Identity Store
|
|
137
|
+
"""
|
|
138
|
+
client = boto3_session.client('identitystore', region_name=region)
|
|
139
|
+
users = []
|
|
140
|
+
|
|
141
|
+
paginator = client.get_paginator('list_users')
|
|
142
|
+
for page in paginator.paginate(IdentityStoreId=identity_store_id):
|
|
143
|
+
user_page = page.get('Users', [])
|
|
144
|
+
for user in user_page:
|
|
145
|
+
if user.get('ExternalIds', None):
|
|
146
|
+
user['ExternalId'] = user.get('ExternalIds')[0].get('Id')
|
|
147
|
+
users.append(user)
|
|
148
|
+
|
|
149
|
+
return users
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@timeit
|
|
153
|
+
def load_sso_users(
|
|
154
|
+
neo4j_session: neo4j.Session,
|
|
155
|
+
users: List[Dict],
|
|
156
|
+
identity_store_id: str,
|
|
157
|
+
region: str,
|
|
158
|
+
aws_account_id: str,
|
|
159
|
+
aws_update_tag: int,
|
|
160
|
+
) -> None:
|
|
161
|
+
"""
|
|
162
|
+
Load SSO users into the graph
|
|
163
|
+
"""
|
|
164
|
+
logger.info(f"Loading {len(users)} SSO users for identity store {identity_store_id} in region {region}")
|
|
165
|
+
|
|
166
|
+
load(
|
|
167
|
+
neo4j_session,
|
|
168
|
+
AWSSSOUserSchema(),
|
|
169
|
+
users,
|
|
170
|
+
lastupdated=aws_update_tag,
|
|
171
|
+
IdentityStoreId=identity_store_id,
|
|
172
|
+
AWS_ID=aws_account_id,
|
|
173
|
+
Region=region,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@timeit
|
|
178
|
+
def get_role_assignments(
|
|
179
|
+
boto3_session: boto3.session.Session,
|
|
180
|
+
users: List[Dict],
|
|
181
|
+
instance_arn: str,
|
|
182
|
+
region: str,
|
|
183
|
+
) -> List[Dict]:
|
|
184
|
+
"""
|
|
185
|
+
Get role assignments for SSO users
|
|
186
|
+
"""
|
|
187
|
+
|
|
188
|
+
logger.info(f"Getting role assignments for {len(users)} users")
|
|
189
|
+
client = boto3_session.client('sso-admin', region_name=region)
|
|
190
|
+
role_assignments = []
|
|
191
|
+
|
|
192
|
+
for user in users:
|
|
193
|
+
user_id = user['UserId']
|
|
194
|
+
paginator = client.get_paginator('list_account_assignments_for_principal')
|
|
195
|
+
for page in paginator.paginate(InstanceArn=instance_arn, PrincipalId=user_id, PrincipalType='USER'):
|
|
196
|
+
for assignment in page.get('AccountAssignments', []):
|
|
197
|
+
role_assignments.append({
|
|
198
|
+
'UserId': user_id,
|
|
199
|
+
'PermissionSetArn': assignment.get('PermissionSetArn'),
|
|
200
|
+
'AccountId': assignment.get('AccountId'),
|
|
201
|
+
})
|
|
202
|
+
|
|
203
|
+
return role_assignments
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@timeit
|
|
207
|
+
def load_role_assignments(
|
|
208
|
+
neo4j_session: neo4j.Session,
|
|
209
|
+
role_assignments: List[Dict],
|
|
210
|
+
aws_update_tag: int,
|
|
211
|
+
) -> None:
|
|
212
|
+
"""
|
|
213
|
+
Load role assignments into the graph
|
|
214
|
+
"""
|
|
215
|
+
logger.info(f"Loading {len(role_assignments)} role assignments")
|
|
216
|
+
if role_assignments:
|
|
217
|
+
neo4j_session.run(
|
|
218
|
+
"""
|
|
219
|
+
UNWIND $role_assignments AS ra
|
|
220
|
+
MATCH (acc:AWSAccount{id:ra.AccountId}) -[:RESOURCE]->
|
|
221
|
+
(role:AWSRole)<-[:ASSIGNED_TO_ROLE]-
|
|
222
|
+
(permset:AWSPermissionSet {id: ra.PermissionSetArn})
|
|
223
|
+
MATCH (sso:AWSSSOUser {id: ra.UserId})
|
|
224
|
+
MERGE (role)-[r:ALLOWED_BY]->(sso)
|
|
225
|
+
SET r.lastupdated = $aws_update_tag,
|
|
226
|
+
r.permission_set_arn = ra.PermissionSetArn
|
|
227
|
+
""",
|
|
228
|
+
role_assignments=role_assignments,
|
|
229
|
+
aws_update_tag=aws_update_tag,
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
|
|
234
|
+
GraphJob.from_node_schema(AWSIdentityCenterInstanceSchema(), common_job_parameters).run(neo4j_session)
|
|
235
|
+
GraphJob.from_node_schema(AWSPermissionSetSchema(), common_job_parameters).run(neo4j_session)
|
|
236
|
+
GraphJob.from_node_schema(AWSSSOUserSchema(), common_job_parameters).run(neo4j_session)
|
|
237
|
+
run_cleanup_job(
|
|
238
|
+
'aws_import_identity_center_cleanup.json',
|
|
239
|
+
neo4j_session,
|
|
240
|
+
common_job_parameters,
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
@timeit
|
|
245
|
+
def sync_identity_center_instances(
|
|
246
|
+
neo4j_session: neo4j.Session,
|
|
247
|
+
boto3_session: boto3.session.Session,
|
|
248
|
+
regions: List[str],
|
|
249
|
+
current_aws_account_id: str,
|
|
250
|
+
update_tag: int,
|
|
251
|
+
common_job_parameters: Dict,
|
|
252
|
+
) -> None:
|
|
253
|
+
"""
|
|
254
|
+
Sync Identity Center instances, their permission sets, and SSO users
|
|
255
|
+
"""
|
|
256
|
+
logger.info(f"Syncing Identity Center instances for regions {regions}")
|
|
257
|
+
for region in regions:
|
|
258
|
+
logger.info(f"Syncing Identity Center instances for region {region}")
|
|
259
|
+
instances = get_identity_center_instances(boto3_session, region)
|
|
260
|
+
load_identity_center_instances(
|
|
261
|
+
neo4j_session,
|
|
262
|
+
instances,
|
|
263
|
+
region,
|
|
264
|
+
current_aws_account_id,
|
|
265
|
+
update_tag,
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
# For each instance, get and load its permission sets and SSO users
|
|
269
|
+
for instance in instances:
|
|
270
|
+
instance_arn = instance['InstanceArn']
|
|
271
|
+
identity_store_id = instance['IdentityStoreId']
|
|
272
|
+
|
|
273
|
+
permission_sets = get_permission_sets(boto3_session, instance_arn, region)
|
|
274
|
+
|
|
275
|
+
load_permission_sets(
|
|
276
|
+
neo4j_session,
|
|
277
|
+
permission_sets,
|
|
278
|
+
instance_arn,
|
|
279
|
+
region,
|
|
280
|
+
current_aws_account_id,
|
|
281
|
+
update_tag,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
users = get_sso_users(boto3_session, identity_store_id, region)
|
|
285
|
+
load_sso_users(
|
|
286
|
+
neo4j_session,
|
|
287
|
+
users,
|
|
288
|
+
identity_store_id,
|
|
289
|
+
region,
|
|
290
|
+
current_aws_account_id,
|
|
291
|
+
update_tag,
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
# Get and load role assignments
|
|
295
|
+
role_assignments = get_role_assignments(
|
|
296
|
+
boto3_session,
|
|
297
|
+
users,
|
|
298
|
+
instance_arn,
|
|
299
|
+
region,
|
|
300
|
+
)
|
|
301
|
+
load_role_assignments(
|
|
302
|
+
neo4j_session,
|
|
303
|
+
role_assignments,
|
|
304
|
+
update_tag,
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
cleanup(neo4j_session, common_job_parameters)
|
|
@@ -10,6 +10,7 @@ from . import elasticache
|
|
|
10
10
|
from . import elasticsearch
|
|
11
11
|
from . import emr
|
|
12
12
|
from . import iam
|
|
13
|
+
from . import identitycenter
|
|
13
14
|
from . import inspector
|
|
14
15
|
from . import kms
|
|
15
16
|
from . import lambda_function
|
|
@@ -32,6 +33,7 @@ from .ec2.key_pairs import sync_ec2_key_pairs
|
|
|
32
33
|
from .ec2.launch_templates import sync_ec2_launch_templates
|
|
33
34
|
from .ec2.load_balancer_v2s import sync_load_balancer_v2s
|
|
34
35
|
from .ec2.load_balancers import sync_load_balancers
|
|
36
|
+
from .ec2.network_acls import sync_network_acls
|
|
35
37
|
from .ec2.network_interfaces import sync_network_interfaces
|
|
36
38
|
from .ec2.reserved_instances import sync_ec2_reserved_instances
|
|
37
39
|
from .ec2.security_groups import sync_ec2_security_groupinfo
|
|
@@ -55,6 +57,7 @@ RESOURCE_FUNCTIONS: Dict = {
|
|
|
55
57
|
'ec2:keypair': sync_ec2_key_pairs,
|
|
56
58
|
'ec2:load_balancer': sync_load_balancers,
|
|
57
59
|
'ec2:load_balancer_v2': sync_load_balancer_v2s,
|
|
60
|
+
'ec2:network_acls': sync_network_acls,
|
|
58
61
|
'ec2:network_interface': sync_network_interfaces,
|
|
59
62
|
'ec2:security_group': sync_ec2_security_groupinfo,
|
|
60
63
|
'ec2:subnet': sync_subnets,
|
|
@@ -86,4 +89,5 @@ RESOURCE_FUNCTIONS: Dict = {
|
|
|
86
89
|
'ssm': ssm.sync,
|
|
87
90
|
'inspector': inspector.sync,
|
|
88
91
|
'config': config.sync,
|
|
92
|
+
'identitycenter': identitycenter.sync_identity_center_instances,
|
|
89
93
|
}
|