cartography 0.117.0__py3-none-any.whl → 0.119.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/_version.py +2 -2
- cartography/cli.py +31 -0
- cartography/client/core/tx.py +19 -3
- cartography/config.py +14 -0
- cartography/data/indexes.cypher +0 -6
- cartography/graph/job.py +13 -7
- cartography/graph/statement.py +4 -0
- cartography/intel/aws/__init__.py +22 -9
- cartography/intel/aws/apigateway.py +18 -5
- cartography/intel/aws/ec2/elastic_ip_addresses.py +3 -1
- cartography/intel/aws/ec2/internet_gateways.py +4 -2
- cartography/intel/aws/ec2/load_balancer_v2s.py +11 -5
- cartography/intel/aws/ec2/network_interfaces.py +4 -0
- cartography/intel/aws/ec2/reserved_instances.py +3 -1
- cartography/intel/aws/ec2/tgw.py +11 -5
- cartography/intel/aws/ec2/volumes.py +1 -1
- cartography/intel/aws/ecr.py +209 -26
- cartography/intel/aws/ecr_image_layers.py +143 -42
- cartography/intel/aws/elasticsearch.py +13 -4
- cartography/intel/aws/identitycenter.py +93 -54
- cartography/intel/aws/inspector.py +90 -46
- cartography/intel/aws/permission_relationships.py +3 -3
- cartography/intel/aws/resourcegroupstaggingapi.py +1 -1
- cartography/intel/aws/s3.py +26 -13
- cartography/intel/aws/ssm.py +3 -5
- cartography/intel/azure/compute.py +9 -4
- cartography/intel/azure/cosmosdb.py +31 -15
- cartography/intel/azure/sql.py +25 -12
- cartography/intel/azure/storage.py +19 -9
- cartography/intel/azure/subscription.py +3 -1
- cartography/intel/crowdstrike/spotlight.py +5 -2
- cartography/intel/entra/app_role_assignments.py +9 -2
- cartography/intel/gcp/__init__.py +26 -9
- cartography/intel/gcp/clients.py +8 -4
- cartography/intel/gcp/compute.py +42 -21
- cartography/intel/gcp/crm/folders.py +9 -3
- cartography/intel/gcp/crm/orgs.py +8 -3
- cartography/intel/gcp/crm/projects.py +14 -3
- cartography/intel/github/repos.py +23 -5
- cartography/intel/gsuite/__init__.py +12 -8
- cartography/intel/gsuite/groups.py +291 -0
- cartography/intel/gsuite/users.py +142 -0
- cartography/intel/jamf/computers.py +7 -1
- cartography/intel/oci/iam.py +23 -9
- cartography/intel/oci/organizations.py +3 -1
- cartography/intel/oci/utils.py +28 -5
- cartography/intel/okta/awssaml.py +9 -8
- cartography/intel/okta/users.py +1 -1
- cartography/intel/ontology/__init__.py +44 -0
- cartography/intel/ontology/devices.py +54 -0
- cartography/intel/ontology/users.py +54 -0
- cartography/intel/ontology/utils.py +121 -0
- cartography/intel/pagerduty/escalation_policies.py +13 -6
- cartography/intel/pagerduty/schedules.py +9 -4
- cartography/intel/pagerduty/services.py +7 -3
- cartography/intel/pagerduty/teams.py +5 -2
- cartography/intel/pagerduty/users.py +3 -1
- cartography/intel/pagerduty/vendors.py +3 -1
- cartography/intel/trivy/__init__.py +109 -58
- cartography/models/airbyte/user.py +4 -0
- cartography/models/anthropic/user.py +4 -0
- cartography/models/aws/ec2/networkinterfaces.py +2 -0
- cartography/models/aws/ecr/image.py +55 -0
- cartography/models/aws/ecr/repository_image.py +1 -1
- cartography/models/aws/iam/group_membership.py +3 -2
- cartography/models/aws/identitycenter/awsssouser.py +3 -1
- cartography/models/bigfix/bigfix_computer.py +1 -1
- cartography/models/cloudflare/member.py +4 -0
- cartography/models/crowdstrike/hosts.py +1 -1
- cartography/models/duo/endpoint.py +1 -1
- cartography/models/duo/phone.py +2 -2
- cartography/models/duo/user.py +4 -0
- cartography/models/entra/user.py +2 -1
- cartography/models/github/users.py +4 -0
- cartography/models/gsuite/__init__.py +0 -0
- cartography/models/gsuite/group.py +218 -0
- cartography/models/gsuite/tenant.py +29 -0
- cartography/models/gsuite/user.py +107 -0
- cartography/models/kandji/device.py +1 -2
- cartography/models/keycloak/user.py +4 -0
- cartography/models/lastpass/user.py +4 -0
- cartography/models/ontology/__init__.py +0 -0
- cartography/models/ontology/device.py +125 -0
- cartography/models/ontology/mapping/__init__.py +16 -0
- cartography/models/ontology/mapping/data/__init__.py +1 -0
- cartography/models/ontology/mapping/data/devices.py +160 -0
- cartography/models/ontology/mapping/data/users.py +239 -0
- cartography/models/ontology/mapping/specs.py +65 -0
- cartography/models/ontology/user.py +52 -0
- cartography/models/openai/user.py +4 -0
- cartography/models/scaleway/iam/user.py +4 -0
- cartography/models/snipeit/asset.py +1 -0
- cartography/models/snipeit/user.py +4 -0
- cartography/models/tailscale/device.py +1 -1
- cartography/models/tailscale/user.py +6 -1
- cartography/rules/data/frameworks/mitre_attack/requirements/t1098_account_manipulation/__init__.py +176 -89
- cartography/sync.py +4 -1
- cartography/util.py +49 -18
- {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/METADATA +3 -3
- {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/RECORD +104 -89
- cartography/data/jobs/cleanup/gsuite_ingest_groups_cleanup.json +0 -23
- cartography/data/jobs/cleanup/gsuite_ingest_users_cleanup.json +0 -11
- cartography/intel/gsuite/api.py +0 -355
- {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/WHEEL +0 -0
- {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/entry_points.txt +0 -0
- {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/licenses/LICENSE +0 -0
- {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import neo4j
|
|
5
|
+
from googleapiclient.discovery import Resource
|
|
6
|
+
from googleapiclient.errors import HttpError
|
|
7
|
+
|
|
8
|
+
from cartography.client.core.tx import load
|
|
9
|
+
from cartography.client.core.tx import load_matchlinks
|
|
10
|
+
from cartography.graph.job import GraphJob
|
|
11
|
+
from cartography.models.gsuite.group import GSuiteGroupSchema
|
|
12
|
+
from cartography.models.gsuite.group import GSuiteGroupToGroupMemberRel
|
|
13
|
+
from cartography.models.gsuite.group import GSuiteGroupToGroupOwnerRel
|
|
14
|
+
from cartography.models.gsuite.tenant import GSuiteTenantSchema
|
|
15
|
+
from cartography.util import timeit
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
GOOGLE_API_NUM_RETRIES = 5
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@timeit
|
|
23
|
+
def get_all_groups(
|
|
24
|
+
admin: Resource, customer_id: str = "my_customer"
|
|
25
|
+
) -> list[dict[str, Any]]:
|
|
26
|
+
"""
|
|
27
|
+
Return list of Google Groups in your organization
|
|
28
|
+
Returns empty list if we are unable to enumerate the groups for any reasons
|
|
29
|
+
|
|
30
|
+
googleapiclient.discovery.build('admin', 'directory_v1', credentials=credentials, cache_discovery=False)
|
|
31
|
+
|
|
32
|
+
:param admin: google's apiclient discovery resource object. From googleapiclient.discovery.build
|
|
33
|
+
See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
|
|
34
|
+
:return: list of Google groups in domain
|
|
35
|
+
"""
|
|
36
|
+
request = admin.groups().list(
|
|
37
|
+
customer=customer_id,
|
|
38
|
+
maxResults=20,
|
|
39
|
+
orderBy="email",
|
|
40
|
+
)
|
|
41
|
+
response_objects = []
|
|
42
|
+
while request is not None:
|
|
43
|
+
try:
|
|
44
|
+
resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
|
|
45
|
+
response_objects.extend(resp.get("groups", []))
|
|
46
|
+
request = admin.groups().list_next(request, resp)
|
|
47
|
+
except HttpError as e:
|
|
48
|
+
if (
|
|
49
|
+
e.resp.status == 403
|
|
50
|
+
and "Request had insufficient authentication scopes" in str(e)
|
|
51
|
+
):
|
|
52
|
+
logger.error(
|
|
53
|
+
"Missing required GSuite scopes. If using the gcloud CLI, "
|
|
54
|
+
"run: gcloud auth application-default login --scopes="
|
|
55
|
+
'"https://www.googleapis.com/auth/admin.directory.user.readonly,'
|
|
56
|
+
"https://www.googleapis.com/auth/admin.directory.group.readonly,"
|
|
57
|
+
"https://www.googleapis.com/auth/admin.directory.group.member.readonly,"
|
|
58
|
+
'https://www.googleapis.com/auth/cloud-platform"'
|
|
59
|
+
)
|
|
60
|
+
raise
|
|
61
|
+
return response_objects
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@timeit
|
|
65
|
+
def get_members_for_groups(
|
|
66
|
+
admin: Resource, groups_email: list[str]
|
|
67
|
+
) -> dict[str, list[dict[str, Any]]]:
|
|
68
|
+
"""Get all members for given groups emails
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
admin (Resource): google's apiclient discovery resource object. From googleapiclient.discovery.build
|
|
72
|
+
See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
|
|
73
|
+
groups_email (list[str]): List of group email addresses to get members for
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
:return: list of dictionaries representing Users or Groups grouped by group email
|
|
77
|
+
"""
|
|
78
|
+
results: dict[str, list[dict]] = {}
|
|
79
|
+
for group_email in groups_email:
|
|
80
|
+
request = admin.members().list(
|
|
81
|
+
groupKey=group_email,
|
|
82
|
+
maxResults=500,
|
|
83
|
+
)
|
|
84
|
+
members: list[dict] = []
|
|
85
|
+
while request is not None:
|
|
86
|
+
resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
|
|
87
|
+
members = members + resp.get("members", [])
|
|
88
|
+
request = admin.members().list_next(request, resp)
|
|
89
|
+
results[group_email] = members
|
|
90
|
+
|
|
91
|
+
return results
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@timeit
|
|
95
|
+
def transform_groups(
|
|
96
|
+
groups: list[dict], group_memberships: dict[str, list[dict[str, Any]]]
|
|
97
|
+
) -> tuple[list[dict], list[dict], list[dict]]:
|
|
98
|
+
"""Strips list of API response objects to return list of group objects only and lists of subgroup relationships
|
|
99
|
+
|
|
100
|
+
:param groups: Raw groups from Google API
|
|
101
|
+
:param group_memberships: Group memberships data
|
|
102
|
+
:return: tuple of (groups, group_member_relationships, group_owner_relationships)
|
|
103
|
+
"""
|
|
104
|
+
transformed_groups: list[dict] = []
|
|
105
|
+
group_member_relationships: list[dict] = []
|
|
106
|
+
group_owner_relationships: list[dict] = []
|
|
107
|
+
|
|
108
|
+
for group in groups:
|
|
109
|
+
group_id = group["id"]
|
|
110
|
+
group_email = group["email"]
|
|
111
|
+
group["member_ids"] = []
|
|
112
|
+
group["owner_ids"] = []
|
|
113
|
+
|
|
114
|
+
for member in group_memberships.get(group_email, []):
|
|
115
|
+
if member["type"] == "GROUP":
|
|
116
|
+
# Create group-to-group relationships
|
|
117
|
+
relationship_data = {
|
|
118
|
+
"parent_group_id": group_id,
|
|
119
|
+
"subgroup_id": member.get("id"),
|
|
120
|
+
"role": member.get("role"),
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if member.get("role") == "OWNER":
|
|
124
|
+
group_owner_relationships.append(relationship_data)
|
|
125
|
+
else:
|
|
126
|
+
group_member_relationships.append(relationship_data)
|
|
127
|
+
continue
|
|
128
|
+
|
|
129
|
+
# Handle user memberships
|
|
130
|
+
if member.get("role") == "OWNER":
|
|
131
|
+
group["owner_ids"].append(member.get("id"))
|
|
132
|
+
group["member_ids"].append(member.get("id"))
|
|
133
|
+
|
|
134
|
+
transformed_groups.append(group)
|
|
135
|
+
|
|
136
|
+
return transformed_groups, group_member_relationships, group_owner_relationships
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@timeit
|
|
140
|
+
def load_gsuite_groups(
|
|
141
|
+
neo4j_session: neo4j.Session,
|
|
142
|
+
groups: list[dict],
|
|
143
|
+
customer_id: str,
|
|
144
|
+
gsuite_update_tag: int,
|
|
145
|
+
) -> None:
|
|
146
|
+
"""
|
|
147
|
+
Load GSuite groups using the modern data model
|
|
148
|
+
"""
|
|
149
|
+
logger.info("Ingesting %d gsuite groups", len(groups))
|
|
150
|
+
|
|
151
|
+
# Load tenant first if it doesn't exist
|
|
152
|
+
tenant_data = [{"id": customer_id}]
|
|
153
|
+
load(
|
|
154
|
+
neo4j_session,
|
|
155
|
+
GSuiteTenantSchema(),
|
|
156
|
+
tenant_data,
|
|
157
|
+
lastupdated=gsuite_update_tag,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
# Load groups with relationship to tenant
|
|
161
|
+
load(
|
|
162
|
+
neo4j_session,
|
|
163
|
+
GSuiteGroupSchema(),
|
|
164
|
+
groups,
|
|
165
|
+
lastupdated=gsuite_update_tag,
|
|
166
|
+
CUSTOMER_ID=customer_id,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@timeit
|
|
171
|
+
def load_gsuite_group_to_group_relationships(
|
|
172
|
+
neo4j_session: neo4j.Session,
|
|
173
|
+
group_member_relationships: list[dict],
|
|
174
|
+
group_owner_relationships: list[dict],
|
|
175
|
+
customer_id: str,
|
|
176
|
+
gsuite_update_tag: int,
|
|
177
|
+
) -> None:
|
|
178
|
+
"""
|
|
179
|
+
Load GSuite group-to-group relationships using MatchLinks
|
|
180
|
+
"""
|
|
181
|
+
logger.info(
|
|
182
|
+
"Ingesting %d group member relationships", len(group_member_relationships)
|
|
183
|
+
)
|
|
184
|
+
logger.info(
|
|
185
|
+
"Ingesting %d group owner relationships", len(group_owner_relationships)
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# Load group member relationships (Group -> Group MEMBER)
|
|
189
|
+
if group_member_relationships:
|
|
190
|
+
load_matchlinks(
|
|
191
|
+
neo4j_session,
|
|
192
|
+
GSuiteGroupToGroupMemberRel(),
|
|
193
|
+
group_member_relationships,
|
|
194
|
+
lastupdated=gsuite_update_tag,
|
|
195
|
+
_sub_resource_label="GSuiteTenant",
|
|
196
|
+
_sub_resource_id=customer_id,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
# Load group owner relationships (Group -> Group OWNER)
|
|
200
|
+
if group_owner_relationships:
|
|
201
|
+
load_matchlinks(
|
|
202
|
+
neo4j_session,
|
|
203
|
+
GSuiteGroupToGroupOwnerRel(),
|
|
204
|
+
group_owner_relationships,
|
|
205
|
+
lastupdated=gsuite_update_tag,
|
|
206
|
+
_sub_resource_label="GSuiteTenant",
|
|
207
|
+
_sub_resource_id=customer_id,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
@timeit
|
|
212
|
+
def cleanup_gsuite_groups(
|
|
213
|
+
neo4j_session: neo4j.Session,
|
|
214
|
+
common_job_parameters: dict[str, Any],
|
|
215
|
+
customer_id: str,
|
|
216
|
+
gsuite_update_tag: int,
|
|
217
|
+
) -> None:
|
|
218
|
+
"""
|
|
219
|
+
Clean up GSuite groups and group-to-group relationships using the modern data model
|
|
220
|
+
"""
|
|
221
|
+
logger.debug("Running GSuite groups cleanup job")
|
|
222
|
+
|
|
223
|
+
# Cleanup group nodes
|
|
224
|
+
GraphJob.from_node_schema(GSuiteGroupSchema(), common_job_parameters).run(
|
|
225
|
+
neo4j_session
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
# Cleanup group-to-group member relationships
|
|
229
|
+
GraphJob.from_matchlink(
|
|
230
|
+
GSuiteGroupToGroupMemberRel(),
|
|
231
|
+
"GSuiteTenant",
|
|
232
|
+
customer_id,
|
|
233
|
+
gsuite_update_tag,
|
|
234
|
+
).run(neo4j_session)
|
|
235
|
+
|
|
236
|
+
# Cleanup group-to-group owner relationships
|
|
237
|
+
GraphJob.from_matchlink(
|
|
238
|
+
GSuiteGroupToGroupOwnerRel(),
|
|
239
|
+
"GSuiteTenant",
|
|
240
|
+
customer_id,
|
|
241
|
+
gsuite_update_tag,
|
|
242
|
+
).run(neo4j_session)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@timeit
|
|
246
|
+
def sync_gsuite_groups(
|
|
247
|
+
neo4j_session: neo4j.Session,
|
|
248
|
+
admin: Resource,
|
|
249
|
+
gsuite_update_tag: int,
|
|
250
|
+
common_job_parameters: dict[str, Any],
|
|
251
|
+
) -> None:
|
|
252
|
+
"""
|
|
253
|
+
GET GSuite group objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
|
|
254
|
+
|
|
255
|
+
:param neo4j_session: The Neo4j session
|
|
256
|
+
:param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
|
|
257
|
+
See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
|
|
258
|
+
:param gsuite_update_tag: The timestamp value to set our new Neo4j nodes with
|
|
259
|
+
:param common_job_parameters: Parameters to carry to the Neo4j jobs
|
|
260
|
+
:return: Nothing
|
|
261
|
+
"""
|
|
262
|
+
logger.debug("Syncing GSuite Groups")
|
|
263
|
+
|
|
264
|
+
customer_id = common_job_parameters.get(
|
|
265
|
+
"CUSTOMER_ID", "my_customer"
|
|
266
|
+
) # Default to "my_customer" for backward compatibility
|
|
267
|
+
|
|
268
|
+
# 1. GET - Fetch data from API
|
|
269
|
+
resp_objs = get_all_groups(admin, customer_id)
|
|
270
|
+
group_members = get_members_for_groups(admin, [resp["email"] for resp in resp_objs])
|
|
271
|
+
|
|
272
|
+
# 2. TRANSFORM - Shape data for ingestion
|
|
273
|
+
groups, group_member_relationships, group_owner_relationships = transform_groups(
|
|
274
|
+
resp_objs, group_members
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
# 3. LOAD - Ingest to Neo4j using data model
|
|
278
|
+
load_gsuite_groups(neo4j_session, groups, customer_id, gsuite_update_tag)
|
|
279
|
+
|
|
280
|
+
# Load group-to-group relationships after groups are loaded
|
|
281
|
+
load_gsuite_group_to_group_relationships(
|
|
282
|
+
neo4j_session,
|
|
283
|
+
group_member_relationships,
|
|
284
|
+
group_owner_relationships,
|
|
285
|
+
customer_id,
|
|
286
|
+
gsuite_update_tag,
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
# 4. CLEANUP - Remove stale data
|
|
290
|
+
cleanup_params = {**common_job_parameters, "CUSTOMER_ID": customer_id}
|
|
291
|
+
cleanup_gsuite_groups(neo4j_session, cleanup_params, customer_id, gsuite_update_tag)
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from collections import defaultdict
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import neo4j
|
|
6
|
+
from googleapiclient.discovery import Resource
|
|
7
|
+
|
|
8
|
+
from cartography.client.core.tx import load
|
|
9
|
+
from cartography.graph.job import GraphJob
|
|
10
|
+
from cartography.models.gsuite.tenant import GSuiteTenantSchema
|
|
11
|
+
from cartography.models.gsuite.user import GSuiteUserSchema
|
|
12
|
+
from cartography.util import timeit
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
GOOGLE_API_NUM_RETRIES = 5
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@timeit
|
|
20
|
+
def get_all_users(admin: Resource) -> list[dict]:
|
|
21
|
+
"""
|
|
22
|
+
Return list of Google Users in your organization
|
|
23
|
+
Returns empty list if we are unable to enumerate the users for any reasons
|
|
24
|
+
https://developers.google.com/admin-sdk/directory/v1/guides/manage-users
|
|
25
|
+
|
|
26
|
+
:param admin: apiclient discovery resource object
|
|
27
|
+
:return: list of Google users in domain
|
|
28
|
+
see https://developers.google.com/admin-sdk/directory/v1/guides/manage-users#get_all_domain_users
|
|
29
|
+
"""
|
|
30
|
+
request = admin.users().list(
|
|
31
|
+
customer="my_customer",
|
|
32
|
+
maxResults=500,
|
|
33
|
+
orderBy="email",
|
|
34
|
+
)
|
|
35
|
+
response_objects = []
|
|
36
|
+
while request is not None:
|
|
37
|
+
resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
|
|
38
|
+
response_objects.append(resp)
|
|
39
|
+
request = admin.users().list_next(request, resp)
|
|
40
|
+
return response_objects
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@timeit
|
|
44
|
+
def transform_users(response_objects: list[dict]) -> dict[str, list[dict[str, Any]]]:
|
|
45
|
+
"""Transform list of API response objects to return list of user objects with flattened structure grouped by customerId
|
|
46
|
+
:param response_objects: Raw API response objects
|
|
47
|
+
:return: list of dictionary objects for data model consumption
|
|
48
|
+
"""
|
|
49
|
+
results = defaultdict(list)
|
|
50
|
+
for response_object in response_objects:
|
|
51
|
+
for user in response_object["users"]:
|
|
52
|
+
# Flatten the nested name structure
|
|
53
|
+
transformed_user = user.copy()
|
|
54
|
+
if "name" in user and isinstance(user["name"], dict):
|
|
55
|
+
transformed_user["name"] = user["name"].get("fullName")
|
|
56
|
+
transformed_user["family_name"] = user["name"].get("familyName")
|
|
57
|
+
transformed_user["given_name"] = user["name"].get("givenName")
|
|
58
|
+
results[transformed_user["customerId"]].append(transformed_user)
|
|
59
|
+
return results
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@timeit
|
|
63
|
+
def load_gsuite_users(
|
|
64
|
+
neo4j_session: neo4j.Session,
|
|
65
|
+
users_by_customer: dict[str, list[dict]],
|
|
66
|
+
gsuite_update_tag: int,
|
|
67
|
+
) -> None:
|
|
68
|
+
"""
|
|
69
|
+
Load GSuite users using the modern data model
|
|
70
|
+
"""
|
|
71
|
+
logger.info("Ingesting %s gsuite tenants", len(users_by_customer))
|
|
72
|
+
tenant_data = [{"id": customer_id} for customer_id in users_by_customer.keys()]
|
|
73
|
+
load(
|
|
74
|
+
neo4j_session,
|
|
75
|
+
GSuiteTenantSchema(),
|
|
76
|
+
tenant_data,
|
|
77
|
+
lastupdated=gsuite_update_tag,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
for customer_id, users in users_by_customer.items():
|
|
81
|
+
logger.info(
|
|
82
|
+
"Ingesting %s gsuite users for customer %s", len(users), customer_id
|
|
83
|
+
)
|
|
84
|
+
# Load users with relationship to tenant
|
|
85
|
+
load(
|
|
86
|
+
neo4j_session,
|
|
87
|
+
GSuiteUserSchema(),
|
|
88
|
+
users,
|
|
89
|
+
lastupdated=gsuite_update_tag,
|
|
90
|
+
CUSTOMER_ID=customer_id,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@timeit
|
|
95
|
+
def cleanup_gsuite_users(
|
|
96
|
+
neo4j_session: neo4j.Session,
|
|
97
|
+
common_job_parameters: dict[str, Any],
|
|
98
|
+
) -> None:
|
|
99
|
+
"""
|
|
100
|
+
Clean up GSuite users using the modern data model
|
|
101
|
+
"""
|
|
102
|
+
logger.debug("Running GSuite users cleanup job")
|
|
103
|
+
GraphJob.from_node_schema(GSuiteUserSchema(), common_job_parameters).run(
|
|
104
|
+
neo4j_session
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@timeit
|
|
109
|
+
def sync_gsuite_users(
|
|
110
|
+
neo4j_session: neo4j.Session,
|
|
111
|
+
admin: Resource,
|
|
112
|
+
gsuite_update_tag: int,
|
|
113
|
+
common_job_parameters: dict[str, Any],
|
|
114
|
+
) -> list[str]:
|
|
115
|
+
"""
|
|
116
|
+
GET GSuite user objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
|
|
117
|
+
|
|
118
|
+
:param neo4j_session: The Neo4j session
|
|
119
|
+
:param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
|
|
120
|
+
See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
|
|
121
|
+
:param gsuite_update_tag: The timestamp value to set our new Neo4j nodes with
|
|
122
|
+
:param common_job_parameters: Parameters to carry to the Neo4j jobs
|
|
123
|
+
:return: list of customer IDs
|
|
124
|
+
"""
|
|
125
|
+
logger.debug("Syncing GSuite Users")
|
|
126
|
+
|
|
127
|
+
# 1. GET - Fetch data from API
|
|
128
|
+
resp_objs = get_all_users(admin)
|
|
129
|
+
|
|
130
|
+
# 2. TRANSFORM - Shape data for ingestion
|
|
131
|
+
users_by_customers = transform_users(resp_objs)
|
|
132
|
+
|
|
133
|
+
# 3. LOAD - Ingest to Neo4j using data model
|
|
134
|
+
load_gsuite_users(neo4j_session, users_by_customers, gsuite_update_tag)
|
|
135
|
+
|
|
136
|
+
# 4. CLEANUP - Remove stale data
|
|
137
|
+
for customer_id in users_by_customers.keys():
|
|
138
|
+
cleanup_params = {**common_job_parameters, "CUSTOMER_ID": customer_id}
|
|
139
|
+
cleanup_gsuite_users(neo4j_session, cleanup_params)
|
|
140
|
+
|
|
141
|
+
# Return the list of customer IDs
|
|
142
|
+
return list(users_by_customers.keys())
|
|
@@ -4,6 +4,7 @@ from typing import List
|
|
|
4
4
|
|
|
5
5
|
import neo4j
|
|
6
6
|
|
|
7
|
+
from cartography.client.core.tx import run_write_query
|
|
7
8
|
from cartography.intel.jamf.util import call_jamf_api
|
|
8
9
|
from cartography.util import run_cleanup_job
|
|
9
10
|
from cartography.util import timeit
|
|
@@ -35,7 +36,12 @@ def load_computer_groups(
|
|
|
35
36
|
g.lastupdated = $UpdateTag
|
|
36
37
|
"""
|
|
37
38
|
groups = data.get("computer_groups")
|
|
38
|
-
|
|
39
|
+
run_write_query(
|
|
40
|
+
neo4j_session,
|
|
41
|
+
ingest_groups,
|
|
42
|
+
JsonData=groups,
|
|
43
|
+
UpdateTag=update_tag,
|
|
44
|
+
)
|
|
39
45
|
|
|
40
46
|
|
|
41
47
|
@timeit
|
cartography/intel/oci/iam.py
CHANGED
|
@@ -10,6 +10,8 @@ from typing import List
|
|
|
10
10
|
import neo4j
|
|
11
11
|
import oci
|
|
12
12
|
|
|
13
|
+
from cartography.client.core.tx import read_list_of_dicts_tx
|
|
14
|
+
from cartography.client.core.tx import run_write_query
|
|
13
15
|
from cartography.util import run_cleanup_job
|
|
14
16
|
|
|
15
17
|
from . import utils
|
|
@@ -89,7 +91,8 @@ def load_compartments(
|
|
|
89
91
|
"""
|
|
90
92
|
|
|
91
93
|
for compartment in compartments:
|
|
92
|
-
|
|
94
|
+
run_write_query(
|
|
95
|
+
neo4j_session,
|
|
93
96
|
ingest_compartment,
|
|
94
97
|
OCID=compartment["id"],
|
|
95
98
|
COMPARTMENT_ID=compartment["compartment-id"],
|
|
@@ -126,7 +129,8 @@ def load_users(
|
|
|
126
129
|
"""
|
|
127
130
|
|
|
128
131
|
for user in users:
|
|
129
|
-
|
|
132
|
+
run_write_query(
|
|
133
|
+
neo4j_session,
|
|
130
134
|
ingest_user,
|
|
131
135
|
OCID=user["id"],
|
|
132
136
|
CREATE_DATE=str(user["time-created"]),
|
|
@@ -206,7 +210,8 @@ def load_groups(
|
|
|
206
210
|
"""
|
|
207
211
|
|
|
208
212
|
for group in groups:
|
|
209
|
-
|
|
213
|
+
run_write_query(
|
|
214
|
+
neo4j_session,
|
|
210
215
|
ingest_group,
|
|
211
216
|
OCID=group["id"],
|
|
212
217
|
CREATE_DATE=str(group["time-created"]),
|
|
@@ -260,7 +265,11 @@ def sync_group_memberships(
|
|
|
260
265
|
"MATCH (group:OCIGroup)<-[:RESOURCE]-(OCITenancy{ocid: $OCI_TENANCY_ID}) "
|
|
261
266
|
"return group.name as name, group.ocid as ocid;"
|
|
262
267
|
)
|
|
263
|
-
groups = neo4j_session.
|
|
268
|
+
groups = neo4j_session.execute_read(
|
|
269
|
+
read_list_of_dicts_tx,
|
|
270
|
+
query,
|
|
271
|
+
OCI_TENANCY_ID=current_tenancy_id,
|
|
272
|
+
)
|
|
264
273
|
groups_membership = {
|
|
265
274
|
group["ocid"]: get_group_membership_data(iam, group["ocid"], current_tenancy_id)
|
|
266
275
|
for group in groups
|
|
@@ -288,7 +297,8 @@ def load_group_memberships(
|
|
|
288
297
|
"""
|
|
289
298
|
for group_ocid, membership_data in group_memberships.items():
|
|
290
299
|
for info in membership_data["GroupMemberships"]:
|
|
291
|
-
|
|
300
|
+
run_write_query(
|
|
301
|
+
neo4j_session,
|
|
292
302
|
ingest_membership,
|
|
293
303
|
COMPARTMENT_ID=info["compartment-id"],
|
|
294
304
|
GROUP_OCID=info["group-id"],
|
|
@@ -317,7 +327,8 @@ def load_policies(
|
|
|
317
327
|
"""
|
|
318
328
|
|
|
319
329
|
for policy in policies:
|
|
320
|
-
|
|
330
|
+
run_write_query(
|
|
331
|
+
neo4j_session,
|
|
321
332
|
ingest_policy,
|
|
322
333
|
OCID=policy["id"],
|
|
323
334
|
POLICY_NAME=policy["name"],
|
|
@@ -386,7 +397,8 @@ def load_oci_policy_group_reference(
|
|
|
386
397
|
ON CREATE SET r.firstseen = timestamp()
|
|
387
398
|
SET r.lastupdated = $oci_update_tag
|
|
388
399
|
"""
|
|
389
|
-
|
|
400
|
+
run_write_query(
|
|
401
|
+
neo4j_session,
|
|
390
402
|
ingest_policy_group_reference,
|
|
391
403
|
POLICY_ID=policy_id,
|
|
392
404
|
GROUP_ID=group_id,
|
|
@@ -408,7 +420,8 @@ def load_oci_policy_compartment_reference(
|
|
|
408
420
|
ON CREATE SET r.firstseen = timestamp()
|
|
409
421
|
SET r.lastupdated = $oci_update_tag
|
|
410
422
|
"""
|
|
411
|
-
|
|
423
|
+
run_write_query(
|
|
424
|
+
neo4j_session,
|
|
412
425
|
ingest_policy_compartment_reference,
|
|
413
426
|
POLICY_ID=policy_id,
|
|
414
427
|
COMPARTMENT_ID=compartment_id,
|
|
@@ -487,7 +500,8 @@ def load_region_subscriptions(
|
|
|
487
500
|
SET r.lastupdated = $oci_update_tag
|
|
488
501
|
"""
|
|
489
502
|
for region in regions:
|
|
490
|
-
|
|
503
|
+
run_write_query(
|
|
504
|
+
neo4j_session,
|
|
491
505
|
query,
|
|
492
506
|
REGION_KEY=region["region-key"],
|
|
493
507
|
REGION_NAME=region["region-name"],
|
|
@@ -11,6 +11,7 @@ from oci.exceptions import ConfigFileNotFound
|
|
|
11
11
|
from oci.exceptions import InvalidConfig
|
|
12
12
|
from oci.exceptions import ProfileNotFound
|
|
13
13
|
|
|
14
|
+
from cartography.client.core.tx import run_write_query
|
|
14
15
|
from cartography.util import run_cleanup_job
|
|
15
16
|
|
|
16
17
|
logger = logging.getLogger(__name__)
|
|
@@ -100,7 +101,8 @@ def load_oci_accounts(
|
|
|
100
101
|
SET aa.lastupdated = $oci_update_tag, aa.name = $ACCOUNT_NAME
|
|
101
102
|
"""
|
|
102
103
|
for name in oci_accounts:
|
|
103
|
-
|
|
104
|
+
run_write_query(
|
|
105
|
+
neo4j_session,
|
|
104
106
|
query,
|
|
105
107
|
TENANCY_ID=oci_accounts[name]["tenancy"],
|
|
106
108
|
ACCOUNT_NAME=name,
|
cartography/intel/oci/utils.py
CHANGED
|
@@ -7,6 +7,8 @@ from typing import List
|
|
|
7
7
|
|
|
8
8
|
import neo4j
|
|
9
9
|
|
|
10
|
+
from cartography.client.core.tx import read_list_of_dicts_tx
|
|
11
|
+
|
|
10
12
|
|
|
11
13
|
# Generic way to turn a OCI python object into the json response that you would see from calling the REST API.
|
|
12
14
|
def oci_object_to_json(in_obj: Any) -> List[Dict[str, Any]]:
|
|
@@ -36,7 +38,11 @@ def get_compartments_in_tenancy(
|
|
|
36
38
|
"return DISTINCT compartment.name as name, compartment.ocid as ocid, "
|
|
37
39
|
"compartment.compartmentid as compartmentid;"
|
|
38
40
|
)
|
|
39
|
-
return neo4j_session.
|
|
41
|
+
return neo4j_session.execute_read(
|
|
42
|
+
read_list_of_dicts_tx,
|
|
43
|
+
query,
|
|
44
|
+
OCI_TENANCY_ID=tenancy_id,
|
|
45
|
+
)
|
|
40
46
|
|
|
41
47
|
|
|
42
48
|
# Grab list of all groups in neo4j already populated by iam.
|
|
@@ -48,7 +54,11 @@ def get_groups_in_tenancy(
|
|
|
48
54
|
"MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(group:OCIGroup)"
|
|
49
55
|
"return DISTINCT group.name as name, group.ocid as ocid;"
|
|
50
56
|
)
|
|
51
|
-
return neo4j_session.
|
|
57
|
+
return neo4j_session.execute_read(
|
|
58
|
+
read_list_of_dicts_tx,
|
|
59
|
+
query,
|
|
60
|
+
OCI_TENANCY_ID=tenancy_id,
|
|
61
|
+
)
|
|
52
62
|
|
|
53
63
|
|
|
54
64
|
# Grab list of all policies in neo4j already populated by iam.
|
|
@@ -61,7 +71,11 @@ def get_policies_in_tenancy(
|
|
|
61
71
|
"return DISTINCT policy.name as name, policy.ocid as ocid, policy.statements as statements, "
|
|
62
72
|
"policy.compartmentid as compartmentid;"
|
|
63
73
|
)
|
|
64
|
-
return neo4j_session.
|
|
74
|
+
return neo4j_session.execute_read(
|
|
75
|
+
read_list_of_dicts_tx,
|
|
76
|
+
query,
|
|
77
|
+
OCI_TENANCY_ID=tenancy_id,
|
|
78
|
+
)
|
|
65
79
|
|
|
66
80
|
|
|
67
81
|
# Grab list of all regions in neo4j already populated by iam.
|
|
@@ -73,7 +87,11 @@ def get_regions_in_tenancy(
|
|
|
73
87
|
"MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-->(region:OCIRegion)"
|
|
74
88
|
"return DISTINCT region.name as name, region.key as key;"
|
|
75
89
|
)
|
|
76
|
-
return neo4j_session.
|
|
90
|
+
return neo4j_session.execute_read(
|
|
91
|
+
read_list_of_dicts_tx,
|
|
92
|
+
query,
|
|
93
|
+
OCI_TENANCY_ID=tenancy_id,
|
|
94
|
+
)
|
|
77
95
|
|
|
78
96
|
|
|
79
97
|
# Grab list of all security groups in neo4j already populated by network. Need to handle regions for this one.
|
|
@@ -88,4 +106,9 @@ def get_security_groups_in_tenancy(
|
|
|
88
106
|
"return DISTINCT security_group.name as name, security_group.ocid as ocid, security_group.compartmentid "
|
|
89
107
|
"as compartmentid;"
|
|
90
108
|
)
|
|
91
|
-
return neo4j_session.
|
|
109
|
+
return neo4j_session.execute_read(
|
|
110
|
+
read_list_of_dicts_tx,
|
|
111
|
+
query,
|
|
112
|
+
OCI_TENANCY_ID=tenancy_id,
|
|
113
|
+
OCI_REGION=region,
|
|
114
|
+
)
|
|
@@ -68,24 +68,25 @@ def query_for_okta_to_aws_role_mapping(
|
|
|
68
68
|
:param neo4j_session: session from the Neo4j server
|
|
69
69
|
:param mapping_regex: the regex used by the organization to map groups to aws roles
|
|
70
70
|
"""
|
|
71
|
-
query =
|
|
71
|
+
query = (
|
|
72
|
+
"MATCH (app:OktaApplication{name:'amazon_aws'})--(group:OktaGroup) "
|
|
73
|
+
"RETURN group.id AS group_id, group.name AS group_name"
|
|
74
|
+
)
|
|
72
75
|
|
|
73
76
|
group_to_role_mapping: List[Dict] = []
|
|
74
|
-
|
|
75
|
-
results = neo4j_session.run(query)
|
|
77
|
+
results = neo4j_session.execute_read(read_list_of_dicts_tx, query)
|
|
76
78
|
|
|
77
79
|
for res in results:
|
|
78
|
-
has_results = True
|
|
79
80
|
# input: okta group id, okta group name. output: aws role arn.
|
|
80
81
|
mapping = transform_okta_group_to_aws_role(
|
|
81
|
-
res["
|
|
82
|
-
res["
|
|
82
|
+
res["group_id"],
|
|
83
|
+
res["group_name"],
|
|
83
84
|
mapping_regex,
|
|
84
85
|
)
|
|
85
86
|
if mapping:
|
|
86
87
|
group_to_role_mapping.append(mapping)
|
|
87
88
|
|
|
88
|
-
if
|
|
89
|
+
if results and not group_to_role_mapping:
|
|
89
90
|
logger.warning(
|
|
90
91
|
"AWS Okta Application present, but no mappings were found. "
|
|
91
92
|
"Please verify the mapping regex is correct",
|
|
@@ -247,7 +248,7 @@ def _load_awssso_tx(
|
|
|
247
248
|
ingest_statement,
|
|
248
249
|
GROUP_TO_ROLE=[g._asdict() for g in group_to_role],
|
|
249
250
|
okta_update_tag=okta_update_tag,
|
|
250
|
-
)
|
|
251
|
+
).consume()
|
|
251
252
|
|
|
252
253
|
|
|
253
254
|
def _load_okta_group_to_awssso_roles(
|
cartography/intel/okta/users.py
CHANGED
|
@@ -161,7 +161,7 @@ def _load_okta_users(
|
|
|
161
161
|
new_user.password_changed = user_data.password_changed,
|
|
162
162
|
new_user.transition_to_status = user_data.transition_to_status,
|
|
163
163
|
new_user.lastupdated = $okta_update_tag,
|
|
164
|
-
new_user
|
|
164
|
+
new_user:UserAccount
|
|
165
165
|
WITH new_user, org
|
|
166
166
|
MERGE (org)-[org_r:RESOURCE]->(new_user)
|
|
167
167
|
ON CREATE SET org_r.firstseen = timestamp()
|