cartography 0.118.0__py3-none-any.whl → 0.119.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/_version.py +2 -2
- cartography/cli.py +20 -0
- cartography/client/core/tx.py +19 -3
- cartography/config.py +9 -0
- cartography/data/indexes.cypher +0 -6
- cartography/graph/job.py +7 -5
- cartography/intel/aws/__init__.py +21 -9
- cartography/intel/aws/ecr.py +7 -0
- cartography/intel/aws/ecr_image_layers.py +143 -42
- cartography/intel/aws/inspector.py +65 -33
- cartography/intel/aws/resourcegroupstaggingapi.py +1 -1
- cartography/intel/gcp/compute.py +3 -3
- cartography/intel/github/repos.py +23 -5
- cartography/intel/gsuite/__init__.py +12 -8
- cartography/intel/gsuite/groups.py +291 -0
- cartography/intel/gsuite/users.py +142 -0
- cartography/intel/okta/awssaml.py +1 -1
- cartography/intel/okta/users.py +1 -1
- cartography/intel/ontology/__init__.py +44 -0
- cartography/intel/ontology/devices.py +54 -0
- cartography/intel/ontology/users.py +54 -0
- cartography/intel/ontology/utils.py +121 -0
- cartography/models/airbyte/user.py +4 -0
- cartography/models/anthropic/user.py +4 -0
- cartography/models/aws/ecr/image.py +47 -0
- cartography/models/aws/iam/group_membership.py +3 -2
- cartography/models/aws/identitycenter/awsssouser.py +3 -1
- cartography/models/bigfix/bigfix_computer.py +1 -1
- cartography/models/cloudflare/member.py +4 -0
- cartography/models/crowdstrike/hosts.py +1 -1
- cartography/models/duo/endpoint.py +1 -1
- cartography/models/duo/phone.py +2 -2
- cartography/models/duo/user.py +4 -0
- cartography/models/entra/user.py +2 -1
- cartography/models/github/users.py +4 -0
- cartography/models/gsuite/__init__.py +0 -0
- cartography/models/gsuite/group.py +218 -0
- cartography/models/gsuite/tenant.py +29 -0
- cartography/models/gsuite/user.py +107 -0
- cartography/models/kandji/device.py +1 -2
- cartography/models/keycloak/user.py +4 -0
- cartography/models/lastpass/user.py +4 -0
- cartography/models/ontology/__init__.py +0 -0
- cartography/models/ontology/device.py +125 -0
- cartography/models/ontology/mapping/__init__.py +16 -0
- cartography/models/ontology/mapping/data/__init__.py +1 -0
- cartography/models/ontology/mapping/data/devices.py +160 -0
- cartography/models/ontology/mapping/data/users.py +239 -0
- cartography/models/ontology/mapping/specs.py +65 -0
- cartography/models/ontology/user.py +52 -0
- cartography/models/openai/user.py +4 -0
- cartography/models/scaleway/iam/user.py +4 -0
- cartography/models/snipeit/asset.py +1 -0
- cartography/models/snipeit/user.py +4 -0
- cartography/models/tailscale/device.py +1 -1
- cartography/models/tailscale/user.py +6 -1
- cartography/rules/data/frameworks/mitre_attack/requirements/t1098_account_manipulation/__init__.py +176 -89
- cartography/sync.py +3 -0
- cartography/util.py +44 -17
- {cartography-0.118.0.dist-info → cartography-0.119.0.dist-info}/METADATA +1 -1
- {cartography-0.118.0.dist-info → cartography-0.119.0.dist-info}/RECORD +65 -50
- cartography/data/jobs/cleanup/gsuite_ingest_groups_cleanup.json +0 -23
- cartography/data/jobs/cleanup/gsuite_ingest_users_cleanup.json +0 -11
- cartography/intel/gsuite/api.py +0 -355
- {cartography-0.118.0.dist-info → cartography-0.119.0.dist-info}/WHEEL +0 -0
- {cartography-0.118.0.dist-info → cartography-0.119.0.dist-info}/entry_points.txt +0 -0
- {cartography-0.118.0.dist-info → cartography-0.119.0.dist-info}/licenses/LICENSE +0 -0
- {cartography-0.118.0.dist-info → cartography-0.119.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from collections import defaultdict
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import neo4j
|
|
6
|
+
from googleapiclient.discovery import Resource
|
|
7
|
+
|
|
8
|
+
from cartography.client.core.tx import load
|
|
9
|
+
from cartography.graph.job import GraphJob
|
|
10
|
+
from cartography.models.gsuite.tenant import GSuiteTenantSchema
|
|
11
|
+
from cartography.models.gsuite.user import GSuiteUserSchema
|
|
12
|
+
from cartography.util import timeit
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
GOOGLE_API_NUM_RETRIES = 5
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@timeit
|
|
20
|
+
def get_all_users(admin: Resource) -> list[dict]:
|
|
21
|
+
"""
|
|
22
|
+
Return list of Google Users in your organization
|
|
23
|
+
Returns empty list if we are unable to enumerate the users for any reasons
|
|
24
|
+
https://developers.google.com/admin-sdk/directory/v1/guides/manage-users
|
|
25
|
+
|
|
26
|
+
:param admin: apiclient discovery resource object
|
|
27
|
+
:return: list of Google users in domain
|
|
28
|
+
see https://developers.google.com/admin-sdk/directory/v1/guides/manage-users#get_all_domain_users
|
|
29
|
+
"""
|
|
30
|
+
request = admin.users().list(
|
|
31
|
+
customer="my_customer",
|
|
32
|
+
maxResults=500,
|
|
33
|
+
orderBy="email",
|
|
34
|
+
)
|
|
35
|
+
response_objects = []
|
|
36
|
+
while request is not None:
|
|
37
|
+
resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
|
|
38
|
+
response_objects.append(resp)
|
|
39
|
+
request = admin.users().list_next(request, resp)
|
|
40
|
+
return response_objects
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@timeit
|
|
44
|
+
def transform_users(response_objects: list[dict]) -> dict[str, list[dict[str, Any]]]:
|
|
45
|
+
"""Transform list of API response objects to return list of user objects with flattened structure grouped by customerId
|
|
46
|
+
:param response_objects: Raw API response objects
|
|
47
|
+
:return: list of dictionary objects for data model consumption
|
|
48
|
+
"""
|
|
49
|
+
results = defaultdict(list)
|
|
50
|
+
for response_object in response_objects:
|
|
51
|
+
for user in response_object["users"]:
|
|
52
|
+
# Flatten the nested name structure
|
|
53
|
+
transformed_user = user.copy()
|
|
54
|
+
if "name" in user and isinstance(user["name"], dict):
|
|
55
|
+
transformed_user["name"] = user["name"].get("fullName")
|
|
56
|
+
transformed_user["family_name"] = user["name"].get("familyName")
|
|
57
|
+
transformed_user["given_name"] = user["name"].get("givenName")
|
|
58
|
+
results[transformed_user["customerId"]].append(transformed_user)
|
|
59
|
+
return results
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@timeit
|
|
63
|
+
def load_gsuite_users(
|
|
64
|
+
neo4j_session: neo4j.Session,
|
|
65
|
+
users_by_customer: dict[str, list[dict]],
|
|
66
|
+
gsuite_update_tag: int,
|
|
67
|
+
) -> None:
|
|
68
|
+
"""
|
|
69
|
+
Load GSuite users using the modern data model
|
|
70
|
+
"""
|
|
71
|
+
logger.info("Ingesting %s gsuite tenants", len(users_by_customer))
|
|
72
|
+
tenant_data = [{"id": customer_id} for customer_id in users_by_customer.keys()]
|
|
73
|
+
load(
|
|
74
|
+
neo4j_session,
|
|
75
|
+
GSuiteTenantSchema(),
|
|
76
|
+
tenant_data,
|
|
77
|
+
lastupdated=gsuite_update_tag,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
for customer_id, users in users_by_customer.items():
|
|
81
|
+
logger.info(
|
|
82
|
+
"Ingesting %s gsuite users for customer %s", len(users), customer_id
|
|
83
|
+
)
|
|
84
|
+
# Load users with relationship to tenant
|
|
85
|
+
load(
|
|
86
|
+
neo4j_session,
|
|
87
|
+
GSuiteUserSchema(),
|
|
88
|
+
users,
|
|
89
|
+
lastupdated=gsuite_update_tag,
|
|
90
|
+
CUSTOMER_ID=customer_id,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@timeit
|
|
95
|
+
def cleanup_gsuite_users(
|
|
96
|
+
neo4j_session: neo4j.Session,
|
|
97
|
+
common_job_parameters: dict[str, Any],
|
|
98
|
+
) -> None:
|
|
99
|
+
"""
|
|
100
|
+
Clean up GSuite users using the modern data model
|
|
101
|
+
"""
|
|
102
|
+
logger.debug("Running GSuite users cleanup job")
|
|
103
|
+
GraphJob.from_node_schema(GSuiteUserSchema(), common_job_parameters).run(
|
|
104
|
+
neo4j_session
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@timeit
|
|
109
|
+
def sync_gsuite_users(
|
|
110
|
+
neo4j_session: neo4j.Session,
|
|
111
|
+
admin: Resource,
|
|
112
|
+
gsuite_update_tag: int,
|
|
113
|
+
common_job_parameters: dict[str, Any],
|
|
114
|
+
) -> list[str]:
|
|
115
|
+
"""
|
|
116
|
+
GET GSuite user objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
|
|
117
|
+
|
|
118
|
+
:param neo4j_session: The Neo4j session
|
|
119
|
+
:param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
|
|
120
|
+
See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
|
|
121
|
+
:param gsuite_update_tag: The timestamp value to set our new Neo4j nodes with
|
|
122
|
+
:param common_job_parameters: Parameters to carry to the Neo4j jobs
|
|
123
|
+
:return: list of customer IDs
|
|
124
|
+
"""
|
|
125
|
+
logger.debug("Syncing GSuite Users")
|
|
126
|
+
|
|
127
|
+
# 1. GET - Fetch data from API
|
|
128
|
+
resp_objs = get_all_users(admin)
|
|
129
|
+
|
|
130
|
+
# 2. TRANSFORM - Shape data for ingestion
|
|
131
|
+
users_by_customers = transform_users(resp_objs)
|
|
132
|
+
|
|
133
|
+
# 3. LOAD - Ingest to Neo4j using data model
|
|
134
|
+
load_gsuite_users(neo4j_session, users_by_customers, gsuite_update_tag)
|
|
135
|
+
|
|
136
|
+
# 4. CLEANUP - Remove stale data
|
|
137
|
+
for customer_id in users_by_customers.keys():
|
|
138
|
+
cleanup_params = {**common_job_parameters, "CUSTOMER_ID": customer_id}
|
|
139
|
+
cleanup_gsuite_users(neo4j_session, cleanup_params)
|
|
140
|
+
|
|
141
|
+
# Return the list of customer IDs
|
|
142
|
+
return list(users_by_customers.keys())
|
cartography/intel/okta/users.py
CHANGED
|
@@ -161,7 +161,7 @@ def _load_okta_users(
|
|
|
161
161
|
new_user.password_changed = user_data.password_changed,
|
|
162
162
|
new_user.transition_to_status = user_data.transition_to_status,
|
|
163
163
|
new_user.lastupdated = $okta_update_tag,
|
|
164
|
-
new_user
|
|
164
|
+
new_user:UserAccount
|
|
165
165
|
WITH new_user, org
|
|
166
166
|
MERGE (org)-[org_r:RESOURCE]->(new_user)
|
|
167
167
|
ON CREATE SET org_r.firstseen = timestamp()
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
import neo4j
|
|
4
|
+
|
|
5
|
+
import cartography.intel.ontology.devices
|
|
6
|
+
import cartography.intel.ontology.users
|
|
7
|
+
from cartography.config import Config
|
|
8
|
+
from cartography.util import timeit
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@timeit
|
|
14
|
+
def run(neo4j_session: neo4j.Session, config: Config) -> None:
|
|
15
|
+
common_job_parameters = {
|
|
16
|
+
"UPDATE_TAG": config.update_tag,
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
# Get source of truth from config
|
|
20
|
+
if config.ontology_users_source:
|
|
21
|
+
users_source_of_truth = [
|
|
22
|
+
source.strip() for source in config.ontology_users_source.split(",")
|
|
23
|
+
]
|
|
24
|
+
else:
|
|
25
|
+
users_source_of_truth = []
|
|
26
|
+
if config.ontology_devices_source:
|
|
27
|
+
computers_source_of_truth = [
|
|
28
|
+
source.strip() for source in config.ontology_devices_source.split(",")
|
|
29
|
+
]
|
|
30
|
+
else:
|
|
31
|
+
computers_source_of_truth = []
|
|
32
|
+
|
|
33
|
+
cartography.intel.ontology.users.sync(
|
|
34
|
+
neo4j_session,
|
|
35
|
+
users_source_of_truth,
|
|
36
|
+
config.update_tag,
|
|
37
|
+
common_job_parameters,
|
|
38
|
+
)
|
|
39
|
+
cartography.intel.ontology.devices.sync(
|
|
40
|
+
neo4j_session,
|
|
41
|
+
computers_source_of_truth,
|
|
42
|
+
config.update_tag,
|
|
43
|
+
common_job_parameters,
|
|
44
|
+
)
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import neo4j
|
|
5
|
+
|
|
6
|
+
from cartography.client.core.tx import load
|
|
7
|
+
from cartography.graph.job import GraphJob
|
|
8
|
+
from cartography.intel.ontology.utils import get_source_nodes_from_graph
|
|
9
|
+
from cartography.intel.ontology.utils import link_ontology_nodes
|
|
10
|
+
from cartography.models.ontology.device import DeviceSchema
|
|
11
|
+
from cartography.util import timeit
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@timeit
|
|
17
|
+
def sync(
|
|
18
|
+
neo4j_session: neo4j.Session,
|
|
19
|
+
source_of_truth: list[str],
|
|
20
|
+
update_tag: int,
|
|
21
|
+
common_job_parameters: dict[str, Any],
|
|
22
|
+
) -> None:
|
|
23
|
+
data = get_source_nodes_from_graph(neo4j_session, source_of_truth, "devices")
|
|
24
|
+
load_devices(
|
|
25
|
+
neo4j_session,
|
|
26
|
+
data,
|
|
27
|
+
update_tag,
|
|
28
|
+
)
|
|
29
|
+
link_ontology_nodes(neo4j_session, "devices", update_tag)
|
|
30
|
+
cleanup(neo4j_session, common_job_parameters)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@timeit
|
|
34
|
+
def load_devices(
|
|
35
|
+
neo4j_session: neo4j.Session,
|
|
36
|
+
data: list[dict[str, Any]],
|
|
37
|
+
update_tag: int,
|
|
38
|
+
) -> None:
|
|
39
|
+
load(
|
|
40
|
+
neo4j_session,
|
|
41
|
+
DeviceSchema(),
|
|
42
|
+
data,
|
|
43
|
+
lastupdated=update_tag,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@timeit
|
|
48
|
+
def cleanup(
|
|
49
|
+
neo4j_session: neo4j.Session,
|
|
50
|
+
common_job_parameters: dict[str, Any],
|
|
51
|
+
) -> None:
|
|
52
|
+
GraphJob.from_node_schema(DeviceSchema(), common_job_parameters).run(
|
|
53
|
+
neo4j_session,
|
|
54
|
+
)
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import neo4j
|
|
5
|
+
|
|
6
|
+
from cartography.client.core.tx import load
|
|
7
|
+
from cartography.graph.job import GraphJob
|
|
8
|
+
from cartography.intel.ontology.utils import get_source_nodes_from_graph
|
|
9
|
+
from cartography.intel.ontology.utils import link_ontology_nodes
|
|
10
|
+
from cartography.models.ontology.user import UserSchema
|
|
11
|
+
from cartography.util import timeit
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@timeit
|
|
17
|
+
def sync(
|
|
18
|
+
neo4j_session: neo4j.Session,
|
|
19
|
+
source_of_truth: list[str],
|
|
20
|
+
update_tag: int,
|
|
21
|
+
common_job_parameters: dict[str, Any],
|
|
22
|
+
) -> None:
|
|
23
|
+
data = get_source_nodes_from_graph(neo4j_session, source_of_truth, "users")
|
|
24
|
+
load_users(
|
|
25
|
+
neo4j_session,
|
|
26
|
+
data,
|
|
27
|
+
update_tag,
|
|
28
|
+
)
|
|
29
|
+
link_ontology_nodes(neo4j_session, "users", update_tag)
|
|
30
|
+
cleanup(neo4j_session, common_job_parameters)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@timeit
|
|
34
|
+
def load_users(
|
|
35
|
+
neo4j_session: neo4j.Session,
|
|
36
|
+
data: list[dict[str, Any]],
|
|
37
|
+
update_tag: int,
|
|
38
|
+
) -> None:
|
|
39
|
+
load(
|
|
40
|
+
neo4j_session,
|
|
41
|
+
UserSchema(),
|
|
42
|
+
data,
|
|
43
|
+
lastupdated=update_tag,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@timeit
|
|
48
|
+
def cleanup(
|
|
49
|
+
neo4j_session: neo4j.Session,
|
|
50
|
+
common_job_parameters: dict[str, Any],
|
|
51
|
+
) -> None:
|
|
52
|
+
GraphJob.from_node_schema(UserSchema(), common_job_parameters).run(
|
|
53
|
+
neo4j_session,
|
|
54
|
+
)
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from dataclasses import asdict
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import neo4j
|
|
6
|
+
|
|
7
|
+
from cartography.client.core.tx import read_list_of_dicts_tx
|
|
8
|
+
from cartography.graph.job import GraphJob
|
|
9
|
+
from cartography.models.ontology.mapping import ONTOLOGY_MAPPING
|
|
10
|
+
from cartography.models.ontology.mapping import ONTOLOGY_MODELS
|
|
11
|
+
from cartography.util import timeit
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@timeit
|
|
17
|
+
def get_source_nodes_from_graph(
|
|
18
|
+
neo4j_session: neo4j.Session,
|
|
19
|
+
source_of_truth: list[str],
|
|
20
|
+
module_name: str,
|
|
21
|
+
) -> list[dict[str, Any]]:
|
|
22
|
+
"""Retrieve source nodes from the Neo4j graph database based on the ontology mapping.
|
|
23
|
+
|
|
24
|
+
This function queries the Neo4j database for nodes that match the labels
|
|
25
|
+
defined in the ontology mapping for the specified module and source of truth.
|
|
26
|
+
It returns a list of dictionaries containing the relevant fields for each node.
|
|
27
|
+
|
|
28
|
+
If no source of truth is provided, default to all sources defined in the mapping.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
neo4j_session (neo4j.Session): The Neo4j session to use for querying the database.
|
|
32
|
+
source_of_truth (list[str]): A list of source of truth identifiers to filter the modules.
|
|
33
|
+
module_name (str): The name of the ontology module to use for the mapping (eg. users, devices, etc.).
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
list[dict[str, Any]]: A list of dictionaries, each containing a node details formatted according to the ontology mapping.
|
|
37
|
+
"""
|
|
38
|
+
results: dict[str, dict[str, Any]] = {}
|
|
39
|
+
modules_mapping = ONTOLOGY_MAPPING[module_name]
|
|
40
|
+
if len(source_of_truth) == 0:
|
|
41
|
+
source_of_truth = list(modules_mapping.keys())
|
|
42
|
+
for source in source_of_truth:
|
|
43
|
+
if source not in modules_mapping:
|
|
44
|
+
logger.warning(
|
|
45
|
+
"Source of truth '%s' is not supported for '%s'.", source, module_name
|
|
46
|
+
)
|
|
47
|
+
continue
|
|
48
|
+
for node in modules_mapping[source].nodes:
|
|
49
|
+
query = f"MATCH (n:{node.node_label}) RETURN n"
|
|
50
|
+
for row in neo4j_session.execute_read(read_list_of_dicts_tx, query):
|
|
51
|
+
node_data = row["n"]
|
|
52
|
+
result: dict[str, Any] = {}
|
|
53
|
+
skip_node: bool = False
|
|
54
|
+
|
|
55
|
+
# Extract only the fields defined in the ontology mapping
|
|
56
|
+
for field in node.fields:
|
|
57
|
+
value = node_data.get(field.node_field)
|
|
58
|
+
# Skip nodes missing required fields
|
|
59
|
+
if field.required and value is None:
|
|
60
|
+
logger.debug(
|
|
61
|
+
"Skipping node with label '%s' due to missing required field '%s'.",
|
|
62
|
+
node.node_label,
|
|
63
|
+
field.node_field,
|
|
64
|
+
)
|
|
65
|
+
skip_node = True
|
|
66
|
+
break
|
|
67
|
+
result[field.ontology_field] = value
|
|
68
|
+
if skip_node:
|
|
69
|
+
continue
|
|
70
|
+
|
|
71
|
+
# Merge results based on the node's id field to avoid duplicates
|
|
72
|
+
id_field = ONTOLOGY_MODELS[module_name]().properties.id.name
|
|
73
|
+
existing = results.get(result[id_field])
|
|
74
|
+
if existing:
|
|
75
|
+
logger.debug(
|
|
76
|
+
"Merging node: %s to %s", result[id_field], existing[id_field]
|
|
77
|
+
)
|
|
78
|
+
# Merge existing data with new data, prioritizing non-None values
|
|
79
|
+
for key, value in result.items():
|
|
80
|
+
if existing.get(key) is None and value is not None:
|
|
81
|
+
existing[key] = value
|
|
82
|
+
else:
|
|
83
|
+
logger.debug("Adding new node: %s", result[id_field])
|
|
84
|
+
results[result[id_field]] = result
|
|
85
|
+
return list(results.values())
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@timeit
|
|
89
|
+
def link_ontology_nodes(
|
|
90
|
+
neo4j_session: neo4j.Session,
|
|
91
|
+
module_name: str,
|
|
92
|
+
update_tag: int,
|
|
93
|
+
) -> None:
|
|
94
|
+
"""Link ontology nodes in the Neo4j graph database based on the ontology mapping.
|
|
95
|
+
|
|
96
|
+
This function retrieves the ontology mapping for the specified module and
|
|
97
|
+
executes the relationship statements defined in the mapping to link nodes
|
|
98
|
+
in the Neo4j graph database.
|
|
99
|
+
|
|
100
|
+
Args:
|
|
101
|
+
neo4j_session (neo4j.Session): The Neo4j session to use for executing the relationship statements.
|
|
102
|
+
module_name (str): The name of the ontology module for which to link nodes (eg. users, devices, etc.).
|
|
103
|
+
update_tag (int): The update tag of the current run, used to tag the changes in the graph.
|
|
104
|
+
"""
|
|
105
|
+
modules_mapping = ONTOLOGY_MAPPING.get(module_name)
|
|
106
|
+
if modules_mapping is None:
|
|
107
|
+
logger.warning("No ontology mapping found for module '%s'.", module_name)
|
|
108
|
+
return
|
|
109
|
+
for source, mapping in modules_mapping.items():
|
|
110
|
+
if len(mapping.rels) == 0:
|
|
111
|
+
continue
|
|
112
|
+
formated_json = {
|
|
113
|
+
"name": f"Linking ontology nodes for {module_name} for source {source}",
|
|
114
|
+
"statements": [asdict(rel) for rel in mapping.rels],
|
|
115
|
+
}
|
|
116
|
+
GraphJob.run_from_json(
|
|
117
|
+
neo4j_session,
|
|
118
|
+
formated_json,
|
|
119
|
+
{"UPDATE_TAG": update_tag},
|
|
120
|
+
short_name=f"ontology.{module_name}.{source}.linking",
|
|
121
|
+
)
|
|
@@ -3,6 +3,7 @@ from dataclasses import dataclass
|
|
|
3
3
|
from cartography.models.core.common import PropertyRef
|
|
4
4
|
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
5
|
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.nodes import ExtraNodeLabels
|
|
6
7
|
from cartography.models.core.relationships import CartographyRelProperties
|
|
7
8
|
from cartography.models.core.relationships import CartographyRelSchema
|
|
8
9
|
from cartography.models.core.relationships import LinkDirection
|
|
@@ -98,6 +99,9 @@ class AirbyteUserToWorkspaceMemberRel(CartographyRelSchema):
|
|
|
98
99
|
@dataclass(frozen=True)
|
|
99
100
|
class AirbyteUserSchema(CartographyNodeSchema):
|
|
100
101
|
label: str = "AirbyteUser"
|
|
102
|
+
extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
|
|
103
|
+
["UserAccount"]
|
|
104
|
+
) # UserAccount label is used for ontology mapping
|
|
101
105
|
properties: AirbyteUserNodeProperties = AirbyteUserNodeProperties()
|
|
102
106
|
sub_resource_relationship: AirbyteUserToOrganizationRel = (
|
|
103
107
|
AirbyteUserToOrganizationRel()
|
|
@@ -3,6 +3,7 @@ from dataclasses import dataclass
|
|
|
3
3
|
from cartography.models.core.common import PropertyRef
|
|
4
4
|
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
5
|
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.nodes import ExtraNodeLabels
|
|
6
7
|
from cartography.models.core.relationships import CartographyRelProperties
|
|
7
8
|
from cartography.models.core.relationships import CartographyRelSchema
|
|
8
9
|
from cartography.models.core.relationships import LinkDirection
|
|
@@ -42,6 +43,9 @@ class AnthropicUserToOrganizationRel(CartographyRelSchema):
|
|
|
42
43
|
@dataclass(frozen=True)
|
|
43
44
|
class AnthropicUserSchema(CartographyNodeSchema):
|
|
44
45
|
label: str = "AnthropicUser"
|
|
46
|
+
extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
|
|
47
|
+
["UserAccount"]
|
|
48
|
+
) # UserAccount label is used for ontology mapping
|
|
45
49
|
properties: AnthropicUserNodeProperties = AnthropicUserNodeProperties()
|
|
46
50
|
sub_resource_relationship: AnthropicUserToOrganizationRel = (
|
|
47
51
|
AnthropicUserToOrganizationRel()
|
|
@@ -26,6 +26,7 @@ class ECRImageNodeProperties(CartographyNodeProperties):
|
|
|
26
26
|
attests_digest: PropertyRef = PropertyRef("attests_digest")
|
|
27
27
|
media_type: PropertyRef = PropertyRef("media_type")
|
|
28
28
|
artifact_media_type: PropertyRef = PropertyRef("artifact_media_type")
|
|
29
|
+
child_image_digests: PropertyRef = PropertyRef("child_image_digests")
|
|
29
30
|
|
|
30
31
|
|
|
31
32
|
@dataclass(frozen=True)
|
|
@@ -86,6 +87,50 @@ class ECRImageToParentImageRel(CartographyRelSchema):
|
|
|
86
87
|
)
|
|
87
88
|
|
|
88
89
|
|
|
90
|
+
@dataclass(frozen=True)
|
|
91
|
+
class ECRImageContainsImageRelProperties(CartographyRelProperties):
|
|
92
|
+
lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass(frozen=True)
|
|
96
|
+
class ECRImageContainsImageRel(CartographyRelSchema):
|
|
97
|
+
"""
|
|
98
|
+
Relationship from a manifest list ECRImage to platform-specific ECRImages it contains.
|
|
99
|
+
Only applies to ECRImage nodes with type="manifest_list".
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
target_node_label: str = "ECRImage"
|
|
103
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
104
|
+
{"digest": PropertyRef("child_image_digests", one_to_many=True)},
|
|
105
|
+
)
|
|
106
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
107
|
+
rel_label: str = "CONTAINS_IMAGE"
|
|
108
|
+
properties: ECRImageContainsImageRelProperties = (
|
|
109
|
+
ECRImageContainsImageRelProperties()
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@dataclass(frozen=True)
|
|
114
|
+
class ECRImageAttestsRelProperties(CartographyRelProperties):
|
|
115
|
+
lastupdated: PropertyRef = PropertyRef("lastupdated", set_in_kwargs=True)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@dataclass(frozen=True)
|
|
119
|
+
class ECRImageAttestsRel(CartographyRelSchema):
|
|
120
|
+
"""
|
|
121
|
+
Relationship from an attestation ECRImage to the ECRImage it attests/validates.
|
|
122
|
+
Only applies to ECRImage nodes with type="attestation".
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
target_node_label: str = "ECRImage"
|
|
126
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
127
|
+
{"digest": PropertyRef("attests_digest")},
|
|
128
|
+
)
|
|
129
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
130
|
+
rel_label: str = "ATTESTS"
|
|
131
|
+
properties: ECRImageAttestsRelProperties = ECRImageAttestsRelProperties()
|
|
132
|
+
|
|
133
|
+
|
|
89
134
|
@dataclass(frozen=True)
|
|
90
135
|
class ECRImageSchema(CartographyNodeSchema):
|
|
91
136
|
label: str = "ECRImage"
|
|
@@ -95,5 +140,7 @@ class ECRImageSchema(CartographyNodeSchema):
|
|
|
95
140
|
[
|
|
96
141
|
ECRImageHasLayerRel(),
|
|
97
142
|
ECRImageToParentImageRel(),
|
|
143
|
+
ECRImageContainsImageRel(),
|
|
144
|
+
ECRImageAttestsRel(),
|
|
98
145
|
],
|
|
99
146
|
)
|
|
@@ -15,12 +15,13 @@ class AWSGroupToAWSUserRelProperties(CartographyRelProperties):
|
|
|
15
15
|
|
|
16
16
|
@dataclass(frozen=True)
|
|
17
17
|
class AWSGroupToAWSUserRel(CartographyRelSchema):
|
|
18
|
+
# AWSUser -MEMBER_AWS_GROUP-> AWSGroup
|
|
18
19
|
target_node_label: str = "AWSUser"
|
|
19
20
|
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
20
21
|
{
|
|
21
|
-
"arn": PropertyRef("
|
|
22
|
+
"arn": PropertyRef("user_arns", one_to_many=True),
|
|
22
23
|
}
|
|
23
24
|
)
|
|
24
|
-
direction: LinkDirection = LinkDirection.
|
|
25
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
25
26
|
rel_label: str = "MEMBER_AWS_GROUP"
|
|
26
27
|
properties: AWSGroupToAWSUserRelProperties = AWSGroupToAWSUserRelProperties()
|
|
@@ -95,7 +95,9 @@ class AWSSSOUserToPermissionSetRel(CartographyRelSchema):
|
|
|
95
95
|
class AWSSSOUserSchema(CartographyNodeSchema):
|
|
96
96
|
label: str = "AWSSSOUser"
|
|
97
97
|
properties: SSOUserProperties = SSOUserProperties()
|
|
98
|
-
extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
|
|
98
|
+
extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
|
|
99
|
+
["UserAccount"]
|
|
100
|
+
) # UserAccount label is used for ontology mapping
|
|
99
101
|
sub_resource_relationship: AWSSSOUserToAWSAccountRel = AWSSSOUserToAWSAccountRel()
|
|
100
102
|
other_relationships: OtherRelationships = OtherRelationships(
|
|
101
103
|
[
|
|
@@ -22,7 +22,7 @@ class BigfixComputerNodeProperties(CartographyNodeProperties):
|
|
|
22
22
|
besrootserver: PropertyRef = PropertyRef("BESRootServer")
|
|
23
23
|
bios: PropertyRef = PropertyRef("BIOS")
|
|
24
24
|
computertype: PropertyRef = PropertyRef("ComputerType")
|
|
25
|
-
computername: PropertyRef = PropertyRef("ComputerName")
|
|
25
|
+
computername: PropertyRef = PropertyRef("ComputerName", extra_index=True)
|
|
26
26
|
cpu: PropertyRef = PropertyRef("CPU")
|
|
27
27
|
devicetype: PropertyRef = PropertyRef("DeviceType")
|
|
28
28
|
distancetobesrelay: PropertyRef = PropertyRef("DistanceToBESRelay")
|
|
@@ -3,6 +3,7 @@ from dataclasses import dataclass
|
|
|
3
3
|
from cartography.models.core.common import PropertyRef
|
|
4
4
|
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
5
|
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.nodes import ExtraNodeLabels
|
|
6
7
|
from cartography.models.core.relationships import CartographyRelProperties
|
|
7
8
|
from cartography.models.core.relationships import CartographyRelSchema
|
|
8
9
|
from cartography.models.core.relationships import LinkDirection
|
|
@@ -71,6 +72,9 @@ class CloudflareMemberToCloudflareRoleRel(CartographyRelSchema):
|
|
|
71
72
|
@dataclass(frozen=True)
|
|
72
73
|
class CloudflareMemberSchema(CartographyNodeSchema):
|
|
73
74
|
label: str = "CloudflareMember"
|
|
75
|
+
extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
|
|
76
|
+
["UserAccount"]
|
|
77
|
+
) # UserAccount label is used for ontology mapping
|
|
74
78
|
properties: CloudflareMemberNodeProperties = CloudflareMemberNodeProperties()
|
|
75
79
|
sub_resource_relationship: CloudflareMemberToAccountRel = (
|
|
76
80
|
CloudflareMemberToAccountRel()
|
|
@@ -12,7 +12,7 @@ class CrowdstrikeHostNodeProperties(CartographyNodeProperties):
|
|
|
12
12
|
instance_id: PropertyRef = PropertyRef("instance_id", extra_index=True)
|
|
13
13
|
serial_number: PropertyRef = PropertyRef("serial_number", extra_index=True)
|
|
14
14
|
status: PropertyRef = PropertyRef("status")
|
|
15
|
-
hostname: PropertyRef = PropertyRef("hostname")
|
|
15
|
+
hostname: PropertyRef = PropertyRef("hostname", extra_index=True)
|
|
16
16
|
machine_domain: PropertyRef = PropertyRef("machine_domain")
|
|
17
17
|
crowdstrike_first_seen: PropertyRef = PropertyRef("first_seen")
|
|
18
18
|
crowdstrike_last_seen: PropertyRef = PropertyRef("last_seen")
|
|
@@ -21,7 +21,7 @@ class DuoEndpointNodeProperties(CartographyNodeProperties):
|
|
|
21
21
|
device_id: PropertyRef = PropertyRef("device_id")
|
|
22
22
|
device_identifier: PropertyRef = PropertyRef("device_identifier")
|
|
23
23
|
device_identifier_type: PropertyRef = PropertyRef("device_identifier_type")
|
|
24
|
-
device_name: PropertyRef = PropertyRef("device_name")
|
|
24
|
+
device_name: PropertyRef = PropertyRef("device_name", extra_index=True)
|
|
25
25
|
device_udid: PropertyRef = PropertyRef("device_udid")
|
|
26
26
|
device_username: PropertyRef = PropertyRef("device_username")
|
|
27
27
|
device_username_type: PropertyRef = PropertyRef("device_username_type")
|
cartography/models/duo/phone.py
CHANGED
|
@@ -22,8 +22,8 @@ class DuoPhoneNodeProperties(CartographyNodeProperties):
|
|
|
22
22
|
fingerprint: PropertyRef = PropertyRef("fingerprint")
|
|
23
23
|
last_seen: PropertyRef = PropertyRef("last_seen")
|
|
24
24
|
model: PropertyRef = PropertyRef("model")
|
|
25
|
-
name: PropertyRef = PropertyRef("name")
|
|
26
|
-
phone_id: PropertyRef = PropertyRef("phone_id"
|
|
25
|
+
name: PropertyRef = PropertyRef("name", extra_index=True)
|
|
26
|
+
phone_id: PropertyRef = PropertyRef("phone_id")
|
|
27
27
|
platform: PropertyRef = PropertyRef("platform")
|
|
28
28
|
postdelay: PropertyRef = PropertyRef("postdelay")
|
|
29
29
|
predelay: PropertyRef = PropertyRef("predelay")
|
cartography/models/duo/user.py
CHANGED
|
@@ -3,6 +3,7 @@ from dataclasses import dataclass
|
|
|
3
3
|
from cartography.models.core.common import PropertyRef
|
|
4
4
|
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
5
|
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.nodes import ExtraNodeLabels
|
|
6
7
|
from cartography.models.core.relationships import CartographyRelProperties
|
|
7
8
|
from cartography.models.core.relationships import CartographyRelSchema
|
|
8
9
|
from cartography.models.core.relationships import LinkDirection
|
|
@@ -149,6 +150,9 @@ class DuoUserToHumanRel(CartographyRelSchema):
|
|
|
149
150
|
@dataclass(frozen=True)
|
|
150
151
|
class DuoUserSchema(CartographyNodeSchema):
|
|
151
152
|
label: str = "DuoUser"
|
|
153
|
+
extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
|
|
154
|
+
["UserAccount"]
|
|
155
|
+
) # UserAccount label is used for ontology mapping
|
|
152
156
|
properties: DuoUserNodeProperties = DuoUserNodeProperties()
|
|
153
157
|
sub_resource_relationship: DuoUserToDuoApiHostRel = DuoUserToDuoApiHostRel()
|
|
154
158
|
other_relationships: OtherRelationships = OtherRelationships(
|