cartography 0.96.0rc1__py3-none-any.whl → 0.96.0rc3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/cli.py +15 -0
- cartography/client/core/tx.py +1 -1
- cartography/config.py +6 -2
- cartography/data/indexes.cypher +1 -2
- cartography/data/jobs/cleanup/aws_import_identity_center_cleanup.json +16 -0
- cartography/data/jobs/cleanup/{github_users_cleanup.json → github_org_and_users_cleanup.json} +5 -0
- cartography/intel/aws/apigateway.py +3 -3
- cartography/intel/aws/identitycenter.py +307 -0
- cartography/intel/aws/resources.py +2 -0
- cartography/intel/cve/__init__.py +1 -1
- cartography/intel/cve/feed.py +4 -4
- cartography/intel/github/users.py +156 -39
- cartography/intel/okta/users.py +2 -1
- cartography/intel/semgrep/__init__.py +1 -1
- cartography/intel/semgrep/dependencies.py +54 -22
- cartography/models/aws/identitycenter/__init__.py +0 -0
- cartography/models/aws/identitycenter/awsidentitycenter.py +44 -0
- cartography/models/aws/identitycenter/awspermissionset.py +84 -0
- cartography/models/aws/identitycenter/awsssouser.py +68 -0
- cartography/models/github/orgs.py +26 -0
- cartography/models/github/users.py +119 -0
- cartography/models/semgrep/dependencies.py +13 -0
- cartography-0.96.0rc3.dist-info/METADATA +53 -0
- {cartography-0.96.0rc1.dist-info → cartography-0.96.0rc3.dist-info}/RECORD +28 -20
- {cartography-0.96.0rc1.dist-info → cartography-0.96.0rc3.dist-info}/WHEEL +1 -1
- cartography-0.96.0rc1.dist-info/METADATA +0 -53
- {cartography-0.96.0rc1.dist-info → cartography-0.96.0rc3.dist-info}/LICENSE +0 -0
- {cartography-0.96.0rc1.dist-info → cartography-0.96.0rc3.dist-info}/entry_points.txt +0 -0
- {cartography-0.96.0rc1.dist-info → cartography-0.96.0rc3.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
from copy import deepcopy
|
|
2
3
|
from typing import Any
|
|
3
4
|
from typing import Dict
|
|
4
5
|
from typing import List
|
|
@@ -6,7 +7,11 @@ from typing import Tuple
|
|
|
6
7
|
|
|
7
8
|
import neo4j
|
|
8
9
|
|
|
10
|
+
from cartography.client.core.tx import load
|
|
9
11
|
from cartography.intel.github.util import fetch_all
|
|
12
|
+
from cartography.models.github.orgs import GitHubOrganizationSchema
|
|
13
|
+
from cartography.models.github.users import GitHubOrganizationUserSchema
|
|
14
|
+
from cartography.models.github.users import GitHubUnaffiliatedUserSchema
|
|
10
15
|
from cartography.stats import get_stats_client
|
|
11
16
|
from cartography.util import merge_module_sync_metadata
|
|
12
17
|
from cartography.util import run_cleanup_job
|
|
@@ -44,17 +49,46 @@ GITHUB_ORG_USERS_PAGINATED_GRAPHQL = """
|
|
|
44
49
|
}
|
|
45
50
|
"""
|
|
46
51
|
|
|
52
|
+
GITHUB_ENTERPRISE_OWNER_USERS_PAGINATED_GRAPHQL = """
|
|
53
|
+
query($login: String!, $cursor: String) {
|
|
54
|
+
organization(login: $login)
|
|
55
|
+
{
|
|
56
|
+
url
|
|
57
|
+
login
|
|
58
|
+
enterpriseOwners(first:100, after: $cursor){
|
|
59
|
+
edges {
|
|
60
|
+
node {
|
|
61
|
+
url
|
|
62
|
+
login
|
|
63
|
+
name
|
|
64
|
+
isSiteAdmin
|
|
65
|
+
email
|
|
66
|
+
company
|
|
67
|
+
}
|
|
68
|
+
organizationRole
|
|
69
|
+
}
|
|
70
|
+
pageInfo{
|
|
71
|
+
endCursor
|
|
72
|
+
hasNextPage
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
"""
|
|
78
|
+
|
|
47
79
|
|
|
48
80
|
@timeit
|
|
49
|
-
def
|
|
81
|
+
def get_users(token: str, api_url: str, organization: str) -> Tuple[List[Dict], Dict]:
|
|
50
82
|
"""
|
|
51
83
|
Retrieve a list of users from the given GitHub organization as described in
|
|
52
84
|
https://docs.github.com/en/graphql/reference/objects#organizationmemberedge.
|
|
53
85
|
:param token: The Github API token as string.
|
|
54
86
|
:param api_url: The Github v4 API endpoint as string.
|
|
55
87
|
:param organization: The name of the target Github organization as string.
|
|
56
|
-
:return: A 2-tuple containing
|
|
57
|
-
|
|
88
|
+
:return: A 2-tuple containing
|
|
89
|
+
1. a list of dicts representing users and
|
|
90
|
+
2. data on the owning GitHub organization
|
|
91
|
+
see tests.data.github.users.GITHUB_USER_DATA for shape of both
|
|
58
92
|
"""
|
|
59
93
|
users, org = fetch_all(
|
|
60
94
|
token,
|
|
@@ -66,56 +100,139 @@ def get(token: str, api_url: str, organization: str) -> Tuple[List[Dict], Dict]:
|
|
|
66
100
|
return users.edges, org
|
|
67
101
|
|
|
68
102
|
|
|
103
|
+
def get_enterprise_owners(token: str, api_url: str, organization: str) -> Tuple[List[Dict], Dict]:
|
|
104
|
+
"""
|
|
105
|
+
Retrieve a list of enterprise owners from the given GitHub organization as described in
|
|
106
|
+
https://docs.github.com/en/graphql/reference/objects#organizationenterpriseowneredge.
|
|
107
|
+
:param token: The Github API token as string.
|
|
108
|
+
:param api_url: The Github v4 API endpoint as string.
|
|
109
|
+
:param organization: The name of the target Github organization as string.
|
|
110
|
+
:return: A 2-tuple containing
|
|
111
|
+
1. a list of dicts representing users who are enterprise owners
|
|
112
|
+
3. data on the owning GitHub organization
|
|
113
|
+
see tests.data.github.users.GITHUB_ENTERPRISE_OWNER_DATA for shape
|
|
114
|
+
"""
|
|
115
|
+
owners, org = fetch_all(
|
|
116
|
+
token,
|
|
117
|
+
api_url,
|
|
118
|
+
organization,
|
|
119
|
+
GITHUB_ENTERPRISE_OWNER_USERS_PAGINATED_GRAPHQL,
|
|
120
|
+
'enterpriseOwners',
|
|
121
|
+
)
|
|
122
|
+
return owners.edges, org
|
|
123
|
+
|
|
124
|
+
|
|
69
125
|
@timeit
|
|
70
|
-
def
|
|
71
|
-
|
|
126
|
+
def transform_users(user_data: List[Dict], owners_data: List[Dict], org_data: Dict) -> Tuple[List[Dict], List[Dict]]:
|
|
127
|
+
"""
|
|
128
|
+
Taking raw user and owner data, return two lists of processed user data:
|
|
129
|
+
* organization users aka affiliated users (users directly affiliated with an organization)
|
|
130
|
+
* unaffiliated users (user who, for example, are enterprise owners but not members of the target organization).
|
|
131
|
+
|
|
132
|
+
:param token: The Github API token as string.
|
|
133
|
+
:param api_url: The Github v4 API endpoint as string.
|
|
134
|
+
:param organization: The name of the target Github organization as string.
|
|
135
|
+
:return: A 2-tuple containing
|
|
136
|
+
1. a list of dicts representing users who are affiliated with the target org
|
|
137
|
+
see tests.data.github.users.GITHUB_USER_DATA for shape
|
|
138
|
+
2. a list of dicts representing users who are not affiliated (e.g. enterprise owners who are not also in
|
|
139
|
+
the target org) — see tests.data.github.users.GITHUB_ENTERPRISE_OWNER_DATA for shape
|
|
140
|
+
3. data on the owning GitHub organization
|
|
141
|
+
"""
|
|
142
|
+
|
|
143
|
+
users_dict = {}
|
|
144
|
+
for user in user_data:
|
|
145
|
+
processed_user = deepcopy(user['node'])
|
|
146
|
+
processed_user['role'] = user['role']
|
|
147
|
+
processed_user['hasTwoFactorEnabled'] = user['hasTwoFactorEnabled']
|
|
148
|
+
processed_user['MEMBER_OF'] = org_data['url']
|
|
149
|
+
users_dict[processed_user['url']] = processed_user
|
|
150
|
+
|
|
151
|
+
owners_dict = {}
|
|
152
|
+
for owner in owners_data:
|
|
153
|
+
processed_owner = deepcopy(owner['node'])
|
|
154
|
+
processed_owner['isEnterpriseOwner'] = True
|
|
155
|
+
if owner['organizationRole'] == 'UNAFFILIATED':
|
|
156
|
+
processed_owner['UNAFFILIATED'] = org_data['url']
|
|
157
|
+
else:
|
|
158
|
+
processed_owner['MEMBER_OF'] = org_data['url']
|
|
159
|
+
owners_dict[processed_owner['url']] = processed_owner
|
|
160
|
+
|
|
161
|
+
affiliated_users = [] # users affiliated with the target org
|
|
162
|
+
for url, user in users_dict.items():
|
|
163
|
+
user['isEnterpriseOwner'] = url in owners_dict
|
|
164
|
+
affiliated_users.append(user)
|
|
165
|
+
|
|
166
|
+
unaffiliated_users = [] # users not affiliated with the target org
|
|
167
|
+
for url, owner in owners_dict.items():
|
|
168
|
+
if url not in users_dict:
|
|
169
|
+
unaffiliated_users.append(owner)
|
|
170
|
+
|
|
171
|
+
return affiliated_users, unaffiliated_users
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@timeit
|
|
175
|
+
def load_users(
|
|
176
|
+
neo4j_session: neo4j.Session,
|
|
177
|
+
node_schema: GitHubOrganizationUserSchema | GitHubUnaffiliatedUserSchema,
|
|
178
|
+
user_data: List[Dict],
|
|
179
|
+
org_data: Dict,
|
|
72
180
|
update_tag: int,
|
|
73
181
|
) -> None:
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
"""
|
|
98
|
-
neo4j_session.run(
|
|
99
|
-
query,
|
|
100
|
-
OrgUrl=org_data['url'],
|
|
101
|
-
OrgLogin=org_data['login'],
|
|
102
|
-
UserData=user_data,
|
|
103
|
-
UpdateTag=update_tag,
|
|
182
|
+
logger.info(f"Loading {len(user_data)} GitHub users to the graph")
|
|
183
|
+
load(
|
|
184
|
+
neo4j_session,
|
|
185
|
+
node_schema,
|
|
186
|
+
user_data,
|
|
187
|
+
lastupdated=update_tag,
|
|
188
|
+
org_url=org_data['url'],
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
@timeit
|
|
193
|
+
def load_organization(
|
|
194
|
+
neo4j_session: neo4j.Session,
|
|
195
|
+
node_schema: GitHubOrganizationSchema,
|
|
196
|
+
org_data: List[Dict[str, Any]],
|
|
197
|
+
update_tag: int,
|
|
198
|
+
) -> None:
|
|
199
|
+
logger.info(f"Loading {len(org_data)} GitHub organization to the graph")
|
|
200
|
+
load(
|
|
201
|
+
neo4j_session,
|
|
202
|
+
node_schema,
|
|
203
|
+
org_data,
|
|
204
|
+
lastupdated=update_tag,
|
|
104
205
|
)
|
|
105
206
|
|
|
106
207
|
|
|
107
208
|
@timeit
|
|
108
209
|
def sync(
|
|
109
210
|
neo4j_session: neo4j.Session,
|
|
110
|
-
common_job_parameters: Dict
|
|
211
|
+
common_job_parameters: Dict,
|
|
111
212
|
github_api_key: str,
|
|
112
213
|
github_url: str,
|
|
113
214
|
organization: str,
|
|
114
215
|
) -> None:
|
|
115
216
|
logger.info("Syncing GitHub users")
|
|
116
|
-
user_data, org_data =
|
|
117
|
-
|
|
118
|
-
|
|
217
|
+
user_data, org_data = get_users(github_api_key, github_url, organization)
|
|
218
|
+
owners_data, org_data = get_enterprise_owners(github_api_key, github_url, organization)
|
|
219
|
+
processed_affiliated_user_data, processed_unaffiliated_user_data = (
|
|
220
|
+
transform_users(user_data, owners_data, org_data)
|
|
221
|
+
)
|
|
222
|
+
load_organization(
|
|
223
|
+
neo4j_session, GitHubOrganizationSchema(), [org_data],
|
|
224
|
+
common_job_parameters['UPDATE_TAG'],
|
|
225
|
+
)
|
|
226
|
+
load_users(
|
|
227
|
+
neo4j_session, GitHubOrganizationUserSchema(), processed_affiliated_user_data, org_data,
|
|
228
|
+
common_job_parameters['UPDATE_TAG'],
|
|
229
|
+
)
|
|
230
|
+
load_users(
|
|
231
|
+
neo4j_session, GitHubUnaffiliatedUserSchema(), processed_unaffiliated_user_data, org_data,
|
|
232
|
+
common_job_parameters['UPDATE_TAG'],
|
|
233
|
+
)
|
|
234
|
+
# no automated cleanup job for users because user node has no sub_resource_relationship
|
|
235
|
+
run_cleanup_job('github_org_and_users_cleanup.json', neo4j_session, common_job_parameters)
|
|
119
236
|
merge_module_sync_metadata(
|
|
120
237
|
neo4j_session,
|
|
121
238
|
group_type='GitHubOrganization',
|
cartography/intel/okta/users.py
CHANGED
|
@@ -150,7 +150,8 @@ def _load_okta_users(
|
|
|
150
150
|
new_user.okta_last_updated = user_data.okta_last_updated,
|
|
151
151
|
new_user.password_changed = user_data.password_changed,
|
|
152
152
|
new_user.transition_to_status = user_data.transition_to_status,
|
|
153
|
-
new_user.lastupdated = $okta_update_tag
|
|
153
|
+
new_user.lastupdated = $okta_update_tag,
|
|
154
|
+
new_user :UserAccount
|
|
154
155
|
WITH new_user, org
|
|
155
156
|
MERGE (org)-[org_r:RESOURCE]->(new_user)
|
|
156
157
|
ON CREATE SET org_r.firstseen = timestamp()
|
|
@@ -26,5 +26,5 @@ def start_semgrep_ingestion(
|
|
|
26
26
|
# sync_deployment must be called first since it populates common_job_parameters
|
|
27
27
|
# with the deployment ID and slug, which are required by the other sync functions
|
|
28
28
|
sync_deployment(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
|
|
29
|
-
sync_dependencies(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
|
|
29
|
+
sync_dependencies(neo4j_session, config.semgrep_app_token, config.semgrep_dependency_ecosystems, config.update_tag, common_job_parameters) # noqa: E501
|
|
30
30
|
sync_findings(neo4j_session, config.semgrep_app_token, config.update_tag, common_job_parameters)
|
|
@@ -12,6 +12,7 @@ from requests.exceptions import ReadTimeout
|
|
|
12
12
|
from cartography.client.core.tx import load
|
|
13
13
|
from cartography.graph.job import GraphJob
|
|
14
14
|
from cartography.models.semgrep.dependencies import SemgrepGoLibrarySchema
|
|
15
|
+
from cartography.models.semgrep.dependencies import SemgrepNpmLibrarySchema
|
|
15
16
|
from cartography.stats import get_stats_client
|
|
16
17
|
from cartography.util import merge_module_sync_metadata
|
|
17
18
|
from cartography.util import timeit
|
|
@@ -22,16 +23,38 @@ _PAGE_SIZE = 10000
|
|
|
22
23
|
_TIMEOUT = (60, 60)
|
|
23
24
|
_MAX_RETRIES = 3
|
|
24
25
|
|
|
26
|
+
# The keys in this dictionary must be in Semgrep's list of supported ecosystems, defined here:
|
|
27
|
+
# https://semgrep.dev/api/v1/docs/#tag/SupplyChainService/operation/semgrep_app.products.sca.handlers.dependency.list_dependencies_conexxion
|
|
28
|
+
ECOSYSTEM_TO_SCHEMA: Dict = {
|
|
29
|
+
'gomod': SemgrepGoLibrarySchema,
|
|
30
|
+
'npm': SemgrepNpmLibrarySchema,
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def parse_and_validate_semgrep_ecosystems(ecosystems: str) -> List[str]:
|
|
35
|
+
validated_ecosystems: List[str] = []
|
|
36
|
+
for ecosystem in ecosystems.split(','):
|
|
37
|
+
ecosystem = ecosystem.strip().lower()
|
|
38
|
+
|
|
39
|
+
if ecosystem in ECOSYSTEM_TO_SCHEMA:
|
|
40
|
+
validated_ecosystems.append(ecosystem)
|
|
41
|
+
else:
|
|
42
|
+
valid_ecosystems: str = ','.join(ECOSYSTEM_TO_SCHEMA.keys())
|
|
43
|
+
raise ValueError(
|
|
44
|
+
f'Error parsing `semgrep-dependency-ecosystems`. You specified "{ecosystems}". '
|
|
45
|
+
f'Please check that your input is formatted as comma-separated values, e.g. "gomod,npm". '
|
|
46
|
+
f'Full list of supported ecosystems: {valid_ecosystems}.',
|
|
47
|
+
)
|
|
48
|
+
return validated_ecosystems
|
|
49
|
+
|
|
25
50
|
|
|
26
51
|
@timeit
|
|
27
|
-
def get_dependencies(semgrep_app_token: str, deployment_id: str,
|
|
52
|
+
def get_dependencies(semgrep_app_token: str, deployment_id: str, ecosystem: str) -> List[Dict[str, Any]]:
|
|
28
53
|
"""
|
|
29
|
-
Gets all dependencies for the given
|
|
54
|
+
Gets all dependencies for the given ecosystem within the given Semgrep deployment ID.
|
|
30
55
|
param: semgrep_app_token: The Semgrep App token to use for authentication.
|
|
31
56
|
param: deployment_id: The Semgrep deployment ID to use for retrieving dependencies.
|
|
32
|
-
param:
|
|
33
|
-
The list of supported ecosystems is defined here:
|
|
34
|
-
https://semgrep.dev/api/v1/docs/#tag/SupplyChainService/operation/semgrep_app.products.sca.handlers.dependency.list_dependencies_conexxion
|
|
57
|
+
param: ecosystem: The ecosystem to import dependencies from, e.g. "gomod" or "npm".
|
|
35
58
|
"""
|
|
36
59
|
all_deps = []
|
|
37
60
|
deps_url = f"https://semgrep.dev/api/v1/deployments/{deployment_id}/dependencies"
|
|
@@ -46,31 +69,31 @@ def get_dependencies(semgrep_app_token: str, deployment_id: str, ecosystems: Lis
|
|
|
46
69
|
request_data: dict[str, Any] = {
|
|
47
70
|
"pageSize": _PAGE_SIZE,
|
|
48
71
|
"dependencyFilter": {
|
|
49
|
-
"ecosystem":
|
|
72
|
+
"ecosystem": [ecosystem],
|
|
50
73
|
},
|
|
51
74
|
}
|
|
52
75
|
|
|
53
|
-
logger.info(f"Retrieving Semgrep dependencies for deployment '{deployment_id}'.")
|
|
76
|
+
logger.info(f"Retrieving Semgrep {ecosystem} dependencies for deployment '{deployment_id}'.")
|
|
54
77
|
while has_more:
|
|
55
78
|
try:
|
|
56
79
|
response = requests.post(deps_url, json=request_data, headers=headers, timeout=_TIMEOUT)
|
|
57
80
|
response.raise_for_status()
|
|
58
81
|
data = response.json()
|
|
59
82
|
except (ReadTimeout, HTTPError):
|
|
60
|
-
logger.warning(f"Failed to retrieve Semgrep dependencies for page {page}. Retrying...")
|
|
83
|
+
logger.warning(f"Failed to retrieve Semgrep {ecosystem} dependencies for page {page}. Retrying...")
|
|
61
84
|
retries += 1
|
|
62
85
|
if retries >= _MAX_RETRIES:
|
|
63
86
|
raise
|
|
64
87
|
continue
|
|
65
88
|
deps = data.get("dependencies", [])
|
|
66
89
|
has_more = data.get("hasMore", False)
|
|
67
|
-
logger.info(f"Processed page {page} of Semgrep dependencies.")
|
|
90
|
+
logger.info(f"Processed page {page} of Semgrep {ecosystem} dependencies.")
|
|
68
91
|
all_deps.extend(deps)
|
|
69
92
|
retries = 0
|
|
70
93
|
page += 1
|
|
71
94
|
request_data["cursor"] = data.get("cursor")
|
|
72
95
|
|
|
73
|
-
logger.info(f"Retrieved {len(all_deps)} Semgrep dependencies in {page} pages.")
|
|
96
|
+
logger.info(f"Retrieved {len(all_deps)} Semgrep {ecosystem} dependencies in {page} pages.")
|
|
74
97
|
return all_deps
|
|
75
98
|
|
|
76
99
|
|
|
@@ -157,19 +180,18 @@ def load_dependencies(
|
|
|
157
180
|
@timeit
|
|
158
181
|
def cleanup(
|
|
159
182
|
neo4j_session: neo4j.Session,
|
|
183
|
+
dependency_schema: Callable,
|
|
160
184
|
common_job_parameters: Dict[str, Any],
|
|
161
185
|
) -> None:
|
|
162
|
-
logger.info("Running Semgrep
|
|
163
|
-
|
|
164
|
-
SemgrepGoLibrarySchema(), common_job_parameters,
|
|
165
|
-
)
|
|
166
|
-
go_libraries_cleanup_job.run(neo4j_session)
|
|
186
|
+
logger.info(f"Running Semgrep Dependencies cleanup job for {dependency_schema().label}.")
|
|
187
|
+
GraphJob.from_node_schema(dependency_schema(), common_job_parameters).run(neo4j_session)
|
|
167
188
|
|
|
168
189
|
|
|
169
190
|
@timeit
|
|
170
191
|
def sync_dependencies(
|
|
171
192
|
neo4j_session: neo4j.Session,
|
|
172
193
|
semgrep_app_token: str,
|
|
194
|
+
ecosystems_str: str,
|
|
173
195
|
update_tag: int,
|
|
174
196
|
common_job_parameters: Dict[str, Any],
|
|
175
197
|
) -> None:
|
|
@@ -177,19 +199,29 @@ def sync_dependencies(
|
|
|
177
199
|
deployment_id = common_job_parameters.get("DEPLOYMENT_ID")
|
|
178
200
|
if not deployment_id:
|
|
179
201
|
logger.warning(
|
|
180
|
-
"Missing Semgrep deployment ID, ensure that sync_deployment() has been called."
|
|
202
|
+
"Missing Semgrep deployment ID, ensure that sync_deployment() has been called. "
|
|
181
203
|
"Skipping Semgrep dependencies sync job.",
|
|
182
204
|
)
|
|
183
205
|
return
|
|
184
206
|
|
|
185
|
-
|
|
207
|
+
if not ecosystems_str:
|
|
208
|
+
logger.warning(
|
|
209
|
+
"Semgrep is not configured to import dependencies for any ecosystems, see docs to configure. "
|
|
210
|
+
"Skipping Semgrep dependencies sync job.",
|
|
211
|
+
)
|
|
212
|
+
return
|
|
213
|
+
|
|
214
|
+
# We don't expect an error here since we've already validated the input in cli.py
|
|
215
|
+
ecosystems = parse_and_validate_semgrep_ecosystems(ecosystems_str)
|
|
186
216
|
|
|
187
|
-
|
|
188
|
-
raw_go_deps = get_dependencies(semgrep_app_token, deployment_id, ecosystems=["gomod"])
|
|
189
|
-
go_deps = transform_dependencies(raw_go_deps)
|
|
190
|
-
load_dependencies(neo4j_session, SemgrepGoLibrarySchema, go_deps, deployment_id, update_tag)
|
|
217
|
+
logger.info("Running Semgrep dependencies sync job.")
|
|
191
218
|
|
|
192
|
-
|
|
219
|
+
for ecosystem in ecosystems:
|
|
220
|
+
schema = ECOSYSTEM_TO_SCHEMA[ecosystem]
|
|
221
|
+
raw_deps = get_dependencies(semgrep_app_token, deployment_id, ecosystem)
|
|
222
|
+
deps = transform_dependencies(raw_deps)
|
|
223
|
+
load_dependencies(neo4j_session, schema, deps, deployment_id, update_tag)
|
|
224
|
+
cleanup(neo4j_session, schema, common_job_parameters)
|
|
193
225
|
|
|
194
226
|
merge_module_sync_metadata(
|
|
195
227
|
neo4j_session=neo4j_session,
|
|
File without changes
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from cartography.models.core.common import PropertyRef
|
|
4
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
7
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
8
|
+
from cartography.models.core.relationships import LinkDirection
|
|
9
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
10
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class IdentityCenterInstanceProperties(CartographyNodeProperties):
|
|
15
|
+
identity_store_id: PropertyRef = PropertyRef('IdentityStoreId')
|
|
16
|
+
arn: PropertyRef = PropertyRef('InstanceArn')
|
|
17
|
+
created_date: PropertyRef = PropertyRef('CreatedDate')
|
|
18
|
+
id: PropertyRef = PropertyRef('InstanceArn')
|
|
19
|
+
status: PropertyRef = PropertyRef('Status')
|
|
20
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class IdentityCenterToAwsAccountRelProperties(CartographyRelProperties):
|
|
25
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
# (:IdentityCenter)<-[:RESOURCE]-(:AWSAccount)
|
|
30
|
+
class IdentityCenterToAWSAccount(CartographyRelSchema):
|
|
31
|
+
target_node_label: str = 'AWSAccount'
|
|
32
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
33
|
+
{'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
|
|
34
|
+
)
|
|
35
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
36
|
+
rel_label: str = "RESOURCE"
|
|
37
|
+
properties: IdentityCenterToAwsAccountRelProperties = IdentityCenterToAwsAccountRelProperties()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class AWSIdentityCenterInstanceSchema(CartographyNodeSchema):
|
|
42
|
+
label: str = 'AWSIdentityCenter'
|
|
43
|
+
properties: IdentityCenterInstanceProperties = IdentityCenterInstanceProperties()
|
|
44
|
+
sub_resource_relationship: IdentityCenterToAWSAccount = IdentityCenterToAWSAccount()
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from cartography.models.core.common import PropertyRef
|
|
4
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
7
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
8
|
+
from cartography.models.core.relationships import LinkDirection
|
|
9
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
10
|
+
from cartography.models.core.relationships import OtherRelationships
|
|
11
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class PermissionSetProperties(CartographyNodeProperties):
|
|
16
|
+
id: PropertyRef = PropertyRef('PermissionSetArn')
|
|
17
|
+
name: PropertyRef = PropertyRef('Name')
|
|
18
|
+
arn: PropertyRef = PropertyRef('PermissionSetArn')
|
|
19
|
+
description: PropertyRef = PropertyRef('Description')
|
|
20
|
+
session_duration: PropertyRef = PropertyRef('SessionDuration')
|
|
21
|
+
instance_arn: PropertyRef = PropertyRef('InstanceArn', set_in_kwargs=True)
|
|
22
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class PermissionSetToInstanceRelProperties(CartographyRelProperties):
|
|
27
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class PermissionSetToInstance(CartographyRelSchema):
|
|
32
|
+
target_node_label: str = 'AWSIdentityCenter'
|
|
33
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
34
|
+
{'arn': PropertyRef('InstanceArn', set_in_kwargs=True)},
|
|
35
|
+
)
|
|
36
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
37
|
+
rel_label: str = "HAS_PERMISSION_SET"
|
|
38
|
+
properties: PermissionSetToInstanceRelProperties = PermissionSetToInstanceRelProperties()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class PermissionSetToAWSRoleRelProperties(CartographyRelProperties):
|
|
43
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class PermissionSetToAWSRole(CartographyRelSchema):
|
|
48
|
+
target_node_label: str = 'AWSRole'
|
|
49
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
50
|
+
{'arn': PropertyRef('RoleHint', fuzzy_and_ignore_case=True)},
|
|
51
|
+
)
|
|
52
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
53
|
+
rel_label: str = "ASSIGNED_TO_ROLE"
|
|
54
|
+
properties: PermissionSetToAWSRoleRelProperties = PermissionSetToAWSRoleRelProperties()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(frozen=True)
|
|
58
|
+
class AWSPermissionSetToAwsAccountRelProperties(CartographyRelProperties):
|
|
59
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
# (:IdentityCenter)<-[:RESOURCE]-(:AWSAccount)
|
|
64
|
+
class AWSPermissionSetToAWSAccount(CartographyRelSchema):
|
|
65
|
+
target_node_label: str = 'AWSAccount'
|
|
66
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
67
|
+
{'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
|
|
68
|
+
)
|
|
69
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
70
|
+
rel_label: str = "RESOURCE"
|
|
71
|
+
properties: AWSPermissionSetToAwsAccountRelProperties = AWSPermissionSetToAwsAccountRelProperties()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True)
|
|
75
|
+
class AWSPermissionSetSchema(CartographyNodeSchema):
|
|
76
|
+
label: str = 'AWSPermissionSet'
|
|
77
|
+
properties: PermissionSetProperties = PermissionSetProperties()
|
|
78
|
+
sub_resource_relationship: AWSPermissionSetToAWSAccount = AWSPermissionSetToAWSAccount()
|
|
79
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
80
|
+
[
|
|
81
|
+
PermissionSetToInstance(),
|
|
82
|
+
PermissionSetToAWSRole(),
|
|
83
|
+
],
|
|
84
|
+
)
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from cartography.models.core.common import PropertyRef
|
|
4
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.nodes import ExtraNodeLabels
|
|
7
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
8
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
9
|
+
from cartography.models.core.relationships import LinkDirection
|
|
10
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
11
|
+
from cartography.models.core.relationships import OtherRelationships
|
|
12
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class SSOUserProperties(CartographyNodeProperties):
|
|
17
|
+
id: PropertyRef = PropertyRef('UserId', extra_index=True)
|
|
18
|
+
user_name: PropertyRef = PropertyRef('UserName')
|
|
19
|
+
identity_store_id: PropertyRef = PropertyRef('IdentityStoreId')
|
|
20
|
+
external_id: PropertyRef = PropertyRef('ExternalId', extra_index=True)
|
|
21
|
+
region: PropertyRef = PropertyRef('Region')
|
|
22
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class SSOUserToOktaUserRelProperties(CartographyRelProperties):
|
|
27
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class SSOUserToOktaUser(CartographyRelSchema):
|
|
32
|
+
target_node_label: str = 'UserAccount'
|
|
33
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
34
|
+
{'id': PropertyRef('ExternalId')},
|
|
35
|
+
)
|
|
36
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
37
|
+
rel_label: str = "CAN_ASSUME_IDENTITY"
|
|
38
|
+
properties: SSOUserToOktaUserRelProperties = SSOUserToOktaUserRelProperties()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class AWSSSOUserToAwsAccountRelProperties(CartographyRelProperties):
|
|
43
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
# (:IdentityCenter)<-[:RESOURCE]-(:AWSAccount)
|
|
48
|
+
class AWSSSOUserToAWSAccount(CartographyRelSchema):
|
|
49
|
+
target_node_label: str = 'AWSAccount'
|
|
50
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
51
|
+
{'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
|
|
52
|
+
)
|
|
53
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
54
|
+
rel_label: str = "RESOURCE"
|
|
55
|
+
properties: AWSSSOUserToAwsAccountRelProperties = AWSSSOUserToAwsAccountRelProperties()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class AWSSSOUserSchema(CartographyNodeSchema):
|
|
60
|
+
label: str = 'AWSSSOUser'
|
|
61
|
+
properties: SSOUserProperties = SSOUserProperties()
|
|
62
|
+
extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(["UserAccount"])
|
|
63
|
+
sub_resource_relationship: AWSSSOUserToAWSAccount = AWSSSOUserToAWSAccount()
|
|
64
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
65
|
+
[
|
|
66
|
+
SSOUserToOktaUser(),
|
|
67
|
+
],
|
|
68
|
+
)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This schema does not handle the org's relationships. Those are handled by other schemas, for example:
|
|
3
|
+
* GitHubTeamSchema defines (GitHubOrganization)-[RESOURCE]->(GitHubTeam)
|
|
4
|
+
* GitHubUserSchema defines (GitHubUser)-[MEMBER_OF|UNAFFILIATED]->(GitHubOrganization)
|
|
5
|
+
(There may be others, these are just two examples.)
|
|
6
|
+
"""
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
from cartography.models.core.common import PropertyRef
|
|
10
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
11
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class GitHubOrganizationNodeProperties(CartographyNodeProperties):
|
|
16
|
+
id: PropertyRef = PropertyRef('url')
|
|
17
|
+
username: PropertyRef = PropertyRef('login', extra_index=True)
|
|
18
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class GitHubOrganizationSchema(CartographyNodeSchema):
|
|
23
|
+
label: str = 'GitHubOrganization'
|
|
24
|
+
properties: GitHubOrganizationNodeProperties = GitHubOrganizationNodeProperties()
|
|
25
|
+
other_relationships = None
|
|
26
|
+
sub_resource_relationship = None
|