cartography 0.94.0rc3__py3-none-any.whl → 0.95.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/cli.py +42 -24
- cartography/config.py +12 -8
- cartography/data/indexes.cypher +0 -2
- cartography/driftdetect/cli.py +1 -1
- cartography/graph/job.py +8 -1
- cartography/intel/aws/permission_relationships.py +6 -2
- cartography/intel/gcp/__init__.py +110 -23
- cartography/intel/kandji/__init__.py +1 -1
- cartography/intel/semgrep/__init__.py +9 -2
- cartography/intel/semgrep/dependencies.py +201 -0
- cartography/intel/semgrep/deployment.py +67 -0
- cartography/intel/semgrep/findings.py +22 -53
- cartography/intel/snipeit/__init__.py +30 -0
- cartography/intel/snipeit/asset.py +74 -0
- cartography/intel/snipeit/user.py +75 -0
- cartography/intel/snipeit/util.py +35 -0
- cartography/models/semgrep/dependencies.py +77 -0
- cartography/models/snipeit/__init__.py +0 -0
- cartography/models/snipeit/asset.py +81 -0
- cartography/models/snipeit/tenant.py +17 -0
- cartography/models/snipeit/user.py +49 -0
- cartography/sync.py +2 -2
- {cartography-0.94.0rc3.dist-info → cartography-0.95.0.dist-info}/LICENSE +1 -1
- {cartography-0.94.0rc3.dist-info → cartography-0.95.0.dist-info}/METADATA +3 -5
- {cartography-0.94.0rc3.dist-info → cartography-0.95.0.dist-info}/RECORD +28 -21
- {cartography-0.94.0rc3.dist-info → cartography-0.95.0.dist-info}/WHEEL +1 -1
- cartography/data/jobs/cleanup/crxcavator_import_cleanup.json +0 -18
- cartography/intel/crxcavator/__init__.py +0 -44
- cartography/intel/crxcavator/crxcavator.py +0 -329
- cartography-0.94.0rc3.dist-info/NOTICE +0 -4
- {cartography-0.94.0rc3.dist-info → cartography-0.95.0.dist-info}/entry_points.txt +0 -0
- {cartography-0.94.0rc3.dist-info → cartography-0.95.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
from typing import Dict
|
|
4
|
+
|
|
5
|
+
import neo4j
|
|
6
|
+
import requests
|
|
7
|
+
|
|
8
|
+
from cartography.client.core.tx import load
|
|
9
|
+
from cartography.models.semgrep.deployment import SemgrepDeploymentSchema
|
|
10
|
+
from cartography.stats import get_stats_client
|
|
11
|
+
from cartography.util import timeit
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
stat_handler = get_stats_client(__name__)
|
|
15
|
+
_TIMEOUT = (60, 60)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@timeit
|
|
19
|
+
def get_deployment(semgrep_app_token: str) -> Dict[str, Any]:
|
|
20
|
+
"""
|
|
21
|
+
Gets the deployment associated with the passed Semgrep App token.
|
|
22
|
+
param: semgrep_app_token: The Semgrep App token to use for authentication.
|
|
23
|
+
"""
|
|
24
|
+
deployment = {}
|
|
25
|
+
deployment_url = "https://semgrep.dev/api/v1/deployments"
|
|
26
|
+
headers = {
|
|
27
|
+
"Content-Type": "application/json",
|
|
28
|
+
"Authorization": f"Bearer {semgrep_app_token}",
|
|
29
|
+
}
|
|
30
|
+
response = requests.get(deployment_url, headers=headers, timeout=_TIMEOUT)
|
|
31
|
+
response.raise_for_status()
|
|
32
|
+
|
|
33
|
+
data = response.json()
|
|
34
|
+
deployment["id"] = data["deployments"][0]["id"]
|
|
35
|
+
deployment["name"] = data["deployments"][0]["name"]
|
|
36
|
+
deployment["slug"] = data["deployments"][0]["slug"]
|
|
37
|
+
|
|
38
|
+
return deployment
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@timeit
|
|
42
|
+
def load_semgrep_deployment(
|
|
43
|
+
neo4j_session: neo4j.Session, deployment: Dict[str, Any], update_tag: int,
|
|
44
|
+
) -> None:
|
|
45
|
+
logger.info(f"Loading SemgrepDeployment {deployment} into the graph.")
|
|
46
|
+
load(
|
|
47
|
+
neo4j_session,
|
|
48
|
+
SemgrepDeploymentSchema(),
|
|
49
|
+
[deployment],
|
|
50
|
+
lastupdated=update_tag,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@timeit
|
|
55
|
+
def sync_deployment(
|
|
56
|
+
neo4j_session: neo4j.Session,
|
|
57
|
+
semgrep_app_token: str,
|
|
58
|
+
update_tag: int,
|
|
59
|
+
common_job_parameters: Dict[str, Any],
|
|
60
|
+
) -> None:
|
|
61
|
+
|
|
62
|
+
semgrep_deployment = get_deployment(semgrep_app_token)
|
|
63
|
+
deployment_id = semgrep_deployment["id"]
|
|
64
|
+
deployment_slug = semgrep_deployment["slug"]
|
|
65
|
+
load_semgrep_deployment(neo4j_session, semgrep_deployment, update_tag)
|
|
66
|
+
common_job_parameters["DEPLOYMENT_ID"] = deployment_id
|
|
67
|
+
common_job_parameters["DEPLOYMENT_SLUG"] = deployment_slug
|
|
@@ -11,7 +11,6 @@ from requests.exceptions import ReadTimeout
|
|
|
11
11
|
|
|
12
12
|
from cartography.client.core.tx import load
|
|
13
13
|
from cartography.graph.job import GraphJob
|
|
14
|
-
from cartography.models.semgrep.deployment import SemgrepDeploymentSchema
|
|
15
14
|
from cartography.models.semgrep.findings import SemgrepSCAFindingSchema
|
|
16
15
|
from cartography.models.semgrep.locations import SemgrepSCALocationSchema
|
|
17
16
|
from cartography.stats import get_stats_client
|
|
@@ -26,29 +25,6 @@ _TIMEOUT = (60, 60)
|
|
|
26
25
|
_MAX_RETRIES = 3
|
|
27
26
|
|
|
28
27
|
|
|
29
|
-
@timeit
|
|
30
|
-
def get_deployment(semgrep_app_token: str) -> Dict[str, Any]:
|
|
31
|
-
"""
|
|
32
|
-
Gets the deployment associated with the passed Semgrep App token.
|
|
33
|
-
param: semgrep_app_token: The Semgrep App token to use for authentication.
|
|
34
|
-
"""
|
|
35
|
-
deployment = {}
|
|
36
|
-
deployment_url = "https://semgrep.dev/api/v1/deployments"
|
|
37
|
-
headers = {
|
|
38
|
-
"Content-Type": "application/json",
|
|
39
|
-
"Authorization": f"Bearer {semgrep_app_token}",
|
|
40
|
-
}
|
|
41
|
-
response = requests.get(deployment_url, headers=headers, timeout=_TIMEOUT)
|
|
42
|
-
response.raise_for_status()
|
|
43
|
-
|
|
44
|
-
data = response.json()
|
|
45
|
-
deployment["id"] = data["deployments"][0]["id"]
|
|
46
|
-
deployment["name"] = data["deployments"][0]["name"]
|
|
47
|
-
deployment["slug"] = data["deployments"][0]["slug"]
|
|
48
|
-
|
|
49
|
-
return deployment
|
|
50
|
-
|
|
51
|
-
|
|
52
28
|
@timeit
|
|
53
29
|
def get_sca_vulns(semgrep_app_token: str, deployment_slug: str) -> List[Dict[str, Any]]:
|
|
54
30
|
"""
|
|
@@ -81,11 +57,11 @@ def get_sca_vulns(semgrep_app_token: str, deployment_slug: str) -> List[Dict[str
|
|
|
81
57
|
response = requests.get(sca_url, params=request_data, headers=headers, timeout=_TIMEOUT)
|
|
82
58
|
response.raise_for_status()
|
|
83
59
|
data = response.json()
|
|
84
|
-
except (ReadTimeout, HTTPError)
|
|
60
|
+
except (ReadTimeout, HTTPError):
|
|
85
61
|
logger.warning(f"Failed to retrieve Semgrep SCA vulns for page {page}. Retrying...")
|
|
86
62
|
retries += 1
|
|
87
63
|
if retries >= _MAX_RETRIES:
|
|
88
|
-
raise
|
|
64
|
+
raise
|
|
89
65
|
continue
|
|
90
66
|
vulns = data["findings"]
|
|
91
67
|
has_more = len(vulns) > 0
|
|
@@ -201,19 +177,6 @@ def transform_sca_vulns(raw_vulns: List[Dict[str, Any]]) -> Tuple[List[Dict[str,
|
|
|
201
177
|
return vulns, usages
|
|
202
178
|
|
|
203
179
|
|
|
204
|
-
@timeit
|
|
205
|
-
def load_semgrep_deployment(
|
|
206
|
-
neo4j_session: neo4j.Session, deployment: Dict[str, Any], update_tag: int,
|
|
207
|
-
) -> None:
|
|
208
|
-
logger.info(f"Loading Semgrep deployment info {deployment} into the graph...")
|
|
209
|
-
load(
|
|
210
|
-
neo4j_session,
|
|
211
|
-
SemgrepDeploymentSchema(),
|
|
212
|
-
[deployment],
|
|
213
|
-
lastupdated=update_tag,
|
|
214
|
-
)
|
|
215
|
-
|
|
216
|
-
|
|
217
180
|
@timeit
|
|
218
181
|
def load_semgrep_sca_vulns(
|
|
219
182
|
neo4j_session: neo4j.Session,
|
|
@@ -221,7 +184,7 @@ def load_semgrep_sca_vulns(
|
|
|
221
184
|
deployment_id: str,
|
|
222
185
|
update_tag: int,
|
|
223
186
|
) -> None:
|
|
224
|
-
logger.info(f"Loading {len(vulns)}
|
|
187
|
+
logger.info(f"Loading {len(vulns)} SemgrepSCAFinding objects into the graph.")
|
|
225
188
|
load(
|
|
226
189
|
neo4j_session,
|
|
227
190
|
SemgrepSCAFindingSchema(),
|
|
@@ -238,7 +201,7 @@ def load_semgrep_sca_usages(
|
|
|
238
201
|
deployment_id: str,
|
|
239
202
|
update_tag: int,
|
|
240
203
|
) -> None:
|
|
241
|
-
logger.info(f"Loading {len(usages)}
|
|
204
|
+
logger.info(f"Loading {len(usages)} SemgrepSCALocation objects into the graph.")
|
|
242
205
|
load(
|
|
243
206
|
neo4j_session,
|
|
244
207
|
SemgrepSCALocationSchema(),
|
|
@@ -265,26 +228,32 @@ def cleanup(
|
|
|
265
228
|
|
|
266
229
|
|
|
267
230
|
@timeit
|
|
268
|
-
def
|
|
269
|
-
|
|
231
|
+
def sync_findings(
|
|
232
|
+
neo4j_session: neo4j.Session,
|
|
270
233
|
semgrep_app_token: str,
|
|
271
234
|
update_tag: int,
|
|
272
235
|
common_job_parameters: Dict[str, Any],
|
|
273
236
|
) -> None:
|
|
237
|
+
|
|
238
|
+
deployment_id = common_job_parameters.get("DEPLOYMENT_ID")
|
|
239
|
+
deployment_slug = common_job_parameters.get("DEPLOYMENT_SLUG")
|
|
240
|
+
if not deployment_id or not deployment_slug:
|
|
241
|
+
logger.warning(
|
|
242
|
+
"Missing Semgrep deployment ID or slug, ensure that sync_deployment() has been called."
|
|
243
|
+
"Skipping SCA findings sync job.",
|
|
244
|
+
)
|
|
245
|
+
return
|
|
246
|
+
|
|
274
247
|
logger.info("Running Semgrep SCA findings sync job.")
|
|
275
|
-
semgrep_deployment = get_deployment(semgrep_app_token)
|
|
276
|
-
deployment_id = semgrep_deployment["id"]
|
|
277
|
-
deployment_slug = semgrep_deployment["slug"]
|
|
278
|
-
load_semgrep_deployment(neo4j_sesion, semgrep_deployment, update_tag)
|
|
279
|
-
common_job_parameters["DEPLOYMENT_ID"] = deployment_id
|
|
280
248
|
raw_vulns = get_sca_vulns(semgrep_app_token, deployment_slug)
|
|
281
249
|
vulns, usages = transform_sca_vulns(raw_vulns)
|
|
282
|
-
load_semgrep_sca_vulns(
|
|
283
|
-
load_semgrep_sca_usages(
|
|
284
|
-
run_scoped_analysis_job('semgrep_sca_risk_analysis.json',
|
|
285
|
-
|
|
250
|
+
load_semgrep_sca_vulns(neo4j_session, vulns, deployment_id, update_tag)
|
|
251
|
+
load_semgrep_sca_usages(neo4j_session, usages, deployment_id, update_tag)
|
|
252
|
+
run_scoped_analysis_job('semgrep_sca_risk_analysis.json', neo4j_session, common_job_parameters)
|
|
253
|
+
|
|
254
|
+
cleanup(neo4j_session, common_job_parameters)
|
|
286
255
|
merge_module_sync_metadata(
|
|
287
|
-
neo4j_session=
|
|
256
|
+
neo4j_session=neo4j_session,
|
|
288
257
|
group_type='Semgrep',
|
|
289
258
|
group_id=deployment_id,
|
|
290
259
|
synced_type='SCA',
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
import neo4j
|
|
4
|
+
|
|
5
|
+
from cartography.config import Config
|
|
6
|
+
from cartography.intel.snipeit import asset
|
|
7
|
+
from cartography.intel.snipeit import user
|
|
8
|
+
from cartography.stats import get_stats_client
|
|
9
|
+
from cartography.util import timeit
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
stat_handler = get_stats_client(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@timeit
|
|
16
|
+
def start_snipeit_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
|
|
17
|
+
if config.snipeit_base_uri is None or config.snipeit_token is None or config.snipeit_tenant_id is None:
|
|
18
|
+
logger.warning(
|
|
19
|
+
"Required parameter(s) missing. Skipping sync.",
|
|
20
|
+
)
|
|
21
|
+
return
|
|
22
|
+
|
|
23
|
+
common_job_parameters = {
|
|
24
|
+
"UPDATE_TAG": config.update_tag,
|
|
25
|
+
"TENANT_ID": config.snipeit_tenant_id,
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
# Ingest SnipeIT users and assets
|
|
29
|
+
user.sync(neo4j_session, common_job_parameters, config.snipeit_base_uri, config.snipeit_token)
|
|
30
|
+
asset.sync(neo4j_session, common_job_parameters, config.snipeit_base_uri, config.snipeit_token)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
from typing import Dict
|
|
4
|
+
from typing import List
|
|
5
|
+
|
|
6
|
+
import neo4j
|
|
7
|
+
|
|
8
|
+
from .util import call_snipeit_api
|
|
9
|
+
from cartography.client.core.tx import load
|
|
10
|
+
from cartography.graph.job import GraphJob
|
|
11
|
+
from cartography.models.snipeit.asset import SnipeitAssetSchema
|
|
12
|
+
from cartography.models.snipeit.tenant import SnipeitTenantSchema
|
|
13
|
+
from cartography.util import timeit
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@timeit
|
|
20
|
+
def get(base_uri: str, token: str) -> List[Dict]:
|
|
21
|
+
api_endpoint = "/api/v1/hardware"
|
|
22
|
+
results: List[Dict[str, Any]] = []
|
|
23
|
+
while True:
|
|
24
|
+
offset = len(results)
|
|
25
|
+
api_endpoint = f"{api_endpoint}?order='asc'&offset={offset}"
|
|
26
|
+
response = call_snipeit_api(api_endpoint, base_uri, token)
|
|
27
|
+
results.extend(response['rows'])
|
|
28
|
+
|
|
29
|
+
total = response['total']
|
|
30
|
+
results_count = len(results)
|
|
31
|
+
if results_count >= total:
|
|
32
|
+
break
|
|
33
|
+
|
|
34
|
+
return results
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@timeit
|
|
38
|
+
def load_assets(
|
|
39
|
+
neo4j_session: neo4j.Session,
|
|
40
|
+
common_job_parameters: Dict,
|
|
41
|
+
data: List[Dict[str, Any]],
|
|
42
|
+
) -> None:
|
|
43
|
+
# Create the SnipeIT Tenant
|
|
44
|
+
load(
|
|
45
|
+
neo4j_session,
|
|
46
|
+
SnipeitTenantSchema(),
|
|
47
|
+
[{'id': common_job_parameters["TENANT_ID"]}],
|
|
48
|
+
lastupdated=common_job_parameters["UPDATE_TAG"],
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
load(
|
|
52
|
+
neo4j_session,
|
|
53
|
+
SnipeitAssetSchema(),
|
|
54
|
+
data,
|
|
55
|
+
lastupdated=common_job_parameters["UPDATE_TAG"],
|
|
56
|
+
TENANT_ID=common_job_parameters["TENANT_ID"],
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@timeit
|
|
61
|
+
def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
|
|
62
|
+
GraphJob.from_node_schema(SnipeitAssetSchema(), common_job_parameters).run(neo4j_session)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@timeit
|
|
66
|
+
def sync(
|
|
67
|
+
neo4j_session: neo4j.Session,
|
|
68
|
+
common_job_parameters: Dict,
|
|
69
|
+
base_uri: str,
|
|
70
|
+
token: str,
|
|
71
|
+
) -> None:
|
|
72
|
+
assets = get(base_uri=base_uri, token=token)
|
|
73
|
+
load_assets(neo4j_session=neo4j_session, common_job_parameters=common_job_parameters, data=assets)
|
|
74
|
+
cleanup(neo4j_session, common_job_parameters)
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
from typing import Dict
|
|
4
|
+
from typing import List
|
|
5
|
+
|
|
6
|
+
import neo4j
|
|
7
|
+
|
|
8
|
+
from .util import call_snipeit_api
|
|
9
|
+
from cartography.client.core.tx import load
|
|
10
|
+
from cartography.graph.job import GraphJob
|
|
11
|
+
from cartography.models.snipeit.tenant import SnipeitTenantSchema
|
|
12
|
+
from cartography.models.snipeit.user import SnipeitUserSchema
|
|
13
|
+
from cartography.util import timeit
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@timeit
|
|
19
|
+
def get(base_uri: str, token: str) -> List[Dict]:
|
|
20
|
+
api_endpoint = "/api/v1/users"
|
|
21
|
+
results: List[Dict[str, Any]] = []
|
|
22
|
+
while True:
|
|
23
|
+
offset = len(results)
|
|
24
|
+
api_endpoint = f"{api_endpoint}?order='asc'&offset={offset}"
|
|
25
|
+
response = call_snipeit_api(api_endpoint, base_uri, token)
|
|
26
|
+
results.extend(response['rows'])
|
|
27
|
+
|
|
28
|
+
total = response['total']
|
|
29
|
+
results_count = len(results)
|
|
30
|
+
if results_count >= total:
|
|
31
|
+
break
|
|
32
|
+
|
|
33
|
+
return results
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@timeit
|
|
37
|
+
def load_users(
|
|
38
|
+
neo4j_session: neo4j.Session,
|
|
39
|
+
common_job_parameters: Dict,
|
|
40
|
+
data: List[Dict[str, Any]],
|
|
41
|
+
) -> None:
|
|
42
|
+
logger.debug(data[0])
|
|
43
|
+
|
|
44
|
+
# Create the SnipeIT Tenant
|
|
45
|
+
load(
|
|
46
|
+
neo4j_session,
|
|
47
|
+
SnipeitTenantSchema(),
|
|
48
|
+
[{'id': common_job_parameters["TENANT_ID"]}],
|
|
49
|
+
lastupdated=common_job_parameters["UPDATE_TAG"],
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
load(
|
|
53
|
+
neo4j_session,
|
|
54
|
+
SnipeitUserSchema(),
|
|
55
|
+
data,
|
|
56
|
+
lastupdated=common_job_parameters["UPDATE_TAG"],
|
|
57
|
+
TENANT_ID=common_job_parameters["TENANT_ID"],
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@timeit
|
|
62
|
+
def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
|
|
63
|
+
GraphJob.from_node_schema(SnipeitUserSchema(), common_job_parameters).run(neo4j_session)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@timeit
|
|
67
|
+
def sync(
|
|
68
|
+
neo4j_session: neo4j.Session,
|
|
69
|
+
common_job_parameters: Dict,
|
|
70
|
+
base_uri: str,
|
|
71
|
+
token: str,
|
|
72
|
+
) -> None:
|
|
73
|
+
users = get(base_uri=base_uri, token=token)
|
|
74
|
+
load_users(neo4j_session, common_job_parameters, users)
|
|
75
|
+
cleanup(neo4j_session, common_job_parameters)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
from typing import Dict
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
|
|
7
|
+
from cartography.util import timeit
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
|
|
11
|
+
_TIMEOUT = (60, 60)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@timeit
|
|
15
|
+
def call_snipeit_api(api_and_parameters: str, base_uri: str, token: str) -> Dict[str, Any]:
|
|
16
|
+
uri = base_uri + api_and_parameters
|
|
17
|
+
try:
|
|
18
|
+
logger.debug(
|
|
19
|
+
"SnipeIT: Get %s", uri,
|
|
20
|
+
)
|
|
21
|
+
response = requests.get(
|
|
22
|
+
uri,
|
|
23
|
+
headers={
|
|
24
|
+
'Accept': 'application/json',
|
|
25
|
+
'Authorization': f'Bearer {token}',
|
|
26
|
+
},
|
|
27
|
+
timeout=_TIMEOUT,
|
|
28
|
+
)
|
|
29
|
+
except requests.exceptions.Timeout:
|
|
30
|
+
# Add context and re-raise for callers to handle
|
|
31
|
+
logger.warning(f"SnipeIT: requests.get('{uri}') timed out.")
|
|
32
|
+
raise
|
|
33
|
+
# if call failed, use requests library to raise an exception
|
|
34
|
+
response.raise_for_status()
|
|
35
|
+
return response.json()
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from cartography.models.core.common import PropertyRef
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
6
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
7
|
+
from cartography.models.core.nodes import ExtraNodeLabels
|
|
8
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
9
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
10
|
+
from cartography.models.core.relationships import LinkDirection
|
|
11
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
12
|
+
from cartography.models.core.relationships import OtherRelationships
|
|
13
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class SemgrepDependencyNodeProperties(CartographyNodeProperties):
|
|
18
|
+
id: PropertyRef = PropertyRef('id')
|
|
19
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
20
|
+
name: PropertyRef = PropertyRef('name')
|
|
21
|
+
ecosystem: PropertyRef = PropertyRef('ecosystem')
|
|
22
|
+
version: PropertyRef = PropertyRef('version')
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class SemgrepDependencyToSemgrepDeploymentRelProperties(CartographyRelProperties):
|
|
27
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
# (:SemgrepDependency)<-[:RESOURCE]-(:SemgrepDeployment)
|
|
32
|
+
class SemgrepDependencyToSemgrepDeploymentSchema(CartographyRelSchema):
|
|
33
|
+
target_node_label: str = 'SemgrepDeployment'
|
|
34
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
35
|
+
{'id': PropertyRef('DEPLOYMENT_ID', set_in_kwargs=True)},
|
|
36
|
+
)
|
|
37
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
38
|
+
rel_label: str = "RESOURCE"
|
|
39
|
+
properties: SemgrepDependencyToSemgrepDeploymentRelProperties = SemgrepDependencyToSemgrepDeploymentRelProperties()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class SemgrepDependencyToGithubRepoRelProperties(CartographyRelProperties):
|
|
44
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
45
|
+
specifier: PropertyRef = PropertyRef('specifier')
|
|
46
|
+
transitivity: PropertyRef = PropertyRef('transitivity')
|
|
47
|
+
url: PropertyRef = PropertyRef('url')
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
# (:SemgrepDependency)<-[:REQUIRES]-(:GitHubRepository)
|
|
52
|
+
class SemgrepDependencyToGithubRepoRel(CartographyRelSchema):
|
|
53
|
+
target_node_label: str = 'GitHubRepository'
|
|
54
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
55
|
+
{'id': PropertyRef('repo_url')},
|
|
56
|
+
)
|
|
57
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
58
|
+
rel_label: str = "REQUIRES"
|
|
59
|
+
properties: SemgrepDependencyToGithubRepoRelProperties = SemgrepDependencyToGithubRepoRelProperties()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class SemgrepSCAFindngToDependencyRelProperties(CartographyRelProperties):
|
|
64
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class SemgrepGoLibrarySchema(CartographyNodeSchema):
|
|
69
|
+
label: str = 'GoLibrary'
|
|
70
|
+
extra_node_labels: Optional[ExtraNodeLabels] = ExtraNodeLabels(['Dependency', 'SemgrepDependency'])
|
|
71
|
+
properties: SemgrepDependencyNodeProperties = SemgrepDependencyNodeProperties()
|
|
72
|
+
sub_resource_relationship: SemgrepDependencyToSemgrepDeploymentSchema = SemgrepDependencyToSemgrepDeploymentSchema()
|
|
73
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
74
|
+
[
|
|
75
|
+
SemgrepDependencyToGithubRepoRel(),
|
|
76
|
+
],
|
|
77
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from cartography.models.core.common import PropertyRef
|
|
4
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
7
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
8
|
+
from cartography.models.core.relationships import LinkDirection
|
|
9
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
10
|
+
from cartography.models.core.relationships import OtherRelationships
|
|
11
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class SnipeitAssetNodeProperties(CartographyNodeProperties):
|
|
16
|
+
"""
|
|
17
|
+
https://snipe-it.readme.io/reference/hardware-list
|
|
18
|
+
"""
|
|
19
|
+
# Common properties
|
|
20
|
+
id: PropertyRef = PropertyRef('id')
|
|
21
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
22
|
+
|
|
23
|
+
# SnipeIT specific properties
|
|
24
|
+
asset_tag: PropertyRef = PropertyRef('asset_tag')
|
|
25
|
+
assigned_to: PropertyRef = PropertyRef('assigned_to.email')
|
|
26
|
+
category: PropertyRef = PropertyRef('category.name')
|
|
27
|
+
company: PropertyRef = PropertyRef('company.name')
|
|
28
|
+
manufacturer: PropertyRef = PropertyRef('manufacturer.name')
|
|
29
|
+
model: PropertyRef = PropertyRef('model.name')
|
|
30
|
+
serial: PropertyRef = PropertyRef('serial', extra_index=True)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
###
|
|
34
|
+
# (:SnipeitAsset)<-[:ASSET]-(:SnipeitTenant)
|
|
35
|
+
###
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class SnipeitTenantToSnipeitAssetRelProperties(CartographyRelProperties):
|
|
38
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class SnipeitTenantToSnipeitAssetRel(CartographyRelSchema):
|
|
43
|
+
target_node_label: str = 'SnipeitTenant'
|
|
44
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
45
|
+
{'id': PropertyRef('TENANT_ID', set_in_kwargs=True)},
|
|
46
|
+
)
|
|
47
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
48
|
+
rel_label: str = "HAS_ASSET"
|
|
49
|
+
properties: SnipeitTenantToSnipeitAssetRelProperties = SnipeitTenantToSnipeitAssetRelProperties()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
###
|
|
53
|
+
# (:SnipeitUser)-[:HAS_CHECKED_OUT]->(:SnipeitAsset)
|
|
54
|
+
###
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class SnipeitUserToSnipeitAssetProperties(CartographyRelProperties):
|
|
57
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class SnipeitUserToSnipeitAssetRel(CartographyRelSchema):
|
|
62
|
+
target_node_label: str = 'SnipeitUser'
|
|
63
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
64
|
+
{'email': PropertyRef('assigned_to.email')},
|
|
65
|
+
)
|
|
66
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
67
|
+
rel_label: str = "HAS_CHECKED_OUT"
|
|
68
|
+
properties: SnipeitUserToSnipeitAssetProperties = SnipeitUserToSnipeitAssetProperties()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
###
|
|
72
|
+
@dataclass(frozen=True)
|
|
73
|
+
class SnipeitAssetSchema(CartographyNodeSchema):
|
|
74
|
+
label: str = 'SnipeitAsset' # The label of the node
|
|
75
|
+
properties: SnipeitAssetNodeProperties = SnipeitAssetNodeProperties() # An object representing all properties
|
|
76
|
+
sub_resource_relationship: SnipeitTenantToSnipeitAssetRel = SnipeitTenantToSnipeitAssetRel()
|
|
77
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
78
|
+
[
|
|
79
|
+
SnipeitUserToSnipeitAssetRel(),
|
|
80
|
+
],
|
|
81
|
+
)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from cartography.models.core.common import PropertyRef
|
|
4
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class SnipeitTenantNodeProperties(CartographyNodeProperties):
|
|
10
|
+
id: PropertyRef = PropertyRef('id')
|
|
11
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class SnipeitTenantSchema(CartographyNodeSchema):
|
|
16
|
+
label: str = 'SnipeitTenant' # The label of the node
|
|
17
|
+
properties: SnipeitTenantNodeProperties = SnipeitTenantNodeProperties() # An object representing all properties
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from cartography.models.core.common import PropertyRef
|
|
4
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
7
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
8
|
+
from cartography.models.core.relationships import LinkDirection
|
|
9
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
10
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class SnipeitUserNodeProperties(CartographyNodeProperties):
|
|
15
|
+
"""
|
|
16
|
+
Ref: https://snipe-it.readme.io/reference/users
|
|
17
|
+
"""
|
|
18
|
+
# Common properties
|
|
19
|
+
id: PropertyRef = PropertyRef('id')
|
|
20
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
21
|
+
|
|
22
|
+
# SnipeIT specific properties
|
|
23
|
+
company: PropertyRef = PropertyRef('company_id.name', extra_index=True)
|
|
24
|
+
email: PropertyRef = PropertyRef('email', extra_index=True)
|
|
25
|
+
username: PropertyRef = PropertyRef('username')
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class SnipeitTenantToSnipeitUserRelProperties(CartographyRelProperties):
|
|
30
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
# (:SnipeitTenant)-[:HAS_USER]->(:SnipeitUser)
|
|
35
|
+
class SnipeitTenantToSnipeitUserRel(CartographyRelSchema):
|
|
36
|
+
target_node_label: str = 'SnipeitTenant'
|
|
37
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
38
|
+
{'id': PropertyRef('TENANT_ID', set_in_kwargs=True)},
|
|
39
|
+
)
|
|
40
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
41
|
+
rel_label: str = "HAS_USER"
|
|
42
|
+
properties: SnipeitTenantToSnipeitUserRelProperties = SnipeitTenantToSnipeitUserRelProperties()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class SnipeitUserSchema(CartographyNodeSchema):
|
|
47
|
+
label: str = 'SnipeitUser' # The label of the node
|
|
48
|
+
properties: SnipeitUserNodeProperties = SnipeitUserNodeProperties() # An object representing all properties
|
|
49
|
+
sub_resource_relationship: SnipeitTenantToSnipeitUserRel = SnipeitTenantToSnipeitUserRel()
|
cartography/sync.py
CHANGED
|
@@ -17,7 +17,6 @@ import cartography.intel.azure
|
|
|
17
17
|
import cartography.intel.bigfix
|
|
18
18
|
import cartography.intel.create_indexes
|
|
19
19
|
import cartography.intel.crowdstrike
|
|
20
|
-
import cartography.intel.crxcavator.crxcavator
|
|
21
20
|
import cartography.intel.cve
|
|
22
21
|
import cartography.intel.digitalocean
|
|
23
22
|
import cartography.intel.duo
|
|
@@ -30,6 +29,7 @@ import cartography.intel.lastpass
|
|
|
30
29
|
import cartography.intel.oci
|
|
31
30
|
import cartography.intel.okta
|
|
32
31
|
import cartography.intel.semgrep
|
|
32
|
+
import cartography.intel.snipeit
|
|
33
33
|
from cartography.config import Config
|
|
34
34
|
from cartography.stats import set_stats_client
|
|
35
35
|
from cartography.util import STATUS_FAILURE
|
|
@@ -45,7 +45,6 @@ TOP_LEVEL_MODULES = OrderedDict({ # preserve order so that the default sync alw
|
|
|
45
45
|
'crowdstrike': cartography.intel.crowdstrike.start_crowdstrike_ingestion,
|
|
46
46
|
'gcp': cartography.intel.gcp.start_gcp_ingestion,
|
|
47
47
|
'gsuite': cartography.intel.gsuite.start_gsuite_ingestion,
|
|
48
|
-
'crxcavator': cartography.intel.crxcavator.start_extension_ingestion,
|
|
49
48
|
'cve': cartography.intel.cve.start_cve_ingestion,
|
|
50
49
|
'oci': cartography.intel.oci.start_oci_ingestion,
|
|
51
50
|
'okta': cartography.intel.okta.start_okta_ingestion,
|
|
@@ -57,6 +56,7 @@ TOP_LEVEL_MODULES = OrderedDict({ # preserve order so that the default sync alw
|
|
|
57
56
|
'bigfix': cartography.intel.bigfix.start_bigfix_ingestion,
|
|
58
57
|
'duo': cartography.intel.duo.start_duo_ingestion,
|
|
59
58
|
'semgrep': cartography.intel.semgrep.start_semgrep_ingestion,
|
|
59
|
+
'snipeit': cartography.intel.snipeit.start_snipeit_ingestion,
|
|
60
60
|
'analysis': cartography.intel.analysis.run,
|
|
61
61
|
})
|
|
62
62
|
|