cartography 0.95.0rc1__py3-none-any.whl → 0.96.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/cli.py +15 -0
- cartography/config.py +4 -0
- cartography/data/indexes.cypher +1 -2
- cartography/data/jobs/cleanup/aws_import_identity_center_cleanup.json +16 -0
- cartography/data/jobs/cleanup/{github_users_cleanup.json → github_org_and_users_cleanup.json} +5 -0
- cartography/graph/querybuilder.py +4 -0
- cartography/intel/aws/ec2/network_acls.py +208 -0
- cartography/intel/aws/identitycenter.py +307 -0
- cartography/intel/aws/resources.py +4 -0
- cartography/intel/github/users.py +156 -39
- cartography/intel/okta/users.py +2 -1
- cartography/intel/semgrep/__init__.py +9 -2
- cartography/intel/semgrep/dependencies.py +233 -0
- cartography/intel/semgrep/deployment.py +67 -0
- cartography/intel/semgrep/findings.py +22 -53
- cartography/models/aws/ec2/network_acl_rules.py +97 -0
- cartography/models/aws/ec2/network_acls.py +86 -0
- cartography/models/core/common.py +18 -1
- cartography/models/github/orgs.py +26 -0
- cartography/models/github/users.py +119 -0
- cartography/models/semgrep/dependencies.py +90 -0
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/METADATA +1 -1
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/RECORD +27 -17
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/WHEEL +1 -1
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/LICENSE +0 -0
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/entry_points.txt +0 -0
- {cartography-0.95.0rc1.dist-info → cartography-0.96.0rc2.dist-info}/top_level.txt +0 -0
|
@@ -11,7 +11,6 @@ from requests.exceptions import ReadTimeout
|
|
|
11
11
|
|
|
12
12
|
from cartography.client.core.tx import load
|
|
13
13
|
from cartography.graph.job import GraphJob
|
|
14
|
-
from cartography.models.semgrep.deployment import SemgrepDeploymentSchema
|
|
15
14
|
from cartography.models.semgrep.findings import SemgrepSCAFindingSchema
|
|
16
15
|
from cartography.models.semgrep.locations import SemgrepSCALocationSchema
|
|
17
16
|
from cartography.stats import get_stats_client
|
|
@@ -26,29 +25,6 @@ _TIMEOUT = (60, 60)
|
|
|
26
25
|
_MAX_RETRIES = 3
|
|
27
26
|
|
|
28
27
|
|
|
29
|
-
@timeit
|
|
30
|
-
def get_deployment(semgrep_app_token: str) -> Dict[str, Any]:
|
|
31
|
-
"""
|
|
32
|
-
Gets the deployment associated with the passed Semgrep App token.
|
|
33
|
-
param: semgrep_app_token: The Semgrep App token to use for authentication.
|
|
34
|
-
"""
|
|
35
|
-
deployment = {}
|
|
36
|
-
deployment_url = "https://semgrep.dev/api/v1/deployments"
|
|
37
|
-
headers = {
|
|
38
|
-
"Content-Type": "application/json",
|
|
39
|
-
"Authorization": f"Bearer {semgrep_app_token}",
|
|
40
|
-
}
|
|
41
|
-
response = requests.get(deployment_url, headers=headers, timeout=_TIMEOUT)
|
|
42
|
-
response.raise_for_status()
|
|
43
|
-
|
|
44
|
-
data = response.json()
|
|
45
|
-
deployment["id"] = data["deployments"][0]["id"]
|
|
46
|
-
deployment["name"] = data["deployments"][0]["name"]
|
|
47
|
-
deployment["slug"] = data["deployments"][0]["slug"]
|
|
48
|
-
|
|
49
|
-
return deployment
|
|
50
|
-
|
|
51
|
-
|
|
52
28
|
@timeit
|
|
53
29
|
def get_sca_vulns(semgrep_app_token: str, deployment_slug: str) -> List[Dict[str, Any]]:
|
|
54
30
|
"""
|
|
@@ -81,11 +57,11 @@ def get_sca_vulns(semgrep_app_token: str, deployment_slug: str) -> List[Dict[str
|
|
|
81
57
|
response = requests.get(sca_url, params=request_data, headers=headers, timeout=_TIMEOUT)
|
|
82
58
|
response.raise_for_status()
|
|
83
59
|
data = response.json()
|
|
84
|
-
except (ReadTimeout, HTTPError)
|
|
60
|
+
except (ReadTimeout, HTTPError):
|
|
85
61
|
logger.warning(f"Failed to retrieve Semgrep SCA vulns for page {page}. Retrying...")
|
|
86
62
|
retries += 1
|
|
87
63
|
if retries >= _MAX_RETRIES:
|
|
88
|
-
raise
|
|
64
|
+
raise
|
|
89
65
|
continue
|
|
90
66
|
vulns = data["findings"]
|
|
91
67
|
has_more = len(vulns) > 0
|
|
@@ -201,19 +177,6 @@ def transform_sca_vulns(raw_vulns: List[Dict[str, Any]]) -> Tuple[List[Dict[str,
|
|
|
201
177
|
return vulns, usages
|
|
202
178
|
|
|
203
179
|
|
|
204
|
-
@timeit
|
|
205
|
-
def load_semgrep_deployment(
|
|
206
|
-
neo4j_session: neo4j.Session, deployment: Dict[str, Any], update_tag: int,
|
|
207
|
-
) -> None:
|
|
208
|
-
logger.info(f"Loading Semgrep deployment info {deployment} into the graph...")
|
|
209
|
-
load(
|
|
210
|
-
neo4j_session,
|
|
211
|
-
SemgrepDeploymentSchema(),
|
|
212
|
-
[deployment],
|
|
213
|
-
lastupdated=update_tag,
|
|
214
|
-
)
|
|
215
|
-
|
|
216
|
-
|
|
217
180
|
@timeit
|
|
218
181
|
def load_semgrep_sca_vulns(
|
|
219
182
|
neo4j_session: neo4j.Session,
|
|
@@ -221,7 +184,7 @@ def load_semgrep_sca_vulns(
|
|
|
221
184
|
deployment_id: str,
|
|
222
185
|
update_tag: int,
|
|
223
186
|
) -> None:
|
|
224
|
-
logger.info(f"Loading {len(vulns)}
|
|
187
|
+
logger.info(f"Loading {len(vulns)} SemgrepSCAFinding objects into the graph.")
|
|
225
188
|
load(
|
|
226
189
|
neo4j_session,
|
|
227
190
|
SemgrepSCAFindingSchema(),
|
|
@@ -238,7 +201,7 @@ def load_semgrep_sca_usages(
|
|
|
238
201
|
deployment_id: str,
|
|
239
202
|
update_tag: int,
|
|
240
203
|
) -> None:
|
|
241
|
-
logger.info(f"Loading {len(usages)}
|
|
204
|
+
logger.info(f"Loading {len(usages)} SemgrepSCALocation objects into the graph.")
|
|
242
205
|
load(
|
|
243
206
|
neo4j_session,
|
|
244
207
|
SemgrepSCALocationSchema(),
|
|
@@ -265,26 +228,32 @@ def cleanup(
|
|
|
265
228
|
|
|
266
229
|
|
|
267
230
|
@timeit
|
|
268
|
-
def
|
|
269
|
-
|
|
231
|
+
def sync_findings(
|
|
232
|
+
neo4j_session: neo4j.Session,
|
|
270
233
|
semgrep_app_token: str,
|
|
271
234
|
update_tag: int,
|
|
272
235
|
common_job_parameters: Dict[str, Any],
|
|
273
236
|
) -> None:
|
|
237
|
+
|
|
238
|
+
deployment_id = common_job_parameters.get("DEPLOYMENT_ID")
|
|
239
|
+
deployment_slug = common_job_parameters.get("DEPLOYMENT_SLUG")
|
|
240
|
+
if not deployment_id or not deployment_slug:
|
|
241
|
+
logger.warning(
|
|
242
|
+
"Missing Semgrep deployment ID or slug, ensure that sync_deployment() has been called."
|
|
243
|
+
"Skipping SCA findings sync job.",
|
|
244
|
+
)
|
|
245
|
+
return
|
|
246
|
+
|
|
274
247
|
logger.info("Running Semgrep SCA findings sync job.")
|
|
275
|
-
semgrep_deployment = get_deployment(semgrep_app_token)
|
|
276
|
-
deployment_id = semgrep_deployment["id"]
|
|
277
|
-
deployment_slug = semgrep_deployment["slug"]
|
|
278
|
-
load_semgrep_deployment(neo4j_sesion, semgrep_deployment, update_tag)
|
|
279
|
-
common_job_parameters["DEPLOYMENT_ID"] = deployment_id
|
|
280
248
|
raw_vulns = get_sca_vulns(semgrep_app_token, deployment_slug)
|
|
281
249
|
vulns, usages = transform_sca_vulns(raw_vulns)
|
|
282
|
-
load_semgrep_sca_vulns(
|
|
283
|
-
load_semgrep_sca_usages(
|
|
284
|
-
run_scoped_analysis_job('semgrep_sca_risk_analysis.json',
|
|
285
|
-
|
|
250
|
+
load_semgrep_sca_vulns(neo4j_session, vulns, deployment_id, update_tag)
|
|
251
|
+
load_semgrep_sca_usages(neo4j_session, usages, deployment_id, update_tag)
|
|
252
|
+
run_scoped_analysis_job('semgrep_sca_risk_analysis.json', neo4j_session, common_job_parameters)
|
|
253
|
+
|
|
254
|
+
cleanup(neo4j_session, common_job_parameters)
|
|
286
255
|
merge_module_sync_metadata(
|
|
287
|
-
neo4j_session=
|
|
256
|
+
neo4j_session=neo4j_session,
|
|
288
257
|
group_type='Semgrep',
|
|
289
258
|
group_id=deployment_id,
|
|
290
259
|
synced_type='SCA',
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from cartography.models.core.common import PropertyRef
|
|
4
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.nodes import ExtraNodeLabels
|
|
7
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
8
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
9
|
+
from cartography.models.core.relationships import LinkDirection
|
|
10
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
11
|
+
from cartography.models.core.relationships import OtherRelationships
|
|
12
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class EC2NetworkAclRuleNodeProperties(CartographyNodeProperties):
|
|
17
|
+
id: PropertyRef = PropertyRef('Id')
|
|
18
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
19
|
+
network_acl_id: PropertyRef = PropertyRef('NetworkAclId')
|
|
20
|
+
protocol: PropertyRef = PropertyRef('Protocol')
|
|
21
|
+
fromport: PropertyRef = PropertyRef('FromPort')
|
|
22
|
+
toport: PropertyRef = PropertyRef('ToPort')
|
|
23
|
+
cidrblock: PropertyRef = PropertyRef('CidrBlock')
|
|
24
|
+
egress: PropertyRef = PropertyRef('Egress')
|
|
25
|
+
rulenumber: PropertyRef = PropertyRef('RuleNumber')
|
|
26
|
+
ruleaction: PropertyRef = PropertyRef('RuleAction')
|
|
27
|
+
region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class EC2NetworkAclRuleAclRelProperties(CartographyRelProperties):
|
|
32
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class EC2NetworkAclRuleToAcl(CartographyRelSchema):
|
|
37
|
+
target_node_label: str = 'EC2NetworkAcl'
|
|
38
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
39
|
+
{'network_acl_id': PropertyRef('NetworkAclId')},
|
|
40
|
+
)
|
|
41
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
42
|
+
rel_label: str = "MEMBER_OF_NACL"
|
|
43
|
+
properties: EC2NetworkAclRuleAclRelProperties = EC2NetworkAclRuleAclRelProperties()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class EC2NetworkAclRuleToAwsAccountRelProperties(CartographyRelProperties):
|
|
48
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True)
|
|
52
|
+
class EC2NetworkAclRuleToAWSAccount(CartographyRelSchema):
|
|
53
|
+
target_node_label: str = 'AWSAccount'
|
|
54
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
55
|
+
{'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
|
|
56
|
+
)
|
|
57
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
58
|
+
rel_label: str = "RESOURCE"
|
|
59
|
+
properties: EC2NetworkAclRuleToAwsAccountRelProperties = EC2NetworkAclRuleToAwsAccountRelProperties()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class EC2NetworkAclInboundRuleSchema(CartographyNodeSchema):
|
|
64
|
+
"""
|
|
65
|
+
Network interface as known by describe-network-interfaces.
|
|
66
|
+
"""
|
|
67
|
+
label: str = 'EC2NetworkAclRule'
|
|
68
|
+
extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
|
|
69
|
+
['IpPermissionInbound'],
|
|
70
|
+
)
|
|
71
|
+
properties: EC2NetworkAclRuleNodeProperties = EC2NetworkAclRuleNodeProperties()
|
|
72
|
+
sub_resource_relationship: EC2NetworkAclRuleToAWSAccount = EC2NetworkAclRuleToAWSAccount()
|
|
73
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
74
|
+
[
|
|
75
|
+
EC2NetworkAclRuleToAcl(),
|
|
76
|
+
],
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class EC2NetworkAclEgressRuleSchema(CartographyNodeSchema):
|
|
82
|
+
"""
|
|
83
|
+
Network interface as known by describe-network-interfaces.
|
|
84
|
+
"""
|
|
85
|
+
label: str = 'EC2NetworkAclRule'
|
|
86
|
+
extra_node_labels: ExtraNodeLabels = ExtraNodeLabels(
|
|
87
|
+
[
|
|
88
|
+
'IpPermissionEgress',
|
|
89
|
+
],
|
|
90
|
+
)
|
|
91
|
+
properties: EC2NetworkAclRuleNodeProperties = EC2NetworkAclRuleNodeProperties()
|
|
92
|
+
sub_resource_relationship: EC2NetworkAclRuleToAWSAccount = EC2NetworkAclRuleToAWSAccount()
|
|
93
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
94
|
+
[
|
|
95
|
+
EC2NetworkAclRuleToAcl(),
|
|
96
|
+
],
|
|
97
|
+
)
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from cartography.models.core.common import PropertyRef
|
|
4
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
6
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
7
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
8
|
+
from cartography.models.core.relationships import LinkDirection
|
|
9
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
10
|
+
from cartography.models.core.relationships import OtherRelationships
|
|
11
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class EC2NetworkAclNodeProperties(CartographyNodeProperties):
|
|
16
|
+
id: PropertyRef = PropertyRef('Arn')
|
|
17
|
+
arn: PropertyRef = PropertyRef('Arn')
|
|
18
|
+
network_acl_id: PropertyRef = PropertyRef('Id')
|
|
19
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
20
|
+
is_default: PropertyRef = PropertyRef('IsDefault')
|
|
21
|
+
region: PropertyRef = PropertyRef('Region', set_in_kwargs=True)
|
|
22
|
+
vpc_id: PropertyRef = PropertyRef('VpcId')
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class EC2NetworkAclToVpcRelProperties(CartographyRelProperties):
|
|
27
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class EC2NetworkAclToVpc(CartographyRelSchema):
|
|
32
|
+
target_node_label: str = 'AWSVpc'
|
|
33
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
34
|
+
{'vpcid': PropertyRef('VpcId')},
|
|
35
|
+
)
|
|
36
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
37
|
+
rel_label: str = "MEMBER_OF_AWS_VPC"
|
|
38
|
+
properties: EC2NetworkAclToVpcRelProperties = EC2NetworkAclToVpcRelProperties()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class EC2NetworkAclToSubnetRelProperties(CartographyRelProperties):
|
|
43
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class EC2NetworkAclToSubnet(CartographyRelSchema):
|
|
48
|
+
target_node_label: str = 'EC2Subnet'
|
|
49
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
50
|
+
{'subnetid': PropertyRef('SubnetId')},
|
|
51
|
+
)
|
|
52
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
53
|
+
rel_label: str = "PART_OF_SUBNET"
|
|
54
|
+
properties: EC2NetworkAclToSubnetRelProperties = EC2NetworkAclToSubnetRelProperties()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(frozen=True)
|
|
58
|
+
class EC2NetworkAclToAwsAccountRelProperties(CartographyRelProperties):
|
|
59
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class EC2NetworkAclToAWSAccount(CartographyRelSchema):
|
|
64
|
+
target_node_label: str = 'AWSAccount'
|
|
65
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
66
|
+
{'id': PropertyRef('AWS_ID', set_in_kwargs=True)},
|
|
67
|
+
)
|
|
68
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
69
|
+
rel_label: str = "RESOURCE"
|
|
70
|
+
properties: EC2NetworkAclToAwsAccountRelProperties = EC2NetworkAclToAwsAccountRelProperties()
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class EC2NetworkAclSchema(CartographyNodeSchema):
|
|
75
|
+
"""
|
|
76
|
+
Network interface as known by describe-network-interfaces.
|
|
77
|
+
"""
|
|
78
|
+
label: str = 'EC2NetworkAcl'
|
|
79
|
+
properties: EC2NetworkAclNodeProperties = EC2NetworkAclNodeProperties()
|
|
80
|
+
sub_resource_relationship: EC2NetworkAclToAWSAccount = EC2NetworkAclToAWSAccount()
|
|
81
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
82
|
+
[
|
|
83
|
+
EC2NetworkAclToVpc(),
|
|
84
|
+
EC2NetworkAclToSubnet(),
|
|
85
|
+
],
|
|
86
|
+
)
|
|
@@ -8,7 +8,14 @@ class PropertyRef:
|
|
|
8
8
|
(PropertyRef.set_in_kwargs=True).
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
|
-
def __init__(
|
|
11
|
+
def __init__(
|
|
12
|
+
self,
|
|
13
|
+
name: str,
|
|
14
|
+
set_in_kwargs=False,
|
|
15
|
+
extra_index=False,
|
|
16
|
+
ignore_case=False,
|
|
17
|
+
fuzzy_and_ignore_case=False,
|
|
18
|
+
):
|
|
12
19
|
"""
|
|
13
20
|
:param name: The name of the property
|
|
14
21
|
:param set_in_kwargs: Optional. If True, the property is not defined on the data dict, and we expect to find the
|
|
@@ -33,11 +40,21 @@ class PropertyRef:
|
|
|
33
40
|
cartography catalog of GitHubUser nodes. Therefore, you would need `ignore_case=True` in the PropertyRef
|
|
34
41
|
that points to the GitHubUser node's name field, otherwise if one of your employees' GitHub usernames
|
|
35
42
|
contains capital letters, you would not be able to map them properly to a GitHubUser node in your graph.
|
|
43
|
+
:param fuzzy_and_ignore_case: If True, performs a fuzzy + case-insensitive match when comparing the value of
|
|
44
|
+
this property using the `CONTAINS` operator.
|
|
45
|
+
query. Defaults to False. This only has effect as part of a TargetNodeMatcher and is not supported for the
|
|
46
|
+
sub resource relationship.
|
|
36
47
|
"""
|
|
37
48
|
self.name = name
|
|
38
49
|
self.set_in_kwargs = set_in_kwargs
|
|
39
50
|
self.extra_index = extra_index
|
|
40
51
|
self.ignore_case = ignore_case
|
|
52
|
+
self.fuzzy_and_ignore_case = fuzzy_and_ignore_case
|
|
53
|
+
if self.fuzzy_and_ignore_case and self.ignore_case:
|
|
54
|
+
raise ValueError(
|
|
55
|
+
f'Error setting PropertyRef "{self.name}": ignore_case cannot be used together with'
|
|
56
|
+
'fuzzy_and_ignore_case. Pick one or the other.',
|
|
57
|
+
)
|
|
41
58
|
|
|
42
59
|
def _parameterize_name(self) -> str:
|
|
43
60
|
return f"${self.name}"
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This schema does not handle the org's relationships. Those are handled by other schemas, for example:
|
|
3
|
+
* GitHubTeamSchema defines (GitHubOrganization)-[RESOURCE]->(GitHubTeam)
|
|
4
|
+
* GitHubUserSchema defines (GitHubUser)-[MEMBER_OF|UNAFFILIATED]->(GitHubOrganization)
|
|
5
|
+
(There may be others, these are just two examples.)
|
|
6
|
+
"""
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
from cartography.models.core.common import PropertyRef
|
|
10
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
11
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class GitHubOrganizationNodeProperties(CartographyNodeProperties):
|
|
16
|
+
id: PropertyRef = PropertyRef('url')
|
|
17
|
+
username: PropertyRef = PropertyRef('login', extra_index=True)
|
|
18
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class GitHubOrganizationSchema(CartographyNodeSchema):
|
|
23
|
+
label: str = 'GitHubOrganization'
|
|
24
|
+
properties: GitHubOrganizationNodeProperties = GitHubOrganizationNodeProperties()
|
|
25
|
+
other_relationships = None
|
|
26
|
+
sub_resource_relationship = None
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""
|
|
2
|
+
RE: Tenant relationship between GitHubUser and GitHubOrganization
|
|
3
|
+
|
|
4
|
+
Note this relationship is implemented via 'other_relationships' and not via the 'sub_resource_relationship'
|
|
5
|
+
as might be expected.
|
|
6
|
+
|
|
7
|
+
The 'sub_resource_relationship' typically describes the relationship of a node to its tenant (the org, project, or
|
|
8
|
+
other resource to which other nodes belong). An assumption of that relationship is that if the tenant goes
|
|
9
|
+
away, all nodes related to it should be cleaned up.
|
|
10
|
+
|
|
11
|
+
In GitHub, though the GitHubUser's tenant seems to be GitHubOrganization, users actually exist independently. There
|
|
12
|
+
is a concept of 'UNAFFILIATED' users (https://docs.github.com/en/graphql/reference/enums#roleinorganization) like
|
|
13
|
+
Enterprise Owners who are related to an org even if they are not direct members of it. You would not want them to be
|
|
14
|
+
cleaned up, if an org goes away, and you could want them in your graph even if they are not members of any org in
|
|
15
|
+
the enterprise.
|
|
16
|
+
|
|
17
|
+
To allow for this in the schema, this relationship is treated as any other node-to-node relationship, via
|
|
18
|
+
'other_relationships', instead of as the typical 'sub_resource_relationship'.
|
|
19
|
+
|
|
20
|
+
RE: GitHubOrganizationUserSchema vs GitHubUnaffiliatedUserSchema
|
|
21
|
+
|
|
22
|
+
As noted above, there are implicitly two types of users, those that are part of, or affiliated, to a target
|
|
23
|
+
GitHubOrganization, and those thare are not part, or unaffiliated. Both are represented as GitHubUser nodes,
|
|
24
|
+
but there are two schemas below to allow for some differences between them, e.g., unaffiliated lack these properties:
|
|
25
|
+
* the 'role' property, because unaffiliated have no 'role' in the target org
|
|
26
|
+
* the 'has_2fa_enabled' property, because the GitHub api does not return it, for these users
|
|
27
|
+
The main importance of having two schemas is to allow the two sets of users to be loaded separately. If we are loading
|
|
28
|
+
an unaffiliated user, but the user already exists in the graph (perhaps they are members of another GitHub orgs for
|
|
29
|
+
example), then loading the unaffiliated user will not blank out the 'role' and 'has_2fa_enabled' properties.
|
|
30
|
+
"""
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
|
|
33
|
+
from cartography.models.core.common import PropertyRef
|
|
34
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
35
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
36
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
37
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
38
|
+
from cartography.models.core.relationships import LinkDirection
|
|
39
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
40
|
+
from cartography.models.core.relationships import OtherRelationships
|
|
41
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class BaseGitHubUserNodeProperties(CartographyNodeProperties):
|
|
46
|
+
# core properties in all GitHubUser nodes
|
|
47
|
+
id: PropertyRef = PropertyRef('url')
|
|
48
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
49
|
+
fullname: PropertyRef = PropertyRef('name')
|
|
50
|
+
username: PropertyRef = PropertyRef('login', extra_index=True)
|
|
51
|
+
is_site_admin: PropertyRef = PropertyRef('isSiteAdmin')
|
|
52
|
+
is_enterprise_owner: PropertyRef = PropertyRef('isEnterpriseOwner')
|
|
53
|
+
email: PropertyRef = PropertyRef('email')
|
|
54
|
+
company: PropertyRef = PropertyRef('company')
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(frozen=True)
|
|
58
|
+
class GitHubOrganizationUserNodeProperties(BaseGitHubUserNodeProperties):
|
|
59
|
+
# specified for affiliated users only. The GitHub api does not return this property for unaffiliated users.
|
|
60
|
+
has_2fa_enabled: PropertyRef = PropertyRef('hasTwoFactorEnabled')
|
|
61
|
+
# specified for affiliated uers only. Unaffiliated users do not have a 'role' in the target organization.
|
|
62
|
+
role: PropertyRef = PropertyRef('role')
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class GitHubUnaffiliatedUserNodeProperties(BaseGitHubUserNodeProperties):
|
|
67
|
+
# No additional properties needed
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True)
|
|
72
|
+
class GitHubUserToOrganizationRelProperties(CartographyRelProperties):
|
|
73
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass(frozen=True)
|
|
77
|
+
class GitHubUserMemberOfOrganizationRel(CartographyRelSchema):
|
|
78
|
+
target_node_label: str = 'GitHubOrganization'
|
|
79
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
80
|
+
{'id': PropertyRef('MEMBER_OF')},
|
|
81
|
+
)
|
|
82
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
83
|
+
rel_label: str = "MEMBER_OF"
|
|
84
|
+
properties: GitHubUserToOrganizationRelProperties = GitHubUserToOrganizationRelProperties()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass(frozen=True)
|
|
88
|
+
class GitHubUserUnaffiliatedOrganizationRel(CartographyRelSchema):
|
|
89
|
+
target_node_label: str = 'GitHubOrganization'
|
|
90
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
91
|
+
{'id': PropertyRef('UNAFFILIATED')},
|
|
92
|
+
)
|
|
93
|
+
direction: LinkDirection = LinkDirection.OUTWARD
|
|
94
|
+
rel_label: str = "UNAFFILIATED"
|
|
95
|
+
properties: GitHubUserToOrganizationRelProperties = GitHubUserToOrganizationRelProperties()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass(frozen=True)
|
|
99
|
+
class GitHubOrganizationUserSchema(CartographyNodeSchema):
|
|
100
|
+
label: str = 'GitHubUser'
|
|
101
|
+
properties: GitHubOrganizationUserNodeProperties = GitHubOrganizationUserNodeProperties()
|
|
102
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
103
|
+
[
|
|
104
|
+
GitHubUserMemberOfOrganizationRel(),
|
|
105
|
+
],
|
|
106
|
+
)
|
|
107
|
+
sub_resource_relationship = None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclass(frozen=True)
|
|
111
|
+
class GitHubUnaffiliatedUserSchema(CartographyNodeSchema):
|
|
112
|
+
label: str = 'GitHubUser'
|
|
113
|
+
properties: GitHubUnaffiliatedUserNodeProperties = GitHubUnaffiliatedUserNodeProperties()
|
|
114
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
115
|
+
[
|
|
116
|
+
GitHubUserUnaffiliatedOrganizationRel(),
|
|
117
|
+
],
|
|
118
|
+
)
|
|
119
|
+
sub_resource_relationship = None
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from cartography.models.core.common import PropertyRef
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
6
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
7
|
+
from cartography.models.core.nodes import ExtraNodeLabels
|
|
8
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
9
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
10
|
+
from cartography.models.core.relationships import LinkDirection
|
|
11
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
12
|
+
from cartography.models.core.relationships import OtherRelationships
|
|
13
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class SemgrepDependencyNodeProperties(CartographyNodeProperties):
|
|
18
|
+
id: PropertyRef = PropertyRef('id')
|
|
19
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
20
|
+
name: PropertyRef = PropertyRef('name')
|
|
21
|
+
ecosystem: PropertyRef = PropertyRef('ecosystem')
|
|
22
|
+
version: PropertyRef = PropertyRef('version')
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class SemgrepDependencyToSemgrepDeploymentRelProperties(CartographyRelProperties):
|
|
27
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
# (:SemgrepDependency)<-[:RESOURCE]-(:SemgrepDeployment)
|
|
32
|
+
class SemgrepDependencyToSemgrepDeploymentSchema(CartographyRelSchema):
|
|
33
|
+
target_node_label: str = 'SemgrepDeployment'
|
|
34
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
35
|
+
{'id': PropertyRef('DEPLOYMENT_ID', set_in_kwargs=True)},
|
|
36
|
+
)
|
|
37
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
38
|
+
rel_label: str = "RESOURCE"
|
|
39
|
+
properties: SemgrepDependencyToSemgrepDeploymentRelProperties = SemgrepDependencyToSemgrepDeploymentRelProperties()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class SemgrepDependencyToGithubRepoRelProperties(CartographyRelProperties):
|
|
44
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
45
|
+
specifier: PropertyRef = PropertyRef('specifier')
|
|
46
|
+
transitivity: PropertyRef = PropertyRef('transitivity')
|
|
47
|
+
url: PropertyRef = PropertyRef('url')
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
# (:SemgrepDependency)<-[:REQUIRES]-(:GitHubRepository)
|
|
52
|
+
class SemgrepDependencyToGithubRepoRel(CartographyRelSchema):
|
|
53
|
+
target_node_label: str = 'GitHubRepository'
|
|
54
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
55
|
+
{'id': PropertyRef('repo_url')},
|
|
56
|
+
)
|
|
57
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
58
|
+
rel_label: str = "REQUIRES"
|
|
59
|
+
properties: SemgrepDependencyToGithubRepoRelProperties = SemgrepDependencyToGithubRepoRelProperties()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class SemgrepSCAFindngToDependencyRelProperties(CartographyRelProperties):
|
|
64
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class SemgrepGoLibrarySchema(CartographyNodeSchema):
|
|
69
|
+
label: str = 'GoLibrary'
|
|
70
|
+
extra_node_labels: Optional[ExtraNodeLabels] = ExtraNodeLabels(['Dependency', 'SemgrepDependency'])
|
|
71
|
+
properties: SemgrepDependencyNodeProperties = SemgrepDependencyNodeProperties()
|
|
72
|
+
sub_resource_relationship: SemgrepDependencyToSemgrepDeploymentSchema = SemgrepDependencyToSemgrepDeploymentSchema()
|
|
73
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
74
|
+
[
|
|
75
|
+
SemgrepDependencyToGithubRepoRel(),
|
|
76
|
+
],
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class SemgrepNpmLibrarySchema(CartographyNodeSchema):
|
|
82
|
+
label: str = 'NpmLibrary'
|
|
83
|
+
extra_node_labels: Optional[ExtraNodeLabels] = ExtraNodeLabels(['Dependency', 'SemgrepDependency'])
|
|
84
|
+
properties: SemgrepDependencyNodeProperties = SemgrepDependencyNodeProperties()
|
|
85
|
+
sub_resource_relationship: SemgrepDependencyToSemgrepDeploymentSchema = SemgrepDependencyToSemgrepDeploymentSchema()
|
|
86
|
+
other_relationships: OtherRelationships = OtherRelationships(
|
|
87
|
+
[
|
|
88
|
+
SemgrepDependencyToGithubRepoRel(),
|
|
89
|
+
],
|
|
90
|
+
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: cartography
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.96.0rc2
|
|
4
4
|
Summary: Explore assets and their relationships across your technical infrastructure.
|
|
5
5
|
Home-page: https://www.github.com/cartography-cncf/cartography
|
|
6
6
|
Maintainer: Cartography Contributors
|