cartography 0.117.0__py3-none-any.whl → 0.118.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/_version.py +2 -2
- cartography/cli.py +11 -0
- cartography/config.py +5 -0
- cartography/graph/job.py +6 -2
- cartography/graph/statement.py +4 -0
- cartography/intel/aws/__init__.py +1 -0
- cartography/intel/aws/apigateway.py +18 -5
- cartography/intel/aws/ec2/elastic_ip_addresses.py +3 -1
- cartography/intel/aws/ec2/internet_gateways.py +4 -2
- cartography/intel/aws/ec2/load_balancer_v2s.py +11 -5
- cartography/intel/aws/ec2/network_interfaces.py +4 -0
- cartography/intel/aws/ec2/reserved_instances.py +3 -1
- cartography/intel/aws/ec2/tgw.py +11 -5
- cartography/intel/aws/ec2/volumes.py +1 -1
- cartography/intel/aws/ecr.py +202 -26
- cartography/intel/aws/elasticsearch.py +13 -4
- cartography/intel/aws/identitycenter.py +93 -54
- cartography/intel/aws/inspector.py +26 -14
- cartography/intel/aws/permission_relationships.py +3 -3
- cartography/intel/aws/s3.py +26 -13
- cartography/intel/aws/ssm.py +3 -5
- cartography/intel/azure/compute.py +9 -4
- cartography/intel/azure/cosmosdb.py +31 -15
- cartography/intel/azure/sql.py +25 -12
- cartography/intel/azure/storage.py +19 -9
- cartography/intel/azure/subscription.py +3 -1
- cartography/intel/crowdstrike/spotlight.py +5 -2
- cartography/intel/entra/app_role_assignments.py +9 -2
- cartography/intel/gcp/__init__.py +26 -9
- cartography/intel/gcp/clients.py +8 -4
- cartography/intel/gcp/compute.py +39 -18
- cartography/intel/gcp/crm/folders.py +9 -3
- cartography/intel/gcp/crm/orgs.py +8 -3
- cartography/intel/gcp/crm/projects.py +14 -3
- cartography/intel/jamf/computers.py +7 -1
- cartography/intel/oci/iam.py +23 -9
- cartography/intel/oci/organizations.py +3 -1
- cartography/intel/oci/utils.py +28 -5
- cartography/intel/okta/awssaml.py +8 -7
- cartography/intel/pagerduty/escalation_policies.py +13 -6
- cartography/intel/pagerduty/schedules.py +9 -4
- cartography/intel/pagerduty/services.py +7 -3
- cartography/intel/pagerduty/teams.py +5 -2
- cartography/intel/pagerduty/users.py +3 -1
- cartography/intel/pagerduty/vendors.py +3 -1
- cartography/intel/trivy/__init__.py +109 -58
- cartography/models/aws/ec2/networkinterfaces.py +2 -0
- cartography/models/aws/ecr/image.py +8 -0
- cartography/models/aws/ecr/repository_image.py +1 -1
- cartography/sync.py +1 -1
- cartography/util.py +5 -1
- {cartography-0.117.0.dist-info → cartography-0.118.0.dist-info}/METADATA +3 -3
- {cartography-0.117.0.dist-info → cartography-0.118.0.dist-info}/RECORD +57 -57
- {cartography-0.117.0.dist-info → cartography-0.118.0.dist-info}/WHEEL +0 -0
- {cartography-0.117.0.dist-info → cartography-0.118.0.dist-info}/entry_points.txt +0 -0
- {cartography-0.117.0.dist-info → cartography-0.118.0.dist-info}/licenses/LICENSE +0 -0
- {cartography-0.117.0.dist-info → cartography-0.118.0.dist-info}/top_level.txt +0 -0
cartography/intel/gcp/compute.py
CHANGED
|
@@ -15,6 +15,7 @@ from googleapiclient.discovery import Resource
|
|
|
15
15
|
from googleapiclient.errors import HttpError
|
|
16
16
|
|
|
17
17
|
from cartography.client.core.tx import load
|
|
18
|
+
from cartography.client.core.tx import run_write_query
|
|
18
19
|
from cartography.graph.job import GraphJob
|
|
19
20
|
from cartography.models.gcp.compute.vpc import GCPVpcSchema
|
|
20
21
|
from cartography.util import run_cleanup_job
|
|
@@ -619,7 +620,8 @@ def load_gcp_instances(
|
|
|
619
620
|
SET r.lastupdated = $gcp_update_tag
|
|
620
621
|
"""
|
|
621
622
|
for instance in data:
|
|
622
|
-
|
|
623
|
+
run_write_query(
|
|
624
|
+
neo4j_session,
|
|
623
625
|
query,
|
|
624
626
|
ProjectId=instance["project_id"],
|
|
625
627
|
PartialUri=instance["partial_uri"],
|
|
@@ -714,7 +716,8 @@ def load_gcp_forwarding_rules(
|
|
|
714
716
|
network = fwd.get("network", None)
|
|
715
717
|
subnetwork = fwd.get("subnetwork", None)
|
|
716
718
|
|
|
717
|
-
|
|
719
|
+
run_write_query(
|
|
720
|
+
neo4j_session,
|
|
718
721
|
query,
|
|
719
722
|
PartialUri=fwd["partial_uri"],
|
|
720
723
|
IPAddress=fwd["ip_address"],
|
|
@@ -760,7 +763,8 @@ def _attach_fwd_rule_to_subnet(
|
|
|
760
763
|
SET p.lastupdated = $gcp_update_tag
|
|
761
764
|
"""
|
|
762
765
|
|
|
763
|
-
|
|
766
|
+
run_write_query(
|
|
767
|
+
neo4j_session,
|
|
764
768
|
query,
|
|
765
769
|
PartialUri=fwd["partial_uri"],
|
|
766
770
|
SubNetworkPartialUri=fwd.get("subnetwork_partial_uri", None),
|
|
@@ -787,7 +791,8 @@ def _attach_fwd_rule_to_vpc(
|
|
|
787
791
|
SET r.lastupdated = $gcp_update_tag
|
|
788
792
|
"""
|
|
789
793
|
|
|
790
|
-
|
|
794
|
+
run_write_query(
|
|
795
|
+
neo4j_session,
|
|
791
796
|
query,
|
|
792
797
|
PartialUri=fwd["partial_uri"],
|
|
793
798
|
NetworkPartialUri=fwd.get("network_partial_uri", None),
|
|
@@ -831,7 +836,8 @@ def _attach_instance_tags(
|
|
|
831
836
|
for tag in instance.get("tags", {}).get("items", []):
|
|
832
837
|
for nic in instance.get("networkInterfaces", []):
|
|
833
838
|
tag_id = _create_gcp_network_tag_id(nic["vpc_partial_uri"], tag)
|
|
834
|
-
|
|
839
|
+
run_write_query(
|
|
840
|
+
neo4j_session,
|
|
835
841
|
query,
|
|
836
842
|
InstanceId=instance["partial_uri"],
|
|
837
843
|
TagId=tag_id,
|
|
@@ -880,7 +886,8 @@ def _attach_gcp_nics(
|
|
|
880
886
|
for nic in instance.get("networkInterfaces", []):
|
|
881
887
|
# Make an ID for GCPNetworkInterface nodes because GCP doesn't define one but we need to uniquely identify them
|
|
882
888
|
nic_id = f"{instance['partial_uri']}/networkinterfaces/{nic['name']}"
|
|
883
|
-
|
|
889
|
+
run_write_query(
|
|
890
|
+
neo4j_session,
|
|
884
891
|
query,
|
|
885
892
|
InstanceId=instance["partial_uri"],
|
|
886
893
|
NicId=nic_id,
|
|
@@ -926,7 +933,8 @@ def _attach_gcp_nic_access_configs(
|
|
|
926
933
|
for ac in nic.get("accessConfigs", []):
|
|
927
934
|
# Make an ID for GCPNicAccessConfig nodes because GCP doesn't define one but we need to uniquely identify them
|
|
928
935
|
access_config_id = f"{nic_id}/accessconfigs/{ac['type']}"
|
|
929
|
-
|
|
936
|
+
run_write_query(
|
|
937
|
+
neo4j_session,
|
|
930
938
|
query,
|
|
931
939
|
NicId=nic_id,
|
|
932
940
|
AccessConfigId=access_config_id,
|
|
@@ -960,7 +968,8 @@ def _attach_gcp_vpc(
|
|
|
960
968
|
ON CREATE SET m.firstseen = timestamp()
|
|
961
969
|
SET m.lastupdated = $gcp_update_tag
|
|
962
970
|
"""
|
|
963
|
-
|
|
971
|
+
run_write_query(
|
|
972
|
+
neo4j_session,
|
|
964
973
|
query,
|
|
965
974
|
InstanceId=instance_id,
|
|
966
975
|
gcp_update_tag=gcp_update_tag,
|
|
@@ -974,10 +983,22 @@ def load_gcp_ingress_firewalls(
|
|
|
974
983
|
gcp_update_tag: int,
|
|
975
984
|
) -> None:
|
|
976
985
|
"""
|
|
977
|
-
Load the firewall list to Neo4j
|
|
986
|
+
Load the firewall list to Neo4j.
|
|
978
987
|
:param fw_list: The transformed list of firewalls
|
|
979
988
|
:return: Nothing
|
|
980
989
|
"""
|
|
990
|
+
neo4j_session.execute_write(
|
|
991
|
+
_load_gcp_ingress_firewalls_tx,
|
|
992
|
+
fw_list,
|
|
993
|
+
gcp_update_tag,
|
|
994
|
+
)
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
def _load_gcp_ingress_firewalls_tx(
|
|
998
|
+
tx: neo4j.Transaction,
|
|
999
|
+
fw_list: List[Resource],
|
|
1000
|
+
gcp_update_tag: int,
|
|
1001
|
+
) -> None:
|
|
981
1002
|
query = """
|
|
982
1003
|
MERGE (fw:GCPFirewall{id:$FwPartialUri})
|
|
983
1004
|
ON CREATE SET fw.firstseen = timestamp(),
|
|
@@ -1000,7 +1021,7 @@ def load_gcp_ingress_firewalls(
|
|
|
1000
1021
|
SET r.lastupdated = $gcp_update_tag
|
|
1001
1022
|
"""
|
|
1002
1023
|
for fw in fw_list:
|
|
1003
|
-
|
|
1024
|
+
tx.run(
|
|
1004
1025
|
query,
|
|
1005
1026
|
FwPartialUri=fw["id"],
|
|
1006
1027
|
Direction=fw["direction"],
|
|
@@ -1012,19 +1033,19 @@ def load_gcp_ingress_firewalls(
|
|
|
1012
1033
|
HasTargetServiceAccounts=fw["has_target_service_accounts"],
|
|
1013
1034
|
gcp_update_tag=gcp_update_tag,
|
|
1014
1035
|
)
|
|
1015
|
-
_attach_firewall_rules(
|
|
1016
|
-
_attach_target_tags(
|
|
1036
|
+
_attach_firewall_rules(tx, fw, gcp_update_tag)
|
|
1037
|
+
_attach_target_tags(tx, fw, gcp_update_tag)
|
|
1017
1038
|
|
|
1018
1039
|
|
|
1019
1040
|
@timeit
|
|
1020
1041
|
def _attach_firewall_rules(
|
|
1021
|
-
|
|
1042
|
+
tx: neo4j.Transaction,
|
|
1022
1043
|
fw: Resource,
|
|
1023
1044
|
gcp_update_tag: int,
|
|
1024
1045
|
) -> None:
|
|
1025
1046
|
"""
|
|
1026
1047
|
Attach the allow_rules to the Firewall object
|
|
1027
|
-
:param
|
|
1048
|
+
:param tx: The Neo4j transaction
|
|
1028
1049
|
:param fw: The Firewall object
|
|
1029
1050
|
:param gcp_update_tag: The timestamp
|
|
1030
1051
|
:return: Nothing
|
|
@@ -1065,7 +1086,7 @@ def _attach_firewall_rules(
|
|
|
1065
1086
|
# If sourceRanges is not specified then the rule must specify sourceTags.
|
|
1066
1087
|
# Since an IP range cannot have a tag applied to it, it is ok if we don't ingest this rule.
|
|
1067
1088
|
for ip_range in fw.get("sourceRanges", []):
|
|
1068
|
-
|
|
1089
|
+
tx.run(
|
|
1069
1090
|
template.safe_substitute(fw_rule_relationship_label=label),
|
|
1070
1091
|
FwPartialUri=fw["id"],
|
|
1071
1092
|
RuleId=rule["ruleid"],
|
|
@@ -1079,13 +1100,13 @@ def _attach_firewall_rules(
|
|
|
1079
1100
|
|
|
1080
1101
|
@timeit
|
|
1081
1102
|
def _attach_target_tags(
|
|
1082
|
-
|
|
1103
|
+
tx: neo4j.Transaction,
|
|
1083
1104
|
fw: Resource,
|
|
1084
1105
|
gcp_update_tag: int,
|
|
1085
1106
|
) -> None:
|
|
1086
1107
|
"""
|
|
1087
1108
|
Attach target tags to the firewall object
|
|
1088
|
-
:param
|
|
1109
|
+
:param tx: The neo4j transaction
|
|
1089
1110
|
:param fw: The firewall object
|
|
1090
1111
|
:param gcp_update_tag: The timestamp
|
|
1091
1112
|
:return: Nothing
|
|
@@ -1105,7 +1126,7 @@ def _attach_target_tags(
|
|
|
1105
1126
|
"""
|
|
1106
1127
|
for tag in fw.get("targetTags", []):
|
|
1107
1128
|
tag_id = _create_gcp_network_tag_id(fw["vpc_partial_uri"], tag)
|
|
1108
|
-
|
|
1129
|
+
tx.run(
|
|
1109
1130
|
query,
|
|
1110
1131
|
FwPartialUri=fw["id"],
|
|
1111
1132
|
TagId=tag_id,
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from typing import Dict
|
|
3
3
|
from typing import List
|
|
4
|
+
from typing import Optional
|
|
4
5
|
|
|
5
6
|
import neo4j
|
|
7
|
+
from google.auth.credentials import Credentials as GoogleCredentials
|
|
6
8
|
from google.cloud import resourcemanager_v3
|
|
7
9
|
|
|
8
10
|
from cartography.client.core.tx import load
|
|
@@ -13,7 +15,10 @@ logger = logging.getLogger(__name__)
|
|
|
13
15
|
|
|
14
16
|
|
|
15
17
|
@timeit
|
|
16
|
-
def get_gcp_folders(
|
|
18
|
+
def get_gcp_folders(
|
|
19
|
+
org_resource_name: str,
|
|
20
|
+
credentials: Optional[GoogleCredentials] = None,
|
|
21
|
+
) -> List[Dict]:
|
|
17
22
|
"""
|
|
18
23
|
Return a list of all descendant GCP folders under the specified organization by traversing the folder tree.
|
|
19
24
|
|
|
@@ -21,7 +26,7 @@ def get_gcp_folders(org_resource_name: str) -> List[Dict]:
|
|
|
21
26
|
:return: List of folder dicts with 'name' field containing full resource names (e.g., "folders/123456")
|
|
22
27
|
"""
|
|
23
28
|
results: List[Dict] = []
|
|
24
|
-
client = resourcemanager_v3.FoldersClient()
|
|
29
|
+
client = resourcemanager_v3.FoldersClient(credentials=credentials)
|
|
25
30
|
# BFS over folders starting at the org root
|
|
26
31
|
queue: List[str] = [org_resource_name]
|
|
27
32
|
seen: set[str] = set()
|
|
@@ -96,6 +101,7 @@ def sync_gcp_folders(
|
|
|
96
101
|
gcp_update_tag: int,
|
|
97
102
|
common_job_parameters: Dict,
|
|
98
103
|
org_resource_name: str,
|
|
104
|
+
credentials: Optional[GoogleCredentials] = None,
|
|
99
105
|
) -> List[Dict]:
|
|
100
106
|
"""
|
|
101
107
|
Get GCP folder data using the CRM v2 resource object and load the data to Neo4j.
|
|
@@ -103,6 +109,6 @@ def sync_gcp_folders(
|
|
|
103
109
|
:return: List of folders synced
|
|
104
110
|
"""
|
|
105
111
|
logger.debug("Syncing GCP folders")
|
|
106
|
-
folders = get_gcp_folders(org_resource_name)
|
|
112
|
+
folders = get_gcp_folders(org_resource_name, credentials=credentials)
|
|
107
113
|
load_gcp_folders(neo4j_session, folders, gcp_update_tag, org_resource_name)
|
|
108
114
|
return folders
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from typing import Dict
|
|
3
3
|
from typing import List
|
|
4
|
+
from typing import Optional
|
|
4
5
|
|
|
5
6
|
import neo4j
|
|
7
|
+
from google.auth.credentials import Credentials as GoogleCredentials
|
|
6
8
|
from google.cloud import resourcemanager_v3
|
|
7
9
|
|
|
8
10
|
from cartography.client.core.tx import load
|
|
@@ -13,13 +15,15 @@ logger = logging.getLogger(__name__)
|
|
|
13
15
|
|
|
14
16
|
|
|
15
17
|
@timeit
|
|
16
|
-
def get_gcp_organizations(
|
|
18
|
+
def get_gcp_organizations(
|
|
19
|
+
credentials: Optional[GoogleCredentials] = None,
|
|
20
|
+
) -> List[Dict]:
|
|
17
21
|
"""
|
|
18
22
|
Return list of GCP organizations that the authenticated principal can access using the high-level client.
|
|
19
23
|
Returns empty list on error.
|
|
20
24
|
:return: List of org dicts with keys: name, displayName, lifecycleState.
|
|
21
25
|
"""
|
|
22
|
-
client = resourcemanager_v3.OrganizationsClient()
|
|
26
|
+
client = resourcemanager_v3.OrganizationsClient(credentials=credentials)
|
|
23
27
|
orgs = []
|
|
24
28
|
for org in client.search_organizations():
|
|
25
29
|
orgs.append(
|
|
@@ -54,12 +58,13 @@ def sync_gcp_organizations(
|
|
|
54
58
|
neo4j_session: neo4j.Session,
|
|
55
59
|
gcp_update_tag: int,
|
|
56
60
|
common_job_parameters: Dict,
|
|
61
|
+
credentials: Optional[GoogleCredentials] = None,
|
|
57
62
|
) -> List[Dict]:
|
|
58
63
|
"""
|
|
59
64
|
Get GCP organization data using the CRM v1 resource object and load the data to Neo4j.
|
|
60
65
|
Returns the list of organizations synced.
|
|
61
66
|
"""
|
|
62
67
|
logger.debug("Syncing GCP organizations")
|
|
63
|
-
data = get_gcp_organizations()
|
|
68
|
+
data = get_gcp_organizations(credentials=credentials)
|
|
64
69
|
load_gcp_organizations(neo4j_session, data, gcp_update_tag)
|
|
65
70
|
return data
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from typing import Dict
|
|
3
3
|
from typing import List
|
|
4
|
+
from typing import Optional
|
|
4
5
|
|
|
5
6
|
import neo4j
|
|
7
|
+
from google.auth.credentials import Credentials as GoogleCredentials
|
|
6
8
|
from google.cloud import resourcemanager_v3
|
|
7
9
|
|
|
8
10
|
from cartography.client.core.tx import load
|
|
@@ -13,7 +15,11 @@ logger = logging.getLogger(__name__)
|
|
|
13
15
|
|
|
14
16
|
|
|
15
17
|
@timeit
|
|
16
|
-
def get_gcp_projects(
|
|
18
|
+
def get_gcp_projects(
|
|
19
|
+
org_resource_name: str,
|
|
20
|
+
folders: List[Dict],
|
|
21
|
+
credentials: Optional[GoogleCredentials] = None,
|
|
22
|
+
) -> List[Dict]:
|
|
17
23
|
"""
|
|
18
24
|
Return list of ACTIVE GCP projects under the specified organization
|
|
19
25
|
and within the specified folders.
|
|
@@ -25,7 +31,7 @@ def get_gcp_projects(org_resource_name: str, folders: List[Dict]) -> List[Dict]:
|
|
|
25
31
|
parents = set([org_resource_name] + folder_names)
|
|
26
32
|
results: List[Dict] = []
|
|
27
33
|
for parent in parents:
|
|
28
|
-
client = resourcemanager_v3.ProjectsClient()
|
|
34
|
+
client = resourcemanager_v3.ProjectsClient(credentials=credentials)
|
|
29
35
|
for proj in client.list_projects(parent=parent):
|
|
30
36
|
# list_projects returns ACTIVE projects by default
|
|
31
37
|
name_field = proj.name # "projects/<number>"
|
|
@@ -96,6 +102,7 @@ def sync_gcp_projects(
|
|
|
96
102
|
folders: List[Dict],
|
|
97
103
|
gcp_update_tag: int,
|
|
98
104
|
common_job_parameters: Dict,
|
|
105
|
+
credentials: Optional[GoogleCredentials] = None,
|
|
99
106
|
) -> List[Dict]:
|
|
100
107
|
"""
|
|
101
108
|
Get and sync GCP project data to Neo4j.
|
|
@@ -104,6 +111,10 @@ def sync_gcp_projects(
|
|
|
104
111
|
:return: List of projects synced
|
|
105
112
|
"""
|
|
106
113
|
logger.debug("Syncing GCP projects")
|
|
107
|
-
projects = get_gcp_projects(
|
|
114
|
+
projects = get_gcp_projects(
|
|
115
|
+
org_resource_name,
|
|
116
|
+
folders,
|
|
117
|
+
credentials=credentials,
|
|
118
|
+
)
|
|
108
119
|
load_gcp_projects(neo4j_session, projects, gcp_update_tag, org_resource_name)
|
|
109
120
|
return projects
|
|
@@ -4,6 +4,7 @@ from typing import List
|
|
|
4
4
|
|
|
5
5
|
import neo4j
|
|
6
6
|
|
|
7
|
+
from cartography.client.core.tx import run_write_query
|
|
7
8
|
from cartography.intel.jamf.util import call_jamf_api
|
|
8
9
|
from cartography.util import run_cleanup_job
|
|
9
10
|
from cartography.util import timeit
|
|
@@ -35,7 +36,12 @@ def load_computer_groups(
|
|
|
35
36
|
g.lastupdated = $UpdateTag
|
|
36
37
|
"""
|
|
37
38
|
groups = data.get("computer_groups")
|
|
38
|
-
|
|
39
|
+
run_write_query(
|
|
40
|
+
neo4j_session,
|
|
41
|
+
ingest_groups,
|
|
42
|
+
JsonData=groups,
|
|
43
|
+
UpdateTag=update_tag,
|
|
44
|
+
)
|
|
39
45
|
|
|
40
46
|
|
|
41
47
|
@timeit
|
cartography/intel/oci/iam.py
CHANGED
|
@@ -10,6 +10,8 @@ from typing import List
|
|
|
10
10
|
import neo4j
|
|
11
11
|
import oci
|
|
12
12
|
|
|
13
|
+
from cartography.client.core.tx import read_list_of_dicts_tx
|
|
14
|
+
from cartography.client.core.tx import run_write_query
|
|
13
15
|
from cartography.util import run_cleanup_job
|
|
14
16
|
|
|
15
17
|
from . import utils
|
|
@@ -89,7 +91,8 @@ def load_compartments(
|
|
|
89
91
|
"""
|
|
90
92
|
|
|
91
93
|
for compartment in compartments:
|
|
92
|
-
|
|
94
|
+
run_write_query(
|
|
95
|
+
neo4j_session,
|
|
93
96
|
ingest_compartment,
|
|
94
97
|
OCID=compartment["id"],
|
|
95
98
|
COMPARTMENT_ID=compartment["compartment-id"],
|
|
@@ -126,7 +129,8 @@ def load_users(
|
|
|
126
129
|
"""
|
|
127
130
|
|
|
128
131
|
for user in users:
|
|
129
|
-
|
|
132
|
+
run_write_query(
|
|
133
|
+
neo4j_session,
|
|
130
134
|
ingest_user,
|
|
131
135
|
OCID=user["id"],
|
|
132
136
|
CREATE_DATE=str(user["time-created"]),
|
|
@@ -206,7 +210,8 @@ def load_groups(
|
|
|
206
210
|
"""
|
|
207
211
|
|
|
208
212
|
for group in groups:
|
|
209
|
-
|
|
213
|
+
run_write_query(
|
|
214
|
+
neo4j_session,
|
|
210
215
|
ingest_group,
|
|
211
216
|
OCID=group["id"],
|
|
212
217
|
CREATE_DATE=str(group["time-created"]),
|
|
@@ -260,7 +265,11 @@ def sync_group_memberships(
|
|
|
260
265
|
"MATCH (group:OCIGroup)<-[:RESOURCE]-(OCITenancy{ocid: $OCI_TENANCY_ID}) "
|
|
261
266
|
"return group.name as name, group.ocid as ocid;"
|
|
262
267
|
)
|
|
263
|
-
groups = neo4j_session.
|
|
268
|
+
groups = neo4j_session.execute_read(
|
|
269
|
+
read_list_of_dicts_tx,
|
|
270
|
+
query,
|
|
271
|
+
OCI_TENANCY_ID=current_tenancy_id,
|
|
272
|
+
)
|
|
264
273
|
groups_membership = {
|
|
265
274
|
group["ocid"]: get_group_membership_data(iam, group["ocid"], current_tenancy_id)
|
|
266
275
|
for group in groups
|
|
@@ -288,7 +297,8 @@ def load_group_memberships(
|
|
|
288
297
|
"""
|
|
289
298
|
for group_ocid, membership_data in group_memberships.items():
|
|
290
299
|
for info in membership_data["GroupMemberships"]:
|
|
291
|
-
|
|
300
|
+
run_write_query(
|
|
301
|
+
neo4j_session,
|
|
292
302
|
ingest_membership,
|
|
293
303
|
COMPARTMENT_ID=info["compartment-id"],
|
|
294
304
|
GROUP_OCID=info["group-id"],
|
|
@@ -317,7 +327,8 @@ def load_policies(
|
|
|
317
327
|
"""
|
|
318
328
|
|
|
319
329
|
for policy in policies:
|
|
320
|
-
|
|
330
|
+
run_write_query(
|
|
331
|
+
neo4j_session,
|
|
321
332
|
ingest_policy,
|
|
322
333
|
OCID=policy["id"],
|
|
323
334
|
POLICY_NAME=policy["name"],
|
|
@@ -386,7 +397,8 @@ def load_oci_policy_group_reference(
|
|
|
386
397
|
ON CREATE SET r.firstseen = timestamp()
|
|
387
398
|
SET r.lastupdated = $oci_update_tag
|
|
388
399
|
"""
|
|
389
|
-
|
|
400
|
+
run_write_query(
|
|
401
|
+
neo4j_session,
|
|
390
402
|
ingest_policy_group_reference,
|
|
391
403
|
POLICY_ID=policy_id,
|
|
392
404
|
GROUP_ID=group_id,
|
|
@@ -408,7 +420,8 @@ def load_oci_policy_compartment_reference(
|
|
|
408
420
|
ON CREATE SET r.firstseen = timestamp()
|
|
409
421
|
SET r.lastupdated = $oci_update_tag
|
|
410
422
|
"""
|
|
411
|
-
|
|
423
|
+
run_write_query(
|
|
424
|
+
neo4j_session,
|
|
412
425
|
ingest_policy_compartment_reference,
|
|
413
426
|
POLICY_ID=policy_id,
|
|
414
427
|
COMPARTMENT_ID=compartment_id,
|
|
@@ -487,7 +500,8 @@ def load_region_subscriptions(
|
|
|
487
500
|
SET r.lastupdated = $oci_update_tag
|
|
488
501
|
"""
|
|
489
502
|
for region in regions:
|
|
490
|
-
|
|
503
|
+
run_write_query(
|
|
504
|
+
neo4j_session,
|
|
491
505
|
query,
|
|
492
506
|
REGION_KEY=region["region-key"],
|
|
493
507
|
REGION_NAME=region["region-name"],
|
|
@@ -11,6 +11,7 @@ from oci.exceptions import ConfigFileNotFound
|
|
|
11
11
|
from oci.exceptions import InvalidConfig
|
|
12
12
|
from oci.exceptions import ProfileNotFound
|
|
13
13
|
|
|
14
|
+
from cartography.client.core.tx import run_write_query
|
|
14
15
|
from cartography.util import run_cleanup_job
|
|
15
16
|
|
|
16
17
|
logger = logging.getLogger(__name__)
|
|
@@ -100,7 +101,8 @@ def load_oci_accounts(
|
|
|
100
101
|
SET aa.lastupdated = $oci_update_tag, aa.name = $ACCOUNT_NAME
|
|
101
102
|
"""
|
|
102
103
|
for name in oci_accounts:
|
|
103
|
-
|
|
104
|
+
run_write_query(
|
|
105
|
+
neo4j_session,
|
|
104
106
|
query,
|
|
105
107
|
TENANCY_ID=oci_accounts[name]["tenancy"],
|
|
106
108
|
ACCOUNT_NAME=name,
|
cartography/intel/oci/utils.py
CHANGED
|
@@ -7,6 +7,8 @@ from typing import List
|
|
|
7
7
|
|
|
8
8
|
import neo4j
|
|
9
9
|
|
|
10
|
+
from cartography.client.core.tx import read_list_of_dicts_tx
|
|
11
|
+
|
|
10
12
|
|
|
11
13
|
# Generic way to turn a OCI python object into the json response that you would see from calling the REST API.
|
|
12
14
|
def oci_object_to_json(in_obj: Any) -> List[Dict[str, Any]]:
|
|
@@ -36,7 +38,11 @@ def get_compartments_in_tenancy(
|
|
|
36
38
|
"return DISTINCT compartment.name as name, compartment.ocid as ocid, "
|
|
37
39
|
"compartment.compartmentid as compartmentid;"
|
|
38
40
|
)
|
|
39
|
-
return neo4j_session.
|
|
41
|
+
return neo4j_session.execute_read(
|
|
42
|
+
read_list_of_dicts_tx,
|
|
43
|
+
query,
|
|
44
|
+
OCI_TENANCY_ID=tenancy_id,
|
|
45
|
+
)
|
|
40
46
|
|
|
41
47
|
|
|
42
48
|
# Grab list of all groups in neo4j already populated by iam.
|
|
@@ -48,7 +54,11 @@ def get_groups_in_tenancy(
|
|
|
48
54
|
"MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(group:OCIGroup)"
|
|
49
55
|
"return DISTINCT group.name as name, group.ocid as ocid;"
|
|
50
56
|
)
|
|
51
|
-
return neo4j_session.
|
|
57
|
+
return neo4j_session.execute_read(
|
|
58
|
+
read_list_of_dicts_tx,
|
|
59
|
+
query,
|
|
60
|
+
OCI_TENANCY_ID=tenancy_id,
|
|
61
|
+
)
|
|
52
62
|
|
|
53
63
|
|
|
54
64
|
# Grab list of all policies in neo4j already populated by iam.
|
|
@@ -61,7 +71,11 @@ def get_policies_in_tenancy(
|
|
|
61
71
|
"return DISTINCT policy.name as name, policy.ocid as ocid, policy.statements as statements, "
|
|
62
72
|
"policy.compartmentid as compartmentid;"
|
|
63
73
|
)
|
|
64
|
-
return neo4j_session.
|
|
74
|
+
return neo4j_session.execute_read(
|
|
75
|
+
read_list_of_dicts_tx,
|
|
76
|
+
query,
|
|
77
|
+
OCI_TENANCY_ID=tenancy_id,
|
|
78
|
+
)
|
|
65
79
|
|
|
66
80
|
|
|
67
81
|
# Grab list of all regions in neo4j already populated by iam.
|
|
@@ -73,7 +87,11 @@ def get_regions_in_tenancy(
|
|
|
73
87
|
"MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-->(region:OCIRegion)"
|
|
74
88
|
"return DISTINCT region.name as name, region.key as key;"
|
|
75
89
|
)
|
|
76
|
-
return neo4j_session.
|
|
90
|
+
return neo4j_session.execute_read(
|
|
91
|
+
read_list_of_dicts_tx,
|
|
92
|
+
query,
|
|
93
|
+
OCI_TENANCY_ID=tenancy_id,
|
|
94
|
+
)
|
|
77
95
|
|
|
78
96
|
|
|
79
97
|
# Grab list of all security groups in neo4j already populated by network. Need to handle regions for this one.
|
|
@@ -88,4 +106,9 @@ def get_security_groups_in_tenancy(
|
|
|
88
106
|
"return DISTINCT security_group.name as name, security_group.ocid as ocid, security_group.compartmentid "
|
|
89
107
|
"as compartmentid;"
|
|
90
108
|
)
|
|
91
|
-
return neo4j_session.
|
|
109
|
+
return neo4j_session.execute_read(
|
|
110
|
+
read_list_of_dicts_tx,
|
|
111
|
+
query,
|
|
112
|
+
OCI_TENANCY_ID=tenancy_id,
|
|
113
|
+
OCI_REGION=region,
|
|
114
|
+
)
|
|
@@ -68,24 +68,25 @@ def query_for_okta_to_aws_role_mapping(
|
|
|
68
68
|
:param neo4j_session: session from the Neo4j server
|
|
69
69
|
:param mapping_regex: the regex used by the organization to map groups to aws roles
|
|
70
70
|
"""
|
|
71
|
-
query =
|
|
71
|
+
query = (
|
|
72
|
+
"MATCH (app:OktaApplication{name:'amazon_aws'})--(group:OktaGroup) "
|
|
73
|
+
"RETURN group.id AS group_id, group.name AS group_name"
|
|
74
|
+
)
|
|
72
75
|
|
|
73
76
|
group_to_role_mapping: List[Dict] = []
|
|
74
|
-
|
|
75
|
-
results = neo4j_session.run(query)
|
|
77
|
+
results = neo4j_session.execute_read(read_list_of_dicts_tx, query)
|
|
76
78
|
|
|
77
79
|
for res in results:
|
|
78
|
-
has_results = True
|
|
79
80
|
# input: okta group id, okta group name. output: aws role arn.
|
|
80
81
|
mapping = transform_okta_group_to_aws_role(
|
|
81
|
-
res["
|
|
82
|
-
res["
|
|
82
|
+
res["group_id"],
|
|
83
|
+
res["group_name"],
|
|
83
84
|
mapping_regex,
|
|
84
85
|
)
|
|
85
86
|
if mapping:
|
|
86
87
|
group_to_role_mapping.append(mapping)
|
|
87
88
|
|
|
88
|
-
if
|
|
89
|
+
if results and not group_to_role_mapping:
|
|
89
90
|
logger.warning(
|
|
90
91
|
"AWS Okta Application present, but no mappings were found. "
|
|
91
92
|
"Please verify the mapping regex is correct",
|
|
@@ -6,6 +6,7 @@ from typing import List
|
|
|
6
6
|
import neo4j
|
|
7
7
|
from pdpyras import APISession
|
|
8
8
|
|
|
9
|
+
from cartography.client.core.tx import run_write_query
|
|
9
10
|
from cartography.util import timeit
|
|
10
11
|
|
|
11
12
|
logger = logging.getLogger(__name__)
|
|
@@ -72,7 +73,8 @@ def load_escalation_policy_data(
|
|
|
72
73
|
for team in policy["teams"]:
|
|
73
74
|
teams.append({"escalation_policy": policy["id"], "team": team["id"]})
|
|
74
75
|
|
|
75
|
-
|
|
76
|
+
run_write_query(
|
|
77
|
+
neo4j_session,
|
|
76
78
|
ingestion_cypher_query,
|
|
77
79
|
EscalationPolicies=data,
|
|
78
80
|
update_tag=update_tag,
|
|
@@ -115,7 +117,8 @@ def _attach_rules(
|
|
|
115
117
|
elif target["type"] == "schedule":
|
|
116
118
|
schedules.append({"rule": rule["id"], "schedule": target["id"]})
|
|
117
119
|
|
|
118
|
-
|
|
120
|
+
run_write_query(
|
|
121
|
+
neo4j_session,
|
|
119
122
|
ingestion_cypher_query,
|
|
120
123
|
Rules=data,
|
|
121
124
|
update_tag=update_tag,
|
|
@@ -140,7 +143,8 @@ def _attach_user_targets(
|
|
|
140
143
|
MERGE (p)-[r:ASSOCIATED_WITH]->(u)
|
|
141
144
|
ON CREATE SET r.firstseen = timestamp()
|
|
142
145
|
"""
|
|
143
|
-
|
|
146
|
+
run_write_query(
|
|
147
|
+
neo4j_session,
|
|
144
148
|
ingestion_cypher_query,
|
|
145
149
|
Relations=data,
|
|
146
150
|
update_tag=update_tag,
|
|
@@ -162,7 +166,8 @@ def _attach_schedule_targets(
|
|
|
162
166
|
MERGE (p)-[r:ASSOCIATED_WITH]->(s)
|
|
163
167
|
ON CREATE SET r.firstseen = timestamp()
|
|
164
168
|
"""
|
|
165
|
-
|
|
169
|
+
run_write_query(
|
|
170
|
+
neo4j_session,
|
|
166
171
|
ingestion_cypher_query,
|
|
167
172
|
Relations=data,
|
|
168
173
|
update_tag=update_tag,
|
|
@@ -184,7 +189,8 @@ def _attach_services(
|
|
|
184
189
|
MERGE (s)-[r:ASSOCIATED_WITH]->(p)
|
|
185
190
|
ON CREATE SET r.firstseen = timestamp()
|
|
186
191
|
"""
|
|
187
|
-
|
|
192
|
+
run_write_query(
|
|
193
|
+
neo4j_session,
|
|
188
194
|
ingestion_cypher_query,
|
|
189
195
|
Relations=data,
|
|
190
196
|
update_tag=update_tag,
|
|
@@ -206,7 +212,8 @@ def _attach_teams(
|
|
|
206
212
|
MERGE (t)-[r:ASSOCIATED_WITH]->(p)
|
|
207
213
|
ON CREATE SET r.firstseen = timestamp()
|
|
208
214
|
"""
|
|
209
|
-
|
|
215
|
+
run_write_query(
|
|
216
|
+
neo4j_session,
|
|
210
217
|
ingestion_cypher_query,
|
|
211
218
|
Relations=data,
|
|
212
219
|
update_tag=update_tag,
|
|
@@ -7,6 +7,7 @@ import dateutil.parser
|
|
|
7
7
|
import neo4j
|
|
8
8
|
from pdpyras import APISession
|
|
9
9
|
|
|
10
|
+
from cartography.client.core.tx import run_write_query
|
|
10
11
|
from cartography.util import timeit
|
|
11
12
|
|
|
12
13
|
logger = logging.getLogger(__name__)
|
|
@@ -63,7 +64,8 @@ def load_schedule_data(
|
|
|
63
64
|
layer["_schedule_id"] = schedule["id"]
|
|
64
65
|
layers.append(layer)
|
|
65
66
|
|
|
66
|
-
|
|
67
|
+
run_write_query(
|
|
68
|
+
neo4j_session,
|
|
67
69
|
ingestion_cypher_query,
|
|
68
70
|
Schedules=data,
|
|
69
71
|
update_tag=update_tag,
|
|
@@ -87,7 +89,8 @@ def _attach_users(
|
|
|
87
89
|
MERGE (u)-[r:MEMBER_OF]->(s)
|
|
88
90
|
ON CREATE SET r.firstseen = timestamp()
|
|
89
91
|
"""
|
|
90
|
-
|
|
92
|
+
run_write_query(
|
|
93
|
+
neo4j_session,
|
|
91
94
|
ingestion_cypher_query,
|
|
92
95
|
Relations=data,
|
|
93
96
|
update_tag=update_tag,
|
|
@@ -129,7 +132,8 @@ def _attach_layers(
|
|
|
129
132
|
users.append(
|
|
130
133
|
{"layer_id": layer["_layer_id"], "user": user["user"]["id"]},
|
|
131
134
|
)
|
|
132
|
-
|
|
135
|
+
run_write_query(
|
|
136
|
+
neo4j_session,
|
|
133
137
|
ingestion_cypher_query,
|
|
134
138
|
Layers=data,
|
|
135
139
|
update_tag=update_tag,
|
|
@@ -152,7 +156,8 @@ def _attach_layer_users(
|
|
|
152
156
|
MERGE (u)-[r:MEMBER_OF]->(l)
|
|
153
157
|
ON CREATE SET r.firstseen = timestamp()
|
|
154
158
|
"""
|
|
155
|
-
|
|
159
|
+
run_write_query(
|
|
160
|
+
neo4j_session,
|
|
156
161
|
ingestion_cypher_query,
|
|
157
162
|
Relations=data,
|
|
158
163
|
update_tag=update_tag,
|