cartography 0.117.0__py3-none-any.whl → 0.119.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cartography might be problematic. Click here for more details.

Files changed (107) hide show
  1. cartography/_version.py +2 -2
  2. cartography/cli.py +31 -0
  3. cartography/client/core/tx.py +19 -3
  4. cartography/config.py +14 -0
  5. cartography/data/indexes.cypher +0 -6
  6. cartography/graph/job.py +13 -7
  7. cartography/graph/statement.py +4 -0
  8. cartography/intel/aws/__init__.py +22 -9
  9. cartography/intel/aws/apigateway.py +18 -5
  10. cartography/intel/aws/ec2/elastic_ip_addresses.py +3 -1
  11. cartography/intel/aws/ec2/internet_gateways.py +4 -2
  12. cartography/intel/aws/ec2/load_balancer_v2s.py +11 -5
  13. cartography/intel/aws/ec2/network_interfaces.py +4 -0
  14. cartography/intel/aws/ec2/reserved_instances.py +3 -1
  15. cartography/intel/aws/ec2/tgw.py +11 -5
  16. cartography/intel/aws/ec2/volumes.py +1 -1
  17. cartography/intel/aws/ecr.py +209 -26
  18. cartography/intel/aws/ecr_image_layers.py +143 -42
  19. cartography/intel/aws/elasticsearch.py +13 -4
  20. cartography/intel/aws/identitycenter.py +93 -54
  21. cartography/intel/aws/inspector.py +90 -46
  22. cartography/intel/aws/permission_relationships.py +3 -3
  23. cartography/intel/aws/resourcegroupstaggingapi.py +1 -1
  24. cartography/intel/aws/s3.py +26 -13
  25. cartography/intel/aws/ssm.py +3 -5
  26. cartography/intel/azure/compute.py +9 -4
  27. cartography/intel/azure/cosmosdb.py +31 -15
  28. cartography/intel/azure/sql.py +25 -12
  29. cartography/intel/azure/storage.py +19 -9
  30. cartography/intel/azure/subscription.py +3 -1
  31. cartography/intel/crowdstrike/spotlight.py +5 -2
  32. cartography/intel/entra/app_role_assignments.py +9 -2
  33. cartography/intel/gcp/__init__.py +26 -9
  34. cartography/intel/gcp/clients.py +8 -4
  35. cartography/intel/gcp/compute.py +42 -21
  36. cartography/intel/gcp/crm/folders.py +9 -3
  37. cartography/intel/gcp/crm/orgs.py +8 -3
  38. cartography/intel/gcp/crm/projects.py +14 -3
  39. cartography/intel/github/repos.py +23 -5
  40. cartography/intel/gsuite/__init__.py +12 -8
  41. cartography/intel/gsuite/groups.py +291 -0
  42. cartography/intel/gsuite/users.py +142 -0
  43. cartography/intel/jamf/computers.py +7 -1
  44. cartography/intel/oci/iam.py +23 -9
  45. cartography/intel/oci/organizations.py +3 -1
  46. cartography/intel/oci/utils.py +28 -5
  47. cartography/intel/okta/awssaml.py +9 -8
  48. cartography/intel/okta/users.py +1 -1
  49. cartography/intel/ontology/__init__.py +44 -0
  50. cartography/intel/ontology/devices.py +54 -0
  51. cartography/intel/ontology/users.py +54 -0
  52. cartography/intel/ontology/utils.py +121 -0
  53. cartography/intel/pagerduty/escalation_policies.py +13 -6
  54. cartography/intel/pagerduty/schedules.py +9 -4
  55. cartography/intel/pagerduty/services.py +7 -3
  56. cartography/intel/pagerduty/teams.py +5 -2
  57. cartography/intel/pagerduty/users.py +3 -1
  58. cartography/intel/pagerduty/vendors.py +3 -1
  59. cartography/intel/trivy/__init__.py +109 -58
  60. cartography/models/airbyte/user.py +4 -0
  61. cartography/models/anthropic/user.py +4 -0
  62. cartography/models/aws/ec2/networkinterfaces.py +2 -0
  63. cartography/models/aws/ecr/image.py +55 -0
  64. cartography/models/aws/ecr/repository_image.py +1 -1
  65. cartography/models/aws/iam/group_membership.py +3 -2
  66. cartography/models/aws/identitycenter/awsssouser.py +3 -1
  67. cartography/models/bigfix/bigfix_computer.py +1 -1
  68. cartography/models/cloudflare/member.py +4 -0
  69. cartography/models/crowdstrike/hosts.py +1 -1
  70. cartography/models/duo/endpoint.py +1 -1
  71. cartography/models/duo/phone.py +2 -2
  72. cartography/models/duo/user.py +4 -0
  73. cartography/models/entra/user.py +2 -1
  74. cartography/models/github/users.py +4 -0
  75. cartography/models/gsuite/__init__.py +0 -0
  76. cartography/models/gsuite/group.py +218 -0
  77. cartography/models/gsuite/tenant.py +29 -0
  78. cartography/models/gsuite/user.py +107 -0
  79. cartography/models/kandji/device.py +1 -2
  80. cartography/models/keycloak/user.py +4 -0
  81. cartography/models/lastpass/user.py +4 -0
  82. cartography/models/ontology/__init__.py +0 -0
  83. cartography/models/ontology/device.py +125 -0
  84. cartography/models/ontology/mapping/__init__.py +16 -0
  85. cartography/models/ontology/mapping/data/__init__.py +1 -0
  86. cartography/models/ontology/mapping/data/devices.py +160 -0
  87. cartography/models/ontology/mapping/data/users.py +239 -0
  88. cartography/models/ontology/mapping/specs.py +65 -0
  89. cartography/models/ontology/user.py +52 -0
  90. cartography/models/openai/user.py +4 -0
  91. cartography/models/scaleway/iam/user.py +4 -0
  92. cartography/models/snipeit/asset.py +1 -0
  93. cartography/models/snipeit/user.py +4 -0
  94. cartography/models/tailscale/device.py +1 -1
  95. cartography/models/tailscale/user.py +6 -1
  96. cartography/rules/data/frameworks/mitre_attack/requirements/t1098_account_manipulation/__init__.py +176 -89
  97. cartography/sync.py +4 -1
  98. cartography/util.py +49 -18
  99. {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/METADATA +3 -3
  100. {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/RECORD +104 -89
  101. cartography/data/jobs/cleanup/gsuite_ingest_groups_cleanup.json +0 -23
  102. cartography/data/jobs/cleanup/gsuite_ingest_users_cleanup.json +0 -11
  103. cartography/intel/gsuite/api.py +0 -355
  104. {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/WHEEL +0 -0
  105. {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/entry_points.txt +0 -0
  106. {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/licenses/LICENSE +0 -0
  107. {cartography-0.117.0.dist-info → cartography-0.119.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,291 @@
1
+ import logging
2
+ from typing import Any
3
+
4
+ import neo4j
5
+ from googleapiclient.discovery import Resource
6
+ from googleapiclient.errors import HttpError
7
+
8
+ from cartography.client.core.tx import load
9
+ from cartography.client.core.tx import load_matchlinks
10
+ from cartography.graph.job import GraphJob
11
+ from cartography.models.gsuite.group import GSuiteGroupSchema
12
+ from cartography.models.gsuite.group import GSuiteGroupToGroupMemberRel
13
+ from cartography.models.gsuite.group import GSuiteGroupToGroupOwnerRel
14
+ from cartography.models.gsuite.tenant import GSuiteTenantSchema
15
+ from cartography.util import timeit
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ GOOGLE_API_NUM_RETRIES = 5
20
+
21
+
22
+ @timeit
23
+ def get_all_groups(
24
+ admin: Resource, customer_id: str = "my_customer"
25
+ ) -> list[dict[str, Any]]:
26
+ """
27
+ Return list of Google Groups in your organization
28
+ Returns empty list if we are unable to enumerate the groups for any reasons
29
+
30
+ googleapiclient.discovery.build('admin', 'directory_v1', credentials=credentials, cache_discovery=False)
31
+
32
+ :param admin: google's apiclient discovery resource object. From googleapiclient.discovery.build
33
+ See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
34
+ :return: list of Google groups in domain
35
+ """
36
+ request = admin.groups().list(
37
+ customer=customer_id,
38
+ maxResults=20,
39
+ orderBy="email",
40
+ )
41
+ response_objects = []
42
+ while request is not None:
43
+ try:
44
+ resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
45
+ response_objects.extend(resp.get("groups", []))
46
+ request = admin.groups().list_next(request, resp)
47
+ except HttpError as e:
48
+ if (
49
+ e.resp.status == 403
50
+ and "Request had insufficient authentication scopes" in str(e)
51
+ ):
52
+ logger.error(
53
+ "Missing required GSuite scopes. If using the gcloud CLI, "
54
+ "run: gcloud auth application-default login --scopes="
55
+ '"https://www.googleapis.com/auth/admin.directory.user.readonly,'
56
+ "https://www.googleapis.com/auth/admin.directory.group.readonly,"
57
+ "https://www.googleapis.com/auth/admin.directory.group.member.readonly,"
58
+ 'https://www.googleapis.com/auth/cloud-platform"'
59
+ )
60
+ raise
61
+ return response_objects
62
+
63
+
64
+ @timeit
65
+ def get_members_for_groups(
66
+ admin: Resource, groups_email: list[str]
67
+ ) -> dict[str, list[dict[str, Any]]]:
68
+ """Get all members for given groups emails
69
+
70
+ Args:
71
+ admin (Resource): google's apiclient discovery resource object. From googleapiclient.discovery.build
72
+ See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
73
+ groups_email (list[str]): List of group email addresses to get members for
74
+
75
+
76
+ :return: list of dictionaries representing Users or Groups grouped by group email
77
+ """
78
+ results: dict[str, list[dict]] = {}
79
+ for group_email in groups_email:
80
+ request = admin.members().list(
81
+ groupKey=group_email,
82
+ maxResults=500,
83
+ )
84
+ members: list[dict] = []
85
+ while request is not None:
86
+ resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
87
+ members = members + resp.get("members", [])
88
+ request = admin.members().list_next(request, resp)
89
+ results[group_email] = members
90
+
91
+ return results
92
+
93
+
94
+ @timeit
95
+ def transform_groups(
96
+ groups: list[dict], group_memberships: dict[str, list[dict[str, Any]]]
97
+ ) -> tuple[list[dict], list[dict], list[dict]]:
98
+ """Strips list of API response objects to return list of group objects only and lists of subgroup relationships
99
+
100
+ :param groups: Raw groups from Google API
101
+ :param group_memberships: Group memberships data
102
+ :return: tuple of (groups, group_member_relationships, group_owner_relationships)
103
+ """
104
+ transformed_groups: list[dict] = []
105
+ group_member_relationships: list[dict] = []
106
+ group_owner_relationships: list[dict] = []
107
+
108
+ for group in groups:
109
+ group_id = group["id"]
110
+ group_email = group["email"]
111
+ group["member_ids"] = []
112
+ group["owner_ids"] = []
113
+
114
+ for member in group_memberships.get(group_email, []):
115
+ if member["type"] == "GROUP":
116
+ # Create group-to-group relationships
117
+ relationship_data = {
118
+ "parent_group_id": group_id,
119
+ "subgroup_id": member.get("id"),
120
+ "role": member.get("role"),
121
+ }
122
+
123
+ if member.get("role") == "OWNER":
124
+ group_owner_relationships.append(relationship_data)
125
+ else:
126
+ group_member_relationships.append(relationship_data)
127
+ continue
128
+
129
+ # Handle user memberships
130
+ if member.get("role") == "OWNER":
131
+ group["owner_ids"].append(member.get("id"))
132
+ group["member_ids"].append(member.get("id"))
133
+
134
+ transformed_groups.append(group)
135
+
136
+ return transformed_groups, group_member_relationships, group_owner_relationships
137
+
138
+
139
+ @timeit
140
+ def load_gsuite_groups(
141
+ neo4j_session: neo4j.Session,
142
+ groups: list[dict],
143
+ customer_id: str,
144
+ gsuite_update_tag: int,
145
+ ) -> None:
146
+ """
147
+ Load GSuite groups using the modern data model
148
+ """
149
+ logger.info("Ingesting %d gsuite groups", len(groups))
150
+
151
+ # Load tenant first if it doesn't exist
152
+ tenant_data = [{"id": customer_id}]
153
+ load(
154
+ neo4j_session,
155
+ GSuiteTenantSchema(),
156
+ tenant_data,
157
+ lastupdated=gsuite_update_tag,
158
+ )
159
+
160
+ # Load groups with relationship to tenant
161
+ load(
162
+ neo4j_session,
163
+ GSuiteGroupSchema(),
164
+ groups,
165
+ lastupdated=gsuite_update_tag,
166
+ CUSTOMER_ID=customer_id,
167
+ )
168
+
169
+
170
+ @timeit
171
+ def load_gsuite_group_to_group_relationships(
172
+ neo4j_session: neo4j.Session,
173
+ group_member_relationships: list[dict],
174
+ group_owner_relationships: list[dict],
175
+ customer_id: str,
176
+ gsuite_update_tag: int,
177
+ ) -> None:
178
+ """
179
+ Load GSuite group-to-group relationships using MatchLinks
180
+ """
181
+ logger.info(
182
+ "Ingesting %d group member relationships", len(group_member_relationships)
183
+ )
184
+ logger.info(
185
+ "Ingesting %d group owner relationships", len(group_owner_relationships)
186
+ )
187
+
188
+ # Load group member relationships (Group -> Group MEMBER)
189
+ if group_member_relationships:
190
+ load_matchlinks(
191
+ neo4j_session,
192
+ GSuiteGroupToGroupMemberRel(),
193
+ group_member_relationships,
194
+ lastupdated=gsuite_update_tag,
195
+ _sub_resource_label="GSuiteTenant",
196
+ _sub_resource_id=customer_id,
197
+ )
198
+
199
+ # Load group owner relationships (Group -> Group OWNER)
200
+ if group_owner_relationships:
201
+ load_matchlinks(
202
+ neo4j_session,
203
+ GSuiteGroupToGroupOwnerRel(),
204
+ group_owner_relationships,
205
+ lastupdated=gsuite_update_tag,
206
+ _sub_resource_label="GSuiteTenant",
207
+ _sub_resource_id=customer_id,
208
+ )
209
+
210
+
211
+ @timeit
212
+ def cleanup_gsuite_groups(
213
+ neo4j_session: neo4j.Session,
214
+ common_job_parameters: dict[str, Any],
215
+ customer_id: str,
216
+ gsuite_update_tag: int,
217
+ ) -> None:
218
+ """
219
+ Clean up GSuite groups and group-to-group relationships using the modern data model
220
+ """
221
+ logger.debug("Running GSuite groups cleanup job")
222
+
223
+ # Cleanup group nodes
224
+ GraphJob.from_node_schema(GSuiteGroupSchema(), common_job_parameters).run(
225
+ neo4j_session
226
+ )
227
+
228
+ # Cleanup group-to-group member relationships
229
+ GraphJob.from_matchlink(
230
+ GSuiteGroupToGroupMemberRel(),
231
+ "GSuiteTenant",
232
+ customer_id,
233
+ gsuite_update_tag,
234
+ ).run(neo4j_session)
235
+
236
+ # Cleanup group-to-group owner relationships
237
+ GraphJob.from_matchlink(
238
+ GSuiteGroupToGroupOwnerRel(),
239
+ "GSuiteTenant",
240
+ customer_id,
241
+ gsuite_update_tag,
242
+ ).run(neo4j_session)
243
+
244
+
245
+ @timeit
246
+ def sync_gsuite_groups(
247
+ neo4j_session: neo4j.Session,
248
+ admin: Resource,
249
+ gsuite_update_tag: int,
250
+ common_job_parameters: dict[str, Any],
251
+ ) -> None:
252
+ """
253
+ GET GSuite group objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
254
+
255
+ :param neo4j_session: The Neo4j session
256
+ :param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
257
+ See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
258
+ :param gsuite_update_tag: The timestamp value to set our new Neo4j nodes with
259
+ :param common_job_parameters: Parameters to carry to the Neo4j jobs
260
+ :return: Nothing
261
+ """
262
+ logger.debug("Syncing GSuite Groups")
263
+
264
+ customer_id = common_job_parameters.get(
265
+ "CUSTOMER_ID", "my_customer"
266
+ ) # Default to "my_customer" for backward compatibility
267
+
268
+ # 1. GET - Fetch data from API
269
+ resp_objs = get_all_groups(admin, customer_id)
270
+ group_members = get_members_for_groups(admin, [resp["email"] for resp in resp_objs])
271
+
272
+ # 2. TRANSFORM - Shape data for ingestion
273
+ groups, group_member_relationships, group_owner_relationships = transform_groups(
274
+ resp_objs, group_members
275
+ )
276
+
277
+ # 3. LOAD - Ingest to Neo4j using data model
278
+ load_gsuite_groups(neo4j_session, groups, customer_id, gsuite_update_tag)
279
+
280
+ # Load group-to-group relationships after groups are loaded
281
+ load_gsuite_group_to_group_relationships(
282
+ neo4j_session,
283
+ group_member_relationships,
284
+ group_owner_relationships,
285
+ customer_id,
286
+ gsuite_update_tag,
287
+ )
288
+
289
+ # 4. CLEANUP - Remove stale data
290
+ cleanup_params = {**common_job_parameters, "CUSTOMER_ID": customer_id}
291
+ cleanup_gsuite_groups(neo4j_session, cleanup_params, customer_id, gsuite_update_tag)
@@ -0,0 +1,142 @@
1
+ import logging
2
+ from collections import defaultdict
3
+ from typing import Any
4
+
5
+ import neo4j
6
+ from googleapiclient.discovery import Resource
7
+
8
+ from cartography.client.core.tx import load
9
+ from cartography.graph.job import GraphJob
10
+ from cartography.models.gsuite.tenant import GSuiteTenantSchema
11
+ from cartography.models.gsuite.user import GSuiteUserSchema
12
+ from cartography.util import timeit
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+ GOOGLE_API_NUM_RETRIES = 5
17
+
18
+
19
+ @timeit
20
+ def get_all_users(admin: Resource) -> list[dict]:
21
+ """
22
+ Return list of Google Users in your organization
23
+ Returns empty list if we are unable to enumerate the users for any reasons
24
+ https://developers.google.com/admin-sdk/directory/v1/guides/manage-users
25
+
26
+ :param admin: apiclient discovery resource object
27
+ :return: list of Google users in domain
28
+ see https://developers.google.com/admin-sdk/directory/v1/guides/manage-users#get_all_domain_users
29
+ """
30
+ request = admin.users().list(
31
+ customer="my_customer",
32
+ maxResults=500,
33
+ orderBy="email",
34
+ )
35
+ response_objects = []
36
+ while request is not None:
37
+ resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
38
+ response_objects.append(resp)
39
+ request = admin.users().list_next(request, resp)
40
+ return response_objects
41
+
42
+
43
+ @timeit
44
+ def transform_users(response_objects: list[dict]) -> dict[str, list[dict[str, Any]]]:
45
+ """Transform list of API response objects to return list of user objects with flattened structure grouped by customerId
46
+ :param response_objects: Raw API response objects
47
+ :return: list of dictionary objects for data model consumption
48
+ """
49
+ results = defaultdict(list)
50
+ for response_object in response_objects:
51
+ for user in response_object["users"]:
52
+ # Flatten the nested name structure
53
+ transformed_user = user.copy()
54
+ if "name" in user and isinstance(user["name"], dict):
55
+ transformed_user["name"] = user["name"].get("fullName")
56
+ transformed_user["family_name"] = user["name"].get("familyName")
57
+ transformed_user["given_name"] = user["name"].get("givenName")
58
+ results[transformed_user["customerId"]].append(transformed_user)
59
+ return results
60
+
61
+
62
+ @timeit
63
+ def load_gsuite_users(
64
+ neo4j_session: neo4j.Session,
65
+ users_by_customer: dict[str, list[dict]],
66
+ gsuite_update_tag: int,
67
+ ) -> None:
68
+ """
69
+ Load GSuite users using the modern data model
70
+ """
71
+ logger.info("Ingesting %s gsuite tenants", len(users_by_customer))
72
+ tenant_data = [{"id": customer_id} for customer_id in users_by_customer.keys()]
73
+ load(
74
+ neo4j_session,
75
+ GSuiteTenantSchema(),
76
+ tenant_data,
77
+ lastupdated=gsuite_update_tag,
78
+ )
79
+
80
+ for customer_id, users in users_by_customer.items():
81
+ logger.info(
82
+ "Ingesting %s gsuite users for customer %s", len(users), customer_id
83
+ )
84
+ # Load users with relationship to tenant
85
+ load(
86
+ neo4j_session,
87
+ GSuiteUserSchema(),
88
+ users,
89
+ lastupdated=gsuite_update_tag,
90
+ CUSTOMER_ID=customer_id,
91
+ )
92
+
93
+
94
+ @timeit
95
+ def cleanup_gsuite_users(
96
+ neo4j_session: neo4j.Session,
97
+ common_job_parameters: dict[str, Any],
98
+ ) -> None:
99
+ """
100
+ Clean up GSuite users using the modern data model
101
+ """
102
+ logger.debug("Running GSuite users cleanup job")
103
+ GraphJob.from_node_schema(GSuiteUserSchema(), common_job_parameters).run(
104
+ neo4j_session
105
+ )
106
+
107
+
108
+ @timeit
109
+ def sync_gsuite_users(
110
+ neo4j_session: neo4j.Session,
111
+ admin: Resource,
112
+ gsuite_update_tag: int,
113
+ common_job_parameters: dict[str, Any],
114
+ ) -> list[str]:
115
+ """
116
+ GET GSuite user objects using the google admin api resource, load the data into Neo4j and clean up stale nodes.
117
+
118
+ :param neo4j_session: The Neo4j session
119
+ :param admin: Google admin resource object created by `googleapiclient.discovery.build()`.
120
+ See https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.discovery-module.html#build.
121
+ :param gsuite_update_tag: The timestamp value to set our new Neo4j nodes with
122
+ :param common_job_parameters: Parameters to carry to the Neo4j jobs
123
+ :return: list of customer IDs
124
+ """
125
+ logger.debug("Syncing GSuite Users")
126
+
127
+ # 1. GET - Fetch data from API
128
+ resp_objs = get_all_users(admin)
129
+
130
+ # 2. TRANSFORM - Shape data for ingestion
131
+ users_by_customers = transform_users(resp_objs)
132
+
133
+ # 3. LOAD - Ingest to Neo4j using data model
134
+ load_gsuite_users(neo4j_session, users_by_customers, gsuite_update_tag)
135
+
136
+ # 4. CLEANUP - Remove stale data
137
+ for customer_id in users_by_customers.keys():
138
+ cleanup_params = {**common_job_parameters, "CUSTOMER_ID": customer_id}
139
+ cleanup_gsuite_users(neo4j_session, cleanup_params)
140
+
141
+ # Return the list of customer IDs
142
+ return list(users_by_customers.keys())
@@ -4,6 +4,7 @@ from typing import List
4
4
 
5
5
  import neo4j
6
6
 
7
+ from cartography.client.core.tx import run_write_query
7
8
  from cartography.intel.jamf.util import call_jamf_api
8
9
  from cartography.util import run_cleanup_job
9
10
  from cartography.util import timeit
@@ -35,7 +36,12 @@ def load_computer_groups(
35
36
  g.lastupdated = $UpdateTag
36
37
  """
37
38
  groups = data.get("computer_groups")
38
- neo4j_session.run(ingest_groups, JsonData=groups, UpdateTag=update_tag)
39
+ run_write_query(
40
+ neo4j_session,
41
+ ingest_groups,
42
+ JsonData=groups,
43
+ UpdateTag=update_tag,
44
+ )
39
45
 
40
46
 
41
47
  @timeit
@@ -10,6 +10,8 @@ from typing import List
10
10
  import neo4j
11
11
  import oci
12
12
 
13
+ from cartography.client.core.tx import read_list_of_dicts_tx
14
+ from cartography.client.core.tx import run_write_query
13
15
  from cartography.util import run_cleanup_job
14
16
 
15
17
  from . import utils
@@ -89,7 +91,8 @@ def load_compartments(
89
91
  """
90
92
 
91
93
  for compartment in compartments:
92
- neo4j_session.run(
94
+ run_write_query(
95
+ neo4j_session,
93
96
  ingest_compartment,
94
97
  OCID=compartment["id"],
95
98
  COMPARTMENT_ID=compartment["compartment-id"],
@@ -126,7 +129,8 @@ def load_users(
126
129
  """
127
130
 
128
131
  for user in users:
129
- neo4j_session.run(
132
+ run_write_query(
133
+ neo4j_session,
130
134
  ingest_user,
131
135
  OCID=user["id"],
132
136
  CREATE_DATE=str(user["time-created"]),
@@ -206,7 +210,8 @@ def load_groups(
206
210
  """
207
211
 
208
212
  for group in groups:
209
- neo4j_session.run(
213
+ run_write_query(
214
+ neo4j_session,
210
215
  ingest_group,
211
216
  OCID=group["id"],
212
217
  CREATE_DATE=str(group["time-created"]),
@@ -260,7 +265,11 @@ def sync_group_memberships(
260
265
  "MATCH (group:OCIGroup)<-[:RESOURCE]-(OCITenancy{ocid: $OCI_TENANCY_ID}) "
261
266
  "return group.name as name, group.ocid as ocid;"
262
267
  )
263
- groups = neo4j_session.run(query, OCI_TENANCY_ID=current_tenancy_id)
268
+ groups = neo4j_session.execute_read(
269
+ read_list_of_dicts_tx,
270
+ query,
271
+ OCI_TENANCY_ID=current_tenancy_id,
272
+ )
264
273
  groups_membership = {
265
274
  group["ocid"]: get_group_membership_data(iam, group["ocid"], current_tenancy_id)
266
275
  for group in groups
@@ -288,7 +297,8 @@ def load_group_memberships(
288
297
  """
289
298
  for group_ocid, membership_data in group_memberships.items():
290
299
  for info in membership_data["GroupMemberships"]:
291
- neo4j_session.run(
300
+ run_write_query(
301
+ neo4j_session,
292
302
  ingest_membership,
293
303
  COMPARTMENT_ID=info["compartment-id"],
294
304
  GROUP_OCID=info["group-id"],
@@ -317,7 +327,8 @@ def load_policies(
317
327
  """
318
328
 
319
329
  for policy in policies:
320
- neo4j_session.run(
330
+ run_write_query(
331
+ neo4j_session,
321
332
  ingest_policy,
322
333
  OCID=policy["id"],
323
334
  POLICY_NAME=policy["name"],
@@ -386,7 +397,8 @@ def load_oci_policy_group_reference(
386
397
  ON CREATE SET r.firstseen = timestamp()
387
398
  SET r.lastupdated = $oci_update_tag
388
399
  """
389
- neo4j_session.run(
400
+ run_write_query(
401
+ neo4j_session,
390
402
  ingest_policy_group_reference,
391
403
  POLICY_ID=policy_id,
392
404
  GROUP_ID=group_id,
@@ -408,7 +420,8 @@ def load_oci_policy_compartment_reference(
408
420
  ON CREATE SET r.firstseen = timestamp()
409
421
  SET r.lastupdated = $oci_update_tag
410
422
  """
411
- neo4j_session.run(
423
+ run_write_query(
424
+ neo4j_session,
412
425
  ingest_policy_compartment_reference,
413
426
  POLICY_ID=policy_id,
414
427
  COMPARTMENT_ID=compartment_id,
@@ -487,7 +500,8 @@ def load_region_subscriptions(
487
500
  SET r.lastupdated = $oci_update_tag
488
501
  """
489
502
  for region in regions:
490
- neo4j_session.run(
503
+ run_write_query(
504
+ neo4j_session,
491
505
  query,
492
506
  REGION_KEY=region["region-key"],
493
507
  REGION_NAME=region["region-name"],
@@ -11,6 +11,7 @@ from oci.exceptions import ConfigFileNotFound
11
11
  from oci.exceptions import InvalidConfig
12
12
  from oci.exceptions import ProfileNotFound
13
13
 
14
+ from cartography.client.core.tx import run_write_query
14
15
  from cartography.util import run_cleanup_job
15
16
 
16
17
  logger = logging.getLogger(__name__)
@@ -100,7 +101,8 @@ def load_oci_accounts(
100
101
  SET aa.lastupdated = $oci_update_tag, aa.name = $ACCOUNT_NAME
101
102
  """
102
103
  for name in oci_accounts:
103
- neo4j_session.run(
104
+ run_write_query(
105
+ neo4j_session,
104
106
  query,
105
107
  TENANCY_ID=oci_accounts[name]["tenancy"],
106
108
  ACCOUNT_NAME=name,
@@ -7,6 +7,8 @@ from typing import List
7
7
 
8
8
  import neo4j
9
9
 
10
+ from cartography.client.core.tx import read_list_of_dicts_tx
11
+
10
12
 
11
13
  # Generic way to turn a OCI python object into the json response that you would see from calling the REST API.
12
14
  def oci_object_to_json(in_obj: Any) -> List[Dict[str, Any]]:
@@ -36,7 +38,11 @@ def get_compartments_in_tenancy(
36
38
  "return DISTINCT compartment.name as name, compartment.ocid as ocid, "
37
39
  "compartment.compartmentid as compartmentid;"
38
40
  )
39
- return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
41
+ return neo4j_session.execute_read(
42
+ read_list_of_dicts_tx,
43
+ query,
44
+ OCI_TENANCY_ID=tenancy_id,
45
+ )
40
46
 
41
47
 
42
48
  # Grab list of all groups in neo4j already populated by iam.
@@ -48,7 +54,11 @@ def get_groups_in_tenancy(
48
54
  "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-[*]->(group:OCIGroup)"
49
55
  "return DISTINCT group.name as name, group.ocid as ocid;"
50
56
  )
51
- return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
57
+ return neo4j_session.execute_read(
58
+ read_list_of_dicts_tx,
59
+ query,
60
+ OCI_TENANCY_ID=tenancy_id,
61
+ )
52
62
 
53
63
 
54
64
  # Grab list of all policies in neo4j already populated by iam.
@@ -61,7 +71,11 @@ def get_policies_in_tenancy(
61
71
  "return DISTINCT policy.name as name, policy.ocid as ocid, policy.statements as statements, "
62
72
  "policy.compartmentid as compartmentid;"
63
73
  )
64
- return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
74
+ return neo4j_session.execute_read(
75
+ read_list_of_dicts_tx,
76
+ query,
77
+ OCI_TENANCY_ID=tenancy_id,
78
+ )
65
79
 
66
80
 
67
81
  # Grab list of all regions in neo4j already populated by iam.
@@ -73,7 +87,11 @@ def get_regions_in_tenancy(
73
87
  "MATCH (OCITenancy{ocid: $OCI_TENANCY_ID})-->(region:OCIRegion)"
74
88
  "return DISTINCT region.name as name, region.key as key;"
75
89
  )
76
- return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id)
90
+ return neo4j_session.execute_read(
91
+ read_list_of_dicts_tx,
92
+ query,
93
+ OCI_TENANCY_ID=tenancy_id,
94
+ )
77
95
 
78
96
 
79
97
  # Grab list of all security groups in neo4j already populated by network. Need to handle regions for this one.
@@ -88,4 +106,9 @@ def get_security_groups_in_tenancy(
88
106
  "return DISTINCT security_group.name as name, security_group.ocid as ocid, security_group.compartmentid "
89
107
  "as compartmentid;"
90
108
  )
91
- return neo4j_session.run(query, OCI_TENANCY_ID=tenancy_id, OCI_REGION=region)
109
+ return neo4j_session.execute_read(
110
+ read_list_of_dicts_tx,
111
+ query,
112
+ OCI_TENANCY_ID=tenancy_id,
113
+ OCI_REGION=region,
114
+ )
@@ -68,24 +68,25 @@ def query_for_okta_to_aws_role_mapping(
68
68
  :param neo4j_session: session from the Neo4j server
69
69
  :param mapping_regex: the regex used by the organization to map groups to aws roles
70
70
  """
71
- query = "MATCH (app:OktaApplication{name:'amazon_aws'})--(group:OktaGroup) return group.id, group.name"
71
+ query = (
72
+ "MATCH (app:OktaApplication{name:'amazon_aws'})--(group:OktaGroup) "
73
+ "RETURN group.id AS group_id, group.name AS group_name"
74
+ )
72
75
 
73
76
  group_to_role_mapping: List[Dict] = []
74
- has_results = False
75
- results = neo4j_session.run(query)
77
+ results = neo4j_session.execute_read(read_list_of_dicts_tx, query)
76
78
 
77
79
  for res in results:
78
- has_results = True
79
80
  # input: okta group id, okta group name. output: aws role arn.
80
81
  mapping = transform_okta_group_to_aws_role(
81
- res["group.id"],
82
- res["group.name"],
82
+ res["group_id"],
83
+ res["group_name"],
83
84
  mapping_regex,
84
85
  )
85
86
  if mapping:
86
87
  group_to_role_mapping.append(mapping)
87
88
 
88
- if has_results and not group_to_role_mapping:
89
+ if results and not group_to_role_mapping:
89
90
  logger.warning(
90
91
  "AWS Okta Application present, but no mappings were found. "
91
92
  "Please verify the mapping regex is correct",
@@ -247,7 +248,7 @@ def _load_awssso_tx(
247
248
  ingest_statement,
248
249
  GROUP_TO_ROLE=[g._asdict() for g in group_to_role],
249
250
  okta_update_tag=okta_update_tag,
250
- )
251
+ ).consume()
251
252
 
252
253
 
253
254
  def _load_okta_group_to_awssso_roles(
@@ -161,7 +161,7 @@ def _load_okta_users(
161
161
  new_user.password_changed = user_data.password_changed,
162
162
  new_user.transition_to_status = user_data.transition_to_status,
163
163
  new_user.lastupdated = $okta_update_tag,
164
- new_user :UserAccount
164
+ new_user:UserAccount
165
165
  WITH new_user, org
166
166
  MERGE (org)-[org_r:RESOURCE]->(new_user)
167
167
  ON CREATE SET org_r.firstseen = timestamp()