cartography 0.100.0rc1__py3-none-any.whl → 0.100.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/_version.py +8 -3
- cartography/cli.py +3 -2
- cartography/intel/aws/ecr.py +10 -2
- cartography/intel/gcp/__init__.py +24 -2
- cartography/intel/gcp/iam.py +222 -0
- cartography/intel/gsuite/__init__.py +21 -4
- cartography/intel/gsuite/api.py +18 -3
- cartography/models/gcp/iam.py +70 -0
- {cartography-0.100.0rc1.dist-info → cartography-0.100.0rc2.dist-info}/METADATA +1 -1
- {cartography-0.100.0rc1.dist-info → cartography-0.100.0rc2.dist-info}/RECORD +14 -12
- {cartography-0.100.0rc1.dist-info → cartography-0.100.0rc2.dist-info}/WHEEL +1 -1
- {cartography-0.100.0rc1.dist-info → cartography-0.100.0rc2.dist-info}/LICENSE +0 -0
- {cartography-0.100.0rc1.dist-info → cartography-0.100.0rc2.dist-info}/entry_points.txt +0 -0
- {cartography-0.100.0rc1.dist-info → cartography-0.100.0rc2.dist-info}/top_level.txt +0 -0
cartography/_version.py
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
# file generated by
|
|
1
|
+
# file generated by setuptools-scm
|
|
2
2
|
# don't change, don't track in version control
|
|
3
|
+
|
|
4
|
+
__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
|
|
5
|
+
|
|
3
6
|
TYPE_CHECKING = False
|
|
4
7
|
if TYPE_CHECKING:
|
|
5
|
-
from typing import Tuple
|
|
8
|
+
from typing import Tuple
|
|
9
|
+
from typing import Union
|
|
10
|
+
|
|
6
11
|
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
7
12
|
else:
|
|
8
13
|
VERSION_TUPLE = object
|
|
@@ -12,5 +17,5 @@ __version__: str
|
|
|
12
17
|
__version_tuple__: VERSION_TUPLE
|
|
13
18
|
version_tuple: VERSION_TUPLE
|
|
14
19
|
|
|
15
|
-
__version__ = version = '0.100.
|
|
20
|
+
__version__ = version = '0.100.0rc2'
|
|
16
21
|
__version_tuple__ = version_tuple = (0, 100, 0)
|
cartography/cli.py
CHANGED
|
@@ -439,9 +439,10 @@ class CLI:
|
|
|
439
439
|
'--gsuite-auth-method',
|
|
440
440
|
type=str,
|
|
441
441
|
default='delegated',
|
|
442
|
-
choices=['delegated', 'oauth'],
|
|
442
|
+
choices=['delegated', 'oauth', 'default'],
|
|
443
443
|
help=(
|
|
444
|
-
'
|
|
444
|
+
'GSuite authentication method. Can be "delegated" for service account or "oauth" for OAuth. '
|
|
445
|
+
'"Default" best if using gcloud CLI.'
|
|
445
446
|
),
|
|
446
447
|
)
|
|
447
448
|
parser.add_argument(
|
cartography/intel/aws/ecr.py
CHANGED
|
@@ -36,7 +36,10 @@ def get_ecr_repository_images(boto3_session: boto3.session.Session, region: str,
|
|
|
36
36
|
paginator = client.get_paginator('list_images')
|
|
37
37
|
ecr_repository_images: List[Dict] = []
|
|
38
38
|
for page in paginator.paginate(repositoryName=repository_name):
|
|
39
|
-
|
|
39
|
+
image_ids = page['imageIds']
|
|
40
|
+
if image_ids:
|
|
41
|
+
response = client.describe_images(repositoryName=repository_name, imageIds=image_ids)
|
|
42
|
+
ecr_repository_images.extend(response['imageDetails'])
|
|
40
43
|
return ecr_repository_images
|
|
41
44
|
|
|
42
45
|
|
|
@@ -103,7 +106,12 @@ def _load_ecr_repo_img_tx(
|
|
|
103
106
|
ON CREATE SET ri.firstseen = timestamp()
|
|
104
107
|
SET ri.lastupdated = $aws_update_tag,
|
|
105
108
|
ri.tag = repo_img.imageTag,
|
|
106
|
-
ri.uri = repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, '')
|
|
109
|
+
ri.uri = repo_img.repo_uri + COALESCE(":" + repo_img.imageTag, ''),
|
|
110
|
+
ri.image_size_bytes = repo_img.imageSizeInBytes,
|
|
111
|
+
ri.image_pushed_at = repo_img.imagePushedAt,
|
|
112
|
+
ri.image_manifest_media_type = repo_img.imageManifestMediaType,
|
|
113
|
+
ri.artifact_media_type = repo_img.artifactMediaType,
|
|
114
|
+
ri.last_recorded_pull_time = repo_img.lastRecordedPullTime
|
|
107
115
|
WITH ri, repo_img
|
|
108
116
|
|
|
109
117
|
MERGE (img:ECRImage{id: repo_img.imageDigest})
|
|
@@ -17,21 +17,23 @@ from cartography.intel.gcp import compute
|
|
|
17
17
|
from cartography.intel.gcp import crm
|
|
18
18
|
from cartography.intel.gcp import dns
|
|
19
19
|
from cartography.intel.gcp import gke
|
|
20
|
+
from cartography.intel.gcp import iam
|
|
20
21
|
from cartography.intel.gcp import storage
|
|
21
22
|
from cartography.util import run_analysis_job
|
|
22
23
|
from cartography.util import timeit
|
|
23
24
|
|
|
24
25
|
logger = logging.getLogger(__name__)
|
|
25
|
-
Resources = namedtuple('Resources', 'compute container crm_v1 crm_v2 dns storage serviceusage')
|
|
26
|
+
Resources = namedtuple('Resources', 'compute container crm_v1 crm_v2 dns storage serviceusage iam')
|
|
26
27
|
|
|
27
28
|
# Mapping of service short names to their full names as in docs. See https://developers.google.com/apis-explorer,
|
|
28
29
|
# and https://cloud.google.com/service-usage/docs/reference/rest/v1/services#ServiceConfig
|
|
29
|
-
Services = namedtuple('Services', 'compute storage gke dns')
|
|
30
|
+
Services = namedtuple('Services', 'compute storage gke dns iam')
|
|
30
31
|
service_names = Services(
|
|
31
32
|
compute='compute.googleapis.com',
|
|
32
33
|
storage='storage.googleapis.com',
|
|
33
34
|
gke='container.googleapis.com',
|
|
34
35
|
dns='dns.googleapis.com',
|
|
36
|
+
iam='iam.googleapis.com',
|
|
35
37
|
)
|
|
36
38
|
|
|
37
39
|
|
|
@@ -112,6 +114,13 @@ def _get_serviceusage_resource(credentials: GoogleCredentials) -> Resource:
|
|
|
112
114
|
return googleapiclient.discovery.build('serviceusage', 'v1', credentials=credentials, cache_discovery=False)
|
|
113
115
|
|
|
114
116
|
|
|
117
|
+
def _get_iam_resource(credentials: GoogleCredentials) -> Resource:
|
|
118
|
+
"""
|
|
119
|
+
Instantiates a Google IAM resource object to call the IAM API.
|
|
120
|
+
"""
|
|
121
|
+
return googleapiclient.discovery.build('iam', 'v1', credentials=credentials, cache_discovery=False)
|
|
122
|
+
|
|
123
|
+
|
|
115
124
|
def _initialize_resources(credentials: GoogleCredentials) -> Resource:
|
|
116
125
|
"""
|
|
117
126
|
Create namedtuple of all resource objects necessary for GCP data gathering.
|
|
@@ -126,6 +135,7 @@ def _initialize_resources(credentials: GoogleCredentials) -> Resource:
|
|
|
126
135
|
container=None,
|
|
127
136
|
dns=None,
|
|
128
137
|
storage=None,
|
|
138
|
+
iam=_get_iam_resource(credentials),
|
|
129
139
|
)
|
|
130
140
|
|
|
131
141
|
|
|
@@ -286,6 +296,18 @@ def _sync_multiple_projects(
|
|
|
286
296
|
logger.info("Syncing GCP project %s for DNS", project_id)
|
|
287
297
|
_sync_single_project_dns(neo4j_session, resources, project_id, gcp_update_tag, common_job_parameters)
|
|
288
298
|
|
|
299
|
+
# IAM data sync
|
|
300
|
+
for project in projects:
|
|
301
|
+
project_id = project['projectId']
|
|
302
|
+
logger.info("Syncing GCP project %s for IAM", project_id)
|
|
303
|
+
iam.sync(
|
|
304
|
+
neo4j_session,
|
|
305
|
+
resources.iam,
|
|
306
|
+
project_id,
|
|
307
|
+
gcp_update_tag,
|
|
308
|
+
common_job_parameters,
|
|
309
|
+
)
|
|
310
|
+
|
|
289
311
|
|
|
290
312
|
@timeit
|
|
291
313
|
def get_gcp_credentials() -> GoogleCredentials:
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
from typing import Dict
|
|
4
|
+
from typing import List
|
|
5
|
+
|
|
6
|
+
import neo4j
|
|
7
|
+
from googleapiclient.discovery import Resource
|
|
8
|
+
|
|
9
|
+
from cartography.client.core.tx import load
|
|
10
|
+
from cartography.graph.job import GraphJob
|
|
11
|
+
from cartography.models.gcp.iam import GCPRoleSchema
|
|
12
|
+
from cartography.models.gcp.iam import GCPServiceAccountSchema
|
|
13
|
+
from cartography.util import timeit
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
# GCP API can be subject to rate limiting, so add small delays between calls
|
|
18
|
+
LIST_SLEEP = 1
|
|
19
|
+
DESCRIBE_SLEEP = 1
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@timeit
|
|
23
|
+
def get_gcp_service_accounts(iam_client: Resource, project_id: str) -> List[Dict[str, Any]]:
|
|
24
|
+
"""
|
|
25
|
+
Retrieve a list of GCP service accounts for a given project.
|
|
26
|
+
|
|
27
|
+
:param iam_client: The IAM resource object created by googleapiclient.discovery.build().
|
|
28
|
+
:param project_id: The GCP Project ID to retrieve service accounts from.
|
|
29
|
+
:return: A list of dictionaries representing GCP service accounts.
|
|
30
|
+
"""
|
|
31
|
+
service_accounts: List[Dict[str, Any]] = []
|
|
32
|
+
try:
|
|
33
|
+
request = iam_client.projects().serviceAccounts().list(
|
|
34
|
+
name=f'projects/{project_id}',
|
|
35
|
+
)
|
|
36
|
+
while request is not None:
|
|
37
|
+
response = request.execute()
|
|
38
|
+
if 'accounts' in response:
|
|
39
|
+
service_accounts.extend(response['accounts'])
|
|
40
|
+
request = iam_client.projects().serviceAccounts().list_next(
|
|
41
|
+
previous_request=request,
|
|
42
|
+
previous_response=response,
|
|
43
|
+
)
|
|
44
|
+
except Exception as e:
|
|
45
|
+
logger.warning(f"Error retrieving service accounts for project {project_id}: {e}")
|
|
46
|
+
return service_accounts
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@timeit
|
|
50
|
+
def get_gcp_roles(iam_client: Resource, project_id: str) -> List[Dict]:
|
|
51
|
+
"""
|
|
52
|
+
Retrieve custom and predefined roles from GCP for a given project.
|
|
53
|
+
|
|
54
|
+
:param iam_client: The IAM resource object created by googleapiclient.discovery.build().
|
|
55
|
+
:param project_id: The GCP Project ID to retrieve roles from.
|
|
56
|
+
:return: A list of dictionaries representing GCP roles.
|
|
57
|
+
"""
|
|
58
|
+
try:
|
|
59
|
+
roles = []
|
|
60
|
+
|
|
61
|
+
# Get custom roles
|
|
62
|
+
custom_req = iam_client.projects().roles().list(parent=f'projects/{project_id}')
|
|
63
|
+
while custom_req is not None:
|
|
64
|
+
resp = custom_req.execute()
|
|
65
|
+
roles.extend(resp.get('roles', []))
|
|
66
|
+
custom_req = iam_client.projects().roles().list_next(custom_req, resp)
|
|
67
|
+
|
|
68
|
+
# Get predefined roles
|
|
69
|
+
predefined_req = iam_client.roles().list(view='FULL')
|
|
70
|
+
while predefined_req is not None:
|
|
71
|
+
resp = predefined_req.execute()
|
|
72
|
+
roles.extend(resp.get('roles', []))
|
|
73
|
+
predefined_req = iam_client.roles().list_next(predefined_req, resp)
|
|
74
|
+
|
|
75
|
+
return roles
|
|
76
|
+
except Exception as e:
|
|
77
|
+
logger.warning(f"Error getting GCP roles - {e}")
|
|
78
|
+
return []
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@timeit
|
|
82
|
+
def load_gcp_service_accounts(
|
|
83
|
+
neo4j_session: neo4j.Session,
|
|
84
|
+
service_accounts: List[Dict[str, Any]],
|
|
85
|
+
project_id: str,
|
|
86
|
+
gcp_update_tag: int,
|
|
87
|
+
) -> None:
|
|
88
|
+
"""
|
|
89
|
+
Load GCP service account data into Neo4j.
|
|
90
|
+
|
|
91
|
+
:param neo4j_session: The Neo4j session.
|
|
92
|
+
:param service_accounts: A list of service account data to load.
|
|
93
|
+
:param project_id: The GCP Project ID associated with the service accounts.
|
|
94
|
+
:param gcp_update_tag: The timestamp of the current sync run.
|
|
95
|
+
"""
|
|
96
|
+
logger.debug(f"Loading {len(service_accounts)} service accounts for project {project_id}")
|
|
97
|
+
transformed_service_accounts = []
|
|
98
|
+
for sa in service_accounts:
|
|
99
|
+
transformed_sa = {
|
|
100
|
+
'id': sa['uniqueId'],
|
|
101
|
+
'email': sa.get('email'),
|
|
102
|
+
'displayName': sa.get('displayName'),
|
|
103
|
+
'oauth2ClientId': sa.get('oauth2ClientId'),
|
|
104
|
+
'uniqueId': sa.get('uniqueId'),
|
|
105
|
+
'disabled': sa.get('disabled', False),
|
|
106
|
+
'projectId': project_id,
|
|
107
|
+
}
|
|
108
|
+
transformed_service_accounts.append(transformed_sa)
|
|
109
|
+
|
|
110
|
+
load(
|
|
111
|
+
neo4j_session,
|
|
112
|
+
GCPServiceAccountSchema(),
|
|
113
|
+
transformed_service_accounts,
|
|
114
|
+
lastupdated=gcp_update_tag,
|
|
115
|
+
projectId=project_id,
|
|
116
|
+
additional_labels=['GCPPrincipal'],
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@timeit
|
|
121
|
+
def load_gcp_roles(
|
|
122
|
+
neo4j_session: neo4j.Session,
|
|
123
|
+
roles: List[Dict[str, Any]],
|
|
124
|
+
project_id: str,
|
|
125
|
+
gcp_update_tag: int,
|
|
126
|
+
) -> None:
|
|
127
|
+
"""
|
|
128
|
+
Load GCP role data into Neo4j.
|
|
129
|
+
|
|
130
|
+
:param neo4j_session: The Neo4j session.
|
|
131
|
+
:param roles: A list of role data to load.
|
|
132
|
+
:param project_id: The GCP Project ID associated with the roles.
|
|
133
|
+
:param gcp_update_tag: The timestamp of the current sync run.
|
|
134
|
+
"""
|
|
135
|
+
logger.debug(f"Loading {len(roles)} roles for project {project_id}")
|
|
136
|
+
transformed_roles = []
|
|
137
|
+
for role in roles:
|
|
138
|
+
role_name = role['name']
|
|
139
|
+
if role_name.startswith('roles/'):
|
|
140
|
+
if role_name in ['roles/owner', 'roles/editor', 'roles/viewer']:
|
|
141
|
+
role_type = 'BASIC'
|
|
142
|
+
else:
|
|
143
|
+
role_type = 'PREDEFINED'
|
|
144
|
+
else:
|
|
145
|
+
role_type = 'CUSTOM'
|
|
146
|
+
|
|
147
|
+
transformed_role = {
|
|
148
|
+
'id': role_name,
|
|
149
|
+
'name': role_name,
|
|
150
|
+
'title': role.get('title'),
|
|
151
|
+
'description': role.get('description'),
|
|
152
|
+
'deleted': role.get('deleted', False),
|
|
153
|
+
'etag': role.get('etag'),
|
|
154
|
+
'includedPermissions': role.get('includedPermissions', []),
|
|
155
|
+
'roleType': role_type,
|
|
156
|
+
'projectId': project_id,
|
|
157
|
+
}
|
|
158
|
+
transformed_roles.append(transformed_role)
|
|
159
|
+
|
|
160
|
+
load(
|
|
161
|
+
neo4j_session,
|
|
162
|
+
GCPRoleSchema(),
|
|
163
|
+
transformed_roles,
|
|
164
|
+
lastupdated=gcp_update_tag,
|
|
165
|
+
projectId=project_id,
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@timeit
|
|
170
|
+
def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
|
|
171
|
+
"""
|
|
172
|
+
Run cleanup jobs for GCP IAM data in Neo4j.
|
|
173
|
+
|
|
174
|
+
:param neo4j_session: The Neo4j session.
|
|
175
|
+
:param common_job_parameters: Common job parameters for cleanup.
|
|
176
|
+
"""
|
|
177
|
+
logger.debug("Running GCP IAM cleanup job")
|
|
178
|
+
job_params = {
|
|
179
|
+
**common_job_parameters,
|
|
180
|
+
'projectId': common_job_parameters.get('PROJECT_ID'),
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
cleanup_jobs = [
|
|
184
|
+
GraphJob.from_node_schema(GCPServiceAccountSchema(), job_params),
|
|
185
|
+
GraphJob.from_node_schema(GCPRoleSchema(), job_params),
|
|
186
|
+
]
|
|
187
|
+
|
|
188
|
+
for cleanup_job in cleanup_jobs:
|
|
189
|
+
cleanup_job.run(neo4j_session)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
@timeit
|
|
193
|
+
def sync(
|
|
194
|
+
neo4j_session: neo4j.Session,
|
|
195
|
+
iam_client: Resource,
|
|
196
|
+
project_id: str,
|
|
197
|
+
gcp_update_tag: int,
|
|
198
|
+
common_job_parameters: Dict[str, Any],
|
|
199
|
+
) -> None:
|
|
200
|
+
"""
|
|
201
|
+
Sync GCP IAM resources for a given project.
|
|
202
|
+
|
|
203
|
+
:param neo4j_session: The Neo4j session.
|
|
204
|
+
:param iam_client: The IAM resource object created by googleapiclient.discovery.build().
|
|
205
|
+
:param project_id: The GCP Project ID to sync.
|
|
206
|
+
:param gcp_update_tag: The timestamp of the current sync run.
|
|
207
|
+
:param common_job_parameters: Common job parameters for the sync.
|
|
208
|
+
"""
|
|
209
|
+
logger.info(f"Syncing GCP IAM for project {project_id}")
|
|
210
|
+
|
|
211
|
+
# Get and load service accounts
|
|
212
|
+
service_accounts = get_gcp_service_accounts(iam_client, project_id)
|
|
213
|
+
logger.info(f"Found {len(service_accounts)} service accounts in project {project_id}")
|
|
214
|
+
load_gcp_service_accounts(neo4j_session, service_accounts, project_id, gcp_update_tag)
|
|
215
|
+
|
|
216
|
+
# Get and load roles
|
|
217
|
+
roles = get_gcp_roles(iam_client, project_id)
|
|
218
|
+
logger.info(f"Found {len(roles)} roles in project {project_id}")
|
|
219
|
+
load_gcp_roles(neo4j_session, roles, project_id, gcp_update_tag)
|
|
220
|
+
|
|
221
|
+
# Run cleanup
|
|
222
|
+
cleanup(neo4j_session, common_job_parameters)
|
|
@@ -6,6 +6,7 @@ from collections import namedtuple
|
|
|
6
6
|
|
|
7
7
|
import googleapiclient.discovery
|
|
8
8
|
import neo4j
|
|
9
|
+
from google.auth import default
|
|
9
10
|
from google.auth.exceptions import DefaultCredentialsError
|
|
10
11
|
from google.auth.transport.requests import Request
|
|
11
12
|
from google.oauth2 import credentials
|
|
@@ -18,10 +19,11 @@ from cartography.config import Config
|
|
|
18
19
|
from cartography.intel.gsuite import api
|
|
19
20
|
from cartography.util import timeit
|
|
20
21
|
|
|
21
|
-
|
|
22
|
+
OAUTH_SCOPES = [
|
|
22
23
|
'https://www.googleapis.com/auth/admin.directory.user.readonly',
|
|
23
24
|
'https://www.googleapis.com/auth/admin.directory.group.readonly',
|
|
24
25
|
'https://www.googleapis.com/auth/admin.directory.group.member',
|
|
26
|
+
'https://www.googleapis.com/auth/cloud-platform',
|
|
25
27
|
]
|
|
26
28
|
|
|
27
29
|
logger = logging.getLogger(__name__)
|
|
@@ -70,7 +72,7 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
|
|
|
70
72
|
try:
|
|
71
73
|
creds = service_account.Credentials.from_service_account_file(
|
|
72
74
|
config.gsuite_config,
|
|
73
|
-
scopes=
|
|
75
|
+
scopes=OAUTH_SCOPES,
|
|
74
76
|
)
|
|
75
77
|
creds = creds.with_subject(os.environ.get('GSUITE_DELEGATED_ADMIN'))
|
|
76
78
|
|
|
@@ -96,10 +98,10 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
|
|
|
96
98
|
refresh_token=auth_tokens['refresh_token'],
|
|
97
99
|
expiry=None,
|
|
98
100
|
token_uri=auth_tokens['token_uri'],
|
|
99
|
-
scopes=
|
|
101
|
+
scopes=OAUTH_SCOPES,
|
|
100
102
|
)
|
|
101
103
|
creds.refresh(Request())
|
|
102
|
-
creds = creds.create_scoped(
|
|
104
|
+
creds = creds.create_scoped(OAUTH_SCOPES)
|
|
103
105
|
except DefaultCredentialsError as e:
|
|
104
106
|
logger.error(
|
|
105
107
|
(
|
|
@@ -111,6 +113,21 @@ def start_gsuite_ingestion(neo4j_session: neo4j.Session, config: Config) -> None
|
|
|
111
113
|
e,
|
|
112
114
|
)
|
|
113
115
|
return
|
|
116
|
+
elif config.gsuite_auth_method == 'default':
|
|
117
|
+
logger.info('Attempting to authenticate to GSuite using default credentials')
|
|
118
|
+
try:
|
|
119
|
+
creds, _ = default(scopes=OAUTH_SCOPES)
|
|
120
|
+
except DefaultCredentialsError as e:
|
|
121
|
+
logger.error(
|
|
122
|
+
(
|
|
123
|
+
"Unable to initialize GSuite creds using default credentials. If you don't have GSuite data or "
|
|
124
|
+
"don't want to load GSuite data then you can ignore this message. Otherwise, the error code is: %s "
|
|
125
|
+
"Make sure you have valid application default credentials configured. "
|
|
126
|
+
"For more details see README"
|
|
127
|
+
),
|
|
128
|
+
e,
|
|
129
|
+
)
|
|
130
|
+
return
|
|
114
131
|
|
|
115
132
|
resources = _initialize_resources(creds)
|
|
116
133
|
api.sync_gsuite_users(neo4j_session, resources.admin, config.update_tag, common_job_parameters)
|
cartography/intel/gsuite/api.py
CHANGED
|
@@ -4,6 +4,7 @@ from typing import List
|
|
|
4
4
|
|
|
5
5
|
import neo4j
|
|
6
6
|
from googleapiclient.discovery import Resource
|
|
7
|
+
from googleapiclient.errors import HttpError
|
|
7
8
|
|
|
8
9
|
from cartography.util import run_cleanup_job
|
|
9
10
|
from cartography.util import timeit
|
|
@@ -30,9 +31,21 @@ def get_all_groups(admin: Resource) -> List[Dict]:
|
|
|
30
31
|
request = admin.groups().list(customer='my_customer', maxResults=20, orderBy='email')
|
|
31
32
|
response_objects = []
|
|
32
33
|
while request is not None:
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
try:
|
|
35
|
+
resp = request.execute(num_retries=GOOGLE_API_NUM_RETRIES)
|
|
36
|
+
response_objects.append(resp)
|
|
37
|
+
request = admin.groups().list_next(request, resp)
|
|
38
|
+
except HttpError as e:
|
|
39
|
+
if e.resp.status == 403 and "Request had insufficient authentication scopes" in str(e):
|
|
40
|
+
logger.error(
|
|
41
|
+
"Missing required GSuite scopes. If using the gcloud CLI, ",
|
|
42
|
+
"run: gcloud auth application-default login --scopes="
|
|
43
|
+
'"https://www.googleapis.com/auth/admin.directory.user.readonly,'
|
|
44
|
+
'https://www.googleapis.com/auth/admin.directory.group.readonly,'
|
|
45
|
+
'https://www.googleapis.com/auth/admin.directory.group.member.readonly,'
|
|
46
|
+
'https://www.googleapis.com/auth/cloud-platform"',
|
|
47
|
+
)
|
|
48
|
+
raise
|
|
36
49
|
return response_objects
|
|
37
50
|
|
|
38
51
|
|
|
@@ -140,6 +153,7 @@ def load_gsuite_groups(neo4j_session: neo4j.Session, groups: List[Dict], gsuite_
|
|
|
140
153
|
g.etag = group.etag,
|
|
141
154
|
g.kind = group.kind,
|
|
142
155
|
g.name = group.name,
|
|
156
|
+
g:GCPPrincipal,
|
|
143
157
|
g.lastupdated = $UpdateTag
|
|
144
158
|
"""
|
|
145
159
|
logger.info(f'Ingesting {len(groups)} gsuite groups')
|
|
@@ -179,6 +193,7 @@ def load_gsuite_users(neo4j_session: neo4j.Session, users: List[Dict], gsuite_up
|
|
|
179
193
|
u.suspended = user.suspended,
|
|
180
194
|
u.thumbnail_photo_etag = user.thumbnailPhotoEtag,
|
|
181
195
|
u.thumbnail_photo_url = user.thumbnailPhotoUrl,
|
|
196
|
+
u:GCPPrincipal,
|
|
182
197
|
u.lastupdated = $UpdateTag
|
|
183
198
|
"""
|
|
184
199
|
logger.info(f'Ingesting {len(users)} gsuite users')
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
|
|
4
|
+
from cartography.models.core.common import PropertyRef
|
|
5
|
+
from cartography.models.core.nodes import CartographyNodeProperties
|
|
6
|
+
from cartography.models.core.nodes import CartographyNodeSchema
|
|
7
|
+
from cartography.models.core.relationships import CartographyRelProperties
|
|
8
|
+
from cartography.models.core.relationships import CartographyRelSchema
|
|
9
|
+
from cartography.models.core.relationships import LinkDirection
|
|
10
|
+
from cartography.models.core.relationships import make_target_node_matcher
|
|
11
|
+
from cartography.models.core.relationships import TargetNodeMatcher
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class GCPServiceAccountNodeProperties(CartographyNodeProperties):
|
|
18
|
+
id: PropertyRef = PropertyRef('id', extra_index=True)
|
|
19
|
+
email: PropertyRef = PropertyRef('email', extra_index=True)
|
|
20
|
+
display_name: PropertyRef = PropertyRef('displayName')
|
|
21
|
+
oauth2_client_id: PropertyRef = PropertyRef('oauth2ClientId')
|
|
22
|
+
unique_id: PropertyRef = PropertyRef('uniqueId')
|
|
23
|
+
disabled: PropertyRef = PropertyRef('disabled')
|
|
24
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
25
|
+
project_id: PropertyRef = PropertyRef('projectId', set_in_kwargs=True)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class GCPRoleNodeProperties(CartographyNodeProperties):
|
|
30
|
+
id: PropertyRef = PropertyRef('name', extra_index=True)
|
|
31
|
+
name: PropertyRef = PropertyRef('name', extra_index=True)
|
|
32
|
+
title: PropertyRef = PropertyRef('title')
|
|
33
|
+
description: PropertyRef = PropertyRef('description')
|
|
34
|
+
deleted: PropertyRef = PropertyRef('deleted')
|
|
35
|
+
etag: PropertyRef = PropertyRef('etag')
|
|
36
|
+
permissions: PropertyRef = PropertyRef('includedPermissions')
|
|
37
|
+
role_type: PropertyRef = PropertyRef('roleType')
|
|
38
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
39
|
+
project_id: PropertyRef = PropertyRef('projectId', set_in_kwargs=True)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class GCPIAMToProjectRelProperties(CartographyRelProperties):
|
|
44
|
+
lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
# (:GCPUser|GCPServiceAccount|GCPRole)<-[:RESOURCE]-(:GCPProject)
|
|
49
|
+
class GCPPrincipalToProject(CartographyRelSchema):
|
|
50
|
+
target_node_label: str = 'GCPProject'
|
|
51
|
+
target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
|
|
52
|
+
{'id': PropertyRef('projectId', set_in_kwargs=True)},
|
|
53
|
+
)
|
|
54
|
+
direction: LinkDirection = LinkDirection.INWARD
|
|
55
|
+
rel_label: str = "RESOURCE"
|
|
56
|
+
properties: GCPIAMToProjectRelProperties = GCPIAMToProjectRelProperties()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class GCPServiceAccountSchema(CartographyNodeSchema):
|
|
61
|
+
label: str = 'GCPServiceAccount'
|
|
62
|
+
properties: GCPServiceAccountNodeProperties = GCPServiceAccountNodeProperties()
|
|
63
|
+
sub_resource_relationship: GCPPrincipalToProject = GCPPrincipalToProject()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class GCPRoleSchema(CartographyNodeSchema):
|
|
68
|
+
label: str = 'GCPRole'
|
|
69
|
+
properties: GCPRoleNodeProperties = GCPRoleNodeProperties()
|
|
70
|
+
sub_resource_relationship: GCPPrincipalToProject = GCPPrincipalToProject()
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
cartography/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
2
|
cartography/__main__.py,sha256=JftXT_nUPkqcEh8uxCCT4n-OyHYqbldEgrDS-4ygy0U,101
|
|
3
|
-
cartography/_version.py,sha256=
|
|
4
|
-
cartography/cli.py,sha256
|
|
3
|
+
cartography/_version.py,sha256=2FDGHtBkuReawM6vCrLyVJwLyBFc1Fmf92-cRLEKxt0,518
|
|
4
|
+
cartography/cli.py,sha256=-77DOKUQn3N-TDIi55V4RHLb3k36ZGZ64o1XgiT0qmE,33370
|
|
5
5
|
cartography/config.py,sha256=ZcadsKmooAkti9Kv0eDl8Ec1PcZDu3lWobtNaCnwY3k,11872
|
|
6
6
|
cartography/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
7
|
cartography/stats.py,sha256=dbybb9V2FuvSuHjjNwz6Vjwnd1hap2C7h960rLoKcl8,4406
|
|
@@ -144,7 +144,7 @@ cartography/intel/aws/__init__.py,sha256=siRhVNypfGxMPNIXHSjfYbLX9qlB6RYyN6aWX64
|
|
|
144
144
|
cartography/intel/aws/apigateway.py,sha256=w4QdxlwnVegKKsZFfoI36i-FGlIUjzxzaayZyz9oQvM,11654
|
|
145
145
|
cartography/intel/aws/config.py,sha256=wrZbz7bc8vImLmRvTLkPcWnjjPzk3tOG4bB_BFS2lq8,7329
|
|
146
146
|
cartography/intel/aws/dynamodb.py,sha256=LZ6LGNThLi0zC3eLMq2JN3mwiSwZeaH58YQQHvsXMGE,5013
|
|
147
|
-
cartography/intel/aws/ecr.py,sha256=
|
|
147
|
+
cartography/intel/aws/ecr.py,sha256=Tkm_H2cJ2wW24FJE3rBsT-Rli2FU9gNEGo6eEk7jBFM,7124
|
|
148
148
|
cartography/intel/aws/ecs.py,sha256=gulrIZ--iEFLpkkPH58MJIkctsxWeWdO2ofM9amDNZA,23654
|
|
149
149
|
cartography/intel/aws/eks.py,sha256=OerAX7qT2uGPbqliPvuy8JZUIgle_KMlnkkHxk8O5fk,3546
|
|
150
150
|
cartography/intel/aws/elasticache.py,sha256=fCI47aDFmIDyE26GiReKYb6XIZUwrzcvsXBQ4ruFhuI,4427
|
|
@@ -220,19 +220,20 @@ cartography/intel/duo/phones.py,sha256=ueJheqSLD2xYcMus5eOiixPYS3_xVjgQzeomjV2a6
|
|
|
220
220
|
cartography/intel/duo/tokens.py,sha256=bEEnjfc4waQnkRHVSnZLAeGE8wHOOZL7FA9m80GGQdQ,2396
|
|
221
221
|
cartography/intel/duo/users.py,sha256=lc7ly_XKeUjJ50szw31WT_GiCrZfGKJv1zVUpmTchh4,4097
|
|
222
222
|
cartography/intel/duo/web_authn_credentials.py,sha256=IbDf3CWqfEyI7f9zJugUvoDd6vZOECfb_7ANZaRYzuk,2636
|
|
223
|
-
cartography/intel/gcp/__init__.py,sha256=
|
|
223
|
+
cartography/intel/gcp/__init__.py,sha256=csFnE0_lznp3UfJZZSCCWF0gzdf_qz7sXuMzzZs_NvY,16516
|
|
224
224
|
cartography/intel/gcp/compute.py,sha256=CH2cBdOwbLZCAbkfRJkkI-sFybXVKRWEUGDJANQmvyA,48333
|
|
225
225
|
cartography/intel/gcp/crm.py,sha256=Uw5PILhVFhpM8gq7uu2v7F_YikDW3gsTZ3d7-e8Z1_k,12324
|
|
226
226
|
cartography/intel/gcp/dns.py,sha256=y2pvbmV04cnrMyuu_nbW3oc7uwHX6yEzn1n7veCsjmk,7748
|
|
227
227
|
cartography/intel/gcp/gke.py,sha256=qaTwsVaxkwNhW5_Mw4bedOk7fgJK8y0LwwcYlUABXDg,7966
|
|
228
|
+
cartography/intel/gcp/iam.py,sha256=S2IHbq2YwtQs51EqcpoRPpWpB6PcVGI06GiBjdyYvuo,7482
|
|
228
229
|
cartography/intel/gcp/storage.py,sha256=oO_ayEhkXlj2Gn7T5MU41ZXiqwRwe6Ud4wzqyRTsyf4,9075
|
|
229
230
|
cartography/intel/github/__init__.py,sha256=y876JJGTDJZEOFCDiNCJfcLNxN24pVj4s2N0YmuuoaE,1914
|
|
230
231
|
cartography/intel/github/repos.py,sha256=MmpxZASDJFQxDeSMxX3pZcpxCHFPos4_uYC_cX9KjOg,29865
|
|
231
232
|
cartography/intel/github/teams.py,sha256=AltQSmBHHmyzBtnRkez9Bo5yChEKBSt3wwzhGcfqmX4,14180
|
|
232
233
|
cartography/intel/github/users.py,sha256=MCLE0V0UCzQm3k3KmrNe6PYkI6usRQZYy2rCN3mT8o0,8948
|
|
233
234
|
cartography/intel/github/util.py,sha256=K0cXOPuhnGvN-aqcSUBO3vTuKQLjufVal9kn2HwOpbo,8110
|
|
234
|
-
cartography/intel/gsuite/__init__.py,sha256=
|
|
235
|
-
cartography/intel/gsuite/api.py,sha256=
|
|
235
|
+
cartography/intel/gsuite/__init__.py,sha256=FZS4WVCjOCvGqrq2E-yhZ_dtViQSy1srs_YEohibpWU,5466
|
|
236
|
+
cartography/intel/gsuite/api.py,sha256=bx0mPn6x6fgssxgm343NHdwjbtFkO6SZTucOsoW0Hgk,11143
|
|
236
237
|
cartography/intel/jamf/__init__.py,sha256=Nof-LrUeevoieo6oP2GyfTwx8k5TUIgreW6hSj53YjQ,419
|
|
237
238
|
cartography/intel/jamf/computers.py,sha256=EfjlupQ-9HYTjOrmuwrGuJDy9ApAnJvk8WrYcp6_Jkk,1673
|
|
238
239
|
cartography/intel/jamf/util.py,sha256=EAyP8VpOY2uAvW3HtX6r7qORNjGa1Tr3fuqezuLQ0j4,1017
|
|
@@ -337,6 +338,7 @@ cartography/models/duo/phone.py,sha256=oxgMmwKLRiCWbAhqrTKE4ILseu0j96GugEIV_hchR
|
|
|
337
338
|
cartography/models/duo/token.py,sha256=BS_AvF-TAGzCY9Owtqxr8g_s6716dnzFOO1IwkckmVA,2668
|
|
338
339
|
cartography/models/duo/user.py,sha256=ih3DH_QveAve4cX9dmIwC5gVN6_RNnuLK3bfJ5I9u6g,6554
|
|
339
340
|
cartography/models/duo/web_authn_credential.py,sha256=OcZnfG5zCMlphxSltRcAXQ12hHYJjxrBt6A9L28g7Vk,2920
|
|
341
|
+
cartography/models/gcp/iam.py,sha256=N7OGmnRlkIFZOv0rh3QGGBmYV7WYy3-xeE4Wv7StGOE,3071
|
|
340
342
|
cartography/models/github/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
341
343
|
cartography/models/github/orgs.py,sha256=EcUmkeyoCJmkmzLsfKdUwwTE0N2IIwyaUrIK32dQybo,1106
|
|
342
344
|
cartography/models/github/teams.py,sha256=qFyFAKKsiiHqFZkMM7Fd9My16dgXgylcFy3BbXHhzng,6069
|
|
@@ -356,9 +358,9 @@ cartography/models/snipeit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
|
|
|
356
358
|
cartography/models/snipeit/asset.py,sha256=FyRAaeXuZjMy0eUQcSDFcgEAF5lbLMlvqp1Tv9d3Lv4,3238
|
|
357
359
|
cartography/models/snipeit/tenant.py,sha256=p4rFnpNNuF1W5ilGBbexDaETWTwavfb38RcQGoImkQI,679
|
|
358
360
|
cartography/models/snipeit/user.py,sha256=MsB4MiCVNTH6JpESime7cOkB89autZOXQpL6Z0l7L6o,2113
|
|
359
|
-
cartography-0.100.
|
|
360
|
-
cartography-0.100.
|
|
361
|
-
cartography-0.100.
|
|
362
|
-
cartography-0.100.
|
|
363
|
-
cartography-0.100.
|
|
364
|
-
cartography-0.100.
|
|
361
|
+
cartography-0.100.0rc2.dist-info/LICENSE,sha256=kvLEBRYaQ1RvUni6y7Ti9uHeooqnjPoo6n_-0JO1ETc,11351
|
|
362
|
+
cartography-0.100.0rc2.dist-info/METADATA,sha256=EvL6jAQNNIVLpRwiNxSRZUANFrXX6RORGzbJk4U4Et0,11860
|
|
363
|
+
cartography-0.100.0rc2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
|
|
364
|
+
cartography-0.100.0rc2.dist-info/entry_points.txt,sha256=GVIAWD0o0_K077qMA_k1oZU4v-M0a8GLKGJR8tZ-qH8,112
|
|
365
|
+
cartography-0.100.0rc2.dist-info/top_level.txt,sha256=BHqsNJQiI6Q72DeypC1IINQJE59SLhU4nllbQjgJi9g,12
|
|
366
|
+
cartography-0.100.0rc2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|