cartography 0.102.0rc2__py3-none-any.whl → 0.103.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cartography might be problematic. Click here for more details.
- cartography/__main__.py +1 -2
- cartography/_version.py +2 -2
- cartography/cli.py +302 -253
- cartography/client/core/tx.py +39 -18
- cartography/config.py +4 -0
- cartography/driftdetect/__main__.py +1 -2
- cartography/driftdetect/add_shortcut.py +10 -2
- cartography/driftdetect/cli.py +71 -75
- cartography/driftdetect/detect_deviations.py +7 -3
- cartography/driftdetect/get_states.py +20 -8
- cartography/driftdetect/model.py +5 -5
- cartography/driftdetect/serializers.py +8 -6
- cartography/driftdetect/storage.py +2 -2
- cartography/graph/cleanupbuilder.py +35 -15
- cartography/graph/job.py +46 -17
- cartography/graph/querybuilder.py +165 -80
- cartography/graph/statement.py +35 -26
- cartography/intel/analysis.py +4 -1
- cartography/intel/aws/__init__.py +114 -55
- cartography/intel/aws/apigateway.py +134 -63
- cartography/intel/aws/cloudtrail.py +127 -0
- cartography/intel/aws/config.py +56 -20
- cartography/intel/aws/dynamodb.py +108 -40
- cartography/intel/aws/ec2/__init__.py +2 -2
- cartography/intel/aws/ec2/auto_scaling_groups.py +181 -78
- cartography/intel/aws/ec2/elastic_ip_addresses.py +41 -13
- cartography/intel/aws/ec2/images.py +49 -20
- cartography/intel/aws/ec2/instances.py +234 -136
- cartography/intel/aws/ec2/internet_gateways.py +40 -11
- cartography/intel/aws/ec2/key_pairs.py +44 -20
- cartography/intel/aws/ec2/launch_templates.py +101 -59
- cartography/intel/aws/ec2/load_balancer_v2s.py +104 -39
- cartography/intel/aws/ec2/load_balancers.py +82 -42
- cartography/intel/aws/ec2/network_acls.py +89 -65
- cartography/intel/aws/ec2/network_interfaces.py +146 -87
- cartography/intel/aws/ec2/reserved_instances.py +45 -16
- cartography/intel/aws/ec2/route_tables.py +138 -98
- cartography/intel/aws/ec2/security_groups.py +71 -21
- cartography/intel/aws/ec2/snapshots.py +61 -22
- cartography/intel/aws/ec2/subnets.py +54 -18
- cartography/intel/aws/ec2/tgw.py +100 -34
- cartography/intel/aws/ec2/util.py +1 -1
- cartography/intel/aws/ec2/volumes.py +69 -41
- cartography/intel/aws/ec2/vpc.py +37 -12
- cartography/intel/aws/ec2/vpc_peerings.py +83 -24
- cartography/intel/aws/ecr.py +88 -32
- cartography/intel/aws/ecs.py +83 -47
- cartography/intel/aws/eks.py +55 -29
- cartography/intel/aws/elasticache.py +42 -18
- cartography/intel/aws/elasticsearch.py +57 -20
- cartography/intel/aws/emr.py +61 -23
- cartography/intel/aws/iam.py +401 -145
- cartography/intel/aws/iam_instance_profiles.py +22 -22
- cartography/intel/aws/identitycenter.py +71 -37
- cartography/intel/aws/inspector.py +159 -89
- cartography/intel/aws/kms.py +92 -38
- cartography/intel/aws/lambda_function.py +103 -34
- cartography/intel/aws/organizations.py +30 -10
- cartography/intel/aws/permission_relationships.py +133 -51
- cartography/intel/aws/rds.py +249 -85
- cartography/intel/aws/redshift.py +107 -46
- cartography/intel/aws/resourcegroupstaggingapi.py +120 -66
- cartography/intel/aws/resources.py +53 -46
- cartography/intel/aws/route53.py +108 -61
- cartography/intel/aws/s3.py +168 -83
- cartography/intel/aws/s3accountpublicaccessblock.py +157 -0
- cartography/intel/aws/secretsmanager.py +24 -12
- cartography/intel/aws/securityhub.py +20 -9
- cartography/intel/aws/sns.py +166 -0
- cartography/intel/aws/sqs.py +60 -28
- cartography/intel/aws/ssm.py +70 -30
- cartography/intel/aws/util/arns.py +7 -7
- cartography/intel/aws/util/common.py +31 -4
- cartography/intel/azure/__init__.py +78 -19
- cartography/intel/azure/compute.py +101 -27
- cartography/intel/azure/cosmosdb.py +496 -170
- cartography/intel/azure/sql.py +296 -105
- cartography/intel/azure/storage.py +322 -113
- cartography/intel/azure/subscription.py +39 -23
- cartography/intel/azure/tenant.py +13 -4
- cartography/intel/azure/util/credentials.py +95 -55
- cartography/intel/bigfix/__init__.py +2 -2
- cartography/intel/bigfix/computers.py +93 -65
- cartography/intel/create_indexes.py +3 -2
- cartography/intel/crowdstrike/__init__.py +11 -9
- cartography/intel/crowdstrike/endpoints.py +5 -1
- cartography/intel/crowdstrike/spotlight.py +8 -3
- cartography/intel/cve/__init__.py +46 -13
- cartography/intel/cve/feed.py +48 -12
- cartography/intel/digitalocean/__init__.py +22 -13
- cartography/intel/digitalocean/compute.py +75 -108
- cartography/intel/digitalocean/management.py +44 -80
- cartography/intel/digitalocean/platform.py +48 -43
- cartography/intel/dns.py +36 -10
- cartography/intel/duo/__init__.py +21 -16
- cartography/intel/duo/api_host.py +14 -9
- cartography/intel/duo/endpoints.py +50 -45
- cartography/intel/duo/groups.py +18 -14
- cartography/intel/duo/phones.py +37 -34
- cartography/intel/duo/tokens.py +26 -23
- cartography/intel/duo/users.py +54 -50
- cartography/intel/duo/web_authn_credentials.py +30 -25
- cartography/intel/entra/__init__.py +25 -7
- cartography/intel/entra/ou.py +112 -0
- cartography/intel/entra/users.py +69 -63
- cartography/intel/gcp/__init__.py +185 -49
- cartography/intel/gcp/compute.py +418 -231
- cartography/intel/gcp/crm.py +96 -43
- cartography/intel/gcp/dns.py +60 -19
- cartography/intel/gcp/gke.py +72 -38
- cartography/intel/gcp/iam.py +61 -41
- cartography/intel/gcp/storage.py +84 -55
- cartography/intel/github/__init__.py +13 -11
- cartography/intel/github/repos.py +270 -137
- cartography/intel/github/teams.py +170 -88
- cartography/intel/github/users.py +70 -39
- cartography/intel/github/util.py +36 -34
- cartography/intel/gsuite/__init__.py +47 -26
- cartography/intel/gsuite/api.py +73 -30
- cartography/intel/jamf/__init__.py +19 -1
- cartography/intel/jamf/computers.py +30 -7
- cartography/intel/jamf/util.py +7 -2
- cartography/intel/kandji/__init__.py +6 -3
- cartography/intel/kandji/devices.py +14 -8
- cartography/intel/kubernetes/namespaces.py +7 -4
- cartography/intel/kubernetes/pods.py +7 -4
- cartography/intel/kubernetes/services.py +8 -4
- cartography/intel/lastpass/__init__.py +2 -2
- cartography/intel/lastpass/users.py +23 -12
- cartography/intel/oci/__init__.py +44 -11
- cartography/intel/oci/iam.py +134 -38
- cartography/intel/oci/organizations.py +13 -6
- cartography/intel/oci/utils.py +43 -20
- cartography/intel/okta/__init__.py +66 -15
- cartography/intel/okta/applications.py +42 -20
- cartography/intel/okta/awssaml.py +93 -33
- cartography/intel/okta/factors.py +16 -4
- cartography/intel/okta/groups.py +56 -29
- cartography/intel/okta/organization.py +5 -1
- cartography/intel/okta/origins.py +6 -2
- cartography/intel/okta/roles.py +15 -5
- cartography/intel/okta/users.py +20 -8
- cartography/intel/okta/utils.py +6 -4
- cartography/intel/pagerduty/__init__.py +8 -7
- cartography/intel/pagerduty/escalation_policies.py +18 -6
- cartography/intel/pagerduty/schedules.py +12 -4
- cartography/intel/pagerduty/services.py +11 -4
- cartography/intel/pagerduty/teams.py +8 -3
- cartography/intel/pagerduty/users.py +3 -1
- cartography/intel/pagerduty/vendors.py +3 -1
- cartography/intel/semgrep/__init__.py +24 -6
- cartography/intel/semgrep/dependencies.py +50 -28
- cartography/intel/semgrep/deployment.py +3 -1
- cartography/intel/semgrep/findings.py +42 -18
- cartography/intel/snipeit/__init__.py +17 -3
- cartography/intel/snipeit/asset.py +12 -6
- cartography/intel/snipeit/user.py +8 -5
- cartography/intel/snipeit/util.py +9 -4
- cartography/models/aws/apigateway.py +21 -17
- cartography/models/aws/apigatewaycertificate.py +28 -22
- cartography/models/aws/apigatewayresource.py +28 -20
- cartography/models/aws/apigatewaystage.py +33 -25
- cartography/models/aws/cloudtrail/__init__.py +0 -0
- cartography/models/aws/cloudtrail/trail.py +61 -0
- cartography/models/aws/dynamodb/gsi.py +30 -22
- cartography/models/aws/dynamodb/tables.py +25 -17
- cartography/models/aws/ec2/auto_scaling_groups.py +102 -82
- cartography/models/aws/ec2/images.py +36 -34
- cartography/models/aws/ec2/instances.py +51 -45
- cartography/models/aws/ec2/keypair.py +21 -16
- cartography/models/aws/ec2/keypair_instance.py +28 -21
- cartography/models/aws/ec2/launch_configurations.py +30 -26
- cartography/models/aws/ec2/launch_template_versions.py +48 -38
- cartography/models/aws/ec2/launch_templates.py +21 -17
- cartography/models/aws/ec2/load_balancer_listeners.py +27 -23
- cartography/models/aws/ec2/load_balancers.py +47 -37
- cartography/models/aws/ec2/network_acl_rules.py +38 -30
- cartography/models/aws/ec2/network_acls.py +38 -29
- cartography/models/aws/ec2/networkinterface_instance.py +52 -39
- cartography/models/aws/ec2/networkinterfaces.py +53 -37
- cartography/models/aws/ec2/privateip_networkinterface.py +32 -22
- cartography/models/aws/ec2/reservations.py +18 -14
- cartography/models/aws/ec2/route_table_associations.py +44 -34
- cartography/models/aws/ec2/route_tables.py +50 -43
- cartography/models/aws/ec2/routes.py +45 -37
- cartography/models/aws/ec2/securitygroup_instance.py +29 -20
- cartography/models/aws/ec2/securitygroup_networkinterface.py +24 -15
- cartography/models/aws/ec2/subnet_instance.py +24 -19
- cartography/models/aws/ec2/subnet_networkinterface.py +40 -31
- cartography/models/aws/ec2/volumes.py +47 -40
- cartography/models/aws/eks/clusters.py +23 -21
- cartography/models/aws/emr.py +32 -30
- cartography/models/aws/iam/instanceprofile.py +33 -24
- cartography/models/aws/identitycenter/awsidentitycenter.py +18 -14
- cartography/models/aws/identitycenter/awspermissionset.py +37 -29
- cartography/models/aws/identitycenter/awsssouser.py +23 -21
- cartography/models/aws/inspector/findings.py +77 -65
- cartography/models/aws/inspector/packages.py +35 -29
- cartography/models/aws/s3/__init__.py +0 -0
- cartography/models/aws/s3/account_public_access_block.py +51 -0
- cartography/models/aws/sns/__init__.py +0 -0
- cartography/models/aws/sns/topic.py +50 -0
- cartography/models/aws/ssm/instance_information.py +51 -39
- cartography/models/aws/ssm/instance_patch.py +32 -26
- cartography/models/bigfix/bigfix_computer.py +42 -38
- cartography/models/bigfix/bigfix_root.py +3 -3
- cartography/models/core/common.py +12 -10
- cartography/models/core/nodes.py +5 -2
- cartography/models/core/relationships.py +14 -6
- cartography/models/crowdstrike/hosts.py +37 -35
- cartography/models/cve/cve.py +34 -32
- cartography/models/cve/cve_feed.py +6 -6
- cartography/models/digitalocean/__init__.py +0 -0
- cartography/models/digitalocean/account.py +21 -0
- cartography/models/digitalocean/droplet.py +56 -0
- cartography/models/digitalocean/project.py +48 -0
- cartography/models/duo/api_host.py +3 -3
- cartography/models/duo/endpoint.py +43 -41
- cartography/models/duo/group.py +14 -14
- cartography/models/duo/phone.py +27 -27
- cartography/models/duo/token.py +16 -16
- cartography/models/duo/user.py +46 -44
- cartography/models/duo/web_authn_credential.py +27 -19
- cartography/models/entra/ou.py +48 -0
- cartography/models/entra/tenant.py +24 -18
- cartography/models/entra/user.py +64 -48
- cartography/models/gcp/iam.py +23 -23
- cartography/models/github/orgs.py +5 -4
- cartography/models/github/teams.py +37 -31
- cartography/models/github/users.py +34 -23
- cartography/models/kandji/device.py +22 -16
- cartography/models/kandji/tenant.py +6 -4
- cartography/models/lastpass/tenant.py +3 -3
- cartography/models/lastpass/user.py +32 -28
- cartography/models/semgrep/dependencies.py +36 -24
- cartography/models/semgrep/deployment.py +5 -5
- cartography/models/semgrep/findings.py +58 -42
- cartography/models/semgrep/locations.py +27 -21
- cartography/models/snipeit/asset.py +30 -21
- cartography/models/snipeit/tenant.py +6 -4
- cartography/models/snipeit/user.py +19 -12
- cartography/stats.py +3 -3
- cartography/sync.py +107 -31
- cartography/util.py +84 -62
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/METADATA +3 -14
- cartography-0.103.0rc1.dist-info/RECORD +396 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/WHEEL +1 -1
- cartography-0.102.0rc2.dist-info/RECORD +0 -381
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/entry_points.txt +0 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/licenses/LICENSE +0 -0
- {cartography-0.102.0rc2.dist-info → cartography-0.103.0rc1.dist-info}/top_level.txt +0 -0
cartography/intel/gcp/compute.py
CHANGED
|
@@ -18,7 +18,7 @@ from cartography.util import run_cleanup_job
|
|
|
18
18
|
from cartography.util import timeit
|
|
19
19
|
|
|
20
20
|
logger = logging.getLogger(__name__)
|
|
21
|
-
InstanceUriPrefix = namedtuple(
|
|
21
|
+
InstanceUriPrefix = namedtuple("InstanceUriPrefix", "zone_name project_id")
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def _get_error_reason(http_error: HttpError) -> str:
|
|
@@ -33,19 +33,23 @@ def _get_error_reason(http_error: HttpError) -> str:
|
|
|
33
33
|
:return: The error reason as a string
|
|
34
34
|
"""
|
|
35
35
|
try:
|
|
36
|
-
data = json.loads(http_error.content.decode(
|
|
36
|
+
data = json.loads(http_error.content.decode("utf-8"))
|
|
37
37
|
if isinstance(data, dict):
|
|
38
|
-
reason = data[
|
|
38
|
+
reason = data["error"]["errors"][0]["reason"]
|
|
39
39
|
else:
|
|
40
|
-
reason = data[0][
|
|
40
|
+
reason = data[0]["error"]["errors"]["reason"]
|
|
41
41
|
except (UnicodeDecodeError, ValueError, KeyError):
|
|
42
42
|
logger.warning(f"HttpError: {data}")
|
|
43
|
-
return
|
|
43
|
+
return ""
|
|
44
44
|
return reason
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
@timeit
|
|
48
|
-
def get_zones_in_project(
|
|
48
|
+
def get_zones_in_project(
|
|
49
|
+
project_id: str,
|
|
50
|
+
compute: Resource,
|
|
51
|
+
max_results: Optional[int] = None,
|
|
52
|
+
) -> Optional[List[Dict]]:
|
|
49
53
|
"""
|
|
50
54
|
Return the zones where the Compute Engine API is enabled for the given project_id.
|
|
51
55
|
See https://cloud.google.com/compute/docs/reference/rest/v1/zones and
|
|
@@ -60,10 +64,10 @@ def get_zones_in_project(project_id: str, compute: Resource, max_results: Option
|
|
|
60
64
|
try:
|
|
61
65
|
req = compute.zones().list(project=project_id, maxResults=max_results)
|
|
62
66
|
res = req.execute()
|
|
63
|
-
return res[
|
|
67
|
+
return res["items"]
|
|
64
68
|
except HttpError as e:
|
|
65
69
|
reason = _get_error_reason(e)
|
|
66
|
-
if reason ==
|
|
70
|
+
if reason == "accessNotConfigured":
|
|
67
71
|
logger.info(
|
|
68
72
|
(
|
|
69
73
|
"Google Compute Engine API access is not configured for project %s; skipping. "
|
|
@@ -73,17 +77,14 @@ def get_zones_in_project(project_id: str, compute: Resource, max_results: Option
|
|
|
73
77
|
e,
|
|
74
78
|
)
|
|
75
79
|
return None
|
|
76
|
-
elif reason ==
|
|
80
|
+
elif reason == "notFound":
|
|
77
81
|
logger.info(
|
|
78
|
-
(
|
|
79
|
-
"Project %s returned a 404 not found error. "
|
|
80
|
-
"Full details: %s"
|
|
81
|
-
),
|
|
82
|
+
("Project %s returned a 404 not found error. " "Full details: %s"),
|
|
82
83
|
project_id,
|
|
83
84
|
e,
|
|
84
85
|
)
|
|
85
86
|
return None
|
|
86
|
-
elif reason ==
|
|
87
|
+
elif reason == "forbidden":
|
|
87
88
|
logger.info(
|
|
88
89
|
(
|
|
89
90
|
"Your GCP identity does not have the compute.zones.list permission for project %s; skipping "
|
|
@@ -98,7 +99,11 @@ def get_zones_in_project(project_id: str, compute: Resource, max_results: Option
|
|
|
98
99
|
|
|
99
100
|
|
|
100
101
|
@timeit
|
|
101
|
-
def get_gcp_instance_responses(
|
|
102
|
+
def get_gcp_instance_responses(
|
|
103
|
+
project_id: str,
|
|
104
|
+
zones: Optional[List[Dict]],
|
|
105
|
+
compute: Resource,
|
|
106
|
+
) -> List[Resource]:
|
|
102
107
|
"""
|
|
103
108
|
Return list of GCP instance response objects for a given project and list of zones
|
|
104
109
|
:param project_id: The project ID
|
|
@@ -111,7 +116,7 @@ def get_gcp_instance_responses(project_id: str, zones: Optional[List[Dict]], com
|
|
|
111
116
|
return []
|
|
112
117
|
response_objects: List[Resource] = []
|
|
113
118
|
for zone in zones:
|
|
114
|
-
req = compute.instances().list(project=project_id, zone=zone[
|
|
119
|
+
req = compute.instances().list(project=project_id, zone=zone["name"])
|
|
115
120
|
res = req.execute()
|
|
116
121
|
response_objects.append(res)
|
|
117
122
|
return response_objects
|
|
@@ -143,7 +148,11 @@ def get_gcp_vpcs(projectid: str, compute: Resource) -> Resource:
|
|
|
143
148
|
|
|
144
149
|
|
|
145
150
|
@timeit
|
|
146
|
-
def get_gcp_regional_forwarding_rules(
|
|
151
|
+
def get_gcp_regional_forwarding_rules(
|
|
152
|
+
project_id: str,
|
|
153
|
+
region: str,
|
|
154
|
+
compute: Resource,
|
|
155
|
+
) -> Resource:
|
|
147
156
|
"""
|
|
148
157
|
Return list of all regional forwarding rules in the given project_id and region
|
|
149
158
|
:param project_id: The project ID
|
|
@@ -189,17 +198,21 @@ def transform_gcp_instances(response_objects: List[Dict]) -> List[Dict]:
|
|
|
189
198
|
"""
|
|
190
199
|
instance_list = []
|
|
191
200
|
for res in response_objects:
|
|
192
|
-
prefix = res[
|
|
201
|
+
prefix = res["id"]
|
|
193
202
|
prefix_fields = _parse_instance_uri_prefix(prefix)
|
|
194
203
|
|
|
195
|
-
for instance in res.get(
|
|
196
|
-
instance[
|
|
197
|
-
instance[
|
|
198
|
-
instance[
|
|
204
|
+
for instance in res.get("items", []):
|
|
205
|
+
instance["partial_uri"] = f"{prefix}/{instance['name']}"
|
|
206
|
+
instance["project_id"] = prefix_fields.project_id
|
|
207
|
+
instance["zone_name"] = prefix_fields.zone_name
|
|
199
208
|
|
|
200
|
-
for nic in instance.get(
|
|
201
|
-
nic[
|
|
202
|
-
|
|
209
|
+
for nic in instance.get("networkInterfaces", []):
|
|
210
|
+
nic["subnet_partial_uri"] = _parse_compute_full_uri_to_partial_uri(
|
|
211
|
+
nic["subnetwork"],
|
|
212
|
+
)
|
|
213
|
+
nic["vpc_partial_uri"] = _parse_compute_full_uri_to_partial_uri(
|
|
214
|
+
nic["network"],
|
|
215
|
+
)
|
|
203
216
|
|
|
204
217
|
instance_list.append(instance)
|
|
205
218
|
return instance_list
|
|
@@ -211,7 +224,7 @@ def _parse_instance_uri_prefix(prefix: str) -> InstanceUriPrefix:
|
|
|
211
224
|
:param prefix: String of the form `projects/{project}/zones/{zone}/instances`
|
|
212
225
|
:return: namedtuple with fields project_id and zone_name
|
|
213
226
|
"""
|
|
214
|
-
split_list = prefix.split(
|
|
227
|
+
split_list = prefix.split("/")
|
|
215
228
|
|
|
216
229
|
return InstanceUriPrefix(
|
|
217
230
|
project_id=split_list[1],
|
|
@@ -219,7 +232,7 @@ def _parse_instance_uri_prefix(prefix: str) -> InstanceUriPrefix:
|
|
|
219
232
|
)
|
|
220
233
|
|
|
221
234
|
|
|
222
|
-
def _parse_compute_full_uri_to_partial_uri(full_uri: str, version: str =
|
|
235
|
+
def _parse_compute_full_uri_to_partial_uri(full_uri: str, version: str = "v1") -> str:
|
|
223
236
|
"""
|
|
224
237
|
Take a GCP Compute object's self_link of the form
|
|
225
238
|
`https://www.googleapis.com/compute/{version}/projects/{project}/{location specifier}/{subtype}/{resource name}`
|
|
@@ -229,7 +242,7 @@ def _parse_compute_full_uri_to_partial_uri(full_uri: str, version: str = 'v1') -
|
|
|
229
242
|
:param version: The version number; default to v1 since at the time of this writing v1 is the only Compute API.
|
|
230
243
|
:return: Partial URI `{project}/{location specifier}/{subtype}/{resource name}`
|
|
231
244
|
"""
|
|
232
|
-
return full_uri.split(f
|
|
245
|
+
return full_uri.split(f"compute/{version}/")[1]
|
|
233
246
|
|
|
234
247
|
|
|
235
248
|
def _create_gcp_network_tag_id(vpc_partial_uri: str, tag: str) -> str:
|
|
@@ -251,19 +264,22 @@ def transform_gcp_vpcs(vpc_res: Dict) -> List[Dict]:
|
|
|
251
264
|
vpc_list = []
|
|
252
265
|
|
|
253
266
|
# prefix has the form `projects/{project ID}/global/networks`
|
|
254
|
-
prefix = vpc_res[
|
|
255
|
-
projectid = prefix.split(
|
|
256
|
-
for v in vpc_res.get(
|
|
267
|
+
prefix = vpc_res["id"]
|
|
268
|
+
projectid = prefix.split("/")[1]
|
|
269
|
+
for v in vpc_res.get("items", []):
|
|
257
270
|
vpc = {}
|
|
258
271
|
partial_uri = f"{prefix}/{v['name']}"
|
|
259
272
|
|
|
260
|
-
vpc[
|
|
261
|
-
vpc[
|
|
262
|
-
vpc[
|
|
263
|
-
vpc[
|
|
264
|
-
vpc[
|
|
265
|
-
vpc[
|
|
266
|
-
vpc[
|
|
273
|
+
vpc["partial_uri"] = partial_uri
|
|
274
|
+
vpc["name"] = v["name"]
|
|
275
|
+
vpc["self_link"] = v["selfLink"]
|
|
276
|
+
vpc["project_id"] = projectid
|
|
277
|
+
vpc["auto_create_subnetworks"] = v.get("autoCreateSubnetworks", None)
|
|
278
|
+
vpc["description"] = v.get("description", None)
|
|
279
|
+
vpc["routing_config_routing_mode"] = v.get("routingConfig", {}).get(
|
|
280
|
+
"routingMode",
|
|
281
|
+
None,
|
|
282
|
+
)
|
|
267
283
|
|
|
268
284
|
vpc_list.append(vpc)
|
|
269
285
|
return vpc_list
|
|
@@ -278,29 +294,29 @@ def transform_gcp_subnets(subnet_res: Dict) -> List[Dict]:
|
|
|
278
294
|
"""
|
|
279
295
|
# The `id` in the response object has the form `projects/{project}/regions/{region}/subnetworks`.
|
|
280
296
|
# We can include this in each subnet object in the list to form the partial_uri later on.
|
|
281
|
-
prefix = subnet_res[
|
|
282
|
-
projectid = prefix.split(
|
|
297
|
+
prefix = subnet_res["id"]
|
|
298
|
+
projectid = prefix.split("/")[1]
|
|
283
299
|
subnet_list: List[Dict] = []
|
|
284
|
-
for s in subnet_res.get(
|
|
300
|
+
for s in subnet_res.get("items", []):
|
|
285
301
|
subnet = {}
|
|
286
302
|
|
|
287
303
|
# Has the form `projects/{project}/regions/{region}/subnetworks/{subnet_name}`
|
|
288
304
|
partial_uri = f"{prefix}/{s['name']}"
|
|
289
|
-
subnet[
|
|
290
|
-
subnet[
|
|
305
|
+
subnet["id"] = partial_uri
|
|
306
|
+
subnet["partial_uri"] = partial_uri
|
|
291
307
|
|
|
292
308
|
# Let's maintain an on-node reference to the VPC that this subnet belongs to.
|
|
293
|
-
subnet[
|
|
294
|
-
subnet[
|
|
309
|
+
subnet["vpc_self_link"] = s["network"]
|
|
310
|
+
subnet["vpc_partial_uri"] = _parse_compute_full_uri_to_partial_uri(s["network"])
|
|
295
311
|
|
|
296
|
-
subnet[
|
|
297
|
-
subnet[
|
|
312
|
+
subnet["name"] = s["name"]
|
|
313
|
+
subnet["project_id"] = projectid
|
|
298
314
|
# Region looks like "https://www.googleapis.com/compute/v1/projects/{project}/regions/{region name}"
|
|
299
|
-
subnet[
|
|
300
|
-
subnet[
|
|
301
|
-
subnet[
|
|
302
|
-
subnet[
|
|
303
|
-
subnet[
|
|
315
|
+
subnet["region"] = s["region"].split("/")[-1]
|
|
316
|
+
subnet["gateway_address"] = s.get("gatewayAddress", None)
|
|
317
|
+
subnet["ip_cidr_range"] = s.get("ipCidrRange", None)
|
|
318
|
+
subnet["self_link"] = s["selfLink"]
|
|
319
|
+
subnet["private_ip_google_access"] = s.get("privateIpGoogleAccess", None)
|
|
304
320
|
|
|
305
321
|
subnet_list.append(subnet)
|
|
306
322
|
return subnet_list
|
|
@@ -314,43 +330,47 @@ def transform_gcp_forwarding_rules(fwd_response: Resource) -> List[Dict]:
|
|
|
314
330
|
:return: A transformed fwd_response
|
|
315
331
|
"""
|
|
316
332
|
fwd_list: List[Dict] = []
|
|
317
|
-
prefix = fwd_response[
|
|
318
|
-
project_id = prefix.split(
|
|
319
|
-
for fwd in fwd_response.get(
|
|
333
|
+
prefix = fwd_response["id"]
|
|
334
|
+
project_id = prefix.split("/")[1]
|
|
335
|
+
for fwd in fwd_response.get("items", []):
|
|
320
336
|
forwarding_rule: Dict[str, Any] = {}
|
|
321
337
|
|
|
322
338
|
fwd_partial_uri = f"{prefix}/{fwd['name']}"
|
|
323
|
-
forwarding_rule[
|
|
324
|
-
forwarding_rule[
|
|
339
|
+
forwarding_rule["id"] = fwd_partial_uri
|
|
340
|
+
forwarding_rule["partial_uri"] = fwd_partial_uri
|
|
325
341
|
|
|
326
|
-
forwarding_rule[
|
|
342
|
+
forwarding_rule["project_id"] = project_id
|
|
327
343
|
# Region looks like "https://www.googleapis.com/compute/v1/projects/{project}/regions/{region name}"
|
|
328
|
-
region = fwd.get(
|
|
329
|
-
forwarding_rule[
|
|
330
|
-
forwarding_rule[
|
|
331
|
-
forwarding_rule[
|
|
332
|
-
forwarding_rule[
|
|
333
|
-
|
|
334
|
-
forwarding_rule[
|
|
335
|
-
forwarding_rule[
|
|
336
|
-
forwarding_rule[
|
|
337
|
-
forwarding_rule[
|
|
338
|
-
forwarding_rule[
|
|
339
|
-
target = fwd.get(
|
|
344
|
+
region = fwd.get("region", None)
|
|
345
|
+
forwarding_rule["region"] = region.split("/")[-1] if region else None
|
|
346
|
+
forwarding_rule["ip_address"] = fwd.get("IPAddress", None)
|
|
347
|
+
forwarding_rule["ip_protocol"] = fwd.get("IPProtocol", None)
|
|
348
|
+
forwarding_rule["allow_global_access"] = fwd.get("allowGlobalAccess", None)
|
|
349
|
+
|
|
350
|
+
forwarding_rule["load_balancing_scheme"] = fwd.get("loadBalancingScheme", None)
|
|
351
|
+
forwarding_rule["name"] = fwd.get("name", None)
|
|
352
|
+
forwarding_rule["port_range"] = fwd.get("portRange", None)
|
|
353
|
+
forwarding_rule["ports"] = fwd.get("ports", None)
|
|
354
|
+
forwarding_rule["self_link"] = fwd.get("selfLink", None)
|
|
355
|
+
target = fwd.get("target", None)
|
|
340
356
|
if target:
|
|
341
|
-
forwarding_rule[
|
|
357
|
+
forwarding_rule["target"] = _parse_compute_full_uri_to_partial_uri(target)
|
|
342
358
|
else:
|
|
343
|
-
forwarding_rule[
|
|
359
|
+
forwarding_rule["target"] = None
|
|
344
360
|
|
|
345
|
-
network = fwd.get(
|
|
361
|
+
network = fwd.get("network", None)
|
|
346
362
|
if network:
|
|
347
|
-
forwarding_rule[
|
|
348
|
-
forwarding_rule[
|
|
363
|
+
forwarding_rule["network"] = network
|
|
364
|
+
forwarding_rule["network_partial_uri"] = (
|
|
365
|
+
_parse_compute_full_uri_to_partial_uri(network)
|
|
366
|
+
)
|
|
349
367
|
|
|
350
|
-
subnetwork = fwd.get(
|
|
368
|
+
subnetwork = fwd.get("subnetwork", None)
|
|
351
369
|
if subnetwork:
|
|
352
|
-
forwarding_rule[
|
|
353
|
-
forwarding_rule[
|
|
370
|
+
forwarding_rule["subnetwork"] = subnetwork
|
|
371
|
+
forwarding_rule["subnetwork_partial_uri"] = (
|
|
372
|
+
_parse_compute_full_uri_to_partial_uri(subnetwork)
|
|
373
|
+
)
|
|
354
374
|
|
|
355
375
|
fwd_list.append(forwarding_rule)
|
|
356
376
|
return fwd_list
|
|
@@ -365,31 +385,45 @@ def transform_gcp_firewall(fw_response: Resource) -> List[Dict]:
|
|
|
365
385
|
:return: List of transformed firewall rule objects.
|
|
366
386
|
"""
|
|
367
387
|
fw_list: List[Dict] = []
|
|
368
|
-
prefix = fw_response[
|
|
369
|
-
for fw in fw_response.get(
|
|
388
|
+
prefix = fw_response["id"]
|
|
389
|
+
for fw in fw_response.get("items", []):
|
|
370
390
|
fw_partial_uri = f"{prefix}/{fw['name']}"
|
|
371
|
-
fw[
|
|
372
|
-
fw[
|
|
391
|
+
fw["id"] = fw_partial_uri
|
|
392
|
+
fw["vpc_partial_uri"] = _parse_compute_full_uri_to_partial_uri(fw["network"])
|
|
373
393
|
|
|
374
|
-
fw[
|
|
375
|
-
fw[
|
|
394
|
+
fw["transformed_allow_list"] = []
|
|
395
|
+
fw["transformed_deny_list"] = []
|
|
376
396
|
# Mark whether this FW is defined on a target service account.
|
|
377
397
|
# In future we will need to ingest GCP IAM objects but for now we simply mark the presence of svc accounts here.
|
|
378
|
-
fw[
|
|
398
|
+
fw["has_target_service_accounts"] = (
|
|
399
|
+
True if "targetServiceAccounts" in fw else False
|
|
400
|
+
)
|
|
379
401
|
|
|
380
|
-
for allow_rule in fw.get(
|
|
381
|
-
transformed_allow_rules = _transform_fw_entry(
|
|
382
|
-
|
|
402
|
+
for allow_rule in fw.get("allowed", []):
|
|
403
|
+
transformed_allow_rules = _transform_fw_entry(
|
|
404
|
+
allow_rule,
|
|
405
|
+
fw_partial_uri,
|
|
406
|
+
is_allow_rule=True,
|
|
407
|
+
)
|
|
408
|
+
fw["transformed_allow_list"].extend(transformed_allow_rules)
|
|
383
409
|
|
|
384
|
-
for deny_rule in fw.get(
|
|
385
|
-
transformed_deny_rules = _transform_fw_entry(
|
|
386
|
-
|
|
410
|
+
for deny_rule in fw.get("denied", []):
|
|
411
|
+
transformed_deny_rules = _transform_fw_entry(
|
|
412
|
+
deny_rule,
|
|
413
|
+
fw_partial_uri,
|
|
414
|
+
is_allow_rule=False,
|
|
415
|
+
)
|
|
416
|
+
fw["transformed_deny_list"].extend(transformed_deny_rules)
|
|
387
417
|
|
|
388
418
|
fw_list.append(fw)
|
|
389
419
|
return fw_list
|
|
390
420
|
|
|
391
421
|
|
|
392
|
-
def _transform_fw_entry(
|
|
422
|
+
def _transform_fw_entry(
|
|
423
|
+
rule: Dict,
|
|
424
|
+
fw_partial_uri: str,
|
|
425
|
+
is_allow_rule: bool,
|
|
426
|
+
) -> List[Dict]:
|
|
393
427
|
"""
|
|
394
428
|
Takes a rule entry from a GCP firewall object's allow or deny list and converts it to a list of one or more
|
|
395
429
|
dicts representing a firewall rule for each port and port range. This format is easier to load into Neo4j.
|
|
@@ -415,21 +449,31 @@ def _transform_fw_entry(rule: Dict, fw_partial_uri: str, is_allow_rule: bool) ->
|
|
|
415
449
|
"""
|
|
416
450
|
result: List[Dict] = []
|
|
417
451
|
# rule['ruleid'] = f"{fw_partial_uri}/"
|
|
418
|
-
protocol = rule[
|
|
452
|
+
protocol = rule["IPProtocol"]
|
|
419
453
|
|
|
420
454
|
# If the protocol covered is TCP or UDP then we need to handle ports
|
|
421
|
-
if protocol ==
|
|
455
|
+
if protocol == "tcp" or protocol == "udp":
|
|
422
456
|
|
|
423
457
|
# If ports are specified then create rules for each port and range
|
|
424
|
-
if
|
|
425
|
-
for port in rule[
|
|
426
|
-
rule = _parse_port_string_to_rule(
|
|
458
|
+
if "ports" in rule:
|
|
459
|
+
for port in rule["ports"]:
|
|
460
|
+
rule = _parse_port_string_to_rule(
|
|
461
|
+
port,
|
|
462
|
+
protocol,
|
|
463
|
+
fw_partial_uri,
|
|
464
|
+
is_allow_rule,
|
|
465
|
+
)
|
|
427
466
|
result.append(rule)
|
|
428
467
|
return result
|
|
429
468
|
|
|
430
469
|
# If ports are not specified then the rule applies to every port
|
|
431
470
|
else:
|
|
432
|
-
rule = _parse_port_string_to_rule(
|
|
471
|
+
rule = _parse_port_string_to_rule(
|
|
472
|
+
"0-65535",
|
|
473
|
+
protocol,
|
|
474
|
+
fw_partial_uri,
|
|
475
|
+
is_allow_rule,
|
|
476
|
+
)
|
|
433
477
|
result.append(rule)
|
|
434
478
|
return result
|
|
435
479
|
|
|
@@ -440,7 +484,12 @@ def _transform_fw_entry(rule: Dict, fw_partial_uri: str, is_allow_rule: bool) ->
|
|
|
440
484
|
return result
|
|
441
485
|
|
|
442
486
|
|
|
443
|
-
def _parse_port_string_to_rule(
|
|
487
|
+
def _parse_port_string_to_rule(
|
|
488
|
+
port: Optional[str],
|
|
489
|
+
protocol: str,
|
|
490
|
+
fw_partial_uri: str,
|
|
491
|
+
is_allow_rule: bool,
|
|
492
|
+
) -> Dict:
|
|
444
493
|
"""
|
|
445
494
|
Takes a string argument representing a GCP firewall rule port or port range and returns a dict that is easier to
|
|
446
495
|
load into Neo4j.
|
|
@@ -467,13 +516,13 @@ def _parse_port_string_to_rule(port: Optional[str], protocol: str, fw_partial_ur
|
|
|
467
516
|
|
|
468
517
|
if port is None:
|
|
469
518
|
# Keep the port range as the empty string
|
|
470
|
-
port_range_str =
|
|
519
|
+
port_range_str = ""
|
|
471
520
|
fromport = None
|
|
472
521
|
toport = None
|
|
473
522
|
else:
|
|
474
523
|
# Case 1 - port range: '12345-12349'.split('-') => ['12345','12349'].
|
|
475
524
|
# Case 2 - single port: '22'.split('-') => ['22'].
|
|
476
|
-
port_split = port.split(
|
|
525
|
+
port_split = port.split("-")
|
|
477
526
|
|
|
478
527
|
# Port range
|
|
479
528
|
if len(port_split) == 2:
|
|
@@ -486,18 +535,22 @@ def _parse_port_string_to_rule(port: Optional[str], protocol: str, fw_partial_ur
|
|
|
486
535
|
fromport = int(port_split[0])
|
|
487
536
|
toport = int(port_split[0])
|
|
488
537
|
|
|
489
|
-
rule_type =
|
|
538
|
+
rule_type = "allow" if is_allow_rule else "deny"
|
|
490
539
|
|
|
491
540
|
return {
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
541
|
+
"ruleid": f"{fw_partial_uri}/{rule_type}/{port_range_str}{protocol}",
|
|
542
|
+
"fromport": fromport,
|
|
543
|
+
"toport": toport,
|
|
544
|
+
"protocol": protocol,
|
|
496
545
|
}
|
|
497
546
|
|
|
498
547
|
|
|
499
548
|
@timeit
|
|
500
|
-
def load_gcp_instances(
|
|
549
|
+
def load_gcp_instances(
|
|
550
|
+
neo4j_session: neo4j.Session,
|
|
551
|
+
data: List[Dict],
|
|
552
|
+
gcp_update_tag: int,
|
|
553
|
+
) -> None:
|
|
501
554
|
"""
|
|
502
555
|
Ingest GCP instance objects to Neo4j
|
|
503
556
|
:param neo4j_session: The Neo4j session object
|
|
@@ -530,22 +583,26 @@ def load_gcp_instances(neo4j_session: neo4j.Session, data: List[Dict], gcp_updat
|
|
|
530
583
|
for instance in data:
|
|
531
584
|
neo4j_session.run(
|
|
532
585
|
query,
|
|
533
|
-
ProjectId=instance[
|
|
534
|
-
PartialUri=instance[
|
|
535
|
-
SelfLink=instance[
|
|
536
|
-
InstanceName=instance[
|
|
537
|
-
ZoneName=instance[
|
|
538
|
-
Hostname=instance.get(
|
|
539
|
-
Status=instance[
|
|
586
|
+
ProjectId=instance["project_id"],
|
|
587
|
+
PartialUri=instance["partial_uri"],
|
|
588
|
+
SelfLink=instance["selfLink"],
|
|
589
|
+
InstanceName=instance["name"],
|
|
590
|
+
ZoneName=instance["zone_name"],
|
|
591
|
+
Hostname=instance.get("hostname", None),
|
|
592
|
+
Status=instance["status"],
|
|
540
593
|
gcp_update_tag=gcp_update_tag,
|
|
541
594
|
)
|
|
542
595
|
_attach_instance_tags(neo4j_session, instance, gcp_update_tag)
|
|
543
596
|
_attach_gcp_nics(neo4j_session, instance, gcp_update_tag)
|
|
544
|
-
_attach_gcp_vpc(neo4j_session, instance[
|
|
597
|
+
_attach_gcp_vpc(neo4j_session, instance["partial_uri"], gcp_update_tag)
|
|
545
598
|
|
|
546
599
|
|
|
547
600
|
@timeit
|
|
548
|
-
def load_gcp_vpcs(
|
|
601
|
+
def load_gcp_vpcs(
|
|
602
|
+
neo4j_session: neo4j.Session,
|
|
603
|
+
vpcs: List[Dict],
|
|
604
|
+
gcp_update_tag: int,
|
|
605
|
+
) -> None:
|
|
549
606
|
"""
|
|
550
607
|
Ingest VPCs to Neo4j
|
|
551
608
|
:param neo4j_session: The Neo4j session object
|
|
@@ -576,19 +633,23 @@ def load_gcp_vpcs(neo4j_session: neo4j.Session, vpcs: List[Dict], gcp_update_tag
|
|
|
576
633
|
for vpc in vpcs:
|
|
577
634
|
neo4j_session.run(
|
|
578
635
|
query,
|
|
579
|
-
ProjectId=vpc[
|
|
580
|
-
PartialUri=vpc[
|
|
581
|
-
SelfLink=vpc[
|
|
582
|
-
VpcName=vpc[
|
|
583
|
-
AutoCreateSubnetworks=vpc[
|
|
584
|
-
RoutingMode=vpc[
|
|
585
|
-
Description=vpc[
|
|
636
|
+
ProjectId=vpc["project_id"],
|
|
637
|
+
PartialUri=vpc["partial_uri"],
|
|
638
|
+
SelfLink=vpc["self_link"],
|
|
639
|
+
VpcName=vpc["name"],
|
|
640
|
+
AutoCreateSubnetworks=vpc["auto_create_subnetworks"],
|
|
641
|
+
RoutingMode=vpc["routing_config_routing_mode"],
|
|
642
|
+
Description=vpc["description"],
|
|
586
643
|
gcp_update_tag=gcp_update_tag,
|
|
587
644
|
)
|
|
588
645
|
|
|
589
646
|
|
|
590
647
|
@timeit
|
|
591
|
-
def load_gcp_subnets(
|
|
648
|
+
def load_gcp_subnets(
|
|
649
|
+
neo4j_session: neo4j.Session,
|
|
650
|
+
subnets: List[Dict],
|
|
651
|
+
gcp_update_tag: int,
|
|
652
|
+
) -> None:
|
|
592
653
|
"""
|
|
593
654
|
Ingest GCP subnet data to Neo4j
|
|
594
655
|
:param neo4j_session: The Neo4j session
|
|
@@ -621,22 +682,26 @@ def load_gcp_subnets(neo4j_session: neo4j.Session, subnets: List[Dict], gcp_upda
|
|
|
621
682
|
for s in subnets:
|
|
622
683
|
neo4j_session.run(
|
|
623
684
|
query,
|
|
624
|
-
VpcPartialUri=s[
|
|
625
|
-
VpcSelfLink=s[
|
|
626
|
-
PartialUri=s[
|
|
627
|
-
SubnetSelfLink=s[
|
|
628
|
-
ProjectId=s[
|
|
629
|
-
SubnetName=s[
|
|
630
|
-
Region=s[
|
|
631
|
-
GatewayAddress=s[
|
|
632
|
-
IpCidrRange=s[
|
|
633
|
-
PrivateIpGoogleAccess=s[
|
|
685
|
+
VpcPartialUri=s["vpc_partial_uri"],
|
|
686
|
+
VpcSelfLink=s["vpc_self_link"],
|
|
687
|
+
PartialUri=s["partial_uri"],
|
|
688
|
+
SubnetSelfLink=s["self_link"],
|
|
689
|
+
ProjectId=s["project_id"],
|
|
690
|
+
SubnetName=s["name"],
|
|
691
|
+
Region=s["region"],
|
|
692
|
+
GatewayAddress=s["gateway_address"],
|
|
693
|
+
IpCidrRange=s["ip_cidr_range"],
|
|
694
|
+
PrivateIpGoogleAccess=s["private_ip_google_access"],
|
|
634
695
|
gcp_update_tag=gcp_update_tag,
|
|
635
696
|
)
|
|
636
697
|
|
|
637
698
|
|
|
638
699
|
@timeit
|
|
639
|
-
def load_gcp_forwarding_rules(
|
|
700
|
+
def load_gcp_forwarding_rules(
|
|
701
|
+
neo4j_session: neo4j.Session,
|
|
702
|
+
fwd_rules: List[Dict],
|
|
703
|
+
gcp_update_tag: int,
|
|
704
|
+
) -> None:
|
|
640
705
|
"""
|
|
641
706
|
Ingest GCP forwarding rules data to Neo4j
|
|
642
707
|
:param neo4j_session: The Neo4j session
|
|
@@ -665,26 +730,26 @@ def load_gcp_forwarding_rules(neo4j_session: neo4j.Session, fwd_rules: List[Dict
|
|
|
665
730
|
"""
|
|
666
731
|
|
|
667
732
|
for fwd in fwd_rules:
|
|
668
|
-
network = fwd.get(
|
|
669
|
-
subnetwork = fwd.get(
|
|
733
|
+
network = fwd.get("network", None)
|
|
734
|
+
subnetwork = fwd.get("subnetwork", None)
|
|
670
735
|
|
|
671
736
|
neo4j_session.run(
|
|
672
737
|
query,
|
|
673
|
-
PartialUri=fwd[
|
|
674
|
-
IPAddress=fwd[
|
|
675
|
-
IPProtocol=fwd[
|
|
676
|
-
LoadBalancingScheme=fwd[
|
|
677
|
-
Name=fwd[
|
|
738
|
+
PartialUri=fwd["partial_uri"],
|
|
739
|
+
IPAddress=fwd["ip_address"],
|
|
740
|
+
IPProtocol=fwd["ip_protocol"],
|
|
741
|
+
LoadBalancingScheme=fwd["load_balancing_scheme"],
|
|
742
|
+
Name=fwd["name"],
|
|
678
743
|
Network=network,
|
|
679
|
-
NetworkPartialUri=fwd.get(
|
|
680
|
-
PortRange=fwd.get(
|
|
681
|
-
Ports=fwd.get(
|
|
682
|
-
ProjectId=fwd[
|
|
683
|
-
Region=fwd.get(
|
|
684
|
-
SelfLink=fwd[
|
|
744
|
+
NetworkPartialUri=fwd.get("network_partial_uri", None),
|
|
745
|
+
PortRange=fwd.get("port_range", None),
|
|
746
|
+
Ports=fwd.get("ports", None),
|
|
747
|
+
ProjectId=fwd["project_id"],
|
|
748
|
+
Region=fwd.get("region", None),
|
|
749
|
+
SelfLink=fwd["self_link"],
|
|
685
750
|
SubNetwork=subnetwork,
|
|
686
|
-
SubNetworkPartialUri=fwd.get(
|
|
687
|
-
TargetPartialUri=fwd[
|
|
751
|
+
SubNetworkPartialUri=fwd.get("subnetwork_partial_uri", None),
|
|
752
|
+
TargetPartialUri=fwd["target"],
|
|
688
753
|
gcp_update_tag=gcp_update_tag,
|
|
689
754
|
)
|
|
690
755
|
|
|
@@ -695,7 +760,11 @@ def load_gcp_forwarding_rules(neo4j_session: neo4j.Session, fwd_rules: List[Dict
|
|
|
695
760
|
|
|
696
761
|
|
|
697
762
|
@timeit
|
|
698
|
-
def _attach_fwd_rule_to_subnet(
|
|
763
|
+
def _attach_fwd_rule_to_subnet(
|
|
764
|
+
neo4j_session: neo4j.Session,
|
|
765
|
+
fwd: Dict,
|
|
766
|
+
gcp_update_tag: int,
|
|
767
|
+
) -> None:
|
|
699
768
|
query = """
|
|
700
769
|
MERGE(subnet:GCPSubnet{id:$SubNetworkPartialUri})
|
|
701
770
|
ON CREATE SET subnet.firstseen = timestamp(),
|
|
@@ -712,14 +781,18 @@ def _attach_fwd_rule_to_subnet(neo4j_session: neo4j.Session, fwd: Dict, gcp_upda
|
|
|
712
781
|
|
|
713
782
|
neo4j_session.run(
|
|
714
783
|
query,
|
|
715
|
-
PartialUri=fwd[
|
|
716
|
-
SubNetworkPartialUri=fwd.get(
|
|
784
|
+
PartialUri=fwd["partial_uri"],
|
|
785
|
+
SubNetworkPartialUri=fwd.get("subnetwork_partial_uri", None),
|
|
717
786
|
gcp_update_tag=gcp_update_tag,
|
|
718
787
|
)
|
|
719
788
|
|
|
720
789
|
|
|
721
790
|
@timeit
|
|
722
|
-
def _attach_fwd_rule_to_vpc(
|
|
791
|
+
def _attach_fwd_rule_to_vpc(
|
|
792
|
+
neo4j_session: neo4j.Session,
|
|
793
|
+
fwd: Dict,
|
|
794
|
+
gcp_update_tag: int,
|
|
795
|
+
) -> None:
|
|
723
796
|
query = """
|
|
724
797
|
MERGE (vpc:GCPVpc{id:$NetworkPartialUri})
|
|
725
798
|
ON CREATE SET vpc.firstseen = timestamp(),
|
|
@@ -735,14 +808,18 @@ def _attach_fwd_rule_to_vpc(neo4j_session: neo4j.Session, fwd: Dict, gcp_update_
|
|
|
735
808
|
|
|
736
809
|
neo4j_session.run(
|
|
737
810
|
query,
|
|
738
|
-
PartialUri=fwd[
|
|
739
|
-
NetworkPartialUri=fwd.get(
|
|
811
|
+
PartialUri=fwd["partial_uri"],
|
|
812
|
+
NetworkPartialUri=fwd.get("network_partial_uri", None),
|
|
740
813
|
gcp_update_tag=gcp_update_tag,
|
|
741
814
|
)
|
|
742
815
|
|
|
743
816
|
|
|
744
817
|
@timeit
|
|
745
|
-
def _attach_instance_tags(
|
|
818
|
+
def _attach_instance_tags(
|
|
819
|
+
neo4j_session: neo4j.Session,
|
|
820
|
+
instance: Resource,
|
|
821
|
+
gcp_update_tag: int,
|
|
822
|
+
) -> None:
|
|
746
823
|
"""
|
|
747
824
|
Attach tags to GCP instance and to the VPCs that they are defined in.
|
|
748
825
|
:param neo4j_session: The session
|
|
@@ -770,21 +847,25 @@ def _attach_instance_tags(neo4j_session: neo4j.Session, instance: Resource, gcp_
|
|
|
770
847
|
ON CREATE SET d.firstseen = timestamp()
|
|
771
848
|
SET d.lastupdated = $gcp_update_tag
|
|
772
849
|
"""
|
|
773
|
-
for tag in instance.get(
|
|
774
|
-
for nic in instance.get(
|
|
775
|
-
tag_id = _create_gcp_network_tag_id(nic[
|
|
850
|
+
for tag in instance.get("tags", {}).get("items", []):
|
|
851
|
+
for nic in instance.get("networkInterfaces", []):
|
|
852
|
+
tag_id = _create_gcp_network_tag_id(nic["vpc_partial_uri"], tag)
|
|
776
853
|
neo4j_session.run(
|
|
777
854
|
query,
|
|
778
|
-
InstanceId=instance[
|
|
855
|
+
InstanceId=instance["partial_uri"],
|
|
779
856
|
TagId=tag_id,
|
|
780
857
|
TagValue=tag,
|
|
781
|
-
VpcPartialUri=nic[
|
|
858
|
+
VpcPartialUri=nic["vpc_partial_uri"],
|
|
782
859
|
gcp_update_tag=gcp_update_tag,
|
|
783
860
|
)
|
|
784
861
|
|
|
785
862
|
|
|
786
863
|
@timeit
|
|
787
|
-
def _attach_gcp_nics(
|
|
864
|
+
def _attach_gcp_nics(
|
|
865
|
+
neo4j_session: neo4j.Session,
|
|
866
|
+
instance: Resource,
|
|
867
|
+
gcp_update_tag: int,
|
|
868
|
+
) -> None:
|
|
788
869
|
"""
|
|
789
870
|
Attach GCP Network Interfaces to GCP Instances and GCP Subnets.
|
|
790
871
|
Then, attach GCP Instances directly to VPCs.
|
|
@@ -815,24 +896,27 @@ def _attach_gcp_nics(neo4j_session: neo4j.Session, instance: Resource, gcp_updat
|
|
|
815
896
|
ON CREATE SET p.firstseen = timestamp()
|
|
816
897
|
SET p.lastupdated = $gcp_update_tag
|
|
817
898
|
"""
|
|
818
|
-
for nic in instance.get(
|
|
899
|
+
for nic in instance.get("networkInterfaces", []):
|
|
819
900
|
# Make an ID for GCPNetworkInterface nodes because GCP doesn't define one but we need to uniquely identify them
|
|
820
901
|
nic_id = f"{instance['partial_uri']}/networkinterfaces/{nic['name']}"
|
|
821
902
|
neo4j_session.run(
|
|
822
903
|
query,
|
|
823
|
-
InstanceId=instance[
|
|
904
|
+
InstanceId=instance["partial_uri"],
|
|
824
905
|
NicId=nic_id,
|
|
825
|
-
NetworkIP=nic.get(
|
|
826
|
-
NicName=nic[
|
|
906
|
+
NetworkIP=nic.get("networkIP"),
|
|
907
|
+
NicName=nic["name"],
|
|
827
908
|
gcp_update_tag=gcp_update_tag,
|
|
828
|
-
SubnetPartialUri=nic[
|
|
909
|
+
SubnetPartialUri=nic["subnet_partial_uri"],
|
|
829
910
|
)
|
|
830
911
|
_attach_gcp_nic_access_configs(neo4j_session, nic_id, nic, gcp_update_tag)
|
|
831
912
|
|
|
832
913
|
|
|
833
914
|
@timeit
|
|
834
915
|
def _attach_gcp_nic_access_configs(
|
|
835
|
-
neo4j_session: neo4j.Session,
|
|
916
|
+
neo4j_session: neo4j.Session,
|
|
917
|
+
nic_id: str,
|
|
918
|
+
nic: Resource,
|
|
919
|
+
gcp_update_tag: int,
|
|
836
920
|
) -> None:
|
|
837
921
|
"""
|
|
838
922
|
Attach an access configuration to the GCP NIC.
|
|
@@ -858,25 +942,29 @@ def _attach_gcp_nic_access_configs(
|
|
|
858
942
|
ON CREATE SET r.firstseen = timestamp()
|
|
859
943
|
SET r.lastupdated = $gcp_update_tag
|
|
860
944
|
"""
|
|
861
|
-
for ac in nic.get(
|
|
945
|
+
for ac in nic.get("accessConfigs", []):
|
|
862
946
|
# Make an ID for GCPNicAccessConfig nodes because GCP doesn't define one but we need to uniquely identify them
|
|
863
947
|
access_config_id = f"{nic_id}/accessconfigs/{ac['type']}"
|
|
864
948
|
neo4j_session.run(
|
|
865
949
|
query,
|
|
866
950
|
NicId=nic_id,
|
|
867
951
|
AccessConfigId=access_config_id,
|
|
868
|
-
Type=ac[
|
|
869
|
-
Name=ac[
|
|
870
|
-
NatIP=ac.get(
|
|
871
|
-
SetPublicPtr=ac.get(
|
|
872
|
-
PublicPtrDomainName=ac.get(
|
|
873
|
-
NetworkTier=ac.get(
|
|
952
|
+
Type=ac["type"],
|
|
953
|
+
Name=ac["name"],
|
|
954
|
+
NatIP=ac.get("natIP", None),
|
|
955
|
+
SetPublicPtr=ac.get("setPublicPtr", None),
|
|
956
|
+
PublicPtrDomainName=ac.get("publicPtrDomainName", None),
|
|
957
|
+
NetworkTier=ac.get("networkTier", None),
|
|
874
958
|
gcp_update_tag=gcp_update_tag,
|
|
875
959
|
)
|
|
876
960
|
|
|
877
961
|
|
|
878
962
|
@timeit
|
|
879
|
-
def _attach_gcp_vpc(
|
|
963
|
+
def _attach_gcp_vpc(
|
|
964
|
+
neo4j_session: neo4j.Session,
|
|
965
|
+
instance_id: str,
|
|
966
|
+
gcp_update_tag: int,
|
|
967
|
+
) -> None:
|
|
880
968
|
"""
|
|
881
969
|
Attach a GCP instance directly to a VPC
|
|
882
970
|
:param neo4j_session: neo4j_session
|
|
@@ -899,7 +987,11 @@ def _attach_gcp_vpc(neo4j_session: neo4j.Session, instance_id: str, gcp_update_t
|
|
|
899
987
|
|
|
900
988
|
|
|
901
989
|
@timeit
|
|
902
|
-
def load_gcp_ingress_firewalls(
|
|
990
|
+
def load_gcp_ingress_firewalls(
|
|
991
|
+
neo4j_session: neo4j.Session,
|
|
992
|
+
fw_list: List[Resource],
|
|
993
|
+
gcp_update_tag: int,
|
|
994
|
+
) -> None:
|
|
903
995
|
"""
|
|
904
996
|
Load the firewall list to Neo4j
|
|
905
997
|
:param fw_list: The transformed list of firewalls
|
|
@@ -929,14 +1021,14 @@ def load_gcp_ingress_firewalls(neo4j_session: neo4j.Session, fw_list: List[Resou
|
|
|
929
1021
|
for fw in fw_list:
|
|
930
1022
|
neo4j_session.run(
|
|
931
1023
|
query,
|
|
932
|
-
FwPartialUri=fw[
|
|
933
|
-
Direction=fw[
|
|
934
|
-
Disabled=fw[
|
|
935
|
-
Name=fw[
|
|
936
|
-
Priority=fw[
|
|
937
|
-
SelfLink=fw[
|
|
938
|
-
VpcPartialUri=fw[
|
|
939
|
-
HasTargetServiceAccounts=fw[
|
|
1024
|
+
FwPartialUri=fw["id"],
|
|
1025
|
+
Direction=fw["direction"],
|
|
1026
|
+
Disabled=fw["disabled"],
|
|
1027
|
+
Name=fw["name"],
|
|
1028
|
+
Priority=fw["priority"],
|
|
1029
|
+
SelfLink=fw["selfLink"],
|
|
1030
|
+
VpcPartialUri=fw["vpc_partial_uri"],
|
|
1031
|
+
HasTargetServiceAccounts=fw["has_target_service_accounts"],
|
|
940
1032
|
gcp_update_tag=gcp_update_tag,
|
|
941
1033
|
)
|
|
942
1034
|
_attach_firewall_rules(neo4j_session, fw, gcp_update_tag)
|
|
@@ -944,7 +1036,11 @@ def load_gcp_ingress_firewalls(neo4j_session: neo4j.Session, fw_list: List[Resou
|
|
|
944
1036
|
|
|
945
1037
|
|
|
946
1038
|
@timeit
|
|
947
|
-
def _attach_firewall_rules(
|
|
1039
|
+
def _attach_firewall_rules(
|
|
1040
|
+
neo4j_session: neo4j.Session,
|
|
1041
|
+
fw: Resource,
|
|
1042
|
+
gcp_update_tag: int,
|
|
1043
|
+
) -> None:
|
|
948
1044
|
"""
|
|
949
1045
|
Attach the allow_rules to the Firewall object
|
|
950
1046
|
:param neo4j_session: The Neo4j session
|
|
@@ -952,7 +1048,8 @@ def _attach_firewall_rules(neo4j_session: neo4j.Session, fw: Resource, gcp_updat
|
|
|
952
1048
|
:param gcp_update_tag: The timestamp
|
|
953
1049
|
:return: Nothing
|
|
954
1050
|
"""
|
|
955
|
-
template = Template(
|
|
1051
|
+
template = Template(
|
|
1052
|
+
"""
|
|
956
1053
|
MATCH (fw:GCPFirewall{id:$FwPartialUri})
|
|
957
1054
|
|
|
958
1055
|
MERGE (rule:IpRule:IpPermissionInbound:GCPIpRule{id:$RuleId})
|
|
@@ -975,9 +1072,10 @@ def _attach_firewall_rules(neo4j_session: neo4j.Session, fw: Resource, gcp_updat
|
|
|
975
1072
|
MERGE (fw)<-[r:$fw_rule_relationship_label]-(rule)
|
|
976
1073
|
ON CREATE SET r.firstseen = timestamp()
|
|
977
1074
|
SET r.lastupdated = $gcp_update_tag
|
|
978
|
-
"""
|
|
979
|
-
|
|
980
|
-
|
|
1075
|
+
""",
|
|
1076
|
+
)
|
|
1077
|
+
for list_type in "transformed_allow_list", "transformed_deny_list":
|
|
1078
|
+
if list_type == "transformed_allow_list":
|
|
981
1079
|
label = "ALLOWED_BY"
|
|
982
1080
|
else:
|
|
983
1081
|
label = "DENIED_BY"
|
|
@@ -985,21 +1083,25 @@ def _attach_firewall_rules(neo4j_session: neo4j.Session, fw: Resource, gcp_updat
|
|
|
985
1083
|
# It is possible for sourceRanges to not be specified for this rule
|
|
986
1084
|
# If sourceRanges is not specified then the rule must specify sourceTags.
|
|
987
1085
|
# Since an IP range cannot have a tag applied to it, it is ok if we don't ingest this rule.
|
|
988
|
-
for ip_range in fw.get(
|
|
1086
|
+
for ip_range in fw.get("sourceRanges", []):
|
|
989
1087
|
neo4j_session.run(
|
|
990
1088
|
template.safe_substitute(fw_rule_relationship_label=label),
|
|
991
|
-
FwPartialUri=fw[
|
|
992
|
-
RuleId=rule[
|
|
993
|
-
Protocol=rule[
|
|
994
|
-
FromPort=rule.get(
|
|
995
|
-
ToPort=rule.get(
|
|
1089
|
+
FwPartialUri=fw["id"],
|
|
1090
|
+
RuleId=rule["ruleid"],
|
|
1091
|
+
Protocol=rule["protocol"],
|
|
1092
|
+
FromPort=rule.get("fromport"),
|
|
1093
|
+
ToPort=rule.get("toport"),
|
|
996
1094
|
Range=ip_range,
|
|
997
1095
|
gcp_update_tag=gcp_update_tag,
|
|
998
1096
|
)
|
|
999
1097
|
|
|
1000
1098
|
|
|
1001
1099
|
@timeit
|
|
1002
|
-
def _attach_target_tags(
|
|
1100
|
+
def _attach_target_tags(
|
|
1101
|
+
neo4j_session: neo4j.Session,
|
|
1102
|
+
fw: Resource,
|
|
1103
|
+
gcp_update_tag: int,
|
|
1104
|
+
) -> None:
|
|
1003
1105
|
"""
|
|
1004
1106
|
Attach target tags to the firewall object
|
|
1005
1107
|
:param neo4j_session: The neo4j session
|
|
@@ -1020,11 +1122,11 @@ def _attach_target_tags(neo4j_session: neo4j.Session, fw: Resource, gcp_update_t
|
|
|
1020
1122
|
ON CREATE SET h.firstseen = timestamp()
|
|
1021
1123
|
SET h.lastupdated = $gcp_update_tag
|
|
1022
1124
|
"""
|
|
1023
|
-
for tag in fw.get(
|
|
1024
|
-
tag_id = _create_gcp_network_tag_id(fw[
|
|
1125
|
+
for tag in fw.get("targetTags", []):
|
|
1126
|
+
tag_id = _create_gcp_network_tag_id(fw["vpc_partial_uri"], tag)
|
|
1025
1127
|
neo4j_session.run(
|
|
1026
1128
|
query,
|
|
1027
|
-
FwPartialUri=fw[
|
|
1129
|
+
FwPartialUri=fw["id"],
|
|
1028
1130
|
TagId=tag_id,
|
|
1029
1131
|
TagValue=tag,
|
|
1030
1132
|
gcp_update_tag=gcp_update_tag,
|
|
@@ -1032,14 +1134,21 @@ def _attach_target_tags(neo4j_session: neo4j.Session, fw: Resource, gcp_update_t
|
|
|
1032
1134
|
|
|
1033
1135
|
|
|
1034
1136
|
@timeit
|
|
1035
|
-
def cleanup_gcp_instances(
|
|
1137
|
+
def cleanup_gcp_instances(
|
|
1138
|
+
neo4j_session: neo4j.Session,
|
|
1139
|
+
common_job_parameters: Dict,
|
|
1140
|
+
) -> None:
|
|
1036
1141
|
"""
|
|
1037
1142
|
Delete out-of-date GCP instance nodes and relationships
|
|
1038
1143
|
:param neo4j_session: The Neo4j session
|
|
1039
1144
|
:param common_job_parameters: dict of other job parameters to pass to Neo4j
|
|
1040
1145
|
:return: Nothing
|
|
1041
1146
|
"""
|
|
1042
|
-
run_cleanup_job(
|
|
1147
|
+
run_cleanup_job(
|
|
1148
|
+
"gcp_compute_instance_cleanup.json",
|
|
1149
|
+
neo4j_session,
|
|
1150
|
+
common_job_parameters,
|
|
1151
|
+
)
|
|
1043
1152
|
|
|
1044
1153
|
|
|
1045
1154
|
@timeit
|
|
@@ -1050,46 +1159,75 @@ def cleanup_gcp_vpcs(neo4j_session: neo4j.Session, common_job_parameters: Dict)
|
|
|
1050
1159
|
:param common_job_parameters: dict of other job parameters to pass to Neo4j
|
|
1051
1160
|
:return: Nothing
|
|
1052
1161
|
"""
|
|
1053
|
-
run_cleanup_job(
|
|
1162
|
+
run_cleanup_job(
|
|
1163
|
+
"gcp_compute_vpc_cleanup.json",
|
|
1164
|
+
neo4j_session,
|
|
1165
|
+
common_job_parameters,
|
|
1166
|
+
)
|
|
1054
1167
|
|
|
1055
1168
|
|
|
1056
1169
|
@timeit
|
|
1057
|
-
def cleanup_gcp_subnets(
|
|
1170
|
+
def cleanup_gcp_subnets(
|
|
1171
|
+
neo4j_session: neo4j.Session,
|
|
1172
|
+
common_job_parameters: Dict,
|
|
1173
|
+
) -> None:
|
|
1058
1174
|
"""
|
|
1059
1175
|
Delete out-of-date GCP VPC subnet nodes and relationships
|
|
1060
1176
|
:param neo4j_session: The Neo4j session
|
|
1061
1177
|
:param common_job_parameters: dict of other job parameters to pass to Neo4j
|
|
1062
1178
|
:return: Nothing
|
|
1063
1179
|
"""
|
|
1064
|
-
run_cleanup_job(
|
|
1180
|
+
run_cleanup_job(
|
|
1181
|
+
"gcp_compute_vpc_subnet_cleanup.json",
|
|
1182
|
+
neo4j_session,
|
|
1183
|
+
common_job_parameters,
|
|
1184
|
+
)
|
|
1065
1185
|
|
|
1066
1186
|
|
|
1067
1187
|
@timeit
|
|
1068
|
-
def cleanup_gcp_forwarding_rules(
|
|
1188
|
+
def cleanup_gcp_forwarding_rules(
|
|
1189
|
+
neo4j_session: neo4j.Session,
|
|
1190
|
+
common_job_parameters: Dict,
|
|
1191
|
+
) -> None:
|
|
1069
1192
|
"""
|
|
1070
1193
|
Delete out-of-date GCP forwarding rules and relationships
|
|
1071
1194
|
:param neo4j_session: The Neo4j session
|
|
1072
1195
|
:param common_job_parameters: dict of other job parameters to pass to Neo4j
|
|
1073
1196
|
:return: Nothing
|
|
1074
1197
|
"""
|
|
1075
|
-
run_cleanup_job(
|
|
1198
|
+
run_cleanup_job(
|
|
1199
|
+
"gcp_compute_forwarding_rules_cleanup.json",
|
|
1200
|
+
neo4j_session,
|
|
1201
|
+
common_job_parameters,
|
|
1202
|
+
)
|
|
1076
1203
|
|
|
1077
1204
|
|
|
1078
1205
|
@timeit
|
|
1079
|
-
def cleanup_gcp_firewall_rules(
|
|
1206
|
+
def cleanup_gcp_firewall_rules(
|
|
1207
|
+
neo4j_session: neo4j.Session,
|
|
1208
|
+
common_job_parameters: Dict,
|
|
1209
|
+
) -> None:
|
|
1080
1210
|
"""
|
|
1081
1211
|
Delete out of date GCP firewalls and their relationships
|
|
1082
1212
|
:param neo4j_session: The Neo4j session
|
|
1083
1213
|
:param common_job_parameters: dict of other job parameters to pass to Neo4j
|
|
1084
1214
|
:return: Nothing
|
|
1085
1215
|
"""
|
|
1086
|
-
run_cleanup_job(
|
|
1216
|
+
run_cleanup_job(
|
|
1217
|
+
"gcp_compute_firewall_cleanup.json",
|
|
1218
|
+
neo4j_session,
|
|
1219
|
+
common_job_parameters,
|
|
1220
|
+
)
|
|
1087
1221
|
|
|
1088
1222
|
|
|
1089
1223
|
@timeit
|
|
1090
1224
|
def sync_gcp_instances(
|
|
1091
|
-
neo4j_session: neo4j.Session,
|
|
1092
|
-
|
|
1225
|
+
neo4j_session: neo4j.Session,
|
|
1226
|
+
compute: Resource,
|
|
1227
|
+
project_id: str,
|
|
1228
|
+
zones: Optional[List[Dict]],
|
|
1229
|
+
gcp_update_tag: int,
|
|
1230
|
+
common_job_parameters: Dict,
|
|
1093
1231
|
) -> None:
|
|
1094
1232
|
"""
|
|
1095
1233
|
Get GCP instances using the Compute resource object, ingest to Neo4j, and clean up old data.
|
|
@@ -1106,13 +1244,16 @@ def sync_gcp_instances(
|
|
|
1106
1244
|
instance_responses = get_gcp_instance_responses(project_id, zones, compute)
|
|
1107
1245
|
instance_list = transform_gcp_instances(instance_responses)
|
|
1108
1246
|
load_gcp_instances(neo4j_session, instance_list, gcp_update_tag)
|
|
1109
|
-
# TODO scope the cleanup to the current project - https://github.com/
|
|
1247
|
+
# TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
|
|
1110
1248
|
cleanup_gcp_instances(neo4j_session, common_job_parameters)
|
|
1111
1249
|
|
|
1112
1250
|
|
|
1113
1251
|
@timeit
|
|
1114
1252
|
def sync_gcp_vpcs(
|
|
1115
|
-
neo4j_session: neo4j.Session,
|
|
1253
|
+
neo4j_session: neo4j.Session,
|
|
1254
|
+
compute: Resource,
|
|
1255
|
+
project_id: str,
|
|
1256
|
+
gcp_update_tag: int,
|
|
1116
1257
|
common_job_parameters: Dict,
|
|
1117
1258
|
) -> None:
|
|
1118
1259
|
"""
|
|
@@ -1127,27 +1268,34 @@ def sync_gcp_vpcs(
|
|
|
1127
1268
|
vpc_res = get_gcp_vpcs(project_id, compute)
|
|
1128
1269
|
vpcs = transform_gcp_vpcs(vpc_res)
|
|
1129
1270
|
load_gcp_vpcs(neo4j_session, vpcs, gcp_update_tag)
|
|
1130
|
-
# TODO scope the cleanup to the current project - https://github.com/
|
|
1271
|
+
# TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
|
|
1131
1272
|
cleanup_gcp_vpcs(neo4j_session, common_job_parameters)
|
|
1132
1273
|
|
|
1133
1274
|
|
|
1134
1275
|
@timeit
|
|
1135
1276
|
def sync_gcp_subnets(
|
|
1136
|
-
neo4j_session: neo4j.Session,
|
|
1277
|
+
neo4j_session: neo4j.Session,
|
|
1278
|
+
compute: Resource,
|
|
1279
|
+
project_id: str,
|
|
1280
|
+
regions: List[str],
|
|
1281
|
+
gcp_update_tag: int,
|
|
1137
1282
|
common_job_parameters: Dict,
|
|
1138
1283
|
) -> None:
|
|
1139
1284
|
for r in regions:
|
|
1140
1285
|
subnet_res = get_gcp_subnets(project_id, r, compute)
|
|
1141
1286
|
subnets = transform_gcp_subnets(subnet_res)
|
|
1142
1287
|
load_gcp_subnets(neo4j_session, subnets, gcp_update_tag)
|
|
1143
|
-
# TODO scope the cleanup to the current project - https://github.com/
|
|
1288
|
+
# TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
|
|
1144
1289
|
cleanup_gcp_subnets(neo4j_session, common_job_parameters)
|
|
1145
1290
|
|
|
1146
1291
|
|
|
1147
1292
|
@timeit
|
|
1148
1293
|
def sync_gcp_forwarding_rules(
|
|
1149
|
-
neo4j_session: neo4j.Session,
|
|
1150
|
-
|
|
1294
|
+
neo4j_session: neo4j.Session,
|
|
1295
|
+
compute: Resource,
|
|
1296
|
+
project_id: str,
|
|
1297
|
+
regions: List[str],
|
|
1298
|
+
gcp_update_tag: int,
|
|
1151
1299
|
common_job_parameters: Dict,
|
|
1152
1300
|
) -> None:
|
|
1153
1301
|
"""
|
|
@@ -1163,20 +1311,23 @@ def sync_gcp_forwarding_rules(
|
|
|
1163
1311
|
global_fwd_response = get_gcp_global_forwarding_rules(project_id, compute)
|
|
1164
1312
|
forwarding_rules = transform_gcp_forwarding_rules(global_fwd_response)
|
|
1165
1313
|
load_gcp_forwarding_rules(neo4j_session, forwarding_rules, gcp_update_tag)
|
|
1166
|
-
# TODO scope the cleanup to the current project - https://github.com/
|
|
1314
|
+
# TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
|
|
1167
1315
|
cleanup_gcp_forwarding_rules(neo4j_session, common_job_parameters)
|
|
1168
1316
|
|
|
1169
1317
|
for r in regions:
|
|
1170
1318
|
fwd_response = get_gcp_regional_forwarding_rules(project_id, r, compute)
|
|
1171
1319
|
forwarding_rules = transform_gcp_forwarding_rules(fwd_response)
|
|
1172
1320
|
load_gcp_forwarding_rules(neo4j_session, forwarding_rules, gcp_update_tag)
|
|
1173
|
-
# TODO scope the cleanup to the current project - https://github.com/
|
|
1321
|
+
# TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
|
|
1174
1322
|
cleanup_gcp_forwarding_rules(neo4j_session, common_job_parameters)
|
|
1175
1323
|
|
|
1176
1324
|
|
|
1177
1325
|
@timeit
|
|
1178
1326
|
def sync_gcp_firewall_rules(
|
|
1179
|
-
neo4j_session: neo4j.Session,
|
|
1327
|
+
neo4j_session: neo4j.Session,
|
|
1328
|
+
compute: Resource,
|
|
1329
|
+
project_id: str,
|
|
1330
|
+
gcp_update_tag: int,
|
|
1180
1331
|
common_job_parameters: Dict,
|
|
1181
1332
|
) -> None:
|
|
1182
1333
|
"""
|
|
@@ -1190,7 +1341,7 @@ def sync_gcp_firewall_rules(
|
|
|
1190
1341
|
fw_response = get_gcp_firewall_ingress_rules(project_id, compute)
|
|
1191
1342
|
fw_list = transform_gcp_firewall(fw_response)
|
|
1192
1343
|
load_gcp_ingress_firewalls(neo4j_session, fw_list, gcp_update_tag)
|
|
1193
|
-
# TODO scope the cleanup to the current project - https://github.com/
|
|
1344
|
+
# TODO scope the cleanup to the current project - https://github.com/cartography-cncf/cartography/issues/381
|
|
1194
1345
|
cleanup_gcp_firewall_rules(neo4j_session, common_job_parameters)
|
|
1195
1346
|
|
|
1196
1347
|
|
|
@@ -1203,13 +1354,16 @@ def _zones_to_regions(zones: List[str]) -> List[Set]:
|
|
|
1203
1354
|
regions = set()
|
|
1204
1355
|
for zone in zones:
|
|
1205
1356
|
# Chop off the last 2 chars to turn the zone to a region
|
|
1206
|
-
region = zone[
|
|
1357
|
+
region = zone["name"][:-2] # type: ignore
|
|
1207
1358
|
regions.add(region)
|
|
1208
|
-
return list(regions)
|
|
1359
|
+
return list(regions) # type: ignore
|
|
1209
1360
|
|
|
1210
1361
|
|
|
1211
1362
|
def sync(
|
|
1212
|
-
neo4j_session: neo4j.Session,
|
|
1363
|
+
neo4j_session: neo4j.Session,
|
|
1364
|
+
compute: Resource,
|
|
1365
|
+
project_id: str,
|
|
1366
|
+
gcp_update_tag: int,
|
|
1213
1367
|
common_job_parameters: dict,
|
|
1214
1368
|
) -> None:
|
|
1215
1369
|
"""
|
|
@@ -1230,8 +1384,41 @@ def sync(
|
|
|
1230
1384
|
return
|
|
1231
1385
|
else:
|
|
1232
1386
|
regions = _zones_to_regions(zones)
|
|
1233
|
-
sync_gcp_vpcs(
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1387
|
+
sync_gcp_vpcs(
|
|
1388
|
+
neo4j_session,
|
|
1389
|
+
compute,
|
|
1390
|
+
project_id,
|
|
1391
|
+
gcp_update_tag,
|
|
1392
|
+
common_job_parameters,
|
|
1393
|
+
)
|
|
1394
|
+
sync_gcp_firewall_rules(
|
|
1395
|
+
neo4j_session,
|
|
1396
|
+
compute,
|
|
1397
|
+
project_id,
|
|
1398
|
+
gcp_update_tag,
|
|
1399
|
+
common_job_parameters,
|
|
1400
|
+
)
|
|
1401
|
+
sync_gcp_subnets(
|
|
1402
|
+
neo4j_session,
|
|
1403
|
+
compute,
|
|
1404
|
+
project_id,
|
|
1405
|
+
regions,
|
|
1406
|
+
gcp_update_tag,
|
|
1407
|
+
common_job_parameters,
|
|
1408
|
+
)
|
|
1409
|
+
sync_gcp_instances(
|
|
1410
|
+
neo4j_session,
|
|
1411
|
+
compute,
|
|
1412
|
+
project_id,
|
|
1413
|
+
zones,
|
|
1414
|
+
gcp_update_tag,
|
|
1415
|
+
common_job_parameters,
|
|
1416
|
+
)
|
|
1417
|
+
sync_gcp_forwarding_rules(
|
|
1418
|
+
neo4j_session,
|
|
1419
|
+
compute,
|
|
1420
|
+
project_id,
|
|
1421
|
+
regions,
|
|
1422
|
+
gcp_update_tag,
|
|
1423
|
+
common_job_parameters,
|
|
1424
|
+
)
|