cs-image-system-gcloud-runtime-plugin 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cs_image_system/gcloud_runtime/__init__.py +3 -0
- cs_image_system/gcloud_runtime/bootstrap.py +351 -0
- cs_image_system/gcloud_runtime/gcp_packer_source.py +135 -0
- cs_image_system/gcloud_runtime/gcp_provider_specific_image.py +38 -0
- cs_image_system/gcloud_runtime/gcp_runtime_builders.py +608 -0
- cs_image_system/gcloud_runtime/gcp_runtime_models.py +168 -0
- cs_image_system/gcloud_runtime/gcp_utils.py +498 -0
- cs_image_system/gcloud_runtime/main.py +28 -0
- cs_image_system/py.typed +0 -0
- cs_image_system_gcloud_runtime_plugin-0.1.1.dist-info/METADATA +1168 -0
- cs_image_system_gcloud_runtime_plugin-0.1.1.dist-info/RECORD +14 -0
- cs_image_system_gcloud_runtime_plugin-0.1.1.dist-info/WHEEL +4 -0
- cs_image_system_gcloud_runtime_plugin-0.1.1.dist-info/entry_points.txt +5 -0
- cs_image_system_gcloud_runtime_plugin-0.1.1.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Mykel Alvis <mykelalvis@infrastructurebuilder.org>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
from dataclasses import field
|
|
6
|
+
from cs_image_system.base.models.model_config import CSIS_MODEL_CONFIG
|
|
7
|
+
from pydantic.dataclasses import dataclass # stage 23: validation at construction
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
|
|
12
|
+
from cs_image_system.gcloud_runtime import gcp_utils
|
|
13
|
+
from cs_image_system.base.helpers.field_helpers import fk_field
|
|
14
|
+
log = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
from cs_image_system.base.constants import DEFAULT, OOPS_DEFAULTS, VCT
|
|
18
|
+
from cs_image_system.base.models.cloud_builder import CloudBuilderModel, CloudNetworkingConfig
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
GCP: str = "gcloud"
|
|
22
|
+
|
|
23
|
+
@dataclass(config=CSIS_MODEL_CONFIG)
|
|
24
|
+
class GCPCloudNetworkingModel(CloudNetworkingConfig):
|
|
25
|
+
"""GCP-specific networking configuration data object."""
|
|
26
|
+
# Network tags select which firewall rules apply to an instance -- the
|
|
27
|
+
# closest GCP analog to AWS security group ids.
|
|
28
|
+
network_tags: list[str] = field(default_factory=list)
|
|
29
|
+
|
|
30
|
+
_model_id: str | None = fk_field(target=VCT.RUNTIME_BUILDER_MODEL,
|
|
31
|
+
init=False,
|
|
32
|
+
default=None, metadata={
|
|
33
|
+
"description": "The name of the runtime model this object is associated with",
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
def __post_init__(self) -> None:
|
|
37
|
+
super().__post_init__()
|
|
38
|
+
self.name = self.name.strip()
|
|
39
|
+
if not self.name:
|
|
40
|
+
raise ValueError(f"Networking configuration name cannot be empty for {self}.")
|
|
41
|
+
|
|
42
|
+
def get_name(self) -> str:
|
|
43
|
+
return self.name
|
|
44
|
+
def get_subnet_id(self) -> str:
|
|
45
|
+
if not self.default_subnet_id:
|
|
46
|
+
raise ValueError(f"Subnet ID is required for GCP networking configuration {self.name} but is not set.")
|
|
47
|
+
return self.default_subnet_id
|
|
48
|
+
def get_network_tags(self) -> list[str]:
|
|
49
|
+
return self.network_tags
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(kw_only=True, config=CSIS_MODEL_CONFIG)
|
|
53
|
+
class GCPCloudBuilderModel(CloudBuilderModel):
|
|
54
|
+
"""GCP cloud provider configuration data object."""
|
|
55
|
+
type = GCP
|
|
56
|
+
# stage 63 item 14: the cfg/executables.yml entry every gcloud call on
|
|
57
|
+
# this runtime runs through (the session command, the inventory, the
|
|
58
|
+
# storage lookups and the generated archive/wipe scripts), as the tofu
|
|
59
|
+
# builders name theirs. `validate` refuses a name that is not declared.
|
|
60
|
+
executable: str | None = "gcloud"
|
|
61
|
+
project_id: str | None = None
|
|
62
|
+
zone: str | None = None
|
|
63
|
+
# Service account ATTACHED to build VMs and instances (config, never a
|
|
64
|
+
# code literal). Without it packer attaches the project's default
|
|
65
|
+
# compute SA, which a least-privilege runner cannot actAs (found live).
|
|
66
|
+
service_account_email: str | None = None
|
|
67
|
+
# Bake disk size in GB for every image baked on this runtime (finding
|
|
68
|
+
# 51): a GCE instance's boot disk is exactly its image's disk, and the
|
|
69
|
+
# OS-builder-level default (200 GB, a RHEL vendor minimum carried
|
|
70
|
+
# forward) cost ~$8/month per instance. None = the image's own value.
|
|
71
|
+
default_disk_size: int | None = None
|
|
72
|
+
# Bake build VMs as preemptible (spot) instances: 60-90 % off bake minutes;
|
|
73
|
+
# a preempted bake simply re-runs (GCP-READINESS §6).
|
|
74
|
+
bake_preemptible: bool = False
|
|
75
|
+
state_configuration: str = fk_field(target=VCT.STATE_BACKEND_MODEL,
|
|
76
|
+
default = DEFAULT,
|
|
77
|
+
metadata={
|
|
78
|
+
"description": "The state manager this config uses",
|
|
79
|
+
"required": True,
|
|
80
|
+
})
|
|
81
|
+
ssh_username: str = DEFAULT # Currently override when needed. (How do we get this? From the image?)
|
|
82
|
+
# Debug session mechanism (V2 §3F1 parity): only "iap" is supported.
|
|
83
|
+
session_mechanism: str | None = None
|
|
84
|
+
networking: GCPCloudNetworkingModel | None = None # type: ignore #Overrides parent
|
|
85
|
+
network_map: dict[str, dict[str, list[dict]]] | None = field(init=False, default=None)
|
|
86
|
+
default_network: str | None = field(init=False, default=None)
|
|
87
|
+
all_firewall_rules: dict[str, dict] = field(init=False, default_factory=dict)
|
|
88
|
+
|
|
89
|
+
def __post_init__(self) -> None:
|
|
90
|
+
super().__post_init__()
|
|
91
|
+
|
|
92
|
+
def finalize(self):
|
|
93
|
+
if self._finalized:
|
|
94
|
+
return
|
|
95
|
+
super().finalize()
|
|
96
|
+
self.update_networking()
|
|
97
|
+
|
|
98
|
+
@classmethod
|
|
99
|
+
def csis_name(cls) -> str:
|
|
100
|
+
return GCP
|
|
101
|
+
|
|
102
|
+
def self_to_gcp_client_config(self) -> dict[str, Any]:
|
|
103
|
+
"""Convert the builder model to a configuration dictionary for GCP client libraries."""
|
|
104
|
+
config: dict[str, Any] = {}
|
|
105
|
+
if self.credentials:
|
|
106
|
+
config["credentials"] = self.get_credentials()
|
|
107
|
+
if self.project_id:
|
|
108
|
+
config["project"] = self.project_id
|
|
109
|
+
if self.region:
|
|
110
|
+
config["region"] = self.region
|
|
111
|
+
if self.zone:
|
|
112
|
+
config["zone"] = self.zone
|
|
113
|
+
return config
|
|
114
|
+
|
|
115
|
+
def update_networking(self) -> None:
|
|
116
|
+
""" Get the network map, default network and firewall rules for the GCP project and validate the networking configuration. """
|
|
117
|
+
session_config = self.self_to_gcp_client_config()
|
|
118
|
+
if not session_config:
|
|
119
|
+
raise ValueError("Session configuration cannot be None or empty")
|
|
120
|
+
if not gcp_utils.resolve_project(session_config):
|
|
121
|
+
log.warning(f"No usable GCP project could be resolved for cloud builder {self.get_display_name()} (no `project_id` declared on the runtime). Skipping network validation; image resolution and build execution will not work until valid GCP credentials are configured.")
|
|
122
|
+
return
|
|
123
|
+
|
|
124
|
+
network_map, default_network, allfw = gcp_utils.get_network_map_and_default_network(session_config)
|
|
125
|
+
self.network_map = network_map
|
|
126
|
+
self.default_network = default_network
|
|
127
|
+
self.all_firewall_rules = allfw
|
|
128
|
+
|
|
129
|
+
if not self.networking:
|
|
130
|
+
log.warning(f"No networking configuration provided for GCP cloud builder {self.get_display_name()}. This is required for image resolution and build execution. Please provide a networking configuration.")
|
|
131
|
+
return
|
|
132
|
+
if self.networking.network in OOPS_DEFAULTS:
|
|
133
|
+
if not default_network:
|
|
134
|
+
errstr = f"No network specified in networking configuration for GCP cloud builder {self.name}, and no 'default' network found in GCP project. A network is required for image resolution and build execution. Please specify a network in the networking configuration or ensure a default network exists in the GCP project."
|
|
135
|
+
log.error(errstr)
|
|
136
|
+
raise ValueError(errstr)
|
|
137
|
+
self.networking.network = default_network
|
|
138
|
+
if self.networking.network not in self.network_map:
|
|
139
|
+
errstr = f"Network {self.networking.network} specified in networking configuration for GCP cloud builder {self.name} not found in GCP project."
|
|
140
|
+
log.error(errstr)
|
|
141
|
+
raise ValueError(errstr)
|
|
142
|
+
# Validate configured subnets against the project's subnetworks (match
|
|
143
|
+
# by name, full self_link, or projects/... path suffix).
|
|
144
|
+
known_subnets: set[str] = set()
|
|
145
|
+
for sn in self.network_map[self.networking.network]["subnets"]:
|
|
146
|
+
known_subnets.add(str(sn.get("subnet_id")))
|
|
147
|
+
self_link = str(sn.get("self_link", ""))
|
|
148
|
+
if self_link:
|
|
149
|
+
known_subnets.add(self_link)
|
|
150
|
+
known_subnets.add(self_link.split("//compute.googleapis.com/")[-1])
|
|
151
|
+
# the canonical path form terraform uses (found live: the
|
|
152
|
+
# real API's self_link host is www.googleapis.com/compute/v1,
|
|
153
|
+
# so the split above never yields projects/...)
|
|
154
|
+
if "/compute/v1/" in self_link:
|
|
155
|
+
known_subnets.add(self_link.split("/compute/v1/", 1)[-1])
|
|
156
|
+
for subnet in self.networking.subnets:
|
|
157
|
+
sid = subnet.get_subnet_id()
|
|
158
|
+
if sid not in known_subnets:
|
|
159
|
+
errstr = f"Subnetwork {sid} specified in networking configuration for GCP cloud builder {self.name} not found in network {self.networking.network}."
|
|
160
|
+
log.error(errstr)
|
|
161
|
+
raise ValueError(errstr)
|
|
162
|
+
# Firewall rules apply via network tags; warn on tags no rule targets.
|
|
163
|
+
all_target_tags: set[str] = set()
|
|
164
|
+
for fw in self.all_firewall_rules.values():
|
|
165
|
+
all_target_tags.update(fw.get("target_tags", []) or [])
|
|
166
|
+
for tag in self.networking.network_tags:
|
|
167
|
+
if tag not in all_target_tags:
|
|
168
|
+
log.warning(f"Network tag {tag} specified in networking configuration for GCP cloud builder {self.name} is not targeted by any firewall rule in the GCP project.")
|
|
@@ -0,0 +1,498 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Mykel Alvis <mykelalvis@infrastructurebuilder.org>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
"""GCP utility functions for Compute Engine image and network queries."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Mapping
|
|
13
|
+
from typing import Any, Sequence, cast
|
|
14
|
+
|
|
15
|
+
from google.api_core import exceptions as gcp_exceptions
|
|
16
|
+
from google.cloud import compute_v1
|
|
17
|
+
|
|
18
|
+
from cs_image_system.base.constants import SELF
|
|
19
|
+
from cs_image_system.base.models.os_builder_runtime_config import OSBuilderBaseImageBuilderSubconfig
|
|
20
|
+
|
|
21
|
+
log = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ImageNotFoundError(Exception):
|
|
25
|
+
"""Raised when no image is found matching the query."""
|
|
26
|
+
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ImageQueryError(Exception):
|
|
31
|
+
"""Raised when an image query fails."""
|
|
32
|
+
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# Generic query filter keys -> Compute Engine image resource fields usable in a
|
|
37
|
+
# list() filter expression.
|
|
38
|
+
GCP_DI_MAP: dict[str, str] = {
|
|
39
|
+
"name": "name",
|
|
40
|
+
"image_name": "name",
|
|
41
|
+
"image_id": "name",
|
|
42
|
+
# The image family this image belongs to.
|
|
43
|
+
"family": "family",
|
|
44
|
+
# The status of the image ( READY | PENDING | FAILED | DELETING ).
|
|
45
|
+
"state": "status",
|
|
46
|
+
"status": "status",
|
|
47
|
+
# The image architecture ( X86_64 | ARM64 ).
|
|
48
|
+
"architecture": "architecture",
|
|
49
|
+
"description": "description",
|
|
50
|
+
# RFC3339 creation timestamp; supports comparison operators.
|
|
51
|
+
"creation_date": "creationTimestamp",
|
|
52
|
+
"creation_timestamp": "creationTimestamp",
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
# Query keys that only make sense for AWS/EC2. They have no Compute Engine
|
|
56
|
+
# equivalent, so they are dropped (with a log line) instead of being pushed
|
|
57
|
+
# into the post-query filter where they would exclude every result.
|
|
58
|
+
AWS_ONLY_QUERY_KEYS: set[str] = {
|
|
59
|
+
"root_device_type",
|
|
60
|
+
"root-device-type",
|
|
61
|
+
"virtualization_type",
|
|
62
|
+
"virtualization-type",
|
|
63
|
+
"ena_support",
|
|
64
|
+
"sriov_net_support",
|
|
65
|
+
"hypervisor",
|
|
66
|
+
"platform",
|
|
67
|
+
"is_public",
|
|
68
|
+
"owner_alias",
|
|
69
|
+
"owner_id",
|
|
70
|
+
"kernel_id",
|
|
71
|
+
"ramdisk_id",
|
|
72
|
+
"free_tier_eligible",
|
|
73
|
+
"image_allowed",
|
|
74
|
+
"manifest_location",
|
|
75
|
+
"product_code",
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
# Common owner aliases -> GCP public image projects.
|
|
79
|
+
GCP_OWNER_PROJECT_MAP: dict[str, str] = {
|
|
80
|
+
"debian": "debian-cloud",
|
|
81
|
+
"ubuntu": "ubuntu-os-cloud",
|
|
82
|
+
"rhel": "rhel-cloud",
|
|
83
|
+
"centos": "centos-cloud",
|
|
84
|
+
"rocky": "rocky-linux-cloud",
|
|
85
|
+
"fedora": "fedora-cloud",
|
|
86
|
+
"suse": "suse-cloud",
|
|
87
|
+
"windows": "windows-cloud",
|
|
88
|
+
"cos": "cos-cloud",
|
|
89
|
+
"google": "debian-cloud",
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
# AWS-specific owner aliases that must never be treated as GCP projects even
|
|
93
|
+
# though they are syntactically valid project ids.
|
|
94
|
+
AWS_ONLY_OWNER_ALIASES: set[str] = {"amazon", "aws-marketplace", "aws-backup-vault"}
|
|
95
|
+
|
|
96
|
+
# Valid GCP project id: 6-30 chars, lowercase letter first, letters/digits/hyphens.
|
|
97
|
+
_PROJECT_ID_RE = re.compile(r"^[a-z][a-z0-9-]{4,28}[a-z0-9]$")
|
|
98
|
+
|
|
99
|
+
_GCP_STATUS_MAP: dict[str, str] = {
|
|
100
|
+
"available": "READY",
|
|
101
|
+
"pending": "PENDING",
|
|
102
|
+
"failed": "FAILED",
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
_GCP_ARCH_MAP: dict[str, str] = {
|
|
106
|
+
"x86_64": "X86_64",
|
|
107
|
+
"amd64": "X86_64",
|
|
108
|
+
"i386": "X86_64",
|
|
109
|
+
"arm64": "ARM64",
|
|
110
|
+
"aarch64": "ARM64",
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _gcp_status(value: Any) -> str:
|
|
115
|
+
return _GCP_STATUS_MAP.get(str(value).lower(), str(value).upper())
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _gcp_architecture(value: Any) -> str:
|
|
119
|
+
return _GCP_ARCH_MAP.get(str(value).lower(), str(value).upper())
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _generic_architecture(value: Any) -> str:
|
|
123
|
+
"""Map a GCP architecture value back to the generic x86_64/arm64 form."""
|
|
124
|
+
v = str(value).lower()
|
|
125
|
+
return v if v in ("x86_64", "arm64") else "x86_64"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _make_credentials(session_config: Mapping[str, Any]) -> Any | None:
|
|
129
|
+
creds_cfg = session_config.get("credentials") or {}
|
|
130
|
+
key_file = creds_cfg.get("service_account_key_file")
|
|
131
|
+
if key_file:
|
|
132
|
+
from google.oauth2 import service_account # noqa: PLC0415
|
|
133
|
+
|
|
134
|
+
return service_account.Credentials.from_service_account_file(key_file)
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _make_images_client(session_config: Mapping[str, Any]) -> compute_v1.ImagesClient:
|
|
139
|
+
creds = _make_credentials(session_config)
|
|
140
|
+
return compute_v1.ImagesClient(credentials=creds) if creds else compute_v1.ImagesClient()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def resolve_project(session_config: Mapping[str, Any]) -> str | None:
|
|
144
|
+
"""Resolve the project id for SELF from the session configuration.
|
|
145
|
+
|
|
146
|
+
Falls back to the ``project_id`` recorded in a service account key file when
|
|
147
|
+
the config does not carry an explicit project.
|
|
148
|
+
"""
|
|
149
|
+
project = session_config.get("project")
|
|
150
|
+
if project:
|
|
151
|
+
return str(project)
|
|
152
|
+
creds_cfg = session_config.get("credentials") or {}
|
|
153
|
+
key_file = creds_cfg.get("service_account_key_file")
|
|
154
|
+
if key_file:
|
|
155
|
+
try:
|
|
156
|
+
with open(key_file) as f:
|
|
157
|
+
return cast(str | None, json.load(f).get("project_id"))
|
|
158
|
+
except (OSError, ValueError) as e:
|
|
159
|
+
log.warning(f"Could not read project_id from key file {key_file}: {e}")
|
|
160
|
+
return None
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def resolve_projects(
|
|
164
|
+
owners: Sequence[Any] | None, session_config: Mapping[str, Any]
|
|
165
|
+
) -> list[str]:
|
|
166
|
+
"""Map generic owner entries to GCP projects to search for images.
|
|
167
|
+
|
|
168
|
+
``self`` resolves to the session project; known distro aliases resolve to
|
|
169
|
+
the corresponding public image projects; entries that cannot be a GCP
|
|
170
|
+
project id (e.g. AWS account numbers, "amazon") are dropped with a warning.
|
|
171
|
+
"""
|
|
172
|
+
projects: list[str] = []
|
|
173
|
+
for owner in owners or [SELF]:
|
|
174
|
+
o = str(owner).strip().strip('"')
|
|
175
|
+
if o.lower() == SELF:
|
|
176
|
+
self_project = resolve_project(session_config)
|
|
177
|
+
if not self_project:
|
|
178
|
+
raise ValueError(
|
|
179
|
+
"Owner 'self' requires a project: set project_id on the GCP runtime "
|
|
180
|
+
"(stage 63: a runtime declares no credentials, so no key file can supply one)."
|
|
181
|
+
)
|
|
182
|
+
projects.append(self_project)
|
|
183
|
+
elif o.lower() in GCP_OWNER_PROJECT_MAP:
|
|
184
|
+
projects.append(GCP_OWNER_PROJECT_MAP[o.lower()])
|
|
185
|
+
elif o.lower() in AWS_ONLY_OWNER_ALIASES:
|
|
186
|
+
log.debug(f"Dropping AWS-only owner alias {o!r} for GCP query.")
|
|
187
|
+
elif _PROJECT_ID_RE.match(o):
|
|
188
|
+
projects.append(o)
|
|
189
|
+
else:
|
|
190
|
+
log.warning(f"Owner {o!r} is not a valid GCP project or known alias; skipping.")
|
|
191
|
+
# Dedup, preserving order.
|
|
192
|
+
seen: set[str] = set()
|
|
193
|
+
return [p for p in projects if not (p in seen or seen.add(p))]
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def build_filter_expression(terms: Mapping[str, Any]) -> str:
|
|
197
|
+
"""Build a Compute Engine list() filter expression from field/value terms."""
|
|
198
|
+
parts: list[str] = []
|
|
199
|
+
for k, v in terms.items():
|
|
200
|
+
if isinstance(v, bool):
|
|
201
|
+
parts.append(f"({k} = {str(v).lower()})")
|
|
202
|
+
else:
|
|
203
|
+
parts.append(f'({k} = "{v}")')
|
|
204
|
+
return " AND ".join(parts)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def remap_for_image_query(
|
|
208
|
+
rc: OSBuilderBaseImageBuilderSubconfig,
|
|
209
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
210
|
+
"""Convert a generic image-builder subconfig query to a GCP image query.
|
|
211
|
+
|
|
212
|
+
Returns a tuple of (query, missed) where ``query`` carries the projects to
|
|
213
|
+
search and a Compute Engine filter expression, and ``missed`` holds
|
|
214
|
+
unmappable keys applied as a post-query filter (mirrors the AWS remap).
|
|
215
|
+
"""
|
|
216
|
+
ret: dict[str, Any] = {}
|
|
217
|
+
owners: Sequence[Any] = rc.get_owners() if rc else []
|
|
218
|
+
ret["projects"] = list(owners) # resolved to real projects at query time
|
|
219
|
+
missed: dict[str, Any] = {}
|
|
220
|
+
filter_terms: dict[str, Any] = {}
|
|
221
|
+
|
|
222
|
+
for topk, topv in rc.get_query().items():
|
|
223
|
+
if topk == "owners":
|
|
224
|
+
ret["projects"] = topv if isinstance(topv, list) else [topv]
|
|
225
|
+
continue
|
|
226
|
+
if topk == "filters":
|
|
227
|
+
if not isinstance(topv, dict):
|
|
228
|
+
raise ValueError(f"Expected 'filters' value to be a dict, got {type(topv)}")
|
|
229
|
+
terms = dict(topv)
|
|
230
|
+
# Ensure we only get READY images, but allow override.
|
|
231
|
+
terms["state"] = terms.get("state", "available")
|
|
232
|
+
for _k, _v in terms.items():
|
|
233
|
+
if _k in AWS_ONLY_QUERY_KEYS or _k.startswith("block_device_mapping"):
|
|
234
|
+
log.debug(f"Dropping AWS-only image query key {_k!r} for GCP query.")
|
|
235
|
+
continue
|
|
236
|
+
if _k.startswith("tag:"):
|
|
237
|
+
filter_terms[f"labels.{_k[4:]}"] = _v
|
|
238
|
+
continue
|
|
239
|
+
field_name = GCP_DI_MAP.get(_k)
|
|
240
|
+
if not field_name:
|
|
241
|
+
missed[_k] = _v
|
|
242
|
+
continue
|
|
243
|
+
if field_name == "status":
|
|
244
|
+
_v = _gcp_status(_v)
|
|
245
|
+
elif field_name == "architecture":
|
|
246
|
+
_v = _gcp_architecture(_v)
|
|
247
|
+
filter_terms[field_name] = _v
|
|
248
|
+
continue
|
|
249
|
+
field_name = GCP_DI_MAP.get(topk)
|
|
250
|
+
if not field_name:
|
|
251
|
+
if topk in AWS_ONLY_QUERY_KEYS:
|
|
252
|
+
log.debug(f"Dropping AWS-only image query key {topk!r} for GCP query.")
|
|
253
|
+
else:
|
|
254
|
+
missed[topk] = topv
|
|
255
|
+
continue
|
|
256
|
+
if field_name == "status":
|
|
257
|
+
topv = _gcp_status(topv)
|
|
258
|
+
elif field_name == "architecture":
|
|
259
|
+
topv = _gcp_architecture(topv)
|
|
260
|
+
filter_terms[field_name] = topv
|
|
261
|
+
|
|
262
|
+
if "status" not in filter_terms:
|
|
263
|
+
filter_terms["status"] = "READY"
|
|
264
|
+
ret["filter"] = build_filter_expression(filter_terms)
|
|
265
|
+
return (ret, missed)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _image_to_dict(image: compute_v1.Image, project: str) -> dict[str, Any]:
|
|
269
|
+
d = cast(dict[str, Any], compute_v1.Image.to_dict(image))
|
|
270
|
+
d["project"] = project
|
|
271
|
+
return d
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def query_image(
|
|
275
|
+
query: dict[str, Any],
|
|
276
|
+
post_query_filter: dict[str, Any] | None = None,
|
|
277
|
+
session_config: dict[str, Any] | None = None,
|
|
278
|
+
images_client: Any | None = None,
|
|
279
|
+
) -> dict[str, Any] | None:
|
|
280
|
+
"""
|
|
281
|
+
Query GCP for a single Compute Engine image matching the provided criteria.
|
|
282
|
+
|
|
283
|
+
Lists images in each candidate project using the provided filter
|
|
284
|
+
expression. If multiple images match, the most recently created one is
|
|
285
|
+
returned.
|
|
286
|
+
|
|
287
|
+
Parameters
|
|
288
|
+
----------
|
|
289
|
+
query : dict[str, Any]
|
|
290
|
+
Dictionary containing query parameters:
|
|
291
|
+
|
|
292
|
+
- ``projects`` (list[str]): Owners/projects to search. ``self``
|
|
293
|
+
resolves to the session project; known distro aliases (``debian``,
|
|
294
|
+
``rhel``, ...) resolve to the public image projects.
|
|
295
|
+
- ``filter`` (str): Compute Engine list() filter expression, e.g.
|
|
296
|
+
``(name = "debian-11-*") AND (status = "READY")``.
|
|
297
|
+
- ``family`` (str): Image family to resolve via getFromFamily when no
|
|
298
|
+
filter expression is provided.
|
|
299
|
+
|
|
300
|
+
post_query_filter : dict[str, Any], optional
|
|
301
|
+
Exact-match constraints applied to the result dictionaries after the
|
|
302
|
+
API query (keys are proto field names, e.g. ``family``).
|
|
303
|
+
|
|
304
|
+
session_config : dict[str, Any], optional
|
|
305
|
+
Session configuration: ``project``, optional ``credentials`` carrying
|
|
306
|
+
``service_account_key_file``, optional ``region``/``zone``.
|
|
307
|
+
|
|
308
|
+
images_client : Any, optional
|
|
309
|
+
Pre-built ImagesClient (used for testing/injection).
|
|
310
|
+
|
|
311
|
+
Returns
|
|
312
|
+
-------
|
|
313
|
+
dict[str, Any] | None
|
|
314
|
+
Dictionary of the image resource (proto field names, e.g. ``name``,
|
|
315
|
+
``self_link``, ``family``, ``architecture``, ``creation_timestamp``)
|
|
316
|
+
plus ``project``, or None if nothing matched.
|
|
317
|
+
|
|
318
|
+
Raises
|
|
319
|
+
------
|
|
320
|
+
ImageQueryError
|
|
321
|
+
If there is an error communicating with GCP.
|
|
322
|
+
ValueError
|
|
323
|
+
If the query or session configuration is empty.
|
|
324
|
+
"""
|
|
325
|
+
if not query:
|
|
326
|
+
raise ValueError("Query dictionary cannot be empty")
|
|
327
|
+
if not session_config:
|
|
328
|
+
raise ValueError("Session configuration cannot be None or empty")
|
|
329
|
+
|
|
330
|
+
try:
|
|
331
|
+
client = images_client or _make_images_client(session_config)
|
|
332
|
+
except (gcp_exceptions.GoogleAPIError, OSError, ValueError) as e:
|
|
333
|
+
raise ImageQueryError(f"Failed to create Compute images client: {e}") from e
|
|
334
|
+
|
|
335
|
+
projects = resolve_projects(query.get("projects"), session_config)
|
|
336
|
+
if not projects:
|
|
337
|
+
raise ValueError(f"No usable GCP projects resolved from query {query}")
|
|
338
|
+
|
|
339
|
+
results: list[dict[str, Any]] = []
|
|
340
|
+
filter_expr = query.get("filter", "")
|
|
341
|
+
family = query.get("family")
|
|
342
|
+
for project in projects:
|
|
343
|
+
if filter_expr:
|
|
344
|
+
results.extend(_query_by_filters(client, project, filter_expr))
|
|
345
|
+
elif family:
|
|
346
|
+
img = _query_by_family(client, project, family)
|
|
347
|
+
if img:
|
|
348
|
+
results.append(img)
|
|
349
|
+
else:
|
|
350
|
+
raise ValueError(
|
|
351
|
+
"No valid query parameters provided. Must include 'filter' or 'family'."
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
ilist: list[dict[str, Any]] = []
|
|
355
|
+
for r in results:
|
|
356
|
+
if post_query_filter:
|
|
357
|
+
match = True
|
|
358
|
+
for k, v in post_query_filter.items():
|
|
359
|
+
if r.get(k) != v:
|
|
360
|
+
match = False
|
|
361
|
+
break
|
|
362
|
+
if not match:
|
|
363
|
+
continue
|
|
364
|
+
ilist.append(r)
|
|
365
|
+
if not ilist:
|
|
366
|
+
return None
|
|
367
|
+
if len(ilist) > 1:
|
|
368
|
+
ilist.sort(key=lambda x: x.get("creation_timestamp", ""), reverse=True)
|
|
369
|
+
|
|
370
|
+
return ilist[0]
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _query_by_family(client: Any, project: str, family: str) -> dict[str, Any] | None:
|
|
374
|
+
"""Resolve the latest non-deprecated image of a family in a project."""
|
|
375
|
+
try:
|
|
376
|
+
image = client.get_from_family(project=project, family=family)
|
|
377
|
+
except gcp_exceptions.NotFound:
|
|
378
|
+
return None
|
|
379
|
+
except gcp_exceptions.GoogleAPIError as e:
|
|
380
|
+
raise ImageQueryError(
|
|
381
|
+
f"Failed to query image family '{family}' in project '{project}': {e}"
|
|
382
|
+
) from e
|
|
383
|
+
return _image_to_dict(image, project)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _query_by_filters(client: Any, project: str, filter_expr: str) -> list[dict[str, Any]]:
|
|
387
|
+
"""List images in a project matching a filter expression (paged)."""
|
|
388
|
+
log.debug(f"Querying project {project} for images with filter: {filter_expr}")
|
|
389
|
+
try:
|
|
390
|
+
request = compute_v1.ListImagesRequest(project=project, filter=filter_expr)
|
|
391
|
+
ilist = [_image_to_dict(image, project) for image in client.list(request=request)]
|
|
392
|
+
except gcp_exceptions.GoogleAPIError as e:
|
|
393
|
+
raise ImageQueryError(
|
|
394
|
+
f"Failed to query images in project '{project}' with filter: {e}"
|
|
395
|
+
) from e
|
|
396
|
+
log.info(f"Found {len(ilist)} images in project {project} matching filter.")
|
|
397
|
+
return ilist
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def get_image_owner(image_info: Mapping[str, Any]) -> dict[str, Any] | None:
|
|
401
|
+
"""Return the owning project of an image result (the GCP owner analog)."""
|
|
402
|
+
project = image_info.get("project")
|
|
403
|
+
if not project:
|
|
404
|
+
self_link = image_info.get("self_link", "")
|
|
405
|
+
m = re.search(r"/projects/([^/]+)/", str(self_link))
|
|
406
|
+
project = m.group(1) if m else None
|
|
407
|
+
if not project:
|
|
408
|
+
return None
|
|
409
|
+
return {"project": project}
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def get_network_map_and_default_network(
|
|
414
|
+
session_config: dict[str, Any],
|
|
415
|
+
clients: Mapping[str, Any] | None = None,
|
|
416
|
+
) -> tuple[dict, str | None, dict]:
|
|
417
|
+
"""Build the network/subnetwork map for a project (GCP analog of the AWS
|
|
418
|
+
VPC map) plus the default network name and the firewall rules.
|
|
419
|
+
|
|
420
|
+
Returns (network_map, default_network_name, firewall_rules) where
|
|
421
|
+
network_map is keyed by network name with a flat ``subnets`` list, and
|
|
422
|
+
firewall_rules is keyed by firewall rule name (the closest analog GCP has
|
|
423
|
+
to security groups).
|
|
424
|
+
"""
|
|
425
|
+
project = resolve_project(session_config)
|
|
426
|
+
if not project:
|
|
427
|
+
raise ValueError(
|
|
428
|
+
"A project is required to enumerate GCP networks; set project_id on "
|
|
429
|
+
"the GCP runtime."
|
|
430
|
+
)
|
|
431
|
+
try:
|
|
432
|
+
creds = _make_credentials(session_config)
|
|
433
|
+
kwargs: dict[str, Any] = {"credentials": creds} if creds else {}
|
|
434
|
+
clients = clients or {}
|
|
435
|
+
networks_client = clients.get("networks") or compute_v1.NetworksClient(**kwargs)
|
|
436
|
+
subnets_client = clients.get("subnetworks") or compute_v1.SubnetworksClient(**kwargs)
|
|
437
|
+
routes_client = clients.get("routes") or compute_v1.RoutesClient(**kwargs)
|
|
438
|
+
firewalls_client = clients.get("firewalls") or compute_v1.FirewallsClient(**kwargs)
|
|
439
|
+
|
|
440
|
+
# Fetch data in bulk
|
|
441
|
+
networks = list(networks_client.list(project=project))
|
|
442
|
+
routes = list(routes_client.list(project=project))
|
|
443
|
+
allfw = {
|
|
444
|
+
fw.name: compute_v1.Firewall.to_dict(fw)
|
|
445
|
+
for fw in firewalls_client.list(project=project)
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
network_map: dict[str, Any] = {}
|
|
449
|
+
default_network_name: str | None = None
|
|
450
|
+
|
|
451
|
+
for net in networks:
|
|
452
|
+
network_map[net.name] = {"subnets": []}
|
|
453
|
+
# GCP has no "default network" flag; the auto-created network is
|
|
454
|
+
# conventionally named "default".
|
|
455
|
+
if net.name == "default":
|
|
456
|
+
default_network_name = net.name
|
|
457
|
+
|
|
458
|
+
# --- ROUTE MAPPING LOGIC ---
|
|
459
|
+
# GCP routes are network-scoped (not subnet-scoped): a network whose
|
|
460
|
+
# default route targets the internet gateway makes all of its subnets
|
|
461
|
+
# internet-capable.
|
|
462
|
+
network_is_public: dict[str, bool] = {}
|
|
463
|
+
for route in routes:
|
|
464
|
+
net_name = str(route.network).rsplit("/", 1)[-1]
|
|
465
|
+
is_default_dest = route.dest_range in ("0.0.0.0/0", "::/0")
|
|
466
|
+
if is_default_dest and route.next_hop_gateway.endswith(
|
|
467
|
+
"/global/gateways/default-internet-gateway"
|
|
468
|
+
):
|
|
469
|
+
network_is_public[net_name] = True
|
|
470
|
+
|
|
471
|
+
# --- PROCESS SUBNETS ---
|
|
472
|
+
agg = subnets_client.aggregated_list(project=project)
|
|
473
|
+
for _scope, scoped_list in agg:
|
|
474
|
+
for sn in getattr(scoped_list, "subnetworks", []) or []:
|
|
475
|
+
net_name = str(sn.network).rsplit("/", 1)[-1]
|
|
476
|
+
if net_name not in network_map:
|
|
477
|
+
continue
|
|
478
|
+
subnet_data = cast(dict[str, Any], compute_v1.Subnetwork.to_dict(sn))
|
|
479
|
+
s_name = subnet_data.pop("name", None)
|
|
480
|
+
cidr = subnet_data.pop("ip_cidr_range", None)
|
|
481
|
+
self_link = subnet_data.get("self_link", "")
|
|
482
|
+
region = str(subnet_data.get("region", "")).rsplit("/", 1)[-1]
|
|
483
|
+
|
|
484
|
+
formatted_subnet = {
|
|
485
|
+
"subnet_id": s_name,
|
|
486
|
+
"self_link": self_link,
|
|
487
|
+
"region": region,
|
|
488
|
+
"cidr": cidr,
|
|
489
|
+
"is_public": network_is_public.get(net_name, False),
|
|
490
|
+
"tags": {}, # GCP subnetworks carry no labels/tags
|
|
491
|
+
"config": subnet_data, # All other remaining raw attributes
|
|
492
|
+
}
|
|
493
|
+
network_map[net_name]["subnets"].append(formatted_subnet)
|
|
494
|
+
|
|
495
|
+
return network_map, default_network_name, allfw
|
|
496
|
+
|
|
497
|
+
except (gcp_exceptions.GoogleAPIError, OSError) as e:
|
|
498
|
+
raise ValueError(f"GCP Error: {e}") from e
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Mykel Alvis <mykelalvis@infrastructurebuilder.org>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
from cs_image_system.base.basic.abstract_plugin_metadata import AbstractPluginMetadata
|
|
8
|
+
from cs_image_system.base.protocols.plugin_metadata import PluginMetadataProtocol
|
|
9
|
+
from .gcp_runtime_builders import GCP_RUNTIME, GCPCLIVersionChecker, GCPCloudBuilder
|
|
10
|
+
from .gcp_runtime_models import GCPCloudBuilderModel
|
|
11
|
+
|
|
12
|
+
log = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
class GcpRuntimePluginMetadata(AbstractPluginMetadata,PluginMetadataProtocol):
|
|
15
|
+
def __init__(self) -> None:
|
|
16
|
+
super().__init__( "1", "3.13",
|
|
17
|
+
{GCP_RUNTIME: [
|
|
18
|
+
GCPCloudBuilderModel,
|
|
19
|
+
GCPCloudBuilder,
|
|
20
|
+
GCPCLIVersionChecker
|
|
21
|
+
]},
|
|
22
|
+
{GCPCloudBuilderModel: GCPCloudBuilder}
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def initialize():
|
|
27
|
+
log.debug("Initializing GCP Runtime Plugin")
|
|
28
|
+
return GcpRuntimePluginMetadata()
|