cs-image-system-gcloud-runtime-plugin 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cs_image_system/gcloud_runtime/__init__.py +3 -0
- cs_image_system/gcloud_runtime/bootstrap.py +351 -0
- cs_image_system/gcloud_runtime/gcp_packer_source.py +135 -0
- cs_image_system/gcloud_runtime/gcp_provider_specific_image.py +38 -0
- cs_image_system/gcloud_runtime/gcp_runtime_builders.py +608 -0
- cs_image_system/gcloud_runtime/gcp_runtime_models.py +168 -0
- cs_image_system/gcloud_runtime/gcp_utils.py +498 -0
- cs_image_system/gcloud_runtime/main.py +28 -0
- cs_image_system/py.typed +0 -0
- cs_image_system_gcloud_runtime_plugin-0.1.1.dist-info/METADATA +1168 -0
- cs_image_system_gcloud_runtime_plugin-0.1.1.dist-info/RECORD +14 -0
- cs_image_system_gcloud_runtime_plugin-0.1.1.dist-info/WHEEL +4 -0
- cs_image_system_gcloud_runtime_plugin-0.1.1.dist-info/entry_points.txt +5 -0
- cs_image_system_gcloud_runtime_plugin-0.1.1.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,608 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Mykel Alvis <mykelalvis@infrastructurebuilder.org>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
from cs_image_system.base.basic.abstract_version_checker import AbstractVersionChecker
|
|
11
|
+
from cs_image_system.base.basic.asset import AssetSet
|
|
12
|
+
from cs_image_system.base.basic.builder_base_cloud import CloudBuilderBase
|
|
13
|
+
from cs_image_system.base.constants import VCT
|
|
14
|
+
from cs_image_system.base.lifecycle import ExecutionLifecyclePhase
|
|
15
|
+
from cs_image_system.base.models.os_builder_runtime_config import OSBuilderBaseImageBuilderSubconfig
|
|
16
|
+
from cs_image_system.base.models.provider_specific_image import ProviderSpecificImage
|
|
17
|
+
from cs_image_system.base import power_state
|
|
18
|
+
from cs_image_system.base.protocols.plugin_metadata import PluginArtifactProtocol
|
|
19
|
+
|
|
20
|
+
from .gcp_provider_specific_image import GcpProviderSpecificImage
|
|
21
|
+
from .gcp_runtime_models import GCPCloudBuilderModel
|
|
22
|
+
from cs_image_system.gcloud_runtime import gcp_utils
|
|
23
|
+
from .gcp_utils import remap_for_image_query, query_image
|
|
24
|
+
|
|
25
|
+
GCP_RUNTIME: str = "gcloud"
|
|
26
|
+
|
|
27
|
+
log = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class GCPCloudBuilder(CloudBuilderBase[GCPCloudBuilderModel], PluginArtifactProtocol):
|
|
31
|
+
"""GCP cloud provider implementation."""
|
|
32
|
+
|
|
33
|
+
@classmethod
|
|
34
|
+
def csis_name(cls) -> str:
|
|
35
|
+
return GCP_RUNTIME
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def model(self) -> GCPCloudBuilderModel:
|
|
39
|
+
return self._model # type: ignore
|
|
40
|
+
|
|
41
|
+
def provider_specific_image_class(self) -> type[ProviderSpecificImage]:
|
|
42
|
+
return GcpProviderSpecificImage
|
|
43
|
+
|
|
44
|
+
def query_provider_image(self, os_builder: OSBuilderBaseImageBuilderSubconfig) -> tuple[str, str, Mapping[str,Any]] | None:
|
|
45
|
+
retval: str | None = None
|
|
46
|
+
q, missed = remap_for_image_query(os_builder) if os_builder else ({}, {})
|
|
47
|
+
kv = query_image(
|
|
48
|
+
query=q,
|
|
49
|
+
post_query_filter=missed,
|
|
50
|
+
session_config=self.model.self_to_gcp_client_config(),
|
|
51
|
+
)
|
|
52
|
+
if kv:
|
|
53
|
+
# The GCP image identifier is its name (unique within a project).
|
|
54
|
+
retval = kv.get("name", None)
|
|
55
|
+
else:
|
|
56
|
+
kv = {}
|
|
57
|
+
|
|
58
|
+
ownerstr = "self"
|
|
59
|
+
if retval:
|
|
60
|
+
owner = gcp_utils.get_image_owner(kv)
|
|
61
|
+
if not owner:
|
|
62
|
+
raise ValueError(f"Could not get owning project for image {retval}, got {kv}")
|
|
63
|
+
_p = owner.get("project", None)
|
|
64
|
+
if not _p:
|
|
65
|
+
raise ValueError(f"Could not get owning project for image {retval}, got {owner}")
|
|
66
|
+
ownerstr = str(_p)
|
|
67
|
+
return (retval, ownerstr, kv) if retval else None
|
|
68
|
+
|
|
69
|
+
def query_provider_image_by_id(self, os_builder: OSBuilderBaseImageBuilderSubconfig,
|
|
70
|
+
image_id: str) -> tuple[str, str, Mapping[str, Any]] | None:
|
|
71
|
+
"""The image an entry pins by ``image_id`` (stage 63): a GCE image is
|
|
72
|
+
named uniquely within its project, so the name is searched in the
|
|
73
|
+
entry's owner projects, as the vendor query is."""
|
|
74
|
+
q, _missed = remap_for_image_query(os_builder)
|
|
75
|
+
q = {k: v for k, v in q.items() if k not in ("family", "filter")}
|
|
76
|
+
q["filter"] = f'name = "{image_id}"'
|
|
77
|
+
kv = query_image(query=q, session_config=self.model.self_to_gcp_client_config())
|
|
78
|
+
if not kv:
|
|
79
|
+
return None
|
|
80
|
+
status = str(kv.get("status") or "")
|
|
81
|
+
if status and status != "READY":
|
|
82
|
+
# stage 67: the vendor query insists on READY; a pinned image must too
|
|
83
|
+
raise ValueError(f"GCP runtime {self.get_name()}: pinned image {image_id!r} is {status}, not READY; "
|
|
84
|
+
"a bake from it would fail")
|
|
85
|
+
owner = gcp_utils.get_image_owner(kv) or {}
|
|
86
|
+
return (str(kv.get("name") or image_id), str(owner.get("project") or "self"), kv)
|
|
87
|
+
|
|
88
|
+
# stage 24: `resolve_image_for_os_builder` was removed here. It was defined
|
|
89
|
+
# only on this builder, had no callers and no base-class declaration, and
|
|
90
|
+
# returned image_from_query_result(...) -- which could never return an
|
|
91
|
+
# Image. The AWS twin was already commented out. Ledger 78.
|
|
92
|
+
|
|
93
|
+
def generate_items_during(self, phase: ExecutionLifecyclePhase) -> AssetSet:
|
|
94
|
+
return super().generate_items_during(phase)
|
|
95
|
+
|
|
96
|
+
# --------------------------------------------- session mechanism (IAP)
|
|
97
|
+
IAP = "iap"
|
|
98
|
+
|
|
99
|
+
def gcloud_binary(self) -> str:
|
|
100
|
+
"""The gcloud this runtime runs: the ``binary`` of the
|
|
101
|
+
``cfg/executables.yml`` entry its ``executable`` names (default
|
|
102
|
+
``gcloud``), the same one ``validate`` version-checks (stage 63 item
|
|
103
|
+
14; every call used to be a bare ``gcloud`` from PATH)."""
|
|
104
|
+
name = self.model.get_executable() or "gcloud"
|
|
105
|
+
exe = (self._get_context().executables or {}).get(name)
|
|
106
|
+
if exe is None:
|
|
107
|
+
raise ValueError(f"GCP runtime {self.get_name()}: executable {name!r} is not declared in "
|
|
108
|
+
f"cfg/executables.yml; declare it (name: {name}, binary: <path to gcloud>)")
|
|
109
|
+
return str(exe.binary or exe.name)
|
|
110
|
+
|
|
111
|
+
def session_mechanism(self) -> str | None:
|
|
112
|
+
mech = getattr(self.model, "session_mechanism", None)
|
|
113
|
+
if not mech:
|
|
114
|
+
return None
|
|
115
|
+
mech = str(mech).strip().lower()
|
|
116
|
+
if mech != self.IAP:
|
|
117
|
+
raise ValueError(f"GCP runtime {self.get_name()}: unknown session mechanism {mech!r} "
|
|
118
|
+
f"(supported: {self.IAP})")
|
|
119
|
+
return mech
|
|
120
|
+
|
|
121
|
+
def session_agent_commands(self, os_family: str | None = None) -> list[str]:
|
|
122
|
+
"""IAP TCP forwarding needs only sshd and the Google guest agent (key
|
|
123
|
+
provisioning via metadata); the IAP firewall rule (35.235.240.0/20 ->
|
|
124
|
+
22) and roles/iap.tunnelResourceAccessor are operator prerequisites."""
|
|
125
|
+
if self.session_mechanism() != self.IAP:
|
|
126
|
+
return []
|
|
127
|
+
fam = (os_family or "").lower()
|
|
128
|
+
install = ("sudo apt-get -o DPkg::Lock::Timeout=600 install -y google-guest-agent" if fam in ("debian", "ubuntu")
|
|
129
|
+
else "sudo yum install -y google-guest-agent")
|
|
130
|
+
return ["# debug session mechanism: IAP TCP forwarding (operator: IAP firewall rule + tunnelResourceAccessor)",
|
|
131
|
+
f"command -v google_guest_agent >/dev/null 2>&1 || {install}",
|
|
132
|
+
"sudo systemctl enable google-guest-agent sshd 2>/dev/null || sudo systemctl enable google-guest-agent ssh"]
|
|
133
|
+
|
|
134
|
+
def session_verify_commands(self, os_family: str | None = None) -> list[str]:
|
|
135
|
+
if self.session_mechanism() != self.IAP:
|
|
136
|
+
return []
|
|
137
|
+
return ["# verify: guest agent baked for IAP sessions",
|
|
138
|
+
"command -v google_guest_agent >/dev/null 2>&1 || test -x /usr/bin/google_guest_agent",
|
|
139
|
+
"systemctl is-enabled google-guest-agent >/dev/null 2>&1"]
|
|
140
|
+
|
|
141
|
+
def default_bake_user(self, family: str) -> str | None:
|
|
142
|
+
# googlecompute has no vendor default user: it creates the account
|
|
143
|
+
# from metadata keys, so any name works; `packer` is the one every GCE
|
|
144
|
+
# chain bakes as unless something names another (finding 43)
|
|
145
|
+
return "packer"
|
|
146
|
+
|
|
147
|
+
def label_key_problem(self, key: str) -> str | None:
|
|
148
|
+
"""Why ``key`` cannot be a GCE label key once it is made one, or None.
|
|
149
|
+
GCE wants a lowercase letter first; ``gce_label`` lowercases and
|
|
150
|
+
replaces what it cannot keep, but it cannot invent a first letter, so
|
|
151
|
+
a key such as ``2024`` failed at apply (stage 63: refused at validate)."""
|
|
152
|
+
from .gcp_packer_source import gce_label
|
|
153
|
+
label = gce_label(key)
|
|
154
|
+
if not label or not ("a" <= label[0] <= "z"):
|
|
155
|
+
return f"GCE label keys must start with a lowercase letter; {key!r} becomes {label!r}"
|
|
156
|
+
return None
|
|
157
|
+
|
|
158
|
+
def one_bake_user_per_chain(self) -> bool:
|
|
159
|
+
return True # finding 43
|
|
160
|
+
|
|
161
|
+
def bake_ssh_username(self, image=None) -> str | None:
|
|
162
|
+
# finding 48: the provisioner must name the user the source bakes as.
|
|
163
|
+
# stage 63 item 23: both ask the one resolver, so an entry-level user
|
|
164
|
+
# no longer bakes as itself while ansible connects as `packer`.
|
|
165
|
+
if image is None:
|
|
166
|
+
return self.default_bake_user("")
|
|
167
|
+
from cs_image_system.base.bake_user import resolve_bake_user
|
|
168
|
+
return resolve_bake_user(self._get_context(), image, self.get_name())
|
|
169
|
+
|
|
170
|
+
def bake_finalize_commands(self, os_family: str | None = None) -> list[str]:
|
|
171
|
+
# finding 47 (found live at the gce-test launch): a baked image
|
|
172
|
+
# carries the bake's ssh user, and at first boot the guest agent
|
|
173
|
+
# (manager + core-plugin generation) removes users absent from
|
|
174
|
+
# metadata -- if that user's google-sudoers membership is missing
|
|
175
|
+
# (the agents race each other over it during bakes), gpasswd exits 3
|
|
176
|
+
# and the agent ABORTS its entire metadata-ssh-key setup, so no
|
|
177
|
+
# operator key is ever provisioned and every SSH/IAP session is
|
|
178
|
+
# refused. Guarantee the membership as the bake's last act so the
|
|
179
|
+
# first-boot removal always succeeds cleanly.
|
|
180
|
+
return ["# finding 47: the build user must leave the image a clean google-sudoers member",
|
|
181
|
+
'getent group google-sudoers >/dev/null 2>&1 && sudo gpasswd -a "$(whoami)" google-sudoers >/dev/null || true']
|
|
182
|
+
|
|
183
|
+
def query_images(self, series: list[str]) -> list[dict[str, Any]]:
|
|
184
|
+
"""Every image in the project labelled with ``csis_series`` (read-only)."""
|
|
185
|
+
from cs_image_system.base.lineage import TAG_PREFIX
|
|
186
|
+
from .gcp_utils import _make_images_client, resolve_project
|
|
187
|
+
cfg = self.model.self_to_gcp_client_config()
|
|
188
|
+
project = resolve_project(cfg)
|
|
189
|
+
if not project:
|
|
190
|
+
raise RuntimeError(f"GCP runtime {self.get_name()}: no project to query")
|
|
191
|
+
client = _make_images_client(cfg)
|
|
192
|
+
out: list[dict[str, Any]] = []
|
|
193
|
+
from google.cloud import compute_v1 # noqa: PLC0415
|
|
194
|
+
request = compute_v1.ListImagesRequest(project=project, filter=f"labels.{TAG_PREFIX}series:*")
|
|
195
|
+
# stage 63: only the images of the series asked about (the argument
|
|
196
|
+
# was ignored, so every labelled image in the project came back); a
|
|
197
|
+
# series label carries gce_label(<series>), so compare in that form
|
|
198
|
+
from .gcp_packer_source import gce_label
|
|
199
|
+
wanted = {gce_label(s) for s in (series or [])}
|
|
200
|
+
for img in client.list(request=request):
|
|
201
|
+
labels = dict(getattr(img, "labels", {}) or {})
|
|
202
|
+
if wanted and labels.get(f"{TAG_PREFIX}series") not in wanted:
|
|
203
|
+
continue
|
|
204
|
+
out.append({"image_id": img.name, "name": img.name, "state": str(getattr(img, "status", "")),
|
|
205
|
+
"created": str(getattr(img, "creation_timestamp", "")), "tags": labels})
|
|
206
|
+
return out
|
|
207
|
+
|
|
208
|
+
def serial_console(self, instance_name: str) -> str:
|
|
209
|
+
"""The instance's serial port 1 output (read-only)."""
|
|
210
|
+
from .gcp_packer_source import gce_name
|
|
211
|
+
from .gcp_utils import _make_credentials, resolve_project
|
|
212
|
+
from google.cloud import compute_v1 # noqa: PLC0415
|
|
213
|
+
cfg = self.model.self_to_gcp_client_config()
|
|
214
|
+
project, zone = resolve_project(cfg), getattr(self.model, "zone", None)
|
|
215
|
+
if not project or not zone:
|
|
216
|
+
raise RuntimeError(f"GCP runtime {self.get_name()}: no project/zone to read a serial console")
|
|
217
|
+
creds = _make_credentials(cfg)
|
|
218
|
+
kw = {"credentials": creds} if creds else {}
|
|
219
|
+
out = compute_v1.InstancesClient(**kw).get_serial_port_output(
|
|
220
|
+
project=project, zone=zone, instance=gce_name(instance_name))
|
|
221
|
+
return str(getattr(out, "contents", "") or "")
|
|
222
|
+
|
|
223
|
+
def verify_instance(self, instance_name: str, expected_build: str | None = None,
|
|
224
|
+
expect_mounts: int = 0, timeout: int = 600,
|
|
225
|
+
mount_points: list[str] | None = None) -> dict[str, Any]:
|
|
226
|
+
"""stage 10.2 -- what `just gce-verify` did by hand: poll the serial
|
|
227
|
+
console until the guest agent reports the startup scripts finished
|
|
228
|
+
(agent >= 20260715: "Finished running startup scripts"; older:
|
|
229
|
+
"startup-script exit status 0"), fail fast on an explicit failure
|
|
230
|
+
line, then compare the booted image with the expected build and
|
|
231
|
+
read each declared mount point's own line from the startup script
|
|
232
|
+
(hygiene XII item 22: the kernel's clean XFS mounts counted the
|
|
233
|
+
image's own filesystems, so the check could not fail); the count
|
|
234
|
+
remains for a caller that names no mount points."""
|
|
235
|
+
import re # noqa: PLC0415
|
|
236
|
+
import time # noqa: PLC0415
|
|
237
|
+
checks: list[dict[str, Any]] = []
|
|
238
|
+
deadline = time.time() + timeout
|
|
239
|
+
out = ""
|
|
240
|
+
while True:
|
|
241
|
+
out = self.serial_console(instance_name)
|
|
242
|
+
if re.search(r"startup-script exit status [1-9]|startup-script.*(failed|error)", out, re.I):
|
|
243
|
+
checks.append({"name": "startup scripts", "ok": False, "detail": "an explicit failure line"})
|
|
244
|
+
break
|
|
245
|
+
if re.search(r"Finished running startup scripts|startup-script exit status 0", out):
|
|
246
|
+
checks.append({"name": "startup scripts", "ok": True, "detail": "finished"})
|
|
247
|
+
break
|
|
248
|
+
if time.time() >= deadline:
|
|
249
|
+
checks.append({"name": "startup scripts", "ok": False,
|
|
250
|
+
"detail": f"no completion on the serial console within {timeout}s"})
|
|
251
|
+
break
|
|
252
|
+
time.sleep(15)
|
|
253
|
+
booted = self.query_instance_boot_image(instance_name)
|
|
254
|
+
if expected_build:
|
|
255
|
+
checks.append({"name": "booted image", "ok": booted == expected_build,
|
|
256
|
+
"detail": f"booted {booted}, expected {expected_build}"})
|
|
257
|
+
else:
|
|
258
|
+
checks.append({"name": "booted image", "ok": booted is not None, "detail": f"booted {booted}"})
|
|
259
|
+
if mount_points:
|
|
260
|
+
from cs_image_system.base.launch_params import MOUNT_MARK # noqa: PLC0415
|
|
261
|
+
seen = [mp for mp in mount_points if re.search(rf"{re.escape(MOUNT_MARK)} {re.escape(mp)} mounted\b", out)]
|
|
262
|
+
missing = [mp for mp in mount_points if mp not in seen]
|
|
263
|
+
kernel = len(re.findall(r"XFS \([a-z0-9]+\): Ending clean mount", out))
|
|
264
|
+
checks.append({"name": "data disks mounted", "ok": not missing,
|
|
265
|
+
"detail": f"mounted: {', '.join(seen) or 'none'}; missing: {', '.join(missing) or 'none'} "
|
|
266
|
+
f"(each declared mount point's own line on the console; the kernel's "
|
|
267
|
+
f"{kernel} clean XFS mount(s) say nothing of which)"})
|
|
268
|
+
elif expect_mounts:
|
|
269
|
+
mounted = len(re.findall(r"XFS \([a-z0-9]+\): Ending clean mount", out))
|
|
270
|
+
checks.append({"name": "data disks mounted", "ok": mounted >= expect_mounts,
|
|
271
|
+
"detail": f"{mounted} clean XFS mount(s) on the console, {expect_mounts} declared"})
|
|
272
|
+
evidence = [ln for ln in out.splitlines()
|
|
273
|
+
if re.search(r"startup|XFS|csis-mount|google_metadata_script_runner", ln)][-20:]
|
|
274
|
+
return {"ok": all(c["ok"] for c in checks), "checks": checks, "evidence": evidence}
|
|
275
|
+
|
|
276
|
+
def ssh_proxy_command(self, instance_name: str) -> list[str] | None:
|
|
277
|
+
"""An IAP TCP tunnel to the instance's port, on stdin/stdout: no
|
|
278
|
+
public address, the IAP firewall rule and grant the session
|
|
279
|
+
mechanism already needs (stage 75 step 6)."""
|
|
280
|
+
from .gcp_packer_source import gce_name
|
|
281
|
+
from .gcp_utils import resolve_project
|
|
282
|
+
if self.session_mechanism() != self.IAP:
|
|
283
|
+
return None
|
|
284
|
+
project = resolve_project(self.model.self_to_gcp_client_config())
|
|
285
|
+
zone = getattr(self.model, "zone", None)
|
|
286
|
+
if not project or not zone:
|
|
287
|
+
return None
|
|
288
|
+
return [self.gcloud_binary(), "compute", "start-iap-tunnel", gce_name(instance_name), "%p",
|
|
289
|
+
"--listen-on-stdin", "--project", str(project), "--zone", str(zone)]
|
|
290
|
+
|
|
291
|
+
def run_session_command(self, instance_name: str, script: str, timeout: int = 300) -> tuple[int, str]:
|
|
292
|
+
"""``gcloud compute ssh --tunnel-through-iap --command`` as the
|
|
293
|
+
operator's gcloud account (the IAP grant, stage 2)."""
|
|
294
|
+
import subprocess # noqa: PLC0415
|
|
295
|
+
from .gcp_packer_source import gce_name
|
|
296
|
+
from .gcp_utils import resolve_project
|
|
297
|
+
if self.session_mechanism() != self.IAP:
|
|
298
|
+
# stage 63 item 14: a runtime that declares no mechanism has no
|
|
299
|
+
# IAP firewall rule or grant to lean on; tunnelling anyway failed
|
|
300
|
+
# as IAP's problem, far from the cause
|
|
301
|
+
raise RuntimeError(f"GCP runtime {self.get_name()} declares no session_mechanism, so no "
|
|
302
|
+
f"session command can reach its instances (supported: {self.IAP})")
|
|
303
|
+
cfg = self.model.self_to_gcp_client_config()
|
|
304
|
+
project, zone = resolve_project(cfg), getattr(self.model, "zone", None)
|
|
305
|
+
if not project or not zone:
|
|
306
|
+
raise RuntimeError(f"GCP runtime {self.get_name()}: no project/zone for a session command")
|
|
307
|
+
cmd = [self.gcloud_binary(), "compute", "ssh", gce_name(instance_name), "--project", str(project), "--zone", str(zone),
|
|
308
|
+
"--tunnel-through-iap", "--quiet", "--command", script]
|
|
309
|
+
res = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=timeout)
|
|
310
|
+
return res.returncode, (res.stdout or "") + (res.stderr or "")
|
|
311
|
+
|
|
312
|
+
def _theirs(self, resource: Any) -> bool:
|
|
313
|
+
"""Stage 93: whether a GCE resource's labels name another configuration."""
|
|
314
|
+
from cs_image_system.base.lineage import belongs_to_another_configuration
|
|
315
|
+
return belongs_to_another_configuration(self._get_context(), dict(getattr(resource, "labels", {}) or {}))
|
|
316
|
+
|
|
317
|
+
def instance_tags(self, instance_name: str) -> dict[str, str] | None:
|
|
318
|
+
"""The named machine's labels (stage 93's relabel reads them); None
|
|
319
|
+
when no such machine stands."""
|
|
320
|
+
from .gcp_packer_source import gce_name
|
|
321
|
+
got = self._instances_client()
|
|
322
|
+
if got is None:
|
|
323
|
+
return None
|
|
324
|
+
client, project, zone = got
|
|
325
|
+
try:
|
|
326
|
+
inst = client.get(project=project, zone=zone, instance=gce_name(instance_name))
|
|
327
|
+
except Exception as e: # noqa: BLE001
|
|
328
|
+
if "not found" in str(e).lower() or "404" in str(e):
|
|
329
|
+
return None
|
|
330
|
+
raise
|
|
331
|
+
return dict(getattr(inst, "labels", {}) or {})
|
|
332
|
+
|
|
333
|
+
def retag_instance(self, instance_name: str, tags: dict[str, str]) -> bool:
|
|
334
|
+
"""``setLabels`` on the named machine (stage 93: the id, in place,
|
|
335
|
+
through the same ``gce_label`` the emission uses)."""
|
|
336
|
+
from google.cloud import compute_v1 # noqa: PLC0415
|
|
337
|
+
from .gcp_packer_source import gce_label, gce_name
|
|
338
|
+
got = self._instances_client()
|
|
339
|
+
if got is None:
|
|
340
|
+
return False
|
|
341
|
+
client, project, zone = got
|
|
342
|
+
inst = client.get(project=project, zone=zone, instance=gce_name(instance_name))
|
|
343
|
+
labels = dict(getattr(inst, "labels", {}) or {})
|
|
344
|
+
labels.update({gce_label(k): gce_label(str(v)) for k, v in tags.items()})
|
|
345
|
+
client.set_labels(project=project, zone=zone, instance=gce_name(instance_name),
|
|
346
|
+
instances_set_labels_request_resource=compute_v1.InstancesSetLabelsRequest(
|
|
347
|
+
labels=labels, label_fingerprint=getattr(inst, "label_fingerprint", None))).result()
|
|
348
|
+
return True
|
|
349
|
+
|
|
350
|
+
def inventory(self) -> dict[str, Any]:
|
|
351
|
+
"""Instances and disks in the runtime's zone, every custom image in
|
|
352
|
+
the project, and the project's buckets (via gcloud, no extra library);
|
|
353
|
+
and, under ``owners``, the ``csis_config`` label each carries (stage
|
|
354
|
+
93), so the emptiness check counts this configuration's alone."""
|
|
355
|
+
import subprocess # noqa: PLC0415
|
|
356
|
+
from google.cloud import compute_v1 # noqa: PLC0415
|
|
357
|
+
from .gcp_utils import _make_credentials, resolve_project
|
|
358
|
+
cfg = self.model.self_to_gcp_client_config()
|
|
359
|
+
project, zone = resolve_project(cfg), getattr(self.model, "zone", None)
|
|
360
|
+
if not project or not zone:
|
|
361
|
+
raise RuntimeError(f"GCP runtime {self.get_name()}: no project/zone to inventory")
|
|
362
|
+
creds = _make_credentials(cfg)
|
|
363
|
+
kw = {"credentials": creds} if creds else {}
|
|
364
|
+
from cs_image_system.base.lineage import CONFIG_TAG
|
|
365
|
+
owners: dict[str, dict[str, str]] = {"instances": {}, "images": {}, "disks": {}, "buckets": {}}
|
|
366
|
+
|
|
367
|
+
def seen(kind: str, name: str, labels: Any) -> str:
|
|
368
|
+
owner = str(dict(labels or {}).get(CONFIG_TAG) or "")
|
|
369
|
+
if owner:
|
|
370
|
+
owners[kind][name] = owner
|
|
371
|
+
return name
|
|
372
|
+
instances = sorted(seen("instances", i.name, getattr(i, "labels", None))
|
|
373
|
+
for i in compute_v1.InstancesClient(**kw).list(project=project, zone=zone))
|
|
374
|
+
images = sorted(seen("images", i.name, getattr(i, "labels", None))
|
|
375
|
+
for i in compute_v1.ImagesClient(**kw).list(project=project))
|
|
376
|
+
disks = sorted(seen("disks", d.name, getattr(d, "labels", None))
|
|
377
|
+
for d in compute_v1.DisksClient(**kw).list(project=project, zone=zone))
|
|
378
|
+
res = subprocess.run([self.gcloud_binary(), "storage", "buckets", "list", "--project", str(project),
|
|
379
|
+
f"--format=csv[no-heading](name,labels.{CONFIG_TAG})"],
|
|
380
|
+
capture_output=True, text=True, check=False)
|
|
381
|
+
if res.returncode != 0:
|
|
382
|
+
raise RuntimeError(f"gcloud storage buckets list: {(res.stderr or '').strip()[-300:]}")
|
|
383
|
+
buckets: list[str] = []
|
|
384
|
+
for ln in (res.stdout or "").splitlines():
|
|
385
|
+
if not ln.strip():
|
|
386
|
+
continue
|
|
387
|
+
name, _, owner = ln.strip().partition(",")
|
|
388
|
+
buckets.append(seen("buckets", name, {CONFIG_TAG: owner} if owner else {}))
|
|
389
|
+
return {"instances": instances, "images": sorted(images), "disks": disks, "buckets": sorted(buckets),
|
|
390
|
+
"owners": owners}
|
|
391
|
+
|
|
392
|
+
def dispose_image(self, build_id: str) -> bool:
|
|
393
|
+
"""Delete the GCE image named by the build id and wait for the
|
|
394
|
+
operation (stage 8.4). NotFound = already gone."""
|
|
395
|
+
from google.api_core.exceptions import NotFound # noqa: PLC0415
|
|
396
|
+
from .gcp_utils import _make_images_client, resolve_project
|
|
397
|
+
cfg = self.model.self_to_gcp_client_config()
|
|
398
|
+
project = resolve_project(cfg)
|
|
399
|
+
if not project:
|
|
400
|
+
raise RuntimeError(f"GCP runtime {self.get_name()}: no project to dispose in")
|
|
401
|
+
client = _make_images_client(cfg)
|
|
402
|
+
try:
|
|
403
|
+
client.delete(project=project, image=build_id).result()
|
|
404
|
+
except NotFound:
|
|
405
|
+
log.warning(f"GCE image {build_id} not found in {project}; nothing to delete")
|
|
406
|
+
return False
|
|
407
|
+
log.info(f"GCE image {build_id} deleted from {project}")
|
|
408
|
+
return True
|
|
409
|
+
|
|
410
|
+
def retag_image(self, image_id: str, tags: dict[str, str]) -> bool:
|
|
411
|
+
"""``setLabels`` on one of our own GCE images (zero-drift-report,
|
|
412
|
+
ledger 66): the labels packer wrote at generation time are merged
|
|
413
|
+
with the lineage truth, through the same ``gce_label`` sanitizer the
|
|
414
|
+
packer source used, so the state query compares like with like."""
|
|
415
|
+
from google.cloud import compute_v1 # noqa: PLC0415
|
|
416
|
+
from .gcp_packer_source import gce_label
|
|
417
|
+
from .gcp_utils import _make_images_client, resolve_project
|
|
418
|
+
cfg = self.model.self_to_gcp_client_config()
|
|
419
|
+
project = resolve_project(cfg)
|
|
420
|
+
if not project:
|
|
421
|
+
raise RuntimeError(f"GCP runtime {self.get_name()}: no project to relabel in")
|
|
422
|
+
client = _make_images_client(cfg)
|
|
423
|
+
current = client.get(project=project, image=image_id)
|
|
424
|
+
labels = dict(getattr(current, "labels", {}) or {})
|
|
425
|
+
labels.update({gce_label(k): gce_label(str(v)) for k, v in tags.items()})
|
|
426
|
+
client.set_labels(
|
|
427
|
+
project=project, resource=image_id,
|
|
428
|
+
global_set_labels_request_resource=compute_v1.GlobalSetLabelsRequest(
|
|
429
|
+
label_fingerprint=str(getattr(current, "label_fingerprint", "") or ""), labels=labels),
|
|
430
|
+
).result()
|
|
431
|
+
log.info(f"GCE image {image_id} relabelled with {sorted(tags)} in {project}")
|
|
432
|
+
return True
|
|
433
|
+
|
|
434
|
+
def can_query_instance_boot_image(self) -> bool:
|
|
435
|
+
return True
|
|
436
|
+
|
|
437
|
+
def query_instance_boot_image(self, instance_name: str) -> str | None:
|
|
438
|
+
"""The GCE image name the instance's boot disk was created from
|
|
439
|
+
(finding 49): instance -> boot disk -> disk.source_image basename.
|
|
440
|
+
Read-only; None when the instance, disk, zone or project cannot be
|
|
441
|
+
resolved, so the caller makes no claim."""
|
|
442
|
+
from .gcp_packer_source import gce_name
|
|
443
|
+
from .gcp_utils import _make_credentials, resolve_project
|
|
444
|
+
try:
|
|
445
|
+
from google.cloud import compute_v1 # noqa: PLC0415
|
|
446
|
+
cfg = self.model.self_to_gcp_client_config()
|
|
447
|
+
project = resolve_project(cfg)
|
|
448
|
+
zone = getattr(self.model, "zone", None)
|
|
449
|
+
if not project or not zone:
|
|
450
|
+
return None
|
|
451
|
+
creds = _make_credentials(cfg)
|
|
452
|
+
kw = {"credentials": creds} if creds else {}
|
|
453
|
+
inst = compute_v1.InstancesClient(**kw).get(project=project, zone=zone,
|
|
454
|
+
instance=gce_name(instance_name))
|
|
455
|
+
if self._theirs(inst):
|
|
456
|
+
return None # stage 93: another configuration's machine of our name
|
|
457
|
+
boot = next((d for d in (inst.disks or []) if getattr(d, "boot", False)), None)
|
|
458
|
+
if boot is None or not getattr(boot, "source", None):
|
|
459
|
+
return None
|
|
460
|
+
disk_name = str(boot.source).rsplit("/", 1)[-1]
|
|
461
|
+
disk = compute_v1.DisksClient(**kw).get(project=project, zone=zone, disk=disk_name)
|
|
462
|
+
src = getattr(disk, "source_image", None)
|
|
463
|
+
return str(src).rsplit("/", 1)[-1] if src else None
|
|
464
|
+
except Exception as e: # noqa: BLE001 - read-only probe: unavailable, never fatal
|
|
465
|
+
log.debug(f"GCE runtime {self.get_name()}: boot image of {instance_name!r} unavailable: {e}")
|
|
466
|
+
return None
|
|
467
|
+
|
|
468
|
+
# ------------------------------------------------ power state (stage 57)
|
|
469
|
+
#: GCE ``Instance.status`` mapped onto the system's vocabulary. The trap
|
|
470
|
+
#: is ``TERMINATED``: on GCE that means STOPPED -- the instance exists and
|
|
471
|
+
#: can be started again -- and NOT deleted, which is the opposite of what
|
|
472
|
+
#: the word means on EC2. A deleted GCE instance is simply not found. This
|
|
473
|
+
#: is exactly why the vocabulary is the system's own and no caller is
|
|
474
|
+
#: allowed to read a provider's spelling.
|
|
475
|
+
_GCE_POWER = {
|
|
476
|
+
"PROVISIONING": power_state.STARTING,
|
|
477
|
+
"STAGING": power_state.STARTING,
|
|
478
|
+
"RUNNING": power_state.RUNNING,
|
|
479
|
+
"STOPPING": power_state.STOPPING,
|
|
480
|
+
"SUSPENDING": power_state.STOPPING,
|
|
481
|
+
"SUSPENDED": power_state.SUSPENDED,
|
|
482
|
+
"TERMINATED": power_state.STOPPED,
|
|
483
|
+
"REPAIRING": power_state.UNKNOWN,
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
def _instances_client(self):
|
|
487
|
+
"""(client, project, zone) for instance calls, or None when the
|
|
488
|
+
runtime is not resolvable."""
|
|
489
|
+
from .gcp_utils import _make_credentials, resolve_project
|
|
490
|
+
from google.cloud import compute_v1 # noqa: PLC0415
|
|
491
|
+
cfg = self.model.self_to_gcp_client_config()
|
|
492
|
+
project, zone = resolve_project(cfg), getattr(self.model, "zone", None)
|
|
493
|
+
if not project or not zone:
|
|
494
|
+
return None
|
|
495
|
+
creds = _make_credentials(cfg)
|
|
496
|
+
kw = {"credentials": creds} if creds else {}
|
|
497
|
+
return compute_v1.InstancesClient(**kw), str(project), str(zone)
|
|
498
|
+
|
|
499
|
+
def can_query_instance_power_state(self) -> bool:
|
|
500
|
+
return True
|
|
501
|
+
|
|
502
|
+
def query_instance_power_state(self, instance_name: str) -> str | None:
|
|
503
|
+
"""GCE's own ``status`` for the named instance. None means this
|
|
504
|
+
runtime could not find out -- never that the machine is off."""
|
|
505
|
+
from .gcp_packer_source import gce_name
|
|
506
|
+
try:
|
|
507
|
+
got = self._instances_client()
|
|
508
|
+
if got is None:
|
|
509
|
+
return None
|
|
510
|
+
client, project, zone = got
|
|
511
|
+
try:
|
|
512
|
+
inst = client.get(project=project, zone=zone, instance=gce_name(instance_name))
|
|
513
|
+
except Exception as e: # noqa: BLE001
|
|
514
|
+
if "not found" in str(e).lower() or "404" in str(e):
|
|
515
|
+
return power_state.ABSENT
|
|
516
|
+
raise
|
|
517
|
+
if self._theirs(inst):
|
|
518
|
+
return power_state.ABSENT # stage 93: theirs stands where ours would; ours is absent
|
|
519
|
+
raw = str(getattr(inst, "status", "") or "")
|
|
520
|
+
mapped = self._GCE_POWER.get(raw)
|
|
521
|
+
if mapped is None:
|
|
522
|
+
log.warning(f"GCE runtime {self.get_name()}: instance {instance_name!r} reports "
|
|
523
|
+
f"unrecognised status {raw!r}; treating it as unknown")
|
|
524
|
+
return power_state.UNKNOWN
|
|
525
|
+
return mapped
|
|
526
|
+
except Exception as e: # noqa: BLE001 - read-only probe: unavailable, never fatal
|
|
527
|
+
log.debug(f"GCE runtime {self.get_name()}: power state of {instance_name!r} unavailable: {e}")
|
|
528
|
+
return None
|
|
529
|
+
|
|
530
|
+
def can_query_instance_identity(self) -> bool:
|
|
531
|
+
return True
|
|
532
|
+
|
|
533
|
+
def query_instance_identity(self, instance_name: str) -> dict[str, str] | None:
|
|
534
|
+
"""GCE's numeric instance id and its internal hostname (a custom
|
|
535
|
+
``hostname`` when set, else ``<name>.c.<project>.internal``). The
|
|
536
|
+
first label of the default is the instance name itself, so on GCE the
|
|
537
|
+
provider-hostname alias usually has nothing to give back."""
|
|
538
|
+
from .gcp_packer_source import gce_name
|
|
539
|
+
try:
|
|
540
|
+
got = self._instances_client()
|
|
541
|
+
if got is None:
|
|
542
|
+
return None
|
|
543
|
+
client, project, zone = got
|
|
544
|
+
inst = client.get(project=project, zone=zone, instance=gce_name(instance_name))
|
|
545
|
+
if self._theirs(inst):
|
|
546
|
+
return None # stage 93
|
|
547
|
+
host = str(getattr(inst, "hostname", "") or "") or f"{gce_name(instance_name)}.c.{project}.internal"
|
|
548
|
+
return {"instance_id": str(getattr(inst, "id", "") or ""), "provider_hostname": host}
|
|
549
|
+
except Exception as e: # noqa: BLE001 - read-only probe
|
|
550
|
+
log.debug(f"GCE runtime {self.get_name()}: identity of {instance_name!r} unavailable: {e}")
|
|
551
|
+
return None
|
|
552
|
+
|
|
553
|
+
def can_set_instance_power_state(self) -> bool:
|
|
554
|
+
return True
|
|
555
|
+
|
|
556
|
+
def start_instance(self, instance_name: str, timeout: int = 300) -> bool:
|
|
557
|
+
from .gcp_packer_source import gce_name
|
|
558
|
+
got = self._instances_client()
|
|
559
|
+
if got is None:
|
|
560
|
+
raise RuntimeError(f"GCP runtime {self.get_name()}: no project/zone to start an instance")
|
|
561
|
+
client, project, zone = got
|
|
562
|
+
client.start(project=project, zone=zone, instance=gce_name(instance_name)).result(timeout=timeout)
|
|
563
|
+
return self.query_instance_power_state(instance_name) == power_state.RUNNING
|
|
564
|
+
|
|
565
|
+
def stop_instance(self, instance_name: str, timeout: int = 300) -> bool:
|
|
566
|
+
from .gcp_packer_source import gce_name
|
|
567
|
+
got = self._instances_client()
|
|
568
|
+
if got is None:
|
|
569
|
+
raise RuntimeError(f"GCP runtime {self.get_name()}: no project/zone to stop an instance")
|
|
570
|
+
client, project, zone = got
|
|
571
|
+
client.stop(project=project, zone=zone, instance=gce_name(instance_name)).result(timeout=timeout)
|
|
572
|
+
return power_state.is_off(self.query_instance_power_state(instance_name))
|
|
573
|
+
|
|
574
|
+
# ------------------------------------------------- packer source hooks
|
|
575
|
+
def packer_source_type(self) -> str:
|
|
576
|
+
from .gcp_packer_source import GOOGLECOMPUTE
|
|
577
|
+
return GOOGLECOMPUTE
|
|
578
|
+
|
|
579
|
+
def packer_source_blocks(self, image: Any, *, runtime: str, source_type: str, subconfig: Any,
|
|
580
|
+
self_subconfig: Any, psi: Any, tags: dict[str, str],
|
|
581
|
+
final_name: str, pinned: str | None) -> list[str]:
|
|
582
|
+
from .gcp_packer_source import googlecompute_source
|
|
583
|
+
return googlecompute_source(self.model, image, runtime=runtime, source_type=source_type,
|
|
584
|
+
subconfig=subconfig, self_subconfig=self_subconfig, psi=psi,
|
|
585
|
+
tags=tags, final_name=final_name, pinned=pinned)
|
|
586
|
+
|
|
587
|
+
def build_id_from_artifact(self, artifact_id: str) -> str:
|
|
588
|
+
"""googlecompute's manifest artifact_id is the image name itself."""
|
|
589
|
+
return artifact_id
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
class GCPCLIVersionChecker(AbstractVersionChecker):
|
|
593
|
+
"""Version checker for GCP."""
|
|
594
|
+
|
|
595
|
+
@classmethod
|
|
596
|
+
def csis_name(cls) -> str:
|
|
597
|
+
return GCP_RUNTIME
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
@classmethod
|
|
601
|
+
def csis_classifier(cls) -> VCT:
|
|
602
|
+
return VCT.VERSION_CHECKER
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def get_regex(self) -> str:
|
|
606
|
+
# the SDK version (`Google Cloud SDK 585.0.0`, the first line), not the
|
|
607
|
+
# `core` component's date-shaped version nobody quotes (stage 48.1)
|
|
608
|
+
return r"Google Cloud SDK ([\d\.]+)"
|