oic-sync 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oic_sync/__init__.py +5 -0
- oic_sync/__main__.py +3 -0
- oic_sync/backends/__init__.py +5 -0
- oic_sync/backends/base.py +76 -0
- oic_sync/backends/openstack.py +461 -0
- oic_sync/backends/proxmox.py +639 -0
- oic_sync/cli.py +480 -0
- oic_sync/engine.py +331 -0
- oic_sync/hashing.py +82 -0
- oic_sync/http.py +27 -0
- oic_sync/index.py +287 -0
- oic_sync/signature.py +115 -0
- oic_sync-0.1.0.dist-info/METADATA +331 -0
- oic_sync-0.1.0.dist-info/RECORD +17 -0
- oic_sync-0.1.0.dist-info/WHEEL +4 -0
- oic_sync-0.1.0.dist-info/entry_points.txt +2 -0
- oic_sync-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,639 @@
|
|
|
1
|
+
"""Proxmox VE backend: one VM template per imported version, on the local node.
|
|
2
|
+
|
|
3
|
+
oic-sync runs on a Proxmox VE node (like pveam) and talks to the local API through the `pvesh`
|
|
4
|
+
command line tool (`pvesh get|create|set|delete <path> --output-format json`), as root: no
|
|
5
|
+
API token and no remote mode in this version. Every call goes through a small client (Pvesh)
|
|
6
|
+
so the tests use a fake.
|
|
7
|
+
|
|
8
|
+
A managed image is a VM template. Ours carry the tags `oic` and `oic-<image id>`, and a
|
|
9
|
+
machine-readable block in the VM description:
|
|
10
|
+
|
|
11
|
+
```oic-sync
|
|
12
|
+
oic_id=debian-13
|
|
13
|
+
version=13-20261103-2651
|
|
14
|
+
sha256=...
|
|
15
|
+
imported_at=2026-11-13T09:00:00Z
|
|
16
|
+
import_file=local:import/oic-9001-debian-13-13-20261103-2651-x86_64.qcow2
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Proxmox has no hidden flag: retired versions get the tag `oic-retired`. VM names must be DNS
|
|
20
|
+
names, so the stable name is the image id ("debian-13", "alpine-linux-uefi") and retired
|
|
21
|
+
versions are "<image id>-<version>".
|
|
22
|
+
|
|
23
|
+
Import, in an order that leaves a trace of every step in the VM itself:
|
|
24
|
+
|
|
25
|
+
1. allocate a VMID and create the VM shell (name, tags, description, hardware), not a
|
|
26
|
+
template yet: a non-template VM carrying our tags is an import in progress;
|
|
27
|
+
2. download the qcow2 into the `import` content of a file storage with `download-url`;
|
|
28
|
+
Proxmox checks the sha256 of the index (the digest the cosign bundle signs);
|
|
29
|
+
3. import the disk (`scsi0: <storage>:0,import-from=<file>`) and add the cloud-init drive;
|
|
30
|
+
4. convert to a template, then delete the downloaded file.
|
|
31
|
+
|
|
32
|
+
On any failure the VM and the file are removed. A crashed run leaves a non-template VM that
|
|
33
|
+
the engine removes after 6 hours (with the file named in its description).
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import json
|
|
39
|
+
import logging
|
|
40
|
+
import re
|
|
41
|
+
import shutil
|
|
42
|
+
import socket
|
|
43
|
+
import subprocess
|
|
44
|
+
import time
|
|
45
|
+
from collections.abc import Callable, Mapping
|
|
46
|
+
from datetime import datetime, timezone
|
|
47
|
+
from typing import Any, Protocol
|
|
48
|
+
from urllib.parse import urlsplit
|
|
49
|
+
|
|
50
|
+
from oic_sync.backends.base import BackendError, ManagedImage
|
|
51
|
+
from oic_sync.index import Target, stable_name
|
|
52
|
+
|
|
53
|
+
log = logging.getLogger(__name__)
|
|
54
|
+
|
|
55
|
+
MANAGED_TAG = "oic"
|
|
56
|
+
RETIRED_TAG = "oic-retired"
|
|
57
|
+
BLOCK_LANG = "oic-sync"
|
|
58
|
+
IMPORT_FORMATS = ("qcow2", "raw", "vmdk")
|
|
59
|
+
|
|
60
|
+
# Disk-like keys of a VM config: a linked clone of template X references base-X-disk-N there.
|
|
61
|
+
DISK_KEY = re.compile(r"^(?:scsi|virtio|sata|ide|unused)\d+$|^(?:efidisk|tpmstate)0$")
|
|
62
|
+
_BLOCK = re.compile(r"```" + BLOCK_LANG + r"[ \t]*\r?\n(.*?)\r?\n?```", re.DOTALL)
|
|
63
|
+
_TAG_SPLIT = re.compile(r"[;,\s]+")
|
|
64
|
+
_EPOCH = datetime.fromtimestamp(0, timezone.utc)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# -- names and tags ----------------------------------------------------------------------
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def dns_name(text: str) -> str:
|
|
71
|
+
"""A valid Proxmox VM name (format dns-name): lowercase labels of [a-z0-9-]."""
|
|
72
|
+
labels = []
|
|
73
|
+
for label in text.lower().split("."):
|
|
74
|
+
label = re.sub(r"[^a-z0-9-]+", "-", label)
|
|
75
|
+
label = re.sub(r"-{2,}", "-", label).strip("-")[:63].strip("-")
|
|
76
|
+
if label:
|
|
77
|
+
labels.append(label)
|
|
78
|
+
if not labels:
|
|
79
|
+
raise ValueError(f"cannot make a VM name out of {text!r}")
|
|
80
|
+
return ".".join(labels)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def tag(text: str) -> str:
|
|
84
|
+
"""A valid Proxmox tag ([a-z0-9_][a-z0-9_+.-]*)."""
|
|
85
|
+
value = re.sub(r"[^a-z0-9_+.-]+", "-", text.lower()).lstrip("-+.")
|
|
86
|
+
if not value:
|
|
87
|
+
raise ValueError(f"cannot make a tag out of {text!r}")
|
|
88
|
+
return value
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def image_tag(image_id: str) -> str:
|
|
92
|
+
return tag(f"oic-{image_id}")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def split_tags(value: str | None) -> list[str]:
|
|
96
|
+
return [t for t in _TAG_SPLIT.split(value or "") if t]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class ProxmoxNaming:
|
|
100
|
+
"""Stable name "debian-13" ("alpine-linux-uefi" for a variant), retired "debian-13-<ver>"."""
|
|
101
|
+
|
|
102
|
+
def stable(self, target: Target) -> str:
|
|
103
|
+
base = target.image.id
|
|
104
|
+
if target.artifact.variant and len(target.image.artifacts) > 1:
|
|
105
|
+
base += f"-{target.artifact.variant}"
|
|
106
|
+
return dns_name(base)
|
|
107
|
+
|
|
108
|
+
def versioned(self, target: Target, version: str, sha256: str | None = None) -> str:
|
|
109
|
+
name = f"{self.stable(target)}-{version}"
|
|
110
|
+
if sha256:
|
|
111
|
+
name += f"-{sha256[:12]}"
|
|
112
|
+
return dns_name(name)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# -- description block -------------------------------------------------------------------
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def parse_description(text: str | None) -> dict[str, str]:
|
|
119
|
+
"""key=value lines of the ```oic-sync block of a VM description ({} when absent)."""
|
|
120
|
+
match = _BLOCK.search(text or "")
|
|
121
|
+
if not match:
|
|
122
|
+
return {}
|
|
123
|
+
meta = {}
|
|
124
|
+
for line in match.group(1).splitlines():
|
|
125
|
+
key, sep, value = line.strip().partition("=")
|
|
126
|
+
if sep and key:
|
|
127
|
+
meta[key.strip()] = value.strip()
|
|
128
|
+
return meta
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
_IMPORT_OWNER = re.compile(r"(?:^|[:/])oic-(\d+)-[^/]*$")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def block_owner(meta: Mapping[str, str]) -> str | None:
|
|
135
|
+
"""VMID the oic-sync block was written for: its vmid key, else the import file name."""
|
|
136
|
+
if meta.get("vmid"):
|
|
137
|
+
return meta["vmid"]
|
|
138
|
+
match = _IMPORT_OWNER.search(meta.get("import_file", ""))
|
|
139
|
+
return match.group(1) if match else None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def render_description(target: Target, meta: Mapping[str, str]) -> str:
|
|
143
|
+
image, artifact = target.image, target.artifact
|
|
144
|
+
title = stable_name(image, artifact) # "Alpine Linux (UEFI)", as in the catalog
|
|
145
|
+
lines = [
|
|
146
|
+
f"**{title}** {artifact.version}, from [openimages.cloud](https://openimages.cloud).",
|
|
147
|
+
"",
|
|
148
|
+
"Imported and rotated by oic-sync: do not edit the block below.",
|
|
149
|
+
]
|
|
150
|
+
if image.login_user:
|
|
151
|
+
lines.append(f"Login user: `{image.login_user}` (cloud-init).")
|
|
152
|
+
lines += ["", f"```{BLOCK_LANG}", *(f"{k}={v}" for k, v in meta.items()), "```"]
|
|
153
|
+
return "\n".join(lines)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _parse_time(value: str | None) -> datetime | None:
|
|
157
|
+
if not value:
|
|
158
|
+
return None
|
|
159
|
+
try:
|
|
160
|
+
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
161
|
+
except ValueError:
|
|
162
|
+
return None
|
|
163
|
+
return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _meta_ctime(meta: str | None) -> datetime | None:
|
|
167
|
+
"""ctime of the VM config `meta` property (creation-qemu=...,ctime=1700000000)."""
|
|
168
|
+
match = re.search(r"(?:^|,)ctime=(\d+)", meta or "")
|
|
169
|
+
return datetime.fromtimestamp(int(match.group(1)), timezone.utc) if match else None
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
# -- pvesh client ------------------------------------------------------------------------
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class PveApi(Protocol):
|
|
176
|
+
"""The local Proxmox VE API: method is get, create (POST), set (PUT) or delete."""
|
|
177
|
+
|
|
178
|
+
def call(self, method: str, path: str, params: Mapping[str, Any] | None = None) -> Any: ...
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class Pvesh:
|
|
182
|
+
"""Runs `pvesh <method> <path> --key value ... --output-format json`."""
|
|
183
|
+
|
|
184
|
+
METHODS = ("get", "create", "set", "delete")
|
|
185
|
+
|
|
186
|
+
def __init__(
|
|
187
|
+
self,
|
|
188
|
+
binary: str = "pvesh",
|
|
189
|
+
*,
|
|
190
|
+
runner: Callable[..., Any] = subprocess.run,
|
|
191
|
+
timeout: float = 600.0,
|
|
192
|
+
) -> None:
|
|
193
|
+
self.binary = binary
|
|
194
|
+
self.runner = runner
|
|
195
|
+
self.timeout = timeout
|
|
196
|
+
|
|
197
|
+
def argv(self, method: str, path: str, params: Mapping[str, Any] | None = None) -> list[str]:
|
|
198
|
+
if method not in self.METHODS:
|
|
199
|
+
raise ValueError(f"unknown pvesh method {method!r}")
|
|
200
|
+
argv = [self.binary, method, path]
|
|
201
|
+
for key, value in (params or {}).items():
|
|
202
|
+
if value is None:
|
|
203
|
+
continue
|
|
204
|
+
if isinstance(value, bool):
|
|
205
|
+
value = int(value)
|
|
206
|
+
argv += [f"--{key}", str(value)]
|
|
207
|
+
return [*argv, "--output-format", "json"]
|
|
208
|
+
|
|
209
|
+
def call(self, method: str, path: str, params: Mapping[str, Any] | None = None) -> Any:
|
|
210
|
+
argv = self.argv(method, path, params)
|
|
211
|
+
log.debug("run %s", " ".join(argv))
|
|
212
|
+
try:
|
|
213
|
+
proc = self.runner(
|
|
214
|
+
argv, capture_output=True, text=True, timeout=self.timeout, check=False
|
|
215
|
+
)
|
|
216
|
+
except (OSError, subprocess.SubprocessError) as e:
|
|
217
|
+
raise BackendError(f"pvesh {method} {path}: {e}") from e
|
|
218
|
+
if proc.returncode != 0:
|
|
219
|
+
err = (proc.stderr or proc.stdout or "").strip().splitlines()
|
|
220
|
+
detail = " / ".join(err[-3:])[:500]
|
|
221
|
+
raise BackendError(f"pvesh {method} {path} failed ({proc.returncode}): {detail}")
|
|
222
|
+
return _decode(proc.stdout)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _decode(stdout: str) -> Any:
|
|
226
|
+
text = (stdout or "").strip()
|
|
227
|
+
if not text:
|
|
228
|
+
return None
|
|
229
|
+
try:
|
|
230
|
+
return json.loads(text)
|
|
231
|
+
except ValueError:
|
|
232
|
+
# Some calls print a task log before (or instead of) the JSON result: keep the UPID.
|
|
233
|
+
match = re.search(r"UPID:[^\s\"]+", text)
|
|
234
|
+
return match.group(0) if match else text
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
# -- backend -----------------------------------------------------------------------------
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
class ProxmoxBackend:
|
|
241
|
+
naming = ProxmoxNaming()
|
|
242
|
+
|
|
243
|
+
def __init__(
|
|
244
|
+
self,
|
|
245
|
+
api: PveApi,
|
|
246
|
+
node: str,
|
|
247
|
+
*,
|
|
248
|
+
import_storage: str = "local",
|
|
249
|
+
disk_storage: str = "local-lvm",
|
|
250
|
+
bridge: str = "vmbr0",
|
|
251
|
+
vmid_base: int | None = None,
|
|
252
|
+
memory: int = 2048,
|
|
253
|
+
cores: int = 2,
|
|
254
|
+
task_timeout: float = 3600.0,
|
|
255
|
+
poll_interval: float = 2.0,
|
|
256
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
257
|
+
clock: Callable[[], float] = time.monotonic,
|
|
258
|
+
now: Callable[[], datetime] = lambda: datetime.now(timezone.utc),
|
|
259
|
+
read_only: bool = False,
|
|
260
|
+
) -> None:
|
|
261
|
+
if vmid_base is not None and not 100 <= vmid_base <= 999_999_999:
|
|
262
|
+
raise ValueError("vmid base must be between 100 and 999999999")
|
|
263
|
+
if memory < 16 or cores < 1:
|
|
264
|
+
raise ValueError("memory must be at least 16 MiB and cores at least 1")
|
|
265
|
+
self.api = api
|
|
266
|
+
self.node = node
|
|
267
|
+
self.import_storage = import_storage
|
|
268
|
+
self.disk_storage = disk_storage
|
|
269
|
+
self.bridge = bridge
|
|
270
|
+
self.vmid_base = vmid_base
|
|
271
|
+
self.memory = memory
|
|
272
|
+
self.cores = cores
|
|
273
|
+
self.task_timeout = task_timeout
|
|
274
|
+
self.poll_interval = poll_interval
|
|
275
|
+
self.sleep = sleep
|
|
276
|
+
self.clock = clock
|
|
277
|
+
self.now = now
|
|
278
|
+
# status only reads VM configs: the storage options do not apply to it.
|
|
279
|
+
self.read_only = read_only
|
|
280
|
+
self._storages_checked = False
|
|
281
|
+
|
|
282
|
+
def describe(self) -> str:
|
|
283
|
+
if self.read_only:
|
|
284
|
+
return f"proxmox node {self.node}"
|
|
285
|
+
return (
|
|
286
|
+
f"proxmox node {self.node}, disks on {self.disk_storage}, "
|
|
287
|
+
f"downloads on {self.import_storage}"
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
def check_storages(self) -> None:
|
|
291
|
+
"""Fail before creating anything when a storage is missing or lacks a content type."""
|
|
292
|
+
if self._storages_checked:
|
|
293
|
+
return
|
|
294
|
+
listed = self.api.call("get", f"/nodes/{self.node}/storage") or []
|
|
295
|
+
by_name = {str(s.get("storage")): s for s in listed if isinstance(s, dict)}
|
|
296
|
+
for name, content, option in (
|
|
297
|
+
(self.import_storage, "import", "--import-storage"),
|
|
298
|
+
(self.disk_storage, "images", "--disk-storage"),
|
|
299
|
+
):
|
|
300
|
+
st = by_name.get(name)
|
|
301
|
+
if st is None:
|
|
302
|
+
known = ", ".join(sorted(by_name)) or "none"
|
|
303
|
+
raise BackendError(
|
|
304
|
+
f"storage {name!r} ({option}) does not exist on node {self.node} "
|
|
305
|
+
f"(storages: {known})"
|
|
306
|
+
)
|
|
307
|
+
if str(st.get("enabled", 1)) in ("0", "false") or str(st.get("active", 1)) in (
|
|
308
|
+
"0",
|
|
309
|
+
"false",
|
|
310
|
+
):
|
|
311
|
+
raise BackendError(f"storage {name!r} ({option}) is disabled or not active")
|
|
312
|
+
types = [t.strip() for t in str(st.get("content") or "").split(",") if t.strip()]
|
|
313
|
+
if content not in types:
|
|
314
|
+
hint = ",".join([*types, content])
|
|
315
|
+
raise BackendError(
|
|
316
|
+
f"storage {name!r} ({option}) is not configured for content type "
|
|
317
|
+
f"{content!r}: pvesm set {name} --content {hint}"
|
|
318
|
+
)
|
|
319
|
+
self._storages_checked = True
|
|
320
|
+
|
|
321
|
+
@property
|
|
322
|
+
def _qemu(self) -> str:
|
|
323
|
+
return f"/nodes/{self.node}/qemu"
|
|
324
|
+
|
|
325
|
+
# -- tasks ---------------------------------------------------------------------------
|
|
326
|
+
|
|
327
|
+
def wait_task(self, upid: Any, what: str) -> None:
|
|
328
|
+
"""Wait for a worker task (UPID) to stop; raise BackendError unless it ended OK."""
|
|
329
|
+
if not isinstance(upid, str) or not upid.startswith("UPID:"):
|
|
330
|
+
return # synchronous call, nothing to wait for
|
|
331
|
+
parts = upid.split(":")
|
|
332
|
+
node = parts[1] if len(parts) > 1 and parts[1] else self.node
|
|
333
|
+
path = f"/nodes/{node}/tasks/{upid}"
|
|
334
|
+
deadline = self.clock() + self.task_timeout
|
|
335
|
+
while True:
|
|
336
|
+
status = self.api.call("get", f"{path}/status") or {}
|
|
337
|
+
if status.get("status") == "stopped":
|
|
338
|
+
break
|
|
339
|
+
if self.clock() >= deadline:
|
|
340
|
+
raise BackendError(f"{what}: task {upid} still running after {self.task_timeout}s")
|
|
341
|
+
self.sleep(self.poll_interval)
|
|
342
|
+
exit_status = str(status.get("exitstatus") or "")
|
|
343
|
+
if exit_status == "OK":
|
|
344
|
+
return
|
|
345
|
+
if exit_status.startswith("WARNINGS"):
|
|
346
|
+
log.warning("%s: task ended with %s", what, exit_status)
|
|
347
|
+
return
|
|
348
|
+
raise BackendError(f"{what} failed: {exit_status or 'unknown'}{self._task_tail(path)}")
|
|
349
|
+
|
|
350
|
+
def _task_tail(self, path: str) -> str:
|
|
351
|
+
try:
|
|
352
|
+
lines = self.api.call("get", f"{path}/log", {"limit": 500}) or []
|
|
353
|
+
tail = [str(line.get("t", "")) for line in lines if isinstance(line, dict)][-3:]
|
|
354
|
+
except BackendError:
|
|
355
|
+
return ""
|
|
356
|
+
return f" ({' / '.join(t for t in tail if t)})" if tail else ""
|
|
357
|
+
|
|
358
|
+
# -- listing -------------------------------------------------------------------------
|
|
359
|
+
|
|
360
|
+
def _config(self, vmid: int | str, node: str | None = None) -> dict[str, Any]:
|
|
361
|
+
cfg = self.api.call("get", f"/nodes/{node or self.node}/qemu/{vmid}/config")
|
|
362
|
+
return cfg if isinstance(cfg, dict) else {}
|
|
363
|
+
|
|
364
|
+
def list_managed(self, target: Target) -> list[ManagedImage]:
|
|
365
|
+
wanted_tag = image_tag(target.image.id)
|
|
366
|
+
managed = []
|
|
367
|
+
for vm in self.api.call("get", self._qemu) or []:
|
|
368
|
+
tags = split_tags(vm.get("tags"))
|
|
369
|
+
if MANAGED_TAG not in tags or wanted_tag not in tags:
|
|
370
|
+
continue
|
|
371
|
+
vmid = vm["vmid"]
|
|
372
|
+
cfg = self._config(vmid)
|
|
373
|
+
meta = parse_description(cfg.get("description"))
|
|
374
|
+
if meta.get("oic_id") != target.image.id:
|
|
375
|
+
continue
|
|
376
|
+
if (meta.get("oic_variant") or None) != target.artifact.variant:
|
|
377
|
+
continue
|
|
378
|
+
if block_owner(meta) != str(vmid):
|
|
379
|
+
# A clone of one of our templates (Proxmox copies the description and the
|
|
380
|
+
# tags), or a VM edited by hand: never ours to promote, retire or clean up.
|
|
381
|
+
log.debug("VM %s carries the block of VM %s; ignored", vmid, block_owner(meta))
|
|
382
|
+
continue
|
|
383
|
+
managed.append(self._managed(vmid, cfg, meta))
|
|
384
|
+
return managed
|
|
385
|
+
|
|
386
|
+
def _managed(
|
|
387
|
+
self, vmid: int | str, cfg: Mapping[str, Any], meta: Mapping[str, str]
|
|
388
|
+
) -> ManagedImage:
|
|
389
|
+
tags = split_tags(cfg.get("tags"))
|
|
390
|
+
lock = str(cfg.get("lock") or "")
|
|
391
|
+
# A template is a finished import, even when locked for a while (backup, migration):
|
|
392
|
+
# only a VM not converted yet is an import in progress (or an abandoned one).
|
|
393
|
+
status = "active" if str(cfg.get("template") or "0") in ("1", "true") else "importing"
|
|
394
|
+
created = _parse_time(meta.get("imported_at")) or _meta_ctime(cfg.get("meta")) or _EPOCH
|
|
395
|
+
return ManagedImage(
|
|
396
|
+
id=str(vmid),
|
|
397
|
+
name=str(cfg.get("name") or ""),
|
|
398
|
+
version=meta.get("version", ""),
|
|
399
|
+
sha256=meta.get("sha256", "").lower(),
|
|
400
|
+
status=status,
|
|
401
|
+
hidden=RETIRED_TAG in tags,
|
|
402
|
+
created_at=created,
|
|
403
|
+
extra={
|
|
404
|
+
"vmid": str(vmid),
|
|
405
|
+
"tags": ";".join(tags),
|
|
406
|
+
"lock": lock,
|
|
407
|
+
"import_file": meta.get("import_file", ""),
|
|
408
|
+
},
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
# -- import --------------------------------------------------------------------------
|
|
412
|
+
|
|
413
|
+
def allocate_vmid(self) -> int:
|
|
414
|
+
if self.vmid_base is None:
|
|
415
|
+
return int(self.api.call("get", "/cluster/nextid"))
|
|
416
|
+
used = {
|
|
417
|
+
int(r["vmid"])
|
|
418
|
+
for r in self.api.call("get", "/cluster/resources", {"type": "vm"}) or []
|
|
419
|
+
if r.get("vmid") is not None
|
|
420
|
+
}
|
|
421
|
+
vmid = self.vmid_base
|
|
422
|
+
while vmid in used:
|
|
423
|
+
vmid += 1
|
|
424
|
+
# Asserts the id is free (fails if it was taken meanwhile or is reserved).
|
|
425
|
+
return int(self.api.call("get", "/cluster/nextid", {"vmid": vmid}))
|
|
426
|
+
|
|
427
|
+
def import_filename(self, target: Target, vmid: int) -> str:
|
|
428
|
+
base = urlsplit(target.artifact.url).path.rsplit("/", 1)[-1]
|
|
429
|
+
base = re.sub(r"[^A-Za-z0-9._-]+", "_", base) or f"image.{target.artifact.format}"
|
|
430
|
+
if not base.endswith(f".{target.artifact.format}"):
|
|
431
|
+
base += f".{target.artifact.format}"
|
|
432
|
+
return f"oic-{vmid}-{base}"
|
|
433
|
+
|
|
434
|
+
def create_params(
|
|
435
|
+
self, target: Target, name: str, vmid: int, import_volid: str
|
|
436
|
+
) -> dict[str, Any]:
|
|
437
|
+
"""Parameters of POST /nodes/<node>/qemu: the VM shell, without the disk."""
|
|
438
|
+
artifact = target.artifact
|
|
439
|
+
meta = {"oic_id": target.image.id}
|
|
440
|
+
if artifact.variant:
|
|
441
|
+
meta["oic_variant"] = artifact.variant
|
|
442
|
+
meta.update(
|
|
443
|
+
# The VMID the block was written for: a clone copies the description and the tags,
|
|
444
|
+
# and must never be taken for one of our imports (see list_managed).
|
|
445
|
+
vmid=str(vmid),
|
|
446
|
+
version=artifact.version,
|
|
447
|
+
sha256=artifact.sha256,
|
|
448
|
+
imported_at=self.now().strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
449
|
+
import_file=import_volid,
|
|
450
|
+
source=artifact.url,
|
|
451
|
+
)
|
|
452
|
+
props = artifact.glance_properties
|
|
453
|
+
params: dict[str, Any] = {
|
|
454
|
+
"vmid": vmid,
|
|
455
|
+
"name": name,
|
|
456
|
+
"tags": ";".join([MANAGED_TAG, image_tag(target.image.id), RETIRED_TAG]),
|
|
457
|
+
"description": render_description(target, meta),
|
|
458
|
+
"memory": self.memory,
|
|
459
|
+
"cores": self.cores,
|
|
460
|
+
"net0": f"virtio,bridge={self.bridge}",
|
|
461
|
+
"scsihw": "virtio-scsi-single",
|
|
462
|
+
"serial0": "socket",
|
|
463
|
+
"vga": "serial0",
|
|
464
|
+
}
|
|
465
|
+
if props.get("os_type", "linux") == "linux":
|
|
466
|
+
params["ostype"] = "l26"
|
|
467
|
+
if props.get("hw_qemu_guest_agent", "yes").lower() not in ("no", "false", "0"):
|
|
468
|
+
params["agent"] = "enabled=1"
|
|
469
|
+
if artifact.firmware == "uefi":
|
|
470
|
+
params.update(bios="ovmf", machine="q35", efidisk0=f"{self.disk_storage}:1,efitype=4m")
|
|
471
|
+
return params
|
|
472
|
+
|
|
473
|
+
def disk_params(self, target: Target, import_volid: str) -> dict[str, Any]:
|
|
474
|
+
"""Parameters of POST /nodes/<node>/qemu/<vmid>/config: disk import and cloud-init."""
|
|
475
|
+
params: dict[str, Any] = {
|
|
476
|
+
"scsi0": f"{self.disk_storage}:0,import-from={import_volid},discard=on",
|
|
477
|
+
"ide2": f"{self.disk_storage}:cloudinit",
|
|
478
|
+
"boot": "order=scsi0",
|
|
479
|
+
"ipconfig0": "ip=dhcp",
|
|
480
|
+
}
|
|
481
|
+
if target.image.login_user:
|
|
482
|
+
params["ciuser"] = target.image.login_user
|
|
483
|
+
return params
|
|
484
|
+
|
|
485
|
+
def import_target(self, target: Target, name: str) -> ManagedImage:
|
|
486
|
+
artifact = target.artifact
|
|
487
|
+
if artifact.format not in IMPORT_FORMATS:
|
|
488
|
+
raise BackendError(f"Proxmox cannot import {artifact.format} images")
|
|
489
|
+
self.check_storages()
|
|
490
|
+
vmid = self.allocate_vmid()
|
|
491
|
+
filename = self.import_filename(target, vmid)
|
|
492
|
+
volid = f"{self.import_storage}:import/{filename}"
|
|
493
|
+
downloaded = False
|
|
494
|
+
try:
|
|
495
|
+
log.info("%s: creating VM %s", target.label, vmid)
|
|
496
|
+
params = self.create_params(target, name, vmid, volid)
|
|
497
|
+
upid = self.api.call("create", self._qemu, params)
|
|
498
|
+
self.wait_task(upid, f"create VM {vmid}")
|
|
499
|
+
|
|
500
|
+
log.info("%s: downloading %s", target.label, artifact.url)
|
|
501
|
+
downloaded = True
|
|
502
|
+
upid = self.api.call(
|
|
503
|
+
"create",
|
|
504
|
+
f"/nodes/{self.node}/storage/{self.import_storage}/download-url",
|
|
505
|
+
{
|
|
506
|
+
"content": "import",
|
|
507
|
+
"filename": filename,
|
|
508
|
+
"url": artifact.url,
|
|
509
|
+
"checksum": artifact.sha256,
|
|
510
|
+
"checksum-algorithm": "sha256",
|
|
511
|
+
},
|
|
512
|
+
)
|
|
513
|
+
self.wait_task(upid, f"download of {artifact.url}")
|
|
514
|
+
|
|
515
|
+
log.info("%s: importing the disk into %s", target.label, self.disk_storage)
|
|
516
|
+
upid = self.api.call(
|
|
517
|
+
"create", f"{self._qemu}/{vmid}/config", self.disk_params(target, volid)
|
|
518
|
+
)
|
|
519
|
+
self.wait_task(upid, f"disk import into VM {vmid}")
|
|
520
|
+
|
|
521
|
+
upid = self.api.call("create", f"{self._qemu}/{vmid}/template")
|
|
522
|
+
self.wait_task(upid, f"conversion of VM {vmid} to a template")
|
|
523
|
+
except BaseException:
|
|
524
|
+
log.warning("%s: removing incomplete VM %s", target.label, vmid)
|
|
525
|
+
self._destroy_if_ours(vmid, volid)
|
|
526
|
+
if downloaded:
|
|
527
|
+
self.remove_import_file(volid)
|
|
528
|
+
raise
|
|
529
|
+
self.remove_import_file(volid)
|
|
530
|
+
cfg = self._config(vmid)
|
|
531
|
+
image = self._managed(vmid, cfg, parse_description(cfg.get("description")))
|
|
532
|
+
if image.status != "active" or image.sha256 != artifact.sha256:
|
|
533
|
+
raise BackendError(f"VM {vmid} is not a template of {artifact.sha256} after import")
|
|
534
|
+
return image
|
|
535
|
+
|
|
536
|
+
def remove_import_file(self, volid: str) -> None:
|
|
537
|
+
"""Delete a downloaded file from the import storage, if it is there (best effort)."""
|
|
538
|
+
storage = volid.split(":", 1)[0]
|
|
539
|
+
base = f"/nodes/{self.node}/storage/{storage}/content"
|
|
540
|
+
try:
|
|
541
|
+
content = self.api.call("get", base, {"content": "import"}) or []
|
|
542
|
+
if not any(item.get("volid") == volid for item in content):
|
|
543
|
+
return
|
|
544
|
+
# With a delay, the API waits for the task and returns null once it is done.
|
|
545
|
+
upid = self.api.call("delete", f"{base}/{volid}", {"delay": 30})
|
|
546
|
+
self.wait_task(upid, f"removal of {volid}")
|
|
547
|
+
except BackendError as e:
|
|
548
|
+
log.warning("could not remove %s: %s", volid, e)
|
|
549
|
+
|
|
550
|
+
# -- rotation ------------------------------------------------------------------------
|
|
551
|
+
|
|
552
|
+
def rename(self, image: ManagedImage, name: str, hidden: bool) -> None:
|
|
553
|
+
tags = [t for t in split_tags(image.extra.get("tags")) if t != RETIRED_TAG]
|
|
554
|
+
if hidden:
|
|
555
|
+
tags.append(RETIRED_TAG)
|
|
556
|
+
self.api.call(
|
|
557
|
+
"set", f"{self._qemu}/{image.id}/config", {"name": name, "tags": ";".join(tags)}
|
|
558
|
+
)
|
|
559
|
+
|
|
560
|
+
def _destroy(self, vmid: int | str, *, skiplock: bool = False, quiet: bool = False) -> None:
|
|
561
|
+
params: dict[str, Any] = {"purge": 1, "destroy-unreferenced-disks": 1}
|
|
562
|
+
if skiplock:
|
|
563
|
+
params["skiplock"] = 1
|
|
564
|
+
try:
|
|
565
|
+
upid = self.api.call("delete", f"{self._qemu}/{vmid}", params)
|
|
566
|
+
self.wait_task(upid, f"destruction of VM {vmid}")
|
|
567
|
+
except BackendError as e:
|
|
568
|
+
if not quiet:
|
|
569
|
+
raise
|
|
570
|
+
log.error("could not remove VM %s: %s", vmid, e)
|
|
571
|
+
|
|
572
|
+
def _destroy_if_ours(self, vmid: int, volid: str) -> None:
|
|
573
|
+
"""Remove the VM of a failed import, only if it is the one this import created.
|
|
574
|
+
|
|
575
|
+
The VMID could have been taken by someone else between allocation and creation:
|
|
576
|
+
the description must name this import's file.
|
|
577
|
+
"""
|
|
578
|
+
try:
|
|
579
|
+
cfg = self._config(vmid)
|
|
580
|
+
except BackendError:
|
|
581
|
+
return # never created
|
|
582
|
+
if parse_description(cfg.get("description")).get("import_file") != volid:
|
|
583
|
+
log.warning("VM %s is not the one this import created; left alone", vmid)
|
|
584
|
+
return
|
|
585
|
+
self._destroy(vmid, skiplock=True, quiet=True)
|
|
586
|
+
|
|
587
|
+
def delete(self, image: ManagedImage) -> None:
|
|
588
|
+
# An abandoned import (not a template, older than the stale delay) may still hold the
|
|
589
|
+
# "create" lock of a crashed run: skip it. Templates are never force-unlocked.
|
|
590
|
+
self._destroy(image.id, skiplock=not image.usable)
|
|
591
|
+
if image.extra.get("import_file"):
|
|
592
|
+
self.remove_import_file(image.extra["import_file"])
|
|
593
|
+
|
|
594
|
+
def in_use(self, image: ManagedImage) -> bool | None:
|
|
595
|
+
"""True when a VM of the cluster is a linked clone of the template.
|
|
596
|
+
|
|
597
|
+
A linked clone references the template base volumes (base-<vmid>-disk-N) in its
|
|
598
|
+
disks; every VM config of the cluster is checked. None when a call fails.
|
|
599
|
+
"""
|
|
600
|
+
base = re.compile(rf"(?:^|[:/])base-{re.escape(image.id)}-disk-\d+")
|
|
601
|
+
try:
|
|
602
|
+
for vm in self.api.call("get", "/cluster/resources", {"type": "vm"}) or []:
|
|
603
|
+
if vm.get("type") != "qemu" or str(vm.get("vmid")) == image.id:
|
|
604
|
+
continue
|
|
605
|
+
cfg = self._config(vm["vmid"], vm.get("node") or self.node)
|
|
606
|
+
for key, value in cfg.items():
|
|
607
|
+
if DISK_KEY.match(key) and isinstance(value, str) and base.search(value):
|
|
608
|
+
log.info("template %s is used by VM %s (%s)", image.id, vm["vmid"], key)
|
|
609
|
+
return True
|
|
610
|
+
except (BackendError, KeyError, TypeError, AttributeError) as e:
|
|
611
|
+
log.warning("usage check for %s failed: %s", image.id, e)
|
|
612
|
+
return None
|
|
613
|
+
return False
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def local_node(api: PveApi, node: str | None = None) -> str:
|
|
617
|
+
"""The node to work on: --node, else the one named like this host, else the only one."""
|
|
618
|
+
nodes = [str(n.get("node")) for n in api.call("get", "/nodes") or [] if n.get("node")]
|
|
619
|
+
if node:
|
|
620
|
+
if nodes and node not in nodes:
|
|
621
|
+
raise BackendError(f"node {node!r} not in the cluster ({', '.join(nodes)})")
|
|
622
|
+
return node
|
|
623
|
+
short = socket.gethostname().split(".", 1)[0]
|
|
624
|
+
if short in nodes:
|
|
625
|
+
return short
|
|
626
|
+
if len(nodes) == 1:
|
|
627
|
+
return nodes[0]
|
|
628
|
+
raise BackendError(f"cannot tell the local node among {', '.join(nodes) or 'none'}: use --node")
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def connect(
|
|
632
|
+
node: str | None = None, *, api: PveApi | None = None, **options: Any
|
|
633
|
+
) -> ProxmoxBackend:
|
|
634
|
+
"""Backend on the local node, through pvesh (must run on a Proxmox VE node, as root)."""
|
|
635
|
+
if api is None:
|
|
636
|
+
if shutil.which("pvesh") is None:
|
|
637
|
+
raise BackendError("pvesh not found: run oic-sync on a Proxmox VE node, as root")
|
|
638
|
+
api = Pvesh()
|
|
639
|
+
return ProxmoxBackend(api, local_node(api, node), **options)
|