oic-sync 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
oic_sync/cli.py ADDED
@@ -0,0 +1,480 @@
1
+ """Command line interface."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import logging
8
+ import os
9
+ import sys
10
+ from collections.abc import Callable, Sequence
11
+ from typing import Any
12
+
13
+ from oic_sync import __version__
14
+ from oic_sync.backends.base import BackendError
15
+ from oic_sync.http import make_client
16
+ from oic_sync.index import (
17
+ DEFAULT_INDEX_URL,
18
+ SERVICE_IMAGES,
19
+ CatalogIndex,
20
+ IndexFormatError,
21
+ Target,
22
+ fetch_index,
23
+ select_targets,
24
+ )
25
+ from oic_sync.signature import SignatureError, SignatureVerifier
26
+
27
+ EXIT_OK = 0
28
+ EXIT_FAILED = 1
29
+ EXIT_USAGE = 2
30
+
31
+ log = logging.getLogger("oic_sync")
32
+
33
+
34
+ def _property(value: str) -> tuple[str, str]:
35
+ key, sep, val = value.partition("=")
36
+ if not sep or not key:
37
+ raise argparse.ArgumentTypeError(f"expected KEY=VALUE, got {value!r}")
38
+ return key, val
39
+
40
+
41
+ def _positive(value: str) -> int:
42
+ n = int(value)
43
+ if n < 1:
44
+ raise argparse.ArgumentTypeError("must be at least 1")
45
+ return n
46
+
47
+
48
+ def build_parser() -> argparse.ArgumentParser:
49
+ common = argparse.ArgumentParser(add_help=False)
50
+ common.add_argument(
51
+ "--index-url",
52
+ default=os.environ.get("OIC_INDEX_URL", DEFAULT_INDEX_URL),
53
+ help="catalog index (default: %(default)s, or $OIC_INDEX_URL)",
54
+ )
55
+ common.add_argument(
56
+ "--image",
57
+ action="append",
58
+ default=[],
59
+ metavar="ID",
60
+ help="image id to handle, e.g. debian-13 (repeatable; default: every image except "
61
+ f"the service images {', '.join(sorted(SERVICE_IMAGES))})",
62
+ )
63
+ common.add_argument(
64
+ "--variant",
65
+ action="append",
66
+ default=[],
67
+ metavar="NAME",
68
+ help="only this variant of multi-variant images, e.g. uefi (repeatable)",
69
+ )
70
+ common.add_argument(
71
+ "--include-service-images",
72
+ action="store_true",
73
+ help="also handle the service images when no --image is given",
74
+ )
75
+ common.add_argument("-v", "--verbose", action="count", default=0, help="more logs")
76
+
77
+ rotation = argparse.ArgumentParser(add_help=False)
78
+ rotation.add_argument(
79
+ "--keep",
80
+ type=_positive,
81
+ default=2,
82
+ help="versions to keep per image, the current one included (default: %(default)s)",
83
+ )
84
+ rotation.add_argument(
85
+ "--delete",
86
+ action="store_true",
87
+ help="delete versions beyond --keep that nothing uses (instance, volume, linked clone)",
88
+ )
89
+
90
+ status_opts = argparse.ArgumentParser(add_help=False)
91
+ status_opts.add_argument("--json", action="store_true", help="JSON output")
92
+ status_opts.add_argument(
93
+ "--check", action="store_true", help="exit 1 when an image is not up to date"
94
+ )
95
+
96
+ sync_opts = argparse.ArgumentParser(add_help=False)
97
+ sync_opts.add_argument(
98
+ "--dry-run",
99
+ action="store_true",
100
+ help="show what would change; only read the platform (signatures are still verified)",
101
+ )
102
+ sync_opts.add_argument(
103
+ "--timeout",
104
+ type=float,
105
+ default=3600.0,
106
+ help="seconds to wait for an import step to finish (default: %(default)s)",
107
+ )
108
+ sync_opts.add_argument(
109
+ "--no-verify-signature",
110
+ action="store_true",
111
+ help="import without checking the cosign signature (not recommended)",
112
+ )
113
+
114
+ openstack_common = argparse.ArgumentParser(add_help=False)
115
+ openstack_common.add_argument(
116
+ "--cloud",
117
+ default=os.environ.get("OS_CLOUD"),
118
+ help="cloud name in clouds.yaml (default: $OS_CLOUD, else OS_* variables)",
119
+ )
120
+ openstack_common.add_argument("--region", help="region name, if not set in clouds.yaml")
121
+
122
+ proxmox_common = argparse.ArgumentParser(add_help=False)
123
+ proxmox_common.add_argument(
124
+ "--node", help="Proxmox VE node (default: the node named like this host)"
125
+ )
126
+
127
+ parser = argparse.ArgumentParser(
128
+ prog="oic-sync",
129
+ description="Keep openimages.cloud signed cloud images in sync with your cloud "
130
+ "(OpenStack Glance or Proxmox VE).",
131
+ )
132
+ parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
133
+ sub = parser.add_subparsers(dest="command", required=True, metavar="COMMAND")
134
+
135
+ p_list = sub.add_parser("list", parents=[common], help="list the images of the index")
136
+ p_list.add_argument("--json", action="store_true", help="JSON output")
137
+ p_list.set_defaults(func=cmd_list)
138
+
139
+ p_verify = sub.add_parser(
140
+ "verify",
141
+ parents=[common],
142
+ help="verify the cosign signature of every selected artifact (no cloud needed)",
143
+ )
144
+ p_verify.set_defaults(func=cmd_verify)
145
+
146
+ p_os = sub.add_parser("openstack", help="OpenStack Glance backend")
147
+ os_sub = p_os.add_subparsers(dest="action", required=True, metavar="ACTION")
148
+
149
+ p_status = os_sub.add_parser(
150
+ "status",
151
+ parents=[common, openstack_common, rotation, status_opts],
152
+ help="compare the cloud with the index (read only)",
153
+ )
154
+ p_status.set_defaults(func=cmd_openstack_status)
155
+
156
+ p_sync = os_sub.add_parser(
157
+ "sync",
158
+ parents=[common, openstack_common, rotation, sync_opts],
159
+ help="import new versions, rotate names, retire old versions",
160
+ )
161
+ p_sync.add_argument(
162
+ "--visibility",
163
+ choices=("private", "shared", "community", "public"),
164
+ default="private",
165
+ help="visibility of new images: public for operators, private or shared for "
166
+ "projects (default: %(default)s)",
167
+ )
168
+ p_sync.add_argument(
169
+ "--import-method",
170
+ choices=("auto", "web-download", "glance-direct", "upload"),
171
+ default="auto",
172
+ help="auto: web-download when Glance offers it, else streaming upload "
173
+ "(default: %(default)s)",
174
+ )
175
+ p_sync.add_argument(
176
+ "--property",
177
+ action="append",
178
+ type=_property,
179
+ default=[],
180
+ metavar="KEY=VALUE",
181
+ help="extra or overriding Glance property for new images (repeatable)",
182
+ )
183
+ p_sync.add_argument(
184
+ "--with-optional-properties",
185
+ action="store_true",
186
+ help="add the index optional properties (hw_rng_model=virtio: the flavors must "
187
+ "allow hw_rng:allowed)",
188
+ )
189
+ p_sync.set_defaults(func=cmd_openstack_sync)
190
+
191
+ p_pve = sub.add_parser(
192
+ "proxmox", help="Proxmox VE backend (VM templates; run on a node, as root)"
193
+ )
194
+ pve_sub = p_pve.add_subparsers(dest="action", required=True, metavar="ACTION")
195
+ p_pstatus = pve_sub.add_parser(
196
+ "status",
197
+ parents=[common, proxmox_common, rotation, status_opts],
198
+ help="compare the node templates with the index (read only)",
199
+ )
200
+ p_pstatus.set_defaults(func=cmd_proxmox_status)
201
+
202
+ p_psync = pve_sub.add_parser(
203
+ "sync",
204
+ parents=[common, proxmox_common, rotation, sync_opts],
205
+ help="import new versions as templates, rotate names, retire old versions",
206
+ )
207
+ p_psync.add_argument(
208
+ "--import-storage",
209
+ default="local",
210
+ help="file storage with the 'import' content type, for downloads (default: %(default)s)",
211
+ )
212
+ p_psync.add_argument(
213
+ "--disk-storage",
214
+ default="local-lvm",
215
+ help="storage of the template disks, EFI and cloud-init drives (default: %(default)s)",
216
+ )
217
+ p_psync.add_argument("--bridge", default="vmbr0", help="bridge of net0 (default: %(default)s)")
218
+ p_psync.add_argument(
219
+ "--vmid-base",
220
+ type=int,
221
+ metavar="N",
222
+ help="first VMID to use for templates, e.g. 9000 (default: the next free VMID)",
223
+ )
224
+ p_psync.add_argument(
225
+ "--memory",
226
+ type=_positive,
227
+ default=2048,
228
+ metavar="MIB",
229
+ help="memory of the templates in MiB (default: %(default)s)",
230
+ )
231
+ p_psync.add_argument(
232
+ "--cores",
233
+ type=_positive,
234
+ default=2,
235
+ help="CPU cores of the templates (default: %(default)s)",
236
+ )
237
+ p_psync.set_defaults(func=cmd_proxmox_sync)
238
+ return parser
239
+
240
+
241
+ def _load(args: argparse.Namespace, client: Any) -> tuple[CatalogIndex, list[Target]]:
242
+ index = fetch_index(client, args.index_url)
243
+ targets = select_targets(
244
+ index, args.image, args.variant, include_service=args.include_service_images
245
+ )
246
+ return index, targets
247
+
248
+
249
+ def cmd_list(args: argparse.Namespace, client: Any, out: Callable[[str], None]) -> int:
250
+ index, targets = _load(args, client)
251
+ if args.json:
252
+ out(
253
+ json.dumps(
254
+ [
255
+ {
256
+ "id": t.image.id,
257
+ "variant": t.artifact.variant,
258
+ "name": t.stable_name,
259
+ "version": t.artifact.version,
260
+ "firmware": t.artifact.firmware,
261
+ "url": t.artifact.url,
262
+ "sha256": t.artifact.sha256,
263
+ "size": t.artifact.size,
264
+ }
265
+ for t in targets
266
+ ],
267
+ indent=2,
268
+ )
269
+ )
270
+ return EXIT_OK
271
+ out(f"# {index.schema}, generated {index.generated_at}")
272
+ for t in targets:
273
+ size = f"{t.artifact.size / 2**20:.0f} MiB" if t.artifact.size else "?"
274
+ out(
275
+ f"{t.label:32} {t.artifact.version:24} {t.artifact.firmware:5} {size:>9} "
276
+ f"{t.stable_name}"
277
+ )
278
+ return EXIT_OK
279
+
280
+
281
+ def cmd_verify(
282
+ args: argparse.Namespace,
283
+ client: Any,
284
+ out: Callable[[str], None],
285
+ verifier: SignatureVerifier | None = None,
286
+ ) -> int:
287
+ index, targets = _load(args, client)
288
+ verifier = verifier or SignatureVerifier(client, index.signing)
289
+ failures = 0
290
+ for t in targets:
291
+ try:
292
+ identity = verifier.verify(t.artifact)
293
+ except SignatureError as e:
294
+ failures += 1
295
+ out(f"{t.label}: FAIL {t.artifact.version}: {e}")
296
+ continue
297
+ out(f"{t.label}: OK {t.artifact.version} signed by {identity}")
298
+ out(f"{len(targets) - failures}/{len(targets)} signatures verified")
299
+ return EXIT_FAILED if failures else EXIT_OK
300
+
301
+
302
+ def _backend_factory(args: argparse.Namespace, client: Any, **options: Any) -> Any:
303
+ from oic_sync.backends.openstack import connect
304
+
305
+ return connect(args.cloud, args.region, client, **options)
306
+
307
+
308
+ def _proxmox_factory(args: argparse.Namespace, **options: Any) -> Any:
309
+ from oic_sync.backends.proxmox import connect
310
+
311
+ return connect(args.node, **options)
312
+
313
+
314
+ def _status(args: argparse.Namespace, client: Any, out: Callable[[str], None], backend: Any) -> int:
315
+ from oic_sync.engine import UP_TO_DATE, Engine
316
+
317
+ _, targets = _load(args, client)
318
+ engine = Engine(backend, verifier=None, keep=args.keep, delete=args.delete, out=out)
319
+ plans = engine.status(targets)
320
+ rows = []
321
+ for p in plans:
322
+ cloud = p.current or (p.older[0] if p.older else None)
323
+ rows.append(
324
+ {
325
+ "id": p.target.image.id,
326
+ "variant": p.target.artifact.variant,
327
+ "name": p.stable_name,
328
+ "index_version": p.target.artifact.version,
329
+ "cloud_version": cloud.version if cloud else None,
330
+ "cloud_image_id": cloud.id if cloud else None,
331
+ "state": p.state,
332
+ "older_versions": len(p.older) - (1 if cloud and not p.current else 0),
333
+ "pending": [a.describe() for a in p.actions],
334
+ "notes": p.notes,
335
+ }
336
+ )
337
+ if args.json:
338
+ out(json.dumps({"backend": backend.describe(), "images": rows}, indent=2))
339
+ else:
340
+ out(f"# {backend.describe()}")
341
+ out(f"{'IMAGE':32} {'INDEX':24} {'CLOUD':24} {'STATE':15} OLDER")
342
+ for p, r in zip(plans, rows, strict=True):
343
+ out(
344
+ f"{p.target.label:32} {r['index_version']:24} {r['cloud_version'] or '-':24} "
345
+ f"{r['state']:15} {r['older_versions']}"
346
+ )
347
+ for pending in r["pending"]:
348
+ out(f" pending: {pending}")
349
+ if args.check and any(p.state != UP_TO_DATE for p in plans):
350
+ return EXIT_FAILED
351
+ return EXIT_OK
352
+
353
+
354
+ def _sync(
355
+ args: argparse.Namespace,
356
+ client: Any,
357
+ out: Callable[[str], None],
358
+ index: CatalogIndex,
359
+ targets: list[Target],
360
+ backend: Any,
361
+ verifier: SignatureVerifier | None,
362
+ ) -> int:
363
+ from oic_sync.engine import Engine
364
+
365
+ if args.no_verify_signature:
366
+ verifier = None
367
+ else:
368
+ verifier = verifier or SignatureVerifier(client, index.signing)
369
+ engine = Engine(
370
+ backend,
371
+ verifier=verifier,
372
+ keep=args.keep,
373
+ delete=args.delete,
374
+ dry_run=args.dry_run,
375
+ out=out,
376
+ )
377
+ out(f"# {backend.describe()}{' (dry run)' if args.dry_run else ''}")
378
+ results = engine.sync(targets)
379
+ failed = [r for r in results if not r.ok]
380
+ out(
381
+ f"{len(results) - len(failed)}/{len(results)} images in sync"
382
+ + (f", {len(failed)} failed: {', '.join(r.target.label for r in failed)}" if failed else "")
383
+ )
384
+ return EXIT_FAILED if failed else EXIT_OK
385
+
386
+
387
+ def cmd_openstack_status(
388
+ args: argparse.Namespace,
389
+ client: Any,
390
+ out: Callable[[str], None],
391
+ backend: Any = None,
392
+ ) -> int:
393
+ return _status(args, client, out, backend or _backend_factory(args, client))
394
+
395
+
396
+ def cmd_openstack_sync(
397
+ args: argparse.Namespace,
398
+ client: Any,
399
+ out: Callable[[str], None],
400
+ backend: Any = None,
401
+ verifier: SignatureVerifier | None = None,
402
+ ) -> int:
403
+ index, targets = _load(args, client)
404
+ backend = backend or _backend_factory(
405
+ args,
406
+ client,
407
+ visibility=args.visibility,
408
+ import_method=args.import_method,
409
+ extra_properties=dict(args.property),
410
+ optional_properties=args.with_optional_properties,
411
+ import_timeout=args.timeout,
412
+ )
413
+ return _sync(args, client, out, index, targets, backend, verifier)
414
+
415
+
416
+ def cmd_proxmox_status(
417
+ args: argparse.Namespace,
418
+ client: Any,
419
+ out: Callable[[str], None],
420
+ backend: Any = None,
421
+ ) -> int:
422
+ return _status(args, client, out, backend or _proxmox_factory(args, read_only=True))
423
+
424
+
425
+ def cmd_proxmox_sync(
426
+ args: argparse.Namespace,
427
+ client: Any,
428
+ out: Callable[[str], None],
429
+ backend: Any = None,
430
+ verifier: SignatureVerifier | None = None,
431
+ ) -> int:
432
+ index, targets = _load(args, client)
433
+ backend = backend or _proxmox_factory(
434
+ args,
435
+ import_storage=args.import_storage,
436
+ disk_storage=args.disk_storage,
437
+ bridge=args.bridge,
438
+ vmid_base=args.vmid_base,
439
+ memory=args.memory,
440
+ cores=args.cores,
441
+ task_timeout=args.timeout,
442
+ )
443
+ return _sync(args, client, out, index, targets, backend, verifier)
444
+
445
+
446
+ def main(argv: Sequence[str] | None = None) -> int:
447
+ parser = build_parser()
448
+ args = parser.parse_args(argv)
449
+ logging.basicConfig(
450
+ level=logging.WARNING - 10 * min(args.verbose, 2),
451
+ format="%(levelname)s %(name)s: %(message)s",
452
+ stream=sys.stderr,
453
+ )
454
+ if args.verbose < 2:
455
+ # openstacksdk and httpx are chatty at INFO level.
456
+ for name in ("openstack", "keystoneauth", "httpx", "httpcore", "urllib3"):
457
+ logging.getLogger(name).setLevel(logging.WARNING)
458
+
459
+ def out(line: str) -> None:
460
+ print(line, flush=True)
461
+
462
+ try:
463
+ with make_client() as client:
464
+ return args.func(args, client, out)
465
+ except BackendError as e: # e.g. missing openstacksdk, pvesh not found, unknown node
466
+ print(f"oic-sync: error: {e}", file=sys.stderr)
467
+ return EXIT_FAILED
468
+ except (IndexFormatError, ValueError) as e:
469
+ print(f"oic-sync: error: {e}", file=sys.stderr)
470
+ return EXIT_USAGE
471
+ except KeyboardInterrupt:
472
+ return 130
473
+ except Exception as e:
474
+ log.debug("fatal error", exc_info=True)
475
+ print(f"oic-sync: error: {type(e).__name__}: {e}", file=sys.stderr)
476
+ return EXIT_FAILED
477
+
478
+
479
+ if __name__ == "__main__":
480
+ raise SystemExit(main())