testcloud 0.9.11__tar.gz → 0.9.13__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. {testcloud-0.9.11/testcloud.egg-info → testcloud-0.9.13}/PKG-INFO +12 -2
  2. {testcloud-0.9.11 → testcloud-0.9.13}/README.md +5 -1
  3. {testcloud-0.9.11 → testcloud-0.9.13}/conf/settings-example.py +18 -3
  4. {testcloud-0.9.11 → testcloud-0.9.13}/docs/source/conf.py +1 -1
  5. {testcloud-0.9.11 → testcloud-0.9.13}/setup.py +2 -0
  6. {testcloud-0.9.11 → testcloud-0.9.13}/test/test_instance.py +11 -1
  7. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/__init__.py +1 -1
  8. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/cli.py +29 -18
  9. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/config.py +26 -37
  10. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/distro_utils/centos.py +5 -1
  11. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/distro_utils/fedora.py +12 -6
  12. testcloud-0.9.13/testcloud/distro_utils/misc.py +43 -0
  13. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/distro_utils/ubuntu.py +4 -3
  14. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/domain_configuration.py +48 -26
  15. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/image.py +5 -2
  16. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/instance.py +39 -29
  17. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/util.py +1 -1
  18. testcloud-0.9.13/testcloud/workarounds.py +75 -0
  19. {testcloud-0.9.11 → testcloud-0.9.13/testcloud.egg-info}/PKG-INFO +12 -2
  20. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud.egg-info/SOURCES.txt +1 -1
  21. testcloud-0.9.13/testcloud.egg-info/requires.txt +7 -0
  22. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud.spec +32 -5
  23. testcloud-0.9.11/conf/domain-template.jinja +0 -55
  24. testcloud-0.9.11/testcloud/distro_utils/misc.py +0 -18
  25. testcloud-0.9.11/testcloud.egg-info/requires.txt +0 -3
  26. {testcloud-0.9.11 → testcloud-0.9.13}/.gitignore +0 -0
  27. {testcloud-0.9.11 → testcloud-0.9.13}/LICENSE +0 -0
  28. {testcloud-0.9.11 → testcloud-0.9.13}/Makefile +0 -0
  29. {testcloud-0.9.11 → testcloud-0.9.13}/conf/99-testcloud-nonroot-libvirt-access.rules +0 -0
  30. {testcloud-0.9.11 → testcloud-0.9.13}/docs/LICENSE +0 -0
  31. {testcloud-0.9.11 → testcloud-0.9.13}/docs/Makefile +0 -0
  32. {testcloud-0.9.11 → testcloud-0.9.13}/docs/source/api.rst +0 -0
  33. {testcloud-0.9.11 → testcloud-0.9.13}/docs/source/indepth.rst +0 -0
  34. {testcloud-0.9.11 → testcloud-0.9.13}/docs/source/index.rst +0 -0
  35. {testcloud-0.9.11 → testcloud-0.9.13}/requirements.txt +0 -0
  36. {testcloud-0.9.11 → testcloud-0.9.13}/run_testcloud.py +0 -0
  37. {testcloud-0.9.11 → testcloud-0.9.13}/setup.cfg +0 -0
  38. {testcloud-0.9.11 → testcloud-0.9.13}/test/__init__.py +0 -0
  39. {testcloud-0.9.11 → testcloud-0.9.13}/test/test_cli.py +0 -0
  40. {testcloud-0.9.11 → testcloud-0.9.13}/test/test_config.py +0 -0
  41. {testcloud-0.9.11 → testcloud-0.9.13}/test/test_image.py +0 -0
  42. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/distro_utils/__init__.py +0 -0
  43. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/distro_utils/alma.py +0 -0
  44. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/distro_utils/debian.py +0 -0
  45. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/distro_utils/oracle.py +0 -0
  46. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/distro_utils/rocky.py +0 -0
  47. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud/exceptions.py +0 -0
  48. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud.egg-info/dependency_links.txt +0 -0
  49. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud.egg-info/entry_points.txt +0 -0
  50. {testcloud-0.9.11 → testcloud-0.9.13}/testcloud.egg-info/top_level.txt +0 -0
  51. {testcloud-0.9.11 → testcloud-0.9.13}/tox.ini +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: testcloud
3
- Version: 0.9.11
3
+ Version: 0.9.13
4
4
  Summary: A tool to download and boot cloud images locally, with an easy to use API.
5
5
  Home-page: https://pagure.io/testcloud
6
6
  Author: Mike Ruckman
@@ -8,6 +8,12 @@ Author-email: roshi@fedoraproject.org
8
8
  License: GPLv2+
9
9
  Description-Content-Type: text/markdown
10
10
  License-File: LICENSE
11
+ Requires-Dist: Jinja2
12
+ Requires-Dist: libvirt-python
13
+ Requires-Dist: requests
14
+ Requires-Dist: packaging
15
+ Provides-Extra: image-resolve-caching
16
+ Requires-Dist: requests_cache>=1.2; extra == "image-resolve-caching"
11
17
 
12
18
  # testcloud
13
19
 
@@ -255,10 +261,14 @@ new instance using **testcloud**.
255
261
  \--nic_number
256
262
 
257
263
  : To provide nic number you want
258
- \--virtiofs
259
264
 
265
+ \--virtiofs
260
266
  : To specify a local directory to mount and mount target like <host path>:<guest path>
261
267
 
268
+ \--iommu
269
+
270
+ : To provide virtual iommu device
271
+
262
272
  There are several additional options that can be used to create a
263
273
  new Coreos instance using **testcloud**.
264
274
 
@@ -244,10 +244,14 @@ new instance using **testcloud**.
244
244
  \--nic_number
245
245
 
246
246
  : To provide nic number you want
247
- \--virtiofs
248
247
 
248
+ \--virtiofs
249
249
  : To specify a local directory to mount and mount target like <host path>:<guest path>
250
250
 
251
+ \--iommu
252
+
253
+ : To provide virtual iommu device
254
+
251
255
  There are several additional options that can be used to create a
252
256
  new Coreos instance using **testcloud**.
253
257
 
@@ -31,9 +31,18 @@
31
31
  ## Read http://cloudinit.readthedocs.io/en/latest/topics/examples.html to see
32
32
  ## what options you can use here.
33
33
  #USER_DATA = """#cloud-config
34
- #password: %s
35
- #chpasswd: { expire: False }
36
- #ssh_pwauth: True
34
+ #ssh_pwauth: true
35
+ #password: ${password}
36
+ #chpasswd:
37
+ # expire: false
38
+ #users:
39
+ # - default
40
+ # - name: cloud-user
41
+ # plain_text_passwd: ${password}
42
+ # sudo: ALL=(ALL) NOPASSWD:ALL
43
+ # lock_passwd: false
44
+ #runcmd:
45
+ #${runcommands}
37
46
  #"""
38
47
 
39
48
  #COREOS_DATA = """variant: fcos
@@ -67,6 +76,12 @@
67
76
  # 0 = unlimited
68
77
  BACKINGSTORE_SIZE = 4
69
78
 
79
+ # Is reusing the local image url cache an option?
80
+ #CACHE_IMAGES = True
81
+
82
+ # # How old can a local image url cache be to take it as viable
83
+ #TRUST_DEADLINE = 7 # Days
84
+
70
85
  # ram size, in MiB
71
86
  #RAM = 768
72
87
  #RAM_COREOS = 2048
@@ -55,7 +55,7 @@ copyright = u'2016-2023, testCloud devs'
55
55
  # built documents.
56
56
  #
57
57
  # The short X.Y version.
58
- version = '0.9.11'
58
+ version = '0.9.13'
59
59
  # The full version, including alpha/beta/rc tags.
60
60
  release = version
61
61
 
@@ -54,5 +54,7 @@ setup(name='testcloud',
54
54
  'Jinja2',
55
55
  'libvirt-python',
56
56
  'requests',
57
+ 'packaging'
57
58
  ],
59
+ extras_require={"image_resolve_caching": ["requests_cache>=1.2"]}
58
60
  )
@@ -9,8 +9,14 @@ import os
9
9
 
10
10
  from unittest import mock
11
11
 
12
- from testcloud import instance, image, config
12
+ from testcloud import instance, image, config, domain_configuration
13
13
 
14
+ class dotdict(dict):
15
+ # https://stackoverflow.com/a/23689767
16
+ """dot.notation access to dictionary attributes"""
17
+ __getattr__ = dict.get
18
+ __setattr__ = dict.__setitem__
19
+ __delattr__ = dict.__delitem__
14
20
 
15
21
  class TestInstance:
16
22
 
@@ -60,6 +66,10 @@ class TestFindInstance(object):
60
66
  stub_listdir.return_value = [ref_name]
61
67
  monkeypatch.setattr(os, 'listdir', stub_listdir)
62
68
 
69
+ stub_data_dir = mock.Mock()
70
+ stub_data_dir.return_value = dotdict({"DATA_DIR": self.conf.DATA_DIR})
71
+ monkeypatch.setattr(config, 'get_config', stub_data_dir)
72
+
63
73
  test_instance = instance.find_instance(ref_name, ref_image)
64
74
 
65
75
  assert test_instance.path == ref_path
@@ -1,2 +1,2 @@
1
1
  # NOTE: if you update version, *make sure* to also update `docs/source/conf.py`
2
- __version__ = '0.9.11'
2
+ __version__ = '0.9.13'
@@ -26,6 +26,7 @@ from testcloud import instance
26
26
 
27
27
  from testcloud.domain_configuration import _get_default_domain_conf
28
28
  from testcloud.util import get_image_url
29
+ from testcloud.workarounds import Workarounds
29
30
  from testcloud.exceptions import TestcloudImageError, TestcloudPermissionsError, TestcloudInstanceError
30
31
 
31
32
 
@@ -58,7 +59,7 @@ def _handle_connection_tip(ip, port, vagrant=False):
58
59
  config_altered = False
59
60
  kind = ""
60
61
 
61
- if "#cloud-config\nssh_pwauth: true\npassword: %s\nchpasswd:\n expire: false\n" not in config_data.USER_DATA:
62
+ if "#cloud-config\nssh_pwauth: true\npassword: ${password}\nchpasswd:\n expire: false\n" not in config_data.USER_DATA:
62
63
  config_altered = True
63
64
 
64
65
  print("-"*80)
@@ -79,7 +80,6 @@ def _handle_connection_tip(ip, port, vagrant=False):
79
80
  print("ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null cloud-user@%s -p %d" % (ip, port))
80
81
 
81
82
  print("-"*80)
82
-
83
83
  if port != 22:
84
84
  print("Due to limitations of tescloud's user session VMs and bugs in some systems,"
85
85
  " the ssh connection may not be available immediately...")
@@ -334,23 +334,17 @@ def _create_instance(args):
334
334
  )
335
335
  sys.exit(1)
336
336
 
337
- url = None
338
- coreos = False
339
-
340
337
  try:
341
338
  url = get_image_url(args.url, arch=args.arch) if not any([prot in args.url for prot in ["http", "file"]]) else args.url
342
- except TestcloudImageError:
343
- pass
344
-
345
- if not url:
339
+ assert url
340
+ except (TestcloudImageError, AssertionError):
346
341
  log.error("Couldn't find the desired image ( %s )..." % args.url)
347
342
  sys.exit(1)
348
343
 
349
- if "coreos" in url:
350
- coreos = True
344
+ coreos = bool(re.search('coreos|rhcos', url.lower()))
351
345
 
346
+ virtiofs_split = args.virtiofs.split(":") if args.virtiofs else [None, None]
352
347
  if args.virtiofs:
353
- virtiofs_split = args.virtiofs.split(":")
354
348
  if len(virtiofs_split) != 2:
355
349
  log.error("Invalid format of virtiofs specification, use <host path>:<guest path>")
356
350
  sys.exit(1)
@@ -358,6 +352,12 @@ def _create_instance(args):
358
352
  log.error("virtiofs is unsupported in session mode by libvirt: https://gitlab.com/libvirt/libvirt/-/issues/535")
359
353
  sys.exit(1)
360
354
 
355
+ # See the iommu support status at https://libvirt.org/formatdomain.html#iommu-devices
356
+ if args.iommu:
357
+ if args.arch not in ["x86_64", "aarch64"]:
358
+ log.error("iommu is not support for the architecture {0}, use x86_64 or aarch64".format(args.arch))
359
+ sys.exit(1)
360
+
361
361
  tc_image = image.Image(url)
362
362
  try:
363
363
  tc_image.prepare()
@@ -380,24 +380,25 @@ def _create_instance(args):
380
380
  domain = _get_default_domain_conf(name=args.name,
381
381
  desired_arch=args.arch,
382
382
  connection=args.connection,
383
- image=tc_image,
383
+ backingstore_image=tc_image,
384
384
  coreos=coreos,
385
385
  vcpus=args.vcpus,
386
386
  ram=ram,
387
387
  disk_size=disk_size,
388
- disk_number=args.disk_number,
389
- nic_number=args.nic_number,
388
+ disk_count=args.disk_number,
389
+ nic_count=args.nic_number,
390
390
  tpm=args.tpm,
391
- serial=args.serial,
391
+ iommu=args.iommu,
392
+ use_disk_serial=args.serial,
392
393
  virtiofs_source=virtiofs_split[0],
393
394
  virtiofs_target=virtiofs_split[1]
394
395
  )
395
396
 
396
- tc_instance = instance.Instance(args.name,
397
+ tc_instance = instance.Instance(domain_configuration=domain,
397
398
  image=tc_image,
398
399
  connection=args.connection,
399
400
  desired_arch=args.arch,
400
- domain_configuration=domain
401
+ workarounds=Workarounds(defaults=True)
401
402
  )
402
403
  log.info("create %s instance %s" % ("coreos" if coreos else "cloud", args.name))
403
404
 
@@ -420,6 +421,10 @@ def _create_instance(args):
420
421
  else:
421
422
  tc_instance._remove_from_disk()
422
423
 
424
+ if args.dry:
425
+ tc_instance._remove_from_disk()
426
+ sys.exit(0)
427
+
423
428
  # create instance domain
424
429
  try:
425
430
  tc_instance.spawn_vm()
@@ -790,6 +795,9 @@ def get_argparser():
790
795
  instarg_create.add_argument("--keep",
791
796
  help="Don't remove instance from disk when something fails, useful for debugging",
792
797
  action="store_true")
798
+ instarg_create.add_argument("--dry",
799
+ help="Don't spawn an actual instance, useful for debugging",
800
+ action="store_true")
793
801
  instarg_create.add_argument("--ssh_path",
794
802
  help="specify your ssh pubkey path",
795
803
  type=str)
@@ -822,6 +830,9 @@ def get_argparser():
822
830
  instarg_create.add_argument("--virtiofs",
823
831
  type=str,
824
832
  help="specify a local directory to mount and mount target like <host path>:<guest path>")
833
+ instarg_create.add_argument("--iommu",
834
+ help="add iommu device",
835
+ action="store_true")
825
836
  imgarg = subparsers.add_parser("image", help="help on image options")
826
837
  imgarg_subp = imgarg.add_subparsers(title="subcommands",
827
838
  description="Types of commands available",
@@ -102,10 +102,6 @@ class ConfigData(object):
102
102
  DATA_DIR = "/var/lib/testcloud"
103
103
  STORE_DIR = "/var/lib/testcloud/backingstores"
104
104
 
105
- # libvirt domain XML Template
106
- # This lives either in the DEFAULT_CONF_DIR or DATA_DIR
107
- XML_TEMPLATE = "domain-template.jinja"
108
-
109
105
  # Data for cloud-init
110
106
 
111
107
  PASSWORD = 'passw0rd'
@@ -116,32 +112,17 @@ local-hostname: %s
116
112
  """
117
113
  USER_DATA = """#cloud-config
118
114
  ssh_pwauth: true
119
- password: %s
115
+ password: ${password}
120
116
  chpasswd:
121
117
  expire: false
122
118
  users:
123
119
  - default
124
120
  - name: cloud-user
125
- plain_text_passwd: %s
121
+ plain_text_passwd: ${password}
126
122
  sudo: ALL=(ALL) NOPASSWD:ALL
127
123
  lock_passwd: false
128
124
  runcmd:
129
- - sed -i -e '/^.*PermitRootLogin/s/^.*$/PermitRootLogin yes/'
130
- -e '/^.*UseDNS/s/^.*$/UseDNS no/'
131
- -e '/^.*GSSAPIAuthentication/s/^.*$/GSSAPIAuthentication no/'
132
- /etc/ssh/sshd_config
133
- - systemctl reload sshd
134
- - [sh, -c, 'if [ ! -f /etc/systemd/network/20-tc-usernet.network ] &&
135
- systemctl status systemd-networkd | grep -q "enabled;\\svendor\\spreset:\\senabled";
136
- then mkdir -p /etc/systemd/network/ &&
137
- echo "[Match]" >> /etc/systemd/network/20-tc-usernet.network &&
138
- echo "Name=en*" >> /etc/systemd/network/20-tc-usernet.network &&
139
- echo "[Network]" >> /etc/systemd/network/20-tc-usernet.network &&
140
- echo "DHCP=yes" >> /etc/systemd/network/20-tc-usernet.network; fi']
141
- - [sh, -c, 'if systemctl status systemd-networkd | grep -q "enabled;\\svendor\\spreset:\\senabled"; then
142
- systemctl restart systemd-networkd; fi']
143
- - [sh, -c, 'if cat /etc/os-release | grep -q platform:el8; then systemctl restart sshd; fi']
144
- - [sh, -c, 'dhclient || :']
125
+ ${runcommands}
145
126
  """
146
127
  COREOS_DATA = """variant: fcos
147
128
  version: 1.4.0
@@ -183,6 +164,12 @@ storage:
183
164
  # 0 = unlimited
184
165
  BACKINGSTORE_SIZE = 0
185
166
 
167
+ # Is reusing the local image url cache an option?
168
+ CACHE_IMAGES = True
169
+
170
+ # How old can a local image url cache be to take it as viable
171
+ TRUST_DEADLINE = 7 # Days
172
+
186
173
  # ram size, in MiB
187
174
  RAM = 768
188
175
  RAM_COREOS = 2048
@@ -234,44 +221,46 @@ storage:
234
221
  # Used to try to auto-fetch the latest qcow2
235
222
  CENTOS_STREAM_URL_PREFIX = "https://cloud.centos.org/centos/{0}-stream/{1}/images/"
236
223
  CENTOS_STREAM_VERSIONS = {
237
- "8": "https://cloud.centos.org/centos/8-stream/{0}/images/CentOS-Stream-GenericCloud-8-20220913.0.{0}.qcow2",
238
- "9": "https://cloud.centos.org/centos/9-stream/{0}/images/CentOS-Stream-GenericCloud-9-20230307.0.{0}.qcow2",
224
+ "8": "https://cloud.centos.org/centos/8-stream/{0}/images/CentOS-Stream-GenericCloud-8-20240603.0.{0}.qcow2",
225
+ "9": "https://cloud.centos.org/centos/9-stream/{0}/images/CentOS-Stream-GenericCloud-9-20240715.0.{0}.qcow2",
226
+ "10": "https://odcs.stream.centos.org/stream-10/production/CentOS-Stream-10-20240709.0/compose/BaseOS/{0}/images/CentOS-Stream-GenericCloud-10-20240709.0.{0}.qcow2",
239
227
  "latest": "9"
240
228
  }
241
229
 
242
230
  ROCKY_URL_PREFIX = "https://download.rockylinux.org/pub/rocky/{0}/images/{1}/"
243
231
  ROCKY_VERSIONS = {
244
- "8": "https://download.rockylinux.org/pub/rocky/8/images/{0}/Rocky-8-GenericCloud-Base-8.7-20230215.0.{0}.qcow2",
245
- "9": "https://download.rockylinux.org/pub/rocky/9/images/{0}/Rocky-9-GenericCloud-Base-9.1-20230215.0.{0}.qcow2",
232
+ "8": "https://download.rockylinux.org/pub/rocky/8/images/{0}/Rocky-8-GenericCloud-Base-8.10-20240528.0.{0}.qcow2",
233
+ "9": "https://download.rockylinux.org/pub/rocky/9/images/{0}/Rocky-9-GenericCloud-Base-9.4-20240609.1.{0}.qcow2",
246
234
  "latest": "9"
247
235
  }
248
236
 
249
- ALMA_URL_PREFIX = "https://ftp.linux.cz/pub/linux/almalinux/{0}/cloud/{1}/images/"
237
+ ALMA_URL_PREFIX = "https://repo.almalinux.org/almalinux/{0}/cloud/{1}/images/"
250
238
  ALMA_VERSIONS = {
251
- "8": "https://ftp.sh.cvut.cz/almalinux/8/cloud/{0}/images/AlmaLinux-8-GenericCloud-8.7-20221111.{0}.qcow2",
252
- "9": "https://ftp.sh.cvut.cz/almalinux/9/cloud/{0}/images/AlmaLinux-9-GenericCloud-9.1-20221118.{0}.qcow2",
239
+ "8": "https://repo.almalinux.org/almalinux/8/cloud/{0}/images/AlmaLinux-8-GenericCloud-8.10-20240530.{0}.qcow2",
240
+ "9": "https://repo.almalinux.org/almalinux/9/cloud/{0}/images/AlmaLinux-9-GenericCloud-9.4-20240507.{0}.qcow2",
253
241
  "latest": "9"
254
242
  }
255
243
 
244
+ # https://yum.oracle.com/oracle-linux-templates.html
256
245
  ORACLE_VERSIONS = {
257
- "7": "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-kvm-b145.qcow",
258
- "8": "https://yum.oracle.com/templates/OracleLinux/OL8/u7/x86_64/OL8U7_x86_64-kvm-b148.qcow",
259
- "9": "https://yum.oracle.com/templates/OracleLinux/OL9/u1/x86_64/OL9U1_x86_64-kvm-b158.qcow",
246
+ "7": "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-kvm-b218.qcow2",
247
+ "8": "https://yum.oracle.com/templates/OracleLinux/OL8/u10/x86_64/OL8U10_x86_64-kvm-b237.qcow2",
248
+ "9": "https://yum.oracle.com/templates/OracleLinux/OL9/u4/x86_64/OL9U4_x86_64-kvm-b234.qcow2",
260
249
  "latest": "9"
261
250
  }
262
251
 
263
252
  ORACLE_A64_VERSIONS = {
264
- "8": "https://yum.oracle.com/templates/OracleLinux/OL8/u7/aarch64/OL8U7_aarch64-kvm-cloud-b11.qcow",
265
- "9": "https://yum.oracle.com/templates/OracleLinux/OL9/u1/aarch64/OL9U1_aarch64-kvm-cloud-b14.qcow",
253
+ "8": "https://yum.oracle.com/templates/OracleLinux/OL8/u10/aarch64/OL8U10_aarch64-kvm-cloud-b100.qcow2",
254
+ "9": "https://yum.oracle.com/templates/OracleLinux/OL9/u4/aarch64/OL9U4_aarch64-kvm-cloud-b90.qcow2",
266
255
  "latest": "9"
267
256
  }
268
257
 
269
- DEBIAN_RELEASE_MAP = {"10": "buster","11": "bullseye"}
270
- DEBIAN_LATEST = "11"
258
+ DEBIAN_RELEASE_MAP = {"10": "buster","11": "bullseye", "12": "bookworm"}
259
+ DEBIAN_LATEST = "12"
271
260
  DEBIAN_IMG_URL = "https://cloud.debian.org/images/cloud/%s/daily/latest/debian-%s-genericcloud-%s-daily.qcow2"
272
261
 
273
262
  UBUNTU_RELEASES_API = "https://api.launchpad.net/devel/ubuntu/series"
274
- UBUNTU_IMG_URL = "https://cloud-images.ubuntu.com/%s/current/%s-server-cloudimg-%s-disk-kvm.img"
263
+ UBUNTU_IMG_URL = "https://cloud-images.ubuntu.com/%s/current/%s-server-cloudimg-%s.img"
275
264
 
276
265
  def merge_object(self, obj):
277
266
  '''Overwrites default values with values from a python object which have
@@ -24,8 +24,12 @@ def _get_centos_image_url(version:str, stream:bool, arch:str) -> str:
24
24
  if version == "latest":
25
25
  version = config_data.CENTOS_STREAM_VERSIONS["latest"]
26
26
 
27
+ # TODO: Until CentOS 10 starts syncing to the main mirror
28
+ if version == "10":
29
+ STREAM_URL_PREFIX = "https://odcs.stream.centos.org/stream-10/production/latest-CentOS-Stream/compose/BaseOS/{0}/images/".format(arch)
27
30
  # Try to dynamically get the latest at first
28
- STREAM_URL_PREFIX = config_data.CENTOS_STREAM_URL_PREFIX.format(version, arch)
31
+ else:
32
+ STREAM_URL_PREFIX = config_data.CENTOS_STREAM_URL_PREFIX.format(version, arch)
29
33
  IMG_NAME = r'CentOS-Stream-GenericCloud-{0}-[0-9.]+.{1}.qcow2'.format(version, arch)
30
34
  try:
31
35
  return parse_latest_qcow(IMG_NAME, STREAM_URL_PREFIX)
@@ -9,6 +9,7 @@ import requests
9
9
 
10
10
  from testcloud import config
11
11
  from testcloud import exceptions
12
+ from testcloud.distro_utils.misc import get_requests_session
12
13
 
13
14
  log = logging.getLogger('testcloud.util')
14
15
  config_data = config.get_config()
@@ -17,6 +18,8 @@ def _process_coreos_url(version:str, arch:str, platform:str) -> str:
17
18
  """
18
19
  Returns an CoreOS url in either qemu or openstack format
19
20
  """
21
+ session = get_requests_session()
22
+
20
23
  if version == "latest":
21
24
  version = "stable"
22
25
  if version not in config_data.STREAM_LIST:
@@ -26,7 +29,7 @@ def _process_coreos_url(version:str, arch:str, platform:str) -> str:
26
29
  log.error("Invalid platform ( %s ) requested for Fedora CoreOS." % platform)
27
30
  raise exceptions.TestcloudImageError
28
31
  try:
29
- result = requests.get("https://builds.coreos.fedoraproject.org/streams/%s.json"%version).json()
32
+ result = session.get("https://builds.coreos.fedoraproject.org/streams/%s.json"%version).json()
30
33
  except (ConnectionError, IndexError, requests.exceptions.JSONDecodeError):
31
34
  log.error("Failed to fetch the image.")
32
35
  raise exceptions.TestcloudImageError
@@ -57,10 +60,12 @@ def get_fedora_image_url(version:str, arch:str) -> str:
57
60
  primary_arches = ["x86_64", "aarch64"]
58
61
  url = ""
59
62
 
63
+ session = get_requests_session()
64
+
60
65
  # get coreos url
61
66
  if version in config_data.STREAM_LIST:
62
67
  try:
63
- result = requests.get("https://builds.coreos.fedoraproject.org/streams/%s.json"%version).json()
68
+ result = session.get("https://builds.coreos.fedoraproject.org/streams/%s.json"%version).json()
64
69
  except (ConnectionError, IndexError, requests.exceptions.JSONDecodeError):
65
70
  log.error("Failed to fetch the image.")
66
71
  raise exceptions.TestcloudImageError
@@ -69,7 +74,7 @@ def get_fedora_image_url(version:str, arch:str) -> str:
69
74
 
70
75
  #get Fedora Cloud url
71
76
  try:
72
- oraculum_releases = requests.get('https://packager-dashboard.fedoraproject.org/api/v1/releases').json()
77
+ oraculum_releases = session.get('https://packager-dashboard.fedoraproject.org/api/v1/releases').json()
73
78
  except (ConnectionError, IndexError, requests.exceptions.JSONDecodeError):
74
79
  log.error("Couldn't fetch Fedora releases from oraculum...")
75
80
  raise exceptions.TestcloudImageError
@@ -89,6 +94,7 @@ def get_fedora_image_url(version:str, arch:str) -> str:
89
94
  log.error("non-x86_64 architecture is not supported with Fedora qa-matrix.")
90
95
  raise exceptions.TestcloudImageError
91
96
  try:
97
+ # Never cache this one
92
98
  nominated_response = requests.get("https://fedoraproject.org/wiki/Test_Results:Current_Installation_Test")
93
99
  return str(re.findall(r'href=\"(.*.%s.qcow2)\"' % arch, nominated_response.text)[0])
94
100
  except (ConnectionError, IndexError):
@@ -98,7 +104,7 @@ def get_fedora_image_url(version:str, arch:str) -> str:
98
104
  if version == "rawhide" or version == "branched":
99
105
  stamp = 0
100
106
  try:
101
- releases = requests.get('https://openqa.fedoraproject.org/nightlies.json').json()
107
+ releases = session.get('https://openqa.fedoraproject.org/nightlies.json').json()
102
108
  except (ConnectionError, IndexError, requests.exceptions.JSONDecodeError):
103
109
  log.error("Failed to fetch the image.")
104
110
  raise exceptions.TestcloudImageError
@@ -116,14 +122,14 @@ def get_fedora_image_url(version:str, arch:str) -> str:
116
122
  version = str(oraculum_releases["fedora"]["stable"])
117
123
 
118
124
  try:
119
- releases = requests.get('https://getfedora.org/releases.json').json()
125
+ releases = session.get('https://getfedora.org/releases.json').json()
120
126
  except (ConnectionError, requests.exceptions.JSONDecodeError):
121
127
  log.error("Couldn't fetch Fedora releases list...")
122
128
  raise exceptions.TestcloudImageError
123
129
 
124
130
  url = ""
125
131
  for release in releases:
126
- if release["version"] == version and release["variant"] == "Cloud" and release["link"].endswith(".qcow2"):
132
+ if release["version"] == version and release["subvariant"] == "Cloud_Base" and release["link"].endswith(".qcow2"):
127
133
  # There are links only to primary architecutres in releases.json... much fun
128
134
  if arch in primary_arches and release["arch"] == arch:
129
135
  url = release["link"]
@@ -0,0 +1,43 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Copyright 2023, Red Hat, Inc.
3
+ # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+>
4
+ # See the LICENSE file for more details on Licensing
5
+
6
+ import logging
7
+ import re
8
+ import requests
9
+
10
+ from testcloud import exceptions, config
11
+ from packaging.version import Version
12
+
13
+ config_data = config.get_config()
14
+ log = logging.getLogger('testcloud.util')
15
+
16
+ def parse_latest_qcow(rule:str, url:str) -> str:
17
+ session = get_requests_session()
18
+
19
+ try:
20
+ resp = session.get(url)
21
+ resp.raise_for_status()
22
+ latest_img_name = sorted(re.findall(rule, resp.text))[-1] or exceptions.TestcloudImageError
23
+ return url + str(latest_img_name)
24
+ except:
25
+ raise exceptions.TestcloudImageError
26
+
27
+
28
+ def get_requests_session():
29
+ try:
30
+ assert config_data.CACHE_IMAGES
31
+ import requests_cache
32
+ assert Version(requests_cache.__version__) >= Version("1.2")
33
+
34
+ log.debug("Using local image url cache...")
35
+ return requests_cache.CachedSession(
36
+ cache_name="{}/testcloud_image_resolve_cache".format(config_data.DATA_DIR),
37
+ backend="sqlite",
38
+ stale_if_error=True,
39
+ expire_after=config_data.TRUST_DEADLINE * 60 * 60 * 24
40
+ )
41
+ except (ImportError, AssertionError):
42
+ log.debug("Not using local image url cache due to config or unmet dependencies...")
43
+ return requests.Session()
@@ -8,13 +8,16 @@ import requests
8
8
 
9
9
  from testcloud import config
10
10
  from testcloud import exceptions
11
+ from testcloud.distro_utils.misc import get_requests_session
11
12
 
12
13
  log = logging.getLogger('testcloud.util')
13
14
  config_data = config.get_config()
14
15
 
15
16
  def get_ubuntu_releases() -> dict:
17
+ session = get_requests_session()
18
+
16
19
  try:
17
- releases_resp = requests.get(config_data.UBUNTU_RELEASES_API).json()
20
+ releases_resp = session.get(config_data.UBUNTU_RELEASES_API).json()
18
21
  except (ConnectionError, IndexError, requests.exceptions.JSONDecodeError):
19
22
  log.error("Failed to fetch Ubuntu releases list.")
20
23
  raise exceptions.TestcloudImageError
@@ -32,8 +35,6 @@ def get_ubuntu_image_url(version:str, arch:str) -> str:
32
35
  log.error("Requested architecture is not supported by testcloud for Ubuntu.")
33
36
  raise exceptions.TestcloudImageError
34
37
 
35
- if arch != "x86_64":
36
- config_data.UBUNTU_IMG_URL = config_data.UBUNTU_IMG_URL.replace("-disk-kvm.img", ".img")
37
38
  releases = get_ubuntu_releases()
38
39
  if len(releases) == 0:
39
40
  raise exceptions.TestcloudImageError