testcloud 0.9.12__tar.gz → 0.9.13__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. {testcloud-0.9.12/testcloud.egg-info → testcloud-0.9.13}/PKG-INFO +12 -2
  2. {testcloud-0.9.12 → testcloud-0.9.13}/README.md +5 -1
  3. {testcloud-0.9.12 → testcloud-0.9.13}/conf/settings-example.py +18 -3
  4. {testcloud-0.9.12 → testcloud-0.9.13}/docs/source/conf.py +1 -1
  5. {testcloud-0.9.12 → testcloud-0.9.13}/setup.py +2 -0
  6. {testcloud-0.9.12 → testcloud-0.9.13}/test/test_instance.py +11 -1
  7. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/__init__.py +1 -1
  8. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/cli.py +28 -16
  9. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/config.py +26 -37
  10. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/distro_utils/centos.py +5 -1
  11. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/distro_utils/fedora.py +12 -6
  12. testcloud-0.9.13/testcloud/distro_utils/misc.py +43 -0
  13. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/distro_utils/ubuntu.py +4 -3
  14. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/domain_configuration.py +48 -26
  15. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/image.py +5 -2
  16. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/instance.py +39 -29
  17. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/util.py +1 -1
  18. testcloud-0.9.13/testcloud/workarounds.py +75 -0
  19. {testcloud-0.9.12 → testcloud-0.9.13/testcloud.egg-info}/PKG-INFO +12 -2
  20. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud.egg-info/SOURCES.txt +1 -1
  21. testcloud-0.9.13/testcloud.egg-info/requires.txt +7 -0
  22. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud.spec +29 -5
  23. testcloud-0.9.12/conf/domain-template.jinja +0 -55
  24. testcloud-0.9.12/testcloud/distro_utils/misc.py +0 -18
  25. testcloud-0.9.12/testcloud.egg-info/requires.txt +0 -3
  26. {testcloud-0.9.12 → testcloud-0.9.13}/.gitignore +0 -0
  27. {testcloud-0.9.12 → testcloud-0.9.13}/LICENSE +0 -0
  28. {testcloud-0.9.12 → testcloud-0.9.13}/Makefile +0 -0
  29. {testcloud-0.9.12 → testcloud-0.9.13}/conf/99-testcloud-nonroot-libvirt-access.rules +0 -0
  30. {testcloud-0.9.12 → testcloud-0.9.13}/docs/LICENSE +0 -0
  31. {testcloud-0.9.12 → testcloud-0.9.13}/docs/Makefile +0 -0
  32. {testcloud-0.9.12 → testcloud-0.9.13}/docs/source/api.rst +0 -0
  33. {testcloud-0.9.12 → testcloud-0.9.13}/docs/source/indepth.rst +0 -0
  34. {testcloud-0.9.12 → testcloud-0.9.13}/docs/source/index.rst +0 -0
  35. {testcloud-0.9.12 → testcloud-0.9.13}/requirements.txt +0 -0
  36. {testcloud-0.9.12 → testcloud-0.9.13}/run_testcloud.py +0 -0
  37. {testcloud-0.9.12 → testcloud-0.9.13}/setup.cfg +0 -0
  38. {testcloud-0.9.12 → testcloud-0.9.13}/test/__init__.py +0 -0
  39. {testcloud-0.9.12 → testcloud-0.9.13}/test/test_cli.py +0 -0
  40. {testcloud-0.9.12 → testcloud-0.9.13}/test/test_config.py +0 -0
  41. {testcloud-0.9.12 → testcloud-0.9.13}/test/test_image.py +0 -0
  42. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/distro_utils/__init__.py +0 -0
  43. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/distro_utils/alma.py +0 -0
  44. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/distro_utils/debian.py +0 -0
  45. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/distro_utils/oracle.py +0 -0
  46. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/distro_utils/rocky.py +0 -0
  47. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud/exceptions.py +0 -0
  48. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud.egg-info/dependency_links.txt +0 -0
  49. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud.egg-info/entry_points.txt +0 -0
  50. {testcloud-0.9.12 → testcloud-0.9.13}/testcloud.egg-info/top_level.txt +0 -0
  51. {testcloud-0.9.12 → testcloud-0.9.13}/tox.ini +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: testcloud
3
- Version: 0.9.12
3
+ Version: 0.9.13
4
4
  Summary: A tool to download and boot cloud images locally, with an easy to use API.
5
5
  Home-page: https://pagure.io/testcloud
6
6
  Author: Mike Ruckman
@@ -8,6 +8,12 @@ Author-email: roshi@fedoraproject.org
8
8
  License: GPLv2+
9
9
  Description-Content-Type: text/markdown
10
10
  License-File: LICENSE
11
+ Requires-Dist: Jinja2
12
+ Requires-Dist: libvirt-python
13
+ Requires-Dist: requests
14
+ Requires-Dist: packaging
15
+ Provides-Extra: image-resolve-caching
16
+ Requires-Dist: requests_cache>=1.2; extra == "image-resolve-caching"
11
17
 
12
18
  # testcloud
13
19
 
@@ -255,10 +261,14 @@ new instance using **testcloud**.
255
261
  \--nic_number
256
262
 
257
263
  : To provide nic number you want
258
- \--virtiofs
259
264
 
265
+ \--virtiofs
260
266
  : To specify a local directory to mount and mount target like <host path>:<guest path>
261
267
 
268
+ \--iommu
269
+
270
+ : To provide virtual iommu device
271
+
262
272
  There are several additional options that can be used to create a
263
273
  new Coreos instance using **testcloud**.
264
274
 
@@ -244,10 +244,14 @@ new instance using **testcloud**.
244
244
  \--nic_number
245
245
 
246
246
  : To provide nic number you want
247
- \--virtiofs
248
247
 
248
+ \--virtiofs
249
249
  : To specify a local directory to mount and mount target like <host path>:<guest path>
250
250
 
251
+ \--iommu
252
+
253
+ : To provide virtual iommu device
254
+
251
255
  There are several additional options that can be used to create a
252
256
  new Coreos instance using **testcloud**.
253
257
 
@@ -31,9 +31,18 @@
31
31
  ## Read http://cloudinit.readthedocs.io/en/latest/topics/examples.html to see
32
32
  ## what options you can use here.
33
33
  #USER_DATA = """#cloud-config
34
- #password: %s
35
- #chpasswd: { expire: False }
36
- #ssh_pwauth: True
34
+ #ssh_pwauth: true
35
+ #password: ${password}
36
+ #chpasswd:
37
+ # expire: false
38
+ #users:
39
+ # - default
40
+ # - name: cloud-user
41
+ # plain_text_passwd: ${password}
42
+ # sudo: ALL=(ALL) NOPASSWD:ALL
43
+ # lock_passwd: false
44
+ #runcmd:
45
+ #${runcommands}
37
46
  #"""
38
47
 
39
48
  #COREOS_DATA = """variant: fcos
@@ -67,6 +76,12 @@
67
76
  # 0 = unlimited
68
77
  BACKINGSTORE_SIZE = 4
69
78
 
79
+ # Is reusing the local image url cache an option?
80
+ #CACHE_IMAGES = True
81
+
82
+ # # How old can a local image url cache be to take it as viable
83
+ #TRUST_DEADLINE = 7 # Days
84
+
70
85
  # ram size, in MiB
71
86
  #RAM = 768
72
87
  #RAM_COREOS = 2048
@@ -55,7 +55,7 @@ copyright = u'2016-2023, testCloud devs'
55
55
  # built documents.
56
56
  #
57
57
  # The short X.Y version.
58
- version = '0.9.12'
58
+ version = '0.9.13'
59
59
  # The full version, including alpha/beta/rc tags.
60
60
  release = version
61
61
 
@@ -54,5 +54,7 @@ setup(name='testcloud',
54
54
  'Jinja2',
55
55
  'libvirt-python',
56
56
  'requests',
57
+ 'packaging'
57
58
  ],
59
+ extras_require={"image_resolve_caching": ["requests_cache>=1.2"]}
58
60
  )
@@ -9,8 +9,14 @@ import os
9
9
 
10
10
  from unittest import mock
11
11
 
12
- from testcloud import instance, image, config
12
+ from testcloud import instance, image, config, domain_configuration
13
13
 
14
+ class dotdict(dict):
15
+ # https://stackoverflow.com/a/23689767
16
+ """dot.notation access to dictionary attributes"""
17
+ __getattr__ = dict.get
18
+ __setattr__ = dict.__setitem__
19
+ __delattr__ = dict.__delitem__
14
20
 
15
21
  class TestInstance:
16
22
 
@@ -60,6 +66,10 @@ class TestFindInstance(object):
60
66
  stub_listdir.return_value = [ref_name]
61
67
  monkeypatch.setattr(os, 'listdir', stub_listdir)
62
68
 
69
+ stub_data_dir = mock.Mock()
70
+ stub_data_dir.return_value = dotdict({"DATA_DIR": self.conf.DATA_DIR})
71
+ monkeypatch.setattr(config, 'get_config', stub_data_dir)
72
+
63
73
  test_instance = instance.find_instance(ref_name, ref_image)
64
74
 
65
75
  assert test_instance.path == ref_path
@@ -1,2 +1,2 @@
1
1
  # NOTE: if you update version, *make sure* to also update `docs/source/conf.py`
2
- __version__ = '0.9.12'
2
+ __version__ = '0.9.13'
@@ -26,6 +26,7 @@ from testcloud import instance
26
26
 
27
27
  from testcloud.domain_configuration import _get_default_domain_conf
28
28
  from testcloud.util import get_image_url
29
+ from testcloud.workarounds import Workarounds
29
30
  from testcloud.exceptions import TestcloudImageError, TestcloudPermissionsError, TestcloudInstanceError
30
31
 
31
32
 
@@ -58,7 +59,7 @@ def _handle_connection_tip(ip, port, vagrant=False):
58
59
  config_altered = False
59
60
  kind = ""
60
61
 
61
- if "#cloud-config\nssh_pwauth: true\npassword: %s\nchpasswd:\n expire: false\n" not in config_data.USER_DATA:
62
+ if "#cloud-config\nssh_pwauth: true\npassword: ${password}\nchpasswd:\n expire: false\n" not in config_data.USER_DATA:
62
63
  config_altered = True
63
64
 
64
65
  print("-"*80)
@@ -333,20 +334,14 @@ def _create_instance(args):
333
334
  )
334
335
  sys.exit(1)
335
336
 
336
- url = None
337
- coreos = False
338
-
339
337
  try:
340
338
  url = get_image_url(args.url, arch=args.arch) if not any([prot in args.url for prot in ["http", "file"]]) else args.url
341
- except TestcloudImageError:
342
- pass
343
-
344
- if not url:
339
+ assert url
340
+ except (TestcloudImageError, AssertionError):
345
341
  log.error("Couldn't find the desired image ( %s )..." % args.url)
346
342
  sys.exit(1)
347
343
 
348
- if "coreos" in url:
349
- coreos = True
344
+ coreos = bool(re.search('coreos|rhcos', url.lower()))
350
345
 
351
346
  virtiofs_split = args.virtiofs.split(":") if args.virtiofs else [None, None]
352
347
  if args.virtiofs:
@@ -357,6 +352,12 @@ def _create_instance(args):
357
352
  log.error("virtiofs is unsupported in session mode by libvirt: https://gitlab.com/libvirt/libvirt/-/issues/535")
358
353
  sys.exit(1)
359
354
 
355
+ # See the iommu support status at https://libvirt.org/formatdomain.html#iommu-devices
356
+ if args.iommu:
357
+ if args.arch not in ["x86_64", "aarch64"]:
358
+ log.error("iommu is not support for the architecture {0}, use x86_64 or aarch64".format(args.arch))
359
+ sys.exit(1)
360
+
360
361
  tc_image = image.Image(url)
361
362
  try:
362
363
  tc_image.prepare()
@@ -379,24 +380,25 @@ def _create_instance(args):
379
380
  domain = _get_default_domain_conf(name=args.name,
380
381
  desired_arch=args.arch,
381
382
  connection=args.connection,
382
- image=tc_image,
383
+ backingstore_image=tc_image,
383
384
  coreos=coreos,
384
385
  vcpus=args.vcpus,
385
386
  ram=ram,
386
387
  disk_size=disk_size,
387
- disk_number=args.disk_number,
388
- nic_number=args.nic_number,
388
+ disk_count=args.disk_number,
389
+ nic_count=args.nic_number,
389
390
  tpm=args.tpm,
390
- serial=args.serial,
391
+ iommu=args.iommu,
392
+ use_disk_serial=args.serial,
391
393
  virtiofs_source=virtiofs_split[0],
392
394
  virtiofs_target=virtiofs_split[1]
393
395
  )
394
396
 
395
- tc_instance = instance.Instance(args.name,
397
+ tc_instance = instance.Instance(domain_configuration=domain,
396
398
  image=tc_image,
397
399
  connection=args.connection,
398
400
  desired_arch=args.arch,
399
- domain_configuration=domain
401
+ workarounds=Workarounds(defaults=True)
400
402
  )
401
403
  log.info("create %s instance %s" % ("coreos" if coreos else "cloud", args.name))
402
404
 
@@ -419,6 +421,10 @@ def _create_instance(args):
419
421
  else:
420
422
  tc_instance._remove_from_disk()
421
423
 
424
+ if args.dry:
425
+ tc_instance._remove_from_disk()
426
+ sys.exit(0)
427
+
422
428
  # create instance domain
423
429
  try:
424
430
  tc_instance.spawn_vm()
@@ -789,6 +795,9 @@ def get_argparser():
789
795
  instarg_create.add_argument("--keep",
790
796
  help="Don't remove instance from disk when something fails, useful for debugging",
791
797
  action="store_true")
798
+ instarg_create.add_argument("--dry",
799
+ help="Don't spawn an actual instance, useful for debugging",
800
+ action="store_true")
792
801
  instarg_create.add_argument("--ssh_path",
793
802
  help="specify your ssh pubkey path",
794
803
  type=str)
@@ -821,6 +830,9 @@ def get_argparser():
821
830
  instarg_create.add_argument("--virtiofs",
822
831
  type=str,
823
832
  help="specify a local directory to mount and mount target like <host path>:<guest path>")
833
+ instarg_create.add_argument("--iommu",
834
+ help="add iommu device",
835
+ action="store_true")
824
836
  imgarg = subparsers.add_parser("image", help="help on image options")
825
837
  imgarg_subp = imgarg.add_subparsers(title="subcommands",
826
838
  description="Types of commands available",
@@ -102,10 +102,6 @@ class ConfigData(object):
102
102
  DATA_DIR = "/var/lib/testcloud"
103
103
  STORE_DIR = "/var/lib/testcloud/backingstores"
104
104
 
105
- # libvirt domain XML Template
106
- # This lives either in the DEFAULT_CONF_DIR or DATA_DIR
107
- XML_TEMPLATE = "domain-template.jinja"
108
-
109
105
  # Data for cloud-init
110
106
 
111
107
  PASSWORD = 'passw0rd'
@@ -116,32 +112,17 @@ local-hostname: %s
116
112
  """
117
113
  USER_DATA = """#cloud-config
118
114
  ssh_pwauth: true
119
- password: %s
115
+ password: ${password}
120
116
  chpasswd:
121
117
  expire: false
122
118
  users:
123
119
  - default
124
120
  - name: cloud-user
125
- plain_text_passwd: %s
121
+ plain_text_passwd: ${password}
126
122
  sudo: ALL=(ALL) NOPASSWD:ALL
127
123
  lock_passwd: false
128
124
  runcmd:
129
- - sed -i -e '/^.*PermitRootLogin/s/^.*$/PermitRootLogin yes/'
130
- -e '/^.*UseDNS/s/^.*$/UseDNS no/'
131
- -e '/^.*GSSAPIAuthentication/s/^.*$/GSSAPIAuthentication no/'
132
- /etc/ssh/sshd_config
133
- - systemctl reload sshd
134
- - [sh, -c, 'if [ ! -f /etc/systemd/network/20-tc-usernet.network ] &&
135
- systemctl status systemd-networkd | grep -q "enabled;\\svendor\\spreset:\\senabled";
136
- then mkdir -p /etc/systemd/network/ &&
137
- echo "[Match]" >> /etc/systemd/network/20-tc-usernet.network &&
138
- echo "Name=en*" >> /etc/systemd/network/20-tc-usernet.network &&
139
- echo "[Network]" >> /etc/systemd/network/20-tc-usernet.network &&
140
- echo "DHCP=yes" >> /etc/systemd/network/20-tc-usernet.network; fi']
141
- - [sh, -c, 'if systemctl status systemd-networkd | grep -q "enabled;\\svendor\\spreset:\\senabled"; then
142
- systemctl restart systemd-networkd; fi']
143
- - [sh, -c, 'if cat /etc/os-release | grep -q platform:el8; then systemctl restart sshd; fi']
144
- - [sh, -c, 'dhclient || :']
125
+ ${runcommands}
145
126
  """
146
127
  COREOS_DATA = """variant: fcos
147
128
  version: 1.4.0
@@ -183,6 +164,12 @@ storage:
183
164
  # 0 = unlimited
184
165
  BACKINGSTORE_SIZE = 0
185
166
 
167
+ # Is reusing the local image url cache an option?
168
+ CACHE_IMAGES = True
169
+
170
+ # How old can a local image url cache be to take it as viable
171
+ TRUST_DEADLINE = 7 # Days
172
+
186
173
  # ram size, in MiB
187
174
  RAM = 768
188
175
  RAM_COREOS = 2048
@@ -234,44 +221,46 @@ storage:
234
221
  # Used to try to auto-fetch the latest qcow2
235
222
  CENTOS_STREAM_URL_PREFIX = "https://cloud.centos.org/centos/{0}-stream/{1}/images/"
236
223
  CENTOS_STREAM_VERSIONS = {
237
- "8": "https://cloud.centos.org/centos/8-stream/{0}/images/CentOS-Stream-GenericCloud-8-20220913.0.{0}.qcow2",
238
- "9": "https://cloud.centos.org/centos/9-stream/{0}/images/CentOS-Stream-GenericCloud-9-20230307.0.{0}.qcow2",
224
+ "8": "https://cloud.centos.org/centos/8-stream/{0}/images/CentOS-Stream-GenericCloud-8-20240603.0.{0}.qcow2",
225
+ "9": "https://cloud.centos.org/centos/9-stream/{0}/images/CentOS-Stream-GenericCloud-9-20240715.0.{0}.qcow2",
226
+ "10": "https://odcs.stream.centos.org/stream-10/production/CentOS-Stream-10-20240709.0/compose/BaseOS/{0}/images/CentOS-Stream-GenericCloud-10-20240709.0.{0}.qcow2",
239
227
  "latest": "9"
240
228
  }
241
229
 
242
230
  ROCKY_URL_PREFIX = "https://download.rockylinux.org/pub/rocky/{0}/images/{1}/"
243
231
  ROCKY_VERSIONS = {
244
- "8": "https://download.rockylinux.org/pub/rocky/8/images/{0}/Rocky-8-GenericCloud-Base-8.7-20230215.0.{0}.qcow2",
245
- "9": "https://download.rockylinux.org/pub/rocky/9/images/{0}/Rocky-9-GenericCloud-Base-9.1-20230215.0.{0}.qcow2",
232
+ "8": "https://download.rockylinux.org/pub/rocky/8/images/{0}/Rocky-8-GenericCloud-Base-8.10-20240528.0.{0}.qcow2",
233
+ "9": "https://download.rockylinux.org/pub/rocky/9/images/{0}/Rocky-9-GenericCloud-Base-9.4-20240609.1.{0}.qcow2",
246
234
  "latest": "9"
247
235
  }
248
236
 
249
- ALMA_URL_PREFIX = "https://ftp.linux.cz/pub/linux/almalinux/{0}/cloud/{1}/images/"
237
+ ALMA_URL_PREFIX = "https://repo.almalinux.org/almalinux/{0}/cloud/{1}/images/"
250
238
  ALMA_VERSIONS = {
251
- "8": "https://ftp.sh.cvut.cz/almalinux/8/cloud/{0}/images/AlmaLinux-8-GenericCloud-8.7-20221111.{0}.qcow2",
252
- "9": "https://ftp.sh.cvut.cz/almalinux/9/cloud/{0}/images/AlmaLinux-9-GenericCloud-9.1-20221118.{0}.qcow2",
239
+ "8": "https://repo.almalinux.org/almalinux/8/cloud/{0}/images/AlmaLinux-8-GenericCloud-8.10-20240530.{0}.qcow2",
240
+ "9": "https://repo.almalinux.org/almalinux/9/cloud/{0}/images/AlmaLinux-9-GenericCloud-9.4-20240507.{0}.qcow2",
253
241
  "latest": "9"
254
242
  }
255
243
 
244
+ # https://yum.oracle.com/oracle-linux-templates.html
256
245
  ORACLE_VERSIONS = {
257
- "7": "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-kvm-b145.qcow",
258
- "8": "https://yum.oracle.com/templates/OracleLinux/OL8/u7/x86_64/OL8U7_x86_64-kvm-b148.qcow",
259
- "9": "https://yum.oracle.com/templates/OracleLinux/OL9/u1/x86_64/OL9U1_x86_64-kvm-b158.qcow",
246
+ "7": "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-kvm-b218.qcow2",
247
+ "8": "https://yum.oracle.com/templates/OracleLinux/OL8/u10/x86_64/OL8U10_x86_64-kvm-b237.qcow2",
248
+ "9": "https://yum.oracle.com/templates/OracleLinux/OL9/u4/x86_64/OL9U4_x86_64-kvm-b234.qcow2",
260
249
  "latest": "9"
261
250
  }
262
251
 
263
252
  ORACLE_A64_VERSIONS = {
264
- "8": "https://yum.oracle.com/templates/OracleLinux/OL8/u7/aarch64/OL8U7_aarch64-kvm-cloud-b11.qcow",
265
- "9": "https://yum.oracle.com/templates/OracleLinux/OL9/u1/aarch64/OL9U1_aarch64-kvm-cloud-b14.qcow",
253
+ "8": "https://yum.oracle.com/templates/OracleLinux/OL8/u10/aarch64/OL8U10_aarch64-kvm-cloud-b100.qcow2",
254
+ "9": "https://yum.oracle.com/templates/OracleLinux/OL9/u4/aarch64/OL9U4_aarch64-kvm-cloud-b90.qcow2",
266
255
  "latest": "9"
267
256
  }
268
257
 
269
- DEBIAN_RELEASE_MAP = {"10": "buster","11": "bullseye"}
270
- DEBIAN_LATEST = "11"
258
+ DEBIAN_RELEASE_MAP = {"10": "buster","11": "bullseye", "12": "bookworm"}
259
+ DEBIAN_LATEST = "12"
271
260
  DEBIAN_IMG_URL = "https://cloud.debian.org/images/cloud/%s/daily/latest/debian-%s-genericcloud-%s-daily.qcow2"
272
261
 
273
262
  UBUNTU_RELEASES_API = "https://api.launchpad.net/devel/ubuntu/series"
274
- UBUNTU_IMG_URL = "https://cloud-images.ubuntu.com/%s/current/%s-server-cloudimg-%s-disk-kvm.img"
263
+ UBUNTU_IMG_URL = "https://cloud-images.ubuntu.com/%s/current/%s-server-cloudimg-%s.img"
275
264
 
276
265
  def merge_object(self, obj):
277
266
  '''Overwrites default values with values from a python object which have
@@ -24,8 +24,12 @@ def _get_centos_image_url(version:str, stream:bool, arch:str) -> str:
24
24
  if version == "latest":
25
25
  version = config_data.CENTOS_STREAM_VERSIONS["latest"]
26
26
 
27
+ # TODO: Until CentOS 10 starts syncing to the main mirror
28
+ if version == "10":
29
+ STREAM_URL_PREFIX = "https://odcs.stream.centos.org/stream-10/production/latest-CentOS-Stream/compose/BaseOS/{0}/images/".format(arch)
27
30
  # Try to dynamically get the latest at first
28
- STREAM_URL_PREFIX = config_data.CENTOS_STREAM_URL_PREFIX.format(version, arch)
31
+ else:
32
+ STREAM_URL_PREFIX = config_data.CENTOS_STREAM_URL_PREFIX.format(version, arch)
29
33
  IMG_NAME = r'CentOS-Stream-GenericCloud-{0}-[0-9.]+.{1}.qcow2'.format(version, arch)
30
34
  try:
31
35
  return parse_latest_qcow(IMG_NAME, STREAM_URL_PREFIX)
@@ -9,6 +9,7 @@ import requests
9
9
 
10
10
  from testcloud import config
11
11
  from testcloud import exceptions
12
+ from testcloud.distro_utils.misc import get_requests_session
12
13
 
13
14
  log = logging.getLogger('testcloud.util')
14
15
  config_data = config.get_config()
@@ -17,6 +18,8 @@ def _process_coreos_url(version:str, arch:str, platform:str) -> str:
17
18
  """
18
19
  Returns an CoreOS url in either qemu or openstack format
19
20
  """
21
+ session = get_requests_session()
22
+
20
23
  if version == "latest":
21
24
  version = "stable"
22
25
  if version not in config_data.STREAM_LIST:
@@ -26,7 +29,7 @@ def _process_coreos_url(version:str, arch:str, platform:str) -> str:
26
29
  log.error("Invalid platform ( %s ) requested for Fedora CoreOS." % platform)
27
30
  raise exceptions.TestcloudImageError
28
31
  try:
29
- result = requests.get("https://builds.coreos.fedoraproject.org/streams/%s.json"%version).json()
32
+ result = session.get("https://builds.coreos.fedoraproject.org/streams/%s.json"%version).json()
30
33
  except (ConnectionError, IndexError, requests.exceptions.JSONDecodeError):
31
34
  log.error("Failed to fetch the image.")
32
35
  raise exceptions.TestcloudImageError
@@ -57,10 +60,12 @@ def get_fedora_image_url(version:str, arch:str) -> str:
57
60
  primary_arches = ["x86_64", "aarch64"]
58
61
  url = ""
59
62
 
63
+ session = get_requests_session()
64
+
60
65
  # get coreos url
61
66
  if version in config_data.STREAM_LIST:
62
67
  try:
63
- result = requests.get("https://builds.coreos.fedoraproject.org/streams/%s.json"%version).json()
68
+ result = session.get("https://builds.coreos.fedoraproject.org/streams/%s.json"%version).json()
64
69
  except (ConnectionError, IndexError, requests.exceptions.JSONDecodeError):
65
70
  log.error("Failed to fetch the image.")
66
71
  raise exceptions.TestcloudImageError
@@ -69,7 +74,7 @@ def get_fedora_image_url(version:str, arch:str) -> str:
69
74
 
70
75
  #get Fedora Cloud url
71
76
  try:
72
- oraculum_releases = requests.get('https://packager-dashboard.fedoraproject.org/api/v1/releases').json()
77
+ oraculum_releases = session.get('https://packager-dashboard.fedoraproject.org/api/v1/releases').json()
73
78
  except (ConnectionError, IndexError, requests.exceptions.JSONDecodeError):
74
79
  log.error("Couldn't fetch Fedora releases from oraculum...")
75
80
  raise exceptions.TestcloudImageError
@@ -89,6 +94,7 @@ def get_fedora_image_url(version:str, arch:str) -> str:
89
94
  log.error("non-x86_64 architecture is not supported with Fedora qa-matrix.")
90
95
  raise exceptions.TestcloudImageError
91
96
  try:
97
+ # Never cache this one
92
98
  nominated_response = requests.get("https://fedoraproject.org/wiki/Test_Results:Current_Installation_Test")
93
99
  return str(re.findall(r'href=\"(.*.%s.qcow2)\"' % arch, nominated_response.text)[0])
94
100
  except (ConnectionError, IndexError):
@@ -98,7 +104,7 @@ def get_fedora_image_url(version:str, arch:str) -> str:
98
104
  if version == "rawhide" or version == "branched":
99
105
  stamp = 0
100
106
  try:
101
- releases = requests.get('https://openqa.fedoraproject.org/nightlies.json').json()
107
+ releases = session.get('https://openqa.fedoraproject.org/nightlies.json').json()
102
108
  except (ConnectionError, IndexError, requests.exceptions.JSONDecodeError):
103
109
  log.error("Failed to fetch the image.")
104
110
  raise exceptions.TestcloudImageError
@@ -116,14 +122,14 @@ def get_fedora_image_url(version:str, arch:str) -> str:
116
122
  version = str(oraculum_releases["fedora"]["stable"])
117
123
 
118
124
  try:
119
- releases = requests.get('https://getfedora.org/releases.json').json()
125
+ releases = session.get('https://getfedora.org/releases.json').json()
120
126
  except (ConnectionError, requests.exceptions.JSONDecodeError):
121
127
  log.error("Couldn't fetch Fedora releases list...")
122
128
  raise exceptions.TestcloudImageError
123
129
 
124
130
  url = ""
125
131
  for release in releases:
126
- if release["version"] == version and release["variant"] == "Cloud" and release["link"].endswith(".qcow2"):
132
+ if release["version"] == version and release["subvariant"] == "Cloud_Base" and release["link"].endswith(".qcow2"):
127
133
  # There are links only to primary architecutres in releases.json... much fun
128
134
  if arch in primary_arches and release["arch"] == arch:
129
135
  url = release["link"]
@@ -0,0 +1,43 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Copyright 2023, Red Hat, Inc.
3
+ # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+>
4
+ # See the LICENSE file for more details on Licensing
5
+
6
+ import logging
7
+ import re
8
+ import requests
9
+
10
+ from testcloud import exceptions, config
11
+ from packaging.version import Version
12
+
13
+ config_data = config.get_config()
14
+ log = logging.getLogger('testcloud.util')
15
+
16
+ def parse_latest_qcow(rule:str, url:str) -> str:
17
+ session = get_requests_session()
18
+
19
+ try:
20
+ resp = session.get(url)
21
+ resp.raise_for_status()
22
+ latest_img_name = sorted(re.findall(rule, resp.text))[-1] or exceptions.TestcloudImageError
23
+ return url + str(latest_img_name)
24
+ except:
25
+ raise exceptions.TestcloudImageError
26
+
27
+
28
+ def get_requests_session():
29
+ try:
30
+ assert config_data.CACHE_IMAGES
31
+ import requests_cache
32
+ assert Version(requests_cache.__version__) >= Version("1.2")
33
+
34
+ log.debug("Using local image url cache...")
35
+ return requests_cache.CachedSession(
36
+ cache_name="{}/testcloud_image_resolve_cache".format(config_data.DATA_DIR),
37
+ backend="sqlite",
38
+ stale_if_error=True,
39
+ expire_after=config_data.TRUST_DEADLINE * 60 * 60 * 24
40
+ )
41
+ except (ImportError, AssertionError):
42
+ log.debug("Not using local image url cache due to config or unmet dependencies...")
43
+ return requests.Session()
@@ -8,13 +8,16 @@ import requests
8
8
 
9
9
  from testcloud import config
10
10
  from testcloud import exceptions
11
+ from testcloud.distro_utils.misc import get_requests_session
11
12
 
12
13
  log = logging.getLogger('testcloud.util')
13
14
  config_data = config.get_config()
14
15
 
15
16
  def get_ubuntu_releases() -> dict:
17
+ session = get_requests_session()
18
+
16
19
  try:
17
- releases_resp = requests.get(config_data.UBUNTU_RELEASES_API).json()
20
+ releases_resp = session.get(config_data.UBUNTU_RELEASES_API).json()
18
21
  except (ConnectionError, IndexError, requests.exceptions.JSONDecodeError):
19
22
  log.error("Failed to fetch Ubuntu releases list.")
20
23
  raise exceptions.TestcloudImageError
@@ -32,8 +35,6 @@ def get_ubuntu_image_url(version:str, arch:str) -> str:
32
35
  log.error("Requested architecture is not supported by testcloud for Ubuntu.")
33
36
  raise exceptions.TestcloudImageError
34
37
 
35
- if arch != "x86_64":
36
- config_data.UBUNTU_IMG_URL = config_data.UBUNTU_IMG_URL.replace("-disk-kvm.img", ".img")
37
38
  releases = get_ubuntu_releases()
38
39
  if len(releases) == 0:
39
40
  raise exceptions.TestcloudImageError