teuthology 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. scripts/describe.py +1 -0
  2. scripts/dispatcher.py +55 -26
  3. scripts/exporter.py +18 -0
  4. scripts/lock.py +1 -1
  5. scripts/node_cleanup.py +58 -0
  6. scripts/openstack.py +9 -9
  7. scripts/results.py +12 -11
  8. scripts/schedule.py +4 -0
  9. scripts/suite.py +57 -16
  10. scripts/supervisor.py +44 -0
  11. scripts/update_inventory.py +10 -4
  12. teuthology/__init__.py +24 -26
  13. teuthology/beanstalk.py +4 -3
  14. teuthology/config.py +16 -6
  15. teuthology/contextutil.py +18 -14
  16. teuthology/describe_tests.py +25 -18
  17. teuthology/dispatcher/__init__.py +210 -35
  18. teuthology/dispatcher/supervisor.py +140 -58
  19. teuthology/exceptions.py +43 -0
  20. teuthology/exporter.py +347 -0
  21. teuthology/kill.py +76 -81
  22. teuthology/lock/cli.py +3 -3
  23. teuthology/lock/ops.py +135 -61
  24. teuthology/lock/query.py +61 -44
  25. teuthology/ls.py +1 -1
  26. teuthology/misc.py +61 -75
  27. teuthology/nuke/__init__.py +12 -353
  28. teuthology/openstack/__init__.py +4 -3
  29. teuthology/openstack/openstack-centos-7.0-user-data.txt +1 -1
  30. teuthology/openstack/openstack-centos-7.1-user-data.txt +1 -1
  31. teuthology/openstack/openstack-centos-7.2-user-data.txt +1 -1
  32. teuthology/openstack/openstack-debian-8.0-user-data.txt +1 -1
  33. teuthology/openstack/openstack-opensuse-42.1-user-data.txt +1 -1
  34. teuthology/openstack/openstack-teuthology.cron +0 -1
  35. teuthology/orchestra/cluster.py +49 -7
  36. teuthology/orchestra/connection.py +17 -4
  37. teuthology/orchestra/console.py +111 -50
  38. teuthology/orchestra/daemon/cephadmunit.py +15 -2
  39. teuthology/orchestra/daemon/state.py +8 -1
  40. teuthology/orchestra/daemon/systemd.py +4 -4
  41. teuthology/orchestra/opsys.py +30 -11
  42. teuthology/orchestra/remote.py +405 -338
  43. teuthology/orchestra/run.py +3 -3
  44. teuthology/packaging.py +19 -16
  45. teuthology/provision/__init__.py +30 -10
  46. teuthology/provision/cloud/openstack.py +12 -6
  47. teuthology/provision/cloud/util.py +1 -2
  48. teuthology/provision/downburst.py +4 -3
  49. teuthology/provision/fog.py +68 -20
  50. teuthology/provision/openstack.py +5 -4
  51. teuthology/provision/pelagos.py +1 -1
  52. teuthology/repo_utils.py +43 -13
  53. teuthology/report.py +57 -35
  54. teuthology/results.py +5 -3
  55. teuthology/run.py +13 -14
  56. teuthology/run_tasks.py +27 -43
  57. teuthology/schedule.py +4 -3
  58. teuthology/scrape.py +28 -22
  59. teuthology/suite/__init__.py +74 -45
  60. teuthology/suite/build_matrix.py +34 -24
  61. teuthology/suite/fragment-merge.lua +105 -0
  62. teuthology/suite/matrix.py +31 -2
  63. teuthology/suite/merge.py +175 -0
  64. teuthology/suite/placeholder.py +6 -9
  65. teuthology/suite/run.py +175 -100
  66. teuthology/suite/util.py +64 -218
  67. teuthology/task/__init__.py +1 -1
  68. teuthology/task/ansible.py +101 -32
  69. teuthology/task/buildpackages.py +2 -2
  70. teuthology/task/ceph_ansible.py +13 -6
  71. teuthology/task/cephmetrics.py +2 -1
  72. teuthology/task/clock.py +33 -14
  73. teuthology/task/exec.py +18 -0
  74. teuthology/task/hadoop.py +2 -2
  75. teuthology/task/install/__init__.py +29 -7
  76. teuthology/task/install/bin/adjust-ulimits +16 -0
  77. teuthology/task/install/bin/daemon-helper +114 -0
  78. teuthology/task/install/bin/stdin-killer +263 -0
  79. teuthology/task/install/deb.py +1 -1
  80. teuthology/task/install/rpm.py +17 -5
  81. teuthology/task/install/util.py +3 -3
  82. teuthology/task/internal/__init__.py +41 -10
  83. teuthology/task/internal/edit_sudoers.sh +10 -0
  84. teuthology/task/internal/lock_machines.py +2 -9
  85. teuthology/task/internal/redhat.py +31 -1
  86. teuthology/task/internal/syslog.py +31 -8
  87. teuthology/task/kernel.py +152 -145
  88. teuthology/task/lockfile.py +1 -1
  89. teuthology/task/mpi.py +10 -10
  90. teuthology/task/pcp.py +1 -1
  91. teuthology/task/selinux.py +16 -8
  92. teuthology/task/ssh_keys.py +4 -4
  93. teuthology/timer.py +3 -3
  94. teuthology/util/loggerfile.py +19 -0
  95. teuthology/util/scanner.py +159 -0
  96. teuthology/util/sentry.py +52 -0
  97. teuthology/util/time.py +52 -0
  98. teuthology-1.2.1.data/scripts/adjust-ulimits +16 -0
  99. teuthology-1.2.1.data/scripts/daemon-helper +114 -0
  100. teuthology-1.2.1.data/scripts/stdin-killer +263 -0
  101. teuthology-1.2.1.dist-info/METADATA +88 -0
  102. teuthology-1.2.1.dist-info/RECORD +168 -0
  103. {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/WHEEL +1 -1
  104. {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/entry_points.txt +3 -2
  105. scripts/nuke.py +0 -47
  106. scripts/worker.py +0 -37
  107. teuthology/lock/test/__init__.py +0 -0
  108. teuthology/lock/test/test_lock.py +0 -7
  109. teuthology/nuke/actions.py +0 -456
  110. teuthology/openstack/test/__init__.py +0 -0
  111. teuthology/openstack/test/openstack-integration.py +0 -286
  112. teuthology/openstack/test/test_config.py +0 -35
  113. teuthology/openstack/test/test_openstack.py +0 -1695
  114. teuthology/orchestra/test/__init__.py +0 -0
  115. teuthology/orchestra/test/integration/__init__.py +0 -0
  116. teuthology/orchestra/test/integration/test_integration.py +0 -94
  117. teuthology/orchestra/test/test_cluster.py +0 -240
  118. teuthology/orchestra/test/test_connection.py +0 -106
  119. teuthology/orchestra/test/test_console.py +0 -217
  120. teuthology/orchestra/test/test_opsys.py +0 -404
  121. teuthology/orchestra/test/test_remote.py +0 -185
  122. teuthology/orchestra/test/test_run.py +0 -286
  123. teuthology/orchestra/test/test_systemd.py +0 -54
  124. teuthology/orchestra/test/util.py +0 -12
  125. teuthology/task/tests/__init__.py +0 -110
  126. teuthology/task/tests/test_locking.py +0 -25
  127. teuthology/task/tests/test_run.py +0 -40
  128. teuthology/test/__init__.py +0 -0
  129. teuthology/test/fake_archive.py +0 -107
  130. teuthology/test/fake_fs.py +0 -92
  131. teuthology/test/integration/__init__.py +0 -0
  132. teuthology/test/integration/test_suite.py +0 -86
  133. teuthology/test/task/__init__.py +0 -205
  134. teuthology/test/task/test_ansible.py +0 -624
  135. teuthology/test/task/test_ceph_ansible.py +0 -176
  136. teuthology/test/task/test_console_log.py +0 -88
  137. teuthology/test/task/test_install.py +0 -337
  138. teuthology/test/task/test_internal.py +0 -57
  139. teuthology/test/task/test_kernel.py +0 -243
  140. teuthology/test/task/test_pcp.py +0 -379
  141. teuthology/test/task/test_selinux.py +0 -35
  142. teuthology/test/test_config.py +0 -189
  143. teuthology/test/test_contextutil.py +0 -68
  144. teuthology/test/test_describe_tests.py +0 -316
  145. teuthology/test/test_email_sleep_before_teardown.py +0 -81
  146. teuthology/test/test_exit.py +0 -97
  147. teuthology/test/test_get_distro.py +0 -47
  148. teuthology/test/test_get_distro_version.py +0 -47
  149. teuthology/test/test_get_multi_machine_types.py +0 -27
  150. teuthology/test/test_job_status.py +0 -60
  151. teuthology/test/test_ls.py +0 -48
  152. teuthology/test/test_misc.py +0 -391
  153. teuthology/test/test_nuke.py +0 -290
  154. teuthology/test/test_packaging.py +0 -763
  155. teuthology/test/test_parallel.py +0 -28
  156. teuthology/test/test_repo_utils.py +0 -225
  157. teuthology/test/test_report.py +0 -77
  158. teuthology/test/test_results.py +0 -155
  159. teuthology/test/test_run.py +0 -239
  160. teuthology/test/test_safepath.py +0 -55
  161. teuthology/test/test_schedule.py +0 -45
  162. teuthology/test/test_scrape.py +0 -167
  163. teuthology/test/test_timer.py +0 -80
  164. teuthology/test/test_vps_os_vers_parameter_checking.py +0 -84
  165. teuthology/test/test_worker.py +0 -303
  166. teuthology/worker.py +0 -354
  167. teuthology-1.1.0.dist-info/METADATA +0 -76
  168. teuthology-1.1.0.dist-info/RECORD +0 -213
  169. {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/LICENSE +0 -0
  170. {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/top_level.txt +0 -0
@@ -9,7 +9,7 @@ from paramiko import ChannelFile
9
9
  import gevent
10
10
  import gevent.event
11
11
  import socket
12
- import pipes
12
+ import shlex
13
13
  import logging
14
14
  import shutil
15
15
 
@@ -252,7 +252,7 @@ def quote(args):
252
252
  if isinstance(a, Raw):
253
253
  yield a.value
254
254
  else:
255
- yield pipes.quote(a)
255
+ yield shlex.quote(a)
256
256
  if isinstance(args, list):
257
257
  return ' '.join(_quote(args))
258
258
  else:
@@ -400,7 +400,7 @@ def run(
400
400
  """
401
401
  Run a command remotely. If any of 'args' contains shell metacharacters
402
402
  that you want to pass unquoted, pass it as an instance of Raw(); otherwise
403
- it will be quoted with pipes.quote() (single quote, and single quotes
403
+ it will be quoted with shlex.quote() (single quote, and single quotes
404
404
  enclosed in double quotes).
405
405
 
406
406
  :param client: SSHConnection to run the command with
teuthology/packaging.py CHANGED
@@ -395,12 +395,12 @@ def _get_config_value_for_remote(ctx, remote, config, key):
395
395
 
396
396
  config = {
397
397
  'all':
398
- {'branch': 'master'},
398
+ {'branch': 'main'},
399
399
  'branch': 'next'
400
400
  }
401
401
  _get_config_value_for_remote(ctx, remote, config, 'branch')
402
402
 
403
- would return 'master'.
403
+ would return 'main'.
404
404
 
405
405
  :param ctx: the argparse.Namespace object
406
406
  :param remote: the teuthology.orchestra.remote.Remote object
@@ -479,7 +479,7 @@ class GitbuilderProject(object):
479
479
  )
480
480
  # when we're initializing with a remote we most likely have
481
481
  # a task config, not the entire teuthology job config
482
- self.flavor = self.job_config.get("flavor", "basic")
482
+ self.flavor = self.job_config.get("flavor", "default")
483
483
  self.tag = self.job_config.get("tag")
484
484
 
485
485
  def _init_from_config(self):
@@ -549,10 +549,6 @@ class GitbuilderProject(object):
549
549
  """
550
550
  The base url that points at this project on gitbuilder.
551
551
 
552
- For example::
553
-
554
- http://gitbuilder.ceph.com/ceph-deb-raring-x86_64-basic/ref/master
555
-
556
552
  :returns: A string of the base url for this project
557
553
  """
558
554
  return self._get_base_url()
@@ -656,9 +652,9 @@ class GitbuilderProject(object):
656
652
  remote, the sha1 from the config will be used.
657
653
 
658
654
  If a tag, branch or sha1 can't be found it will default to use the
659
- build from the master branch.
655
+ build from the main branch.
660
656
 
661
- :returns: A string URI. Ex: ref/master
657
+ :returns: A string URI. Ex: ref/main
662
658
  """
663
659
  ref_name, ref_val = next(iter(self._choose_reference().items()))
664
660
  if ref_name == 'sha1':
@@ -673,7 +669,7 @@ class GitbuilderProject(object):
673
669
  Decide which to use.
674
670
 
675
671
  :returns: a single-key dict containing the name and value of the
676
- reference to use, e.g. {'branch': 'master'}
672
+ reference to use, e.g. {'branch': 'main'}
677
673
  """
678
674
  tag = branch = sha1 = None
679
675
  if self.remote:
@@ -716,8 +712,8 @@ class GitbuilderProject(object):
716
712
  warn('sha1')
717
713
  return dict(sha1=sha1)
718
714
  else:
719
- log.warning("defaulting to master branch")
720
- return dict(branch='master')
715
+ log.warning("defaulting to main branch")
716
+ return dict(branch='main')
721
717
 
722
718
  def _get_base_url(self):
723
719
  """
@@ -879,8 +875,6 @@ class ShamanProject(GitbuilderProject):
879
875
  @property
880
876
  def _search_uri(self):
881
877
  flavor = self.flavor
882
- if flavor == 'basic':
883
- flavor = 'default'
884
878
  req_obj = OrderedDict()
885
879
  req_obj['status'] = 'ready'
886
880
  req_obj['project'] = self.project
@@ -983,6 +977,9 @@ class ShamanProject(GitbuilderProject):
983
977
  except VersionNotFoundError:
984
978
  return False
985
979
 
980
+ # self._result has status, project, flavor, distros, arch, and sha1
981
+ # restrictions, so the only reason for multiples should be "multiple
982
+ # builds of the same sha1 etc."; the first entry is the newest
986
983
  search_result = self._result.json()[0]
987
984
 
988
985
  # now look for the build complete status
@@ -996,14 +993,20 @@ class ShamanProject(GitbuilderProject):
996
993
  resp.raise_for_status()
997
994
  except requests.HttpError:
998
995
  return False
996
+ log.debug(f'looking for {self.distro} {self.arch} {self.flavor}')
999
997
  for build in resp.json():
998
+ log.debug(f'build: {build["distro"]}/{build["distro_version"]} {build["distro_arch"]} {build["flavor"]}')
1000
999
  if (
1000
+ # we must compare build arch to self.arch, since shaman's
1001
+ # results can have multiple arches but we're searching
1002
+ # for precisely one here
1001
1003
  build['distro'] == search_result['distro'] and
1002
1004
  build['distro_version'] == search_result['distro_version'] and
1003
1005
  build['flavor'] == search_result['flavor'] and
1004
- build['distro_arch'] in search_result['archs']
1006
+ build['distro_arch'] == self.arch and
1007
+ build['status'] == 'completed'
1005
1008
  ):
1006
- return build['status'] == 'completed'
1009
+ return True
1007
1010
  return False
1008
1011
 
1009
1012
  def _get_repo(self):
@@ -1,5 +1,7 @@
1
1
  import logging
2
+ import os
2
3
 
4
+ import teuthology.exporter
3
5
  import teuthology.lock.query
4
6
  from teuthology.misc import decanonicalize_hostname, get_distro, get_distro_version
5
7
 
@@ -8,19 +10,19 @@ from teuthology.provision import downburst
8
10
  from teuthology.provision import fog
9
11
  from teuthology.provision import openstack
10
12
  from teuthology.provision import pelagos
11
- import os
12
13
 
13
14
  log = logging.getLogger(__name__)
14
15
 
15
16
 
16
- def _logfile(ctx, shortname):
17
- if hasattr(ctx, 'config') and ctx.config.get('archive_path'):
18
- return os.path.join(ctx.config['archive_path'],
19
- shortname + '.downburst.log')
17
+ def _logfile(shortname: str, archive_path: str = ""):
18
+ if os.path.isfile(archive_path):
19
+ return f"{archive_path}/{shortname}.downburst.log"
20
+
20
21
 
21
22
  def get_reimage_types():
22
23
  return pelagos.get_types() + fog.get_types()
23
24
 
25
+
24
26
  def reimage(ctx, machine_name, machine_type):
25
27
  os_type = get_distro(ctx)
26
28
  os_version = get_distro_version(ctx)
@@ -36,7 +38,21 @@ def reimage(ctx, machine_name, machine_type):
36
38
  else:
37
39
  raise Exception("The machine_type '%s' is not known to any "
38
40
  "of configured provisioners" % machine_type)
39
- return obj.create()
41
+ status = "fail"
42
+ try:
43
+ result = obj.create()
44
+ status = "success"
45
+ except Exception:
46
+ # We only need this clause so that we avoid triggering the finally
47
+ # clause below in cases where the exception raised is KeyboardInterrupt
48
+ # or SystemExit
49
+ raise
50
+ finally:
51
+ teuthology.exporter.NodeReimagingResults().record(
52
+ machine_type=machine_type,
53
+ status=status,
54
+ )
55
+ return result
40
56
 
41
57
 
42
58
  def create_if_vm(ctx, machine_name, _downburst=None):
@@ -78,8 +94,12 @@ def create_if_vm(ctx, machine_name, _downburst=None):
78
94
  return dbrst.create()
79
95
 
80
96
 
81
- def destroy_if_vm(ctx, machine_name, user=None, description=None,
82
- _downburst=None):
97
+ def destroy_if_vm(
98
+ machine_name: str,
99
+ user: str = "",
100
+ description: str = "",
101
+ _downburst=None
102
+ ):
83
103
  """
84
104
  Use downburst to destroy a virtual machine
85
105
 
@@ -99,7 +119,7 @@ def destroy_if_vm(ctx, machine_name, user=None, description=None,
99
119
  log.error(msg.format(node=machine_name, as_user=user,
100
120
  locked_by=status_info['locked_by']))
101
121
  return False
102
- if (description is not None and description !=
122
+ if (description and description !=
103
123
  status_info['description']):
104
124
  msg = "Tried to destroy {node} with description {desc_arg} " + \
105
125
  "but it is locked with description {desc_lock}"
@@ -117,5 +137,5 @@ def destroy_if_vm(ctx, machine_name, user=None, description=None,
117
137
  dbrst = _downburst or \
118
138
  downburst.Downburst(name=machine_name, os_type=None,
119
139
  os_version=None, status=status_info,
120
- logfile=_logfile(ctx, shortname))
140
+ logfile=_logfile(description, shortname))
121
141
  return dbrst.destroy()
@@ -84,7 +84,13 @@ class OpenStackProvider(Provider):
84
84
  @property
85
85
  def images(self):
86
86
  if not hasattr(self, '_images'):
87
- self._images = retry(self.driver.list_images)
87
+ exclude_image = self.conf.get('exclude_image', [])
88
+ if exclude_image and not isinstance(exclude_image, list):
89
+ exclude_image = [exclude_image]
90
+ exclude_re = [re.compile(x) for x in exclude_image]
91
+ images = retry(self.driver.list_images)
92
+ self._images = [_ for _ in images
93
+ if not any(x.match(_.name) for x in exclude_re)]
88
94
  return self._images
89
95
 
90
96
  @property
@@ -126,7 +132,7 @@ class OpenStackProvider(Provider):
126
132
  else:
127
133
  self._networks = list()
128
134
  except AttributeError:
129
- log.warn("Unable to list networks for %s", self.driver)
135
+ log.warning("Unable to list networks for %s", self.driver)
130
136
  self._networks = list()
131
137
  return self._networks
132
138
 
@@ -144,7 +150,7 @@ class OpenStackProvider(Provider):
144
150
  self.driver.ex_list_security_groups
145
151
  )
146
152
  except AttributeError:
147
- log.warn("Unable to list security groups for %s", self.driver)
153
+ log.warning("Unable to list security groups for %s", self.driver)
148
154
  self._security_groups = list()
149
155
  return self._security_groups
150
156
 
@@ -420,7 +426,7 @@ class OpenStackProvisioner(base.Provisioner):
420
426
  msg = "Unknown error locating %s"
421
427
  if not matches:
422
428
  msg = "No nodes found with name '%s'" % self.name
423
- log.warn(msg)
429
+ log.warning(msg)
424
430
  return
425
431
  elif len(matches) > 1:
426
432
  msg = "More than one node found with name '%s'"
@@ -438,9 +444,9 @@ class OpenStackProvisioner(base.Provisioner):
438
444
  self._destroy_volumes()
439
445
  nodes = self._find_nodes()
440
446
  if not nodes:
441
- log.warn("Didn't find any nodes named '%s' to destroy!", self.name)
447
+ log.warning("Didn't find any nodes named '%s' to destroy!", self.name)
442
448
  return True
443
449
  if len(nodes) > 1:
444
- log.warn("Found multiple nodes named '%s' to destroy!", self.name)
450
+ log.warning("Found multiple nodes named '%s' to destroy!", self.name)
445
451
  log.info("Destroying nodes: %s", nodes)
446
452
  return all([node.destroy() for node in nodes])
@@ -1,5 +1,4 @@
1
1
  import datetime
2
- import dateutil.tz
3
2
  import dateutil.parser
4
3
  import json
5
4
  import os
@@ -103,7 +102,7 @@ class AuthToken(object):
103
102
  def expired(self):
104
103
  if self.expires is None:
105
104
  return True
106
- utcnow = datetime.datetime.now(dateutil.tz.tzutc())
105
+ utcnow = datetime.datetime.now(datetime.timezone.utc)
107
106
  offset = datetime.timedelta(minutes=30)
108
107
  return self.expires < (utcnow + offset)
109
108
 
@@ -44,6 +44,7 @@ def downburst_executable():
44
44
 
45
45
  def downburst_environment():
46
46
  env = dict()
47
+ env['PATH'] = os.environ.get('PATH')
47
48
  discover_url = os.environ.get('DOWNBURST_DISCOVER_URL')
48
49
  if config.downburst and not discover_url:
49
50
  if isinstance(config.downburst, dict):
@@ -165,7 +166,7 @@ class Downburst(object):
165
166
  if proc.returncode != 0:
166
167
  not_found_msg = "no domain with matching name '%s'" % self.shortname
167
168
  if not_found_msg in err:
168
- log.warn("Ignoring error during destroy: %s", err)
169
+ log.warning("Ignoring error during destroy: %s", err)
169
170
  return True
170
171
  log.error("Error destroying %s: %s", self.name, err)
171
172
  return False
@@ -215,7 +216,7 @@ class Downburst(object):
215
216
  'additional-disks-size': machine['volumes']['size'],
216
217
  'arch': 'x86_64',
217
218
  }
218
- fqdn = self.name.split('@')[1]
219
+ fqdn = self.name.split('@')[-1]
219
220
  file_out = {
220
221
  'downburst': file_info,
221
222
  'local-hostname': fqdn,
@@ -307,7 +308,7 @@ def get_distro_from_downburst():
307
308
  executable_cmd = downburst_executable()
308
309
  environment_dict = downburst_environment()
309
310
  if not executable_cmd:
310
- log.warn("Downburst not found!")
311
+ log.warning("Downburst not found!")
311
312
  log.info('Using default values for supported os_type/os_version')
312
313
  return default_table
313
314
  try:
@@ -1,10 +1,10 @@
1
+ import datetime
1
2
  import json
2
3
  import logging
3
4
  import requests
4
5
  import socket
5
6
  import re
6
7
 
7
- from datetime import datetime
8
8
  from paramiko import SSHException
9
9
  from paramiko.ssh_exception import NoValidConnectionsError
10
10
 
@@ -13,6 +13,7 @@ import teuthology.orchestra
13
13
  from teuthology.config import config
14
14
  from teuthology.contextutil import safe_while
15
15
  from teuthology.exceptions import MaxWhileTries
16
+ from teuthology.orchestra.opsys import OS
16
17
  from teuthology import misc
17
18
 
18
19
  log = logging.getLogger(__name__)
@@ -29,7 +30,7 @@ def enabled(warn=False):
29
30
  params = ['endpoint', 'api_token', 'user_token', 'machine_types']
30
31
  unset = [param for param in params if not fog_conf.get(param)]
31
32
  if unset and warn:
32
- log.warn(
33
+ log.warning(
33
34
  "FOG disabled; set the following config options to enable: %s",
34
35
  ' '.join(unset),
35
36
  )
@@ -89,6 +90,7 @@ class FOG(object):
89
90
  raise
90
91
  self._wait_for_ready()
91
92
  self._fix_hostname()
93
+ self._verify_installed_os()
92
94
  self.log.info("Deploy complete!")
93
95
 
94
96
  def do_request(self, url_suffix, data=None, method='GET', verify=True):
@@ -147,18 +149,39 @@ class FOG(object):
147
149
  represents it
148
150
  :returns: A dict describing the image
149
151
  """
150
- name = '_'.join([
151
- self.remote.machine_type, self.os_type.lower(), self.os_version])
152
- resp = self.do_request(
153
- '/image',
154
- data=json.dumps(dict(name=name)),
155
- )
156
- obj = resp.json()
157
- if not obj['count']:
152
+ def do_get(name):
153
+ resp = self.do_request(
154
+ '/image',
155
+ data=json.dumps(dict(name=name)),
156
+ )
157
+ obj = resp.json()
158
+ if obj['count']:
159
+ return obj['images'][0]
160
+
161
+ os_type = self.os_type.lower()
162
+ os_version = self.os_version
163
+ name = f"{self.remote.machine_type}_{os_type}_{os_version}"
164
+ if image := do_get(name):
165
+ return image
166
+ elif os_type == 'centos' and not os_version.endswith('.stream'):
167
+ image = do_get(f"{name}.stream")
168
+ if image:
169
+ return image
170
+ else:
158
171
  raise RuntimeError(
159
- "Could not find an image for %s %s" %
160
- (self.os_type, self.os_version))
161
- return obj['images'][0]
172
+ "Fog has no %s image. Available %s images: %s" %
173
+ (name, self.remote.machine_type, self.suggest_image_names()))
174
+
175
+ def suggest_image_names(self):
176
+ """
177
+ Suggest available image names for this machine type.
178
+
179
+ :returns: A list of image names.
180
+ """
181
+ resp = self.do_request('/image/search/%s' % self.remote.machine_type)
182
+ obj = resp.json()
183
+ images = obj['images']
184
+ return [image['name'] for image in images]
162
185
 
163
186
  def set_image(self, host_id):
164
187
  """
@@ -167,6 +190,8 @@ class FOG(object):
167
190
  """
168
191
  image_data = self.get_image_data()
169
192
  image_id = int(image_data['id'])
193
+ image_name = image_data.get("name")
194
+ self.log.debug(f"Requesting image {image_name} (ID {image_id})")
170
195
  self.do_request(
171
196
  '/host/%s' % host_id,
172
197
  method='PUT',
@@ -202,8 +227,8 @@ class FOG(object):
202
227
  for task in host_tasks:
203
228
  timestamp = task['createdTime']
204
229
  time_delta = (
205
- datetime.utcnow() - datetime.strptime(
206
- timestamp, self.timestamp_format)
230
+ datetime.datetime.now(datetime.timezone.utc) - datetime.datetime.strptime(
231
+ timestamp, self.timestamp_format).replace(tzinfo=datetime.timezone.utc)
207
232
  ).total_seconds()
208
233
  # There should only be one deploy task matching our host. Just in
209
234
  # case there are multiple, select a very recent one.
@@ -240,13 +265,14 @@ class FOG(object):
240
265
  completed)
241
266
  """
242
267
  self.log.info("Waiting for deploy to finish")
243
- with safe_while(sleep=15, tries=60) as proceed:
268
+ with safe_while(sleep=15, tries=120, timeout=config.fog_reimage_timeout) as proceed:
244
269
  while proceed():
245
270
  if not self.deploy_task_active(task_id):
246
271
  break
247
272
 
248
273
  def cancel_deploy_task(self, task_id):
249
274
  """ Cancel an active deploy task """
275
+ self.log.debug(f"Canceling deploy task with ID {task_id}")
250
276
  resp = self.do_request(
251
277
  '/task/cancel',
252
278
  method='DELETE',
@@ -256,7 +282,7 @@ class FOG(object):
256
282
 
257
283
  def _wait_for_ready(self):
258
284
  """ Attempt to connect to the machine via SSH """
259
- with safe_while(sleep=6, tries=100) as proceed:
285
+ with safe_while(sleep=6, timeout=config.fog_wait_for_ssh_timeout) as proceed:
260
286
  while proceed():
261
287
  try:
262
288
  self.remote.connect()
@@ -267,12 +293,26 @@ class FOG(object):
267
293
  NoValidConnectionsError,
268
294
  MaxWhileTries,
269
295
  EOFError,
270
- ):
271
- pass
296
+ ) as e:
297
+ # log this, because otherwise lots of failures just
298
+ # keep retrying without any notification (like, say,
299
+ # a mismatched host key in ~/.ssh/known_hosts, or
300
+ # something)
301
+ self.log.warning(e)
272
302
  sentinel_file = config.fog.get('sentinel_file', None)
273
303
  if sentinel_file:
274
304
  cmd = "while [ ! -e '%s' ]; do sleep 5; done" % sentinel_file
275
- self.remote.run(args=cmd, timeout=600)
305
+ action = f"wait for sentinel on {self.shortname}"
306
+ with safe_while(action=action, timeout=1800, increment=3) as proceed:
307
+ while proceed():
308
+ try:
309
+ self.remote.run(args=cmd, timeout=600)
310
+ break
311
+ except (
312
+ ConnectionError,
313
+ EOFError,
314
+ ) as e:
315
+ log.error(f"{e} on {self.shortname}")
276
316
  self.log.info("Node is ready")
277
317
 
278
318
  def _fix_hostname(self):
@@ -307,6 +347,14 @@ class FOG(object):
307
347
  check_status=False,
308
348
  )
309
349
 
350
+ def _verify_installed_os(self):
351
+ wanted_os = OS(name=self.os_type, version=self.os_version)
352
+ if self.remote.os != wanted_os:
353
+ raise RuntimeError(
354
+ f"Expected {self.remote.shortname}'s OS to be {wanted_os} but "
355
+ f"found {self.remote.os}"
356
+ )
357
+
310
358
  def destroy(self):
311
359
  """A no-op; we just leave idle nodes as-is"""
312
360
  pass
@@ -27,7 +27,8 @@ class ProvisionOpenStack(OpenStack):
27
27
  """
28
28
  def __init__(self):
29
29
  super(ProvisionOpenStack, self).__init__()
30
- self.user_data = tempfile.mktemp()
30
+ fd, self.user_data = tempfile.mkstemp()
31
+ os.close(fd)
31
32
  log.debug("ProvisionOpenStack: " + str(config.openstack))
32
33
  self.basename = 'target'
33
34
  self.up_string = 'The system is finally up'
@@ -77,7 +78,7 @@ class ProvisionOpenStack(OpenStack):
77
78
  if 'No volume with a name or ID' not in e.output:
78
79
  raise e
79
80
  if volume_id:
80
- log.warn("Volume {} already exists with ID {}; using it"
81
+ log.warning("Volume {} already exists with ID {}; using it"
81
82
  .format(volume_name, volume_id))
82
83
  volume_id = self._openstack(
83
84
  "volume create %s" % config['openstack'].get('volume-create','')
@@ -106,7 +107,7 @@ class ProvisionOpenStack(OpenStack):
106
107
  log.debug("volume %s not in '%s' status yet"
107
108
  % (volume_id, status))
108
109
  except subprocess.CalledProcessError:
109
- log.warn("volume " + volume_id +
110
+ log.warning("volume " + volume_id +
110
111
  " not information available yet")
111
112
 
112
113
  def _attach_volume(self, volume_id, name):
@@ -151,7 +152,7 @@ class ProvisionOpenStack(OpenStack):
151
152
  """
152
153
  return the instance name suffixed with the IP address.
153
154
  """
154
- digits = map(int, re.findall('(\d+)\.(\d+)\.(\d+)\.(\d+)', ip)[0])
155
+ digits = map(int, re.findall(r'(\d+)\.(\d+)\.(\d+)\.(\d+)', ip)[0])
155
156
  return prefix + "%03d%03d%03d%03d" % tuple(digits)
156
157
 
157
158
  def create(self, num, os_type, os_version, arch, resources_hint):
@@ -26,7 +26,7 @@ def enabled(warn=False):
26
26
  params = ['endpoint', 'machine_types']
27
27
  unset = [_ for _ in params if not conf.get(_)]
28
28
  if unset and warn:
29
- log.warn(
29
+ log.warning(
30
30
  "Pelagos is disabled; set the following config options to enable: %s",
31
31
  ' '.join(unset),
32
32
  )
teuthology/repo_utils.py CHANGED
@@ -1,3 +1,4 @@
1
+ import functools
1
2
  import logging
2
3
  import os
3
4
  import re
@@ -5,6 +6,8 @@ import shutil
5
6
  import subprocess
6
7
  import time
7
8
 
9
+ import teuthology.exporter as exporter
10
+
8
11
  from teuthology import misc
9
12
  from teuthology.util.flock import FileLock
10
13
  from teuthology.config import config
@@ -50,10 +53,11 @@ def build_git_url(project, project_owner='ceph'):
50
53
  base = config.get_ceph_git_url()
51
54
  else:
52
55
  base = 'https://github.com/{project_owner}/{project}'
53
- url_templ = re.sub('\.git$', '', base)
56
+ url_templ = re.sub(r'\.git$', '', base)
54
57
  return url_templ.format(project_owner=project_owner, project=project)
55
58
 
56
59
 
60
+ @functools.lru_cache()
57
61
  def ls_remote(url, ref):
58
62
  """
59
63
  Return the current sha1 for a given repository and ref
@@ -70,6 +74,27 @@ def ls_remote(url, ref):
70
74
  return sha1
71
75
 
72
76
 
77
+ def current_branch(path: str) -> str:
78
+ """
79
+ Return the current branch for a given on-disk repository.
80
+
81
+ :returns: the current branch, or an empty string if none is found.
82
+ """
83
+ # git branch --show-current was added in 2.22.0, and we can't assume
84
+ # our version is new enough.
85
+ cmd = "git rev-parse --abbrev-ref HEAD"
86
+ result = subprocess.Popen(
87
+ cmd,
88
+ shell=True,
89
+ cwd=path,
90
+ stdout=subprocess.PIPE,
91
+ stderr=subprocess.DEVNULL,
92
+ ).communicate()[0].strip().decode()
93
+ if result == "HEAD":
94
+ return ""
95
+ return result
96
+
97
+
73
98
  def enforce_repo_state(repo_url, dest_path, branch, commit=None, remove_on_error=True):
74
99
  """
75
100
  Use git to either clone or update a given repo, forcing it to switch to the
@@ -96,8 +121,6 @@ def enforce_repo_state(repo_url, dest_path, branch, commit=None, remove_on_error
96
121
  set_remote(dest_path, repo_url)
97
122
  fetch_branch(dest_path, branch)
98
123
  touch_file(sentinel)
99
- else:
100
- log.info("%s was just updated or references a specific commit; assuming it is current", dest_path)
101
124
 
102
125
  if commit and os.path.exists(repo_reset):
103
126
  return
@@ -142,8 +165,9 @@ def clone_repo(repo_url, dest_path, branch, shallow=True):
142
165
  result = proc.wait()
143
166
  # Newer git versions will bail if the branch is not found, but older ones
144
167
  # will not. Fortunately they both output similar text.
145
- if not_found_str in out:
168
+ if result != 0:
146
169
  log.error(out)
170
+ if not_found_str in out:
147
171
  if result == 0:
148
172
  # Old git left a repo with the wrong branch. Remove it.
149
173
  shutil.rmtree(dest_path, ignore_errors=True)
@@ -251,7 +275,7 @@ def fetch_branch(repo_path, branch, shallow=True):
251
275
  GitError for other errors
252
276
  """
253
277
  validate_branch(branch)
254
- log.info("Fetching %s from origin", branch)
278
+ log.info("Fetching %s from origin", repo_path.split("/")[-1])
255
279
  args = ['git', 'fetch']
256
280
  if shallow:
257
281
  args.extend(['--depth', '1'])
@@ -288,7 +312,7 @@ def reset_repo(repo_url, dest_path, branch, commit=None):
288
312
  else:
289
313
  reset_branch = 'origin/%s' % branch
290
314
  reset_ref = commit or reset_branch
291
- log.info('Resetting repo at %s to %s', dest_path, reset_ref)
315
+ log.debug('Resetting repo at %s to %s', dest_path, reset_ref)
292
316
  # This try/except block will notice if the requested branch doesn't
293
317
  # exist, whether it was cloned or fetched.
294
318
  try:
@@ -334,7 +358,7 @@ def fetch_repo(url, branch, commit=None, bootstrap=None, lock=True):
334
358
  # only let one worker create/update the checkout at a time
335
359
  lock_path = dest_path.rstrip('/') + '.lock'
336
360
  with FileLock(lock_path, noop=not lock):
337
- with safe_while(sleep=10, tries=60) as proceed:
361
+ with safe_while(sleep=10, tries=6) as proceed:
338
362
  try:
339
363
  while proceed():
340
364
  try:
@@ -416,6 +440,7 @@ def fetch_teuthology(branch, commit=None, lock=True):
416
440
 
417
441
 
418
442
  def bootstrap_teuthology(dest_path):
443
+ with exporter.BootstrapTime().time():
419
444
  log.info("Bootstrapping %s", dest_path)
420
445
  # This magic makes the bootstrap script not attempt to clobber an
421
446
  # existing virtualenv. But the branch's bootstrap needs to actually
@@ -423,15 +448,20 @@ def bootstrap_teuthology(dest_path):
423
448
  env = os.environ.copy()
424
449
  env['NO_CLOBBER'] = '1'
425
450
  cmd = './bootstrap'
426
- boot_proc = subprocess.Popen(cmd, shell=True, cwd=dest_path, env=env,
427
- stdout=subprocess.PIPE,
428
- stderr=subprocess.STDOUT)
429
- out, err = boot_proc.communicate()
451
+ boot_proc = subprocess.Popen(
452
+ cmd, shell=True,
453
+ cwd=dest_path,
454
+ env=env,
455
+ stdout=subprocess.PIPE,
456
+ stderr=subprocess.STDOUT,
457
+ universal_newlines=True
458
+ )
459
+ out, _ = boot_proc.communicate()
430
460
  returncode = boot_proc.wait()
431
461
  log.info("Bootstrap exited with status %s", returncode)
432
462
  if returncode != 0:
433
- for line in out.split():
434
- log.warn(line.strip())
463
+ for line in out.split("\n"):
464
+ log.warning(line.strip())
435
465
  venv_path = os.path.join(dest_path, 'virtualenv')
436
466
  log.info("Removing %s", venv_path)
437
467
  shutil.rmtree(venv_path, ignore_errors=True)