teuthology 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scripts/describe.py +1 -0
- scripts/dispatcher.py +55 -26
- scripts/exporter.py +18 -0
- scripts/lock.py +1 -1
- scripts/node_cleanup.py +58 -0
- scripts/openstack.py +9 -9
- scripts/results.py +12 -11
- scripts/schedule.py +4 -0
- scripts/suite.py +57 -16
- scripts/supervisor.py +44 -0
- scripts/update_inventory.py +10 -4
- teuthology/__init__.py +24 -26
- teuthology/beanstalk.py +4 -3
- teuthology/config.py +16 -6
- teuthology/contextutil.py +18 -14
- teuthology/describe_tests.py +25 -18
- teuthology/dispatcher/__init__.py +210 -35
- teuthology/dispatcher/supervisor.py +140 -58
- teuthology/exceptions.py +43 -0
- teuthology/exporter.py +347 -0
- teuthology/kill.py +76 -81
- teuthology/lock/cli.py +3 -3
- teuthology/lock/ops.py +135 -61
- teuthology/lock/query.py +61 -44
- teuthology/ls.py +1 -1
- teuthology/misc.py +61 -75
- teuthology/nuke/__init__.py +12 -353
- teuthology/openstack/__init__.py +4 -3
- teuthology/openstack/openstack-centos-7.0-user-data.txt +1 -1
- teuthology/openstack/openstack-centos-7.1-user-data.txt +1 -1
- teuthology/openstack/openstack-centos-7.2-user-data.txt +1 -1
- teuthology/openstack/openstack-debian-8.0-user-data.txt +1 -1
- teuthology/openstack/openstack-opensuse-42.1-user-data.txt +1 -1
- teuthology/openstack/openstack-teuthology.cron +0 -1
- teuthology/orchestra/cluster.py +49 -7
- teuthology/orchestra/connection.py +17 -4
- teuthology/orchestra/console.py +111 -50
- teuthology/orchestra/daemon/cephadmunit.py +15 -2
- teuthology/orchestra/daemon/state.py +8 -1
- teuthology/orchestra/daemon/systemd.py +4 -4
- teuthology/orchestra/opsys.py +30 -11
- teuthology/orchestra/remote.py +405 -338
- teuthology/orchestra/run.py +3 -3
- teuthology/packaging.py +19 -16
- teuthology/provision/__init__.py +30 -10
- teuthology/provision/cloud/openstack.py +12 -6
- teuthology/provision/cloud/util.py +1 -2
- teuthology/provision/downburst.py +4 -3
- teuthology/provision/fog.py +68 -20
- teuthology/provision/openstack.py +5 -4
- teuthology/provision/pelagos.py +1 -1
- teuthology/repo_utils.py +43 -13
- teuthology/report.py +57 -35
- teuthology/results.py +5 -3
- teuthology/run.py +13 -14
- teuthology/run_tasks.py +27 -43
- teuthology/schedule.py +4 -3
- teuthology/scrape.py +28 -22
- teuthology/suite/__init__.py +74 -45
- teuthology/suite/build_matrix.py +34 -24
- teuthology/suite/fragment-merge.lua +105 -0
- teuthology/suite/matrix.py +31 -2
- teuthology/suite/merge.py +175 -0
- teuthology/suite/placeholder.py +6 -9
- teuthology/suite/run.py +175 -100
- teuthology/suite/util.py +64 -218
- teuthology/task/__init__.py +1 -1
- teuthology/task/ansible.py +101 -32
- teuthology/task/buildpackages.py +2 -2
- teuthology/task/ceph_ansible.py +13 -6
- teuthology/task/cephmetrics.py +2 -1
- teuthology/task/clock.py +33 -14
- teuthology/task/exec.py +18 -0
- teuthology/task/hadoop.py +2 -2
- teuthology/task/install/__init__.py +29 -7
- teuthology/task/install/bin/adjust-ulimits +16 -0
- teuthology/task/install/bin/daemon-helper +114 -0
- teuthology/task/install/bin/stdin-killer +263 -0
- teuthology/task/install/deb.py +1 -1
- teuthology/task/install/rpm.py +17 -5
- teuthology/task/install/util.py +3 -3
- teuthology/task/internal/__init__.py +41 -10
- teuthology/task/internal/edit_sudoers.sh +10 -0
- teuthology/task/internal/lock_machines.py +2 -9
- teuthology/task/internal/redhat.py +31 -1
- teuthology/task/internal/syslog.py +31 -8
- teuthology/task/kernel.py +152 -145
- teuthology/task/lockfile.py +1 -1
- teuthology/task/mpi.py +10 -10
- teuthology/task/pcp.py +1 -1
- teuthology/task/selinux.py +16 -8
- teuthology/task/ssh_keys.py +4 -4
- teuthology/timer.py +3 -3
- teuthology/util/loggerfile.py +19 -0
- teuthology/util/scanner.py +159 -0
- teuthology/util/sentry.py +52 -0
- teuthology/util/time.py +52 -0
- teuthology-1.2.1.data/scripts/adjust-ulimits +16 -0
- teuthology-1.2.1.data/scripts/daemon-helper +114 -0
- teuthology-1.2.1.data/scripts/stdin-killer +263 -0
- teuthology-1.2.1.dist-info/METADATA +88 -0
- teuthology-1.2.1.dist-info/RECORD +168 -0
- {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/WHEEL +1 -1
- {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/entry_points.txt +3 -2
- scripts/nuke.py +0 -47
- scripts/worker.py +0 -37
- teuthology/lock/test/__init__.py +0 -0
- teuthology/lock/test/test_lock.py +0 -7
- teuthology/nuke/actions.py +0 -456
- teuthology/openstack/test/__init__.py +0 -0
- teuthology/openstack/test/openstack-integration.py +0 -286
- teuthology/openstack/test/test_config.py +0 -35
- teuthology/openstack/test/test_openstack.py +0 -1695
- teuthology/orchestra/test/__init__.py +0 -0
- teuthology/orchestra/test/integration/__init__.py +0 -0
- teuthology/orchestra/test/integration/test_integration.py +0 -94
- teuthology/orchestra/test/test_cluster.py +0 -240
- teuthology/orchestra/test/test_connection.py +0 -106
- teuthology/orchestra/test/test_console.py +0 -217
- teuthology/orchestra/test/test_opsys.py +0 -404
- teuthology/orchestra/test/test_remote.py +0 -185
- teuthology/orchestra/test/test_run.py +0 -286
- teuthology/orchestra/test/test_systemd.py +0 -54
- teuthology/orchestra/test/util.py +0 -12
- teuthology/task/tests/__init__.py +0 -110
- teuthology/task/tests/test_locking.py +0 -25
- teuthology/task/tests/test_run.py +0 -40
- teuthology/test/__init__.py +0 -0
- teuthology/test/fake_archive.py +0 -107
- teuthology/test/fake_fs.py +0 -92
- teuthology/test/integration/__init__.py +0 -0
- teuthology/test/integration/test_suite.py +0 -86
- teuthology/test/task/__init__.py +0 -205
- teuthology/test/task/test_ansible.py +0 -624
- teuthology/test/task/test_ceph_ansible.py +0 -176
- teuthology/test/task/test_console_log.py +0 -88
- teuthology/test/task/test_install.py +0 -337
- teuthology/test/task/test_internal.py +0 -57
- teuthology/test/task/test_kernel.py +0 -243
- teuthology/test/task/test_pcp.py +0 -379
- teuthology/test/task/test_selinux.py +0 -35
- teuthology/test/test_config.py +0 -189
- teuthology/test/test_contextutil.py +0 -68
- teuthology/test/test_describe_tests.py +0 -316
- teuthology/test/test_email_sleep_before_teardown.py +0 -81
- teuthology/test/test_exit.py +0 -97
- teuthology/test/test_get_distro.py +0 -47
- teuthology/test/test_get_distro_version.py +0 -47
- teuthology/test/test_get_multi_machine_types.py +0 -27
- teuthology/test/test_job_status.py +0 -60
- teuthology/test/test_ls.py +0 -48
- teuthology/test/test_misc.py +0 -391
- teuthology/test/test_nuke.py +0 -290
- teuthology/test/test_packaging.py +0 -763
- teuthology/test/test_parallel.py +0 -28
- teuthology/test/test_repo_utils.py +0 -225
- teuthology/test/test_report.py +0 -77
- teuthology/test/test_results.py +0 -155
- teuthology/test/test_run.py +0 -239
- teuthology/test/test_safepath.py +0 -55
- teuthology/test/test_schedule.py +0 -45
- teuthology/test/test_scrape.py +0 -167
- teuthology/test/test_timer.py +0 -80
- teuthology/test/test_vps_os_vers_parameter_checking.py +0 -84
- teuthology/test/test_worker.py +0 -303
- teuthology/worker.py +0 -354
- teuthology-1.1.0.dist-info/METADATA +0 -76
- teuthology-1.1.0.dist-info/RECORD +0 -213
- {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/LICENSE +0 -0
- {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/top_level.txt +0 -0
teuthology/orchestra/run.py
CHANGED
@@ -9,7 +9,7 @@ from paramiko import ChannelFile
|
|
9
9
|
import gevent
|
10
10
|
import gevent.event
|
11
11
|
import socket
|
12
|
-
import
|
12
|
+
import shlex
|
13
13
|
import logging
|
14
14
|
import shutil
|
15
15
|
|
@@ -252,7 +252,7 @@ def quote(args):
|
|
252
252
|
if isinstance(a, Raw):
|
253
253
|
yield a.value
|
254
254
|
else:
|
255
|
-
yield
|
255
|
+
yield shlex.quote(a)
|
256
256
|
if isinstance(args, list):
|
257
257
|
return ' '.join(_quote(args))
|
258
258
|
else:
|
@@ -400,7 +400,7 @@ def run(
|
|
400
400
|
"""
|
401
401
|
Run a command remotely. If any of 'args' contains shell metacharacters
|
402
402
|
that you want to pass unquoted, pass it as an instance of Raw(); otherwise
|
403
|
-
it will be quoted with
|
403
|
+
it will be quoted with shlex.quote() (single quote, and single quotes
|
404
404
|
enclosed in double quotes).
|
405
405
|
|
406
406
|
:param client: SSHConnection to run the command with
|
teuthology/packaging.py
CHANGED
@@ -395,12 +395,12 @@ def _get_config_value_for_remote(ctx, remote, config, key):
|
|
395
395
|
|
396
396
|
config = {
|
397
397
|
'all':
|
398
|
-
{'branch': '
|
398
|
+
{'branch': 'main'},
|
399
399
|
'branch': 'next'
|
400
400
|
}
|
401
401
|
_get_config_value_for_remote(ctx, remote, config, 'branch')
|
402
402
|
|
403
|
-
would return '
|
403
|
+
would return 'main'.
|
404
404
|
|
405
405
|
:param ctx: the argparse.Namespace object
|
406
406
|
:param remote: the teuthology.orchestra.remote.Remote object
|
@@ -479,7 +479,7 @@ class GitbuilderProject(object):
|
|
479
479
|
)
|
480
480
|
# when we're initializing with a remote we most likely have
|
481
481
|
# a task config, not the entire teuthology job config
|
482
|
-
self.flavor = self.job_config.get("flavor", "
|
482
|
+
self.flavor = self.job_config.get("flavor", "default")
|
483
483
|
self.tag = self.job_config.get("tag")
|
484
484
|
|
485
485
|
def _init_from_config(self):
|
@@ -549,10 +549,6 @@ class GitbuilderProject(object):
|
|
549
549
|
"""
|
550
550
|
The base url that points at this project on gitbuilder.
|
551
551
|
|
552
|
-
For example::
|
553
|
-
|
554
|
-
http://gitbuilder.ceph.com/ceph-deb-raring-x86_64-basic/ref/master
|
555
|
-
|
556
552
|
:returns: A string of the base url for this project
|
557
553
|
"""
|
558
554
|
return self._get_base_url()
|
@@ -656,9 +652,9 @@ class GitbuilderProject(object):
|
|
656
652
|
remote, the sha1 from the config will be used.
|
657
653
|
|
658
654
|
If a tag, branch or sha1 can't be found it will default to use the
|
659
|
-
build from the
|
655
|
+
build from the main branch.
|
660
656
|
|
661
|
-
:returns: A string URI. Ex: ref/
|
657
|
+
:returns: A string URI. Ex: ref/main
|
662
658
|
"""
|
663
659
|
ref_name, ref_val = next(iter(self._choose_reference().items()))
|
664
660
|
if ref_name == 'sha1':
|
@@ -673,7 +669,7 @@ class GitbuilderProject(object):
|
|
673
669
|
Decide which to use.
|
674
670
|
|
675
671
|
:returns: a single-key dict containing the name and value of the
|
676
|
-
reference to use, e.g. {'branch': '
|
672
|
+
reference to use, e.g. {'branch': 'main'}
|
677
673
|
"""
|
678
674
|
tag = branch = sha1 = None
|
679
675
|
if self.remote:
|
@@ -716,8 +712,8 @@ class GitbuilderProject(object):
|
|
716
712
|
warn('sha1')
|
717
713
|
return dict(sha1=sha1)
|
718
714
|
else:
|
719
|
-
log.warning("defaulting to
|
720
|
-
return dict(branch='
|
715
|
+
log.warning("defaulting to main branch")
|
716
|
+
return dict(branch='main')
|
721
717
|
|
722
718
|
def _get_base_url(self):
|
723
719
|
"""
|
@@ -879,8 +875,6 @@ class ShamanProject(GitbuilderProject):
|
|
879
875
|
@property
|
880
876
|
def _search_uri(self):
|
881
877
|
flavor = self.flavor
|
882
|
-
if flavor == 'basic':
|
883
|
-
flavor = 'default'
|
884
878
|
req_obj = OrderedDict()
|
885
879
|
req_obj['status'] = 'ready'
|
886
880
|
req_obj['project'] = self.project
|
@@ -983,6 +977,9 @@ class ShamanProject(GitbuilderProject):
|
|
983
977
|
except VersionNotFoundError:
|
984
978
|
return False
|
985
979
|
|
980
|
+
# self._result has status, project, flavor, distros, arch, and sha1
|
981
|
+
# restrictions, so the only reason for multiples should be "multiple
|
982
|
+
# builds of the same sha1 etc."; the first entry is the newest
|
986
983
|
search_result = self._result.json()[0]
|
987
984
|
|
988
985
|
# now look for the build complete status
|
@@ -996,14 +993,20 @@ class ShamanProject(GitbuilderProject):
|
|
996
993
|
resp.raise_for_status()
|
997
994
|
except requests.HttpError:
|
998
995
|
return False
|
996
|
+
log.debug(f'looking for {self.distro} {self.arch} {self.flavor}')
|
999
997
|
for build in resp.json():
|
998
|
+
log.debug(f'build: {build["distro"]}/{build["distro_version"]} {build["distro_arch"]} {build["flavor"]}')
|
1000
999
|
if (
|
1000
|
+
# we must compare build arch to self.arch, since shaman's
|
1001
|
+
# results can have multiple arches but we're searching
|
1002
|
+
# for precisely one here
|
1001
1003
|
build['distro'] == search_result['distro'] and
|
1002
1004
|
build['distro_version'] == search_result['distro_version'] and
|
1003
1005
|
build['flavor'] == search_result['flavor'] and
|
1004
|
-
build['distro_arch']
|
1006
|
+
build['distro_arch'] == self.arch and
|
1007
|
+
build['status'] == 'completed'
|
1005
1008
|
):
|
1006
|
-
return
|
1009
|
+
return True
|
1007
1010
|
return False
|
1008
1011
|
|
1009
1012
|
def _get_repo(self):
|
teuthology/provision/__init__.py
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
import logging
|
2
|
+
import os
|
2
3
|
|
4
|
+
import teuthology.exporter
|
3
5
|
import teuthology.lock.query
|
4
6
|
from teuthology.misc import decanonicalize_hostname, get_distro, get_distro_version
|
5
7
|
|
@@ -8,19 +10,19 @@ from teuthology.provision import downburst
|
|
8
10
|
from teuthology.provision import fog
|
9
11
|
from teuthology.provision import openstack
|
10
12
|
from teuthology.provision import pelagos
|
11
|
-
import os
|
12
13
|
|
13
14
|
log = logging.getLogger(__name__)
|
14
15
|
|
15
16
|
|
16
|
-
def _logfile(
|
17
|
-
if
|
18
|
-
return
|
19
|
-
|
17
|
+
def _logfile(shortname: str, archive_path: str = ""):
|
18
|
+
if os.path.isfile(archive_path):
|
19
|
+
return f"{archive_path}/{shortname}.downburst.log"
|
20
|
+
|
20
21
|
|
21
22
|
def get_reimage_types():
|
22
23
|
return pelagos.get_types() + fog.get_types()
|
23
24
|
|
25
|
+
|
24
26
|
def reimage(ctx, machine_name, machine_type):
|
25
27
|
os_type = get_distro(ctx)
|
26
28
|
os_version = get_distro_version(ctx)
|
@@ -36,7 +38,21 @@ def reimage(ctx, machine_name, machine_type):
|
|
36
38
|
else:
|
37
39
|
raise Exception("The machine_type '%s' is not known to any "
|
38
40
|
"of configured provisioners" % machine_type)
|
39
|
-
|
41
|
+
status = "fail"
|
42
|
+
try:
|
43
|
+
result = obj.create()
|
44
|
+
status = "success"
|
45
|
+
except Exception:
|
46
|
+
# We only need this clause so that we avoid triggering the finally
|
47
|
+
# clause below in cases where the exception raised is KeyboardInterrupt
|
48
|
+
# or SystemExit
|
49
|
+
raise
|
50
|
+
finally:
|
51
|
+
teuthology.exporter.NodeReimagingResults().record(
|
52
|
+
machine_type=machine_type,
|
53
|
+
status=status,
|
54
|
+
)
|
55
|
+
return result
|
40
56
|
|
41
57
|
|
42
58
|
def create_if_vm(ctx, machine_name, _downburst=None):
|
@@ -78,8 +94,12 @@ def create_if_vm(ctx, machine_name, _downburst=None):
|
|
78
94
|
return dbrst.create()
|
79
95
|
|
80
96
|
|
81
|
-
def destroy_if_vm(
|
82
|
-
|
97
|
+
def destroy_if_vm(
|
98
|
+
machine_name: str,
|
99
|
+
user: str = "",
|
100
|
+
description: str = "",
|
101
|
+
_downburst=None
|
102
|
+
):
|
83
103
|
"""
|
84
104
|
Use downburst to destroy a virtual machine
|
85
105
|
|
@@ -99,7 +119,7 @@ def destroy_if_vm(ctx, machine_name, user=None, description=None,
|
|
99
119
|
log.error(msg.format(node=machine_name, as_user=user,
|
100
120
|
locked_by=status_info['locked_by']))
|
101
121
|
return False
|
102
|
-
if (description
|
122
|
+
if (description and description !=
|
103
123
|
status_info['description']):
|
104
124
|
msg = "Tried to destroy {node} with description {desc_arg} " + \
|
105
125
|
"but it is locked with description {desc_lock}"
|
@@ -117,5 +137,5 @@ def destroy_if_vm(ctx, machine_name, user=None, description=None,
|
|
117
137
|
dbrst = _downburst or \
|
118
138
|
downburst.Downburst(name=machine_name, os_type=None,
|
119
139
|
os_version=None, status=status_info,
|
120
|
-
logfile=_logfile(
|
140
|
+
logfile=_logfile(description, shortname))
|
121
141
|
return dbrst.destroy()
|
@@ -84,7 +84,13 @@ class OpenStackProvider(Provider):
|
|
84
84
|
@property
|
85
85
|
def images(self):
|
86
86
|
if not hasattr(self, '_images'):
|
87
|
-
|
87
|
+
exclude_image = self.conf.get('exclude_image', [])
|
88
|
+
if exclude_image and not isinstance(exclude_image, list):
|
89
|
+
exclude_image = [exclude_image]
|
90
|
+
exclude_re = [re.compile(x) for x in exclude_image]
|
91
|
+
images = retry(self.driver.list_images)
|
92
|
+
self._images = [_ for _ in images
|
93
|
+
if not any(x.match(_.name) for x in exclude_re)]
|
88
94
|
return self._images
|
89
95
|
|
90
96
|
@property
|
@@ -126,7 +132,7 @@ class OpenStackProvider(Provider):
|
|
126
132
|
else:
|
127
133
|
self._networks = list()
|
128
134
|
except AttributeError:
|
129
|
-
log.
|
135
|
+
log.warning("Unable to list networks for %s", self.driver)
|
130
136
|
self._networks = list()
|
131
137
|
return self._networks
|
132
138
|
|
@@ -144,7 +150,7 @@ class OpenStackProvider(Provider):
|
|
144
150
|
self.driver.ex_list_security_groups
|
145
151
|
)
|
146
152
|
except AttributeError:
|
147
|
-
log.
|
153
|
+
log.warning("Unable to list security groups for %s", self.driver)
|
148
154
|
self._security_groups = list()
|
149
155
|
return self._security_groups
|
150
156
|
|
@@ -420,7 +426,7 @@ class OpenStackProvisioner(base.Provisioner):
|
|
420
426
|
msg = "Unknown error locating %s"
|
421
427
|
if not matches:
|
422
428
|
msg = "No nodes found with name '%s'" % self.name
|
423
|
-
log.
|
429
|
+
log.warning(msg)
|
424
430
|
return
|
425
431
|
elif len(matches) > 1:
|
426
432
|
msg = "More than one node found with name '%s'"
|
@@ -438,9 +444,9 @@ class OpenStackProvisioner(base.Provisioner):
|
|
438
444
|
self._destroy_volumes()
|
439
445
|
nodes = self._find_nodes()
|
440
446
|
if not nodes:
|
441
|
-
log.
|
447
|
+
log.warning("Didn't find any nodes named '%s' to destroy!", self.name)
|
442
448
|
return True
|
443
449
|
if len(nodes) > 1:
|
444
|
-
log.
|
450
|
+
log.warning("Found multiple nodes named '%s' to destroy!", self.name)
|
445
451
|
log.info("Destroying nodes: %s", nodes)
|
446
452
|
return all([node.destroy() for node in nodes])
|
@@ -1,5 +1,4 @@
|
|
1
1
|
import datetime
|
2
|
-
import dateutil.tz
|
3
2
|
import dateutil.parser
|
4
3
|
import json
|
5
4
|
import os
|
@@ -103,7 +102,7 @@ class AuthToken(object):
|
|
103
102
|
def expired(self):
|
104
103
|
if self.expires is None:
|
105
104
|
return True
|
106
|
-
utcnow = datetime.datetime.now(
|
105
|
+
utcnow = datetime.datetime.now(datetime.timezone.utc)
|
107
106
|
offset = datetime.timedelta(minutes=30)
|
108
107
|
return self.expires < (utcnow + offset)
|
109
108
|
|
@@ -44,6 +44,7 @@ def downburst_executable():
|
|
44
44
|
|
45
45
|
def downburst_environment():
|
46
46
|
env = dict()
|
47
|
+
env['PATH'] = os.environ.get('PATH')
|
47
48
|
discover_url = os.environ.get('DOWNBURST_DISCOVER_URL')
|
48
49
|
if config.downburst and not discover_url:
|
49
50
|
if isinstance(config.downburst, dict):
|
@@ -165,7 +166,7 @@ class Downburst(object):
|
|
165
166
|
if proc.returncode != 0:
|
166
167
|
not_found_msg = "no domain with matching name '%s'" % self.shortname
|
167
168
|
if not_found_msg in err:
|
168
|
-
log.
|
169
|
+
log.warning("Ignoring error during destroy: %s", err)
|
169
170
|
return True
|
170
171
|
log.error("Error destroying %s: %s", self.name, err)
|
171
172
|
return False
|
@@ -215,7 +216,7 @@ class Downburst(object):
|
|
215
216
|
'additional-disks-size': machine['volumes']['size'],
|
216
217
|
'arch': 'x86_64',
|
217
218
|
}
|
218
|
-
fqdn = self.name.split('@')[1]
|
219
|
+
fqdn = self.name.split('@')[-1]
|
219
220
|
file_out = {
|
220
221
|
'downburst': file_info,
|
221
222
|
'local-hostname': fqdn,
|
@@ -307,7 +308,7 @@ def get_distro_from_downburst():
|
|
307
308
|
executable_cmd = downburst_executable()
|
308
309
|
environment_dict = downburst_environment()
|
309
310
|
if not executable_cmd:
|
310
|
-
log.
|
311
|
+
log.warning("Downburst not found!")
|
311
312
|
log.info('Using default values for supported os_type/os_version')
|
312
313
|
return default_table
|
313
314
|
try:
|
teuthology/provision/fog.py
CHANGED
@@ -1,10 +1,10 @@
|
|
1
|
+
import datetime
|
1
2
|
import json
|
2
3
|
import logging
|
3
4
|
import requests
|
4
5
|
import socket
|
5
6
|
import re
|
6
7
|
|
7
|
-
from datetime import datetime
|
8
8
|
from paramiko import SSHException
|
9
9
|
from paramiko.ssh_exception import NoValidConnectionsError
|
10
10
|
|
@@ -13,6 +13,7 @@ import teuthology.orchestra
|
|
13
13
|
from teuthology.config import config
|
14
14
|
from teuthology.contextutil import safe_while
|
15
15
|
from teuthology.exceptions import MaxWhileTries
|
16
|
+
from teuthology.orchestra.opsys import OS
|
16
17
|
from teuthology import misc
|
17
18
|
|
18
19
|
log = logging.getLogger(__name__)
|
@@ -29,7 +30,7 @@ def enabled(warn=False):
|
|
29
30
|
params = ['endpoint', 'api_token', 'user_token', 'machine_types']
|
30
31
|
unset = [param for param in params if not fog_conf.get(param)]
|
31
32
|
if unset and warn:
|
32
|
-
log.
|
33
|
+
log.warning(
|
33
34
|
"FOG disabled; set the following config options to enable: %s",
|
34
35
|
' '.join(unset),
|
35
36
|
)
|
@@ -89,6 +90,7 @@ class FOG(object):
|
|
89
90
|
raise
|
90
91
|
self._wait_for_ready()
|
91
92
|
self._fix_hostname()
|
93
|
+
self._verify_installed_os()
|
92
94
|
self.log.info("Deploy complete!")
|
93
95
|
|
94
96
|
def do_request(self, url_suffix, data=None, method='GET', verify=True):
|
@@ -147,18 +149,39 @@ class FOG(object):
|
|
147
149
|
represents it
|
148
150
|
:returns: A dict describing the image
|
149
151
|
"""
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
152
|
+
def do_get(name):
|
153
|
+
resp = self.do_request(
|
154
|
+
'/image',
|
155
|
+
data=json.dumps(dict(name=name)),
|
156
|
+
)
|
157
|
+
obj = resp.json()
|
158
|
+
if obj['count']:
|
159
|
+
return obj['images'][0]
|
160
|
+
|
161
|
+
os_type = self.os_type.lower()
|
162
|
+
os_version = self.os_version
|
163
|
+
name = f"{self.remote.machine_type}_{os_type}_{os_version}"
|
164
|
+
if image := do_get(name):
|
165
|
+
return image
|
166
|
+
elif os_type == 'centos' and not os_version.endswith('.stream'):
|
167
|
+
image = do_get(f"{name}.stream")
|
168
|
+
if image:
|
169
|
+
return image
|
170
|
+
else:
|
158
171
|
raise RuntimeError(
|
159
|
-
"
|
160
|
-
(self.
|
161
|
-
|
172
|
+
"Fog has no %s image. Available %s images: %s" %
|
173
|
+
(name, self.remote.machine_type, self.suggest_image_names()))
|
174
|
+
|
175
|
+
def suggest_image_names(self):
|
176
|
+
"""
|
177
|
+
Suggest available image names for this machine type.
|
178
|
+
|
179
|
+
:returns: A list of image names.
|
180
|
+
"""
|
181
|
+
resp = self.do_request('/image/search/%s' % self.remote.machine_type)
|
182
|
+
obj = resp.json()
|
183
|
+
images = obj['images']
|
184
|
+
return [image['name'] for image in images]
|
162
185
|
|
163
186
|
def set_image(self, host_id):
|
164
187
|
"""
|
@@ -167,6 +190,8 @@ class FOG(object):
|
|
167
190
|
"""
|
168
191
|
image_data = self.get_image_data()
|
169
192
|
image_id = int(image_data['id'])
|
193
|
+
image_name = image_data.get("name")
|
194
|
+
self.log.debug(f"Requesting image {image_name} (ID {image_id})")
|
170
195
|
self.do_request(
|
171
196
|
'/host/%s' % host_id,
|
172
197
|
method='PUT',
|
@@ -202,8 +227,8 @@ class FOG(object):
|
|
202
227
|
for task in host_tasks:
|
203
228
|
timestamp = task['createdTime']
|
204
229
|
time_delta = (
|
205
|
-
datetime.
|
206
|
-
timestamp, self.timestamp_format)
|
230
|
+
datetime.datetime.now(datetime.timezone.utc) - datetime.datetime.strptime(
|
231
|
+
timestamp, self.timestamp_format).replace(tzinfo=datetime.timezone.utc)
|
207
232
|
).total_seconds()
|
208
233
|
# There should only be one deploy task matching our host. Just in
|
209
234
|
# case there are multiple, select a very recent one.
|
@@ -240,13 +265,14 @@ class FOG(object):
|
|
240
265
|
completed)
|
241
266
|
"""
|
242
267
|
self.log.info("Waiting for deploy to finish")
|
243
|
-
with safe_while(sleep=15, tries=
|
268
|
+
with safe_while(sleep=15, tries=120, timeout=config.fog_reimage_timeout) as proceed:
|
244
269
|
while proceed():
|
245
270
|
if not self.deploy_task_active(task_id):
|
246
271
|
break
|
247
272
|
|
248
273
|
def cancel_deploy_task(self, task_id):
|
249
274
|
""" Cancel an active deploy task """
|
275
|
+
self.log.debug(f"Canceling deploy task with ID {task_id}")
|
250
276
|
resp = self.do_request(
|
251
277
|
'/task/cancel',
|
252
278
|
method='DELETE',
|
@@ -256,7 +282,7 @@ class FOG(object):
|
|
256
282
|
|
257
283
|
def _wait_for_ready(self):
|
258
284
|
""" Attempt to connect to the machine via SSH """
|
259
|
-
with safe_while(sleep=6,
|
285
|
+
with safe_while(sleep=6, timeout=config.fog_wait_for_ssh_timeout) as proceed:
|
260
286
|
while proceed():
|
261
287
|
try:
|
262
288
|
self.remote.connect()
|
@@ -267,12 +293,26 @@ class FOG(object):
|
|
267
293
|
NoValidConnectionsError,
|
268
294
|
MaxWhileTries,
|
269
295
|
EOFError,
|
270
|
-
):
|
271
|
-
|
296
|
+
) as e:
|
297
|
+
# log this, because otherwise lots of failures just
|
298
|
+
# keep retrying without any notification (like, say,
|
299
|
+
# a mismatched host key in ~/.ssh/known_hosts, or
|
300
|
+
# something)
|
301
|
+
self.log.warning(e)
|
272
302
|
sentinel_file = config.fog.get('sentinel_file', None)
|
273
303
|
if sentinel_file:
|
274
304
|
cmd = "while [ ! -e '%s' ]; do sleep 5; done" % sentinel_file
|
275
|
-
self.
|
305
|
+
action = f"wait for sentinel on {self.shortname}"
|
306
|
+
with safe_while(action=action, timeout=1800, increment=3) as proceed:
|
307
|
+
while proceed():
|
308
|
+
try:
|
309
|
+
self.remote.run(args=cmd, timeout=600)
|
310
|
+
break
|
311
|
+
except (
|
312
|
+
ConnectionError,
|
313
|
+
EOFError,
|
314
|
+
) as e:
|
315
|
+
log.error(f"{e} on {self.shortname}")
|
276
316
|
self.log.info("Node is ready")
|
277
317
|
|
278
318
|
def _fix_hostname(self):
|
@@ -307,6 +347,14 @@ class FOG(object):
|
|
307
347
|
check_status=False,
|
308
348
|
)
|
309
349
|
|
350
|
+
def _verify_installed_os(self):
|
351
|
+
wanted_os = OS(name=self.os_type, version=self.os_version)
|
352
|
+
if self.remote.os != wanted_os:
|
353
|
+
raise RuntimeError(
|
354
|
+
f"Expected {self.remote.shortname}'s OS to be {wanted_os} but "
|
355
|
+
f"found {self.remote.os}"
|
356
|
+
)
|
357
|
+
|
310
358
|
def destroy(self):
|
311
359
|
"""A no-op; we just leave idle nodes as-is"""
|
312
360
|
pass
|
@@ -27,7 +27,8 @@ class ProvisionOpenStack(OpenStack):
|
|
27
27
|
"""
|
28
28
|
def __init__(self):
|
29
29
|
super(ProvisionOpenStack, self).__init__()
|
30
|
-
self.user_data = tempfile.
|
30
|
+
fd, self.user_data = tempfile.mkstemp()
|
31
|
+
os.close(fd)
|
31
32
|
log.debug("ProvisionOpenStack: " + str(config.openstack))
|
32
33
|
self.basename = 'target'
|
33
34
|
self.up_string = 'The system is finally up'
|
@@ -77,7 +78,7 @@ class ProvisionOpenStack(OpenStack):
|
|
77
78
|
if 'No volume with a name or ID' not in e.output:
|
78
79
|
raise e
|
79
80
|
if volume_id:
|
80
|
-
log.
|
81
|
+
log.warning("Volume {} already exists with ID {}; using it"
|
81
82
|
.format(volume_name, volume_id))
|
82
83
|
volume_id = self._openstack(
|
83
84
|
"volume create %s" % config['openstack'].get('volume-create','')
|
@@ -106,7 +107,7 @@ class ProvisionOpenStack(OpenStack):
|
|
106
107
|
log.debug("volume %s not in '%s' status yet"
|
107
108
|
% (volume_id, status))
|
108
109
|
except subprocess.CalledProcessError:
|
109
|
-
log.
|
110
|
+
log.warning("volume " + volume_id +
|
110
111
|
" not information available yet")
|
111
112
|
|
112
113
|
def _attach_volume(self, volume_id, name):
|
@@ -151,7 +152,7 @@ class ProvisionOpenStack(OpenStack):
|
|
151
152
|
"""
|
152
153
|
return the instance name suffixed with the IP address.
|
153
154
|
"""
|
154
|
-
digits = map(int, re.findall('(\d+)\.(\d+)\.(\d+)\.(\d+)', ip)[0])
|
155
|
+
digits = map(int, re.findall(r'(\d+)\.(\d+)\.(\d+)\.(\d+)', ip)[0])
|
155
156
|
return prefix + "%03d%03d%03d%03d" % tuple(digits)
|
156
157
|
|
157
158
|
def create(self, num, os_type, os_version, arch, resources_hint):
|
teuthology/provision/pelagos.py
CHANGED
@@ -26,7 +26,7 @@ def enabled(warn=False):
|
|
26
26
|
params = ['endpoint', 'machine_types']
|
27
27
|
unset = [_ for _ in params if not conf.get(_)]
|
28
28
|
if unset and warn:
|
29
|
-
log.
|
29
|
+
log.warning(
|
30
30
|
"Pelagos is disabled; set the following config options to enable: %s",
|
31
31
|
' '.join(unset),
|
32
32
|
)
|
teuthology/repo_utils.py
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
import functools
|
1
2
|
import logging
|
2
3
|
import os
|
3
4
|
import re
|
@@ -5,6 +6,8 @@ import shutil
|
|
5
6
|
import subprocess
|
6
7
|
import time
|
7
8
|
|
9
|
+
import teuthology.exporter as exporter
|
10
|
+
|
8
11
|
from teuthology import misc
|
9
12
|
from teuthology.util.flock import FileLock
|
10
13
|
from teuthology.config import config
|
@@ -50,10 +53,11 @@ def build_git_url(project, project_owner='ceph'):
|
|
50
53
|
base = config.get_ceph_git_url()
|
51
54
|
else:
|
52
55
|
base = 'https://github.com/{project_owner}/{project}'
|
53
|
-
url_templ = re.sub('\.git$', '', base)
|
56
|
+
url_templ = re.sub(r'\.git$', '', base)
|
54
57
|
return url_templ.format(project_owner=project_owner, project=project)
|
55
58
|
|
56
59
|
|
60
|
+
@functools.lru_cache()
|
57
61
|
def ls_remote(url, ref):
|
58
62
|
"""
|
59
63
|
Return the current sha1 for a given repository and ref
|
@@ -70,6 +74,27 @@ def ls_remote(url, ref):
|
|
70
74
|
return sha1
|
71
75
|
|
72
76
|
|
77
|
+
def current_branch(path: str) -> str:
|
78
|
+
"""
|
79
|
+
Return the current branch for a given on-disk repository.
|
80
|
+
|
81
|
+
:returns: the current branch, or an empty string if none is found.
|
82
|
+
"""
|
83
|
+
# git branch --show-current was added in 2.22.0, and we can't assume
|
84
|
+
# our version is new enough.
|
85
|
+
cmd = "git rev-parse --abbrev-ref HEAD"
|
86
|
+
result = subprocess.Popen(
|
87
|
+
cmd,
|
88
|
+
shell=True,
|
89
|
+
cwd=path,
|
90
|
+
stdout=subprocess.PIPE,
|
91
|
+
stderr=subprocess.DEVNULL,
|
92
|
+
).communicate()[0].strip().decode()
|
93
|
+
if result == "HEAD":
|
94
|
+
return ""
|
95
|
+
return result
|
96
|
+
|
97
|
+
|
73
98
|
def enforce_repo_state(repo_url, dest_path, branch, commit=None, remove_on_error=True):
|
74
99
|
"""
|
75
100
|
Use git to either clone or update a given repo, forcing it to switch to the
|
@@ -96,8 +121,6 @@ def enforce_repo_state(repo_url, dest_path, branch, commit=None, remove_on_error
|
|
96
121
|
set_remote(dest_path, repo_url)
|
97
122
|
fetch_branch(dest_path, branch)
|
98
123
|
touch_file(sentinel)
|
99
|
-
else:
|
100
|
-
log.info("%s was just updated or references a specific commit; assuming it is current", dest_path)
|
101
124
|
|
102
125
|
if commit and os.path.exists(repo_reset):
|
103
126
|
return
|
@@ -142,8 +165,9 @@ def clone_repo(repo_url, dest_path, branch, shallow=True):
|
|
142
165
|
result = proc.wait()
|
143
166
|
# Newer git versions will bail if the branch is not found, but older ones
|
144
167
|
# will not. Fortunately they both output similar text.
|
145
|
-
if
|
168
|
+
if result != 0:
|
146
169
|
log.error(out)
|
170
|
+
if not_found_str in out:
|
147
171
|
if result == 0:
|
148
172
|
# Old git left a repo with the wrong branch. Remove it.
|
149
173
|
shutil.rmtree(dest_path, ignore_errors=True)
|
@@ -251,7 +275,7 @@ def fetch_branch(repo_path, branch, shallow=True):
|
|
251
275
|
GitError for other errors
|
252
276
|
"""
|
253
277
|
validate_branch(branch)
|
254
|
-
log.info("Fetching %s from origin",
|
278
|
+
log.info("Fetching %s from origin", repo_path.split("/")[-1])
|
255
279
|
args = ['git', 'fetch']
|
256
280
|
if shallow:
|
257
281
|
args.extend(['--depth', '1'])
|
@@ -288,7 +312,7 @@ def reset_repo(repo_url, dest_path, branch, commit=None):
|
|
288
312
|
else:
|
289
313
|
reset_branch = 'origin/%s' % branch
|
290
314
|
reset_ref = commit or reset_branch
|
291
|
-
log.
|
315
|
+
log.debug('Resetting repo at %s to %s', dest_path, reset_ref)
|
292
316
|
# This try/except block will notice if the requested branch doesn't
|
293
317
|
# exist, whether it was cloned or fetched.
|
294
318
|
try:
|
@@ -334,7 +358,7 @@ def fetch_repo(url, branch, commit=None, bootstrap=None, lock=True):
|
|
334
358
|
# only let one worker create/update the checkout at a time
|
335
359
|
lock_path = dest_path.rstrip('/') + '.lock'
|
336
360
|
with FileLock(lock_path, noop=not lock):
|
337
|
-
with safe_while(sleep=10, tries=
|
361
|
+
with safe_while(sleep=10, tries=6) as proceed:
|
338
362
|
try:
|
339
363
|
while proceed():
|
340
364
|
try:
|
@@ -416,6 +440,7 @@ def fetch_teuthology(branch, commit=None, lock=True):
|
|
416
440
|
|
417
441
|
|
418
442
|
def bootstrap_teuthology(dest_path):
|
443
|
+
with exporter.BootstrapTime().time():
|
419
444
|
log.info("Bootstrapping %s", dest_path)
|
420
445
|
# This magic makes the bootstrap script not attempt to clobber an
|
421
446
|
# existing virtualenv. But the branch's bootstrap needs to actually
|
@@ -423,15 +448,20 @@ def bootstrap_teuthology(dest_path):
|
|
423
448
|
env = os.environ.copy()
|
424
449
|
env['NO_CLOBBER'] = '1'
|
425
450
|
cmd = './bootstrap'
|
426
|
-
boot_proc = subprocess.Popen(
|
427
|
-
|
428
|
-
|
429
|
-
|
451
|
+
boot_proc = subprocess.Popen(
|
452
|
+
cmd, shell=True,
|
453
|
+
cwd=dest_path,
|
454
|
+
env=env,
|
455
|
+
stdout=subprocess.PIPE,
|
456
|
+
stderr=subprocess.STDOUT,
|
457
|
+
universal_newlines=True
|
458
|
+
)
|
459
|
+
out, _ = boot_proc.communicate()
|
430
460
|
returncode = boot_proc.wait()
|
431
461
|
log.info("Bootstrap exited with status %s", returncode)
|
432
462
|
if returncode != 0:
|
433
|
-
for line in out.split():
|
434
|
-
log.
|
463
|
+
for line in out.split("\n"):
|
464
|
+
log.warning(line.strip())
|
435
465
|
venv_path = os.path.join(dest_path, 'virtualenv')
|
436
466
|
log.info("Removing %s", venv_path)
|
437
467
|
shutil.rmtree(venv_path, ignore_errors=True)
|