teuthology 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. scripts/describe.py +1 -0
  2. scripts/dispatcher.py +55 -26
  3. scripts/exporter.py +18 -0
  4. scripts/lock.py +1 -1
  5. scripts/node_cleanup.py +58 -0
  6. scripts/openstack.py +9 -9
  7. scripts/results.py +12 -11
  8. scripts/schedule.py +4 -0
  9. scripts/suite.py +57 -16
  10. scripts/supervisor.py +44 -0
  11. scripts/update_inventory.py +10 -4
  12. teuthology/__init__.py +24 -26
  13. teuthology/beanstalk.py +4 -3
  14. teuthology/config.py +16 -6
  15. teuthology/contextutil.py +18 -14
  16. teuthology/describe_tests.py +25 -18
  17. teuthology/dispatcher/__init__.py +210 -35
  18. teuthology/dispatcher/supervisor.py +140 -58
  19. teuthology/exceptions.py +43 -0
  20. teuthology/exporter.py +347 -0
  21. teuthology/kill.py +76 -81
  22. teuthology/lock/cli.py +3 -3
  23. teuthology/lock/ops.py +135 -61
  24. teuthology/lock/query.py +61 -44
  25. teuthology/ls.py +1 -1
  26. teuthology/misc.py +61 -75
  27. teuthology/nuke/__init__.py +12 -353
  28. teuthology/openstack/__init__.py +4 -3
  29. teuthology/openstack/openstack-centos-7.0-user-data.txt +1 -1
  30. teuthology/openstack/openstack-centos-7.1-user-data.txt +1 -1
  31. teuthology/openstack/openstack-centos-7.2-user-data.txt +1 -1
  32. teuthology/openstack/openstack-debian-8.0-user-data.txt +1 -1
  33. teuthology/openstack/openstack-opensuse-42.1-user-data.txt +1 -1
  34. teuthology/openstack/openstack-teuthology.cron +0 -1
  35. teuthology/orchestra/cluster.py +49 -7
  36. teuthology/orchestra/connection.py +17 -4
  37. teuthology/orchestra/console.py +111 -50
  38. teuthology/orchestra/daemon/cephadmunit.py +15 -2
  39. teuthology/orchestra/daemon/state.py +8 -1
  40. teuthology/orchestra/daemon/systemd.py +4 -4
  41. teuthology/orchestra/opsys.py +30 -11
  42. teuthology/orchestra/remote.py +405 -338
  43. teuthology/orchestra/run.py +3 -3
  44. teuthology/packaging.py +19 -16
  45. teuthology/provision/__init__.py +30 -10
  46. teuthology/provision/cloud/openstack.py +12 -6
  47. teuthology/provision/cloud/util.py +1 -2
  48. teuthology/provision/downburst.py +4 -3
  49. teuthology/provision/fog.py +68 -20
  50. teuthology/provision/openstack.py +5 -4
  51. teuthology/provision/pelagos.py +1 -1
  52. teuthology/repo_utils.py +43 -13
  53. teuthology/report.py +57 -35
  54. teuthology/results.py +5 -3
  55. teuthology/run.py +13 -14
  56. teuthology/run_tasks.py +27 -43
  57. teuthology/schedule.py +4 -3
  58. teuthology/scrape.py +28 -22
  59. teuthology/suite/__init__.py +74 -45
  60. teuthology/suite/build_matrix.py +34 -24
  61. teuthology/suite/fragment-merge.lua +105 -0
  62. teuthology/suite/matrix.py +31 -2
  63. teuthology/suite/merge.py +175 -0
  64. teuthology/suite/placeholder.py +6 -9
  65. teuthology/suite/run.py +175 -100
  66. teuthology/suite/util.py +64 -218
  67. teuthology/task/__init__.py +1 -1
  68. teuthology/task/ansible.py +101 -32
  69. teuthology/task/buildpackages.py +2 -2
  70. teuthology/task/ceph_ansible.py +13 -6
  71. teuthology/task/cephmetrics.py +2 -1
  72. teuthology/task/clock.py +33 -14
  73. teuthology/task/exec.py +18 -0
  74. teuthology/task/hadoop.py +2 -2
  75. teuthology/task/install/__init__.py +29 -7
  76. teuthology/task/install/bin/adjust-ulimits +16 -0
  77. teuthology/task/install/bin/daemon-helper +114 -0
  78. teuthology/task/install/bin/stdin-killer +263 -0
  79. teuthology/task/install/deb.py +1 -1
  80. teuthology/task/install/rpm.py +17 -5
  81. teuthology/task/install/util.py +3 -3
  82. teuthology/task/internal/__init__.py +41 -10
  83. teuthology/task/internal/edit_sudoers.sh +10 -0
  84. teuthology/task/internal/lock_machines.py +2 -9
  85. teuthology/task/internal/redhat.py +31 -1
  86. teuthology/task/internal/syslog.py +31 -8
  87. teuthology/task/kernel.py +152 -145
  88. teuthology/task/lockfile.py +1 -1
  89. teuthology/task/mpi.py +10 -10
  90. teuthology/task/pcp.py +1 -1
  91. teuthology/task/selinux.py +16 -8
  92. teuthology/task/ssh_keys.py +4 -4
  93. teuthology/timer.py +3 -3
  94. teuthology/util/loggerfile.py +19 -0
  95. teuthology/util/scanner.py +159 -0
  96. teuthology/util/sentry.py +52 -0
  97. teuthology/util/time.py +52 -0
  98. teuthology-1.2.1.data/scripts/adjust-ulimits +16 -0
  99. teuthology-1.2.1.data/scripts/daemon-helper +114 -0
  100. teuthology-1.2.1.data/scripts/stdin-killer +263 -0
  101. teuthology-1.2.1.dist-info/METADATA +88 -0
  102. teuthology-1.2.1.dist-info/RECORD +168 -0
  103. {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/WHEEL +1 -1
  104. {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/entry_points.txt +3 -2
  105. scripts/nuke.py +0 -47
  106. scripts/worker.py +0 -37
  107. teuthology/lock/test/__init__.py +0 -0
  108. teuthology/lock/test/test_lock.py +0 -7
  109. teuthology/nuke/actions.py +0 -456
  110. teuthology/openstack/test/__init__.py +0 -0
  111. teuthology/openstack/test/openstack-integration.py +0 -286
  112. teuthology/openstack/test/test_config.py +0 -35
  113. teuthology/openstack/test/test_openstack.py +0 -1695
  114. teuthology/orchestra/test/__init__.py +0 -0
  115. teuthology/orchestra/test/integration/__init__.py +0 -0
  116. teuthology/orchestra/test/integration/test_integration.py +0 -94
  117. teuthology/orchestra/test/test_cluster.py +0 -240
  118. teuthology/orchestra/test/test_connection.py +0 -106
  119. teuthology/orchestra/test/test_console.py +0 -217
  120. teuthology/orchestra/test/test_opsys.py +0 -404
  121. teuthology/orchestra/test/test_remote.py +0 -185
  122. teuthology/orchestra/test/test_run.py +0 -286
  123. teuthology/orchestra/test/test_systemd.py +0 -54
  124. teuthology/orchestra/test/util.py +0 -12
  125. teuthology/task/tests/__init__.py +0 -110
  126. teuthology/task/tests/test_locking.py +0 -25
  127. teuthology/task/tests/test_run.py +0 -40
  128. teuthology/test/__init__.py +0 -0
  129. teuthology/test/fake_archive.py +0 -107
  130. teuthology/test/fake_fs.py +0 -92
  131. teuthology/test/integration/__init__.py +0 -0
  132. teuthology/test/integration/test_suite.py +0 -86
  133. teuthology/test/task/__init__.py +0 -205
  134. teuthology/test/task/test_ansible.py +0 -624
  135. teuthology/test/task/test_ceph_ansible.py +0 -176
  136. teuthology/test/task/test_console_log.py +0 -88
  137. teuthology/test/task/test_install.py +0 -337
  138. teuthology/test/task/test_internal.py +0 -57
  139. teuthology/test/task/test_kernel.py +0 -243
  140. teuthology/test/task/test_pcp.py +0 -379
  141. teuthology/test/task/test_selinux.py +0 -35
  142. teuthology/test/test_config.py +0 -189
  143. teuthology/test/test_contextutil.py +0 -68
  144. teuthology/test/test_describe_tests.py +0 -316
  145. teuthology/test/test_email_sleep_before_teardown.py +0 -81
  146. teuthology/test/test_exit.py +0 -97
  147. teuthology/test/test_get_distro.py +0 -47
  148. teuthology/test/test_get_distro_version.py +0 -47
  149. teuthology/test/test_get_multi_machine_types.py +0 -27
  150. teuthology/test/test_job_status.py +0 -60
  151. teuthology/test/test_ls.py +0 -48
  152. teuthology/test/test_misc.py +0 -391
  153. teuthology/test/test_nuke.py +0 -290
  154. teuthology/test/test_packaging.py +0 -763
  155. teuthology/test/test_parallel.py +0 -28
  156. teuthology/test/test_repo_utils.py +0 -225
  157. teuthology/test/test_report.py +0 -77
  158. teuthology/test/test_results.py +0 -155
  159. teuthology/test/test_run.py +0 -239
  160. teuthology/test/test_safepath.py +0 -55
  161. teuthology/test/test_schedule.py +0 -45
  162. teuthology/test/test_scrape.py +0 -167
  163. teuthology/test/test_timer.py +0 -80
  164. teuthology/test/test_vps_os_vers_parameter_checking.py +0 -84
  165. teuthology/test/test_worker.py +0 -303
  166. teuthology/worker.py +0 -354
  167. teuthology-1.1.0.dist-info/METADATA +0 -76
  168. teuthology-1.1.0.dist-info/RECORD +0 -213
  169. {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/LICENSE +0 -0
  170. {teuthology-1.1.0.dist-info → teuthology-1.2.1.dist-info}/top_level.txt +0 -0
@@ -1,391 +0,0 @@
1
- import argparse
2
- from datetime import datetime
3
-
4
- from unittest.mock import Mock, patch
5
- from teuthology.orchestra import cluster
6
- from teuthology.config import config
7
- from teuthology import misc
8
- import subprocess
9
-
10
- import pytest
11
-
12
-
13
- class FakeRemote(object):
14
- pass
15
-
16
-
17
- def test_sh_normal(caplog):
18
- assert misc.sh("/bin/echo ABC") == "ABC\n"
19
- assert "truncated" not in caplog.text
20
-
21
-
22
- def test_sh_truncate(caplog):
23
- assert misc.sh("/bin/echo -n AB ; /bin/echo C", 2) == "ABC\n"
24
- assert "truncated" in caplog.text
25
- assert "ABC" not in caplog.text
26
-
27
-
28
- def test_sh_fail(caplog):
29
- with pytest.raises(subprocess.CalledProcessError) as excinfo:
30
- misc.sh("/bin/echo -n AB ; /bin/echo C ; exit 111", 2) == "ABC\n"
31
- assert excinfo.value.returncode == 111
32
- for record in caplog.records:
33
- if record.levelname == 'ERROR':
34
- assert ('replay full' in record.message or
35
- 'ABC\n' == record.message)
36
-
37
- def test_sh_progress(caplog):
38
- misc.sh("echo AB ; sleep 5 ; /bin/echo C", 2) == "ABC\n"
39
- records = caplog.records
40
- assert ':sh: ' in records[0].message
41
- assert 'AB' == records[1].message
42
- assert 'C' == records[2].message
43
- #
44
- # With a sleep 5 between the first and the second message,
45
- # there must be at least 2 seconds between the log record
46
- # of the first message and the log record of the second one
47
- #
48
- t1 = datetime.strptime(records[1].asctime.split(',')[0], "%Y-%m-%d %H:%M:%S")
49
- t2 = datetime.strptime(records[2].asctime.split(',')[0], "%Y-%m-%d %H:%M:%S")
50
- assert (t2 - t1).total_seconds() > 2
51
-
52
-
53
- def test_wait_until_osds_up():
54
- ctx = argparse.Namespace()
55
- ctx.daemons = Mock()
56
- ctx.daemons.iter_daemons_of_role.return_value = list()
57
- remote = FakeRemote()
58
-
59
- def s(self, **kwargs):
60
- return 'IGNORED\n{"osds":[{"state":["up"]}]}'
61
-
62
- remote.sh = s
63
- ctx.cluster = cluster.Cluster(
64
- remotes=[
65
- (remote, ['osd.0', 'client.1'])
66
- ],
67
- )
68
- with patch.multiple(
69
- misc,
70
- get_testdir=lambda ctx: "TESTDIR",
71
- ):
72
- misc.wait_until_osds_up(ctx, ctx.cluster, remote)
73
-
74
-
75
- def test_get_clients_simple():
76
- ctx = argparse.Namespace()
77
- remote = FakeRemote()
78
- ctx.cluster = cluster.Cluster(
79
- remotes=[
80
- (remote, ['client.0', 'client.1'])
81
- ],
82
- )
83
- g = misc.get_clients(ctx=ctx, roles=['client.1'])
84
- got = next(g)
85
- assert len(got) == 2
86
- assert got[0] == ('1')
87
- assert got[1] is remote
88
- with pytest.raises(StopIteration):
89
- next(g)
90
-
91
-
92
- def test_get_mon_names():
93
- expected = [
94
- ([['mon.a', 'osd.0', 'mon.c']], 'ceph', ['mon.a', 'mon.c']),
95
- ([['ceph.mon.a', 'osd.0', 'ceph.mon.c']], 'ceph', ['ceph.mon.a', 'ceph.mon.c']),
96
- ([['mon.a', 'osd.0', 'mon.c'], ['ceph.mon.b']], 'ceph', ['mon.a', 'mon.c', 'ceph.mon.b']),
97
- ([['mon.a', 'osd.0', 'mon.c'], ['foo.mon.a']], 'ceph', ['mon.a', 'mon.c']),
98
- ([['mon.a', 'osd.0', 'mon.c'], ['foo.mon.a']], 'foo', ['foo.mon.a']),
99
- ]
100
- for remote_roles, cluster_name, expected_mons in expected:
101
- ctx = argparse.Namespace()
102
- ctx.cluster = Mock()
103
- ctx.cluster.remotes = {i: roles for i, roles in enumerate(remote_roles)}
104
- mons = misc.get_mon_names(ctx, cluster_name)
105
- assert expected_mons == mons
106
-
107
-
108
- def test_get_first_mon():
109
- expected = [
110
- ([['mon.a', 'osd.0', 'mon.c']], 'ceph', 'mon.a'),
111
- ([['ceph.mon.a', 'osd.0', 'ceph.mon.c']], 'ceph', 'ceph.mon.a'),
112
- ([['mon.a', 'osd.0', 'mon.c'], ['ceph.mon.b']], 'ceph', 'ceph.mon.b'),
113
- ([['mon.a', 'osd.0', 'mon.c'], ['foo.mon.a']], 'ceph', 'mon.a'),
114
- ([['foo.mon.b', 'osd.0', 'mon.c'], ['foo.mon.a']], 'foo', 'foo.mon.a'),
115
- ]
116
- for remote_roles, cluster_name, expected_mon in expected:
117
- ctx = argparse.Namespace()
118
- ctx.cluster = Mock()
119
- ctx.cluster.remotes = {i: roles for i, roles in enumerate(remote_roles)}
120
- mon = misc.get_first_mon(ctx, None, cluster_name)
121
- assert expected_mon == mon
122
-
123
-
124
- def test_roles_of_type():
125
- expected = [
126
- (['client.0', 'osd.0', 'ceph.osd.1'], 'osd', ['0', '1']),
127
- (['client.0', 'osd.0', 'ceph.osd.1'], 'client', ['0']),
128
- (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'mon', []),
129
- (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'client',
130
- ['1', '2.3']),
131
- ]
132
- for roles_for_host, type_, expected_ids in expected:
133
- ids = list(misc.roles_of_type(roles_for_host, type_))
134
- assert ids == expected_ids
135
-
136
-
137
- def test_cluster_roles_of_type():
138
- expected = [
139
- (['client.0', 'osd.0', 'ceph.osd.1'], 'osd', 'ceph',
140
- ['osd.0', 'ceph.osd.1']),
141
- (['client.0', 'osd.0', 'ceph.osd.1'], 'client', 'ceph',
142
- ['client.0']),
143
- (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'mon', None, []),
144
- (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'client', None,
145
- ['foo.client.1', 'bar.client.2.3']),
146
- (['foo.client.1', 'bar.client.2.3', 'baz.osd.1'], 'client', 'bar',
147
- ['bar.client.2.3']),
148
- ]
149
- for roles_for_host, type_, cluster_, expected_roles in expected:
150
- roles = list(misc.cluster_roles_of_type(roles_for_host, type_, cluster_))
151
- assert roles == expected_roles
152
-
153
-
154
- def test_all_roles_of_type():
155
- expected = [
156
- ([['client.0', 'osd.0', 'ceph.osd.1'], ['bar.osd.2']],
157
- 'osd', ['0', '1', '2']),
158
- ([['client.0', 'osd.0', 'ceph.osd.1'], ['bar.osd.2', 'baz.client.1']],
159
- 'client', ['0', '1']),
160
- ([['foo.client.1', 'bar.client.2.3'], ['baz.osd.1']], 'mon', []),
161
- ([['foo.client.1', 'bar.client.2.3'], ['baz.osd.1', 'ceph.client.bar']],
162
- 'client', ['1', '2.3', 'bar']),
163
- ]
164
- for host_roles, type_, expected_ids in expected:
165
- cluster_ = Mock()
166
- cluster_.remotes = dict(enumerate(host_roles))
167
- ids = list(misc.all_roles_of_type(cluster_, type_))
168
- assert ids == expected_ids
169
-
170
-
171
- def test_get_http_log_path():
172
- # Fake configuration
173
- archive_server = "http://example.com/server_root"
174
- config.archive_server = archive_server
175
- archive_dir = "/var/www/archives"
176
-
177
- path = misc.get_http_log_path(archive_dir)
178
- assert path == "http://example.com/server_root/archives/"
179
-
180
- job_id = '12345'
181
- path = misc.get_http_log_path(archive_dir, job_id)
182
- assert path == "http://example.com/server_root/archives/12345/"
183
-
184
- # Inktank configuration
185
- archive_server = "http://qa-proxy.ceph.com/teuthology/"
186
- config.archive_server = archive_server
187
- archive_dir = "/var/lib/teuthworker/archive/teuthology-2013-09-12_11:49:50-ceph-deploy-master-testing-basic-vps"
188
- job_id = 31087
189
- path = misc.get_http_log_path(archive_dir, job_id)
190
- assert path == "http://qa-proxy.ceph.com/teuthology/teuthology-2013-09-12_11:49:50-ceph-deploy-master-testing-basic-vps/31087/"
191
-
192
- path = misc.get_http_log_path(archive_dir)
193
- assert path == "http://qa-proxy.ceph.com/teuthology/teuthology-2013-09-12_11:49:50-ceph-deploy-master-testing-basic-vps/"
194
-
195
-
196
- def test_is_type():
197
- is_client = misc.is_type('client')
198
- assert is_client('client.0')
199
- assert is_client('ceph.client.0')
200
- assert is_client('foo.client.0')
201
- assert is_client('foo.client.bar.baz')
202
-
203
- with pytest.raises(ValueError):
204
- is_client('')
205
- is_client('client')
206
- assert not is_client('foo.bar.baz')
207
- assert not is_client('ceph.client')
208
- assert not is_client('hadoop.master.0')
209
-
210
-
211
- def test_is_type_in_cluster():
212
- is_c1_osd = misc.is_type('osd', 'c1')
213
- with pytest.raises(ValueError):
214
- is_c1_osd('')
215
- assert not is_c1_osd('osd.0')
216
- assert not is_c1_osd('ceph.osd.0')
217
- assert not is_c1_osd('ceph.osd.0')
218
- assert not is_c1_osd('c11.osd.0')
219
- assert is_c1_osd('c1.osd.0')
220
- assert is_c1_osd('c1.osd.999')
221
-
222
-
223
- def test_get_mons():
224
- ips = ['1.1.1.1', '2.2.2.2', '3.3.3.3']
225
- addrs = ['1.1.1.1:6789', '1.1.1.1:6790', '1.1.1.1:6791']
226
-
227
- mons = misc.get_mons([['mon.a']], ips)
228
- assert mons == {'mon.a': addrs[0]}
229
-
230
- mons = misc.get_mons([['cluster-a.mon.foo', 'client.b'], ['osd.0']], ips)
231
- assert mons == {'cluster-a.mon.foo': addrs[0]}
232
-
233
- mons = misc.get_mons([['mon.a', 'mon.b', 'ceph.mon.c']], ips)
234
- assert mons == {'mon.a': addrs[0],
235
- 'mon.b': addrs[1],
236
- 'ceph.mon.c': addrs[2]}
237
-
238
- mons = misc.get_mons([['mon.a'], ['mon.b'], ['ceph.mon.c']], ips)
239
- assert mons == {'mon.a': addrs[0],
240
- 'mon.b': ips[1] + ':6789',
241
- 'ceph.mon.c': ips[2] + ':6789'}
242
-
243
-
244
- def test_split_role():
245
- expected = {
246
- 'client.0': ('ceph', 'client', '0'),
247
- 'foo.client.0': ('foo', 'client', '0'),
248
- 'bar.baz.x.y.z': ('bar', 'baz', 'x.y.z'),
249
- 'mds.a-s-b': ('ceph', 'mds', 'a-s-b'),
250
- }
251
-
252
- for role, expected_split in expected.items():
253
- actual_split = misc.split_role(role)
254
- assert actual_split == expected_split
255
-
256
- class TestHostnames(object):
257
- def setup(self):
258
- config._conf = dict()
259
-
260
- def teardown(self):
261
- config.load()
262
-
263
- def test_canonicalize_hostname(self):
264
- host_base = 'box1'
265
- result = misc.canonicalize_hostname(host_base)
266
- assert result == 'ubuntu@box1.front.sepia.ceph.com'
267
-
268
- def test_decanonicalize_hostname(self):
269
- host = 'ubuntu@box1.front.sepia.ceph.com'
270
- result = misc.decanonicalize_hostname(host)
271
- assert result == 'box1'
272
-
273
- def test_canonicalize_hostname_nouser(self):
274
- host_base = 'box1'
275
- result = misc.canonicalize_hostname(host_base, user=None)
276
- assert result == 'box1.front.sepia.ceph.com'
277
-
278
- def test_decanonicalize_hostname_nouser(self):
279
- host = 'box1.front.sepia.ceph.com'
280
- result = misc.decanonicalize_hostname(host)
281
- assert result == 'box1'
282
-
283
- def test_canonicalize_hostname_otherlab(self):
284
- config.lab_domain = 'example.com'
285
- host_base = 'box1'
286
- result = misc.canonicalize_hostname(host_base)
287
- assert result == 'ubuntu@box1.example.com'
288
-
289
- def test_decanonicalize_hostname_otherlab(self):
290
- config.lab_domain = 'example.com'
291
- host = 'ubuntu@box1.example.com'
292
- result = misc.decanonicalize_hostname(host)
293
- assert result == 'box1'
294
-
295
- def test_canonicalize_hostname_nodomain(self):
296
- config.lab_domain = ''
297
- host = 'box2'
298
- result = misc.canonicalize_hostname(host)
299
- assert result == 'ubuntu@' + host
300
-
301
- def test_decanonicalize_hostname_nodomain(self):
302
- config.lab_domain = ''
303
- host = 'ubuntu@box2'
304
- result = misc.decanonicalize_hostname(host)
305
- assert result == 'box2'
306
-
307
- def test_canonicalize_hostname_full_other_user(self):
308
- config.lab_domain = 'example.com'
309
- host = 'user1@box1.example.come'
310
- result = misc.canonicalize_hostname(host)
311
- assert result == 'user1@box1.example.com'
312
-
313
- def test_decanonicalize_hostname_full_other_user(self):
314
- config.lab_domain = 'example.com'
315
- host = 'user1@box1.example.come'
316
- result = misc.decanonicalize_hostname(host)
317
- assert result == 'box1'
318
-
319
- class TestMergeConfigs(object):
320
- """ Tests merge_config and deep_merge in teuthology.misc """
321
-
322
- @patch("os.path.exists")
323
- @patch("yaml.safe_load")
324
- @patch("teuthology.misc.open")
325
- def test_merge_configs(self, m_open, m_safe_load, m_exists):
326
- """ Only tests with one yaml file being passed, mainly just to test
327
- the loop logic. The actual merge will be tested in subsequent
328
- tests.
329
- """
330
- expected = {"a": "b", "b": "c"}
331
- m_exists.return_value = True
332
- m_safe_load.return_value = expected
333
- result = misc.merge_configs(["path/to/config1"])
334
- assert result == expected
335
- m_open.assert_called_once_with("path/to/config1")
336
-
337
- def test_merge_configs_empty(self):
338
- assert misc.merge_configs([]) == {}
339
-
340
- def test_deep_merge(self):
341
- a = {"a": "b"}
342
- b = {"b": "c"}
343
- result = misc.deep_merge(a, b)
344
- assert result == {"a": "b", "b": "c"}
345
-
346
- def test_overwrite_deep_merge(self):
347
- a = {"a": "b"}
348
- b = {"a": "overwritten", "b": "c"}
349
- result = misc.deep_merge(a, b)
350
- assert result == {"a": "overwritten", "b": "c"}
351
-
352
- def test_list_deep_merge(self):
353
- a = [1, 2]
354
- b = [3, 4]
355
- result = misc.deep_merge(a, b)
356
- assert result == [1, 2, 3, 4]
357
-
358
- def test_missing_list_deep_merge(self):
359
- a = [1, 2]
360
- b = "not a list"
361
- with pytest.raises(AssertionError):
362
- misc.deep_merge(a, b)
363
-
364
- def test_missing_a_deep_merge(self):
365
- result = misc.deep_merge(None, [1, 2])
366
- assert result == [1, 2]
367
-
368
- def test_missing_b_deep_merge(self):
369
- result = misc.deep_merge([1, 2], None)
370
- assert result == [1, 2]
371
-
372
- def test_invalid_b_deep_merge(self):
373
- with pytest.raises(AssertionError):
374
- misc.deep_merge({"a": "b"}, "invalid")
375
-
376
-
377
- class TestIsInDict(object):
378
- def test_simple_membership(self):
379
- assert misc.is_in_dict('a', 'foo', {'a':'foo', 'b':'bar'})
380
-
381
- def test_dict_membership(self):
382
- assert misc.is_in_dict(
383
- 'a', {'sub1':'key1', 'sub2':'key2'},
384
- {'a':{'sub1':'key1', 'sub2':'key2', 'sub3':'key3'}}
385
- )
386
-
387
- def test_simple_nonmembership(self):
388
- assert not misc.is_in_dict('a', 'foo', {'a':'bar', 'b':'foo'})
389
-
390
- def test_nonmembership_with_presence_at_lower_level(self):
391
- assert not misc.is_in_dict('a', 'foo', {'a':{'a': 'foo'}})
@@ -1,290 +0,0 @@
1
- import datetime
2
- import json
3
- import os
4
- import pytest
5
- import subprocess
6
-
7
- from unittest.mock import patch, Mock, DEFAULT, ANY
8
-
9
- from teuthology import nuke
10
- from teuthology import misc
11
- from teuthology.config import config
12
- from teuthology.dispatcher.supervisor import create_fake_context
13
-
14
- class TestNuke(object):
15
-
16
- #@pytest.mark.skipif('OS_AUTH_URL' not in os.environ,
17
- # reason="no OS_AUTH_URL environment variable")
18
- def test_stale_openstack_volumes(self):
19
- ctx = Mock()
20
- ctx.teuthology_config = config
21
- ctx.dry_run = False
22
- now = datetime.datetime.strftime(datetime.datetime.now(),
23
- "%Y-%m-%dT%H:%M:%S.000000")
24
- id = '4bee3af9-febb-40c1-a17e-ff63edb415c5'
25
- name = 'target1-0'
26
- volume_list = json.loads(
27
- '[{'
28
- ' "ID": "' + id + '"'
29
- '}]'
30
- )
31
- #
32
- # A volume created a second ago is left untouched
33
- #
34
- volume_show = (
35
- '{"id": "' + id + '", '
36
- '"created_at": "' + now + '", '
37
- '"display_name": "' + name + '"}'
38
- )
39
-
40
- with patch('teuthology.nuke.openstack_delete_volume') as m_os_del_vol:
41
- with patch.object(nuke.OpenStack, 'run') as m_os_run:
42
- m_os_run.return_value = volume_show
43
- nuke.stale_openstack_volumes(ctx, volume_list)
44
- m_os_del_vol.assert_not_called()
45
-
46
-
47
- #
48
- # A volume created long ago is destroyed
49
- #
50
- ancient = "2000-11-02T15:43:12.000000"
51
- volume_show = (
52
- '{"id": "' + id + '", '
53
- '"created_at": "' + ancient + '", '
54
- '"display_name": "' + name + '"}'
55
- )
56
-
57
- with patch('teuthology.nuke.openstack_delete_volume') as m_os_del_vol:
58
- with patch.object(nuke.OpenStack, 'run') as m_os_run:
59
- m_os_run.return_value = volume_show
60
- nuke.stale_openstack_volumes(ctx, volume_list)
61
- m_os_del_vol.assert_called_with(id)
62
-
63
- #
64
- # A volume that no longer exists is ignored
65
- #
66
- with patch('teuthology.nuke.openstack_delete_volume') as m_os_del_vol:
67
- with patch.object(nuke.OpenStack, 'run') as m_os_run:
68
- m_os_run.side_effect = subprocess.CalledProcessError('ERROR', 'FAIL')
69
- nuke.stale_openstack_volumes(ctx, volume_list)
70
- m_os_del_vol.assert_not_called()
71
-
72
- def test_stale_openstack_nodes(self):
73
- ctx = Mock()
74
- ctx.teuthology_config = config
75
- ctx.dry_run = False
76
- name = 'target1'
77
- uuid = 'UUID1'
78
- now = datetime.datetime.strftime(datetime.datetime.now(),
79
- "%Y-%m-%d %H:%M:%S.%f")
80
- #
81
- # A node is not of type openstack is left untouched
82
- #
83
- with patch.multiple(
84
- nuke,
85
- unlock_one=DEFAULT,
86
- ) as m:
87
- nuke.stale_openstack_nodes(ctx, {
88
- }, {
89
- name: { 'locked_since': now,
90
- 'machine_type': 'mira', },
91
- })
92
- m['unlock_one'].assert_not_called()
93
- #
94
- # A node that was just locked and does not have
95
- # an instance yet is left untouched
96
- #
97
- with patch.multiple(
98
- nuke,
99
- unlock_one=DEFAULT,
100
- ) as m:
101
- nuke.stale_openstack_nodes(ctx, {
102
- }, {
103
- name: { 'locked_since': now,
104
- 'machine_type': 'openstack', },
105
- })
106
- m['unlock_one'].assert_not_called()
107
- #
108
- # A node that has been locked for some time and
109
- # has no instance is unlocked.
110
- #
111
- ancient = "2000-11-02 15:43:12.000000"
112
- me = 'loic@dachary.org'
113
- with patch.multiple(
114
- nuke,
115
- unlock_one=DEFAULT,
116
- ) as m:
117
- nuke.stale_openstack_nodes(ctx, {
118
- }, {
119
- name: { 'locked_since': ancient,
120
- 'locked_by': me,
121
- 'machine_type': 'openstack', },
122
- })
123
- m['unlock_one'].assert_called_with(
124
- ctx, name, me)
125
- #
126
- # A node that has been locked for some time and
127
- # has an instance is left untouched
128
- #
129
- with patch.multiple(
130
- nuke,
131
- unlock_one=DEFAULT,
132
- ) as m:
133
- nuke.stale_openstack_nodes(ctx, {
134
- uuid: {
135
- 'ID': uuid,
136
- 'Name': name,
137
- },
138
- }, {
139
- name: { 'locked_since': ancient,
140
- 'machine_type': 'openstack', },
141
- })
142
- m['unlock_one'].assert_not_called()
143
-
144
- def test_stale_openstack_instances(self):
145
- if 'OS_AUTH_URL' not in os.environ:
146
- pytest.skip('no OS_AUTH_URL environment variable')
147
- ctx = Mock()
148
- ctx.teuthology_config = config
149
- ctx.dry_run = False
150
- name = 'target1'
151
- uuid = 'UUID1'
152
- #
153
- # An instance created a second ago is left untouched,
154
- # even when it is not locked.
155
- #
156
- with patch.multiple(
157
- nuke.OpenStackInstance,
158
- exists=lambda _: True,
159
- get_created=lambda _: 1,
160
- __getitem__=lambda _, key: name,
161
- destroy=DEFAULT,
162
- ) as m:
163
- nuke.stale_openstack_instances(ctx, {
164
- uuid: { 'Name': name, },
165
- }, {
166
- })
167
- m['destroy'].assert_not_called()
168
- #
169
- # An instance created a very long time ago is destroyed
170
- #
171
- with patch.multiple(
172
- nuke.OpenStackInstance,
173
- exists=lambda _: True,
174
- get_created=lambda _: 1000000000,
175
- __getitem__=lambda _, key: name,
176
- destroy=DEFAULT,
177
- ) as m:
178
- nuke.stale_openstack_instances(ctx, {
179
- uuid: { 'Name': name, },
180
- }, {
181
- misc.canonicalize_hostname(name, user=None): {},
182
- })
183
- m['destroy'].assert_called_with()
184
- #
185
- # An instance that turns out to not exist any longer
186
- # is ignored.
187
- #
188
- with patch.multiple(
189
- nuke.OpenStackInstance,
190
- exists=lambda _: False,
191
- __getitem__=lambda _, key: name,
192
- destroy=DEFAULT,
193
- ) as m:
194
- nuke.stale_openstack_instances(ctx, {
195
- uuid: { 'Name': name, },
196
- }, {
197
- misc.canonicalize_hostname(name, user=None): {},
198
- })
199
- m['destroy'].assert_not_called()
200
- #
201
- # An instance created but not locked after a while is
202
- # destroyed.
203
- #
204
- with patch.multiple(
205
- nuke.OpenStackInstance,
206
- exists=lambda _: True,
207
- get_created=lambda _: nuke.OPENSTACK_DELAY + 1,
208
- __getitem__=lambda _, key: name,
209
- destroy=DEFAULT,
210
- ) as m:
211
- nuke.stale_openstack_instances(ctx, {
212
- uuid: { 'Name': name, },
213
- }, {
214
- })
215
- m['destroy'].assert_called_with()
216
- #
217
- # An instance created within the expected lifetime
218
- # of a job and locked is left untouched.
219
- #
220
- with patch.multiple(
221
- nuke.OpenStackInstance,
222
- exists=lambda _: True,
223
- get_created=lambda _: nuke.OPENSTACK_DELAY + 1,
224
- __getitem__=lambda _, key: name,
225
- destroy=DEFAULT,
226
- ) as m:
227
- nuke.stale_openstack_instances(ctx, {
228
- uuid: { 'Name': name, },
229
- }, {
230
- misc.canonicalize_hostname(name, user=None): {},
231
- })
232
- m['destroy'].assert_not_called()
233
-
234
- def test_nuke_internal():
235
- job_config = dict(
236
- owner='test_owner',
237
- targets={'user@host1': 'key1', 'user@host2': 'key2'},
238
- archive_path='/path/to/test_run',
239
- machine_type='test_machine',
240
- os_type='centos',
241
- os_version='8.3',
242
- name='test_name',
243
- )
244
- locks = [{'name': target, 'description': job_config['name']} for target
245
- in job_config['targets'].keys()]
246
- ctx = create_fake_context(job_config)
247
-
248
- # minimal call using defaults
249
- with patch.multiple(
250
- nuke,
251
- nuke_helper=DEFAULT,
252
- list_locks=lambda: locks,
253
- unlock_one=DEFAULT,
254
- ) as m:
255
- nuke.nuke(ctx, True)
256
- m['nuke_helper'].assert_called_with(ANY, True, False, True)
257
- m['unlock_one'].assert_called()
258
-
259
- # don't unlock
260
- with patch.multiple(
261
- nuke,
262
- nuke_helper=DEFAULT,
263
- list_locks=lambda: locks,
264
- unlock_one=DEFAULT,
265
- ) as m:
266
- nuke.nuke(ctx, False)
267
- m['nuke_helper'].assert_called_with(ANY, False, False, True)
268
- m['unlock_one'].assert_not_called()
269
-
270
- # mimicing what teuthology-dispatcher --supervisor does
271
- with patch.multiple(
272
- nuke,
273
- nuke_helper=DEFAULT,
274
- list_locks=lambda: locks,
275
- unlock_one=DEFAULT,
276
- ) as m:
277
- nuke.nuke(ctx, False, True, False, True, False)
278
- m['nuke_helper'].assert_called_with(ANY, False, True, False)
279
- m['unlock_one'].assert_not_called()
280
-
281
- # no targets
282
- del ctx.config['targets']
283
- with patch.multiple(
284
- nuke,
285
- nuke_helper=DEFAULT,
286
- unlock_one=DEFAULT,
287
- ) as m:
288
- nuke.nuke(ctx, True)
289
- m['nuke_helper'].assert_not_called()
290
- m['unlock_one'].assert_not_called()