ironic-python-agent 10.0.0__py3-none-any.whl → 10.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ironic_python_agent/agent.py +21 -20
- ironic_python_agent/api/app.py +17 -1
- ironic_python_agent/burnin.py +128 -39
- ironic_python_agent/config.py +6 -1
- ironic_python_agent/device_hints.py +314 -0
- ironic_python_agent/disk_partitioner.py +5 -4
- ironic_python_agent/disk_utils.py +14 -15
- ironic_python_agent/encoding.py +0 -12
- ironic_python_agent/errors.py +37 -0
- ironic_python_agent/extensions/clean.py +1 -2
- ironic_python_agent/extensions/deploy.py +1 -2
- ironic_python_agent/extensions/rescue.py +9 -3
- ironic_python_agent/extensions/service.py +1 -2
- ironic_python_agent/extensions/standby.py +292 -7
- ironic_python_agent/hardware.py +199 -90
- ironic_python_agent/hardware_managers/nvidia/nvidia_fw_update.py +25 -25
- ironic_python_agent/inject_files.py +1 -2
- ironic_python_agent/inspect.py +1 -2
- ironic_python_agent/inspector.py +1 -1
- ironic_python_agent/mdns.py +211 -0
- ironic_python_agent/metrics_lib/__init__.py +0 -0
- ironic_python_agent/metrics_lib/metrics.py +5 -5
- ironic_python_agent/metrics_lib/metrics_utils.py +3 -3
- ironic_python_agent/netutils.py +18 -0
- ironic_python_agent/partition_utils.py +20 -22
- ironic_python_agent/qemu_img.py +2 -2
- ironic_python_agent/raid_utils.py +4 -4
- ironic_python_agent/tests/functional/base.py +1 -1
- ironic_python_agent/tests/functional/test_commands.py +4 -2
- ironic_python_agent/tests/unit/base.py +3 -4
- ironic_python_agent/tests/unit/extensions/test_image.py +2 -2
- ironic_python_agent/tests/unit/extensions/test_rescue.py +25 -5
- ironic_python_agent/tests/unit/extensions/test_standby.py +476 -7
- ironic_python_agent/tests/unit/metrics_lib/test_metrics_utils.py +4 -4
- ironic_python_agent/tests/unit/test_agent.py +6 -7
- ironic_python_agent/tests/unit/test_api.py +96 -0
- ironic_python_agent/tests/unit/test_base.py +8 -12
- ironic_python_agent/tests/unit/test_burnin.py +41 -7
- ironic_python_agent/tests/unit/test_device_hints.py +330 -0
- ironic_python_agent/tests/unit/test_disk_partitioner.py +6 -7
- ironic_python_agent/tests/unit/test_disk_utils.py +25 -14
- ironic_python_agent/tests/unit/test_encoding.py +0 -9
- ironic_python_agent/tests/unit/test_hardware.py +358 -157
- ironic_python_agent/tests/unit/test_inspector.py +1 -1
- ironic_python_agent/tests/unit/test_mdns.py +234 -0
- ironic_python_agent/tests/unit/test_netutils.py +24 -0
- ironic_python_agent/tests/unit/test_partition_utils.py +29 -19
- ironic_python_agent/tests/unit/test_qemu_img.py +8 -8
- ironic_python_agent/tests/unit/test_raid_utils.py +8 -9
- ironic_python_agent/tests/unit/test_utils.py +195 -25
- ironic_python_agent/utils.py +169 -11
- {ironic_python_agent-10.0.0.dist-info → ironic_python_agent-10.2.0.dist-info}/AUTHORS +2 -0
- {ironic_python_agent-10.0.0.dist-info → ironic_python_agent-10.2.0.dist-info}/METADATA +26 -27
- {ironic_python_agent-10.0.0.dist-info → ironic_python_agent-10.2.0.dist-info}/RECORD +59 -55
- {ironic_python_agent-10.0.0.dist-info → ironic_python_agent-10.2.0.dist-info}/WHEEL +1 -1
- {ironic_python_agent-10.0.0.dist-info → ironic_python_agent-10.2.0.dist-info}/entry_points.txt +0 -1
- ironic_python_agent-10.2.0.dist-info/pbr.json +1 -0
- ironic_python_agent/metrics_lib/metrics_exception.py +0 -34
- ironic_python_agent-10.0.0.dist-info/pbr.json +0 -1
- {ironic_python_agent-10.0.0.dist-info → ironic_python_agent-10.2.0.dist-info}/LICENSE +0 -0
- {ironic_python_agent-10.0.0.dist-info → ironic_python_agent-10.2.0.dist-info}/top_level.txt +0 -0
ironic_python_agent/agent.py
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import collections
|
|
16
16
|
import importlib.metadata
|
|
17
17
|
import ipaddress
|
|
18
|
+
import os
|
|
18
19
|
import random
|
|
19
20
|
import socket
|
|
20
21
|
import threading
|
|
@@ -22,8 +23,6 @@ import time
|
|
|
22
23
|
from urllib import parse as urlparse
|
|
23
24
|
|
|
24
25
|
import eventlet
|
|
25
|
-
from ironic_lib import exception as lib_exc
|
|
26
|
-
from ironic_lib import mdns
|
|
27
26
|
from oslo_concurrency import processutils
|
|
28
27
|
from oslo_config import cfg
|
|
29
28
|
from oslo_log import log
|
|
@@ -36,6 +35,7 @@ from ironic_python_agent.extensions import base
|
|
|
36
35
|
from ironic_python_agent import hardware
|
|
37
36
|
from ironic_python_agent import inspector
|
|
38
37
|
from ironic_python_agent import ironic_api_client
|
|
38
|
+
from ironic_python_agent import mdns
|
|
39
39
|
from ironic_python_agent import netutils
|
|
40
40
|
from ironic_python_agent import utils
|
|
41
41
|
|
|
@@ -48,9 +48,6 @@ NETWORK_WAIT_TIMEOUT = 60
|
|
|
48
48
|
# Time(in seconds) to wait before reattempt
|
|
49
49
|
NETWORK_WAIT_RETRY = 5
|
|
50
50
|
|
|
51
|
-
cfg.CONF.import_group('metrics', 'ironic_lib.metrics_utils')
|
|
52
|
-
cfg.CONF.import_group('metrics_statsd', 'ironic_lib.metrics_statsd')
|
|
53
|
-
|
|
54
51
|
Host = collections.namedtuple('Host', ['hostname', 'port'])
|
|
55
52
|
|
|
56
53
|
|
|
@@ -213,7 +210,7 @@ class IronicPythonAgent(base.ExecuteCommandMixin):
|
|
|
213
210
|
if (not api_url or api_url == 'mdns') and not standalone:
|
|
214
211
|
try:
|
|
215
212
|
api_url, params = mdns.get_endpoint('baremetal')
|
|
216
|
-
except
|
|
213
|
+
except errors.ServiceLookupFailure:
|
|
217
214
|
if api_url:
|
|
218
215
|
# mDNS explicitly requested, report failure.
|
|
219
216
|
raise
|
|
@@ -272,19 +269,11 @@ class IronicPythonAgent(base.ExecuteCommandMixin):
|
|
|
272
269
|
|
|
273
270
|
def validate_agent_token(self, token):
|
|
274
271
|
# We did not get a token, i.e. None and
|
|
275
|
-
# we've
|
|
276
|
-
#
|
|
277
|
-
if (not token
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
LOG.warning('Agent token for requests are not required '
|
|
281
|
-
'by the conductor, yet we received a token. '
|
|
282
|
-
'Cluster may be mid-upgrade. Support to '
|
|
283
|
-
'not fail in this condition will be removed in '
|
|
284
|
-
'the Victoria development cycle.')
|
|
285
|
-
# Tell the API everything is okay.
|
|
286
|
-
return True
|
|
287
|
-
|
|
272
|
+
# or we've not seen a token yet, so we
|
|
273
|
+
# cannot make a comparison, thus False.
|
|
274
|
+
if (not token or not self.agent_token):
|
|
275
|
+
return False
|
|
276
|
+
# Otherwise, compare the values.
|
|
288
277
|
return self.agent_token == token
|
|
289
278
|
|
|
290
279
|
def _get_route_source(self, dest):
|
|
@@ -568,7 +557,19 @@ class IronicPythonAgent(base.ExecuteCommandMixin):
|
|
|
568
557
|
LOG.error('Neither ipa-api-url nor inspection_callback_url'
|
|
569
558
|
'found, please check your pxe append parameters.')
|
|
570
559
|
|
|
571
|
-
|
|
560
|
+
in_rescued_mode = os.path.exists('/etc/.rescued')
|
|
561
|
+
if not in_rescued_mode:
|
|
562
|
+
self.serve_ipa_api()
|
|
563
|
+
else:
|
|
564
|
+
# NOTE(cid): In rescued state, we don't call _lockdown_system() as
|
|
565
|
+
# it brings down network interfaces which should be preserved for
|
|
566
|
+
# rescue operations.
|
|
567
|
+
LOG.info('Found rescue state marker file, locking down IPA '
|
|
568
|
+
'in disabled mode')
|
|
569
|
+
self.heartbeater.stop()
|
|
570
|
+
self.serve_api = False
|
|
571
|
+
while True:
|
|
572
|
+
time.sleep(100)
|
|
572
573
|
|
|
573
574
|
if not self.standalone and self.api_urls:
|
|
574
575
|
self.heartbeater.stop()
|
ironic_python_agent/api/app.py
CHANGED
|
@@ -113,6 +113,7 @@ class Application(object):
|
|
|
113
113
|
routing.Rule('/commands/', endpoint='run_command',
|
|
114
114
|
methods=['POST']),
|
|
115
115
|
])
|
|
116
|
+
self.security_get_token_support = False
|
|
116
117
|
|
|
117
118
|
def __call__(self, environ, start_response):
|
|
118
119
|
"""WSGI entry point."""
|
|
@@ -199,11 +200,27 @@ class Application(object):
|
|
|
199
200
|
status = self.agent.get_status()
|
|
200
201
|
return jsonify(status)
|
|
201
202
|
|
|
203
|
+
def require_agent_token_for_command(func):
|
|
204
|
+
def wrapper(self, request, *args, **kwargs):
|
|
205
|
+
token = request.args.get('agent_token', None)
|
|
206
|
+
if token:
|
|
207
|
+
# TODO(TheJulia): At some point down the road, remove the
|
|
208
|
+
# self.security_get_token_support flag and use the same
|
|
209
|
+
# decorator for the api_run_command endpoint.
|
|
210
|
+
self.security_get_token_support = True
|
|
211
|
+
if (self.security_get_token_support
|
|
212
|
+
and not self.agent.validate_agent_token(token)):
|
|
213
|
+
raise http_exc.Unauthorized('Token invalid.')
|
|
214
|
+
return func(self, request, *args, **kwargs)
|
|
215
|
+
return wrapper
|
|
216
|
+
|
|
217
|
+
@require_agent_token_for_command
|
|
202
218
|
def api_list_commands(self, request):
|
|
203
219
|
with metrics_utils.get_metrics_logger(__name__).timer('list_commands'):
|
|
204
220
|
results = self.agent.list_command_results()
|
|
205
221
|
return jsonify({'commands': results})
|
|
206
222
|
|
|
223
|
+
@require_agent_token_for_command
|
|
207
224
|
def api_get_command(self, request, cmd):
|
|
208
225
|
with metrics_utils.get_metrics_logger(__name__).timer('get_command'):
|
|
209
226
|
result = self.agent.get_command_result(cmd)
|
|
@@ -219,7 +236,6 @@ class Application(object):
|
|
|
219
236
|
if ('name' not in body or 'params' not in body
|
|
220
237
|
or not isinstance(body['params'], dict)):
|
|
221
238
|
raise http_exc.BadRequest('Missing or invalid name or params')
|
|
222
|
-
|
|
223
239
|
token = request.args.get('agent_token', None)
|
|
224
240
|
if not self.agent.validate_agent_token(token):
|
|
225
241
|
raise http_exc.Unauthorized(
|
ironic_python_agent/burnin.py
CHANGED
|
@@ -14,13 +14,13 @@ import json
|
|
|
14
14
|
import socket
|
|
15
15
|
import time
|
|
16
16
|
|
|
17
|
-
from ironic_lib import utils
|
|
18
17
|
from oslo_concurrency import processutils
|
|
19
18
|
from oslo_log import log
|
|
20
19
|
from tooz import coordination
|
|
21
20
|
|
|
22
21
|
from ironic_python_agent import errors
|
|
23
22
|
from ironic_python_agent import hardware
|
|
23
|
+
from ironic_python_agent import utils
|
|
24
24
|
|
|
25
25
|
LOG = log.getLogger(__name__)
|
|
26
26
|
|
|
@@ -28,71 +28,79 @@ NETWORK_BURNIN_ROLES = frozenset(['writer', 'reader'])
|
|
|
28
28
|
NETWORK_READER_CYCLE = 30
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
def
|
|
32
|
-
"""
|
|
31
|
+
def stress_ng(node, stressor_type, default_timeout=86400):
|
|
32
|
+
"""Run stress-ng for different stressor types
|
|
33
33
|
|
|
34
|
-
|
|
35
|
-
a configurable amount of time
|
|
36
|
-
all CPUs and stress them for 24 hours.
|
|
34
|
+
Burn-in a configurable number of CPU/VM with stress-ng,
|
|
35
|
+
for a configurable amount of time but default of 24 hours.
|
|
37
36
|
|
|
38
37
|
:param node: Ironic node object
|
|
38
|
+
:param stressor_type: 'cpu' or 'vm'
|
|
39
|
+
:param default_timeout: Default timeout in seconds (default: 86400)
|
|
40
|
+
|
|
41
|
+
:raises: ValueError if an unknown stressor_type is provided
|
|
39
42
|
:raises: CommandExecutionError if the execution of stress-ng fails.
|
|
40
43
|
"""
|
|
44
|
+
stressor_type = stressor_type.lower()
|
|
45
|
+
if stressor_type not in ['cpu', 'vm']:
|
|
46
|
+
raise ValueError("Unknown stressor type: %s" % stressor_type)
|
|
47
|
+
|
|
41
48
|
info = node.get('driver_info', {})
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
49
|
+
stressor_suffix = {'cpu': 'cpu'}
|
|
50
|
+
|
|
51
|
+
if stressor_type == 'vm':
|
|
52
|
+
count_key = 'agent_burnin_%s_vm' % stressor_type
|
|
53
|
+
bytes_key = 'agent_burnin_%s_vm-bytes' % stressor_type
|
|
54
|
+
else:
|
|
55
|
+
count_key = 'agent_burnin_%s_%s' % (stressor_type,
|
|
56
|
+
stressor_suffix[stressor_type])
|
|
57
|
+
bytes_key = None
|
|
58
|
+
|
|
59
|
+
timeout_key = 'agent_burnin_%s_timeout' % stressor_type
|
|
60
|
+
outputfile_key = 'agent_burnin_%s_outputfile' % stressor_type
|
|
61
|
+
|
|
62
|
+
count = info.get(count_key, 0)
|
|
63
|
+
timeout = info.get(timeout_key, default_timeout)
|
|
64
|
+
outputfile = info.get(outputfile_key)
|
|
65
|
+
|
|
66
|
+
args = ['stress-ng', '--%s' % stressor_type, count, '--timeout', timeout]
|
|
67
|
+
|
|
68
|
+
if stressor_type == 'vm':
|
|
69
|
+
vm_bytes = info.get(bytes_key, '98%')
|
|
70
|
+
args.extend(['--vm-bytes', vm_bytes])
|
|
71
|
+
|
|
72
|
+
args.extend(['--metrics-brief'])
|
|
45
73
|
|
|
46
|
-
args = ('stress-ng', '--cpu', cpu, '--timeout', timeout,
|
|
47
|
-
'--metrics-brief')
|
|
48
74
|
if outputfile:
|
|
49
|
-
args
|
|
75
|
+
args.extend(['--log-file', outputfile])
|
|
50
76
|
|
|
51
|
-
LOG.debug('Burn-in
|
|
77
|
+
LOG.debug('Burn-in stress_ng_%s command: %s', stressor_type, args)
|
|
52
78
|
|
|
53
79
|
try:
|
|
54
80
|
_, err = utils.execute(*args)
|
|
55
81
|
# stress-ng reports on stderr only
|
|
56
82
|
LOG.info(err)
|
|
57
83
|
except (processutils.ProcessExecutionError, OSError) as e:
|
|
58
|
-
error_msg =
|
|
84
|
+
error_msg = 'stress-ng (%s) failed with error %s' % (stressor_type, e)
|
|
59
85
|
LOG.error(error_msg)
|
|
60
86
|
raise errors.CommandExecutionError(error_msg)
|
|
61
87
|
|
|
62
88
|
|
|
89
|
+
def stress_ng_cpu(node):
|
|
90
|
+
"""Burn-in the CPU with stress-ng"""
|
|
91
|
+
stress_ng(node, 'cpu')
|
|
92
|
+
|
|
93
|
+
|
|
63
94
|
def stress_ng_vm(node):
|
|
64
|
-
"""Burn-in the memory with the vm stressor in stress-ng
|
|
95
|
+
"""Burn-in the memory with the vm stressor in stress-ng.
|
|
65
96
|
|
|
66
97
|
Run stress-ng with a configurable number of workers on
|
|
67
98
|
a configurable amount of the available memory for
|
|
68
99
|
a configurable amount of time. Without config use
|
|
69
100
|
as many workers as CPUs, 98% of the memory and stress
|
|
70
101
|
it for 24 hours.
|
|
71
|
-
|
|
72
|
-
:param node: Ironic node object
|
|
73
|
-
:raises: CommandExecutionError if the execution of stress-ng fails.
|
|
74
102
|
"""
|
|
75
|
-
|
|
76
|
-
vm = info.get('agent_burnin_vm_vm', 0)
|
|
77
|
-
vm_bytes = info.get('agent_burnin_vm_vm-bytes', '98%')
|
|
78
|
-
timeout = info.get('agent_burnin_vm_timeout', 86400)
|
|
79
|
-
outputfile = info.get('agent_burnin_vm_outputfile', None)
|
|
80
|
-
|
|
81
|
-
args = ('stress-ng', '--vm', vm, '--vm-bytes', vm_bytes,
|
|
82
|
-
'--timeout', timeout, '--metrics-brief')
|
|
83
|
-
if outputfile:
|
|
84
|
-
args += ('--log-file', outputfile,)
|
|
85
|
-
|
|
86
|
-
LOG.debug('Burn-in stress_ng_vm command: %s', args)
|
|
87
|
-
|
|
88
|
-
try:
|
|
89
|
-
_, err = utils.execute(*args)
|
|
90
|
-
# stress-ng reports on stderr only
|
|
91
|
-
LOG.info(err)
|
|
92
|
-
except (processutils.ProcessExecutionError, OSError) as e:
|
|
93
|
-
error_msg = "stress-ng (vm) failed with error %s" % e
|
|
94
|
-
LOG.error(error_msg)
|
|
95
|
-
raise errors.CommandExecutionError(error_msg)
|
|
103
|
+
stress_ng(node, 'vm')
|
|
96
104
|
|
|
97
105
|
|
|
98
106
|
def _smart_test_status(device):
|
|
@@ -420,3 +428,84 @@ def fio_network(node):
|
|
|
420
428
|
irole = "reader" if (role == "writer") else "writer"
|
|
421
429
|
logfilename = outputfile + '.' + irole
|
|
422
430
|
_do_fio_network(not role == 'writer', runtime, partner, logfilename)
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _gpu_burn_check_count(install_dir, count):
|
|
434
|
+
"""Check the count of GPUs with gpu-burn
|
|
435
|
+
|
|
436
|
+
Run a check to confirm how many GPUs are seen by the OS.
|
|
437
|
+
|
|
438
|
+
:param install_dir: The location where gpu-burn has been installed.
|
|
439
|
+
:param count: The number of expected GPUs.
|
|
440
|
+
|
|
441
|
+
:raises: CleaningError if the incorrect number of GPUs found.
|
|
442
|
+
:raises: CommandExecutionError if the execution of gpu-burn fails.
|
|
443
|
+
|
|
444
|
+
"""
|
|
445
|
+
args = ['./gpu_burn', '-l']
|
|
446
|
+
LOG.debug('Burn-in gpu count command: %s', args)
|
|
447
|
+
try:
|
|
448
|
+
out, _ = utils.execute(*args, cwd=install_dir)
|
|
449
|
+
# gpu-burn reports on stdout
|
|
450
|
+
LOG.debug(out)
|
|
451
|
+
except (processutils.ProcessExecutionError, OSError) as e:
|
|
452
|
+
error_msg = 'gpu-burn failed with error %s' % e
|
|
453
|
+
LOG.error(error_msg)
|
|
454
|
+
raise errors.CommandExecutionError(error_msg)
|
|
455
|
+
|
|
456
|
+
gpu_data = [i for i in out.splitlines() if i.startswith('ID')]
|
|
457
|
+
gpu_count = len(gpu_data)
|
|
458
|
+
if gpu_count != count:
|
|
459
|
+
error_msg = ("gpu-burn failed to find the correct number of gpus. "
|
|
460
|
+
"%s found but %s expected." % (gpu_count, count))
|
|
461
|
+
LOG.error(error_msg)
|
|
462
|
+
raise errors.CleaningError(error_msg)
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _gpu_burn_run(install_dir, memory, timeout=86400):
|
|
466
|
+
"""Burn-in the GPU with gpu-burn
|
|
467
|
+
|
|
468
|
+
Run a GPU burn-in job for a configurable amount of time.
|
|
469
|
+
|
|
470
|
+
:param install_dir: The location where gpu-burn has been installed.
|
|
471
|
+
:param memory: Use N% or X MB of the available GPU memory.
|
|
472
|
+
:param timeout: Timeout in seconds (default: 86400).
|
|
473
|
+
|
|
474
|
+
:raises: CommandExecutionError if the execution of gpu-burn fails.
|
|
475
|
+
"""
|
|
476
|
+
|
|
477
|
+
args = ['./gpu_burn', '-m', memory, timeout]
|
|
478
|
+
LOG.debug('Burn-in gpu command: %s', args)
|
|
479
|
+
try:
|
|
480
|
+
out, _ = utils.execute(*args, cwd=install_dir)
|
|
481
|
+
# gpu-burn reports on stdout
|
|
482
|
+
LOG.debug(out)
|
|
483
|
+
except (processutils.ProcessExecutionError, OSError) as e:
|
|
484
|
+
error_msg = 'gpu-burn failed with error %s' % e
|
|
485
|
+
LOG.error(error_msg)
|
|
486
|
+
raise errors.CommandExecutionError(error_msg)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def gpu_burn(node):
|
|
490
|
+
"""Burn-in and check correct count of GPUs using gpu-burn
|
|
491
|
+
|
|
492
|
+
Check that the expected number of GPUs are available on the node
|
|
493
|
+
and run a GPU burn-in job for a configurable amount of time.
|
|
494
|
+
|
|
495
|
+
:param node: Ironic node object
|
|
496
|
+
"""
|
|
497
|
+
info = node.get('driver_info', {})
|
|
498
|
+
|
|
499
|
+
install_dir = info.get('agent_burnin_gpu_install_dir', '/opt/gpu-burn')
|
|
500
|
+
timeout = info.get('agent_burnin_gpu_timeout', 86400)
|
|
501
|
+
memory = info.get('agent_burnin_gpu_memory', '95%')
|
|
502
|
+
count = info.get('agent_burnin_gpu_count', 0)
|
|
503
|
+
|
|
504
|
+
# Only check count if an expected number of GPUs has been configured
|
|
505
|
+
if count > 0:
|
|
506
|
+
_gpu_burn_check_count(install_dir, count)
|
|
507
|
+
else:
|
|
508
|
+
LOG.debug("Burn-in gpu skipping expected number of GPUs check as "
|
|
509
|
+
"'agent_burnin_gpu_count' set to 0")
|
|
510
|
+
|
|
511
|
+
_gpu_burn_run(install_dir, memory, timeout)
|
ironic_python_agent/config.py
CHANGED
|
@@ -256,7 +256,7 @@ cli_opts = [
|
|
|
256
256
|
cfg.StrOpt('ntp_server',
|
|
257
257
|
default=APARAMS.get('ipa-ntp-server', None),
|
|
258
258
|
help='Address of a single NTP server against which the '
|
|
259
|
-
'agent should sync the
|
|
259
|
+
'agent should sync the system software clock prior to '
|
|
260
260
|
'rebooting to an instance.'),
|
|
261
261
|
cfg.BoolOpt('fail_if_clock_not_set',
|
|
262
262
|
default=False,
|
|
@@ -388,6 +388,11 @@ cli_opts = [
|
|
|
388
388
|
'image validation logic will fail the deployment '
|
|
389
389
|
'process. This check is skipped if deep image '
|
|
390
390
|
'inspection is disabled.'),
|
|
391
|
+
cfg.BoolOpt('disable_bootc_deploy',
|
|
392
|
+
default=False,
|
|
393
|
+
help='This disables bootc deployment methods in the ramdisk '
|
|
394
|
+
'because the bootc command inside of the ramdisk '
|
|
395
|
+
'comes from the supplied image to be deployed.'),
|
|
391
396
|
]
|
|
392
397
|
|
|
393
398
|
disk_utils_opts = [
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
2
|
+
# you may not use this file except in compliance with the License.
|
|
3
|
+
# You may obtain a copy of the License at
|
|
4
|
+
#
|
|
5
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
#
|
|
7
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
8
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
9
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
10
|
+
# See the License for the specific language governing permissions and
|
|
11
|
+
# limitations under the License.
|
|
12
|
+
import copy
|
|
13
|
+
import re
|
|
14
|
+
from urllib import parse as urlparse
|
|
15
|
+
|
|
16
|
+
from oslo_log import log as logging
|
|
17
|
+
from oslo_utils import specs_matcher
|
|
18
|
+
from oslo_utils import strutils
|
|
19
|
+
from oslo_utils import units
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
LOG = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# A dictionary in the form {hint name: hint type}
|
|
26
|
+
VALID_ROOT_DEVICE_HINTS = {
|
|
27
|
+
'size': int, 'model': str, 'wwn': str, 'serial': str, 'vendor': str,
|
|
28
|
+
'wwn_with_extension': str, 'wwn_vendor_extension': str, 'name': str,
|
|
29
|
+
'rotational': bool, 'hctl': str, 'by_path': str,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
ROOT_DEVICE_HINTS_GRAMMAR = specs_matcher.make_grammar()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _extract_hint_operator_and_values(hint_expression, hint_name):
|
|
36
|
+
"""Extract the operator and value(s) of a root device hint expression.
|
|
37
|
+
|
|
38
|
+
A root device hint expression could contain one or more values
|
|
39
|
+
depending on the operator. This method extracts the operator and
|
|
40
|
+
value(s) and returns a dictionary containing both.
|
|
41
|
+
|
|
42
|
+
:param hint_expression: The hint expression string containing value(s)
|
|
43
|
+
and operator (optionally).
|
|
44
|
+
:param hint_name: The name of the hint. Used for logging.
|
|
45
|
+
:raises: ValueError if the hint_expression is empty.
|
|
46
|
+
:returns: A dictionary containing:
|
|
47
|
+
|
|
48
|
+
:op: The operator. An empty string in case of None.
|
|
49
|
+
:values: A list of values stripped and converted to lowercase.
|
|
50
|
+
"""
|
|
51
|
+
expression = str(hint_expression).strip().lower()
|
|
52
|
+
if not expression:
|
|
53
|
+
raise ValueError(f'Root device hint {hint_name} expression is empty')
|
|
54
|
+
|
|
55
|
+
# parseString() returns a list of tokens which the operator (if
|
|
56
|
+
# present) is always the first element.
|
|
57
|
+
ast = ROOT_DEVICE_HINTS_GRAMMAR.parseString(expression)
|
|
58
|
+
if len(ast) <= 1:
|
|
59
|
+
# hint_expression had no operator
|
|
60
|
+
return {'op': '', 'values': [expression]}
|
|
61
|
+
|
|
62
|
+
op = ast[0]
|
|
63
|
+
return {'values': [v.strip() for v in re.split(op, expression) if v],
|
|
64
|
+
'op': op}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _normalize_hint_expression(hint_expression, hint_name):
|
|
68
|
+
"""Normalize a string type hint expression.
|
|
69
|
+
|
|
70
|
+
A string-type hint expression contains one or more operators and
|
|
71
|
+
one or more values: [<op>] <value> [<op> <value>]*. This normalizes
|
|
72
|
+
the values by url-encoding white spaces and special characters. The
|
|
73
|
+
operators are not normalized. For example: the hint value of "<or>
|
|
74
|
+
foo bar <or> bar" will become "<or> foo%20bar <or> bar".
|
|
75
|
+
|
|
76
|
+
:param hint_expression: The hint expression string containing value(s)
|
|
77
|
+
and operator (optionally).
|
|
78
|
+
:param hint_name: The name of the hint. Used for logging.
|
|
79
|
+
:raises: ValueError if the hint_expression is empty.
|
|
80
|
+
:returns: A normalized string.
|
|
81
|
+
"""
|
|
82
|
+
hdict = _extract_hint_operator_and_values(hint_expression, hint_name)
|
|
83
|
+
result = hdict['op'].join([' %s ' % urlparse.quote(t)
|
|
84
|
+
for t in hdict['values']])
|
|
85
|
+
return (hdict['op'] + result).strip()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _append_operator_to_hints(root_device):
|
|
89
|
+
"""Add an equal (s== or ==) operator to the hints.
|
|
90
|
+
|
|
91
|
+
For backwards compatibility, for root device hints where no operator
|
|
92
|
+
means equal, this method adds the equal operator to the hint. This is
|
|
93
|
+
needed when using oslo.utils.specs_matcher methods.
|
|
94
|
+
|
|
95
|
+
:param root_device: The root device hints dictionary.
|
|
96
|
+
"""
|
|
97
|
+
for name, expression in root_device.items():
|
|
98
|
+
# NOTE(lucasagomes): The specs_matcher from oslo.utils does not
|
|
99
|
+
# support boolean, so we don't need to append any operator
|
|
100
|
+
# for it.
|
|
101
|
+
if VALID_ROOT_DEVICE_HINTS[name] is bool:
|
|
102
|
+
continue
|
|
103
|
+
|
|
104
|
+
expression = str(expression)
|
|
105
|
+
ast = ROOT_DEVICE_HINTS_GRAMMAR.parseString(expression)
|
|
106
|
+
if len(ast) > 1:
|
|
107
|
+
continue
|
|
108
|
+
|
|
109
|
+
op = 's== %s' if VALID_ROOT_DEVICE_HINTS[name] is str else '== %s'
|
|
110
|
+
root_device[name] = op % expression
|
|
111
|
+
|
|
112
|
+
return root_device
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def parse_root_device_hints(root_device):
|
|
116
|
+
"""Parse the root_device property of a node.
|
|
117
|
+
|
|
118
|
+
Parses and validates the root_device property of a node. These are
|
|
119
|
+
hints for how a node's root device is created. The 'size' hint
|
|
120
|
+
should be a positive integer. The 'rotational' hint should be a
|
|
121
|
+
Boolean value.
|
|
122
|
+
|
|
123
|
+
:param root_device: the root_device dictionary from the node's property.
|
|
124
|
+
:returns: a dictionary with the root device hints parsed or
|
|
125
|
+
None if there are no hints.
|
|
126
|
+
:raises: ValueError, if some information is invalid.
|
|
127
|
+
|
|
128
|
+
"""
|
|
129
|
+
if not root_device:
|
|
130
|
+
return
|
|
131
|
+
|
|
132
|
+
root_device = copy.deepcopy(root_device)
|
|
133
|
+
|
|
134
|
+
invalid_hints = set(root_device) - set(VALID_ROOT_DEVICE_HINTS)
|
|
135
|
+
if invalid_hints:
|
|
136
|
+
raise ValueError('The hints "%(invalid_hints)s" are invalid. '
|
|
137
|
+
'Valid hints are: "%(valid_hints)s"' %
|
|
138
|
+
{'invalid_hints': ', '.join(invalid_hints),
|
|
139
|
+
'valid_hints': ', '.join(VALID_ROOT_DEVICE_HINTS)})
|
|
140
|
+
|
|
141
|
+
for name, expression in root_device.items():
|
|
142
|
+
hint_type = VALID_ROOT_DEVICE_HINTS[name]
|
|
143
|
+
if hint_type is str:
|
|
144
|
+
if not isinstance(expression, str):
|
|
145
|
+
raise ValueError(
|
|
146
|
+
'Root device hint "%(name)s" is not a string value. '
|
|
147
|
+
'Hint expression: %(expression)s' %
|
|
148
|
+
{'name': name, 'expression': expression})
|
|
149
|
+
root_device[name] = _normalize_hint_expression(expression, name)
|
|
150
|
+
|
|
151
|
+
elif hint_type is int:
|
|
152
|
+
for v in _extract_hint_operator_and_values(expression,
|
|
153
|
+
name)['values']:
|
|
154
|
+
try:
|
|
155
|
+
integer = int(v)
|
|
156
|
+
except ValueError:
|
|
157
|
+
raise ValueError(
|
|
158
|
+
'Root device hint "%(name)s" is not an integer '
|
|
159
|
+
'value. Current value: %(expression)s' %
|
|
160
|
+
{'name': name, 'expression': expression})
|
|
161
|
+
|
|
162
|
+
if integer <= 0:
|
|
163
|
+
raise ValueError(
|
|
164
|
+
'Root device hint "%(name)s" should be a positive '
|
|
165
|
+
'integer. Current value: %(expression)s' %
|
|
166
|
+
{'name': name, 'expression': expression})
|
|
167
|
+
|
|
168
|
+
elif hint_type is bool:
|
|
169
|
+
try:
|
|
170
|
+
root_device[name] = strutils.bool_from_string(
|
|
171
|
+
expression, strict=True)
|
|
172
|
+
except ValueError:
|
|
173
|
+
raise ValueError(
|
|
174
|
+
'Root device hint "%(name)s" is not a Boolean value. '
|
|
175
|
+
'Current value: %(expression)s' %
|
|
176
|
+
{'name': name, 'expression': expression})
|
|
177
|
+
|
|
178
|
+
return _append_operator_to_hints(root_device)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def find_devices_by_hints(devices, root_device_hints):
|
|
182
|
+
"""Find all devices that match the root device hints.
|
|
183
|
+
|
|
184
|
+
Try to find devices that match the root device hints. In order
|
|
185
|
+
for a device to be matched it needs to satisfy all the given hints.
|
|
186
|
+
|
|
187
|
+
:param devices: A list of dictionaries representing the devices
|
|
188
|
+
containing one or more of the following keys:
|
|
189
|
+
|
|
190
|
+
:name: (String) The device name, e.g /dev/sda
|
|
191
|
+
:size: (Integer) Size of the device in *bytes*
|
|
192
|
+
:model: (String) Device model
|
|
193
|
+
:vendor: (String) Device vendor name
|
|
194
|
+
:serial: (String) Device serial number
|
|
195
|
+
:wwn: (String) Unique storage identifier
|
|
196
|
+
:wwn_with_extension: (String): Unique storage identifier with
|
|
197
|
+
the vendor extension appended
|
|
198
|
+
:wwn_vendor_extension: (String): United vendor storage identifier
|
|
199
|
+
:rotational: (Boolean) Whether it's a rotational device or
|
|
200
|
+
not. Useful to distinguish HDDs (rotational) and SSDs
|
|
201
|
+
(not rotational).
|
|
202
|
+
:hctl: (String): The SCSI address: Host, channel, target and lun.
|
|
203
|
+
For example: '1:0:0:0'.
|
|
204
|
+
:by_path: (String): The alternative device name,
|
|
205
|
+
e.g. /dev/disk/by-path/pci-0000:00
|
|
206
|
+
|
|
207
|
+
:param root_device_hints: A dictionary with the root device hints.
|
|
208
|
+
:raises: ValueError, if some information is invalid.
|
|
209
|
+
:returns: A generator with all matching devices as dictionaries.
|
|
210
|
+
"""
|
|
211
|
+
LOG.debug('Trying to find devices from "%(devs)s" that match the '
|
|
212
|
+
'device hints "%(hints)s"',
|
|
213
|
+
{'devs': ', '.join([d.get('name') for d in devices]),
|
|
214
|
+
'hints': root_device_hints})
|
|
215
|
+
parsed_hints = parse_root_device_hints(root_device_hints)
|
|
216
|
+
for dev in devices:
|
|
217
|
+
device_name = dev.get('name')
|
|
218
|
+
|
|
219
|
+
for hint in parsed_hints:
|
|
220
|
+
hint_type = VALID_ROOT_DEVICE_HINTS[hint]
|
|
221
|
+
device_value = dev.get(hint)
|
|
222
|
+
hint_value = parsed_hints[hint]
|
|
223
|
+
|
|
224
|
+
if hint_type is str:
|
|
225
|
+
try:
|
|
226
|
+
device_value = _normalize_hint_expression(device_value,
|
|
227
|
+
hint)
|
|
228
|
+
except ValueError:
|
|
229
|
+
LOG.warning(
|
|
230
|
+
'The attribute "%(attr)s" of the device "%(dev)s" '
|
|
231
|
+
'has an empty value. Skipping device.',
|
|
232
|
+
{'attr': hint, 'dev': device_name})
|
|
233
|
+
break
|
|
234
|
+
|
|
235
|
+
if hint == 'size':
|
|
236
|
+
# Since we don't support units yet we expect the size
|
|
237
|
+
# in GiB for now
|
|
238
|
+
device_value = device_value / units.Gi
|
|
239
|
+
|
|
240
|
+
LOG.debug('Trying to match the device hint "%(hint)s" '
|
|
241
|
+
'with a value of "%(hint_value)s" against the same '
|
|
242
|
+
'device\'s (%(dev)s) attribute with a value of '
|
|
243
|
+
'"%(dev_value)s"', {'hint': hint, 'dev': device_name,
|
|
244
|
+
'hint_value': hint_value,
|
|
245
|
+
'dev_value': device_value})
|
|
246
|
+
|
|
247
|
+
# NOTE(lucasagomes): Boolean hints are not supported by
|
|
248
|
+
# specs_matcher.match(), so we need to do the comparison
|
|
249
|
+
# ourselves
|
|
250
|
+
if hint_type is bool:
|
|
251
|
+
try:
|
|
252
|
+
device_value = strutils.bool_from_string(device_value,
|
|
253
|
+
strict=True)
|
|
254
|
+
except ValueError:
|
|
255
|
+
LOG.warning('The attribute "%(attr)s" (with value '
|
|
256
|
+
'"%(value)s") of device "%(dev)s" is not '
|
|
257
|
+
'a valid Boolean. Skipping device.',
|
|
258
|
+
{'attr': hint, 'value': device_value,
|
|
259
|
+
'dev': device_name})
|
|
260
|
+
break
|
|
261
|
+
if device_value == hint_value:
|
|
262
|
+
continue
|
|
263
|
+
|
|
264
|
+
elif specs_matcher.match(device_value, hint_value):
|
|
265
|
+
continue
|
|
266
|
+
|
|
267
|
+
LOG.debug('The attribute "%(attr)s" (with value "%(value)s") '
|
|
268
|
+
'of device "%(dev)s" does not match the hint %(hint)s',
|
|
269
|
+
{'attr': hint, 'value': device_value,
|
|
270
|
+
'dev': device_name, 'hint': hint_value})
|
|
271
|
+
break
|
|
272
|
+
else:
|
|
273
|
+
yield dev
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def match_root_device_hints(devices, root_device_hints):
|
|
277
|
+
"""Try to find a device that matches the root device hints.
|
|
278
|
+
|
|
279
|
+
Try to find a device that matches the root device hints. In order
|
|
280
|
+
for a device to be matched it needs to satisfy all the given hints.
|
|
281
|
+
|
|
282
|
+
:param devices: A list of dictionaries representing the devices
|
|
283
|
+
containing one or more of the following keys:
|
|
284
|
+
|
|
285
|
+
:name: (String) The device name, e.g /dev/sda
|
|
286
|
+
:size: (Integer) Size of the device in *bytes*
|
|
287
|
+
:model: (String) Device model
|
|
288
|
+
:vendor: (String) Device vendor name
|
|
289
|
+
:serial: (String) Device serial number
|
|
290
|
+
:wwn: (String) Unique storage identifier
|
|
291
|
+
:wwn_with_extension: (String): Unique storage identifier with
|
|
292
|
+
the vendor extension appended
|
|
293
|
+
:wwn_vendor_extension: (String): United vendor storage identifier
|
|
294
|
+
:rotational: (Boolean) Whether it's a rotational device or
|
|
295
|
+
not. Useful to distinguish HDDs (rotational) and SSDs
|
|
296
|
+
(not rotational).
|
|
297
|
+
:hctl: (String): The SCSI address: Host, channel, target and lun.
|
|
298
|
+
For example: '1:0:0:0'.
|
|
299
|
+
:by_path: (String): The alternative device name,
|
|
300
|
+
e.g. /dev/disk/by-path/pci-0000:00
|
|
301
|
+
|
|
302
|
+
:param root_device_hints: A dictionary with the root device hints.
|
|
303
|
+
:raises: ValueError, if some information is invalid.
|
|
304
|
+
:returns: The first device to match all the hints or None.
|
|
305
|
+
"""
|
|
306
|
+
try:
|
|
307
|
+
dev = next(find_devices_by_hints(devices, root_device_hints))
|
|
308
|
+
except StopIteration:
|
|
309
|
+
LOG.warning('No device found that matches the root device hints %s',
|
|
310
|
+
root_device_hints)
|
|
311
|
+
else:
|
|
312
|
+
LOG.info('Root device found! The device "%s" matches the root '
|
|
313
|
+
'device hints %s', dev, root_device_hints)
|
|
314
|
+
return dev
|