atex 0.9__py3-none-any.whl → 0.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atex/aggregator/__init__.py +60 -0
- atex/{orchestrator/aggregator.py → aggregator/json.py} +6 -21
- atex/cli/__init__.py +11 -1
- atex/cli/libvirt.py +3 -2
- atex/cli/testingfarm.py +48 -3
- atex/connection/podman.py +2 -4
- atex/connection/ssh.py +7 -14
- atex/executor/executor.py +18 -17
- atex/executor/scripts.py +5 -3
- atex/executor/testcontrol.py +1 -1
- atex/orchestrator/__init__.py +76 -3
- atex/orchestrator/{orchestrator.py → adhoc.py} +183 -103
- atex/{provision → provisioner}/__init__.py +49 -37
- atex/{provision → provisioner}/libvirt/libvirt.py +21 -14
- atex/{provision → provisioner}/libvirt/locking.py +3 -1
- atex/provisioner/podman/__init__.py +2 -0
- atex/provisioner/podman/podman.py +169 -0
- atex/{provision → provisioner}/testingfarm/api.py +53 -44
- atex/{provision → provisioner}/testingfarm/testingfarm.py +17 -23
- atex/util/log.py +62 -67
- atex/util/subprocess.py +46 -12
- atex/util/threads.py +7 -0
- atex-0.10.dist-info/METADATA +86 -0
- atex-0.10.dist-info/RECORD +44 -0
- atex/provision/podman/__init__.py +0 -1
- atex/provision/podman/podman.py +0 -274
- atex-0.9.dist-info/METADATA +0 -178
- atex-0.9.dist-info/RECORD +0 -43
- /atex/{provision → provisioner}/libvirt/VM_PROVISION +0 -0
- /atex/{provision → provisioner}/libvirt/__init__.py +0 -0
- /atex/{provision → provisioner}/libvirt/setup-libvirt.sh +0 -0
- /atex/{provision → provisioner}/testingfarm/__init__.py +0 -0
- {atex-0.9.dist-info → atex-0.10.dist-info}/WHEEL +0 -0
- {atex-0.9.dist-info → atex-0.10.dist-info}/entry_points.txt +0 -0
- {atex-0.9.dist-info → atex-0.10.dist-info}/licenses/COPYING.txt +0 -0
atex/util/log.py
CHANGED
|
@@ -1,76 +1,71 @@
|
|
|
1
|
-
import
|
|
1
|
+
import os
|
|
2
2
|
import logging
|
|
3
|
-
|
|
3
|
+
import inspect
|
|
4
4
|
|
|
5
5
|
_logger = logging.getLogger("atex")
|
|
6
6
|
|
|
7
|
+
# which functions to skip when determining the logger function caller;
|
|
8
|
+
# typically, these are wrappers and we want to see their caller in the trace
|
|
9
|
+
# instead of them
|
|
10
|
+
#
|
|
11
|
+
# ( file basename , qualname )
|
|
12
|
+
# where qualname is '<module>' or 'funcname' or 'Classname.funcname'
|
|
13
|
+
skip_levels = {
|
|
14
|
+
("subprocess.py", "subprocess_run"),
|
|
15
|
+
("subprocess.py", "subprocess_output"),
|
|
16
|
+
("subprocess.py", "subprocess_Popen"),
|
|
17
|
+
("subprocess.py", "subprocess_stream"),
|
|
18
|
+
("subprocess.py", "subprocess_log"),
|
|
7
19
|
|
|
8
|
-
|
|
9
|
-
"""
|
|
10
|
-
Return True if the root logger is using the DEBUG (or more verbose) level.
|
|
11
|
-
"""
|
|
12
|
-
# TODO: use _logger.isEnabledFor() ?
|
|
13
|
-
root_level = logging.getLogger().level
|
|
14
|
-
return root_level > 0 and root_level <= logging.DEBUG
|
|
20
|
+
("podman.py", "PodmanConnection.cmd"),
|
|
21
|
+
("podman.py", "PodmanConnection.rsync"),
|
|
15
22
|
|
|
23
|
+
("ssh.py", "StatelessSSHConnection.cmd"),
|
|
24
|
+
("ssh.py", "StatelessSSHConnection.rsync"),
|
|
25
|
+
("ssh.py", "ManagedSSHConnection.forward"),
|
|
26
|
+
("ssh.py", "ManagedSSHConnection.cmd"),
|
|
27
|
+
("ssh.py", "ManagedSSHConnection.rsync"),
|
|
28
|
+
}
|
|
16
29
|
|
|
17
|
-
def _format_msg(msg, *, skip_frames=0):
|
|
18
|
-
stack = inspect.stack()
|
|
19
|
-
if len(stack)-1 <= skip_frames:
|
|
20
|
-
raise SyntaxError("skip_frames exceeds call stack (frame count)")
|
|
21
|
-
stack = stack[skip_frames+1:]
|
|
22
30
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
31
|
+
def _log_msg(logger_func, *args, stacklevel=1, **kwargs):
|
|
32
|
+
# inspect.stack() is MUCH slower
|
|
33
|
+
caller = inspect.currentframe().f_back.f_back
|
|
34
|
+
extra_levels = 2 # skip this func and the debug/info/warning parent
|
|
35
|
+
while caller.f_back:
|
|
36
|
+
code = caller.f_code
|
|
37
|
+
# pathlib is much slower
|
|
38
|
+
basename = os.path.basename(code.co_filename) # noqa: PTH119
|
|
39
|
+
qualname = code.co_qualname
|
|
40
|
+
if (basename, qualname) in skip_levels:
|
|
41
|
+
extra_levels += 1
|
|
42
|
+
caller = caller.f_back
|
|
43
|
+
else:
|
|
26
44
|
break
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
else:
|
|
55
|
-
return f"{Path(parent.filename).name}:{parent.lineno}: {msg}"
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def debug(msg, *, skip_frames=0):
|
|
59
|
-
if in_debug_mode():
|
|
60
|
-
_logger.debug(_format_msg(msg, skip_frames=skip_frames+1))
|
|
61
|
-
else:
|
|
62
|
-
_logger.debug(msg)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def info(msg, *, skip_frames=0):
|
|
66
|
-
if in_debug_mode():
|
|
67
|
-
_logger.info(_format_msg(msg, skip_frames=skip_frames+1))
|
|
68
|
-
else:
|
|
69
|
-
_logger.info(msg)
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
def warning(msg, *, skip_frames=0):
|
|
73
|
-
if in_debug_mode():
|
|
74
|
-
_logger.warning(_format_msg(msg, skip_frames=skip_frames+1))
|
|
75
|
-
else:
|
|
76
|
-
_logger.warning(msg)
|
|
45
|
+
return logger_func(*args, stacklevel=stacklevel+extra_levels, **kwargs)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def warning(*args, **kwargs):
|
|
49
|
+
return _log_msg(_logger.warning, *args, **kwargs)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def info(*args, **kwargs):
|
|
53
|
+
return _log_msg(_logger.info, *args, **kwargs)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def debug(*args, **kwargs):
|
|
57
|
+
return _log_msg(_logger.debug, *args, **kwargs)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# add a log level more verbose than logging.DEBUG, for verbose command
|
|
61
|
+
# outputs, big JSON / XML printouts, and other outputs unsuitable for
|
|
62
|
+
# large parallel runs; to be used in targeted debugging
|
|
63
|
+
#
|
|
64
|
+
# logging.DEBUG is 10, and programs tend to add TRACE as 5, so be somewhere
|
|
65
|
+
# in between
|
|
66
|
+
EXTRADEBUG = 8
|
|
67
|
+
logging.addLevelName(EXTRADEBUG, "EXTRADEBUG")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def extradebug(*args, **kwargs):
|
|
71
|
+
return _log_msg(_logger.log, EXTRADEBUG, *args, **kwargs)
|
atex/util/subprocess.py
CHANGED
|
@@ -1,52 +1,86 @@
|
|
|
1
1
|
import subprocess
|
|
2
2
|
|
|
3
|
-
from .log import debug
|
|
3
|
+
from .log import debug, extradebug
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
def subprocess_run(cmd,
|
|
6
|
+
def subprocess_run(cmd, **kwargs):
|
|
7
7
|
"""
|
|
8
8
|
A simple wrapper for the real subprocess.run() that logs the command used.
|
|
9
9
|
"""
|
|
10
10
|
# when logging, skip current stack frame - report the place we were called
|
|
11
11
|
# from, not util.subprocess_run itself
|
|
12
|
-
debug(f"running: {cmd}
|
|
12
|
+
debug(f"running: '{cmd}' with {kwargs=}")
|
|
13
13
|
return subprocess.run(cmd, **kwargs)
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def subprocess_output(cmd, *,
|
|
16
|
+
def subprocess_output(cmd, *, check=True, text=True, **kwargs):
|
|
17
17
|
"""
|
|
18
18
|
A wrapper simulating subprocess.check_output() via a modern .run() API.
|
|
19
19
|
"""
|
|
20
|
-
debug(f"running: {cmd}
|
|
20
|
+
debug(f"running: '{cmd}' with {check=}, {text=} and {kwargs=}")
|
|
21
21
|
proc = subprocess.run(cmd, check=check, text=text, stdout=subprocess.PIPE, **kwargs)
|
|
22
22
|
return proc.stdout.rstrip("\n") if text else proc.stdout
|
|
23
23
|
|
|
24
24
|
|
|
25
|
-
def subprocess_Popen(cmd,
|
|
25
|
+
def subprocess_Popen(cmd, **kwargs): # noqa: N802
|
|
26
26
|
"""
|
|
27
27
|
A simple wrapper for the real subprocess.Popen() that logs the command used.
|
|
28
28
|
"""
|
|
29
|
-
debug(f"running: {cmd}
|
|
29
|
+
debug(f"running: '{cmd}' with {kwargs=}")
|
|
30
30
|
return subprocess.Popen(cmd, **kwargs)
|
|
31
31
|
|
|
32
32
|
|
|
33
|
-
def subprocess_stream(cmd, *, check=False,
|
|
33
|
+
def subprocess_stream(cmd, *, stream="stdout", check=False, input=None, **kwargs):
|
|
34
34
|
"""
|
|
35
35
|
Run 'cmd' via subprocess.Popen() and return an iterator over any lines
|
|
36
36
|
the command outputs on stdout, in text mode.
|
|
37
37
|
|
|
38
|
+
The 'stream' is a subprocess.Popen attribute (either 'stdout' or 'stderr')
|
|
39
|
+
to read from.
|
|
40
|
+
To capture both stdout and stderr as yielded lines, use 'stream="stdout"'
|
|
41
|
+
and pass an additional 'stderr=subprocess.STDOUT'.
|
|
42
|
+
|
|
38
43
|
With 'check' set to True, raise a CalledProcessError if the 'cmd' failed.
|
|
39
44
|
|
|
40
|
-
|
|
45
|
+
Similarly, 'input' simulates the 'input' arg of subprocess.run().
|
|
46
|
+
Note that the input is written to stdin of the process *before* any outputs
|
|
47
|
+
are streamed, so it should be sufficiently small and/or not cause a deadlock
|
|
48
|
+
with the process waiting for outputs to be read before consuming more input.
|
|
49
|
+
Use 'stdin=subprocess.PIPE' and write to it manually if you need more.
|
|
41
50
|
"""
|
|
42
|
-
|
|
43
|
-
|
|
51
|
+
all_kwargs = {
|
|
52
|
+
"text": True,
|
|
53
|
+
stream: subprocess.PIPE,
|
|
54
|
+
}
|
|
55
|
+
if input is not None:
|
|
56
|
+
all_kwargs["stdin"] = subprocess.PIPE
|
|
57
|
+
all_kwargs |= kwargs
|
|
58
|
+
|
|
59
|
+
debug(f"running: '{cmd}' with {all_kwargs=}")
|
|
60
|
+
proc = subprocess.Popen(cmd, **all_kwargs)
|
|
44
61
|
|
|
45
62
|
def generate_lines():
|
|
46
|
-
|
|
63
|
+
if input is not None:
|
|
64
|
+
proc.stdin.write(input)
|
|
65
|
+
proc.stdin.close()
|
|
66
|
+
line_stream = getattr(proc, stream)
|
|
67
|
+
for line in line_stream:
|
|
47
68
|
yield line.rstrip("\n")
|
|
48
69
|
code = proc.wait()
|
|
49
70
|
if code > 0 and check:
|
|
50
71
|
raise subprocess.CalledProcessError(cmd=cmd, returncode=code)
|
|
51
72
|
|
|
52
73
|
return (proc, generate_lines())
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def subprocess_log(cmd, **kwargs):
|
|
77
|
+
"""
|
|
78
|
+
A wrapper to stream every (text) line output from the process to the
|
|
79
|
+
logging module.
|
|
80
|
+
|
|
81
|
+
Uses subprocess_stream() to gather the lines.
|
|
82
|
+
"""
|
|
83
|
+
debug(f"running: '{cmd}' with {kwargs=}")
|
|
84
|
+
_, lines = subprocess_stream(cmd, **kwargs)
|
|
85
|
+
for line in lines:
|
|
86
|
+
extradebug(line)
|
atex/util/threads.py
CHANGED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: atex
|
|
3
|
+
Version: 0.10
|
|
4
|
+
Summary: Ad-hoc Test EXecutor
|
|
5
|
+
Project-URL: Homepage, https://github.com/RHSecurityCompliance/atex
|
|
6
|
+
License-Expression: GPL-3.0-or-later
|
|
7
|
+
License-File: COPYING.txt
|
|
8
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Topic :: Software Development :: Testing
|
|
11
|
+
Requires-Python: >=3.11
|
|
12
|
+
Requires-Dist: fmf>=1.6
|
|
13
|
+
Requires-Dist: pyyaml
|
|
14
|
+
Requires-Dist: urllib3<3,>=2
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# ATEX = Ad-hoc Test EXecutor
|
|
18
|
+
|
|
19
|
+
A collections of Python APIs to provision operating systems, collect
|
|
20
|
+
and execute [FMF](https://github.com/teemtee/fmf/)-style tests, gather
|
|
21
|
+
and organize their results and generate reports from those results.
|
|
22
|
+
|
|
23
|
+
The name comes from a (fairly unique to FMF/TMT ecosystem) approach that
|
|
24
|
+
allows provisioning a pool of systems and scheduling tests on them as one would
|
|
25
|
+
on an ad-hoc pool of thread/process workers - once a worker becomes free,
|
|
26
|
+
it receives a test to run.
|
|
27
|
+
This is in contrast to splitting a large list of N tests onto M workers
|
|
28
|
+
like N/M, which yields significant time penalties due to tests having
|
|
29
|
+
very varies runtimes.
|
|
30
|
+
|
|
31
|
+
Above all, this project is meant to be a toolbox, not a silver-plate solution.
|
|
32
|
+
Use its Python APIs to build a CLI tool for your specific use case.
|
|
33
|
+
The CLI tool provided here is just for demonstration / testing, not for serious
|
|
34
|
+
use - we want to avoid huge modular CLIs for Every Possible Scenario. That's
|
|
35
|
+
the job of the Python API. Any CLI should be simple by nature.
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
|
|
39
|
+
## License
|
|
40
|
+
|
|
41
|
+
Unless specified otherwise, any content within this repository is distributed
|
|
42
|
+
under the GNU GPLv3 license, see the [COPYING.txt](COPYING.txt) file for more.
|
|
43
|
+
|
|
44
|
+
## Environment variables
|
|
45
|
+
|
|
46
|
+
- `ATEX_DEBUG_TEST`
|
|
47
|
+
- Set to `1` to print out detailed runner-related trace within the test output
|
|
48
|
+
stream (as if it was printed out by the test).
|
|
49
|
+
|
|
50
|
+
## Testing this project
|
|
51
|
+
|
|
52
|
+
There are some limited sanity tests provided via `pytest`, although:
|
|
53
|
+
|
|
54
|
+
- Some require additional variables (ie. Testing Farm) and will ERROR
|
|
55
|
+
without them.
|
|
56
|
+
- Some take a long time (ie. Testing Farm) due to system provisioning
|
|
57
|
+
taking a long time, so install `pytest-xdist` and run with a large `-n`.
|
|
58
|
+
|
|
59
|
+
Currently, the recommended approach is to split the execution:
|
|
60
|
+
|
|
61
|
+
```
|
|
62
|
+
# synchronously, because podman CLI has concurrency issues
|
|
63
|
+
pytest tests/provision/test_podman.py
|
|
64
|
+
|
|
65
|
+
# in parallel, because provisioning takes a long time
|
|
66
|
+
export TESTING_FARM_API_TOKEN=...
|
|
67
|
+
export TESTING_FARM_COMPOSE=...
|
|
68
|
+
pytest -n 20 tests/provision/test_podman.py
|
|
69
|
+
|
|
70
|
+
# fast enough for synchronous execution
|
|
71
|
+
pytest tests/fmf
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Unsorted notes
|
|
75
|
+
|
|
76
|
+
TODO: codestyle from contest
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
- this is not tmt, the goal is to make a python toolbox *for* making runcontest
|
|
80
|
+
style tools easily, not to replace those tools with tmt-style CLI syntax
|
|
81
|
+
|
|
82
|
+
- the whole point is to make usecase-targeted easy-to-use tools that don't
|
|
83
|
+
intimidate users with 1 KB long command line, and runcontest is a nice example
|
|
84
|
+
|
|
85
|
+
- TL;DR - use a modular pythonic approach, not a gluetool-style long CLI
|
|
86
|
+
```
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
atex/__init__.py,sha256=LdX67gprtHYeAkjLhFPKzpc7ECv2rHxUbHKDGbGXO1c,517
|
|
2
|
+
atex/fmf.py,sha256=gkJXIaRO7_KvwJR-V6Tc1NVn4a9Hq7hoBLQLhxYIdbg,8834
|
|
3
|
+
atex/aggregator/__init__.py,sha256=uNnYSyDGXjknxckI8MFfl-C8_gin8FwQchiq-UOyP6I,1744
|
|
4
|
+
atex/aggregator/json.py,sha256=x1zim9O2olzBh185NYWo5N96fixB2oxCamoOZwmgR9w,3330
|
|
5
|
+
atex/cli/__init__.py,sha256=X5XxkDEDXE4tJAjwt5ShRHCFTXDK-2zvxQ34opmueUc,2768
|
|
6
|
+
atex/cli/fmf.py,sha256=HfbTgFbCwK4Nuyq6vtGutcq_4-4kj-tmoqzXUn3AYtY,3573
|
|
7
|
+
atex/cli/libvirt.py,sha256=6tt5ANb8XBBRXOQsYPTWILThKqf-gvt5AZh5Dctg2PA,3782
|
|
8
|
+
atex/cli/testingfarm.py,sha256=HGlqrkhanUMo2CqKxmM3ACgptWtxm0gICyEGf7O6Qc0,9078
|
|
9
|
+
atex/connection/__init__.py,sha256=dj8ZBcEspom7Z_UjecfLGBRNvLZ3dyGR9q19i_B4xpY,3880
|
|
10
|
+
atex/connection/podman.py,sha256=1T56gh1TgbcQWpTIJHL4NaxZOI6aMg7Xp7sn6PQQyBk,1911
|
|
11
|
+
atex/connection/ssh.py,sha256=9A57b9YR_HI-kIu06Asic1y__JPVXEheDZxjbG2Qcsc,13460
|
|
12
|
+
atex/executor/__init__.py,sha256=XCfhi7QDELjey7N1uzhMjc46Kp1Jsd5bOCf52I27SCE,85
|
|
13
|
+
atex/executor/duration.py,sha256=x06sItKOZi6XA8KszQwZGpIb1Z_L-HWqIwZKo2SDo0s,1759
|
|
14
|
+
atex/executor/executor.py,sha256=JLFR9cZjSlUdAlAlLct6WuzmYbtjGtSobxvsToQum6M,15738
|
|
15
|
+
atex/executor/reporter.py,sha256=MceFmHFt0bTEClBZbRI1WnFbfMhR0e1noOzcu7gjKuQ,3403
|
|
16
|
+
atex/executor/scripts.py,sha256=riJAQWsV-BFGkJwR2Dmf3R0ZRRZJs9w9iYnPpYaQNaE,5618
|
|
17
|
+
atex/executor/testcontrol.py,sha256=mVrLwQUnDRfUq-5diz-80UvCWWxn1TkcBgmAKhKNb5E,12696
|
|
18
|
+
atex/orchestrator/__init__.py,sha256=8Q1YknyibilXLjWRYkHm_Mr2HMm0SRw8Zv39KypeASM,2059
|
|
19
|
+
atex/orchestrator/adhoc.py,sha256=GnvHLlCHeJ_nQ8doEjMuDzqmu4XZorI7ZzOtG_C08tU,18451
|
|
20
|
+
atex/provisioner/__init__.py,sha256=2eepmEznq94tbam9VSWbsGFrZZpWeNSVlsTczGxjNuQ,4667
|
|
21
|
+
atex/provisioner/libvirt/VM_PROVISION,sha256=7pkZ-ozgTyK4qNGC-E-HUznr4IhbosWSASbB72Gknl8,2664
|
|
22
|
+
atex/provisioner/libvirt/__init__.py,sha256=pKG5IpZSC2IHs5wL2ecQx_fd9AzAXEbZmDzA7RyZsfM,119
|
|
23
|
+
atex/provisioner/libvirt/libvirt.py,sha256=rtxowv5DpgcWsGRXYF29n6S9x_cgXRVgqY41DiFu920,18431
|
|
24
|
+
atex/provisioner/libvirt/locking.py,sha256=AXtDyidZNmUoMmrit26g9iTHDqInrzL_RSQEoc_EAXw,5669
|
|
25
|
+
atex/provisioner/libvirt/setup-libvirt.sh,sha256=oCMy9SCnbC_QuAzO2sFwvB5ui1kMQ6uviHsgdXyoFXc,2428
|
|
26
|
+
atex/provisioner/podman/__init__.py,sha256=dM0JzQXWX7edtWSc0KH0cMFXAjArFn2Vme4j_ZMsdYA,138
|
|
27
|
+
atex/provisioner/podman/podman.py,sha256=ztRypoakSf-jF04iER58tEMUZ4Y6AuzIpNpFXp44bB4,4997
|
|
28
|
+
atex/provisioner/testingfarm/__init__.py,sha256=kZncgLGdRCR4FMaRQr2GTwJ8vjlA-24ri8JO2ueZJuw,113
|
|
29
|
+
atex/provisioner/testingfarm/api.py,sha256=UcMN61nBr3wqEd5KSR5Xhv1-TS7nSPFvk2byb6PdIs8,21811
|
|
30
|
+
atex/provisioner/testingfarm/testingfarm.py,sha256=OI-a99xALaiYf-y5037WFVxY1g2H2y1xEKxHBdUQvfg,8271
|
|
31
|
+
atex/util/__init__.py,sha256=cWHFbtQ4mDlKe6lXyPDWRmWJOTcHDGfVuW_-GYa8hB0,1473
|
|
32
|
+
atex/util/dedent.py,sha256=SEuJMtLzqz3dQ7g7qyZzEJ9VYynVlk52tQCJY-FveXo,603
|
|
33
|
+
atex/util/libvirt.py,sha256=kDZmT6xLYEZkQNLZY98gJ2M48DDWXxHF8rQY9PnjB3U,660
|
|
34
|
+
atex/util/log.py,sha256=KVR7ep8n5wtghsvBFCtHiPsMAQBdAmK83E_Jec5t4cU,2230
|
|
35
|
+
atex/util/named_mapping.py,sha256=UBMe9TetjV-DGPhjYjJ42YtC40FVPKAAEROXl9MA5fo,4700
|
|
36
|
+
atex/util/path.py,sha256=x-kXqiWCVodfZWbEwtC5A8LFvutpDIPYv2m0boZSlXU,504
|
|
37
|
+
atex/util/ssh_keygen.py,sha256=9yuSl2yBV7pG3Qfsf9tossVC00nbIUrAeLdbwTykpjk,384
|
|
38
|
+
atex/util/subprocess.py,sha256=PQBxcQJPapP1ZLO4LqENyrxxCbNAxtJDNNlBV5DcD9k,2953
|
|
39
|
+
atex/util/threads.py,sha256=c8hsEc-8SqJGodInorv_6JxpiHiSkGFGob4qbMmOD2M,3531
|
|
40
|
+
atex-0.10.dist-info/METADATA,sha256=evOBYvVboY2T8eGrAKy64UpyeuFKSMLGz8pUz8Sstm8,3050
|
|
41
|
+
atex-0.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
42
|
+
atex-0.10.dist-info/entry_points.txt,sha256=pLqJdcfeyQTgup2h6dWb6SvkHhtOl-W5Eg9zV8moK0o,39
|
|
43
|
+
atex-0.10.dist-info/licenses/COPYING.txt,sha256=oEuj51jdmbXcCUy7pZ-KE0BNcJTR1okudRp5zQ0yWnU,670
|
|
44
|
+
atex-0.10.dist-info/RECORD,,
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
from .podman import PodmanProvisioner, PodmanRemote # noqa: F401
|
atex/provision/podman/podman.py
DELETED
|
@@ -1,274 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import time
|
|
3
|
-
import enum
|
|
4
|
-
import threading
|
|
5
|
-
import subprocess
|
|
6
|
-
|
|
7
|
-
from ... import connection, util
|
|
8
|
-
from .. import Provisioner, Remote
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class PodmanRemote(Remote, connection.podman.PodmanConn):
|
|
12
|
-
"""
|
|
13
|
-
Built on the official Remote API, pulling in the Connection API
|
|
14
|
-
as implemented by ManagedSSHConn.
|
|
15
|
-
"""
|
|
16
|
-
|
|
17
|
-
def __init__(self, image, container, *, release_hook):
|
|
18
|
-
"""
|
|
19
|
-
'image' is an image tag (used for repr()).
|
|
20
|
-
|
|
21
|
-
'container' is a podman container id / name.
|
|
22
|
-
|
|
23
|
-
'release_hook' is a callable called on .release() in addition
|
|
24
|
-
to disconnecting the connection.
|
|
25
|
-
"""
|
|
26
|
-
super().__init__(container=container)
|
|
27
|
-
self.lock = threading.RLock()
|
|
28
|
-
self.image = image
|
|
29
|
-
self.container = container
|
|
30
|
-
self.release_called = False
|
|
31
|
-
self.release_hook = release_hook
|
|
32
|
-
|
|
33
|
-
def release(self):
|
|
34
|
-
with self.lock:
|
|
35
|
-
if self.release_called:
|
|
36
|
-
return
|
|
37
|
-
else:
|
|
38
|
-
self.release_called = True
|
|
39
|
-
self.release_hook(self)
|
|
40
|
-
self.disconnect()
|
|
41
|
-
util.subprocess_run(
|
|
42
|
-
("podman", "container", "rm", "-f", "-t", "0", self.container),
|
|
43
|
-
check=False, # ignore if it fails
|
|
44
|
-
stdout=subprocess.DEVNULL,
|
|
45
|
-
)
|
|
46
|
-
|
|
47
|
-
# not /technically/ a valid repr(), but meh
|
|
48
|
-
def __repr__(self):
|
|
49
|
-
class_name = self.__class__.__name__
|
|
50
|
-
|
|
51
|
-
if "/" in self.image:
|
|
52
|
-
image = self.image.rsplit("/",1)[1]
|
|
53
|
-
elif len(self.image) > 20:
|
|
54
|
-
image = f"{self.image[:17]}..."
|
|
55
|
-
else:
|
|
56
|
-
image = self.image
|
|
57
|
-
|
|
58
|
-
name = f"{self.container[:17]}..." if len(self.container) > 20 else self.container
|
|
59
|
-
|
|
60
|
-
return f"{class_name}({image}, {name})"
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
class PodmanProvisioner(Provisioner):
|
|
64
|
-
class State(enum.Enum):
|
|
65
|
-
WAITING_FOR_PULL = enum.auto()
|
|
66
|
-
CREATING_CONTAINER = enum.auto()
|
|
67
|
-
WAITING_FOR_CREATION = enum.auto()
|
|
68
|
-
SETTING_UP_CONTAINER = enum.auto()
|
|
69
|
-
WAITING_FOR_SETUP = enum.auto()
|
|
70
|
-
|
|
71
|
-
# NOTE: this uses a single Popen process to run podman commands,
|
|
72
|
-
# to avoid double downloads/pulls, but also to avoid SQLite errors
|
|
73
|
-
# when creating multiple containers in parallel
|
|
74
|
-
|
|
75
|
-
def __init__(self, image, run_options=None, *, pull=True, max_systems=1):
|
|
76
|
-
"""
|
|
77
|
-
'image' is a string of image tag/id to create containers from.
|
|
78
|
-
It can be a local identifier or an URL.
|
|
79
|
-
|
|
80
|
-
'run_options' is an iterable with additional CLI options passed
|
|
81
|
-
to 'podman container run'.
|
|
82
|
-
|
|
83
|
-
'pull' specifies whether to attempt 'podman image pull' on the specified
|
|
84
|
-
image tag/id before any container creation.
|
|
85
|
-
|
|
86
|
-
'max_systems' is a maximum number of containers running at any one time.
|
|
87
|
-
"""
|
|
88
|
-
self.lock = threading.RLock()
|
|
89
|
-
self.image = image
|
|
90
|
-
self.run_options = run_options or ()
|
|
91
|
-
self.pull = pull
|
|
92
|
-
self.max_systems = max_systems
|
|
93
|
-
|
|
94
|
-
self.image_id = None
|
|
95
|
-
self.container_id = None
|
|
96
|
-
self.worker = None
|
|
97
|
-
self.worker_output = bytearray()
|
|
98
|
-
self.state = None
|
|
99
|
-
# created PodmanRemote instances, ready to be handed over to the user,
|
|
100
|
-
# or already in use by the user
|
|
101
|
-
self.remotes = []
|
|
102
|
-
|
|
103
|
-
@staticmethod
|
|
104
|
-
def _spawn_proc(cmd):
|
|
105
|
-
proc = util.subprocess_Popen(
|
|
106
|
-
cmd,
|
|
107
|
-
stdin=subprocess.DEVNULL,
|
|
108
|
-
stdout=subprocess.PIPE,
|
|
109
|
-
stderr=subprocess.STDOUT,
|
|
110
|
-
)
|
|
111
|
-
os.set_blocking(proc.stdout.fileno(), False)
|
|
112
|
-
return proc
|
|
113
|
-
|
|
114
|
-
# @staticmethod
|
|
115
|
-
# def _poll_proc(proc):
|
|
116
|
-
# # read from the process to un-block any kernel buffers
|
|
117
|
-
# try:
|
|
118
|
-
# out = proc.stdout.read() # non-blocking
|
|
119
|
-
# except BlockingIOError:
|
|
120
|
-
# out = ""
|
|
121
|
-
# return (proc.poll(), out)
|
|
122
|
-
|
|
123
|
-
def _make_remote(self, container):
|
|
124
|
-
def release_hook(remote):
|
|
125
|
-
# remove from the list of remotes inside this Provisioner
|
|
126
|
-
with self.lock:
|
|
127
|
-
try:
|
|
128
|
-
self.remotes.remove(remote)
|
|
129
|
-
except ValueError:
|
|
130
|
-
pass
|
|
131
|
-
|
|
132
|
-
remote = PodmanRemote(
|
|
133
|
-
self.image,
|
|
134
|
-
container,
|
|
135
|
-
release_hook=release_hook,
|
|
136
|
-
)
|
|
137
|
-
self.remotes.append(remote)
|
|
138
|
-
return remote
|
|
139
|
-
|
|
140
|
-
def start(self):
|
|
141
|
-
if not self.image:
|
|
142
|
-
raise ValueError("image cannot be empty")
|
|
143
|
-
|
|
144
|
-
if not self.pull:
|
|
145
|
-
self.image_id = self.image
|
|
146
|
-
self.state = self.State.CREATING_CONTAINER
|
|
147
|
-
else:
|
|
148
|
-
self.worker = self._spawn_proc(
|
|
149
|
-
("podman", "image", "pull", "--quiet", self.image),
|
|
150
|
-
)
|
|
151
|
-
self.state = self.State.WAITING_FOR_PULL
|
|
152
|
-
|
|
153
|
-
def stop(self):
|
|
154
|
-
with self.lock:
|
|
155
|
-
while self.remotes:
|
|
156
|
-
self.remotes.pop().release()
|
|
157
|
-
worker = self.worker
|
|
158
|
-
self.worker = None
|
|
159
|
-
|
|
160
|
-
if worker:
|
|
161
|
-
worker.kill()
|
|
162
|
-
# don"t zombie forever, return EPIPE on any attempts to write to us
|
|
163
|
-
worker.stdout.close()
|
|
164
|
-
worker.wait()
|
|
165
|
-
|
|
166
|
-
def stop_defer(self):
|
|
167
|
-
# avoid SQLite errors by removing containers sequentially
|
|
168
|
-
return self.stop
|
|
169
|
-
|
|
170
|
-
@staticmethod
|
|
171
|
-
def _nonblock_read(fobj):
|
|
172
|
-
"""Return b'' if there was nothing to read, instead of None."""
|
|
173
|
-
data = fobj.read()
|
|
174
|
-
return b"" if data is None else data
|
|
175
|
-
|
|
176
|
-
def _get_remote_nonblock(self):
|
|
177
|
-
if self.state is None:
|
|
178
|
-
raise RuntimeError("the provisioner is in an invalid state")
|
|
179
|
-
|
|
180
|
-
# NOTE: these are not 'elif' statements explicitly to allow a next block
|
|
181
|
-
# to follow a previous one (if the condition is met)
|
|
182
|
-
if self.state is self.State.WAITING_FOR_PULL:
|
|
183
|
-
self.worker_output += self._nonblock_read(self.worker.stdout)
|
|
184
|
-
rc = self.worker.poll()
|
|
185
|
-
if rc is None:
|
|
186
|
-
return None # still running
|
|
187
|
-
elif rc != 0:
|
|
188
|
-
out = self.worker_output.decode().rstrip("\n")
|
|
189
|
-
self.worker_output.clear()
|
|
190
|
-
self.worker = None
|
|
191
|
-
self.state = None
|
|
192
|
-
raise RuntimeError(f"podman image pull failed with {rc}:\n{out}")
|
|
193
|
-
else:
|
|
194
|
-
self.image_id = self.worker_output.decode().rstrip("\n")
|
|
195
|
-
self.worker_output.clear()
|
|
196
|
-
self.worker = None
|
|
197
|
-
self.state = self.State.CREATING_CONTAINER
|
|
198
|
-
|
|
199
|
-
if self.state is self.State.CREATING_CONTAINER:
|
|
200
|
-
if len(self.remotes) < self.max_systems:
|
|
201
|
-
self.worker = self._spawn_proc(
|
|
202
|
-
(
|
|
203
|
-
"podman", "container", "run", "--quiet", "--detach", "--pull", "never",
|
|
204
|
-
*self.run_options, self.image_id, "sleep", "inf",
|
|
205
|
-
),
|
|
206
|
-
)
|
|
207
|
-
self.state = self.State.WAITING_FOR_CREATION
|
|
208
|
-
else:
|
|
209
|
-
# too many remotes requested
|
|
210
|
-
return None
|
|
211
|
-
|
|
212
|
-
if self.state is self.State.WAITING_FOR_CREATION:
|
|
213
|
-
self.worker_output += self._nonblock_read(self.worker.stdout)
|
|
214
|
-
rc = self.worker.poll()
|
|
215
|
-
if rc is None:
|
|
216
|
-
return None # still running
|
|
217
|
-
elif rc != 0:
|
|
218
|
-
out = self.worker_output.decode().rstrip("\n")
|
|
219
|
-
self.worker_output.clear()
|
|
220
|
-
self.worker = None
|
|
221
|
-
self.state = None
|
|
222
|
-
raise RuntimeError(f"podman run failed with {rc}:\n{out}")
|
|
223
|
-
else:
|
|
224
|
-
self.container_id = self.worker_output.decode().rstrip("\n")
|
|
225
|
-
self.worker_output.clear()
|
|
226
|
-
self.worker = None
|
|
227
|
-
self.state = self.State.SETTING_UP_CONTAINER
|
|
228
|
-
|
|
229
|
-
if self.state is self.State.SETTING_UP_CONTAINER:
|
|
230
|
-
cmd = ("dnf", "install", "-y", "-q", "--setopt=install_weak_deps=False", "rsync")
|
|
231
|
-
self.worker = self._spawn_proc(
|
|
232
|
-
("podman", "container", "exec", self.container_id, *cmd),
|
|
233
|
-
)
|
|
234
|
-
self.state = self.State.WAITING_FOR_SETUP
|
|
235
|
-
|
|
236
|
-
if self.state is self.State.WAITING_FOR_SETUP:
|
|
237
|
-
self.worker_output += self._nonblock_read(self.worker.stdout)
|
|
238
|
-
rc = self.worker.poll()
|
|
239
|
-
if rc is None:
|
|
240
|
-
return None # still running
|
|
241
|
-
elif rc != 0:
|
|
242
|
-
out = self.worker_output.decode().rstrip("\n")
|
|
243
|
-
self.worker_output.clear()
|
|
244
|
-
self.worker = None
|
|
245
|
-
self.state = None
|
|
246
|
-
raise RuntimeError(f"setting up failed with {rc}:\n{out}")
|
|
247
|
-
else:
|
|
248
|
-
# everything ready, give the Remote to the caller and reset
|
|
249
|
-
remote = self._make_remote(self.container_id)
|
|
250
|
-
self.worker_output.clear()
|
|
251
|
-
self.worker = None
|
|
252
|
-
self.state = self.State.CREATING_CONTAINER
|
|
253
|
-
return remote
|
|
254
|
-
|
|
255
|
-
raise AssertionError(f"reached end (invalid state {self.state}?)")
|
|
256
|
-
|
|
257
|
-
def get_remote(self, block=True):
|
|
258
|
-
if not block:
|
|
259
|
-
with self.lock:
|
|
260
|
-
return self._get_remote_nonblock()
|
|
261
|
-
else:
|
|
262
|
-
while True:
|
|
263
|
-
with self.lock:
|
|
264
|
-
if remote := self._get_remote_nonblock():
|
|
265
|
-
return remote
|
|
266
|
-
time.sleep(0.1)
|
|
267
|
-
|
|
268
|
-
# not /technically/ a valid repr(), but meh
|
|
269
|
-
def __repr__(self):
|
|
270
|
-
class_name = self.__class__.__name__
|
|
271
|
-
return (
|
|
272
|
-
f"{class_name}({self.image}, {len(self.remotes)}/{self.max_systems} remotes, "
|
|
273
|
-
f"{hex(id(self))})"
|
|
274
|
-
)
|