atex 0.9__py3-none-any.whl → 0.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. atex/aggregator/__init__.py +60 -0
  2. atex/{orchestrator/aggregator.py → aggregator/json.py} +6 -21
  3. atex/cli/__init__.py +11 -1
  4. atex/cli/libvirt.py +3 -2
  5. atex/cli/testingfarm.py +48 -3
  6. atex/connection/podman.py +2 -4
  7. atex/connection/ssh.py +7 -14
  8. atex/executor/executor.py +18 -17
  9. atex/executor/scripts.py +5 -3
  10. atex/executor/testcontrol.py +1 -1
  11. atex/orchestrator/__init__.py +76 -3
  12. atex/orchestrator/{orchestrator.py → adhoc.py} +183 -103
  13. atex/{provision → provisioner}/__init__.py +49 -37
  14. atex/{provision → provisioner}/libvirt/libvirt.py +21 -14
  15. atex/{provision → provisioner}/libvirt/locking.py +3 -1
  16. atex/provisioner/podman/__init__.py +2 -0
  17. atex/provisioner/podman/podman.py +169 -0
  18. atex/{provision → provisioner}/testingfarm/api.py +53 -44
  19. atex/{provision → provisioner}/testingfarm/testingfarm.py +17 -23
  20. atex/util/log.py +62 -67
  21. atex/util/subprocess.py +46 -12
  22. atex/util/threads.py +7 -0
  23. atex-0.10.dist-info/METADATA +86 -0
  24. atex-0.10.dist-info/RECORD +44 -0
  25. atex/provision/podman/__init__.py +0 -1
  26. atex/provision/podman/podman.py +0 -274
  27. atex-0.9.dist-info/METADATA +0 -178
  28. atex-0.9.dist-info/RECORD +0 -43
  29. /atex/{provision → provisioner}/libvirt/VM_PROVISION +0 -0
  30. /atex/{provision → provisioner}/libvirt/__init__.py +0 -0
  31. /atex/{provision → provisioner}/libvirt/setup-libvirt.sh +0 -0
  32. /atex/{provision → provisioner}/testingfarm/__init__.py +0 -0
  33. {atex-0.9.dist-info → atex-0.10.dist-info}/WHEEL +0 -0
  34. {atex-0.9.dist-info → atex-0.10.dist-info}/entry_points.txt +0 -0
  35. {atex-0.9.dist-info → atex-0.10.dist-info}/licenses/COPYING.txt +0 -0
atex/util/log.py CHANGED
@@ -1,76 +1,71 @@
1
- import inspect
1
+ import os
2
2
  import logging
3
- from pathlib import Path
3
+ import inspect
4
4
 
5
5
  _logger = logging.getLogger("atex")
6
6
 
7
+ # which functions to skip when determining the logger function caller;
8
+ # typically, these are wrappers and we want to see their caller in the trace
9
+ # instead of them
10
+ #
11
+ # ( file basename , qualname )
12
+ # where qualname is '<module>' or 'funcname' or 'Classname.funcname'
13
+ skip_levels = {
14
+ ("subprocess.py", "subprocess_run"),
15
+ ("subprocess.py", "subprocess_output"),
16
+ ("subprocess.py", "subprocess_Popen"),
17
+ ("subprocess.py", "subprocess_stream"),
18
+ ("subprocess.py", "subprocess_log"),
7
19
 
8
- def in_debug_mode():
9
- """
10
- Return True if the root logger is using the DEBUG (or more verbose) level.
11
- """
12
- # TODO: use _logger.isEnabledFor() ?
13
- root_level = logging.getLogger().level
14
- return root_level > 0 and root_level <= logging.DEBUG
20
+ ("podman.py", "PodmanConnection.cmd"),
21
+ ("podman.py", "PodmanConnection.rsync"),
15
22
 
23
+ ("ssh.py", "StatelessSSHConnection.cmd"),
24
+ ("ssh.py", "StatelessSSHConnection.rsync"),
25
+ ("ssh.py", "ManagedSSHConnection.forward"),
26
+ ("ssh.py", "ManagedSSHConnection.cmd"),
27
+ ("ssh.py", "ManagedSSHConnection.rsync"),
28
+ }
16
29
 
17
- def _format_msg(msg, *, skip_frames=0):
18
- stack = inspect.stack()
19
- if len(stack)-1 <= skip_frames:
20
- raise SyntaxError("skip_frames exceeds call stack (frame count)")
21
- stack = stack[skip_frames+1:]
22
30
 
23
- # bottom of the stack, or runpy executed module
24
- for frame_info in stack:
25
- if frame_info.function == "<module>":
31
+ def _log_msg(logger_func, *args, stacklevel=1, **kwargs):
32
+ # inspect.stack() is MUCH slower
33
+ caller = inspect.currentframe().f_back.f_back
34
+ extra_levels = 2 # skip this func and the debug/info/warning parent
35
+ while caller.f_back:
36
+ code = caller.f_code
37
+ # pathlib is much slower
38
+ basename = os.path.basename(code.co_filename) # noqa: PTH119
39
+ qualname = code.co_qualname
40
+ if (basename, qualname) in skip_levels:
41
+ extra_levels += 1
42
+ caller = caller.f_back
43
+ else:
26
44
  break
27
- module = frame_info
28
-
29
- # last (topmost) function that isn't us
30
- parent = stack[0]
31
- function = parent.function
32
-
33
- # if the function has 'self' and it looks like a class instance,
34
- # prepend it to the function name
35
- argvals = inspect.getargvalues(parent.frame)
36
- if argvals.args:
37
- if argvals.args[0] == "self":
38
- self = argvals.locals["self"]
39
- if hasattr(self, "__class__") and inspect.isclass(self.__class__):
40
- function = f"{self.__class__.__name__}.{function}"
41
- elif argvals.args[0] == "cls":
42
- cls = argvals.locals["cls"]
43
- if inspect.isclass(cls):
44
- function = f"{cls.__name__}.{function}"
45
-
46
- # don't report module name of a function if it's the same as running module
47
- if parent.filename != module.filename:
48
- parent_modname = parent.frame.f_globals["__name__"]
49
- # avoid everything having the package name prefixed
50
- parent_modname = parent_modname.partition(".")[2] or parent_modname
51
- return f"{parent_modname}.{function}:{parent.lineno}: {msg}"
52
- elif parent.function != "<module>":
53
- return f"{function}:{parent.lineno}: {msg}"
54
- else:
55
- return f"{Path(parent.filename).name}:{parent.lineno}: {msg}"
56
-
57
-
58
- def debug(msg, *, skip_frames=0):
59
- if in_debug_mode():
60
- _logger.debug(_format_msg(msg, skip_frames=skip_frames+1))
61
- else:
62
- _logger.debug(msg)
63
-
64
-
65
- def info(msg, *, skip_frames=0):
66
- if in_debug_mode():
67
- _logger.info(_format_msg(msg, skip_frames=skip_frames+1))
68
- else:
69
- _logger.info(msg)
70
-
71
-
72
- def warning(msg, *, skip_frames=0):
73
- if in_debug_mode():
74
- _logger.warning(_format_msg(msg, skip_frames=skip_frames+1))
75
- else:
76
- _logger.warning(msg)
45
+ return logger_func(*args, stacklevel=stacklevel+extra_levels, **kwargs)
46
+
47
+
48
+ def warning(*args, **kwargs):
49
+ return _log_msg(_logger.warning, *args, **kwargs)
50
+
51
+
52
+ def info(*args, **kwargs):
53
+ return _log_msg(_logger.info, *args, **kwargs)
54
+
55
+
56
+ def debug(*args, **kwargs):
57
+ return _log_msg(_logger.debug, *args, **kwargs)
58
+
59
+
60
+ # add a log level more verbose than logging.DEBUG, for verbose command
61
+ # outputs, big JSON / XML printouts, and other outputs unsuitable for
62
+ # large parallel runs; to be used in targeted debugging
63
+ #
64
+ # logging.DEBUG is 10, and programs tend to add TRACE as 5, so be somewhere
65
+ # in between
66
+ EXTRADEBUG = 8
67
+ logging.addLevelName(EXTRADEBUG, "EXTRADEBUG")
68
+
69
+
70
+ def extradebug(*args, **kwargs):
71
+ return _log_msg(_logger.log, EXTRADEBUG, *args, **kwargs)
atex/util/subprocess.py CHANGED
@@ -1,52 +1,86 @@
1
1
  import subprocess
2
2
 
3
- from .log import debug
3
+ from .log import debug, extradebug
4
4
 
5
5
 
6
- def subprocess_run(cmd, *, skip_frames=0, **kwargs):
6
+ def subprocess_run(cmd, **kwargs):
7
7
  """
8
8
  A simple wrapper for the real subprocess.run() that logs the command used.
9
9
  """
10
10
  # when logging, skip current stack frame - report the place we were called
11
11
  # from, not util.subprocess_run itself
12
- debug(f"running: {cmd}", skip_frames=skip_frames+1)
12
+ debug(f"running: '{cmd}' with {kwargs=}")
13
13
  return subprocess.run(cmd, **kwargs)
14
14
 
15
15
 
16
- def subprocess_output(cmd, *, skip_frames=0, check=True, text=True, **kwargs):
16
+ def subprocess_output(cmd, *, check=True, text=True, **kwargs):
17
17
  """
18
18
  A wrapper simulating subprocess.check_output() via a modern .run() API.
19
19
  """
20
- debug(f"running: {cmd}", skip_frames=skip_frames+1)
20
+ debug(f"running: '{cmd}' with {check=}, {text=} and {kwargs=}")
21
21
  proc = subprocess.run(cmd, check=check, text=text, stdout=subprocess.PIPE, **kwargs)
22
22
  return proc.stdout.rstrip("\n") if text else proc.stdout
23
23
 
24
24
 
25
- def subprocess_Popen(cmd, *, skip_frames=0, **kwargs): # noqa: N802
25
+ def subprocess_Popen(cmd, **kwargs): # noqa: N802
26
26
  """
27
27
  A simple wrapper for the real subprocess.Popen() that logs the command used.
28
28
  """
29
- debug(f"running: {cmd}", skip_frames=skip_frames+1)
29
+ debug(f"running: '{cmd}' with {kwargs=}")
30
30
  return subprocess.Popen(cmd, **kwargs)
31
31
 
32
32
 
33
- def subprocess_stream(cmd, *, check=False, skip_frames=0, **kwargs):
33
+ def subprocess_stream(cmd, *, stream="stdout", check=False, input=None, **kwargs):
34
34
  """
35
35
  Run 'cmd' via subprocess.Popen() and return an iterator over any lines
36
36
  the command outputs on stdout, in text mode.
37
37
 
38
+ The 'stream' is a subprocess.Popen attribute (either 'stdout' or 'stderr')
39
+ to read from.
40
+ To capture both stdout and stderr as yielded lines, use 'stream="stdout"'
41
+ and pass an additional 'stderr=subprocess.STDOUT'.
42
+
38
43
  With 'check' set to True, raise a CalledProcessError if the 'cmd' failed.
39
44
 
40
- To capture both stdout and stderr as yielded lines, use subprocess.STDOUT.
45
+ Similarly, 'input' simulates the 'input' arg of subprocess.run().
46
+ Note that the input is written to stdin of the process *before* any outputs
47
+ are streamed, so it should be sufficiently small and/or not cause a deadlock
48
+ with the process waiting for outputs to be read before consuming more input.
49
+ Use 'stdin=subprocess.PIPE' and write to it manually if you need more.
41
50
  """
42
- debug(f"running: {cmd}", skip_frames=skip_frames+1)
43
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True, **kwargs)
51
+ all_kwargs = {
52
+ "text": True,
53
+ stream: subprocess.PIPE,
54
+ }
55
+ if input is not None:
56
+ all_kwargs["stdin"] = subprocess.PIPE
57
+ all_kwargs |= kwargs
58
+
59
+ debug(f"running: '{cmd}' with {all_kwargs=}")
60
+ proc = subprocess.Popen(cmd, **all_kwargs)
44
61
 
45
62
  def generate_lines():
46
- for line in proc.stdout:
63
+ if input is not None:
64
+ proc.stdin.write(input)
65
+ proc.stdin.close()
66
+ line_stream = getattr(proc, stream)
67
+ for line in line_stream:
47
68
  yield line.rstrip("\n")
48
69
  code = proc.wait()
49
70
  if code > 0 and check:
50
71
  raise subprocess.CalledProcessError(cmd=cmd, returncode=code)
51
72
 
52
73
  return (proc, generate_lines())
74
+
75
+
76
+ def subprocess_log(cmd, **kwargs):
77
+ """
78
+ A wrapper to stream every (text) line output from the process to the
79
+ logging module.
80
+
81
+ Uses subprocess_stream() to gather the lines.
82
+ """
83
+ debug(f"running: '{cmd}' with {kwargs=}")
84
+ _, lines = subprocess_stream(cmd, **kwargs)
85
+ for line in lines:
86
+ extradebug(line)
atex/util/threads.py CHANGED
@@ -97,3 +97,10 @@ class ThreadQueue:
97
97
  except KeyError:
98
98
  break
99
99
  thread.join()
100
+
101
+ def qsize(self):
102
+ """
103
+ Return the amount of elements .get() can retrieve before it raises
104
+ queue.Empty.
105
+ """
106
+ return self.queue.qsize()
@@ -0,0 +1,86 @@
1
+ Metadata-Version: 2.4
2
+ Name: atex
3
+ Version: 0.10
4
+ Summary: Ad-hoc Test EXecutor
5
+ Project-URL: Homepage, https://github.com/RHSecurityCompliance/atex
6
+ License-Expression: GPL-3.0-or-later
7
+ License-File: COPYING.txt
8
+ Classifier: Operating System :: POSIX :: Linux
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Topic :: Software Development :: Testing
11
+ Requires-Python: >=3.11
12
+ Requires-Dist: fmf>=1.6
13
+ Requires-Dist: pyyaml
14
+ Requires-Dist: urllib3<3,>=2
15
+ Description-Content-Type: text/markdown
16
+
17
+ # ATEX = Ad-hoc Test EXecutor
18
+
19
+ A collections of Python APIs to provision operating systems, collect
20
+ and execute [FMF](https://github.com/teemtee/fmf/)-style tests, gather
21
+ and organize their results and generate reports from those results.
22
+
23
+ The name comes from a (fairly unique to FMF/TMT ecosystem) approach that
24
+ allows provisioning a pool of systems and scheduling tests on them as one would
25
+ on an ad-hoc pool of thread/process workers - once a worker becomes free,
26
+ it receives a test to run.
27
+ This is in contrast to splitting a large list of N tests onto M workers
28
+ like N/M, which yields significant time penalties due to tests having
29
+ very varies runtimes.
30
+
31
+ Above all, this project is meant to be a toolbox, not a silver-plate solution.
32
+ Use its Python APIs to build a CLI tool for your specific use case.
33
+ The CLI tool provided here is just for demonstration / testing, not for serious
34
+ use - we want to avoid huge modular CLIs for Every Possible Scenario. That's
35
+ the job of the Python API. Any CLI should be simple by nature.
36
+
37
+ ---
38
+
39
+ ## License
40
+
41
+ Unless specified otherwise, any content within this repository is distributed
42
+ under the GNU GPLv3 license, see the [COPYING.txt](COPYING.txt) file for more.
43
+
44
+ ## Environment variables
45
+
46
+ - `ATEX_DEBUG_TEST`
47
+ - Set to `1` to print out detailed runner-related trace within the test output
48
+ stream (as if it was printed out by the test).
49
+
50
+ ## Testing this project
51
+
52
+ There are some limited sanity tests provided via `pytest`, although:
53
+
54
+ - Some require additional variables (ie. Testing Farm) and will ERROR
55
+ without them.
56
+ - Some take a long time (ie. Testing Farm) due to system provisioning
57
+ taking a long time, so install `pytest-xdist` and run with a large `-n`.
58
+
59
+ Currently, the recommended approach is to split the execution:
60
+
61
+ ```
62
+ # synchronously, because podman CLI has concurrency issues
63
+ pytest tests/provision/test_podman.py
64
+
65
+ # in parallel, because provisioning takes a long time
66
+ export TESTING_FARM_API_TOKEN=...
67
+ export TESTING_FARM_COMPOSE=...
68
+ pytest -n 20 tests/provision/test_podman.py
69
+
70
+ # fast enough for synchronous execution
71
+ pytest tests/fmf
72
+ ```
73
+
74
+ ## Unsorted notes
75
+
76
+ TODO: codestyle from contest
77
+
78
+ ```
79
+ - this is not tmt, the goal is to make a python toolbox *for* making runcontest
80
+ style tools easily, not to replace those tools with tmt-style CLI syntax
81
+
82
+ - the whole point is to make usecase-targeted easy-to-use tools that don't
83
+ intimidate users with 1 KB long command line, and runcontest is a nice example
84
+
85
+ - TL;DR - use a modular pythonic approach, not a gluetool-style long CLI
86
+ ```
@@ -0,0 +1,44 @@
1
+ atex/__init__.py,sha256=LdX67gprtHYeAkjLhFPKzpc7ECv2rHxUbHKDGbGXO1c,517
2
+ atex/fmf.py,sha256=gkJXIaRO7_KvwJR-V6Tc1NVn4a9Hq7hoBLQLhxYIdbg,8834
3
+ atex/aggregator/__init__.py,sha256=uNnYSyDGXjknxckI8MFfl-C8_gin8FwQchiq-UOyP6I,1744
4
+ atex/aggregator/json.py,sha256=x1zim9O2olzBh185NYWo5N96fixB2oxCamoOZwmgR9w,3330
5
+ atex/cli/__init__.py,sha256=X5XxkDEDXE4tJAjwt5ShRHCFTXDK-2zvxQ34opmueUc,2768
6
+ atex/cli/fmf.py,sha256=HfbTgFbCwK4Nuyq6vtGutcq_4-4kj-tmoqzXUn3AYtY,3573
7
+ atex/cli/libvirt.py,sha256=6tt5ANb8XBBRXOQsYPTWILThKqf-gvt5AZh5Dctg2PA,3782
8
+ atex/cli/testingfarm.py,sha256=HGlqrkhanUMo2CqKxmM3ACgptWtxm0gICyEGf7O6Qc0,9078
9
+ atex/connection/__init__.py,sha256=dj8ZBcEspom7Z_UjecfLGBRNvLZ3dyGR9q19i_B4xpY,3880
10
+ atex/connection/podman.py,sha256=1T56gh1TgbcQWpTIJHL4NaxZOI6aMg7Xp7sn6PQQyBk,1911
11
+ atex/connection/ssh.py,sha256=9A57b9YR_HI-kIu06Asic1y__JPVXEheDZxjbG2Qcsc,13460
12
+ atex/executor/__init__.py,sha256=XCfhi7QDELjey7N1uzhMjc46Kp1Jsd5bOCf52I27SCE,85
13
+ atex/executor/duration.py,sha256=x06sItKOZi6XA8KszQwZGpIb1Z_L-HWqIwZKo2SDo0s,1759
14
+ atex/executor/executor.py,sha256=JLFR9cZjSlUdAlAlLct6WuzmYbtjGtSobxvsToQum6M,15738
15
+ atex/executor/reporter.py,sha256=MceFmHFt0bTEClBZbRI1WnFbfMhR0e1noOzcu7gjKuQ,3403
16
+ atex/executor/scripts.py,sha256=riJAQWsV-BFGkJwR2Dmf3R0ZRRZJs9w9iYnPpYaQNaE,5618
17
+ atex/executor/testcontrol.py,sha256=mVrLwQUnDRfUq-5diz-80UvCWWxn1TkcBgmAKhKNb5E,12696
18
+ atex/orchestrator/__init__.py,sha256=8Q1YknyibilXLjWRYkHm_Mr2HMm0SRw8Zv39KypeASM,2059
19
+ atex/orchestrator/adhoc.py,sha256=GnvHLlCHeJ_nQ8doEjMuDzqmu4XZorI7ZzOtG_C08tU,18451
20
+ atex/provisioner/__init__.py,sha256=2eepmEznq94tbam9VSWbsGFrZZpWeNSVlsTczGxjNuQ,4667
21
+ atex/provisioner/libvirt/VM_PROVISION,sha256=7pkZ-ozgTyK4qNGC-E-HUznr4IhbosWSASbB72Gknl8,2664
22
+ atex/provisioner/libvirt/__init__.py,sha256=pKG5IpZSC2IHs5wL2ecQx_fd9AzAXEbZmDzA7RyZsfM,119
23
+ atex/provisioner/libvirt/libvirt.py,sha256=rtxowv5DpgcWsGRXYF29n6S9x_cgXRVgqY41DiFu920,18431
24
+ atex/provisioner/libvirt/locking.py,sha256=AXtDyidZNmUoMmrit26g9iTHDqInrzL_RSQEoc_EAXw,5669
25
+ atex/provisioner/libvirt/setup-libvirt.sh,sha256=oCMy9SCnbC_QuAzO2sFwvB5ui1kMQ6uviHsgdXyoFXc,2428
26
+ atex/provisioner/podman/__init__.py,sha256=dM0JzQXWX7edtWSc0KH0cMFXAjArFn2Vme4j_ZMsdYA,138
27
+ atex/provisioner/podman/podman.py,sha256=ztRypoakSf-jF04iER58tEMUZ4Y6AuzIpNpFXp44bB4,4997
28
+ atex/provisioner/testingfarm/__init__.py,sha256=kZncgLGdRCR4FMaRQr2GTwJ8vjlA-24ri8JO2ueZJuw,113
29
+ atex/provisioner/testingfarm/api.py,sha256=UcMN61nBr3wqEd5KSR5Xhv1-TS7nSPFvk2byb6PdIs8,21811
30
+ atex/provisioner/testingfarm/testingfarm.py,sha256=OI-a99xALaiYf-y5037WFVxY1g2H2y1xEKxHBdUQvfg,8271
31
+ atex/util/__init__.py,sha256=cWHFbtQ4mDlKe6lXyPDWRmWJOTcHDGfVuW_-GYa8hB0,1473
32
+ atex/util/dedent.py,sha256=SEuJMtLzqz3dQ7g7qyZzEJ9VYynVlk52tQCJY-FveXo,603
33
+ atex/util/libvirt.py,sha256=kDZmT6xLYEZkQNLZY98gJ2M48DDWXxHF8rQY9PnjB3U,660
34
+ atex/util/log.py,sha256=KVR7ep8n5wtghsvBFCtHiPsMAQBdAmK83E_Jec5t4cU,2230
35
+ atex/util/named_mapping.py,sha256=UBMe9TetjV-DGPhjYjJ42YtC40FVPKAAEROXl9MA5fo,4700
36
+ atex/util/path.py,sha256=x-kXqiWCVodfZWbEwtC5A8LFvutpDIPYv2m0boZSlXU,504
37
+ atex/util/ssh_keygen.py,sha256=9yuSl2yBV7pG3Qfsf9tossVC00nbIUrAeLdbwTykpjk,384
38
+ atex/util/subprocess.py,sha256=PQBxcQJPapP1ZLO4LqENyrxxCbNAxtJDNNlBV5DcD9k,2953
39
+ atex/util/threads.py,sha256=c8hsEc-8SqJGodInorv_6JxpiHiSkGFGob4qbMmOD2M,3531
40
+ atex-0.10.dist-info/METADATA,sha256=evOBYvVboY2T8eGrAKy64UpyeuFKSMLGz8pUz8Sstm8,3050
41
+ atex-0.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
42
+ atex-0.10.dist-info/entry_points.txt,sha256=pLqJdcfeyQTgup2h6dWb6SvkHhtOl-W5Eg9zV8moK0o,39
43
+ atex-0.10.dist-info/licenses/COPYING.txt,sha256=oEuj51jdmbXcCUy7pZ-KE0BNcJTR1okudRp5zQ0yWnU,670
44
+ atex-0.10.dist-info/RECORD,,
@@ -1 +0,0 @@
1
- from .podman import PodmanProvisioner, PodmanRemote # noqa: F401
@@ -1,274 +0,0 @@
1
- import os
2
- import time
3
- import enum
4
- import threading
5
- import subprocess
6
-
7
- from ... import connection, util
8
- from .. import Provisioner, Remote
9
-
10
-
11
- class PodmanRemote(Remote, connection.podman.PodmanConn):
12
- """
13
- Built on the official Remote API, pulling in the Connection API
14
- as implemented by ManagedSSHConn.
15
- """
16
-
17
- def __init__(self, image, container, *, release_hook):
18
- """
19
- 'image' is an image tag (used for repr()).
20
-
21
- 'container' is a podman container id / name.
22
-
23
- 'release_hook' is a callable called on .release() in addition
24
- to disconnecting the connection.
25
- """
26
- super().__init__(container=container)
27
- self.lock = threading.RLock()
28
- self.image = image
29
- self.container = container
30
- self.release_called = False
31
- self.release_hook = release_hook
32
-
33
- def release(self):
34
- with self.lock:
35
- if self.release_called:
36
- return
37
- else:
38
- self.release_called = True
39
- self.release_hook(self)
40
- self.disconnect()
41
- util.subprocess_run(
42
- ("podman", "container", "rm", "-f", "-t", "0", self.container),
43
- check=False, # ignore if it fails
44
- stdout=subprocess.DEVNULL,
45
- )
46
-
47
- # not /technically/ a valid repr(), but meh
48
- def __repr__(self):
49
- class_name = self.__class__.__name__
50
-
51
- if "/" in self.image:
52
- image = self.image.rsplit("/",1)[1]
53
- elif len(self.image) > 20:
54
- image = f"{self.image[:17]}..."
55
- else:
56
- image = self.image
57
-
58
- name = f"{self.container[:17]}..." if len(self.container) > 20 else self.container
59
-
60
- return f"{class_name}({image}, {name})"
61
-
62
-
63
- class PodmanProvisioner(Provisioner):
64
- class State(enum.Enum):
65
- WAITING_FOR_PULL = enum.auto()
66
- CREATING_CONTAINER = enum.auto()
67
- WAITING_FOR_CREATION = enum.auto()
68
- SETTING_UP_CONTAINER = enum.auto()
69
- WAITING_FOR_SETUP = enum.auto()
70
-
71
- # NOTE: this uses a single Popen process to run podman commands,
72
- # to avoid double downloads/pulls, but also to avoid SQLite errors
73
- # when creating multiple containers in parallel
74
-
75
- def __init__(self, image, run_options=None, *, pull=True, max_systems=1):
76
- """
77
- 'image' is a string of image tag/id to create containers from.
78
- It can be a local identifier or an URL.
79
-
80
- 'run_options' is an iterable with additional CLI options passed
81
- to 'podman container run'.
82
-
83
- 'pull' specifies whether to attempt 'podman image pull' on the specified
84
- image tag/id before any container creation.
85
-
86
- 'max_systems' is a maximum number of containers running at any one time.
87
- """
88
- self.lock = threading.RLock()
89
- self.image = image
90
- self.run_options = run_options or ()
91
- self.pull = pull
92
- self.max_systems = max_systems
93
-
94
- self.image_id = None
95
- self.container_id = None
96
- self.worker = None
97
- self.worker_output = bytearray()
98
- self.state = None
99
- # created PodmanRemote instances, ready to be handed over to the user,
100
- # or already in use by the user
101
- self.remotes = []
102
-
103
- @staticmethod
104
- def _spawn_proc(cmd):
105
- proc = util.subprocess_Popen(
106
- cmd,
107
- stdin=subprocess.DEVNULL,
108
- stdout=subprocess.PIPE,
109
- stderr=subprocess.STDOUT,
110
- )
111
- os.set_blocking(proc.stdout.fileno(), False)
112
- return proc
113
-
114
- # @staticmethod
115
- # def _poll_proc(proc):
116
- # # read from the process to un-block any kernel buffers
117
- # try:
118
- # out = proc.stdout.read() # non-blocking
119
- # except BlockingIOError:
120
- # out = ""
121
- # return (proc.poll(), out)
122
-
123
- def _make_remote(self, container):
124
- def release_hook(remote):
125
- # remove from the list of remotes inside this Provisioner
126
- with self.lock:
127
- try:
128
- self.remotes.remove(remote)
129
- except ValueError:
130
- pass
131
-
132
- remote = PodmanRemote(
133
- self.image,
134
- container,
135
- release_hook=release_hook,
136
- )
137
- self.remotes.append(remote)
138
- return remote
139
-
140
- def start(self):
141
- if not self.image:
142
- raise ValueError("image cannot be empty")
143
-
144
- if not self.pull:
145
- self.image_id = self.image
146
- self.state = self.State.CREATING_CONTAINER
147
- else:
148
- self.worker = self._spawn_proc(
149
- ("podman", "image", "pull", "--quiet", self.image),
150
- )
151
- self.state = self.State.WAITING_FOR_PULL
152
-
153
- def stop(self):
154
- with self.lock:
155
- while self.remotes:
156
- self.remotes.pop().release()
157
- worker = self.worker
158
- self.worker = None
159
-
160
- if worker:
161
- worker.kill()
162
- # don"t zombie forever, return EPIPE on any attempts to write to us
163
- worker.stdout.close()
164
- worker.wait()
165
-
166
- def stop_defer(self):
167
- # avoid SQLite errors by removing containers sequentially
168
- return self.stop
169
-
170
- @staticmethod
171
- def _nonblock_read(fobj):
172
- """Return b'' if there was nothing to read, instead of None."""
173
- data = fobj.read()
174
- return b"" if data is None else data
175
-
176
- def _get_remote_nonblock(self):
177
- if self.state is None:
178
- raise RuntimeError("the provisioner is in an invalid state")
179
-
180
- # NOTE: these are not 'elif' statements explicitly to allow a next block
181
- # to follow a previous one (if the condition is met)
182
- if self.state is self.State.WAITING_FOR_PULL:
183
- self.worker_output += self._nonblock_read(self.worker.stdout)
184
- rc = self.worker.poll()
185
- if rc is None:
186
- return None # still running
187
- elif rc != 0:
188
- out = self.worker_output.decode().rstrip("\n")
189
- self.worker_output.clear()
190
- self.worker = None
191
- self.state = None
192
- raise RuntimeError(f"podman image pull failed with {rc}:\n{out}")
193
- else:
194
- self.image_id = self.worker_output.decode().rstrip("\n")
195
- self.worker_output.clear()
196
- self.worker = None
197
- self.state = self.State.CREATING_CONTAINER
198
-
199
- if self.state is self.State.CREATING_CONTAINER:
200
- if len(self.remotes) < self.max_systems:
201
- self.worker = self._spawn_proc(
202
- (
203
- "podman", "container", "run", "--quiet", "--detach", "--pull", "never",
204
- *self.run_options, self.image_id, "sleep", "inf",
205
- ),
206
- )
207
- self.state = self.State.WAITING_FOR_CREATION
208
- else:
209
- # too many remotes requested
210
- return None
211
-
212
- if self.state is self.State.WAITING_FOR_CREATION:
213
- self.worker_output += self._nonblock_read(self.worker.stdout)
214
- rc = self.worker.poll()
215
- if rc is None:
216
- return None # still running
217
- elif rc != 0:
218
- out = self.worker_output.decode().rstrip("\n")
219
- self.worker_output.clear()
220
- self.worker = None
221
- self.state = None
222
- raise RuntimeError(f"podman run failed with {rc}:\n{out}")
223
- else:
224
- self.container_id = self.worker_output.decode().rstrip("\n")
225
- self.worker_output.clear()
226
- self.worker = None
227
- self.state = self.State.SETTING_UP_CONTAINER
228
-
229
- if self.state is self.State.SETTING_UP_CONTAINER:
230
- cmd = ("dnf", "install", "-y", "-q", "--setopt=install_weak_deps=False", "rsync")
231
- self.worker = self._spawn_proc(
232
- ("podman", "container", "exec", self.container_id, *cmd),
233
- )
234
- self.state = self.State.WAITING_FOR_SETUP
235
-
236
- if self.state is self.State.WAITING_FOR_SETUP:
237
- self.worker_output += self._nonblock_read(self.worker.stdout)
238
- rc = self.worker.poll()
239
- if rc is None:
240
- return None # still running
241
- elif rc != 0:
242
- out = self.worker_output.decode().rstrip("\n")
243
- self.worker_output.clear()
244
- self.worker = None
245
- self.state = None
246
- raise RuntimeError(f"setting up failed with {rc}:\n{out}")
247
- else:
248
- # everything ready, give the Remote to the caller and reset
249
- remote = self._make_remote(self.container_id)
250
- self.worker_output.clear()
251
- self.worker = None
252
- self.state = self.State.CREATING_CONTAINER
253
- return remote
254
-
255
- raise AssertionError(f"reached end (invalid state {self.state}?)")
256
-
257
- def get_remote(self, block=True):
258
- if not block:
259
- with self.lock:
260
- return self._get_remote_nonblock()
261
- else:
262
- while True:
263
- with self.lock:
264
- if remote := self._get_remote_nonblock():
265
- return remote
266
- time.sleep(0.1)
267
-
268
- # not /technically/ a valid repr(), but meh
269
- def __repr__(self):
270
- class_name = self.__class__.__name__
271
- return (
272
- f"{class_name}({self.image}, {len(self.remotes)}/{self.max_systems} remotes, "
273
- f"{hex(id(self))})"
274
- )