marsh-lib 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- marsh/__init__.py +2 -0
- marsh/bash/__init__.py +3 -0
- marsh/bash/bash_factory.py +74 -0
- marsh/bash/bash_grammar.py +44 -0
- marsh/bash/bash_runner_decorators.py +0 -0
- marsh/bash/bash_script.py +71 -0
- marsh/constants.py +5 -0
- marsh/core/__init__.py +8 -0
- marsh/core/authenticator.py +7 -0
- marsh/core/cmd_run_decorator.py +289 -0
- marsh/core/command_grammar.py +45 -0
- marsh/core/connector.py +77 -0
- marsh/core/conveyor.py +105 -0
- marsh/core/executor.py +296 -0
- marsh/core/expression.py +208 -0
- marsh/core/script.py +38 -0
- marsh/dag/__init__.py +11 -0
- marsh/dag/dag.py +748 -0
- marsh/dag/node.py +52 -0
- marsh/dag/startable.py +48 -0
- marsh/docker/__init__.py +2 -0
- marsh/docker/docker_command_grammar.py +31 -0
- marsh/docker/docker_executor.py +181 -0
- marsh/exceptions.py +16 -0
- marsh/logger.py +116 -0
- marsh/modifier_functions/__init__.py +2 -0
- marsh/modifier_functions/case_conversion.py +17 -0
- marsh/modifier_functions/readers.py +11 -0
- marsh/powershell/__init__.py +0 -0
- marsh/processor_functions/__init__.py +3 -0
- marsh/processor_functions/printers.py +63 -0
- marsh/processor_functions/raisers.py +8 -0
- marsh/processor_functions/redirections.py +39 -0
- marsh/signals.py +1 -0
- marsh/ssh/__init__.py +3 -0
- marsh/ssh/ssh_command_grammar.py +50 -0
- marsh/ssh/ssh_connector.py +71 -0
- marsh/ssh/ssh_factory.py +93 -0
- marsh/utils/__init__.py +16 -0
- marsh/utils/output_streams.py +13 -0
- marsh_lib-0.2.0.dist-info/LICENSE +21 -0
- marsh_lib-0.2.0.dist-info/METADATA +380 -0
- marsh_lib-0.2.0.dist-info/RECORD +44 -0
- marsh_lib-0.2.0.dist-info/WHEEL +4 -0
marsh/core/conveyor.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
from typing import Callable, Tuple, Sequence
|
|
2
|
+
|
|
3
|
+
from .cmd_run_decorator import CmdRunDecorator
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Conveyor:
|
|
7
|
+
"""
|
|
8
|
+
A class that chains multiple command runners, allowing sequential execution of commands.
|
|
9
|
+
|
|
10
|
+
Each command runner is a callable that accepts two `bytes` arguments (representing standard
|
|
11
|
+
output and standard error) and returns a tuple of two `bytes` values as the updated outputs.
|
|
12
|
+
|
|
13
|
+
This class supports adding command runners, decorating them, and invoking them in sequence.
|
|
14
|
+
|
|
15
|
+
Example:
|
|
16
|
+
>>> def cmd_runner_1(stdout: bytes, stderr: bytes) -> Tuple[bytes, bytes]:
|
|
17
|
+
... return stdout + b"Cmd1", stderr
|
|
18
|
+
|
|
19
|
+
>>> def cmd_runner_2(stdout: bytes, stderr: bytes) -> Tuple[bytes, bytes]:
|
|
20
|
+
... return stdout + b"Cmd2", stderr
|
|
21
|
+
|
|
22
|
+
>>> conveyor = Conveyor()
|
|
23
|
+
>>> conveyor = conveyor.add_cmd_runner(cmd_runner_1).add_cmd_runner(cmd_runner_2)
|
|
24
|
+
>>> result_stdout, result_stderr = conveyor(b"Start", b"")
|
|
25
|
+
>>> print(result_stdout)
|
|
26
|
+
b'StartCmd1Cmd2'
|
|
27
|
+
"""
|
|
28
|
+
def __init__(self, cmds_=None) -> None:
|
|
29
|
+
self._cmd_runners = cmds_ or []
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def cmd_run_triples(self) -> Sequence[Tuple[Callable, Tuple, dict]]:
|
|
33
|
+
"""
|
|
34
|
+
Returns the sequence of triples containing the registered command runner, its positional arguments and keyword arguments.
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
Sequence[Tuple[Callable, Tuple, dict]]:
|
|
38
|
+
A sequence of command runners with their arguments.
|
|
39
|
+
"""
|
|
40
|
+
return self._cmd_runners
|
|
41
|
+
|
|
42
|
+
def add_cmd_runner(self,
|
|
43
|
+
cmd_runner: Callable[[bytes, bytes], Tuple[bytes, bytes]],
|
|
44
|
+
*args,
|
|
45
|
+
cmd_runner_decorator: CmdRunDecorator | None = None,
|
|
46
|
+
**kwargs
|
|
47
|
+
) -> "Conveyor":
|
|
48
|
+
"""
|
|
49
|
+
Adds a new command runner to the Conveyor.
|
|
50
|
+
|
|
51
|
+
Optionally applies a decorator to the command runner before adding it to the chain.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
cmd_runner (Callable[[bytes, bytes], Tuple[bytes, bytes]]):
|
|
55
|
+
A callable that processes two `bytes` inputs (stdout, stderr)
|
|
56
|
+
and returns a tuple of two `bytes` outputs.
|
|
57
|
+
*args:
|
|
58
|
+
Positional arguments to pass to the `cmd_runner` during invocation.
|
|
59
|
+
cmd_runner_decorator (CmdRunDecorator, optional):
|
|
60
|
+
A command runner decorator for decorating the `cmd_runner`. Defaults to None.
|
|
61
|
+
**kwargs:
|
|
62
|
+
Keyword arguments to pass to the `cmd_runner` during invocation.
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
Conveyor: A new Conveyor instance with the added command runner.
|
|
66
|
+
|
|
67
|
+
Raises:
|
|
68
|
+
TypeError: If `cmd_runner` is not callable.
|
|
69
|
+
"""
|
|
70
|
+
if not callable(cmd_runner):
|
|
71
|
+
raise TypeError("cmd_runner must be callable with 2 required bytes arguments.")
|
|
72
|
+
|
|
73
|
+
# Apply the pre/post processors and modifiers to command runner, if given
|
|
74
|
+
decorated_cmd_runner = cmd_runner_decorator.decorate(cmd_runner) if isinstance(cmd_runner_decorator, CmdRunDecorator) else cmd_runner
|
|
75
|
+
|
|
76
|
+
return Conveyor(cmds_=self._cmd_runners + [(decorated_cmd_runner, args, kwargs)])
|
|
77
|
+
|
|
78
|
+
def __call__(self, x_stdout: bytes = b"", x_stderr: bytes = b"", callback_list_: Sequence = None) -> Tuple[bytes, bytes]:
|
|
79
|
+
"""
|
|
80
|
+
Executes the command runners sequentially.
|
|
81
|
+
|
|
82
|
+
Starts with the given `stdout` and `stderr` values and passes the results through
|
|
83
|
+
each command runner in the chain.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
x_stdout (bytes, optional): Initial standard output. Defaults to `b""`.
|
|
87
|
+
x_stderr (bytes, optional): Initial standard error. Defaults to `b""`.
|
|
88
|
+
callback_list_ (Sequence, optional):
|
|
89
|
+
A list of command runners to execute. Defaults to the command runners in the Conveyor.
|
|
90
|
+
|
|
91
|
+
Returns:
|
|
92
|
+
Tuple[bytes, bytes]:
|
|
93
|
+
A tuple containing the final standard output (`stdout`) and standard error (`stderr`).
|
|
94
|
+
"""
|
|
95
|
+
callback_list = callback_list_ or self._cmd_runners
|
|
96
|
+
if len(callback_list) == 1:
|
|
97
|
+
callback, args, kwargs = callback_list[0]
|
|
98
|
+
return callback(x_stdout, x_stderr, *args, **kwargs)
|
|
99
|
+
|
|
100
|
+
if len(self._cmd_runners) == 0:
|
|
101
|
+
return x_stdout, x_stderr
|
|
102
|
+
|
|
103
|
+
callback, args, kwargs = callback_list[0]
|
|
104
|
+
new_x_stdout, new_x_stderr = callback(x_stdout, x_stderr, *args, **kwargs)
|
|
105
|
+
return self(x_stdout=new_x_stdout, x_stderr=new_x_stderr, callback_list_=callback_list[1:])
|
marsh/core/executor.py
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
import textwrap
|
|
2
|
+
import pickle
|
|
3
|
+
import subprocess
|
|
4
|
+
from abc import abstractmethod, ABC
|
|
5
|
+
from typing import Callable, Tuple, Any
|
|
6
|
+
|
|
7
|
+
from ..utils.output_streams import suppress_output
|
|
8
|
+
from .command_grammar import CommandGrammar
|
|
9
|
+
from .connector import Connector
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Executor(ABC):
|
|
13
|
+
"""
|
|
14
|
+
Abstract base class for command execution.
|
|
15
|
+
|
|
16
|
+
This class defines a common interface for running commands, where subclasses implement
|
|
17
|
+
the `run` method to execute commands in specific environments (e.g., local, remote).
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def run(self, x_stdout: bytes, x_stderr: bytes, *args, **kwargs) -> Tuple[bytes, bytes]:
|
|
22
|
+
"""
|
|
23
|
+
Abstract method to run a command.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
x_stdout (bytes): Standard output from a previous command.
|
|
27
|
+
x_stderr (bytes): Standard error from a previous command.
|
|
28
|
+
*args: Additional positional arguments.
|
|
29
|
+
**kwargs: Additional keyword arguments.
|
|
30
|
+
|
|
31
|
+
Returns:
|
|
32
|
+
Tuple[bytes, bytes]: A tuple containing standard output and standard error.
|
|
33
|
+
|
|
34
|
+
Raises:
|
|
35
|
+
NotImplementedError: If the subclass does not override this method.
|
|
36
|
+
"""
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class LocalCommandExecutor(Executor):
|
|
41
|
+
"""Executes commands locally as subprocesses."""
|
|
42
|
+
|
|
43
|
+
def __init__(self,
|
|
44
|
+
command_grammar: CommandGrammar,
|
|
45
|
+
pipe_prev_stdout: bool = False,
|
|
46
|
+
timeout: float | None = None,
|
|
47
|
+
):
|
|
48
|
+
"""
|
|
49
|
+
Initializes a LocalCommandExecutor.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
command_grammar (CommandGrammar): A CommandGrammar object to build the command.
|
|
53
|
+
pipe_prev_stdout (bool, optional): Whether to pipe the previous standard output as input. Defaults to False.
|
|
54
|
+
timeout (float | None, optional): Timeout for command execution in seconds. Defaults to None.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
self.command_grammar = command_grammar # Already parameterized the command grammar
|
|
58
|
+
self.pipe_prev_stdout = pipe_prev_stdout # (Unix) Pipe the previous STDOUT as STDIN for current command runner
|
|
59
|
+
self.timeout = timeout
|
|
60
|
+
|
|
61
|
+
@staticmethod
|
|
62
|
+
def create_popen_with_pipe(command: list[str], *args, **kwargs) -> subprocess.Popen:
|
|
63
|
+
"""
|
|
64
|
+
Creates a subprocess with pipes for stdin, stdout, and stderr.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
command (list[str]): The command to execute as a list of strings.
|
|
68
|
+
*args: Additional positional arguments for `subprocess.Popen`.
|
|
69
|
+
**kwargs: Additional keyword arguments for `subprocess.Popen`.
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
subprocess.Popen: A subprocess instance with pipes.
|
|
73
|
+
"""
|
|
74
|
+
return subprocess.Popen(command, *args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
75
|
+
**kwargs)
|
|
76
|
+
|
|
77
|
+
def run(self,
|
|
78
|
+
x_stdout: bytes,
|
|
79
|
+
x_stderr: bytes,
|
|
80
|
+
*args,
|
|
81
|
+
callback: Callable[[subprocess.Popen, bytes, bytes], Tuple[bytes, bytes]] = None,
|
|
82
|
+
popen_args=(),
|
|
83
|
+
popen_kwargs=None,
|
|
84
|
+
**kwargs
|
|
85
|
+
) -> Tuple[bytes, bytes]:
|
|
86
|
+
"""
|
|
87
|
+
Runs a command locally.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
x_stdout (bytes): Standard output to pass to the command.
|
|
91
|
+
x_stderr (bytes): Standard error to pass to the command.
|
|
92
|
+
*args: Additional positional arguments.
|
|
93
|
+
callback (Callable, optional): A custom callback function that takes subprocess.Popen, stdout, and stderr.
|
|
94
|
+
This callback must return tuple[bytes, bytes] which represents the result.
|
|
95
|
+
popen_args (tuple, optional): Arguments for `subprocess.Popen`. Defaults to ().
|
|
96
|
+
popen_kwargs (dict, optional): Keyword arguments for `subprocess.Popen`. Defaults to None.
|
|
97
|
+
**kwargs: Additional keyword arguments for `subprocess.Popen`.
|
|
98
|
+
|
|
99
|
+
Returns:
|
|
100
|
+
Tuple[bytes, bytes]: A tuple containing standard output and standard error.
|
|
101
|
+
|
|
102
|
+
Raises:
|
|
103
|
+
ValueError: If the provided callback does not return a tuple of bytes.
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
popen_kwargs = popen_kwargs or dict()
|
|
107
|
+
|
|
108
|
+
# Build the Command as List of Strings
|
|
109
|
+
command = self.command_grammar.build_cmd()
|
|
110
|
+
|
|
111
|
+
# Create subprocess.Popen
|
|
112
|
+
process = self.create_popen_with_pipe(command, *args, **kwargs)
|
|
113
|
+
|
|
114
|
+
# Use the custom callback provided by user/client
|
|
115
|
+
if callback:
|
|
116
|
+
result = callback(process, x_stdout, x_stderr, *popen_args, **popen_kwargs)
|
|
117
|
+
match result:
|
|
118
|
+
# The `process.communicate(...)` must be invoked in the `callback` to return (stdout, stderr).
|
|
119
|
+
case (stdout, stderr) if isinstance(stdout, bytes) and isinstance(stderr, bytes):
|
|
120
|
+
return stdout, stderr
|
|
121
|
+
case _:
|
|
122
|
+
raise ValueError("Given callback must return tuple[bytes, bytes]")
|
|
123
|
+
|
|
124
|
+
# Use the default procedure for running programs
|
|
125
|
+
stdout, stderr = process.communicate(input=x_stdout,
|
|
126
|
+
timeout=self.timeout) if self.pipe_prev_stdout else process.communicate(
|
|
127
|
+
timeout=self.timeout)
|
|
128
|
+
return stdout, stderr
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class RemoteCommandExecutor(Executor):
|
|
132
|
+
"""
|
|
133
|
+
Executes commands on remote systems via a connector interface.
|
|
134
|
+
|
|
135
|
+
This class uses a `Connector` to establish a connection to a remote host
|
|
136
|
+
and execute a command generated by a `CommandGrammar`. The command is sent
|
|
137
|
+
to the remote system, and the standard output and error are retrieved.
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
def __init__(self, connector: Connector, command_grammar: CommandGrammar):
|
|
141
|
+
"""
|
|
142
|
+
Initializes a RemoteCommandExecutor.
|
|
143
|
+
|
|
144
|
+
Args:
|
|
145
|
+
connector (Connector): An object responsible for managing the connection to the remote system.
|
|
146
|
+
command_grammar (CommandGrammar): An object to build and format the command to be executed remotely.
|
|
147
|
+
"""
|
|
148
|
+
self.connector = connector
|
|
149
|
+
self.command_grammar = command_grammar
|
|
150
|
+
|
|
151
|
+
def run(self,
|
|
152
|
+
x_stdout: bytes,
|
|
153
|
+
x_stderr: bytes,
|
|
154
|
+
*build_cmd_args,
|
|
155
|
+
conn_args=(),
|
|
156
|
+
exec_cmd_args=(),
|
|
157
|
+
exec_cmd_kwargs=None,
|
|
158
|
+
conn_kwargs=None,
|
|
159
|
+
**build_cmd_kwargs,
|
|
160
|
+
) -> Tuple[bytes, bytes]:
|
|
161
|
+
"""
|
|
162
|
+
Runs a command on a remote system and retrieves its output.
|
|
163
|
+
|
|
164
|
+
This method connects to a remote host using the provided `Connector`, runs the command
|
|
165
|
+
generated by `CommandGrammar`, and returns the standard output and standard error.
|
|
166
|
+
|
|
167
|
+
Args:
|
|
168
|
+
x_stdout (bytes): Standard output to pass as input, if required by the command.
|
|
169
|
+
x_stderr (bytes): Standard error to pass as input, if required by the command.
|
|
170
|
+
*build_cmd_args: Additional arguments passed to `CommandGrammar.build_cmd`.
|
|
171
|
+
conn_args (tuple, optional): Positional arguments for the `Connector.connect` method.
|
|
172
|
+
exec_cmd_args (tuple, optional): Positional arguments for the `Connector.exec_cmd` method.
|
|
173
|
+
exec_cmd_kwargs (dict, optional): Keyword arguments for `Connector.exec_cmd`. Defaults to None.
|
|
174
|
+
conn_kwargs (dict, optional): Keyword arguments for the `Connector.connect` method. Defaults to None.
|
|
175
|
+
**build_cmd_kwargs: Additional keyword arguments for `CommandGrammar.build_cmd`.
|
|
176
|
+
|
|
177
|
+
Returns:
|
|
178
|
+
Tuple[bytes, bytes]: A tuple containing:
|
|
179
|
+
- stdout (bytes): The standard output from the remote command execution.
|
|
180
|
+
- stderr (bytes): The standard error from the remote command execution.
|
|
181
|
+
"""
|
|
182
|
+
exec_cmd_kwargs = exec_cmd_kwargs or dict()
|
|
183
|
+
conn_kwargs = conn_kwargs or dict()
|
|
184
|
+
connection = None
|
|
185
|
+
# TODO: Enhance and include Pipes.
|
|
186
|
+
try:
|
|
187
|
+
connection = self.connector.connect(*conn_args, **conn_kwargs)
|
|
188
|
+
stdout, stderr = self.connector.exec_cmd(
|
|
189
|
+
self.command_grammar.build_cmd(*build_cmd_args, **build_cmd_kwargs),
|
|
190
|
+
connection,
|
|
191
|
+
*exec_cmd_args,
|
|
192
|
+
**exec_cmd_kwargs
|
|
193
|
+
)
|
|
194
|
+
return stdout, stderr
|
|
195
|
+
except Exception as e:
|
|
196
|
+
return b"", str(e).encode()
|
|
197
|
+
finally:
|
|
198
|
+
self.connector.disconnect(connection)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class PythonExecutor(Executor):
|
|
202
|
+
"""
|
|
203
|
+
Runs Python code in the specified mode ('eval' or 'exec') within a given namespace.
|
|
204
|
+
|
|
205
|
+
The PythonExecutor class allows for the execution or evaluation of Python code in a specific namespace,
|
|
206
|
+
with optional support for serializing results via pickle.
|
|
207
|
+
"""
|
|
208
|
+
|
|
209
|
+
def __init__(self,
|
|
210
|
+
py_code: str,
|
|
211
|
+
mode: str = "eval",
|
|
212
|
+
namespace: dict | None = None,
|
|
213
|
+
use_pickle: bool = False,
|
|
214
|
+
):
|
|
215
|
+
"""
|
|
216
|
+
Initializes a PythonExecutor.
|
|
217
|
+
|
|
218
|
+
Args:
|
|
219
|
+
py_code (str): The Python code to execute or evaluate. This should be a valid python expression for `eval()`,
|
|
220
|
+
or a valid python statement(s) for `exec()`.
|
|
221
|
+
mode (str, optional): The mode for executing the code, either 'eval' (default) or 'exec'.
|
|
222
|
+
namespace (dict, optional): The namespace in which to execute the code. Defaults to None.
|
|
223
|
+
The namespace would contain "x_stdout" and "x_stderr" for both `eval` and `exec` modes. This is for
|
|
224
|
+
referencing the previous results. The result of `exec` mode can be stored in a variable `exec_result`
|
|
225
|
+
to be referenced by the next command runner. If this is not specified, the value of `stdout` will be
|
|
226
|
+
'None'.
|
|
227
|
+
use_pickle (bool, optional): Whether to pickle the result. Defaults to False.
|
|
228
|
+
|
|
229
|
+
Raises:
|
|
230
|
+
ValueError: If the mode is not 'eval' or 'exec', or if the Python code is empty.
|
|
231
|
+
"""
|
|
232
|
+
if mode not in ["eval", "exec"]:
|
|
233
|
+
raise ValueError("`mode` must be 'eval' or 'exec'.")
|
|
234
|
+
|
|
235
|
+
if not py_code.strip():
|
|
236
|
+
raise ValueError("Python code is empty.")
|
|
237
|
+
|
|
238
|
+
self._mode = mode
|
|
239
|
+
self._namespace = namespace or {}
|
|
240
|
+
self._py_code = textwrap.dedent(py_code)
|
|
241
|
+
self._use_pickle = use_pickle
|
|
242
|
+
|
|
243
|
+
def run(self,
|
|
244
|
+
x_stdout: bytes,
|
|
245
|
+
x_stderr: bytes,
|
|
246
|
+
encoding: str = "utf-8",
|
|
247
|
+
pickle_kwargs: dict | None = None,
|
|
248
|
+
**kwargs
|
|
249
|
+
) -> Tuple[bytes, bytes]:
|
|
250
|
+
"""
|
|
251
|
+
Executes or evaluates Python code in the provided namespace with the given standard output and error.
|
|
252
|
+
|
|
253
|
+
Args:
|
|
254
|
+
x_stdout (bytes): The standard output stream to pass to the code.
|
|
255
|
+
x_stderr (bytes): The standard error stream to pass to the code.
|
|
256
|
+
encoding (str, optional): The encoding for the result. Defaults to "utf-8".
|
|
257
|
+
pickle_kwargs (dict | None, optional): Additional keyword arguments for pickle dumps. Defaults to None.
|
|
258
|
+
**kwargs (dict, optional): Additional keyword arguments for either `eval` or `exec`.
|
|
259
|
+
|
|
260
|
+
Returns:
|
|
261
|
+
Tuple[bytes, bytes]: A tuple containing:
|
|
262
|
+
- The result of the execution or evaluation as a byte string.
|
|
263
|
+
- An empty byte string if no error occurs, or an error message in byte string form.
|
|
264
|
+
"""
|
|
265
|
+
|
|
266
|
+
pickle_kwargs = pickle_kwargs or {}
|
|
267
|
+
|
|
268
|
+
# Inject x_stdout and x_stderr into the namespace
|
|
269
|
+
self._namespace["x_stdout"] = x_stdout
|
|
270
|
+
self._namespace["x_stderr"] = x_stderr
|
|
271
|
+
|
|
272
|
+
# Use the context manager to suppress stdout and stderr during execution
|
|
273
|
+
with suppress_output():
|
|
274
|
+
try:
|
|
275
|
+
if self._mode == "exec":
|
|
276
|
+
self._namespace["exec_result"] = None # To store results for code.
|
|
277
|
+
exec(self._py_code, self._namespace, **kwargs)
|
|
278
|
+
result = self._namespace["exec_result"]
|
|
279
|
+
else:
|
|
280
|
+
result = eval(self._py_code, self._namespace, **kwargs)
|
|
281
|
+
|
|
282
|
+
if self._use_pickle:
|
|
283
|
+
return pickle.dumps(result, **pickle_kwargs), b""
|
|
284
|
+
|
|
285
|
+
return str(result).encode(encoding), b""
|
|
286
|
+
|
|
287
|
+
except Exception as err:
|
|
288
|
+
return b"", str(err).encode(encoding)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
__all__ = (
|
|
292
|
+
"Executor",
|
|
293
|
+
"LocalCommandExecutor",
|
|
294
|
+
"RemoteCommandExecutor",
|
|
295
|
+
"PythonExecutor",
|
|
296
|
+
)
|
marsh/core/expression.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Callable, Tuple
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from marsh.exceptions import CommandError
|
|
7
|
+
from marsh.core.conveyor import Conveyor
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class CommandExpression:
|
|
12
|
+
def evaluate(self, *args, **kwargs) -> Tuple[bytes, bytes]:
|
|
13
|
+
raise NotImplementedError("Subclasses of CommandExpression must implement 'evaluate'.")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(order=True)
|
|
17
|
+
class UnaryExpression(CommandExpression):
|
|
18
|
+
conveyor: Conveyor
|
|
19
|
+
def evaluate(self, *args, **kwargs):
|
|
20
|
+
return self.conveyor(*args, **kwargs)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(order=True)
|
|
24
|
+
class AndExpression(CommandExpression):
|
|
25
|
+
left: CommandExpression | Conveyor
|
|
26
|
+
right: CommandExpression | Conveyor
|
|
27
|
+
|
|
28
|
+
def evaluate(self, *args, **kwargs) -> Tuple[bytes, bytes]:
|
|
29
|
+
# Same as Unix's `command1 && command2`
|
|
30
|
+
# PASS & PASS => PASS
|
|
31
|
+
# PASS & FAIL => FAIL
|
|
32
|
+
# FAIL & NORUN => FAIL
|
|
33
|
+
match self.left:
|
|
34
|
+
case CommandExpression(): # If it's a CommandExpression
|
|
35
|
+
left_stdout, left_stderr = self.left.evaluate(*args, **kwargs)
|
|
36
|
+
case Conveyor(): # If it's a Conveyor
|
|
37
|
+
left_stdout, left_stderr = self.left(*args, **kwargs)
|
|
38
|
+
case _:
|
|
39
|
+
raise ValueError("Invalid type for left side of AndExpression.")
|
|
40
|
+
|
|
41
|
+
if left_stderr.decode("utf-8").strip(): # Left expression is a fail
|
|
42
|
+
# Left expression failed
|
|
43
|
+
raise CommandError(left_stderr.decode("utf-8").strip())
|
|
44
|
+
|
|
45
|
+
match self.right:
|
|
46
|
+
case CommandExpression(): # If it's a CommandExpression
|
|
47
|
+
right_stdout, right_stderr = self.right.evaluate(x_stdout=left_stdout, x_stderr=left_stderr)
|
|
48
|
+
case Conveyor(): # If it's a Conveyor
|
|
49
|
+
right_stdout, right_stderr = self.right(x_stdout=left_stdout, x_stderr=left_stderr)
|
|
50
|
+
case _:
|
|
51
|
+
raise ValueError("Invalid type for right side of AndExpression.")
|
|
52
|
+
|
|
53
|
+
# If right stderr is empty, we pass the stdout from the right side.
|
|
54
|
+
if right_stderr.decode("utf-8").strip(): # Right expression failed
|
|
55
|
+
raise CommandError(right_stderr.decode("utf-8").strip())
|
|
56
|
+
else:
|
|
57
|
+
return right_stdout, right_stderr
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(order=True)
|
|
61
|
+
class OrExpression(CommandExpression):
|
|
62
|
+
left: CommandExpression | Conveyor
|
|
63
|
+
right: CommandExpression | Conveyor
|
|
64
|
+
|
|
65
|
+
def evaluate(self, *args, **kwargs) -> Tuple[bytes, bytes]:
|
|
66
|
+
# Same as Unix's `command1 || command2`
|
|
67
|
+
# PASS or NORUN => PASS
|
|
68
|
+
# FAIL or PASS => PASS
|
|
69
|
+
# FAIL or FAIL => FAIL
|
|
70
|
+
match self.left:
|
|
71
|
+
case CommandExpression(): # If it's a CommandExpression
|
|
72
|
+
left_stdout, left_stderr = self.left.evaluate(*args, **kwargs)
|
|
73
|
+
case Conveyor(): # If it's a Conveyor
|
|
74
|
+
left_stdout, left_stderr = self.left(*args, **kwargs)
|
|
75
|
+
case _:
|
|
76
|
+
raise ValueError("Invalid type for left side of AndExpression.")
|
|
77
|
+
|
|
78
|
+
# Left expression is a fail
|
|
79
|
+
if left_stderr.decode("utf-8").strip():
|
|
80
|
+
match self.right:
|
|
81
|
+
case CommandExpression():
|
|
82
|
+
right_stdout, right_stderr = self.right.evaluate(x_stdout=left_stdout, x_stderr=left_stderr)
|
|
83
|
+
case Conveyor():
|
|
84
|
+
right_stdout, right_stderr = self.right(x_stdout=left_stdout, x_stderr=left_stderr)
|
|
85
|
+
case _:
|
|
86
|
+
raise ValueError("Invalid type for right side of AndExpression.")
|
|
87
|
+
|
|
88
|
+
# Right expression is a fail
|
|
89
|
+
if right_stderr.decode().strip():
|
|
90
|
+
raise CommandError(right_stderr.decode().strip())
|
|
91
|
+
return right_stdout, right_stderr
|
|
92
|
+
# Left expression is a pass
|
|
93
|
+
return left_stdout, left_stderr
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(order=True)
|
|
97
|
+
class JunctionExpression(CommandExpression):
|
|
98
|
+
"""
|
|
99
|
+
(command1 > command2) > (command3 > command4)
|
|
100
|
+
|
|
101
|
+
It may look similar to `command1 & command2` but this does not necessarily raise error if there was an error on left-side.
|
|
102
|
+
"""
|
|
103
|
+
left: CommandExpression | Conveyor
|
|
104
|
+
right: CommandExpression | Conveyor
|
|
105
|
+
|
|
106
|
+
def evaluate(self, *args, **kwargs) -> Tuple[bytes, bytes]:
|
|
107
|
+
match self.left:
|
|
108
|
+
case CommandExpression(): # If it's a CommandExpression
|
|
109
|
+
left_stdout, left_stderr = self.left.evaluate(*args, **kwargs)
|
|
110
|
+
case Conveyor(): # If it's a Conveyor
|
|
111
|
+
left_stdout, left_stderr = self.left(*args, **kwargs)
|
|
112
|
+
case _:
|
|
113
|
+
raise ValueError("Invalid type for left side of AndExpression.")
|
|
114
|
+
|
|
115
|
+
match self.right:
|
|
116
|
+
case CommandExpression():
|
|
117
|
+
right_stdout, right_stderr = self.right.evaluate(x_stdout=left_stdout, x_stderr=left_stderr)
|
|
118
|
+
case Conveyor():
|
|
119
|
+
right_stdout, right_stderr = self.right(x_stdout=left_stdout, x_stderr=left_stderr)
|
|
120
|
+
case _:
|
|
121
|
+
raise ValueError("Invalid type for right side of AndExpression.")
|
|
122
|
+
return right_stdout, right_stderr
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class Command:
|
|
126
|
+
def __init__(self, cmd_expr):
|
|
127
|
+
if not isinstance(cmd_expr, (CommandExpression, Conveyor)):
|
|
128
|
+
raise TypeError("The argument for Command constructor must be a CommandExpression or Conveyor.")
|
|
129
|
+
|
|
130
|
+
self._cmd_expr = cmd_expr if isinstance(cmd_expr, CommandExpression) else UnaryExpression(cmd_expr)
|
|
131
|
+
|
|
132
|
+
@property
|
|
133
|
+
def expression(self) -> CommandExpression:
|
|
134
|
+
return self._cmd_expr
|
|
135
|
+
|
|
136
|
+
@classmethod
|
|
137
|
+
def make_junctions(cls, *command_expressions: list[CommandExpression|Conveyor]) -> "Command":
|
|
138
|
+
# new_command = Command.make_junctions(<cmd_expr|cmd_pipe>, <cmd_expr|cmd_pipe>, ...)
|
|
139
|
+
# new_command(...)
|
|
140
|
+
return cls(functools.reduce(JunctionExpression, command_expressions))
|
|
141
|
+
|
|
142
|
+
def __or__(self, right: CommandExpression) -> "Command":
|
|
143
|
+
if isinstance(right, CommandExpression):
|
|
144
|
+
return Command(OrExpression(self._cmd_expr, right))
|
|
145
|
+
else:
|
|
146
|
+
return Command(OrExpression(self._cmd_expr, right.expression))
|
|
147
|
+
|
|
148
|
+
def __and__(self, right: CommandExpression) -> "Command":
|
|
149
|
+
if isinstance(right, CommandExpression):
|
|
150
|
+
return Command(AndExpression(self._cmd_expr, right))
|
|
151
|
+
else:
|
|
152
|
+
return Command(AndExpression(self._cmd_expr, right.expression))
|
|
153
|
+
|
|
154
|
+
def _write_stdout(self, file_path_: str | Path, mode: str) -> None:
|
|
155
|
+
file_path = file_path_ if isinstance(file_path_, Path) else Path(file_path_).resolve()
|
|
156
|
+
stdout, _ = self()
|
|
157
|
+
with file_path.open(mode=mode) as file:
|
|
158
|
+
file.write(stdout.decode("utf-8").strip() + "\n")
|
|
159
|
+
|
|
160
|
+
def __gt__(self, right: str | Path) -> None:
|
|
161
|
+
# Write (overwrite) STDOUT to file
|
|
162
|
+
# command > "/path/to/file"
|
|
163
|
+
# command > "/dev/null"
|
|
164
|
+
|
|
165
|
+
# Decision: The following evaluates the expression immediately.
|
|
166
|
+
# Warn: We are only considering the STDOUT and not STDERR.
|
|
167
|
+
# command = Command(...)
|
|
168
|
+
# command > "file.txt"
|
|
169
|
+
|
|
170
|
+
if isinstance(right, (str, Path)):
|
|
171
|
+
file_path = right if isinstance(right, Path) else Path(right).resolve()
|
|
172
|
+
self._write_stdout(file_path, "w")
|
|
173
|
+
|
|
174
|
+
if isinstance(right, Command):
|
|
175
|
+
# Both methods still require pathentesis
|
|
176
|
+
# (command1 > command2) > (command3 > command4)
|
|
177
|
+
|
|
178
|
+
# Method: We can achieve Junction without parenthesis using Side-effect
|
|
179
|
+
# self._cmd_expr = JunctionExpression(self._cmd_expr, right.expression)
|
|
180
|
+
# return self
|
|
181
|
+
|
|
182
|
+
# Method: Using New Copy
|
|
183
|
+
return Command(JunctionExpression(self._cmd_expr, right.expression))
|
|
184
|
+
|
|
185
|
+
def __rshift__(self, right: str | Path) -> None:
|
|
186
|
+
# Append STDOUT to file
|
|
187
|
+
# command >> /path/to/file
|
|
188
|
+
# command >> /dev/null
|
|
189
|
+
|
|
190
|
+
# Decision: The following evaluates the expression immediately.
|
|
191
|
+
# Warn: We are only considering the STDOUT and not STDERR.
|
|
192
|
+
# command = Command(...)
|
|
193
|
+
# command >> "file.txt"
|
|
194
|
+
file_path = right if isinstance(right, Path) else Path(right).resolve()
|
|
195
|
+
self._write_stdout(file_path, "a")
|
|
196
|
+
|
|
197
|
+
def __call__(self) -> Tuple[bytes, bytes]:
|
|
198
|
+
return self._cmd_expr.evaluate()
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
__all__ = (
|
|
202
|
+
"CommandExpression",
|
|
203
|
+
"UnaryExpression",
|
|
204
|
+
"AndExpression",
|
|
205
|
+
"OrExpression",
|
|
206
|
+
"JunctionExpression",
|
|
207
|
+
"Command"
|
|
208
|
+
)
|
marsh/core/script.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
from abc import ABC, abstractmethod
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Script(ABC):
|
|
6
|
+
"""
|
|
7
|
+
Abstract base class for generating scripts from templates.
|
|
8
|
+
Subclasses must implement the `generate` method to produce
|
|
9
|
+
a rendered script as a string.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
def __init__(self, script_template: Any):
|
|
13
|
+
"""
|
|
14
|
+
Initializes a Script instance with a template to be rendered.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
`script_template` (Any): A template object (e.g., a `string.Template`,
|
|
18
|
+
a Jinja2 template, or a plain string)
|
|
19
|
+
that serves as the base for script generation.
|
|
20
|
+
"""
|
|
21
|
+
self.script_template = script_template # e.g. string.Template, Jinja2 Templates, or str
|
|
22
|
+
|
|
23
|
+
@abstractmethod
|
|
24
|
+
def generate(self, *args, **kwargs) -> str:
|
|
25
|
+
"""
|
|
26
|
+
Abstract method to render the script template into a string.
|
|
27
|
+
|
|
28
|
+
This method should be implemented in subclasses to provide specific logic
|
|
29
|
+
for rendering the `script_template` using the provided arguments.
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
*args: Positional arguments to customize the script rendering.
|
|
33
|
+
**kwargs: Keyword arguments to customize the script rendering.
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
str: A string containing the rendered script, ready to be executed or used.
|
|
37
|
+
"""
|
|
38
|
+
pass
|