marsh-lib 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. marsh/__init__.py +2 -0
  2. marsh/bash/__init__.py +3 -0
  3. marsh/bash/bash_factory.py +74 -0
  4. marsh/bash/bash_grammar.py +44 -0
  5. marsh/bash/bash_runner_decorators.py +0 -0
  6. marsh/bash/bash_script.py +71 -0
  7. marsh/constants.py +5 -0
  8. marsh/core/__init__.py +8 -0
  9. marsh/core/authenticator.py +7 -0
  10. marsh/core/cmd_run_decorator.py +289 -0
  11. marsh/core/command_grammar.py +45 -0
  12. marsh/core/connector.py +77 -0
  13. marsh/core/conveyor.py +105 -0
  14. marsh/core/executor.py +296 -0
  15. marsh/core/expression.py +208 -0
  16. marsh/core/script.py +38 -0
  17. marsh/dag/__init__.py +11 -0
  18. marsh/dag/dag.py +748 -0
  19. marsh/dag/node.py +52 -0
  20. marsh/dag/startable.py +48 -0
  21. marsh/docker/__init__.py +2 -0
  22. marsh/docker/docker_command_grammar.py +31 -0
  23. marsh/docker/docker_executor.py +181 -0
  24. marsh/exceptions.py +16 -0
  25. marsh/logger.py +116 -0
  26. marsh/modifier_functions/__init__.py +2 -0
  27. marsh/modifier_functions/case_conversion.py +17 -0
  28. marsh/modifier_functions/readers.py +11 -0
  29. marsh/powershell/__init__.py +0 -0
  30. marsh/processor_functions/__init__.py +3 -0
  31. marsh/processor_functions/printers.py +63 -0
  32. marsh/processor_functions/raisers.py +8 -0
  33. marsh/processor_functions/redirections.py +39 -0
  34. marsh/signals.py +1 -0
  35. marsh/ssh/__init__.py +3 -0
  36. marsh/ssh/ssh_command_grammar.py +50 -0
  37. marsh/ssh/ssh_connector.py +71 -0
  38. marsh/ssh/ssh_factory.py +93 -0
  39. marsh/utils/__init__.py +16 -0
  40. marsh/utils/output_streams.py +13 -0
  41. marsh_lib-0.2.0.dist-info/LICENSE +21 -0
  42. marsh_lib-0.2.0.dist-info/METADATA +380 -0
  43. marsh_lib-0.2.0.dist-info/RECORD +44 -0
  44. marsh_lib-0.2.0.dist-info/WHEEL +4 -0
marsh/core/conveyor.py ADDED
@@ -0,0 +1,105 @@
1
+ from typing import Callable, Tuple, Sequence
2
+
3
+ from .cmd_run_decorator import CmdRunDecorator
4
+
5
+
6
+ class Conveyor:
7
+ """
8
+ A class that chains multiple command runners, allowing sequential execution of commands.
9
+
10
+ Each command runner is a callable that accepts two `bytes` arguments (representing standard
11
+ output and standard error) and returns a tuple of two `bytes` values as the updated outputs.
12
+
13
+ This class supports adding command runners, decorating them, and invoking them in sequence.
14
+
15
+ Example:
16
+ >>> def cmd_runner_1(stdout: bytes, stderr: bytes) -> Tuple[bytes, bytes]:
17
+ ... return stdout + b"Cmd1", stderr
18
+
19
+ >>> def cmd_runner_2(stdout: bytes, stderr: bytes) -> Tuple[bytes, bytes]:
20
+ ... return stdout + b"Cmd2", stderr
21
+
22
+ >>> conveyor = Conveyor()
23
+ >>> conveyor = conveyor.add_cmd_runner(cmd_runner_1).add_cmd_runner(cmd_runner_2)
24
+ >>> result_stdout, result_stderr = conveyor(b"Start", b"")
25
+ >>> print(result_stdout)
26
+ b'StartCmd1Cmd2'
27
+ """
28
+ def __init__(self, cmds_=None) -> None:
29
+ self._cmd_runners = cmds_ or []
30
+
31
+ @property
32
+ def cmd_run_triples(self) -> Sequence[Tuple[Callable, Tuple, dict]]:
33
+ """
34
+ Returns the sequence of triples containing the registered command runner, its positional arguments and keyword arguments.
35
+
36
+ Returns:
37
+ Sequence[Tuple[Callable, Tuple, dict]]:
38
+ A sequence of command runners with their arguments.
39
+ """
40
+ return self._cmd_runners
41
+
42
+ def add_cmd_runner(self,
43
+ cmd_runner: Callable[[bytes, bytes], Tuple[bytes, bytes]],
44
+ *args,
45
+ cmd_runner_decorator: CmdRunDecorator | None = None,
46
+ **kwargs
47
+ ) -> "Conveyor":
48
+ """
49
+ Adds a new command runner to the Conveyor.
50
+
51
+ Optionally applies a decorator to the command runner before adding it to the chain.
52
+
53
+ Args:
54
+ cmd_runner (Callable[[bytes, bytes], Tuple[bytes, bytes]]):
55
+ A callable that processes two `bytes` inputs (stdout, stderr)
56
+ and returns a tuple of two `bytes` outputs.
57
+ *args:
58
+ Positional arguments to pass to the `cmd_runner` during invocation.
59
+ cmd_runner_decorator (CmdRunDecorator, optional):
60
+ A command runner decorator for decorating the `cmd_runner`. Defaults to None.
61
+ **kwargs:
62
+ Keyword arguments to pass to the `cmd_runner` during invocation.
63
+
64
+ Returns:
65
+ Conveyor: A new Conveyor instance with the added command runner.
66
+
67
+ Raises:
68
+ TypeError: If `cmd_runner` is not callable.
69
+ """
70
+ if not callable(cmd_runner):
71
+ raise TypeError("cmd_runner must be callable with 2 required bytes arguments.")
72
+
73
+ # Apply the pre/post processors and modifiers to command runner, if given
74
+ decorated_cmd_runner = cmd_runner_decorator.decorate(cmd_runner) if isinstance(cmd_runner_decorator, CmdRunDecorator) else cmd_runner
75
+
76
+ return Conveyor(cmds_=self._cmd_runners + [(decorated_cmd_runner, args, kwargs)])
77
+
78
+ def __call__(self, x_stdout: bytes = b"", x_stderr: bytes = b"", callback_list_: Sequence = None) -> Tuple[bytes, bytes]:
79
+ """
80
+ Executes the command runners sequentially.
81
+
82
+ Starts with the given `stdout` and `stderr` values and passes the results through
83
+ each command runner in the chain.
84
+
85
+ Args:
86
+ x_stdout (bytes, optional): Initial standard output. Defaults to `b""`.
87
+ x_stderr (bytes, optional): Initial standard error. Defaults to `b""`.
88
+ callback_list_ (Sequence, optional):
89
+ A list of command runners to execute. Defaults to the command runners in the Conveyor.
90
+
91
+ Returns:
92
+ Tuple[bytes, bytes]:
93
+ A tuple containing the final standard output (`stdout`) and standard error (`stderr`).
94
+ """
95
+ callback_list = callback_list_ or self._cmd_runners
96
+ if len(callback_list) == 1:
97
+ callback, args, kwargs = callback_list[0]
98
+ return callback(x_stdout, x_stderr, *args, **kwargs)
99
+
100
+ if len(self._cmd_runners) == 0:
101
+ return x_stdout, x_stderr
102
+
103
+ callback, args, kwargs = callback_list[0]
104
+ new_x_stdout, new_x_stderr = callback(x_stdout, x_stderr, *args, **kwargs)
105
+ return self(x_stdout=new_x_stdout, x_stderr=new_x_stderr, callback_list_=callback_list[1:])
marsh/core/executor.py ADDED
@@ -0,0 +1,296 @@
1
+ import textwrap
2
+ import pickle
3
+ import subprocess
4
+ from abc import abstractmethod, ABC
5
+ from typing import Callable, Tuple, Any
6
+
7
+ from ..utils.output_streams import suppress_output
8
+ from .command_grammar import CommandGrammar
9
+ from .connector import Connector
10
+
11
+
12
+ class Executor(ABC):
13
+ """
14
+ Abstract base class for command execution.
15
+
16
+ This class defines a common interface for running commands, where subclasses implement
17
+ the `run` method to execute commands in specific environments (e.g., local, remote).
18
+ """
19
+
20
+ @abstractmethod
21
+ def run(self, x_stdout: bytes, x_stderr: bytes, *args, **kwargs) -> Tuple[bytes, bytes]:
22
+ """
23
+ Abstract method to run a command.
24
+
25
+ Args:
26
+ x_stdout (bytes): Standard output from a previous command.
27
+ x_stderr (bytes): Standard error from a previous command.
28
+ *args: Additional positional arguments.
29
+ **kwargs: Additional keyword arguments.
30
+
31
+ Returns:
32
+ Tuple[bytes, bytes]: A tuple containing standard output and standard error.
33
+
34
+ Raises:
35
+ NotImplementedError: If the subclass does not override this method.
36
+ """
37
+ pass
38
+
39
+
40
+ class LocalCommandExecutor(Executor):
41
+ """Executes commands locally as subprocesses."""
42
+
43
+ def __init__(self,
44
+ command_grammar: CommandGrammar,
45
+ pipe_prev_stdout: bool = False,
46
+ timeout: float | None = None,
47
+ ):
48
+ """
49
+ Initializes a LocalCommandExecutor.
50
+
51
+ Args:
52
+ command_grammar (CommandGrammar): A CommandGrammar object to build the command.
53
+ pipe_prev_stdout (bool, optional): Whether to pipe the previous standard output as input. Defaults to False.
54
+ timeout (float | None, optional): Timeout for command execution in seconds. Defaults to None.
55
+ """
56
+
57
+ self.command_grammar = command_grammar # Already parameterized the command grammar
58
+ self.pipe_prev_stdout = pipe_prev_stdout # (Unix) Pipe the previous STDOUT as STDIN for current command runner
59
+ self.timeout = timeout
60
+
61
+ @staticmethod
62
+ def create_popen_with_pipe(command: list[str], *args, **kwargs) -> subprocess.Popen:
63
+ """
64
+ Creates a subprocess with pipes for stdin, stdout, and stderr.
65
+
66
+ Args:
67
+ command (list[str]): The command to execute as a list of strings.
68
+ *args: Additional positional arguments for `subprocess.Popen`.
69
+ **kwargs: Additional keyword arguments for `subprocess.Popen`.
70
+
71
+ Returns:
72
+ subprocess.Popen: A subprocess instance with pipes.
73
+ """
74
+ return subprocess.Popen(command, *args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
75
+ **kwargs)
76
+
77
+ def run(self,
78
+ x_stdout: bytes,
79
+ x_stderr: bytes,
80
+ *args,
81
+ callback: Callable[[subprocess.Popen, bytes, bytes], Tuple[bytes, bytes]] = None,
82
+ popen_args=(),
83
+ popen_kwargs=None,
84
+ **kwargs
85
+ ) -> Tuple[bytes, bytes]:
86
+ """
87
+ Runs a command locally.
88
+
89
+ Args:
90
+ x_stdout (bytes): Standard output to pass to the command.
91
+ x_stderr (bytes): Standard error to pass to the command.
92
+ *args: Additional positional arguments.
93
+ callback (Callable, optional): A custom callback function that takes subprocess.Popen, stdout, and stderr.
94
+ This callback must return tuple[bytes, bytes] which represents the result.
95
+ popen_args (tuple, optional): Arguments for `subprocess.Popen`. Defaults to ().
96
+ popen_kwargs (dict, optional): Keyword arguments for `subprocess.Popen`. Defaults to None.
97
+ **kwargs: Additional keyword arguments for `subprocess.Popen`.
98
+
99
+ Returns:
100
+ Tuple[bytes, bytes]: A tuple containing standard output and standard error.
101
+
102
+ Raises:
103
+ ValueError: If the provided callback does not return a tuple of bytes.
104
+ """
105
+
106
+ popen_kwargs = popen_kwargs or dict()
107
+
108
+ # Build the Command as List of Strings
109
+ command = self.command_grammar.build_cmd()
110
+
111
+ # Create subprocess.Popen
112
+ process = self.create_popen_with_pipe(command, *args, **kwargs)
113
+
114
+ # Use the custom callback provided by user/client
115
+ if callback:
116
+ result = callback(process, x_stdout, x_stderr, *popen_args, **popen_kwargs)
117
+ match result:
118
+ # The `process.communicate(...)` must be invoked in the `callback` to return (stdout, stderr).
119
+ case (stdout, stderr) if isinstance(stdout, bytes) and isinstance(stderr, bytes):
120
+ return stdout, stderr
121
+ case _:
122
+ raise ValueError("Given callback must return tuple[bytes, bytes]")
123
+
124
+ # Use the default procedure for running programs
125
+ stdout, stderr = process.communicate(input=x_stdout,
126
+ timeout=self.timeout) if self.pipe_prev_stdout else process.communicate(
127
+ timeout=self.timeout)
128
+ return stdout, stderr
129
+
130
+
131
+ class RemoteCommandExecutor(Executor):
132
+ """
133
+ Executes commands on remote systems via a connector interface.
134
+
135
+ This class uses a `Connector` to establish a connection to a remote host
136
+ and execute a command generated by a `CommandGrammar`. The command is sent
137
+ to the remote system, and the standard output and error are retrieved.
138
+ """
139
+
140
+ def __init__(self, connector: Connector, command_grammar: CommandGrammar):
141
+ """
142
+ Initializes a RemoteCommandExecutor.
143
+
144
+ Args:
145
+ connector (Connector): An object responsible for managing the connection to the remote system.
146
+ command_grammar (CommandGrammar): An object to build and format the command to be executed remotely.
147
+ """
148
+ self.connector = connector
149
+ self.command_grammar = command_grammar
150
+
151
+ def run(self,
152
+ x_stdout: bytes,
153
+ x_stderr: bytes,
154
+ *build_cmd_args,
155
+ conn_args=(),
156
+ exec_cmd_args=(),
157
+ exec_cmd_kwargs=None,
158
+ conn_kwargs=None,
159
+ **build_cmd_kwargs,
160
+ ) -> Tuple[bytes, bytes]:
161
+ """
162
+ Runs a command on a remote system and retrieves its output.
163
+
164
+ This method connects to a remote host using the provided `Connector`, runs the command
165
+ generated by `CommandGrammar`, and returns the standard output and standard error.
166
+
167
+ Args:
168
+ x_stdout (bytes): Standard output to pass as input, if required by the command.
169
+ x_stderr (bytes): Standard error to pass as input, if required by the command.
170
+ *build_cmd_args: Additional arguments passed to `CommandGrammar.build_cmd`.
171
+ conn_args (tuple, optional): Positional arguments for the `Connector.connect` method.
172
+ exec_cmd_args (tuple, optional): Positional arguments for the `Connector.exec_cmd` method.
173
+ exec_cmd_kwargs (dict, optional): Keyword arguments for `Connector.exec_cmd`. Defaults to None.
174
+ conn_kwargs (dict, optional): Keyword arguments for the `Connector.connect` method. Defaults to None.
175
+ **build_cmd_kwargs: Additional keyword arguments for `CommandGrammar.build_cmd`.
176
+
177
+ Returns:
178
+ Tuple[bytes, bytes]: A tuple containing:
179
+ - stdout (bytes): The standard output from the remote command execution.
180
+ - stderr (bytes): The standard error from the remote command execution.
181
+ """
182
+ exec_cmd_kwargs = exec_cmd_kwargs or dict()
183
+ conn_kwargs = conn_kwargs or dict()
184
+ connection = None
185
+ # TODO: Enhance and include Pipes.
186
+ try:
187
+ connection = self.connector.connect(*conn_args, **conn_kwargs)
188
+ stdout, stderr = self.connector.exec_cmd(
189
+ self.command_grammar.build_cmd(*build_cmd_args, **build_cmd_kwargs),
190
+ connection,
191
+ *exec_cmd_args,
192
+ **exec_cmd_kwargs
193
+ )
194
+ return stdout, stderr
195
+ except Exception as e:
196
+ return b"", str(e).encode()
197
+ finally:
198
+ self.connector.disconnect(connection)
199
+
200
+
201
+ class PythonExecutor(Executor):
202
+ """
203
+ Runs Python code in the specified mode ('eval' or 'exec') within a given namespace.
204
+
205
+ The PythonExecutor class allows for the execution or evaluation of Python code in a specific namespace,
206
+ with optional support for serializing results via pickle.
207
+ """
208
+
209
+ def __init__(self,
210
+ py_code: str,
211
+ mode: str = "eval",
212
+ namespace: dict | None = None,
213
+ use_pickle: bool = False,
214
+ ):
215
+ """
216
+ Initializes a PythonExecutor.
217
+
218
+ Args:
219
+ py_code (str): The Python code to execute or evaluate. This should be a valid python expression for `eval()`,
220
+ or a valid python statement(s) for `exec()`.
221
+ mode (str, optional): The mode for executing the code, either 'eval' (default) or 'exec'.
222
+ namespace (dict, optional): The namespace in which to execute the code. Defaults to None.
223
+ The namespace would contain "x_stdout" and "x_stderr" for both `eval` and `exec` modes. This is for
224
+ referencing the previous results. The result of `exec` mode can be stored in a variable `exec_result`
225
+ to be referenced by the next command runner. If this is not specified, the value of `stdout` will be
226
+ 'None'.
227
+ use_pickle (bool, optional): Whether to pickle the result. Defaults to False.
228
+
229
+ Raises:
230
+ ValueError: If the mode is not 'eval' or 'exec', or if the Python code is empty.
231
+ """
232
+ if mode not in ["eval", "exec"]:
233
+ raise ValueError("`mode` must be 'eval' or 'exec'.")
234
+
235
+ if not py_code.strip():
236
+ raise ValueError("Python code is empty.")
237
+
238
+ self._mode = mode
239
+ self._namespace = namespace or {}
240
+ self._py_code = textwrap.dedent(py_code)
241
+ self._use_pickle = use_pickle
242
+
243
+ def run(self,
244
+ x_stdout: bytes,
245
+ x_stderr: bytes,
246
+ encoding: str = "utf-8",
247
+ pickle_kwargs: dict | None = None,
248
+ **kwargs
249
+ ) -> Tuple[bytes, bytes]:
250
+ """
251
+ Executes or evaluates Python code in the provided namespace with the given standard output and error.
252
+
253
+ Args:
254
+ x_stdout (bytes): The standard output stream to pass to the code.
255
+ x_stderr (bytes): The standard error stream to pass to the code.
256
+ encoding (str, optional): The encoding for the result. Defaults to "utf-8".
257
+ pickle_kwargs (dict | None, optional): Additional keyword arguments for pickle dumps. Defaults to None.
258
+ **kwargs (dict, optional): Additional keyword arguments for either `eval` or `exec`.
259
+
260
+ Returns:
261
+ Tuple[bytes, bytes]: A tuple containing:
262
+ - The result of the execution or evaluation as a byte string.
263
+ - An empty byte string if no error occurs, or an error message in byte string form.
264
+ """
265
+
266
+ pickle_kwargs = pickle_kwargs or {}
267
+
268
+ # Inject x_stdout and x_stderr into the namespace
269
+ self._namespace["x_stdout"] = x_stdout
270
+ self._namespace["x_stderr"] = x_stderr
271
+
272
+ # Use the context manager to suppress stdout and stderr during execution
273
+ with suppress_output():
274
+ try:
275
+ if self._mode == "exec":
276
+ self._namespace["exec_result"] = None # To store results for code.
277
+ exec(self._py_code, self._namespace, **kwargs)
278
+ result = self._namespace["exec_result"]
279
+ else:
280
+ result = eval(self._py_code, self._namespace, **kwargs)
281
+
282
+ if self._use_pickle:
283
+ return pickle.dumps(result, **pickle_kwargs), b""
284
+
285
+ return str(result).encode(encoding), b""
286
+
287
+ except Exception as err:
288
+ return b"", str(err).encode(encoding)
289
+
290
+
291
+ __all__ = (
292
+ "Executor",
293
+ "LocalCommandExecutor",
294
+ "RemoteCommandExecutor",
295
+ "PythonExecutor",
296
+ )
@@ -0,0 +1,208 @@
1
+ import functools
2
+ from dataclasses import dataclass
3
+ from typing import Callable, Tuple
4
+ from pathlib import Path
5
+
6
+ from marsh.exceptions import CommandError
7
+ from marsh.core.conveyor import Conveyor
8
+
9
+
10
+ @dataclass
11
+ class CommandExpression:
12
+ def evaluate(self, *args, **kwargs) -> Tuple[bytes, bytes]:
13
+ raise NotImplementedError("Subclasses of CommandExpression must implement 'evaluate'.")
14
+
15
+
16
+ @dataclass(order=True)
17
+ class UnaryExpression(CommandExpression):
18
+ conveyor: Conveyor
19
+ def evaluate(self, *args, **kwargs):
20
+ return self.conveyor(*args, **kwargs)
21
+
22
+
23
+ @dataclass(order=True)
24
+ class AndExpression(CommandExpression):
25
+ left: CommandExpression | Conveyor
26
+ right: CommandExpression | Conveyor
27
+
28
+ def evaluate(self, *args, **kwargs) -> Tuple[bytes, bytes]:
29
+ # Same as Unix's `command1 && command2`
30
+ # PASS & PASS => PASS
31
+ # PASS & FAIL => FAIL
32
+ # FAIL & NORUN => FAIL
33
+ match self.left:
34
+ case CommandExpression(): # If it's a CommandExpression
35
+ left_stdout, left_stderr = self.left.evaluate(*args, **kwargs)
36
+ case Conveyor(): # If it's a Conveyor
37
+ left_stdout, left_stderr = self.left(*args, **kwargs)
38
+ case _:
39
+ raise ValueError("Invalid type for left side of AndExpression.")
40
+
41
+ if left_stderr.decode("utf-8").strip(): # Left expression is a fail
42
+ # Left expression failed
43
+ raise CommandError(left_stderr.decode("utf-8").strip())
44
+
45
+ match self.right:
46
+ case CommandExpression(): # If it's a CommandExpression
47
+ right_stdout, right_stderr = self.right.evaluate(x_stdout=left_stdout, x_stderr=left_stderr)
48
+ case Conveyor(): # If it's a Conveyor
49
+ right_stdout, right_stderr = self.right(x_stdout=left_stdout, x_stderr=left_stderr)
50
+ case _:
51
+ raise ValueError("Invalid type for right side of AndExpression.")
52
+
53
+ # If right stderr is empty, we pass the stdout from the right side.
54
+ if right_stderr.decode("utf-8").strip(): # Right expression failed
55
+ raise CommandError(right_stderr.decode("utf-8").strip())
56
+ else:
57
+ return right_stdout, right_stderr
58
+
59
+
60
+ @dataclass(order=True)
61
+ class OrExpression(CommandExpression):
62
+ left: CommandExpression | Conveyor
63
+ right: CommandExpression | Conveyor
64
+
65
+ def evaluate(self, *args, **kwargs) -> Tuple[bytes, bytes]:
66
+ # Same as Unix's `command1 || command2`
67
+ # PASS or NORUN => PASS
68
+ # FAIL or PASS => PASS
69
+ # FAIL or FAIL => FAIL
70
+ match self.left:
71
+ case CommandExpression(): # If it's a CommandExpression
72
+ left_stdout, left_stderr = self.left.evaluate(*args, **kwargs)
73
+ case Conveyor(): # If it's a Conveyor
74
+ left_stdout, left_stderr = self.left(*args, **kwargs)
75
+ case _:
76
+ raise ValueError("Invalid type for left side of AndExpression.")
77
+
78
+ # Left expression is a fail
79
+ if left_stderr.decode("utf-8").strip():
80
+ match self.right:
81
+ case CommandExpression():
82
+ right_stdout, right_stderr = self.right.evaluate(x_stdout=left_stdout, x_stderr=left_stderr)
83
+ case Conveyor():
84
+ right_stdout, right_stderr = self.right(x_stdout=left_stdout, x_stderr=left_stderr)
85
+ case _:
86
+ raise ValueError("Invalid type for right side of AndExpression.")
87
+
88
+ # Right expression is a fail
89
+ if right_stderr.decode().strip():
90
+ raise CommandError(right_stderr.decode().strip())
91
+ return right_stdout, right_stderr
92
+ # Left expression is a pass
93
+ return left_stdout, left_stderr
94
+
95
+
96
+ @dataclass(order=True)
97
+ class JunctionExpression(CommandExpression):
98
+ """
99
+ (command1 > command2) > (command3 > command4)
100
+
101
+ It may look similar to `command1 & command2` but this does not necessarily raise error if there was an error on left-side.
102
+ """
103
+ left: CommandExpression | Conveyor
104
+ right: CommandExpression | Conveyor
105
+
106
+ def evaluate(self, *args, **kwargs) -> Tuple[bytes, bytes]:
107
+ match self.left:
108
+ case CommandExpression(): # If it's a CommandExpression
109
+ left_stdout, left_stderr = self.left.evaluate(*args, **kwargs)
110
+ case Conveyor(): # If it's a Conveyor
111
+ left_stdout, left_stderr = self.left(*args, **kwargs)
112
+ case _:
113
+ raise ValueError("Invalid type for left side of AndExpression.")
114
+
115
+ match self.right:
116
+ case CommandExpression():
117
+ right_stdout, right_stderr = self.right.evaluate(x_stdout=left_stdout, x_stderr=left_stderr)
118
+ case Conveyor():
119
+ right_stdout, right_stderr = self.right(x_stdout=left_stdout, x_stderr=left_stderr)
120
+ case _:
121
+ raise ValueError("Invalid type for right side of AndExpression.")
122
+ return right_stdout, right_stderr
123
+
124
+
125
+ class Command:
126
+ def __init__(self, cmd_expr):
127
+ if not isinstance(cmd_expr, (CommandExpression, Conveyor)):
128
+ raise TypeError("The argument for Command constructor must be a CommandExpression or Conveyor.")
129
+
130
+ self._cmd_expr = cmd_expr if isinstance(cmd_expr, CommandExpression) else UnaryExpression(cmd_expr)
131
+
132
+ @property
133
+ def expression(self) -> CommandExpression:
134
+ return self._cmd_expr
135
+
136
+ @classmethod
137
+ def make_junctions(cls, *command_expressions: list[CommandExpression|Conveyor]) -> "Command":
138
+ # new_command = Command.make_junctions(<cmd_expr|cmd_pipe>, <cmd_expr|cmd_pipe>, ...)
139
+ # new_command(...)
140
+ return cls(functools.reduce(JunctionExpression, command_expressions))
141
+
142
+ def __or__(self, right: CommandExpression) -> "Command":
143
+ if isinstance(right, CommandExpression):
144
+ return Command(OrExpression(self._cmd_expr, right))
145
+ else:
146
+ return Command(OrExpression(self._cmd_expr, right.expression))
147
+
148
+ def __and__(self, right: CommandExpression) -> "Command":
149
+ if isinstance(right, CommandExpression):
150
+ return Command(AndExpression(self._cmd_expr, right))
151
+ else:
152
+ return Command(AndExpression(self._cmd_expr, right.expression))
153
+
154
+ def _write_stdout(self, file_path_: str | Path, mode: str) -> None:
155
+ file_path = file_path_ if isinstance(file_path_, Path) else Path(file_path_).resolve()
156
+ stdout, _ = self()
157
+ with file_path.open(mode=mode) as file:
158
+ file.write(stdout.decode("utf-8").strip() + "\n")
159
+
160
+ def __gt__(self, right: str | Path) -> None:
161
+ # Write (overwrite) STDOUT to file
162
+ # command > "/path/to/file"
163
+ # command > "/dev/null"
164
+
165
+ # Decision: The following evaluates the expression immediately.
166
+ # Warn: We are only considering the STDOUT and not STDERR.
167
+ # command = Command(...)
168
+ # command > "file.txt"
169
+
170
+ if isinstance(right, (str, Path)):
171
+ file_path = right if isinstance(right, Path) else Path(right).resolve()
172
+ self._write_stdout(file_path, "w")
173
+
174
+ if isinstance(right, Command):
175
+ # Both methods still require pathentesis
176
+ # (command1 > command2) > (command3 > command4)
177
+
178
+ # Method: We can achieve Junction without parenthesis using Side-effect
179
+ # self._cmd_expr = JunctionExpression(self._cmd_expr, right.expression)
180
+ # return self
181
+
182
+ # Method: Using New Copy
183
+ return Command(JunctionExpression(self._cmd_expr, right.expression))
184
+
185
+ def __rshift__(self, right: str | Path) -> None:
186
+ # Append STDOUT to file
187
+ # command >> /path/to/file
188
+ # command >> /dev/null
189
+
190
+ # Decision: The following evaluates the expression immediately.
191
+ # Warn: We are only considering the STDOUT and not STDERR.
192
+ # command = Command(...)
193
+ # command >> "file.txt"
194
+ file_path = right if isinstance(right, Path) else Path(right).resolve()
195
+ self._write_stdout(file_path, "a")
196
+
197
+ def __call__(self) -> Tuple[bytes, bytes]:
198
+ return self._cmd_expr.evaluate()
199
+
200
+
201
+ __all__ = (
202
+ "CommandExpression",
203
+ "UnaryExpression",
204
+ "AndExpression",
205
+ "OrExpression",
206
+ "JunctionExpression",
207
+ "Command"
208
+ )
marsh/core/script.py ADDED
@@ -0,0 +1,38 @@
1
+ from typing import Any
2
+ from abc import ABC, abstractmethod
3
+
4
+
5
+ class Script(ABC):
6
+ """
7
+ Abstract base class for generating scripts from templates.
8
+ Subclasses must implement the `generate` method to produce
9
+ a rendered script as a string.
10
+ """
11
+
12
+ def __init__(self, script_template: Any):
13
+ """
14
+ Initializes a Script instance with a template to be rendered.
15
+
16
+ Args:
17
+ `script_template` (Any): A template object (e.g., a `string.Template`,
18
+ a Jinja2 template, or a plain string)
19
+ that serves as the base for script generation.
20
+ """
21
+ self.script_template = script_template # e.g. string.Template, Jinja2 Templates, or str
22
+
23
+ @abstractmethod
24
+ def generate(self, *args, **kwargs) -> str:
25
+ """
26
+ Abstract method to render the script template into a string.
27
+
28
+ This method should be implemented in subclasses to provide specific logic
29
+ for rendering the `script_template` using the provided arguments.
30
+
31
+ Args:
32
+ *args: Positional arguments to customize the script rendering.
33
+ **kwargs: Keyword arguments to customize the script rendering.
34
+
35
+ Returns:
36
+ str: A string containing the rendered script, ready to be executed or used.
37
+ """
38
+ pass
marsh/dag/__init__.py ADDED
@@ -0,0 +1,11 @@
1
+ from .startable import Startable
2
+ from .node import Node
3
+ from .dag import (
4
+ Dag,
5
+ SyncDag,
6
+ AsyncDag,
7
+ ThreadPoolDag,
8
+ ThreadDag,
9
+ MultiprocessDag,
10
+ ProcessPoolDag
11
+ )