marsh-lib 0.2.0__py3-none-any.whl

marsh/dag/node.py

@@ -0,0 +1,52 @@
+from typing import Tuple
+
+from ..logger import result_logging_decorator
+from ..core import Conveyor
+from .startable import Startable
+
+
+class Node(Startable):
+    """
+    A Node represents an object that can be started, typically as part of a workflow or graph.
+
+    This class extends `Startable` and represents a node in a directed acyclic graph (DAG) or similar structure.
+    Each node is responsible for starting a process using a `Conveyor` object. The `start` method invokes the `Conveyor`
+    with the provided keyword arguments, returning the result as a 2-tuple of bytes.
+
+    Attributes:
+        name (str): The name of the node, inherited from the `Startable` base class.
+        conveyor (Conveyor): The `Conveyor` object responsible for carrying out the node's operation.
+        _kwargs (dict): A dictionary of keyword arguments used when calling the `Conveyor`.
+
+    Methods:
+        start() -> Tuple[bytes, bytes]:
+            Executes the `Conveyor` with the provided arguments, returning the result as a tuple of bytes.
+    """
+
+    def __init__(self, name: str, conveyor: Conveyor, **kwargs):
+        """
+        Initializes a Node object with a name and a Conveyor.
+
+        Args:
+            name (str): The name of the node.
+            conveyor (Conveyor): The `Conveyor` object that will perform the operation associated with this node.
+            **kwargs: Additional keyword arguments to be passed to the `Conveyor` when called.
+        """
+        super().__init__(name)
+        self.conveyor = conveyor
+
+        # Conveyor keyword arguments for __call__
+        self._kwargs = kwargs
+
+    @result_logging_decorator(__name__)
+    def start(self) -> Tuple[bytes, bytes]:
+        """
+        Starts the operation associated with the Node by invoking the Conveyor.
+
+        This method calls the `Conveyor` object with the stored keyword arguments and returns a tuple of bytes
+        representing the result of the operation.
+
+        Returns:
+            Tuple[bytes, bytes]: A tuple containing two byte sequences as the result of the Conveyor operation.
+        """
+        return self.conveyor(**self._kwargs)

marsh/dag/startable.py

@@ -0,0 +1,48 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Any
+
+
+class Startable(ABC):
+    """
+    Abstract base class for objects that can be started, such as nodes or DAGs.
+
+    This class defines a common interface for objects that need to be started.
+    Subclasses like `Node` and `Dag` will implement the `start` method with their own logic,
+    while inheriting from `Startable` to ensure they follow the expected contract for starting.
+
+    Attributes:
+        name (str): A hashable name that identifies the object within a node or directed acyclic graph (DAG).
+            This will be used to define the graph in `graphlib.TopologicalSorter`.
+
+    Methods:
+        start(*args, **kwargs):
+            Starts the object with optional arguments. Subclasses must implement this method with their own logic,
+            defining how the object is started. It may return any type or `None`.
+    """
+
+    def __init__(self, name: str):
+        """
+        Initializes a Startable object with a given name.
+
+        Args:
+            name (str): A hashable name for the node or DAG.
+                This name is used to identify and reference the object.
+        """
+        self.name = name  # Hashable name for Node and Dag
+
+    @abstractmethod
+    def start(self, *args, **kwargs) -> Optional[Any]:
+        """
+        Abstract method that starts the object.
+
+        This method must be implemented by subclasses such as `Node` or `Dag` to define the specific behavior
+        for starting the object. It should handle the logic for initiating the object's state or execution.
+
+        Args:
+            *args: Positional arguments passed to the method.
+            **kwargs: Keyword arguments passed to the method.
+
+        Returns:
+            Optional[Any]: The result of the start operation, which may be of any type, or `None`.
+        """
+        pass

marsh/docker/__init__.py

@@ -0,0 +1,2 @@
+from marsh.docker.docker_executor import DockerContainer, DockerCommandExecutor
+from marsh.docker.docker_command_grammar import DockerCommandGrammar

marsh/docker/docker_command_grammar.py

@@ -0,0 +1,31 @@
+from marsh import CommandGrammar
+
+
+class DockerCommandGrammar(CommandGrammar):
+    def __init__(self, shell_command: str = "/bin/sh -c"):
+        if not shell_command.strip():
+            raise ValueError("The shell command cannot be empty. Example: Use '/bin/sh -c'")
+        self.shell_command = shell_command
+
+    def build_cmd(self,
+                  command: str | list[str],
+                  x_stdout: str | bytes | None = None
+                  ) -> list[str]:
+        shell_command = self.shell_command.split()
+
+        x_stdout = x_stdout.decode() if isinstance(x_stdout, bytes)\
+            else x_stdout if isinstance(x_stdout, str)\
+            else None
+
+        if isinstance(command, str):
+            full_command = [*shell_command, command]
+        else:
+            full_command = [*shell_command, " ".join(command)]
+
+        if x_stdout:
+            full_command[-1] = full_command[-1] + " " + x_stdout.strip()
+
+        return full_command
+
+    def pipe_prev_stdout(self, x_stdout: str | bytes) -> list[str]:
+        pass

marsh/docker/docker_executor.py

@@ -0,0 +1,181 @@
+import uuid
+import time
+import threading
+from typing import Optional
+
+import docker
+from docker.models.containers import Container
+from docker.errors import DockerException, NotFound
+
+from marsh.exceptions import DockerError, DockerClientError
+from marsh import Executor
+from .docker_command_grammar import DockerCommandGrammar
+
+
+def generate_random_container_name(prefix="ephemeral-container"):
+    return prefix + "-" + str(uuid.uuid4()).replace('-', '')[:8]
+
+
+class DockerContainer:
+    def __init__(self,
+                 image: str,
+                 *create_args,
+                 client_args=(),
+                 client_kwargs=None,
+                 name: str | None = None,
+                 timeout: int = 600,  # in seconds
+                 start_timeout: int = 1.5,  # Wait for few seconds after a container started
+                 **create_kwargs
+                 ):
+        client_kwargs = client_kwargs or {}
+
+        self._image = image
+        self._name = name or generate_random_container_name()
+
+        self._start_timeout = start_timeout
+        self._timeout = timeout
+        self._timer: threading.Timer | None = None
+
+        self._container: Optional[Container] = None
+
+        try:
+            self._client = docker.DockerClient(*client_args, **client_kwargs)
+        except Exception as err:
+            raise DockerClientError(err)
+
+        self._create_args = create_args
+        self._create_kwargs = create_kwargs
+
+    def __enter__(self) -> Container:
+        try:
+            self._container = self._client.containers.create(
+                self._image,
+                *self._create_args,
+                command="tail -f /dev/null",  # To run the container continuously
+                name=self._name,
+                detach=True,
+                auto_remove=True,
+                **self._create_kwargs
+            )
+
+            # Start the container
+            self._container.start()
+            time.sleep(self._start_timeout)  # Wait for few seconds
+
+            # Set the Timer to stop the container on timeout
+            self._timer = threading.Timer(self._timeout, self._throw_timeout_error)
+            self._timer.start()
+
+        except Exception as err:
+            self._clean()
+            raise DockerError(err)
+
+        return self._container
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        # Clean the resources
+        self._clean()
+
+        if exc_type is TimeoutError:
+            return False
+
+        return True
+
+    def _throw_timeout_error(self) -> None:
+        self._clean()
+        raise TimeoutError(f"Timeout reached for container '{self._name}'.")
+
+    def _clean(self) -> None:
+        # 1. Cancel the Timer
+        # 2. Remove the Container
+        # 3. Close the Docker Client
+
+        # Cancel the Timer
+        if self._timer:
+            self._timer.cancel()
+
+        # Remove the Container
+        all_containers: list[Container] = self._client.containers.list(all=True)
+        for container in all_containers:
+            if container.name == self._name:
+                try:
+                    container.stop(timeout=0)
+                except NotFound as err:
+                    # This occurs when timeout event occurred.
+                    pass
+
+        # Close the docker client
+        try:
+            self._client.close()
+        except DockerException as err:
+            raise  DockerClientError(err)
+
+
+class DockerCommandExecutor(Executor):
+    def __init__(self,
+                 image: str,
+                 *create_args,
+                 container_name: str | None = None,
+                 pipe_prev_stdout: bool = False,
+                 timeout: int = 600,
+                 shell_command="/bin/sh -c",
+                 client_args: tuple = (),
+                 client_kwargs: dict | None = None,
+                 **create_kwargs,
+                 ):
+
+        self._client_args = client_args
+        self._client_kwargs = client_kwargs or {}
+
+        self._create_args = create_args
+        self._create_kwargs = create_kwargs
+
+        self._container_name = container_name
+        self.timeout = timeout
+        self.image = image  # Docker container image
+
+        # Create a DockerCommandGrammar and clean the given command
+        self._command_grammar = DockerCommandGrammar(shell_command=shell_command)
+
+        self._pipe_prev_stdout = pipe_prev_stdout
+
+    def run(self,
+            x_stdout: bytes,
+            x_stderr: bytes,
+            command: str | list[str],
+            **run_kwargs
+            ) -> tuple[bytes, bytes]:
+
+        stdout: bytes = b""
+        stderr: bytes = b""
+
+        with DockerContainer(self.image,
+                             *self._create_args,
+                             name=self._container_name,
+                             timeout=self.timeout,
+                             client_args=self._client_args,
+                             client_kwargs=self._client_kwargs,
+                             **self._create_kwargs,
+                             ) as container:
+
+            # Unix Pipes, if specified
+            if self._pipe_prev_stdout:
+                full_command = self._command_grammar.build_cmd(command, x_stdout)
+            else:
+                full_command = self._command_grammar.build_cmd(command)
+
+            # Run a command in the container
+            try:
+                result = container.exec_run(full_command, **run_kwargs)
+            except TimeoutError:
+                raise
+
+            # Check exit code and update output streams
+            if result.exit_code == 0:
+                # Update STDOUT (no strip)
+                stdout = result.output
+            else:
+                # Update STDERR (no strip)
+                stderr = result.output
+
+        return stdout, stderr

marsh/exceptions.py

@@ -0,0 +1,16 @@
+class CommandError(Exception):
+    """Base Class for all Command Errors."""
+    pass
+
+
+class ScriptError(Exception):
+    """Base Class for all Script Errors."""
+    pass
+
+
+class DockerError(Exception):
+    """Base Class for all Docker Errors."""
+
+
+class DockerClientError(DockerError):
+    pass

marsh/logger.py

@@ -0,0 +1,116 @@
+import os
+import functools
+import logging
+from logging.handlers import RotatingFileHandler
+
+
+class LoggerTemplate:
+    def __init__(self,
+                 logger_name: str,
+                 log_level: int,
+                 log_handlers: list,
+                 log_formatter: logging.Formatter
+                 ):
+        self.logger_name = logger_name
+        self.log_level = log_level
+        self.log_handlers = log_handlers
+        self.log_formatter = log_formatter
+
+    def create_logger(self) -> logging.Logger:
+        logger = logging.getLogger(self.logger_name)
+        logger.setLevel(self.log_level)
+
+        for handler in self.log_handlers:
+            handler.setFormatter(self.log_formatter)
+            logger.addHandler(handler)
+
+        return logger
+
+
+class CustomRotatingFileHandler(RotatingFileHandler):
+    def rotation_filename(self, default_name):
+        base_name = default_name.split(".")[0]
+        count = default_name.split(".")[-1]
+        return f"{base_name}_{count}.log"
+
+
+def create_rotating_file_logger(logger_name: str,
+                                log_file_path: str,
+                                size_mb: int = 1,
+                                bak_count: int = 2
+                                ) -> logging.Logger:
+    # Define the log format
+    log_formatter = logging.Formatter(
+        "[%(levelname)s] %(asctime)s | %(name)s | Line %(lineno)d | %(message)s"
+    )
+
+    file_path = str(LOG_DIR_PATH / log_file_path)
+
+    # Set up the rotating file handler
+    rotating_handler = CustomRotatingFileHandler(
+        file_path,
+        maxBytes=size_mb * 1024 * 1024,
+        backupCount=bak_count
+    )
+
+    # Create a logger using the template
+    logger_template = LoggerTemplate(
+        logger_name=logger_name,
+        log_level=logging.DEBUG,
+        log_handler=rotating_handler,
+        log_formatter=log_formatter
+    )
+
+    return logger_template.create_logger()
+
+
+def create_console_logger(logger_name: str) -> logging.Logger:
+    # Define the log format
+    log_formatter = logging.Formatter(
+        "[%(levelname)s] %(asctime)s | %(name)s | Line %(lineno)d | %(message)s"
+    )
+
+    console_handler = logging.StreamHandler()
+
+    # Create a logger using the template
+    logger_template = LoggerTemplate(
+        logger_name=logger_name,
+        log_level=logging.DEBUG,
+        log_handlers=[console_handler],
+        log_formatter=log_formatter
+    )
+
+    return logger_template.create_logger()
+
+
+def result_logging_decorator(name: str):
+    """
+    Result logging decorator.
+
+    Usage:
+        Decorate any callable with return values
+        Step 1: Create Environment Variable `MARSH_USE_LOGGER` with value `true` in terminal.
+            - On Linux/MacOS: `export MARSH_USE_LOGGER=true`
+            - On Windows (PowerShell): `$env:MARSH_USE_LOGGER="true"`
+        Step 2: Decorate any callable with return values.
+            `@result_logging_decorator(__name__)`
+    """
+
+    use_logger = True if os.getenv("MARSH_USE_LOGGER") == "true" else False
+    logger = create_console_logger(name)
+
+    def outer(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                result = func(*args, **kwargs)
+                if use_logger:
+                    logger.info(f"{func.__name__} result: {result}")
+                return result
+            except Exception as err:
+                if use_logger:
+                    logger.error(str(err))
+
+        return wrapper
+
+    return outer

marsh/modifier_functions/__init__.py

@@ -0,0 +1,2 @@
+from .case_conversion import *
+from .readers import *

marsh/modifier_functions/case_conversion.py

@@ -0,0 +1,17 @@
+def case_conversion(inp_stdout: bytes,
+                    inp_stderr: bytes,
+                    upper=True,
+                    output_stream="stdout",
+                    ) -> tuple[bytes, bytes]:
+    if output_stream not in ["stdout", "stderr"]:
+        raise ValueError("Output stream must be 'stdout' or 'stderr'.")
+
+    if output_stream == "stdout":
+        return inp_stdout.upper() if upper else inp_stdout.lower(), inp_stderr
+    else:
+        return inp_stderr.upper() if upper else inp_stderr.lower(), inp_stderr
+
+
+__all__ = (
+    "case_conversion",
+)

marsh/modifier_functions/readers.py

@@ -0,0 +1,11 @@
+def read_file(inp_stdout: bytes, inp_stderr: bytes, file_path: str, encoding='utf-8') -> tuple[bytes, bytes]:
+    try:
+        with open(file_path, 'r') as file:
+            return file.read().encode(encoding), b""
+    except (FileNotFoundError, PermissionError, IsADirectoryError, OSError, UnicodeEncodeError, IOError) as e:
+        return b"", str(e).encode(encoding)
+
+
+__all__ = (
+    "read_file",
+)

marsh/processor_functions/__init__.py

@@ -0,0 +1,3 @@
+from .printers import *
+from .raisers import *
+from .redirections import *

marsh/processor_functions/printers.py

@@ -0,0 +1,63 @@
+import logging
+
+
+def print_output_stream(inp_stdout: bytes,
+                        inp_stderr: bytes,
+                        *args,
+                        output_stream="stdout",
+                        encoding='utf-8',
+                        **kwargs
+                        ) -> None:
+    if output_stream not in ["stdout", "stderr"]:
+        raise ValueError("Output stream must be 'stdout' or 'stderr'.")
+
+    if output_stream == "stdout":
+        if inp_stdout.strip():
+            print(inp_stdout.decode(encoding).strip(), *args, **kwargs)
+    else:
+        if inp_stderr.strip():
+            print(inp_stderr.decode(encoding).strip(), *args, **kwargs)
+
+
+def print_stdout(inp_stdout: bytes, inp_stderr: bytes, *args, encoding='utf-8', **kwargs) -> None:
+    print_output_stream(inp_stdout, inp_stderr, *args, output_stream="stdout", encoding=encoding, **kwargs)
+
+
+def print_stderr(inp_stdout: bytes, inp_stderr: bytes, *args, encoding='utf-8', **kwargs) -> None:
+    print_output_stream(inp_stdout, inp_stderr, *args, output_stream="stderr", encoding=encoding, **kwargs)
+
+
+def print_all_output_streams(inp_stdout: bytes, inp_stderr: bytes, *args, encoding='utf-8', **kwargs) -> None:
+    print_stdout(inp_stdout, inp_stderr, *args, encoding=encoding, **kwargs)
+    print_stderr(inp_stdout, inp_stderr, *args, encoding=encoding, **kwargs)
+
+
+def log_output_streams(inp_stdout: bytes,
+                       inp_stderr: bytes,
+                       name="ConsoleLogger",
+                       log_level=logging.DEBUG,
+                       format_="[%(levelname)s] %(asctime)s | %(message)s",
+                       encoding='utf-8'
+                       ) -> None:
+    logger = logging.getLogger(name)
+    logger.setLevel(log_level)
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(logging.DEBUG)
+    formatter = logging.Formatter(format_)
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
+
+    if inp_stderr.strip():
+        logger.error(inp_stderr.decode(encoding).strip())
+
+    if inp_stdout.strip():
+        logger.info(inp_stdout.decode(encoding).strip())
+
+
+__all__ = (
+    "print_output_stream",
+    "print_stdout",
+    "print_stderr",
+    "print_all_output_streams",
+    "log_output_streams"
+)

marsh/processor_functions/raisers.py

@@ -0,0 +1,8 @@
+def raise_stderr(inp_stdout: bytes, inp_stderr: bytes, exception: type[Exception], *args, encoding='utf-8', **kwargs) -> None:
+    if inp_stderr.strip():
+        raise exception(inp_stderr.decode(encoding).strip(), *args, **kwargs)
+
+
+__all__ = (
+    "raise_stderr",
+)

marsh/processor_functions/redirections.py

@@ -0,0 +1,39 @@
+def redirect_output_stream(inp_stdout: bytes,
+                           inp_stderr: bytes,
+                           file_path: str,
+                           output_stream="stdout",
+                           mode='w', 
+                           encoding='utf-8',
+                           ) -> None:
+    if output_stream not in ["stdout", "stderr"]:
+        raise ValueError("Output stream must be 'stdout' or 'stderr'.")
+
+    with open(file_path, mode) as file:
+        if output_stream == "stdout":
+            file.write(inp_stdout.decode(encoding).strip())
+        else:
+            file.write(inp_stderr.decode(encoding).strip())
+
+
+def redirect_stdout(inp_stdout: bytes,
+                    inp_stderr: bytes,
+                    file_path: str,
+                    mode='w',
+                    encoding='utf-8') -> None:
+    redirect_output_stream(inp_stdout, inp_stderr, file_path, output_stream="stdout", mode=mode, encoding=encoding)
+
+
+def redirect_stderr(inp_stdout: bytes,
+                    inp_stderr: bytes,
+                    file_path: str,
+                    mode='w',
+                    encoding='utf-8'
+                    ) -> None:
+    redirect_output_stream(inp_stdout, inp_stderr, file_path, output_stream="stderr", mode=mode, encoding=encoding)
+
+
+__all__ = (
+    "redirect_output_stream",
+    "redirect_stdout",
+    "redirect_stderr",
+)

marsh/signals.py

@@ -0,0 +1 @@
+import signal

marsh/ssh/__init__.py

@@ -0,0 +1,3 @@
+from .ssh_connector import SshConnector
+from .ssh_command_grammar import SshCommandGrammar
+from .ssh_factory import SshFactory

marsh/ssh/ssh_command_grammar.py

@@ -0,0 +1,50 @@
+from typing import Optional
+
+from marsh import CommandGrammar
+
+
+class SshCommandGrammar(CommandGrammar):
+    """
+    SSH Command Grammar for building SSH commands by adding custom logic.
+
+    Inherits from `CommandGrammar` and provides a method to build commands with 
+    optional piping logic.
+    """
+    def build_cmd(self,
+                  commands: list[str],
+                  prev_stdout: Optional[bytes] = None,
+                  *pipe_args,
+                  **pipe_kwargs,
+                  ) -> list[str]:
+        """
+        Builds the full SSH command by optionally piping the stdout of a previous command to the next.
+
+        Args:
+            commands (list[str]): The list of commands to execute.
+            prev_stdout (Optional[bytes]): The stdout from the previous command to pipe into the current one.
+            *pipe_args: Arguments for the piping logic.
+            **pipe_kwargs: Keyword arguments for the piping logic.
+
+        Returns:
+            list[str]: A list of strings representing the final SSH command with any necessary piping logic.
+        """
+        if prev_stdout:
+            return commands + [self._pipe_stdout(prev_stdout, *pipe_args, **pipe_kwargs)]
+        return commands
+
+    @staticmethod
+    def _pipe_stdout(stdout: str | bytes, encoding: str = "utf-8") -> str:
+        """
+        Creates a formatted string to pipe the stdout of a previous command into the next.
+
+        Args:
+            stdout (str | bytes): The stdout to be piped.
+            encoding (str): The encoding to use if `stdout` is a byte string. Default is "utf-8".
+
+        Returns:
+            str: The formatted string to pipe the stdout.
+        """
+        return rf"""<<'EOF'
+{stdout if isinstance(stdout, str) else stdout.decode(encoding)}
+EOF
+"""