pyallel 1.1.0__tar.gz → 1.2.1__tar.gz

{pyallel-1.1.0 → pyallel-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyallel
-Version: 1.1.0
+Version: 1.2.1
 Summary: Run and handle the output of multiple executables in pyallel (as in parallel)
 Home-page: https://github.com/Danthewaann/pyallel
 License: MIT
@@ -48,10 +48,10 @@ usage: pyallel [-h] [-t] [-n] [-V] [--colour {yes,no,auto}] [commands ...]
 Run and handle the output of multiple executables in pyallel (as in parallel)
 
 positional arguments:
-  commands              list of quoted commands to run e.g "mypy ." "black ."
+  commands              list of quoted commands to run in parallel e.g "mypy ." "black ."
 
                         each command is executed inside a shell, so shell syntax is supported as
-                        if you were running the command directly in a shell, some examples are below:
+                        if you were running the command directly in a shell, some examples are below
 
                              "MYPY_FORCE_COLOR=1 mypy ."          <- provide environment variables
                              "mypy | tee -a mypy.log"             <- use pipes to redirect output
@@ -60,6 +60,14 @@ positional arguments:
                              "echo \$SHELL" or "\$(echo mypy .)"  <- expand variables and commands to evaluate (must be escaped)
                              "pytest . && mypy . || echo failed!" <- use AND (&&) and OR (||) to run commands conditionally
 
+                        commands can be grouped using the group separator symbol (:::)
+
+                             "echo boil kettle" "sleep 1" ::: "echo make coffee"
+
+                        the above will print "boil kettle" and sleep for 1 second first before printing "make coffee"
+
+                        command groups are ran in the sequence you provide them, and if a command group fails
+                        (if a command fails inside the command group) the rest of the command groups in the sequence are not run
 
 options:
   -h, --help            show this help message and exit
@@ -90,26 +98,53 @@ You can also build an executable with the following (executables will be written
 > The `arch=x86_64` values in the following code blocks can be replaced with `arch=aarch64` and
 > any other architecture that is supported by docker to build an executable for that given architecture
 
+> [!NOTE]
+> To build aarch64 binaries on an x86_64 host machine, you will need to run the following
+> commands to setup qemu to allow this to work
+
+```bash
+sudo apt-get install qemu binfmt-support qemu-user-static && \
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+```
+
 #### Build for generic linux
 
 ```bash
-docker build --tag pyallel --build-arg 'arch=x86_64' . && docker run -e 'arch=x86_64' --rm --volume "$(pwd):/src" pyallel
+docker build --tag pyallel --build-arg 'arch=x86_64' --build-arg "uid=$(id -u)" . && \
+    docker run -e 'arch=x86_64' --rm --volume "$(pwd):/src" pyallel
 ```
 
 #### Build for alpine linux
 
 ```bash
-docker build --tag pyallel-alpine --build-arg 'arch=x86_64' --file Dockerfile.alpine . && docker run -e 'arch=x86_64' --rm --volume "$(pwd):/src" pyallel-alpine
+docker build --tag pyallel-alpine --build-arg 'arch=x86_64' --build-arg "uid=$(id -u)" --file Dockerfile.alpine . && \
+    docker run -e 'arch=x86_64' --rm --volume "$(pwd):/src" pyallel-alpine
 ```
 
 #### Build locally
 
 ```bash
-python -m venv .venv && source .venv/bin/activate && pip install . -r requirements_build.txt && ./build.sh
+python -m venv .venv && \
+  source .venv/bin/activate && \
+  pip install . -r requirements_build.txt && \
+  ./build.sh
+```
+
+#### Build all
+
+```bash
+./build_all.sh
 ```
 
 ## TODOs
 
+- [x] Add support to have commands depend on other commands (some commands must complete
+      before a given command can start)
+- [ ] Add a debug mode that logs debug information to a log file
+- [ ] Fix wrapping of long commands in the command status line
+- [ ] Fix wrapping of very long lines in command output that pushes other commands off the
+      screen
+- [ ] Add support to state how many lines a command can use for it's output in interactive mode
 - [ ] Maybe add support to allow the user to provide stdin for commands that request it
       (such as a REPL)
 - [ ] Add custom parsing of command output to support filtering for errors (like vim's

{pyallel-1.1.0 → pyallel-1.2.1}/README.md

@@ -26,10 +26,10 @@ usage: pyallel [-h] [-t] [-n] [-V] [--colour {yes,no,auto}] [commands ...]
 Run and handle the output of multiple executables in pyallel (as in parallel)
 
 positional arguments:
-  commands              list of quoted commands to run e.g "mypy ." "black ."
+  commands              list of quoted commands to run in parallel e.g "mypy ." "black ."
 
                         each command is executed inside a shell, so shell syntax is supported as
-                        if you were running the command directly in a shell, some examples are below:
+                        if you were running the command directly in a shell, some examples are below
 
                              "MYPY_FORCE_COLOR=1 mypy ."          <- provide environment variables
                              "mypy | tee -a mypy.log"             <- use pipes to redirect output
@@ -38,6 +38,14 @@ positional arguments:
                              "echo \$SHELL" or "\$(echo mypy .)"  <- expand variables and commands to evaluate (must be escaped)
                              "pytest . && mypy . || echo failed!" <- use AND (&&) and OR (||) to run commands conditionally
 
+                        commands can be grouped using the group separator symbol (:::)
+
+                             "echo boil kettle" "sleep 1" ::: "echo make coffee"
+
+                        the above will print "boil kettle" and sleep for 1 second first before printing "make coffee"
+
+                        command groups are ran in the sequence you provide them, and if a command group fails
+                        (if a command fails inside the command group) the rest of the command groups in the sequence are not run
 
 options:
   -h, --help            show this help message and exit
@@ -68,26 +76,53 @@ You can also build an executable with the following (executables will be written
 > The `arch=x86_64` values in the following code blocks can be replaced with `arch=aarch64` and
 > any other architecture that is supported by docker to build an executable for that given architecture
 
+> [!NOTE]
+> To build aarch64 binaries on an x86_64 host machine, you will need to run the following
+> commands to setup qemu to allow this to work
+
+```bash
+sudo apt-get install qemu binfmt-support qemu-user-static && \
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+```
+
 #### Build for generic linux
 
 ```bash
-docker build --tag pyallel --build-arg 'arch=x86_64' . && docker run -e 'arch=x86_64' --rm --volume "$(pwd):/src" pyallel
+docker build --tag pyallel --build-arg 'arch=x86_64' --build-arg "uid=$(id -u)" . && \
+    docker run -e 'arch=x86_64' --rm --volume "$(pwd):/src" pyallel
 ```
 
 #### Build for alpine linux
 
 ```bash
-docker build --tag pyallel-alpine --build-arg 'arch=x86_64' --file Dockerfile.alpine . && docker run -e 'arch=x86_64' --rm --volume "$(pwd):/src" pyallel-alpine
+docker build --tag pyallel-alpine --build-arg 'arch=x86_64' --build-arg "uid=$(id -u)" --file Dockerfile.alpine . && \
+    docker run -e 'arch=x86_64' --rm --volume "$(pwd):/src" pyallel-alpine
 ```
 
 #### Build locally
 
 ```bash
-python -m venv .venv && source .venv/bin/activate && pip install . -r requirements_build.txt && ./build.sh
+python -m venv .venv && \
+  source .venv/bin/activate && \
+  pip install . -r requirements_build.txt && \
+  ./build.sh
+```
+
+#### Build all
+
+```bash
+./build_all.sh
 ```
 
 ## TODOs
 
+- [x] Add support to have commands depend on other commands (some commands must complete
+      before a given command can start)
+- [ ] Add a debug mode that logs debug information to a log file
+- [ ] Fix wrapping of long commands in the command status line
+- [ ] Fix wrapping of very long lines in command output that pushes other commands off the
+      screen
+- [ ] Add support to state how many lines a command can use for it's output in interactive mode
 - [ ] Maybe add support to allow the user to provide stdin for commands that request it
       (such as a REPL)
 - [ ] Add custom parsing of command output to support filtering for errors (like vim's

{pyallel-1.1.0 → pyallel-1.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyallel"
-version = "1.1.0"
+version = "1.2.1"
 description = "Run and handle the output of multiple executables in pyallel (as in parallel)"
 authors = ["Daniel Black <danielcrblack@gmail.com>"]
 license = "MIT"

{pyallel-1.1.0 → pyallel-1.2.1}/src/pyallel/constants.py

@@ -5,7 +5,7 @@ import sys
 IN_TTY = sys.stdout.isatty()
 CLEAR_LINE = "\033[2K"
 UP_LINE = "\033[1F"
-ANSI_ESCAPE = re.compile(r"(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]")
+ANSI_ESCAPE = re.compile(r"(\x9B|\x1B\[|\x1B\()[0-?]*[ -\/]*[@-~]")
 
 if IN_TTY:
 

{pyallel-1.1.0 → pyallel-1.2.1}/src/pyallel/main.py

@@ -8,23 +8,23 @@ from pyallel import constants
 from pyallel.colours import Colours
 from pyallel.errors import InvalidExecutableErrors
 from pyallel.parser import Arguments, create_parser
-from pyallel.process import ProcessGroup
+from pyallel.process_group_manager import ProcessGroupManager
 
 
 def main_loop(
-    *commands: str,
+    *args: str,
     colours: Colours,
     interactive: bool = False,
     timer: bool = False,
 ) -> int:
-    process_group = ProcessGroup.from_commands(
-        *commands,
+    process_group_manager = ProcessGroupManager.from_args(
+        *args,
         colours=colours,
         interactive=interactive,
         timer=timer,
     )
 
-    return process_group.stream()
+    return process_group_manager.stream()
 
 
 def run(*args: str) -> int:

{pyallel-1.1.0 → pyallel-1.2.1}/src/pyallel/parser.py

@@ -19,10 +19,10 @@ class Arguments:
         return msg
 
 
-COMMANDS_HELP = r"""list of quoted commands to run e.g "mypy ." "black ."
+COMMANDS_HELP = r"""list of quoted commands to run in parallel e.g "mypy ." "black ."
 
 each command is executed inside a shell, so shell syntax is supported as
-if you were running the command directly in a shell, some examples are below:
+if you were running the command directly in a shell, some examples are below
 
      "MYPY_FORCE_COLOR=1 mypy ."          <- provide environment variables
      "mypy | tee -a mypy.log"             <- use pipes to redirect output
@@ -31,6 +31,14 @@ if you were running the command directly in a shell, some examples are below:
      "echo \$SHELL" or "\$(echo mypy .)"  <- expand variables and commands to evaluate (must be escaped)
      "pytest . && mypy . || echo failed!" <- use AND (&&) and OR (||) to run commands conditionally
 
+commands can be grouped using the group separator symbol (:::)
+
+     "echo boil kettle" "sleep 1" ::: "echo make coffee"
+
+the above will print "boil kettle" and sleep for 1 second first before printing "make coffee"
+
+command groups are ran in the sequence you provide them, and if a command group fails
+(if a command fails inside the command group) the rest of the command groups in the sequence are not run
 """
 
 

pyallel-1.2.1/src/pyallel/process.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import signal
+import subprocess
+import tempfile
+import time
+from dataclasses import dataclass, field
+from typing import BinaryIO
+
+
+@dataclass
+class Process:
+    id: int
+    command: str
+    start: float = 0.0
+    end: float = 0.0
+    _fd: BinaryIO | None = field(init=False, repr=False, compare=False, default=None)
+    _process: subprocess.Popen[bytes] | None = field(
+        init=False, repr=False, compare=False, default=None
+    )
+
+    def run(self) -> None:
+        self.start = time.perf_counter()
+        fd, fd_name = tempfile.mkstemp()
+        self._fd = open(fd_name, "rb")
+        self._process = subprocess.Popen(
+            self.command,
+            stdin=subprocess.DEVNULL,
+            stdout=fd,
+            stderr=subprocess.STDOUT,
+            shell=True,
+        )
+
+    def __del__(self) -> None:
+        if self._fd:
+            self._fd.close()
+
+    def poll(self) -> int | None:
+        if self._process:
+            poll = self._process.poll()
+            if poll is not None and not self.end:
+                self.end = time.perf_counter()
+            return poll
+        return None
+
+    def read(self) -> bytes:
+        if self._fd:
+            return self._fd.read()
+        return b""
+
+    def readline(self) -> bytes:
+        if self._fd:
+            return self._fd.readline()
+        return b""
+
+    def return_code(self) -> int | None:
+        if self._process:
+            return self._process.returncode
+        return None
+
+    def interrupt(self) -> None:
+        if self._process:
+            self._process.send_signal(signal.SIGINT)
+
+    def kill(self) -> None:
+        if self._process:
+            self._process.send_signal(signal.SIGKILL)
+
+    def wait(self) -> int:
+        if self._process:
+            return self._process.wait()
+        return -1

pyallel-1.1.0/src/pyallel/process.py → pyallel-1.2.1/src/pyallel/process_group.py

@@ -1,17 +1,13 @@
 from __future__ import annotations
 
-import signal
-import subprocess
-import tempfile
 import time
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Any, BinaryIO
-from uuid import UUID, uuid4
 
 from pyallel import constants
 from pyallel.colours import Colours
 from pyallel.errors import InvalidExecutableError, InvalidExecutableErrors
+from pyallel.process import Process
 
 
 def get_num_lines(output: str, columns: int | None = None) -> int:
@@ -36,15 +32,18 @@ class ProcessGroup:
     processes: list[Process]
     interactive: bool = False
     timer: bool = False
-    output: dict[UUID, list[str]] = field(default_factory=lambda: defaultdict(list))
+    output: dict[int, list[str]] = field(default_factory=lambda: defaultdict(list))
     process_lines: list[int] = field(default_factory=list)
-    completed_processes: set[UUID] = field(default_factory=set)
+    completed_processes: set[int] = field(default_factory=set)
     exit_code: int = 0
     interrupt_count: int = 0
     passed: bool = True
     icon: int = 0
     colours: Colours = field(default_factory=Colours)
 
+    def __post_init__(self) -> None:
+        self.process_lines = [0 for _ in self.processes]
+
     def stream(self) -> int:
         for process in self.processes:
             process.run()
@@ -83,11 +82,6 @@ class ProcessGroup:
         running_process = None
         interrupted = False
 
-        print(
-            f"{self.colours.dim_on}=>{self.colours.dim_off} {self.colours.white_bold}Running commands...{self.colours.reset_colour}\n{self.colours.dim_on}=>{self.colours.dim_off} ",
-            flush=True,
-        )
-
         while True:
             output = ""
             for process in self.processes:
@@ -212,7 +206,7 @@ class ProcessGroup:
 
         return output
 
-    def _handle_signal(self, signum: int, _frame: Any) -> None:
+    def handle_signal(self, signum: int) -> None:
         for process in self.processes:
             if self.interrupt_count == 0:
                 process.interrupt()
@@ -226,16 +220,17 @@ class ProcessGroup:
     def from_commands(
         cls,
         *commands: str,
-        colours: Colours,
+        colours: Colours | None = None,
         interactive: bool = False,
         timer: bool = False,
     ) -> ProcessGroup:
+        colours = colours or Colours()
         processes: list[Process] = []
         errors: list[InvalidExecutableError] = []
 
-        for command in commands:
+        for i, command in enumerate(commands):
             try:
-                processes.append(Process.from_command(command))
+                processes.append(Process(i + 1, command))
             except InvalidExecutableError as e:
                 errors.append(e)
 
@@ -249,23 +244,15 @@ class ProcessGroup:
             colours=colours,
         )
 
-        signal.signal(signal.SIGINT, process_group._handle_signal)
-        signal.signal(signal.SIGTERM, process_group._handle_signal)
-
         return process_group
 
-    def complete_output(self, tail: int = 20, all: bool = False) -> str:
+    def complete_output(self, all: bool = False) -> str:
         num_processes = len(self.processes)
         lines = constants.LINES() - (2 * num_processes)
         remainder = lines % num_processes
         tail = lines // num_processes
-
-        if self.process_lines:
-            self.process_lines = []
-
-        for process in self.processes:
-            self.process_lines.append(tail)
-
+        for i in range(num_processes):
+            self.process_lines[i] = tail
         if remainder:
             self.process_lines[-1] += remainder - 2
         else:
@@ -319,71 +306,3 @@ class ProcessGroup:
             output += f"\n{self.colours.red_bold}Abort!{self.colours.reset_colour}"
 
         return output
-
-
-@dataclass
-class Process:
-    id: UUID = field(repr=False, compare=False)
-    command: str
-    start: float = 0.0
-    end: float = 0.0
-    _fd: BinaryIO | None = field(init=False, repr=False, compare=False, default=None)
-    _process: subprocess.Popen[bytes] | None = field(
-        init=False, repr=False, compare=False, default=None
-    )
-
-    def run(self) -> None:
-        self.start = time.perf_counter()
-        fd, fd_name = tempfile.mkstemp()
-        self._fd = open(fd_name, "rb")
-        self._process = subprocess.Popen(
-            self.command,
-            stdin=subprocess.DEVNULL,
-            stdout=fd,
-            stderr=subprocess.STDOUT,
-            shell=True,
-        )
-
-    def __del__(self) -> None:
-        if self._fd:
-            self._fd.close()
-
-    def poll(self) -> int | None:
-        if self._process:
-            poll = self._process.poll()
-            if poll is not None and not self.end:
-                self.end = time.perf_counter()
-            return poll
-        return None
-
-    def read(self) -> bytes:
-        if self._fd:
-            return self._fd.read()
-        return b""
-
-    def readline(self) -> bytes:
-        if self._fd:
-            return self._fd.readline()
-        return b""
-
-    def return_code(self) -> int | None:
-        if self._process:
-            return self._process.returncode
-        return None
-
-    def interrupt(self) -> None:
-        if self._process:
-            self._process.send_signal(signal.SIGINT)
-
-    def kill(self) -> None:
-        if self._process:
-            self._process.send_signal(signal.SIGKILL)
-
-    def wait(self) -> int:
-        if self._process:
-            return self._process.wait()
-        return -1
-
-    @classmethod
-    def from_command(cls, command: str) -> Process:
-        return cls(id=uuid4(), command=command)

pyallel-1.2.1/src/pyallel/process_group_manager.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import signal
+from dataclasses import dataclass, field
+from typing import Any
+
+from pyallel.colours import Colours
+from pyallel.process_group import ProcessGroup
+
+
+@dataclass
+class ProcessGroupManager:
+    process_groups: list[ProcessGroup]
+    interactive: bool = False
+    colours: Colours = field(default_factory=Colours)
+
+    def stream(self) -> int:
+        exit_code = 0
+
+        if not self.interactive:
+            print(
+                f"{self.colours.dim_on}=>{self.colours.dim_off} {self.colours.white_bold}Running commands...{self.colours.reset_colour}\n{self.colours.dim_on}=>{self.colours.dim_off} ",
+                flush=True,
+            )
+
+        for process_group in self.process_groups:
+            exit_code = process_group.stream()
+            if exit_code > 0:
+                break
+
+        return exit_code
+
+    def handle_signal(self, signum: int, _frame: Any) -> None:
+        for process_group in self.process_groups:
+            process_group.handle_signal(signum)
+
+    @classmethod
+    def from_args(
+        cls,
+        *args: str,
+        colours: Colours | None = None,
+        interactive: bool = False,
+        timer: bool = False,
+    ) -> ProcessGroupManager:
+        colours = colours or Colours()
+        last_separator_index = 0
+        commands: list[str] = []
+        process_groups: list[ProcessGroup] = []
+
+        for i, arg in enumerate(args):
+            if arg == ":::":
+                if i - 1 == 0:
+                    process_groups.append(
+                        ProcessGroup.from_commands(
+                            args[0],
+                            colours=colours,
+                            interactive=interactive,
+                            timer=timer,
+                        )
+                    )
+                else:
+                    process_groups.append(
+                        ProcessGroup.from_commands(
+                            *commands[last_separator_index:],
+                            colours=colours,
+                            interactive=interactive,
+                            timer=timer,
+                        )
+                    )
+
+                last_separator_index = i
+                continue
+
+            commands.append(arg)
+
+        if len(process_groups) > 1:
+            last_separator_index -= 1
+
+        process_groups.append(
+            ProcessGroup.from_commands(
+                *commands[last_separator_index:],
+                colours=colours,
+                interactive=interactive,
+                timer=timer,
+            )
+        )
+
+        process_group_manager = cls(
+            process_groups=process_groups, interactive=interactive, colours=colours
+        )
+
+        signal.signal(signal.SIGINT, process_group_manager.handle_signal)
+        signal.signal(signal.SIGTERM, process_group_manager.handle_signal)
+
+        return process_group_manager