pyallel 1.0.1__tar.gz → 1.2.0__tar.gz

{pyallel-1.0.1 → pyallel-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyallel
-Version: 1.0.1
+Version: 1.2.0
 Summary: Run and handle the output of multiple executables in pyallel (as in parallel)
 Home-page: https://github.com/Danthewaann/pyallel
 License: MIT
@@ -43,24 +43,38 @@ pip install pyallel
 Once installed, you can run `pyallel` to see usage information, like so:
 
 ```
-usage: pyallel [-h] [-t] [-n] [-V] [-v] [--colour {yes,no,auto}] [commands ...]
+usage: pyallel [-h] [-t] [-n] [-V] [--colour {yes,no,auto}] [commands ...]
 
 Run and handle the output of multiple executables in pyallel (as in parallel)
 
 positional arguments:
-  commands              list of quoted commands to run e.g "mypy ." "black ."
+  commands              list of quoted commands to run in parallel e.g "mypy ." "black ."
 
-                        can provide environment variables to each command like so:
+                        each command is executed inside a shell, so shell syntax is supported as
+                        if you were running the command directly in a shell, some examples are below
 
-                             "MYPY_FORCE_COLOR=1 mypy ."
+                             "MYPY_FORCE_COLOR=1 mypy ."          <- provide environment variables
+                             "mypy | tee -a mypy.log"             <- use pipes to redirect output
+                             "cat > test.log < other.log"         <- use input and output redirection
+                             "mypy .; pytest ."                   <- run commands one at a time in sequence
+                             "echo \$SHELL" or "\$(echo mypy .)"  <- expand variables and commands to evaluate (must be escaped)
+                             "pytest . && mypy . || echo failed!" <- use AND (&&) and OR (||) to run commands conditionally
+
+                        commands can be grouped using the group separator symbol (:::)
+
+                             "echo boil kettle" "sleep 1" ::: "echo make coffee"
+
+                        the above will print "boil kettle" and sleep for 1 second first before printing "make coffee"
+
+                        command groups are ran in the sequence you provide them, and if a command group fails
+                        (if a command fails inside the command group) the rest of the command groups in the sequence are not run
 
 options:
   -h, --help            show this help message and exit
   -t, --no-timer        don't time how long each command is taking
   -n, --non-interactive
                         run in non-interactive mode
-  -V, --verbose         run in verbose mode
-  -v, --version         print version and exit
+  -V, --version         print version and exit
   --colour {yes,no,auto}
                         colour terminal output, defaults to "auto"
 ```
@@ -104,8 +118,11 @@ python -m venv .venv && source .venv/bin/activate && pip install . -r requiremen
 
 ## TODOs
 
-- [ ] Maybe add support for allowing commands to contain shell idiom's (such as piping
-      e.g. `echo hi | tee test.log`)
+- [x] Add support to have commands depend on other commands (some commands must complete
+      before a given command can start)
+- [ ] Add support to state how many lines a command can use for it's output in interactive mode
+- [ ] Maybe add support to allow the user to provide stdin for commands that request it
+      (such as a REPL)
 - [ ] Add custom parsing of command output to support filtering for errors (like vim's
       `errorformat`)
 - [ ] Allow list of files to be provided to supply as input arguments to each command

{pyallel-1.0.1 → pyallel-1.2.0}/README.md

@@ -21,24 +21,38 @@ pip install pyallel
 Once installed, you can run `pyallel` to see usage information, like so:
 
 ```
-usage: pyallel [-h] [-t] [-n] [-V] [-v] [--colour {yes,no,auto}] [commands ...]
+usage: pyallel [-h] [-t] [-n] [-V] [--colour {yes,no,auto}] [commands ...]
 
 Run and handle the output of multiple executables in pyallel (as in parallel)
 
 positional arguments:
-  commands              list of quoted commands to run e.g "mypy ." "black ."
+  commands              list of quoted commands to run in parallel e.g "mypy ." "black ."
 
-                        can provide environment variables to each command like so:
+                        each command is executed inside a shell, so shell syntax is supported as
+                        if you were running the command directly in a shell, some examples are below
 
-                             "MYPY_FORCE_COLOR=1 mypy ."
+                             "MYPY_FORCE_COLOR=1 mypy ."          <- provide environment variables
+                             "mypy | tee -a mypy.log"             <- use pipes to redirect output
+                             "cat > test.log < other.log"         <- use input and output redirection
+                             "mypy .; pytest ."                   <- run commands one at a time in sequence
+                             "echo \$SHELL" or "\$(echo mypy .)"  <- expand variables and commands to evaluate (must be escaped)
+                             "pytest . && mypy . || echo failed!" <- use AND (&&) and OR (||) to run commands conditionally
+
+                        commands can be grouped using the group separator symbol (:::)
+
+                             "echo boil kettle" "sleep 1" ::: "echo make coffee"
+
+                        the above will print "boil kettle" and sleep for 1 second first before printing "make coffee"
+
+                        command groups are ran in the sequence you provide them, and if a command group fails
+                        (if a command fails inside the command group) the rest of the command groups in the sequence are not run
 
 options:
   -h, --help            show this help message and exit
   -t, --no-timer        don't time how long each command is taking
   -n, --non-interactive
                         run in non-interactive mode
-  -V, --verbose         run in verbose mode
-  -v, --version         print version and exit
+  -V, --version         print version and exit
   --colour {yes,no,auto}
                         colour terminal output, defaults to "auto"
 ```
@@ -82,8 +96,11 @@ python -m venv .venv && source .venv/bin/activate && pip install . -r requiremen
 
 ## TODOs
 
-- [ ] Maybe add support for allowing commands to contain shell idiom's (such as piping
-      e.g. `echo hi | tee test.log`)
+- [x] Add support to have commands depend on other commands (some commands must complete
+      before a given command can start)
+- [ ] Add support to state how many lines a command can use for it's output in interactive mode
+- [ ] Maybe add support to allow the user to provide stdin for commands that request it
+      (such as a REPL)
 - [ ] Add custom parsing of command output to support filtering for errors (like vim's
       `errorformat`)
 - [ ] Allow list of files to be provided to supply as input arguments to each command

{pyallel-1.0.1 → pyallel-1.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyallel"
-version = "1.0.1"
+version = "1.2.0"
 description = "Run and handle the output of multiple executables in pyallel (as in parallel)"
 authors = ["Daniel Black <danielcrblack@gmail.com>"]
 license = "MIT"

{pyallel-1.0.1 → pyallel-1.2.0}/src/pyallel/main.py

@@ -8,25 +8,23 @@ from pyallel import constants
 from pyallel.colours import Colours
 from pyallel.errors import InvalidExecutableErrors
 from pyallel.parser import Arguments, create_parser
-from pyallel.process import ProcessGroup
+from pyallel.process_group_manager import ProcessGroupManager
 
 
 def main_loop(
-    *commands: str,
+    *args: str,
     colours: Colours,
     interactive: bool = False,
     timer: bool = False,
-    verbose: bool = False,
 ) -> int:
-    process_group = ProcessGroup.from_commands(
-        *commands,
+    process_group_manager = ProcessGroupManager.from_args(
+        *args,
         colours=colours,
         interactive=interactive,
         timer=timer,
-        verbose=verbose,
     )
 
-    return process_group.stream()
+    return process_group_manager.stream()
 
 
 def run(*args: str) -> int:
@@ -57,7 +55,6 @@ def run(*args: str) -> int:
             colours=colours,
             interactive=interactive,
             timer=parsed_args.timer,
-            verbose=parsed_args.verbose,
         )
     except InvalidExecutableErrors as e:
         exit_code = 1

{pyallel-1.0.1 → pyallel-1.2.0}/src/pyallel/parser.py

@@ -9,7 +9,6 @@ class Arguments:
     commands: list[str]
     interactive: bool
     timer: bool
-    verbose: bool
     version: bool
 
     def __repr__(self) -> str:
@@ -20,11 +19,26 @@ class Arguments:
         return msg
 
 
-COMMANDS_HELP = """list of quoted commands to run e.g "mypy ." "black ."
+COMMANDS_HELP = r"""list of quoted commands to run in parallel e.g "mypy ." "black ."
 
-can provide environment variables to each command like so:
+each command is executed inside a shell, so shell syntax is supported as
+if you were running the command directly in a shell, some examples are below
 
-     "MYPY_FORCE_COLOR=1 mypy ."
+     "MYPY_FORCE_COLOR=1 mypy ."          <- provide environment variables
+     "mypy | tee -a mypy.log"             <- use pipes to redirect output
+     "cat > test.log < other.log"         <- use input and output redirection
+     "mypy .; pytest ."                   <- run commands one at a time in sequence
+     "echo \$SHELL" or "\$(echo mypy .)"  <- expand variables and commands to evaluate (must be escaped)
+     "pytest . && mypy . || echo failed!" <- use AND (&&) and OR (||) to run commands conditionally
+
+commands can be grouped using the group separator symbol (:::)
+
+     "echo boil kettle" "sleep 1" ::: "echo make coffee"
+
+the above will print "boil kettle" and sleep for 1 second first before printing "make coffee"
+
+command groups are ran in the sequence you provide them, and if a command group fails
+(if a command fails inside the command group) the rest of the command groups in the sequence are not run
 """
 
 
@@ -57,13 +71,6 @@ def create_parser() -> ArgumentParser:
     )
     parser.add_argument(
         "-V",
-        "--verbose",
-        help="run in verbose mode",
-        action="store_true",
-        default=False,
-    )
-    parser.add_argument(
-        "-v",
         "--version",
         help="print version and exit",
         action="store_true",

pyallel-1.2.0/src/pyallel/process.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import signal
+import subprocess
+import tempfile
+import time
+from dataclasses import dataclass, field
+from typing import BinaryIO
+
+
+@dataclass
+class Process:
+    id: int
+    command: str
+    start: float = 0.0
+    end: float = 0.0
+    _fd: BinaryIO | None = field(init=False, repr=False, compare=False, default=None)
+    _process: subprocess.Popen[bytes] | None = field(
+        init=False, repr=False, compare=False, default=None
+    )
+
+    def run(self) -> None:
+        self.start = time.perf_counter()
+        fd, fd_name = tempfile.mkstemp()
+        self._fd = open(fd_name, "rb")
+        self._process = subprocess.Popen(
+            self.command,
+            stdin=subprocess.DEVNULL,
+            stdout=fd,
+            stderr=subprocess.STDOUT,
+            shell=True,
+        )
+
+    def __del__(self) -> None:
+        if self._fd:
+            self._fd.close()
+
+    def poll(self) -> int | None:
+        if self._process:
+            poll = self._process.poll()
+            if poll is not None and not self.end:
+                self.end = time.perf_counter()
+            return poll
+        return None
+
+    def read(self) -> bytes:
+        if self._fd:
+            return self._fd.read()
+        return b""
+
+    def readline(self) -> bytes:
+        if self._fd:
+            return self._fd.readline()
+        return b""
+
+    def return_code(self) -> int | None:
+        if self._process:
+            return self._process.returncode
+        return None
+
+    def interrupt(self) -> None:
+        if self._process:
+            self._process.send_signal(signal.SIGINT)
+
+    def kill(self) -> None:
+        if self._process:
+            self._process.send_signal(signal.SIGKILL)
+
+    def wait(self) -> int:
+        if self._process:
+            return self._process.wait()
+        return -1
+
+    @classmethod
+    def from_command(cls, id: int, command: str) -> Process:
+        return cls(id=id, command=command)

pyallel-1.0.1/src/pyallel/process.py → pyallel-1.2.0/src/pyallel/process_group.py

@@ -1,20 +1,13 @@
 from __future__ import annotations
 
-import os
-import shlex
-import shutil
-import signal
-import subprocess
-import tempfile
 import time
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Any, BinaryIO
-from uuid import UUID, uuid4
 
 from pyallel import constants
 from pyallel.colours import Colours
 from pyallel.errors import InvalidExecutableError, InvalidExecutableErrors
+from pyallel.process import Process
 
 
 def get_num_lines(output: str, columns: int | None = None) -> int:
@@ -39,10 +32,9 @@ class ProcessGroup:
     processes: list[Process]
     interactive: bool = False
     timer: bool = False
-    verbose: bool = False
-    output: dict[UUID, list[str]] = field(default_factory=lambda: defaultdict(list))
+    output: dict[int, list[str]] = field(default_factory=lambda: defaultdict(list))
     process_lines: list[int] = field(default_factory=list)
-    completed_processes: set[UUID] = field(default_factory=set)
+    completed_processes: set[int] = field(default_factory=set)
     exit_code: int = 0
     interrupt_count: int = 0
     passed: bool = True
@@ -87,11 +79,6 @@ class ProcessGroup:
         running_process = None
         interrupted = False
 
-        print(
-            f"{self.colours.dim_on}=>{self.colours.dim_off} {self.colours.white_bold}Running commands...{self.colours.reset_colour}\n{self.colours.dim_on}=>{self.colours.dim_off} ",
-            flush=True,
-        )
-
         while True:
             output = ""
             for process in self.processes:
@@ -99,7 +86,7 @@ class ProcessGroup:
                     running_process is None
                     and process.id not in self.completed_processes
                 ):
-                    output += self._get_command_status(process, verbose=self.verbose)
+                    output += self._get_command_status(process)
                     output += "\n"
                     running_process = process
                 elif running_process is not process:
@@ -138,7 +125,6 @@ class ProcessGroup:
                     output += self._get_command_status(
                         process,
                         passed=process.return_code() == 0,
-                        verbose=self.verbose,
                         timer=self.timer,
                     )
                     output += f"\n{self.colours.dim_on}=>{self.colours.dim_off} \n"
@@ -189,7 +175,6 @@ class ProcessGroup:
         process: Process,
         icon: str | None = None,
         passed: bool | None = None,
-        verbose: bool = False,
         timer: bool = False,
     ) -> str:
         if passed is True:
@@ -207,12 +192,7 @@ class ProcessGroup:
             if not icon:
                 msg += "..."
 
-        output = f"{self.colours.dim_on}=>{self.colours.dim_off} {self.colours.white_bold}[{self.colours.reset_colour}{self.colours.blue_bold}{process.name}"
-
-        if verbose:
-            output += f" {' '.join(process.args)}"
-
-        output += f"{self.colours.reset_colour}{self.colours.white_bold}]{self.colours.reset_colour}{colour} {msg} {icon}{self.colours.reset_colour}"
+        output = f"{self.colours.dim_on}=>{self.colours.dim_off} {self.colours.white_bold}[{self.colours.reset_colour}{self.colours.blue_bold}{process.command}{self.colours.reset_colour}{self.colours.white_bold}]{self.colours.reset_colour}{colour} {msg} {icon}{self.colours.reset_colour}"
 
         if timer:
             end = process.end
@@ -223,7 +203,7 @@ class ProcessGroup:
 
         return output
 
-    def _handle_signal(self, signum: int, _frame: Any) -> None:
+    def handle_signal(self, signum: int) -> None:
         for process in self.processes:
             if self.interrupt_count == 0:
                 process.interrupt()
@@ -237,17 +217,17 @@ class ProcessGroup:
     def from_commands(
         cls,
         *commands: str,
-        colours: Colours,
+        colours: Colours | None = None,
         interactive: bool = False,
         timer: bool = False,
-        verbose: bool = False,
     ) -> ProcessGroup:
+        colours = colours or Colours()
         processes: list[Process] = []
         errors: list[InvalidExecutableError] = []
 
-        for command in commands:
+        for i, command in enumerate(commands):
             try:
-                processes.append(Process.from_command(command))
+                processes.append(Process.from_command(i + 1, command))
             except InvalidExecutableError as e:
                 errors.append(e)
 
@@ -258,13 +238,9 @@ class ProcessGroup:
             processes=processes,
             interactive=interactive,
             timer=timer,
-            verbose=verbose,
             colours=colours,
         )
 
-        signal.signal(signal.SIGINT, process_group._handle_signal)
-        signal.signal(signal.SIGTERM, process_group._handle_signal)
-
         return process_group
 
     def complete_output(self, tail: int = 20, all: bool = False) -> str:
@@ -293,7 +269,6 @@ class ProcessGroup:
                 output += self._get_command_status(
                     process,
                     passed=process.return_code() == 0,
-                    verbose=self.verbose,
                     timer=self.timer,
                 )
                 output += "\n"
@@ -301,7 +276,6 @@ class ProcessGroup:
                 output += self._get_command_status(
                     process,
                     icon=constants.ICONS[self.icon],
-                    verbose=self.verbose,
                     timer=self.timer,
                 )
                 output += "\n"
@@ -334,93 +308,3 @@ class ProcessGroup:
             output += f"\n{self.colours.red_bold}Abort!{self.colours.reset_colour}"
 
         return output
-
-
-@dataclass
-class Process:
-    id: UUID = field(repr=False, compare=False)
-    name: str
-    args: list[str] = field(default_factory=list)
-    env: dict[str, str] = field(default_factory=dict)
-    start: float = 0.0
-    end: float = 0.0
-    _fd: BinaryIO | None = field(init=False, repr=False, compare=False, default=None)
-    _process: subprocess.Popen[bytes] | None = field(
-        init=False, repr=False, compare=False, default=None
-    )
-
-    def run(self) -> None:
-        self.start = time.perf_counter()
-        fd, fd_name = tempfile.mkstemp()
-        self._fd = open(fd_name, "rb")
-        self._process = subprocess.Popen(
-            [self.name, *self.args],
-            stdin=subprocess.DEVNULL,
-            stdout=fd,
-            stderr=subprocess.STDOUT,
-            env=self.env,
-        )
-
-    def __del__(self) -> None:
-        if self._fd:
-            self._fd.close()
-
-    def poll(self) -> int | None:
-        if self._process:
-            poll = self._process.poll()
-            if poll is not None and not self.end:
-                self.end = time.perf_counter()
-            return poll
-        return None
-
-    def read(self) -> bytes:
-        if self._fd:
-            return self._fd.read()
-        return b""
-
-    def readline(self) -> bytes:
-        if self._fd:
-            return self._fd.readline()
-        return b""
-
-    def return_code(self) -> int | None:
-        if self._process:
-            return self._process.returncode
-        return None
-
-    def interrupt(self) -> None:
-        if self._process:
-            self._process.send_signal(signal.SIGINT)
-
-    def kill(self) -> None:
-        if self._process:
-            self._process.send_signal(signal.SIGKILL)
-
-    def wait(self) -> int:
-        if self._process:
-            return self._process.wait()
-        return -1
-
-    @classmethod
-    def from_command(cls, command: str) -> Process:
-        env = os.environ.copy()
-        args = command.split()
-
-        parsed_args: list[str] = []
-        for arg in args:
-            if "=" in arg:
-                name, env_value = arg.split("=")
-                env[name] = env_value
-            else:
-                parsed_args.append(arg)
-
-        if not shutil.which(parsed_args[0]):
-            raise InvalidExecutableError(parsed_args[0])
-
-        str_args = shlex.split(" ".join(parsed_args[1:]))
-        return cls(
-            id=uuid4(),
-            name=parsed_args[0],
-            args=str_args,
-            env=env,
-        )

pyallel-1.2.0/src/pyallel/process_group_manager.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import signal
+from dataclasses import dataclass, field
+from typing import Any
+
+from pyallel.colours import Colours
+from pyallel.process_group import ProcessGroup
+
+
+@dataclass
+class ProcessGroupManager:
+    process_groups: list[ProcessGroup]
+    interactive: bool = False
+    colours: Colours = field(default_factory=Colours)
+
+    def stream(self) -> int:
+        exit_code = 0
+
+        if not self.interactive:
+            print(
+                f"{self.colours.dim_on}=>{self.colours.dim_off} {self.colours.white_bold}Running commands...{self.colours.reset_colour}\n{self.colours.dim_on}=>{self.colours.dim_off} ",
+                flush=True,
+            )
+
+        for process_group in self.process_groups:
+            exit_code = process_group.stream()
+            if exit_code > 0:
+                break
+
+        return exit_code
+
+    def handle_signal(self, signum: int, _frame: Any) -> None:
+        for process_group in self.process_groups:
+            process_group.handle_signal(signum)
+
+    @classmethod
+    def from_args(
+        cls,
+        *args: str,
+        colours: Colours | None = None,
+        interactive: bool = False,
+        timer: bool = False,
+    ) -> ProcessGroupManager:
+        colours = colours or Colours()
+        last_separator_index = 0
+        commands: list[str] = []
+        process_groups: list[ProcessGroup] = []
+
+        for i, arg in enumerate(args):
+            if arg == ":::":
+                if i - 1 == 0:
+                    process_groups.append(
+                        ProcessGroup.from_commands(
+                            args[0],
+                            colours=colours,
+                            interactive=interactive,
+                            timer=timer,
+                        )
+                    )
+                else:
+                    process_groups.append(
+                        ProcessGroup.from_commands(
+                            *commands[last_separator_index:],
+                            colours=colours,
+                            interactive=interactive,
+                            timer=timer,
+                        )
+                    )
+
+                last_separator_index = i
+                continue
+
+            commands.append(arg)
+
+        if len(process_groups) > 1:
+            last_separator_index -= 1
+
+        process_groups.append(
+            ProcessGroup.from_commands(
+                *commands[last_separator_index:],
+                colours=colours,
+                interactive=interactive,
+                timer=timer,
+            )
+        )
+
+        process_group_manager = cls(
+            process_groups=process_groups, interactive=interactive, colours=colours
+        )
+
+        signal.signal(signal.SIGINT, process_group_manager.handle_signal)
+        signal.signal(signal.SIGTERM, process_group_manager.handle_signal)
+
+        return process_group_manager