tasktree 0.0.21__py3-none-any.whl → 0.0.23__py3-none-any.whl

tasktree/process_runner.py

@@ -0,0 +1,411 @@
+"""Process execution abstraction layer.
+
+This module provides an interface for running subprocesses, allowing for
+better testability and dependency injection.
+"""
+
+import subprocess
+import sys
+from abc import ABC, abstractmethod
+from enum import Enum
+from subprocess import Popen
+from threading import Thread
+from typing import Any
+
+__all__ = [
+    "ProcessRunner",
+    "PassthroughProcessRunner",
+    "SilentProcessRunner",
+    "StdoutOnlyProcessRunner",
+    "StderrOnlyProcessRunner",
+    "StderrOnlyOnFailureProcessRunner",
+    "TaskOutputTypes",
+    "make_process_runner",
+    "stream_output"
+]
+
+from tasktree.logging import Logger
+
+
+class TaskOutputTypes(Enum):
+    """
+    Enum defining task output control modes.
+    @athena: TBD
+    """
+
+    ALL = "all"
+    NONE = "none"
+    OUT = "out"
+    ERR = "err"
+    ON_ERR = "on-err"
+
+
+class ProcessRunner(ABC):
+    """
+    Abstract interface for running subprocess commands.
+    @athena: 78720f594104
+    """
+
+    @abstractmethod
+    def run(self, *args: Any, **kwargs: Any) -> subprocess.CompletedProcess[Any]:
+        """
+        Run a subprocess command.
+
+        This method signature matches subprocess.run() to allow for direct
+        substitution in existing code.
+
+        Args:
+        *args: Positional arguments passed to subprocess.run
+        **kwargs: Keyword arguments passed to subprocess.run
+
+        Returns:
+        subprocess.CompletedProcess: The completed process result
+
+        Raises:
+        subprocess.CalledProcessError: If check=True and process exits non-zero
+        subprocess.TimeoutExpired: If timeout is exceeded
+        @athena: c056d217be2e
+        """
+        ...
+
+
+class PassthroughProcessRunner(ProcessRunner):
+    """
+    Process runner that directly delegates to subprocess.run.
+    @athena: 470e2ca46355
+    """
+
+    def __init__(self, logger: Logger) -> None:
+        self._logger = logger
+
+    def run(self, *args: Any, **kwargs: Any) -> subprocess.CompletedProcess[Any]:
+        """
+        Run a subprocess command via subprocess.run.
+
+        Args:
+        *args: Positional arguments passed to subprocess.run
+        **kwargs: Keyword arguments passed to subprocess.run
+
+        Returns:
+        subprocess.CompletedProcess: The completed process result
+
+        Raises:
+        subprocess.CalledProcessError: If check=True and process exits non-zero
+        subprocess.TimeoutExpired: If timeout is exceeded
+        @athena: 9f6363a621f2
+        """
+        return subprocess.run(*args, **kwargs)
+
+
+class SilentProcessRunner(ProcessRunner):
+    """
+    Process runner that suppresses all subprocess output by redirecting to DEVNULL.
+    @athena: TBD
+    """
+
+    def __init__(self, logger: Logger) -> None:
+        self._logger = logger
+
+    def run(self, *args: Any, **kwargs: Any) -> subprocess.CompletedProcess[Any]:
+        """
+        Run a subprocess command with stdout and stderr suppressed.
+
+        This implementation forces stdout=DEVNULL and stderr=DEVNULL to discard
+        all subprocess output, regardless of what the caller requests.
+
+        Args:
+        *args: Positional arguments passed to subprocess.run
+        **kwargs: Keyword arguments passed to subprocess.run
+
+        Returns:
+        subprocess.CompletedProcess: The completed process result
+
+        Raises:
+        subprocess.CalledProcessError: If check=True and process exits non-zero
+        subprocess.TimeoutExpired: If timeout is exceeded
+        @athena: TBD
+        """
+        kwargs["stdout"] = subprocess.DEVNULL
+        kwargs["stderr"] = subprocess.DEVNULL
+        return subprocess.run(*args, **kwargs)
+
+
+def stream_output(pipe: Any, target: Any) -> None:
+    """
+    Stream output from a pipe to a target stream.
+
+    Handles exceptions gracefully to avoid silent thread failures.
+    If the pipe is closed or an error occurs during reading/writing,
+    the function returns without raising an exception.
+
+    Args:
+        pipe: Input pipe to read from
+        target: Output stream to write to
+    @athena: TBD
+    """
+    if pipe:
+        try:
+            for line in pipe:
+                target.write(line)
+                target.flush()
+        except (OSError, ValueError):
+            # Pipe closed or other I/O error - this is expected when
+            # process is killed or stdout is closed
+            pass
+
+
+def _start_thread_and_wait_to_complete(process: Popen[str], stream: Any, thread: Thread, process_allowed_runtime: float | None, logger: Logger) -> int:
+    join_timeout_secs = 1.0
+
+    thread.start()
+
+    try:
+        process_return_code = process.wait(timeout=process_allowed_runtime)
+    except subprocess.TimeoutExpired:
+        process.kill()
+        process.wait()
+        if stream:
+            stream.close()
+            stream = None
+        thread.join(timeout=join_timeout_secs)
+        raise
+    finally:
+        if stream:
+            stream.close()
+            stream = None
+
+    thread.join(timeout=join_timeout_secs)
+    if thread.is_alive():
+        logger.warn(f"Stream thread did not complete within timeout of {join_timeout_secs} seconds")
+
+    return process_return_code
+
+
+def _check_result_if_necessary(raise_on_failure: bool, proc_ret_code: int, *args, **kwargs) -> subprocess.CompletedProcess[Any]:
+    if raise_on_failure and proc_ret_code != 0:
+        raise subprocess.CalledProcessError(
+            proc_ret_code, args[0] if args else kwargs.get("args", [])
+        )
+
+    # Return a CompletedProcess object for interface compatibility
+    return subprocess.CompletedProcess(
+        args=args[0] if args else kwargs.get("args", []),
+        returncode=proc_ret_code,
+        stdout=None,
+        stderr=None,  # We streamed it, so don't capture it
+    )
+
+
+class StdoutOnlyProcessRunner(ProcessRunner):
+    """
+    Process runner that streams stdout while suppressing stderr.
+
+    This implementation uses threading to asynchronously stream stdout from the
+    subprocess while discarding stderr output.
+    @athena: TBD
+    """
+
+    def __init__(self, logger: Logger) -> None:
+        self._logger = logger
+
+    def run(self, *args: Any, **kwargs: Any) -> subprocess.CompletedProcess[Any]:
+        """
+        Run a subprocess command with stdout streamed and stderr suppressed.
+
+        This implementation uses subprocess.Popen with threading to stream stdout
+        in real-time while discarding stderr. The interface remains synchronous
+        from the caller's perspective.
+
+        Buffering strategy: Uses line buffering (bufsize=1) to ensure output
+        appears promptly while maintaining reasonable performance.
+
+        Args:
+            *args: Positional arguments passed to subprocess.Popen
+            **kwargs: Keyword arguments passed to subprocess.Popen
+
+        Returns:
+            subprocess.CompletedProcess: The completed process result
+
+        Raises:
+            subprocess.CalledProcessError: If check=True and process exits non-zero
+            subprocess.TimeoutExpired: If timeout is exceeded
+        @athena: TBD
+        """
+        # Extract parameters that need special handling
+        check = kwargs.pop("check", False)
+        timeout = kwargs.pop("timeout", None)
+        # Remove capture_output if present - not supported by Popen
+        kwargs.pop("capture_output", None)
+
+        # Force stdout/stderr handling
+        kwargs["stdout"] = subprocess.PIPE
+        kwargs["stderr"] = subprocess.DEVNULL
+        kwargs["text"] = True
+        kwargs["bufsize"] = 1
+
+        # Start the process
+        process = subprocess.Popen(*args, **kwargs)
+
+        # Start thread to stream stdout with a descriptive name for debugging
+        thread = Thread(
+            target=stream_output,
+            args=(process.stdout, sys.stdout),
+            name="stdout-streamer",
+            daemon=True,
+        )
+
+        process_return_code = _start_thread_and_wait_to_complete(process, process.stdout, thread, timeout, self._logger)
+        return _check_result_if_necessary(check, process_return_code, *args, **kwargs)
+
+
+class StderrOnlyProcessRunner(ProcessRunner):
+    """
+    Process runner that streams stderr while suppressing stdout.
+
+    This implementation uses threading to asynchronously stream stderr from the
+    subprocess while discarding stdout output.
+    @athena: TBD
+    """
+
+    def __init__(self, logger: Logger) -> None:
+        self._logger = logger
+
+    def run(self, *args: Any, **kwargs: Any) -> subprocess.CompletedProcess[Any]:
+        """
+        Run a subprocess command with stderr streamed and stdout suppressed.
+
+        This implementation uses subprocess.Popen with threading to stream stderr
+        in real-time while discarding stdout. The interface remains synchronous
+        from the caller's perspective.
+
+        Buffering strategy: Uses line buffering (bufsize=1) to ensure output
+        appears promptly while maintaining reasonable performance.
+
+        Args:
+            *args: Positional arguments passed to subprocess.Popen
+            **kwargs: Keyword arguments passed to subprocess.Popen
+
+        Returns:
+            subprocess.CompletedProcess: The completed process result
+
+        Raises:
+            subprocess.CalledProcessError: If check=True and process exits non-zero
+            subprocess.TimeoutExpired: If timeout is exceeded
+        @athena: TBD
+        """
+        # Extract parameters that need special handling
+        check = kwargs.pop("check", False)
+        timeout = kwargs.pop("timeout", None)
+        # Remove capture_output if present - not supported by Popen
+        kwargs.pop("capture_output", None)
+
+        # Force stdout/stderr handling
+        kwargs["stdout"] = subprocess.DEVNULL
+        kwargs["stderr"] = subprocess.PIPE
+        kwargs["text"] = True
+        kwargs["bufsize"] = 1
+
+        # Start the process
+        process = subprocess.Popen(*args, **kwargs)
+
+        # Start thread to stream stderr with a descriptive name for debugging
+        thread = Thread(
+            target=stream_output,
+            args=(process.stderr, sys.stderr),
+            name="stderr-streamer",
+            daemon=True,
+        )
+
+        process_return_code = _start_thread_and_wait_to_complete(process, process.stderr, thread, timeout, self._logger)
+        return _check_result_if_necessary(check, process_return_code, *args, **kwargs)
+
+
+class StderrOnlyOnFailureProcessRunner(ProcessRunner):
+    """
+    Process runner that buffers stderr and only outputs it on failure.
+
+    This implementation ignores stdout completely (sends to DEVNULL) and captures
+    stderr. The buffered stderr is only output if the process exits with a non-zero
+    code.
+    """
+
+    def __init__(self, logger: Logger) -> None:
+        self._logger = logger
+
+    def run(self, *args: Any, **kwargs: Any) -> subprocess.CompletedProcess[Any]:
+        """
+        Run a subprocess command, buffering stderr and outputting only on failure.
+
+        Stdout is completely ignored (sent to DEVNULL). Stderr is collected in a
+        buffer during execution. If the process exits with non-zero code, the
+        buffered stderr is written to sys.stderr before returning/raising.
+
+        Args:
+            *args: Positional arguments passed to subprocess.run
+            **kwargs: Keyword arguments passed to subprocess.run
+
+        Returns:
+            subprocess.CompletedProcess: The completed process result
+
+        Raises:
+            subprocess.CalledProcessError: If check=True and process exits non-zero
+            subprocess.TimeoutExpired: If timeout is exceeded
+        """
+        check = kwargs.pop("check", False)
+        timeout = kwargs.pop("timeout", None)
+        kwargs.pop("capture_output", None)  # Remove if present
+        kwargs.pop("stdout", None)  # Remove if present
+        kwargs.pop("stderr", None)  # Remove if present
+
+        result = subprocess.run(
+            *args,
+            **kwargs,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+            text=True,
+            timeout=timeout,
+            check=False,
+        )
+
+        if result.returncode != 0 and result.stderr:
+            sys.stderr.write(result.stderr)
+            sys.stderr.flush()
+
+        if check and result.returncode != 0:
+            raise subprocess.CalledProcessError(
+                result.returncode,
+                result.args,
+                output=result.stdout,
+                stderr=result.stderr,
+            )
+
+        return result
+
+
+def make_process_runner(output_type: TaskOutputTypes, logger: Logger) -> ProcessRunner:
+    """
+    Factory function for creating ProcessRunner instances.
+
+    Args:
+    output_type: The type of output control to use
+
+    Returns:
+    ProcessRunner: A new ProcessRunner instance
+
+    Raises:
+    ValueError: If an invalid TaskOutputTypes value is provided
+    @athena: ba1d2e048716
+    """
+    match output_type:
+        case TaskOutputTypes.ALL:
+            return PassthroughProcessRunner(logger)
+        case TaskOutputTypes.NONE:
+            return SilentProcessRunner(logger)
+        case TaskOutputTypes.OUT:
+            return StdoutOnlyProcessRunner(logger)
+        case TaskOutputTypes.ERR:
+            return StderrOnlyProcessRunner(logger)
+        case TaskOutputTypes.ON_ERR:
+            return StderrOnlyOnFailureProcessRunner(logger)
+        case _:
+            raise ValueError(f"Invalid TaskOutputTypes: {output_type}")

tasktree/state.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import json
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any
+from typing import Any, Set
 
 
 @dataclass
@@ -43,7 +43,7 @@ class TaskState:
 class StateManager:
     """
     Manages the .tasktree-state file.
-    @athena: 44713c70e04e
+    @athena: 3dd3447bb53b
     """
 
     STATE_FILE = ".tasktree-state"
@@ -64,17 +64,16 @@ class StateManager:
     def load(self) -> None:
         """
         Load state from file if it exists.
-        @athena: 11748af0886c
+        @athena: e0cf9097c590
         """
         if self.state_path.exists():
             try:
                 with open(self.state_path, "r") as f:
                     data = json.load(f)
                     self._state = {
-                        key: TaskState.from_dict(value)
-                        for key, value in data.items()
+                        key: TaskState.from_dict(value) for key, value in data.items()
                     }
-            except (json.JSONDecodeError, KeyError) as e:
+            except (json.JSONDecodeError, KeyError):
                 # If state file is corrupted, start fresh
                 self._state = {}
         self._loaded = True
@@ -116,13 +115,13 @@ class StateManager:
             self.load()
         self._state[cache_key] = state
 
-    def prune(self, valid_task_hashes: set[str]) -> None:
+    def prune(self, valid_task_hashes: Set[str]) -> None:
         """
         Remove state entries for tasks that no longer exist.
 
         Args:
         valid_task_hashes: Set of valid task hashes from current recipe
-        @athena: ce21bb523d49
+        @athena: 2717c6c244d3
         """
         if not self._loaded:
             self.load()

tasktree/substitution.py

@@ -7,30 +7,30 @@ and {{ env.NAME }} placeholders with their corresponding values.
 """
 
 import re
-from random import choice
 from typing import Any
 
-
 # Pattern matches: {{ prefix.name }} with optional whitespace
 # Groups: (1) prefix (var|arg|env|tt), (2) name (identifier)
 PLACEHOLDER_PATTERN = re.compile(
-    r'\{\{\s*(var|arg|env|tt)\.([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}'
+    r"\{\{\s*(var|arg|env|tt)\.([a-zA-Z_][a-zA-Z0-9_]*)\s*}}"
 )
 
 # Pattern matches: {{ dep.task_name.outputs.output_name }} with optional whitespace
 # Groups: (1) task_name (can include dots for namespacing), (2) output_name (identifier)
 DEP_OUTPUT_PATTERN = re.compile(
-    r'\{\{\s*dep\.([a-zA-Z_][a-zA-Z0-9_.-]*)\.outputs\.([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}'
+    r"\{\{\s*dep\.([a-zA-Z_][a-zA-Z0-9_.-]*)\.outputs\.([a-zA-Z_][a-zA-Z0-9_]*)\s*}}"
 )
 
 # Pattern matches: {{ self.(inputs|outputs).name }} or {{ self.(inputs|outputs).0 }} with optional whitespace
 # Groups: (1) field (inputs|outputs), (2) name (identifier) or index (numeric)
 SELF_REFERENCE_PATTERN = re.compile(
-    r'\{\{\s*self\.(inputs|outputs)\.([a-zA-Z_][a-zA-Z0-9_]*|[0-9]+)\s*\}\}'
+    r"\{\{\s*self\.(inputs|outputs)\.([a-zA-Z_][a-zA-Z0-9_]*|[0-9]+)\s*}}"
 )
 
 
-def substitute_variables(text: str | dict[str, Any], variables: dict[str, str]) -> str | dict[str, Any]:
+def substitute_variables(
+    text: str | dict[str, Any], variables: dict[str, str]
+) -> str | dict[str, Any]:
     """
     Substitute {{ var.name }} placeholders with variable values.
 
@@ -52,13 +52,18 @@ def substitute_variables(text: str | dict[str, Any], variables: dict[str, str])
 
         for arg_name in text.keys():
             # Pull out and substitute the individual fields of an argument one at a time
-            for field in  [ "default", "min", "max" ]:
+            for field in ["default", "min", "max"]:
                 if field in text[arg_name]:
-                    text[arg_name][field] = substitute_variables(text[arg_name][field], variables)
+                    text[arg_name][field] = substitute_variables(
+                        text[arg_name][field], variables
+                    )
 
             # choices is a list of things
             if "choices" in text[arg_name]:
-                text[arg_name]["choices"] = [substitute_variables(c, variables) for c in text[arg_name]["choices"]]
+                text[arg_name]["choices"] = [
+                    substitute_variables(c, variables)
+                    for c in text[arg_name]["choices"]
+                ]
 
             return text
         else:
@@ -88,7 +93,9 @@ def substitute_variables(text: str | dict[str, Any], variables: dict[str, str])
         return PLACEHOLDER_PATTERN.sub(replace_match, text)
 
 
-def substitute_arguments(text: str, args: dict[str, Any], exported_args: set[str] | None = None) -> str:
+def substitute_arguments(
+    text: str, args: dict[str, Any], exported_args: set[str] | None = None
+) -> str:
     """
     Substitute {{ arg.name }} placeholders with argument values.
 
@@ -172,9 +179,7 @@ def substitute_environment(text: str) -> str:
 
         value = os.environ.get(name)
         if value is None:
-            raise ValueError(
-                f"Environment variable '{name}' is not set"
-            )
+            raise ValueError(f"Environment variable '{name}' is not set")
 
         return value
 
@@ -203,6 +208,7 @@ def substitute_builtin_variables(text: str, builtin_vars: dict[str, str]) -> str
     'Root: /home/user/project'
     @athena: 716250e3a71f
     """
+
     def replace_match(match: re.Match) -> str:
         prefix = match.group(1)
         name = match.group(2)
@@ -226,7 +232,7 @@ def substitute_dependency_args(
     template_value: str,
     parent_task_name: str,
     parent_args: dict[str, Any],
-    exported_args: set[str] | None = None
+    exported_args: set[str] | None = None,
 ) -> str:
     """
     Substitute {{ arg.* }} templates in dependency argument values.
@@ -250,7 +256,7 @@ def substitute_dependency_args(
     Example:
     >>> substitute_dependency_args("{{ arg.mode }}", "build", {"mode": "debug"})
     'debug'
-    @athena: 3d07a1b4e6bc
+    @athena: 4ffd5664e3ec
     """
     # Check for disallowed placeholder types in dependency args
     # Only {{ arg.* }} is allowed, not {{ var.* }}, {{ env.* }}, or {{ tt.* }}
@@ -355,8 +361,9 @@ def substitute_dependency_outputs(
     ...     {"build": build_task}
     ... )
     'Deploy dist/app.js'
-    @athena: 1e537c8d579c
+    @athena: 3fbf79c15ee9
     """
+
     def replacer(match: re.Match) -> str:
         dep_task_name = match.group(1)
         output_name = match.group(2)
@@ -383,7 +390,11 @@ def substitute_dependency_outputs(
         # Look up the named output
         if output_name not in dep_task._output_map:
             available = list(dep_task._output_map.keys())
-            available_msg = ", ".join(available) if available else "(none - all outputs are anonymous)"
+            available_msg = (
+                ", ".join(available)
+                if available
+                else "(none - all outputs are anonymous)"
+            )
             raise ValueError(
                 f"Task '{current_task_name}' references output '{output_name}' "
                 f"from task '{dep_task_name}', but '{dep_task_name}' has no output named '{output_name}'.\n"
@@ -447,6 +458,7 @@ def substitute_self_references(
     'cp *.txt out/result.txt'
     @athena: 9d997ff08eef
     """
+
     def replacer(match: re.Match) -> str:
         field = match.group(1)  # "inputs" or "outputs"
         identifier = match.group(2)  # name or numeric index