tasktree 0.0.21__py3-none-any.whl → 0.0.23__py3-none-any.whl

tasktree/executor.py

@@ -2,22 +2,32 @@
 
 from __future__ import annotations
 
+import io
 import os
 import platform
 import stat
 import subprocess
+import sys
 import tempfile
 import time
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any
+from typing import Any, Callable
 
 from tasktree import docker as docker_module
-from tasktree.graph import get_implicit_inputs, resolve_execution_order, resolve_dependency_output_references, resolve_self_references
+from tasktree.graph import (
+    get_implicit_inputs,
+    resolve_execution_order,
+    resolve_dependency_output_references,
+    resolve_self_references,
+)
 from tasktree.hasher import hash_args, hash_task, make_cache_key
+from tasktree.logging import Logger, LogLevel
 from tasktree.parser import Recipe, Task, Environment
+from tasktree.process_runner import ProcessRunner, TaskOutputTypes
 from tasktree.state import StateManager, TaskState
+from tasktree.hasher import hash_environment_definition
 
 
 @dataclass
@@ -47,35 +57,46 @@ class ExecutionError(Exception):
 class Executor:
     """
     Executes tasks with incremental execution logic.
-    @athena: ac1e2fc7b82b
+    @athena: 779b12944194
     """
 
     # Protected environment variables that cannot be overridden by exported args
     PROTECTED_ENV_VARS = {
-        'PATH',
-        'LD_LIBRARY_PATH',
-        'LD_PRELOAD',
-        'PYTHONPATH',
-        'HOME',
-        'SHELL',
-        'USER',
-        'LOGNAME',
+        "PATH",
+        "LD_LIBRARY_PATH",
+        "LD_PRELOAD",
+        "PYTHONPATH",
+        "HOME",
+        "SHELL",
+        "USER",
+        "LOGNAME",
     }
 
-    def __init__(self, recipe: Recipe, state_manager: StateManager):
+    def __init__(
+        self,
+        recipe: Recipe,
+        state_manager: StateManager,
+        logger: Logger,
+        process_runner_factory: Callable[[TaskOutputTypes, Logger], ProcessRunner]
+    ):
         """
         Initialize executor.
 
         Args:
         recipe: Parsed recipe containing all tasks
         state_manager: State manager for tracking task execution
-        @athena: 21b65db48bca
+        logger_fn: Logger function for output (matches Console.print signature)
+        process_runner_factory: Factory function for creating ProcessRunner instances
+        @athena: d09e6a537c99
         """
         self.recipe = recipe
         self.state = state_manager
+        self.logger = logger
+        self._process_runner_factory = process_runner_factory
         self.docker_manager = docker_module.DockerManager(recipe.project_root)
 
-    def _has_regular_args(self, task: Task) -> bool:
+    @staticmethod
+    def _has_regular_args(task: Task) -> bool:
         """
         Check if a task has any regular (non-exported) arguments.
 
@@ -84,7 +105,7 @@ class Executor:
 
         Returns:
         True if task has at least one regular (non-exported) argument, False otherwise
-        @athena: 0fc46146eed3
+        @athena: c529cda63cce
         """
         if not task.args:
             return False
@@ -94,18 +115,19 @@ class Executor:
             # Handle both string and dict arg specs
             if isinstance(arg_spec, str):
                 # Remove default value part if present
-                arg_name = arg_spec.split('=')[0].split(':')[0].strip()
-                if not arg_name.startswith('$'):
+                arg_name = arg_spec.split("=")[0].split(":")[0].strip()
+                if not arg_name.startswith("$"):
                     return True
             elif isinstance(arg_spec, dict):
                 # Dict format: { argname: { ... } } or { $argname: { ... } }
                 for key in arg_spec.keys():
-                    if not key.startswith('$'):
+                    if not key.startswith("$"):
                         return True
 
         return False
 
-    def _filter_regular_args(self, task: Task, task_args: dict[str, Any]) -> dict[str, Any]:
+    @staticmethod
+    def _filter_regular_args(task: Task, task_args: dict[str, Any]) -> dict[str, Any]:
         """
         Filter task_args to only include regular (non-exported) arguments.
 
@@ -115,7 +137,7 @@ class Executor:
 
         Returns:
         Dictionary containing only regular (non-exported) arguments
-        @athena: 811abd3a56f9
+        @athena: 1ae863406335
         """
         if not task.args or not task_args:
             return {}
@@ -124,18 +146,20 @@ class Executor:
         exported_names = set()
         for arg_spec in task.args:
             if isinstance(arg_spec, str):
-                arg_name = arg_spec.split('=')[0].split(':')[0].strip()
-                if arg_name.startswith('$'):
+                arg_name = arg_spec.split("=")[0].split(":")[0].strip()
+                if arg_name.startswith("$"):
                     exported_names.add(arg_name[1:])  # Remove $ prefix
             elif isinstance(arg_spec, dict):
                 for key in arg_spec.keys():
-                    if key.startswith('$'):
+                    if key.startswith("$"):
                         exported_names.add(key[1:])  # Remove $ prefix
 
         # Filter out exported args
         return {k: v for k, v in task_args.items() if k not in exported_names}
 
-    def _collect_early_builtin_variables(self, task: Task, timestamp: datetime) -> dict[str, str]:
+    def _collect_early_builtin_variables(
+        self, task: Task, timestamp: datetime
+    ) -> dict[str, str]:
         """
         Collect built-in variables that don't depend on working_dir.
 
@@ -150,31 +174,27 @@ class Executor:
 
         Raises:
         ExecutionError: If any built-in variable fails to resolve
-        @athena: 0b348e67ce4c
+        @athena: a0c1316fd713
         """
         import os
 
-        builtin_vars = {}
-
-        # {{ tt.project_root }} - Absolute path to project root
-        builtin_vars['project_root'] = str(self.recipe.project_root.resolve())
-
-        # {{ tt.recipe_dir }} - Absolute path to directory containing the recipe file
-        builtin_vars['recipe_dir'] = str(self.recipe.recipe_path.parent.resolve())
-
-        # {{ tt.task_name }} - Name of currently executing task
-        builtin_vars['task_name'] = task.name
-
-        # {{ tt.timestamp }} - ISO8601 timestamp when task started execution
-        builtin_vars['timestamp'] = timestamp.strftime('%Y-%m-%dT%H:%M:%SZ')
-
-        # {{ tt.timestamp_unix }} - Unix epoch timestamp when task started
-        builtin_vars['timestamp_unix'] = str(int(timestamp.timestamp()))
+        builtin_vars = {
+            # {{ tt.project_root }} - Absolute path to project root
+            "project_root": str(self.recipe.project_root.resolve()),
+            # {{ tt.recipe_dir }} - Absolute path to directory containing the recipe file
+            "recipe_dir": str(self.recipe.recipe_path.parent.resolve()),
+            # {{ tt.task_name }} - Name of currently executing task
+            "task_name": task.name,
+            # {{ tt.timestamp }} - ISO8601 timestamp when task started execution
+            "timestamp": timestamp.strftime("%Y-%m-%dT%H:%M:%SZ"),
+            # {{ tt.timestamp_unix }} - Unix epoch timestamp when task started
+            "timestamp_unix": str(int(timestamp.timestamp())),
+        }
 
         # {{ tt.user_home }} - Current user's home directory (cross-platform)
         try:
             user_home = Path.home()
-            builtin_vars['user_home'] = str(user_home)
+            builtin_vars["user_home"] = str(user_home)
         except Exception as e:
             raise ExecutionError(
                 f"Failed to get user home directory for {{ tt.user_home }}: {e}"
@@ -185,12 +205,16 @@ class Executor:
             user_name = os.getlogin()
         except OSError:
             # Fallback to environment variables if os.getlogin() fails
-            user_name = os.environ.get('USER') or os.environ.get('USERNAME') or 'unknown'
-        builtin_vars['user_name'] = user_name
+            user_name = (
+                os.environ.get("USER") or os.environ.get("USERNAME") or "unknown"
+            )
+        builtin_vars["user_name"] = user_name
 
         return builtin_vars
 
-    def _collect_builtin_variables(self, task: Task, working_dir: Path, timestamp: datetime) -> dict[str, str]:
+    def _collect_builtin_variables(
+        self, task: Task, working_dir: Path, timestamp: datetime
+    ) -> dict[str, str]:
         """
         Collect built-in variables for task execution.
 
@@ -204,18 +228,20 @@ class Executor:
 
         Raises:
         ExecutionError: If any built-in variable fails to resolve
-        @athena: bb8c385cb0a5
+        @athena: 7f6203e8d617
         """
         # Get early builtin vars (those that don't depend on working_dir)
         builtin_vars = self._collect_early_builtin_variables(task, timestamp)
 
         # {{ tt.working_dir }} - Absolute path to task's effective working directory
         # This is added after working_dir is resolved to avoid circular dependency
-        builtin_vars['working_dir'] = str(working_dir.resolve())
+        builtin_vars["working_dir"] = str(working_dir.resolve())
 
         return builtin_vars
 
-    def _prepare_env_with_exports(self, exported_env_vars: dict[str, str] | None = None) -> dict[str, str]:
+    def _prepare_env_with_exports(
+        self, exported_env_vars: dict[str, str] | None = None
+    ) -> dict[str, str]:
         """
         Prepare environment with exported arguments.
 
@@ -241,19 +267,20 @@ class Executor:
             env.update(exported_env_vars)
         return env
 
-    def _get_platform_default_environment(self) -> tuple[str, list[str]]:
+    @staticmethod
+    def _get_platform_default_environment() -> tuple[str, list[str]]:
         """
         Get default shell and args for current platform.
 
         Returns:
         Tuple of (shell, args) for platform default
-        @athena: b67799671787
+        @athena: 8b7fa81073af
         """
         is_windows = platform.system() == "Windows"
         if is_windows:
-            return ("cmd", ["/c"])
+            return "cmd", ["/c"]
         else:
-            return ("bash", ["-c"])
+            return "bash", ["-c"]
 
     def _get_effective_env_name(self, task: Task) -> str:
         """
@@ -287,7 +314,7 @@ class Executor:
         # Platform default (no env name)
         return ""
 
-    def _resolve_environment(self, task: Task) -> tuple[str, list[str], str]:
+    def _resolve_environment(self, task: Task) -> tuple[str, str]:
         """
         Resolve which environment to use for a task.
 
@@ -301,8 +328,8 @@ class Executor:
         task: Task to resolve environment for
 
         Returns:
-        Tuple of (shell, args, preamble)
-        @athena: b919568f73fc
+        Tuple of (shell, preamble)
+        @athena: 15cad76d7c80
         """
         # Check for global override first
         env_name = self.recipe.global_env_override
@@ -319,17 +346,18 @@ class Executor:
         if env_name:
             env = self.recipe.get_environment(env_name)
             if env:
-                return (env.shell, env.args, env.preamble)
+                return env.shell, env.preamble
             # If env not found, fall through to platform default
 
         # Use platform default
-        shell, args = self._get_platform_default_environment()
-        return (shell, args, "")
+        shell, _ = self._get_platform_default_environment()
+        return shell, ""
 
     def check_task_status(
         self,
         task: Task,
         args_dict: dict[str, Any],
+        process_runner: ProcessRunner,
         force: bool = False,
     ) -> TaskStatus:
         """
@@ -348,11 +376,12 @@ class Executor:
         Args:
         task: Task to check
         args_dict: Arguments for this task execution
+        process_runner: ProcessRunner instance for subprocess execution
         force: If True, ignore freshness and force execution
 
         Returns:
         TaskStatus indicating whether task will run and why
-        @athena: 7252f5db8a4d
+        @athena: 03922de1bd23
         """
         # If force flag is set, always run
         if force:
@@ -364,7 +393,14 @@ class Executor:
 
         # Compute hashes (include effective environment and dependencies)
         effective_env = self._get_effective_env_name(task)
-        task_hash = hash_task(task.cmd, task.outputs, task.working_dir, task.args, effective_env, task.deps)
+        task_hash = hash_task(
+            task.cmd,
+            task.outputs,
+            task.working_dir,
+            task.args,
+            effective_env,
+            task.deps,
+        )
         args_hash = hash_args(args_dict) if args_dict else None
         cache_key = make_cache_key(task_hash, args_hash)
 
@@ -386,8 +422,9 @@ class Executor:
                 reason="never_run",
             )
 
-        # Check if environment definition has changed
-        env_changed = self._check_environment_changed(task, cached_state, effective_env)
+        env_changed = self._check_environment_changed(
+            task, cached_state, effective_env, process_runner
+        )
         if env_changed:
             return TaskStatus(
                 task_name=task.name,
@@ -426,9 +463,21 @@ class Executor:
             last_run=datetime.fromtimestamp(cached_state.last_run),
         )
 
+    @staticmethod
+    def _get_task_output_type(user_inputted_value: TaskOutputTypes | None, task: Task) -> TaskOutputTypes:
+        if user_inputted_value is None:
+            if task.task_output is not None:
+                return task.task_output
+
+            return TaskOutputTypes.ALL
+
+        return user_inputted_value
+
+
     def execute_task(
         self,
         task_name: str,
+        user_inputted_task_output_types: TaskOutputTypes | None,
         args_dict: dict[str, Any] | None = None,
         force: bool = False,
         only: bool = False,
@@ -438,6 +487,7 @@ class Executor:
 
         Args:
         task_name: Name of task to execute
+        task_output_type: TaskOutputTypes enum value for controlling subprocess output
         args_dict: Arguments to pass to the task
         force: If True, ignore freshness and re-run all tasks
         only: If True, run only the specified task without dependencies (implies force=True)
@@ -447,7 +497,7 @@ class Executor:
 
         Raises:
         ExecutionError: If task execution fails
-        @athena: 1c293ee6a6fa
+        @athena: 4773fc590d9a
         """
         if args_dict is None:
             args_dict = {}
@@ -480,20 +530,31 @@ class Executor:
             # Convert None to {} for internal use (None is used to distinguish simple deps in graph)
             args_dict_for_execution = task_args if task_args is not None else {}
 
+            process_runner = self._process_runner_factory(self._get_task_output_type(user_inputted_task_output_types, task), self.logger)
+
             # Check if task needs to run (based on CURRENT filesystem state)
-            status = self.check_task_status(task, args_dict_for_execution, force=force)
+            status = self.check_task_status(
+                task, args_dict_for_execution, process_runner, force=force
+            )
 
             # Use a key that includes args for status tracking
             # Only include regular (non-exported) args in status key for parameterized dependencies
             # For the root task (invoked from CLI), status key is always just the task name
             # For dependencies with parameterized invocations, include the regular args
-            is_root_task = (name == task_name)
-            if not is_root_task and args_dict_for_execution and self._has_regular_args(task):
+            is_root_task = name == task_name
+            if (
+                not is_root_task
+                and args_dict_for_execution
+                and self._has_regular_args(task)
+            ):
                 import json
+
                 # Filter to only include regular (non-exported) args
                 regular_args = self._filter_regular_args(task, args_dict_for_execution)
                 if regular_args:
-                    args_str = json.dumps(regular_args, sort_keys=True, separators=(",", ":"))
+                    args_str = json.dumps(
+                        regular_args, sort_keys=True, separators=(",", ":")
+                    )
                     status_key = f"{name}({args_str})"
                 else:
                     status_key = name
@@ -505,27 +566,29 @@ class Executor:
             if status.will_run:
                 # Warn if re-running due to missing outputs
                 if status.reason == "outputs_missing":
-                    import sys
-                    print(
+                    self.logger.log(
+                        LogLevel.WARN,
                         f"Warning: Re-running task '{name}' because declared outputs are missing",
-                        file=sys.stderr,
                     )
 
-                self._run_task(task, args_dict_for_execution)
+                self._run_task(task, args_dict_for_execution, process_runner)
 
         return statuses
 
-    def _run_task(self, task: Task, args_dict: dict[str, Any]) -> None:
+    def _run_task(
+        self, task: Task, args_dict: dict[str, Any], process_runner: ProcessRunner
+    ) -> None:
         """
         Execute a single task.
 
         Args:
         task: Task to execute
         args_dict: Arguments to substitute in command
+        process_runner: ProcessRunner instance for subprocess execution
 
         Raises:
         ExecutionError: If task execution fails
-        @athena: 885c66658550
+        @athena: b5abffeef10a
         """
         # Capture timestamp at task start for consistency (in UTC)
         task_start_time = datetime.now(timezone.utc)
@@ -533,6 +596,7 @@ class Executor:
         # Parse task arguments to identify exported args
         # Note: args_dict already has defaults applied by CLI (cli.py:413-424)
         from tasktree.parser import parse_arg_spec
+
         exported_args = set()
         regular_args = {}
         exported_env_vars = {}
@@ -551,18 +615,24 @@ class Executor:
 
         # Collect early built-in variables (those that don't depend on working_dir)
         # These can be used in the working_dir field itself
-        early_builtin_vars = self._collect_early_builtin_variables(task, task_start_time)
+        early_builtin_vars = self._collect_early_builtin_variables(
+            task, task_start_time
+        )
 
         # Resolve working directory
         # Validate that working_dir doesn't contain {{ tt.working_dir }} (circular dependency)
         self._validate_no_working_dir_circular_ref(task.working_dir)
         working_dir_str = self._substitute_builtin(task.working_dir, early_builtin_vars)
-        working_dir_str = self._substitute_args(working_dir_str, regular_args, exported_args)
+        working_dir_str = self._substitute_args(
+            working_dir_str, regular_args, exported_args
+        )
         working_dir_str = self._substitute_env(working_dir_str)
         working_dir = self.recipe.project_root / working_dir_str
 
         # Collect all built-in variables (including tt.working_dir now that it's resolved)
-        builtin_vars = self._collect_builtin_variables(task, working_dir, task_start_time)
+        builtin_vars = self._collect_builtin_variables(
+            task, working_dir, task_start_time
+        )
 
         # Substitute built-in variables, arguments, and environment variables in command
         cmd = self._substitute_builtin(task.cmd, builtin_vars)
@@ -576,80 +646,65 @@ class Executor:
             env = self.recipe.get_environment(env_name)
 
         # Execute command
-        print(f"Running: {task.name}")
+        self.logger.log(LogLevel.INFO, f"Running: {task.name}")
 
         # Route to Docker execution or regular execution
         if env and env.dockerfile:
             # Docker execution path
-            self._run_task_in_docker(task, env, cmd, working_dir, exported_env_vars)
+            self._run_task_in_docker(
+                task,
+                env,
+                cmd,
+                working_dir,
+                process_runner,
+                exported_env_vars,
+            )
         else:
-            # Regular execution path
-            shell, shell_args, preamble = self._resolve_environment(task)
-
-            # Detect multi-line commands (ignore trailing newlines from YAML folded blocks)
-            if "\n" in cmd.rstrip():
-                self._run_multiline_command(cmd, working_dir, task.name, shell, preamble, exported_env_vars)
-            else:
-                self._run_single_line_command(cmd, working_dir, task.name, shell, shell_args, exported_env_vars)
+            # Regular execution path - use unified script-based execution
+            shell, preamble = self._resolve_environment(task)
+            self._run_command_as_script(
+                cmd,
+                working_dir,
+                task.name,
+                shell,
+                preamble,
+                process_runner,
+                exported_env_vars,
+            )
 
         # Update state
         self._update_state(task, args_dict)
 
-    def _run_single_line_command(
-        self, cmd: str, working_dir: Path, task_name: str, shell: str, shell_args: list[str],
-        exported_env_vars: dict[str, str] | None = None
+    def _run_command_as_script(
+        self,
+        cmd: str,
+        working_dir: Path,
+        task_name: str,
+        shell: str,
+        preamble: str,
+        process_runner: ProcessRunner,
+        exported_env_vars: dict[str, str] | None = None,
     ) -> None:
         """
-        Execute a single-line command via shell.
-
-        Args:
-        cmd: Command string
-        working_dir: Working directory
-        task_name: Task name (for error messages)
-        shell: Shell executable to use
-        shell_args: Arguments to pass to shell
-        exported_env_vars: Exported arguments to set as environment variables
-
-        Raises:
-        ExecutionError: If command execution fails
-        @athena: 46849e6a0bbb
-        """
-        # Prepare environment with exported args
-        env = self._prepare_env_with_exports(exported_env_vars)
-
-        try:
-            # Build command: shell + args + cmd
-            full_cmd = [shell] + shell_args + [cmd]
-            subprocess.run(
-                full_cmd,
-                cwd=working_dir,
-                check=True,
-                capture_output=False,
-                env=env,
-            )
-        except subprocess.CalledProcessError as e:
-            raise ExecutionError(
-                f"Task '{task_name}' failed with exit code {e.returncode}"
-            )
+        Execute a command via temporary script file (unified execution path).
 
-    def _run_multiline_command(
-        self, cmd: str, working_dir: Path, task_name: str, shell: str, preamble: str,
-        exported_env_vars: dict[str, str] | None = None
-    ) -> None:
-        """
-        Execute a multi-line command via temporary script file.
+        This method handles both single-line and multi-line commands by writing
+        them to a temporary script file and executing the script. This provides
+        consistent behavior and allows preamble to work with all commands.
 
         Args:
-        cmd: Multi-line command string
+        cmd: Command string (single-line or multi-line)
         working_dir: Working directory
         task_name: Task name (for error messages)
         shell: Shell to use for script execution
         preamble: Preamble text to prepend to script
+        process_runner: ProcessRunner instance to use for subprocess execution
         exported_env_vars: Exported arguments to set as environment variables
 
         Raises:
         ExecutionError: If command execution fails
-        @athena: 825892b6db05
+        @athena: TBD
+        @athena: 228cc00e7665
         """
         # Prepare environment with exported args
         env = self._prepare_env_with_exports(exported_env_vars)
@@ -689,13 +744,56 @@ class Executor:
 
             # Execute script file
             try:
-                subprocess.run(
-                    [script_path],
-                    cwd=working_dir,
-                    check=True,
-                    capture_output=False,
-                    env=env,
-                )
+                # Check if stdout/stderr support fileno() (real file descriptors)
+                # CliRunner uses StringIO which has fileno() method but raises when called
+                def supports_fileno(stream):
+                    """Check if a stream has a working fileno() method."""
+                    try:
+                        stream.fileno()
+                        return True
+                    except (AttributeError, OSError, io.UnsupportedOperation):
+                        return False
+
+                # Determine output targets based on task_output mode
+                # For "all" mode: show everything
+                # Future modes: use subprocess.DEVNULL for suppression
+                should_suppress = False  # Will be: self.task_output == "none", etc.
+
+                if should_suppress:
+                    stdout_target = subprocess.DEVNULL
+                    stderr_target = subprocess.DEVNULL
+                else:
+                    stdout_target = sys.stdout
+                    stderr_target = sys.stderr
+
+                # If streams support fileno, pass target streams directly (most efficient)
+                # Otherwise capture and manually write (CliRunner compatibility)
+                if not should_suppress and not (
+                    supports_fileno(sys.stdout) and supports_fileno(sys.stderr)
+                ):
+                    # CliRunner path: capture and write manually
+                    result = process_runner.run(
+                        [script_path],
+                        cwd=working_dir,
+                        check=True,
+                        capture_output=True,
+                        text=True,
+                        env=env,
+                    )
+                    if result.stdout:
+                        sys.stdout.write(result.stdout)
+                    if result.stderr:
+                        sys.stderr.write(result.stderr)
+                else:
+                    # Normal execution path: use target streams (including DEVNULL when suppressing)
+                    process_runner.run(
+                        [script_path],
+                        cwd=working_dir,
+                        check=True,
+                        stdout=stdout_target,
+                        stderr=stderr_target,
+                        env=env,
+                    )
             except subprocess.CalledProcessError as e:
                 raise ExecutionError(
                     f"Task '{task_name}' failed with exit code {e.returncode}"
@@ -707,7 +805,9 @@ class Executor:
             except OSError:
                 pass  # Ignore cleanup errors
 
-    def _substitute_builtin_in_environment(self, env: Environment, builtin_vars: dict[str, str]) -> Environment:
+    def _substitute_builtin_in_environment(
+        self, env: Environment, builtin_vars: dict[str, str]
+    ) -> Environment:
         """
         Substitute builtin and environment variables in environment fields.
 
@@ -720,34 +820,56 @@ class Executor:
 
         Raises:
         ValueError: If builtin variable or environment variable is not defined
-        @athena: 21e2ccd27dbb
+        @athena: eba6e3d62062
         """
         from dataclasses import replace
 
         # Substitute in volumes (builtin vars first, then env vars)
-        substituted_volumes = [
-            self._substitute_env(self._substitute_builtin(vol, builtin_vars)) for vol in env.volumes
-        ] if env.volumes else []
+        substituted_volumes = (
+            [
+                self._substitute_env(self._substitute_builtin(vol, builtin_vars))
+                for vol in env.volumes
+            ]
+            if env.volumes
+            else []
+        )
 
         # Substitute in env_vars values (builtin vars first, then env vars)
-        substituted_env_vars = {
-            key: self._substitute_env(self._substitute_builtin(value, builtin_vars))
-            for key, value in env.env_vars.items()
-        } if env.env_vars else {}
+        substituted_env_vars = (
+            {
+                key: self._substitute_env(self._substitute_builtin(value, builtin_vars))
+                for key, value in env.env_vars.items()
+            }
+            if env.env_vars
+            else {}
+        )
 
         # Substitute in ports (builtin vars first, then env vars)
-        substituted_ports = [
-            self._substitute_env(self._substitute_builtin(port, builtin_vars)) for port in env.ports
-        ] if env.ports else []
+        substituted_ports = (
+            [
+                self._substitute_env(self._substitute_builtin(port, builtin_vars))
+                for port in env.ports
+            ]
+            if env.ports
+            else []
+        )
 
         # Substitute in working_dir (builtin vars first, then env vars)
-        substituted_working_dir = self._substitute_env(self._substitute_builtin(env.working_dir, builtin_vars)) if env.working_dir else ""
+        substituted_working_dir = (
+            self._substitute_env(
+                self._substitute_builtin(env.working_dir, builtin_vars)
+            )
+            if env.working_dir
+            else ""
+        )
 
         # Substitute in build args (for Docker environments, args is a dict)
         # Apply builtin vars first, then env vars
         if isinstance(env.args, dict):
             substituted_args = {
-                key: self._substitute_env(self._substitute_builtin(str(value), builtin_vars))
+                key: self._substitute_env(
+                    self._substitute_builtin(str(value), builtin_vars)
+                )
                 for key, value in env.args.items()
             }
         else:
@@ -760,12 +882,17 @@ class Executor:
             env_vars=substituted_env_vars,
             ports=substituted_ports,
             working_dir=substituted_working_dir,
-            args=substituted_args
+            args=substituted_args,
         )
 
     def _run_task_in_docker(
-        self, task: Task, env: Any, cmd: str, working_dir: Path,
-        exported_env_vars: dict[str, str] | None = None
+        self,
+        task: Task,
+        env: Any,
+        cmd: str,
+        working_dir: Path,
+        process_runner: ProcessRunner,
+        exported_env_vars: dict[str, str] | None = None,
     ) -> None:
         """
         Execute task inside Docker container.
@@ -775,15 +902,19 @@ class Executor:
         env: Docker environment configuration
         cmd: Command to execute
         working_dir: Host working directory
+        process_runner: ProcessRunner instance to use for subprocess execution
         exported_env_vars: Exported arguments to set as environment variables
+        task_output: Control task subprocess output (all, out, err, on-err, none)
 
         Raises:
         ExecutionError: If Docker execution fails
-        @athena: fe972e4c97a3
+        @athena: 61725a57e304
         """
         # Get builtin variables for substitution in environment fields
         task_start_time = datetime.now(timezone.utc)
-        builtin_vars = self._collect_builtin_variables(task, working_dir, task_start_time)
+        builtin_vars = self._collect_builtin_variables(
+            task, working_dir, task_start_time
+        )
 
         # Substitute builtin variables in environment fields (volumes, env_vars, etc.)
         env = self._substitute_builtin_in_environment(env, builtin_vars)
@@ -807,6 +938,7 @@ class Executor:
 
         # Create modified environment with merged env vars using dataclass replace
         from dataclasses import replace
+
         modified_env = replace(env, env_vars=docker_env_vars)
 
         # Execute in container
@@ -816,11 +948,13 @@ class Executor:
                 cmd=cmd,
                 working_dir=working_dir,
                 container_working_dir=container_working_dir,
+                process_runner=process_runner,
             )
         except docker_module.DockerError as e:
             raise ExecutionError(str(e)) from e
 
-    def _validate_no_working_dir_circular_ref(self, text: str) -> None:
+    @staticmethod
+    def _validate_no_working_dir_circular_ref(text: str) -> None:
         """
         Validate that working_dir field does not contain {{ tt.working_dir }}.
 
@@ -831,20 +965,22 @@ class Executor:
 
         Raises:
         ExecutionError: If {{ tt.working_dir }} placeholder is found
-        @athena: 5dc6ee41d403
+        @athena: 82822f02716a
         """
         import re
+
         # Pattern to match {{ tt.working_dir }} specifically
-        pattern = re.compile(r'\{\{\s*tt\s*\.\s*working_dir\s*\}\}')
+        pattern = re.compile(r"\{\{\s*tt\s*\.\s*working_dir\s*}}")
 
         if pattern.search(text):
             raise ExecutionError(
-                f"Cannot use {{{{ tt.working_dir }}}} in the 'working_dir' field.\n\n"
-                f"This creates a circular dependency (working_dir cannot reference itself).\n"
-                f"Other built-in variables like {{{{ tt.task_name }}}} or {{{{ tt.timestamp }}}} are allowed."
+                "Cannot use {{ tt.working_dir }} in the 'working_dir' field.\n\n"
+                "This creates a circular dependency (working_dir cannot reference itself).\n"
+                "Other built-in variables like {{ tt.task_name }} or {{ tt.timestamp }} are allowed."
             )
 
-    def _substitute_builtin(self, text: str, builtin_vars: dict[str, str]) -> str:
+    @staticmethod
+    def _substitute_builtin(text: str, builtin_vars: dict[str, str]) -> str:
         """
         Substitute {{ tt.name }} placeholders in text.
 
@@ -859,12 +995,16 @@ class Executor:
 
         Raises:
         ValueError: If built-in variable is not defined
-        @athena: 463600a203f4
+        @athena: fe47afe87b52
         """
         from tasktree.substitution import substitute_builtin_variables
+
         return substitute_builtin_variables(text, builtin_vars)
 
-    def _substitute_args(self, cmd: str, args_dict: dict[str, Any], exported_args: set[str] | None = None) -> str:
+    @staticmethod
+    def _substitute_args(
+        cmd: str, args_dict: dict[str, Any], exported_args: set[str] | None = None
+    ) -> str:
         """
         Substitute {{ arg.name }} placeholders in command string.
 
@@ -881,12 +1021,14 @@ class Executor:
 
         Raises:
         ValueError: If an exported argument is used in template substitution
-        @athena: 4261a91c6a98
+        @athena: 9a931179f270
         """
         from tasktree.substitution import substitute_arguments
+
         return substitute_arguments(cmd, args_dict, exported_args)
 
-    def _substitute_env(self, text: str) -> str:
+    @staticmethod
+    def _substitute_env(text: str) -> str:
         """
         Substitute {{ env.NAME }} placeholders in text.
 
@@ -900,9 +1042,10 @@ class Executor:
 
         Raises:
         ValueError: If environment variable is not set
-        @athena: 63becab531cd
+        @athena: 1bbe24759451
         """
         from tasktree.substitution import substitute_environment
+
         return substitute_environment(text)
 
     def _get_all_inputs(self, task: Task) -> list[str]:
@@ -929,8 +1072,13 @@ class Executor:
         all_inputs.extend(implicit_inputs)
         return all_inputs
 
+    # TODO: Understand why task isn't used
     def _check_environment_changed(
-        self, task: Task, cached_state: TaskState, env_name: str
+        self,
+        task: Task,
+        cached_state: TaskState,
+        env_name: str,
+        process_runner: ProcessRunner,
     ) -> bool:
         """
         Check if environment definition has changed since last run.
@@ -942,10 +1090,11 @@ class Executor:
         task: Task to check
         cached_state: Cached state from previous run
         env_name: Effective environment name (from _get_effective_env_name)
+        process_runner: ProcessRunner instance for subprocess execution
 
         Returns:
         True if environment definition changed, False otherwise
-        @athena: 052561b75455
+        @athena: e206e104150a
         """
         # If using platform default (no environment), no definition to track
         if not env_name:
@@ -976,13 +1125,19 @@ class Executor:
 
         # For Docker environments, also check if image ID changed
         if env.dockerfile:
-            return self._check_docker_image_changed(env, cached_state, env_name)
+            return self._check_docker_image_changed(
+                env, cached_state, env_name, process_runner
+            )
 
         # Shell environment with unchanged hash
         return False
 
     def _check_docker_image_changed(
-        self, env: Environment, cached_state: TaskState, env_name: str
+        self,
+        env: Environment,
+        cached_state: TaskState,
+        env_name: str,
+        process_runner: ProcessRunner,
     ) -> bool:
         """
         Check if Docker image ID has changed.
@@ -994,15 +1149,18 @@ class Executor:
         env: Docker environment definition
         cached_state: Cached state from previous run
         env_name: Environment name
+        process_runner: ProcessRunner instance for subprocess execution
 
         Returns:
         True if image ID changed, False otherwise
-        @athena: 8af77cb1be44
+        @athena: bc954288e4ad
         """
         # Build/ensure image is built and get its ID
         try:
-            image_tag, current_image_id = self.docker_manager.ensure_image_built(env)
-        except Exception as e:
+            image_tag, current_image_id = self.docker_manager.ensure_image_built(
+                env, process_runner
+            )
+        except Exception:
             # If we can't build, treat as changed (will fail later with better error)
             return True
 
@@ -1095,7 +1253,9 @@ class Executor:
 
                         # Check if digests changed
                         if current_digests != cached_digests:
-                            changed_files.append(f"Docker base image digests in {dockerfile_name}")
+                            changed_files.append(
+                                f"Docker base image digests in {dockerfile_name}"
+                            )
                     except (OSError, IOError):
                         # Can't read Dockerfile - consider changed
                         changed_files.append(f"Dockerfile: {dockerfile_name}")
@@ -1115,7 +1275,8 @@ class Executor:
 
         return changed_files
 
-    def _expand_output_paths(self, task: Task) -> list[str]:
+    @staticmethod
+    def _expand_output_paths(task: Task) -> list[str]:
         """
         Extract all output paths from task outputs (both named and anonymous).
 
@@ -1124,7 +1285,7 @@ class Executor:
 
         Returns:
         List of output path patterns (glob patterns as strings)
-        @athena: 848a28564b14
+        @athena: 21da23ad5dcf
         """
         paths = []
         for output in task.outputs:
@@ -1193,25 +1354,48 @@ class Executor:
     def _update_state(self, task: Task, args_dict: dict[str, Any]) -> None:
         """
         Update state after task execution.
+        @athena: f4d3efdaac7c
+        """
+        cache_key = self._cache_key(task, args_dict)
+        input_state = self._input_files_to_modified_times(task)
 
-        Args:
-        task: Task that was executed
-        args_dict: Arguments used for execution
-        @athena: 1fcfdfcb9be9
+        env_name = self._get_effective_env_name(task)
+        if env_name:
+            env = self.recipe.get_environment(env_name)
+            if env:
+                input_state[f"_env_hash_{env_name}"] = hash_environment_definition(env)
+                if env.dockerfile:
+                    input_state |= self._docker_inputs_to_modified_times(env_name, env)
+
+        new_state = TaskState(last_run=time.time(), input_state=input_state)
+        self.state.set(cache_key, new_state)
+        self.state.save()
+
+    def _cache_key(self, task: Task, args_dict: dict[str, Any]) -> str:
+        """
+        @athena: d20ce4090741
         """
-        # Compute hashes (include effective environment and dependencies)
         effective_env = self._get_effective_env_name(task)
-        task_hash = hash_task(task.cmd, task.outputs, task.working_dir, task.args, effective_env, task.deps)
+        task_hash = hash_task(
+            task.cmd,
+            task.outputs,
+            task.working_dir,
+            task.args,
+            effective_env,
+            task.deps,
+        )
         args_hash = hash_args(args_dict) if args_dict else None
-        cache_key = make_cache_key(task_hash, args_hash)
+        return make_cache_key(task_hash, args_hash)
 
-        # Get all inputs and their current mtimes
-        all_inputs = self._get_all_inputs(task)
-        input_files = self._expand_globs(all_inputs, task.working_dir)
+    def _input_files_to_modified_times(self, task: Task) -> dict[str, float]:
+        """
+        @athena: 7e5ba779a41f
+        """
+        input_files = self._expand_globs(self._get_all_inputs(task), task.working_dir)
 
         input_state = {}
         for file_path in input_files:
-            # Skip Docker special markers (handled separately below)
+            # Skip Docker special markers (handled separately)
             if file_path.startswith("_docker_"):
                 continue
 
@@ -1219,59 +1403,47 @@ class Executor:
             if file_path_obj.exists():
                 input_state[file_path] = file_path_obj.stat().st_mtime
 
-        # Record Docker-specific inputs if task uses Docker environment
-        env_name = self._get_effective_env_name(task)
-        if env_name:
-            env = self.recipe.get_environment(env_name)
-            if env and env.dockerfile:
-                # Record Dockerfile mtime
-                dockerfile_path = self.recipe.project_root / env.dockerfile
-                if dockerfile_path.exists():
-                    input_state[env.dockerfile] = dockerfile_path.stat().st_mtime
-
-                # Record .dockerignore mtime if exists
-                context_path = self.recipe.project_root / env.context
-                dockerignore_path = context_path / ".dockerignore"
-                if dockerignore_path.exists():
-                    relative_dockerignore = str(
-                        dockerignore_path.relative_to(self.recipe.project_root)
-                    )
-                    input_state[relative_dockerignore] = dockerignore_path.stat().st_mtime
-
-                # Record context check timestamp
-                input_state[f"_context_{env.context}"] = time.time()
-
-                # Parse and record base image digests from Dockerfile
-                try:
-                    dockerfile_content = dockerfile_path.read_text()
-                    digests = docker_module.parse_base_image_digests(dockerfile_content)
-                    for digest in digests:
-                        # Store digest with Dockerfile's mtime
-                        input_state[f"_digest_{digest}"] = dockerfile_path.stat().st_mtime
-                except (OSError, IOError):
-                    # If we can't read Dockerfile, skip digest tracking
-                    pass
-
-            # Record environment definition hash for all environments (shell and Docker)
-            if env:
-                from tasktree.hasher import hash_environment_definition
+        return input_state
 
-                env_hash = hash_environment_definition(env)
-                input_state[f"_env_hash_{env_name}"] = env_hash
+    def _docker_inputs_to_modified_times(
+        self, env_name: str, env: Environment
+    ) -> dict[str, float]:
+        """
+        @athena: bfe53b0d56cd
+        """
+        input_state = dict()
+        # Record Dockerfile mtime
+        dockerfile_path = self.recipe.project_root / env.dockerfile
+        if dockerfile_path.exists():
+            input_state[env.dockerfile] = dockerfile_path.stat().st_mtime
+
+        # Record .dockerignore mtime if exists
+        context_path = self.recipe.project_root / env.context
+        dockerignore_path = context_path / ".dockerignore"
+        if dockerignore_path.exists():
+            relative_dockerignore = str(
+                dockerignore_path.relative_to(self.recipe.project_root)
+            )
+            input_state[relative_dockerignore] = dockerignore_path.stat().st_mtime
 
-                # For Docker environments, also store the image ID
-                if env.dockerfile:
-                    # Image was already built during check phase or task execution
-                    if env_name in self.docker_manager._built_images:
-                        image_tag, image_id = self.docker_manager._built_images[env_name]
-                        input_state[f"_docker_image_id_{env_name}"] = image_id
-
-        # Create new state
-        state = TaskState(
-            last_run=time.time(),
-            input_state=input_state,
-        )
+        # Record context check timestamp
+        input_state[f"_context_{env.context}"] = time.time()
 
-        # Save state
-        self.state.set(cache_key, state)
-        self.state.save()
+        # Parse and record base image digests from Dockerfile
+        try:
+            dockerfile_content = dockerfile_path.read_text()
+            digests = docker_module.parse_base_image_digests(dockerfile_content)
+            for digest in digests:
+                # Store digest with Dockerfile's mtime
+                input_state[f"_digest_{digest}"] = dockerfile_path.stat().st_mtime
+        except (OSError, IOError):
+            # If we can't read Dockerfile, skip digest tracking
+            pass
+
+        # For Docker environments, also store the image ID
+        # Image was already built during check phase or task execution
+        if env_name in self.docker_manager._built_images:
+            image_tag, image_id = self.docker_manager._built_images[env_name]
+            input_state[f"_docker_image_id_{env_name}"] = image_id
+
+        return input_state