tasktree 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl

tasktree/executor.py

@@ -13,6 +13,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import Any
 
+from tasktree import docker as docker_module
 from tasktree.graph import get_implicit_inputs, resolve_execution_order
 from tasktree.hasher import hash_args, hash_task, make_cache_key
 from tasktree.parser import Recipe, Task
@@ -26,7 +27,7 @@ class TaskStatus:
     task_name: str
     will_run: bool
     reason: str  # "fresh", "inputs_changed", "definition_changed",
-    # "never_run", "dependency_triggered", "no_outputs"
+    # "never_run", "no_outputs", "outputs_missing", "forced", "environment_changed"
     changed_files: list[str] = field(default_factory=list)
     last_run: datetime | None = None
 
@@ -49,6 +50,7 @@ class Executor:
         """
         self.recipe = recipe
         self.state = state_manager
+        self.docker_manager = docker_module.DockerManager(recipe.project_root)
 
     def _get_platform_default_environment(self) -> tuple[str, list[str]]:
         """Get default shell and args for current platform.
@@ -133,7 +135,6 @@ class Executor:
         self,
         task: Task,
         args_dict: dict[str, Any],
-        dep_statuses: dict[str, TaskStatus],
         force: bool = False,
     ) -> TaskStatus:
         """Check if a task needs to run.
@@ -141,16 +142,16 @@ class Executor:
         A task executes if ANY of these conditions are met:
         1. Force flag is set (--force)
         2. Task definition hash differs from cached state
-        3. Any explicit inputs have newer mtime than last_run
-        4. Any implicit inputs (from deps) have changed
-        5. No cached state exists for this task+args combination
-        6. Task has no inputs AND no outputs (always runs)
-        7. Different arguments than any cached execution
+        3. Environment definition has changed
+        4. Any explicit inputs have newer mtime than last_run
+        5. Any implicit inputs (from deps) have changed
+        6. No cached state exists for this task+args combination
+        7. Task has no inputs AND no outputs (always runs)
+        8. Different arguments than any cached execution
 
         Args:
             task: Task to check
             args_dict: Arguments for this task execution
-            dep_statuses: Status of dependencies
             force: If True, ignore freshness and force execution
 
         Returns:
@@ -179,21 +180,23 @@ class Executor:
                 reason="no_outputs",
             )
 
-        # Check if any dependency triggered
-        if any(status.will_run for status in dep_statuses.values()):
+        # Check cached state
+        cached_state = self.state.get(cache_key)
+        if cached_state is None:
             return TaskStatus(
                 task_name=task.name,
                 will_run=True,
-                reason="dependency_triggered",
+                reason="never_run",
             )
 
-        # Check cached state
-        cached_state = self.state.get(cache_key)
-        if cached_state is None:
+        # Check if environment definition has changed
+        env_changed = self._check_environment_changed(task, cached_state, effective_env)
+        if env_changed:
             return TaskStatus(
                 task_name=task.name,
                 will_run=True,
-                reason="never_run",
+                reason="environment_changed",
+                last_run=datetime.fromtimestamp(cached_state.last_run),
             )
 
         # Check if inputs have changed
@@ -262,23 +265,19 @@ class Executor:
             # Execute task and all dependencies
             execution_order = resolve_execution_order(self.recipe, task_name)
 
-        # Check status of all tasks
+        # Single phase: Check and execute incrementally
         statuses: dict[str, TaskStatus] = {}
         for name in execution_order:
             task = self.recipe.tasks[name]
 
-            # Get status of dependencies
-            dep_statuses = {dep: statuses[dep] for dep in task.deps if dep in statuses}
-
             # Determine task-specific args (only for target task)
             task_args = args_dict if name == task_name else {}
 
-            status = self.check_task_status(task, task_args, dep_statuses, force=force)
+            # Check if task needs to run (based on CURRENT filesystem state)
+            status = self.check_task_status(task, task_args, force=force)
             statuses[name] = status
 
-        # Execute tasks that need to run
-        for name in execution_order:
-            status = statuses[name]
+            # Execute immediately if needed
             if status.will_run:
                 # Warn if re-running due to missing outputs
                 if status.reason == "outputs_missing":
@@ -288,8 +287,6 @@ class Executor:
                         file=sys.stderr,
                     )
 
-                task = self.recipe.tasks[name]
-                task_args = args_dict if name == task_name else {}
                 self._run_task(task, task_args)
 
         return statuses
@@ -310,17 +307,28 @@ class Executor:
         # Determine working directory
         working_dir = self.recipe.project_root / task.working_dir
 
-        # Resolve environment for this task
-        shell, shell_args, preamble = self._resolve_environment(task)
+        # Check if task uses Docker environment
+        env_name = self._get_effective_env_name(task)
+        env = None
+        if env_name:
+            env = self.recipe.get_environment(env_name)
 
         # Execute command
         print(f"Running: {task.name}")
 
-        # Detect multi-line commands (ignore trailing newlines from YAML folded blocks)
-        if "\n" in cmd.rstrip():
-            self._run_multiline_command(cmd, working_dir, task.name, shell, preamble)
+        # Route to Docker execution or regular execution
+        if env and env.dockerfile:
+            # Docker execution path
+            self._run_task_in_docker(task, env, cmd, working_dir)
         else:
-            self._run_single_line_command(cmd, working_dir, task.name, shell, shell_args)
+            # Regular execution path
+            shell, shell_args, preamble = self._resolve_environment(task)
+
+            # Detect multi-line commands (ignore trailing newlines from YAML folded blocks)
+            if "\n" in cmd.rstrip():
+                self._run_multiline_command(cmd, working_dir, task.name, shell, preamble)
+            else:
+                self._run_single_line_command(cmd, working_dir, task.name, shell, shell_args)
 
         # Update state
         self._update_state(task, args_dict)
@@ -421,6 +429,36 @@ class Executor:
             except OSError:
                 pass  # Ignore cleanup errors
 
+    def _run_task_in_docker(
+        self, task: Task, env: Any, cmd: str, working_dir: Path
+    ) -> None:
+        """Execute task inside Docker container.
+
+        Args:
+            task: Task to execute
+            env: Docker environment configuration
+            cmd: Command to execute
+            working_dir: Host working directory
+
+        Raises:
+            ExecutionError: If Docker execution fails
+        """
+        # Resolve container working directory
+        container_working_dir = docker_module.resolve_container_working_dir(
+            env.working_dir, task.working_dir
+        )
+
+        # Execute in container
+        try:
+            self.docker_manager.run_in_container(
+                env=env,
+                cmd=cmd,
+                working_dir=working_dir,
+                container_working_dir=container_working_dir,
+            )
+        except docker_module.DockerError as e:
+            raise ExecutionError(str(e)) from e
+
     def _substitute_args(self, cmd: str, args_dict: dict[str, Any]) -> str:
         """Substitute arguments in command string.
 
@@ -451,11 +489,100 @@ class Executor:
         all_inputs.extend(implicit_inputs)
         return all_inputs
 
+    def _check_environment_changed(
+        self, task: Task, cached_state: TaskState, env_name: str
+    ) -> bool:
+        """Check if environment definition has changed since last run.
+
+        For shell environments: checks YAML definition hash
+        For Docker environments: checks YAML hash AND Docker image ID
+
+        Args:
+            task: Task to check
+            cached_state: Cached state from previous run
+            env_name: Effective environment name (from _get_effective_env_name)
+
+        Returns:
+            True if environment definition changed, False otherwise
+        """
+        # If using platform default (no environment), no definition to track
+        if not env_name:
+            return False
+
+        # Get environment definition
+        env = self.recipe.get_environment(env_name)
+        if env is None:
+            # Environment was deleted - treat as changed
+            return True
+
+        # Compute current environment hash (YAML definition)
+        from tasktree.hasher import hash_environment_definition
+
+        current_env_hash = hash_environment_definition(env)
+
+        # Get cached environment hash
+        marker_key = f"_env_hash_{env_name}"
+        cached_env_hash = cached_state.input_state.get(marker_key)
+
+        # If no cached hash (old state file), treat as changed to establish baseline
+        if cached_env_hash is None:
+            return True
+
+        # Check if YAML definition changed
+        if current_env_hash != cached_env_hash:
+            return True  # YAML changed, no need to check image
+
+        # For Docker environments, also check if image ID changed
+        if env.dockerfile:
+            return self._check_docker_image_changed(env, cached_state, env_name)
+
+        # Shell environment with unchanged hash
+        return False
+
+    def _check_docker_image_changed(
+        self, env: Environment, cached_state: TaskState, env_name: str
+    ) -> bool:
+        """Check if Docker image ID has changed.
+
+        Builds the image and compares the resulting image ID with the cached ID.
+        This detects changes from unpinned base images, network-dependent builds, etc.
+
+        Args:
+            env: Docker environment definition
+            cached_state: Cached state from previous run
+            env_name: Environment name
+
+        Returns:
+            True if image ID changed, False otherwise
+        """
+        # Build/ensure image is built and get its ID
+        try:
+            image_tag, current_image_id = self.docker_manager.ensure_image_built(env)
+        except Exception as e:
+            # If we can't build, treat as changed (will fail later with better error)
+            return True
+
+        # Get cached image ID
+        image_id_key = f"_docker_image_id_{env_name}"
+        cached_image_id = cached_state.input_state.get(image_id_key)
+
+        # If no cached ID (first run or old state), treat as changed
+        if cached_image_id is None:
+            return True
+
+        # Compare image IDs
+        return current_image_id != cached_image_id
+
     def _check_inputs_changed(
         self, task: Task, cached_state: TaskState, all_inputs: list[str]
     ) -> list[str]:
         """Check if any input files have changed since last run.
 
+        Handles both regular file inputs and Docker-specific inputs:
+        - Regular files: checked via mtime
+        - Docker context: checked via directory walk with early exit
+        - Dockerfile digests: checked via parsing and comparison
+
         Args:
             task: Task to check
             cached_state: Cached state from previous run
@@ -469,7 +596,66 @@ class Executor:
         # Expand glob patterns
         input_files = self._expand_globs(all_inputs, task.working_dir)
 
+        # Check if task uses Docker environment
+        env_name = self._get_effective_env_name(task)
+        docker_env = None
+        if env_name:
+            docker_env = self.recipe.get_environment(env_name)
+            if docker_env and not docker_env.dockerfile:
+                docker_env = None  # Not a Docker environment
+
         for file_path in input_files:
+            # Handle Docker context directory check
+            if file_path.startswith("_docker_context_"):
+                if docker_env:
+                    context_name = file_path.replace("_docker_context_", "")
+                    context_path = self.recipe.project_root / context_name
+                    dockerignore_path = context_path / ".dockerignore"
+
+                    # Get last context check time
+                    cached_context_time = cached_state.input_state.get(
+                        f"_context_{context_name}"
+                    )
+                    if cached_context_time is None:
+                        # Never checked before - consider changed
+                        changed_files.append(f"Docker context: {context_name}")
+                        continue
+
+                    # Check if context changed (with early exit optimization)
+                    if docker_module.context_changed_since(
+                        context_path, dockerignore_path, cached_context_time
+                    ):
+                        changed_files.append(f"Docker context: {context_name}")
+                continue
+
+            # Handle Docker Dockerfile digest check
+            if file_path.startswith("_docker_dockerfile_"):
+                if docker_env:
+                    dockerfile_name = file_path.replace("_docker_dockerfile_", "")
+                    dockerfile_path = self.recipe.project_root / dockerfile_name
+
+                    try:
+                        dockerfile_content = dockerfile_path.read_text()
+                        current_digests = set(
+                            docker_module.parse_base_image_digests(dockerfile_content)
+                        )
+
+                        # Get cached digests
+                        cached_digests = set()
+                        for key in cached_state.input_state:
+                            if key.startswith("_digest_"):
+                                digest = key.replace("_digest_", "")
+                                cached_digests.add(digest)
+
+                        # Check if digests changed
+                        if current_digests != cached_digests:
+                            changed_files.append(f"Docker base image digests in {dockerfile_name}")
+                    except (OSError, IOError):
+                        # Can't read Dockerfile - consider changed
+                        changed_files.append(f"Dockerfile: {dockerfile_name}")
+                continue
+
+            # Regular file check
             file_path_obj = self.recipe.project_root / task.working_dir / file_path
             if not file_path_obj.exists():
                 continue
@@ -549,10 +735,61 @@ class Executor:
 
         input_state = {}
         for file_path in input_files:
+            # Skip Docker special markers (handled separately below)
+            if file_path.startswith("_docker_"):
+                continue
+
             file_path_obj = self.recipe.project_root / task.working_dir / file_path
             if file_path_obj.exists():
                 input_state[file_path] = file_path_obj.stat().st_mtime
 
+        # Record Docker-specific inputs if task uses Docker environment
+        env_name = self._get_effective_env_name(task)
+        if env_name:
+            env = self.recipe.get_environment(env_name)
+            if env and env.dockerfile:
+                # Record Dockerfile mtime
+                dockerfile_path = self.recipe.project_root / env.dockerfile
+                if dockerfile_path.exists():
+                    input_state[env.dockerfile] = dockerfile_path.stat().st_mtime
+
+                # Record .dockerignore mtime if exists
+                context_path = self.recipe.project_root / env.context
+                dockerignore_path = context_path / ".dockerignore"
+                if dockerignore_path.exists():
+                    relative_dockerignore = str(
+                        dockerignore_path.relative_to(self.recipe.project_root)
+                    )
+                    input_state[relative_dockerignore] = dockerignore_path.stat().st_mtime
+
+                # Record context check timestamp
+                input_state[f"_context_{env.context}"] = time.time()
+
+                # Parse and record base image digests from Dockerfile
+                try:
+                    dockerfile_content = dockerfile_path.read_text()
+                    digests = docker_module.parse_base_image_digests(dockerfile_content)
+                    for digest in digests:
+                        # Store digest with Dockerfile's mtime
+                        input_state[f"_digest_{digest}"] = dockerfile_path.stat().st_mtime
+                except (OSError, IOError):
+                    # If we can't read Dockerfile, skip digest tracking
+                    pass
+
+            # Record environment definition hash for all environments (shell and Docker)
+            if env:
+                from tasktree.hasher import hash_environment_definition
+
+                env_hash = hash_environment_definition(env)
+                input_state[f"_env_hash_{env_name}"] = env_hash
+
+                # For Docker environments, also store the image ID
+                if env.dockerfile:
+                    # Image was already built during check phase or task execution
+                    if env_name in self.docker_manager._built_images:
+                        image_tag, image_id = self.docker_manager._built_images[env_name]
+                        input_state[f"_docker_image_id_{env_name}"] = image_id
+
         # Create new state
         state = TaskState(
             last_run=time.time(),

tasktree/graph.py

@@ -1,6 +1,7 @@
 """Dependency resolution using topological sorting."""
 
 from graphlib import TopologicalSorter
+from pathlib import Path
 
 from tasktree.parser import Recipe, Task
 
@@ -71,16 +72,21 @@ def get_implicit_inputs(recipe: Recipe, task: Task) -> list[str]:
     Tasks automatically inherit inputs from dependencies:
     1. All outputs from dependency tasks become implicit inputs
     2. All inputs from dependency tasks that don't declare outputs are inherited
+    3. If task uses a Docker environment, Docker artifacts become implicit inputs:
+       - Dockerfile
+       - .dockerignore (if present)
+       - Special markers for context directory and base image digests
 
     Args:
         recipe: Parsed recipe containing all tasks
         task: Task to get implicit inputs for
 
     Returns:
-        List of glob patterns for implicit inputs
+        List of glob patterns for implicit inputs, including Docker-specific markers
     """
     implicit_inputs = []
 
+    # Inherit from dependencies
     for dep_name in task.deps:
         dep_task = recipe.tasks.get(dep_name)
         if dep_task is None:
@@ -93,6 +99,29 @@ def get_implicit_inputs(recipe: Recipe, task: Task) -> list[str]:
         elif dep_task.inputs:
             implicit_inputs.extend(dep_task.inputs)
 
+    # Add Docker-specific implicit inputs if task uses Docker environment
+    env_name = task.env or recipe.default_env
+    if env_name:
+        env = recipe.get_environment(env_name)
+        if env and env.dockerfile:
+            # Add Dockerfile as input
+            implicit_inputs.append(env.dockerfile)
+
+            # Add .dockerignore if it exists in context directory
+            context_path = recipe.project_root / env.context
+            dockerignore_path = context_path / ".dockerignore"
+            if dockerignore_path.exists():
+                relative_dockerignore = str(
+                    dockerignore_path.relative_to(recipe.project_root)
+                )
+                implicit_inputs.append(relative_dockerignore)
+
+            # Add special markers for context directory and digest tracking
+            # These are tracked differently in state management (not file paths)
+            # The executor will handle these specially
+            implicit_inputs.append(f"_docker_context_{env.context}")
+            implicit_inputs.append(f"_docker_dockerfile_{env.dockerfile}")
+
     return implicit_inputs
 
 

tasktree/hasher.py

@@ -21,6 +21,33 @@ def hash_args(args_dict: dict[str, Any]) -> str:
     return hashlib.sha256(serialized.encode()).hexdigest()[:8]
 
 
+def hash_environment_definition(env) -> str:
+    """Hash environment definition fields that affect task execution.
+
+    Args:
+        env: Environment to hash
+
+    Returns:
+        16-character hash of environment definition
+    """
+    # Import inside function to avoid circular dependency
+    from tasktree.parser import Environment
+
+    data = {
+        "shell": env.shell,
+        "args": sorted(env.args),  # Sort for determinism
+        "preamble": env.preamble,
+        "dockerfile": env.dockerfile,
+        "context": env.context,
+        "volumes": sorted(env.volumes),
+        "ports": sorted(env.ports),
+        "env_vars": dict(sorted(env.env_vars.items())),
+        "working_dir": env.working_dir,
+    }
+    serialized = json.dumps(data, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(serialized.encode()).hexdigest()[:16]
+
+
 def make_cache_key(task_hash: str, args_hash: Optional[str] = None) -> str:
     if args_hash:
         return f"{task_hash}__{args_hash}"