tasktree 0.0.7__py3-none-any.whl → 0.0.8__py3-none-any.whl

tasktree/cli.py

@@ -375,8 +375,8 @@ def _parse_task_args(arg_specs: list[str], arg_values: list[str]) -> dict[str, A
 
     parsed_specs = []
     for spec in arg_specs:
-        name, arg_type, default = parse_arg_spec(spec)
-        parsed_specs.append((name, arg_type, default))
+        parsed = parse_arg_spec(spec)
+        parsed_specs.append(parsed)
 
     args_dict = {}
     positional_index = 0
@@ -386,41 +386,49 @@ def _parse_task_args(arg_specs: list[str], arg_values: list[str]) -> dict[str, A
         if "=" in value_str:
             arg_name, arg_value = value_str.split("=", 1)
             # Find the spec for this argument
-            spec = next((s for s in parsed_specs if s[0] == arg_name), None)
+            spec = next((s for s in parsed_specs if s.name == arg_name), None)
             if spec is None:
                 console.print(f"[red]Unknown argument: {arg_name}[/red]")
                 raise typer.Exit(1)
-            name, arg_type, default = spec
         else:
             # Positional argument
             if positional_index >= len(parsed_specs):
                 console.print(f"[red]Too many arguments[/red]")
                 raise typer.Exit(1)
-            name, arg_type, default = parsed_specs[positional_index]
+            spec = parsed_specs[positional_index]
             arg_value = value_str
             positional_index += 1
 
-        # Convert value to appropriate type
+        # Convert value to appropriate type (exported args are always strings)
         try:
-            click_type = get_click_type(arg_type)
+            click_type = get_click_type(spec.arg_type, min_val=spec.min_val, max_val=spec.max_val)
             converted_value = click_type.convert(arg_value, None, None)
-            args_dict[name] = converted_value
+
+            # Validate choices after type conversion
+            if spec.choices is not None and converted_value not in spec.choices:
+                console.print(f"[red]Invalid value for {spec.name}: {converted_value!r}[/red]")
+                console.print(f"Valid choices: {', '.join(repr(c) for c in spec.choices)}")
+                raise typer.Exit(1)
+
+            args_dict[spec.name] = converted_value
+        except typer.Exit:
+            raise  # Re-raise typer.Exit without wrapping
         except Exception as e:
-            console.print(f"[red]Invalid value for {name}: {e}[/red]")
+            console.print(f"[red]Invalid value for {spec.name}: {e}[/red]")
             raise typer.Exit(1)
 
     # Fill in defaults for missing arguments
-    for name, arg_type, default in parsed_specs:
-        if name not in args_dict:
-            if default is not None:
+    for spec in parsed_specs:
+        if spec.name not in args_dict:
+            if spec.default is not None:
                 try:
-                    click_type = get_click_type(arg_type)
-                    args_dict[name] = click_type.convert(default, None, None)
+                    click_type = get_click_type(spec.arg_type, min_val=spec.min_val, max_val=spec.max_val)
+                    args_dict[spec.name] = click_type.convert(spec.default, None, None)
                 except Exception as e:
-                    console.print(f"[red]Invalid default value for {name}: {e}[/red]")
+                    console.print(f"[red]Invalid default value for {spec.name}: {e}[/red]")
                     raise typer.Exit(1)
             else:
-                console.print(f"[red]Missing required argument: {name}[/red]")
+                console.print(f"[red]Missing required argument: {spec.name}[/red]")
                 raise typer.Exit(1)
 
     return args_dict

tasktree/docker.py

@@ -6,6 +6,7 @@ Provides Docker image building and container execution capabilities.
 from __future__ import annotations
 
 import os
+import platform
 import re
 import subprocess
 import time
@@ -39,6 +40,22 @@ class DockerManager:
         self._project_root = project_root
         self._built_images: dict[str, tuple[str, str]] = {}  # env_name -> (image_tag, image_id) cache
 
+    def _should_add_user_flag(self) -> bool:
+        """Check if --user flag should be added to docker run.
+
+        Returns False on Windows (where Docker Desktop handles UID mapping automatically).
+        Returns True on Linux/macOS where os.getuid() and os.getgid() are available.
+
+        Returns:
+            True if --user flag should be added, False otherwise
+        """
+        # Skip on Windows - Docker Desktop handles UID mapping differently
+        if platform.system() == "Windows":
+            return False
+
+        # Check if os.getuid() and os.getgid() are available (Linux/macOS)
+        return hasattr(os, "getuid") and hasattr(os, "getgid")
+
     def ensure_image_built(self, env: Environment) -> tuple[str, str]:
         """Build Docker image if not already built this invocation.
 
@@ -127,6 +144,14 @@ class DockerManager:
         # Build docker run command
         docker_cmd = ["docker", "run", "--rm"]
 
+        # Add user mapping (run as current host user) unless explicitly disabled or on Windows
+        if not env.run_as_root and self._should_add_user_flag():
+            uid = os.getuid()
+            gid = os.getgid()
+            docker_cmd.extend(["--user", f"{uid}:{gid}"])
+
+        docker_cmd.extend(env.extra_args)
+
         # Add volume mounts
         for volume in env.volumes:
             # Resolve volume paths

tasktree/executor.py

@@ -9,14 +9,14 @@ import subprocess
 import tempfile
 import time
 from dataclasses import dataclass, field
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
 from tasktree import docker as docker_module
 from tasktree.graph import get_implicit_inputs, resolve_execution_order
 from tasktree.hasher import hash_args, hash_task, make_cache_key
-from tasktree.parser import Recipe, Task
+from tasktree.parser import Recipe, Task, Environment
 from tasktree.state import StateManager, TaskState
 
 
@@ -41,6 +41,18 @@ class ExecutionError(Exception):
 class Executor:
     """Executes tasks with incremental execution logic."""
 
+    # Protected environment variables that cannot be overridden by exported args
+    PROTECTED_ENV_VARS = {
+        'PATH',
+        'LD_LIBRARY_PATH',
+        'LD_PRELOAD',
+        'PYTHONPATH',
+        'HOME',
+        'SHELL',
+        'USER',
+        'LOGNAME',
+    }
+
     def __init__(self, recipe: Recipe, state_manager: StateManager):
         """Initialize executor.
 
@@ -52,6 +64,106 @@ class Executor:
         self.state = state_manager
         self.docker_manager = docker_module.DockerManager(recipe.project_root)
 
+    def _collect_early_builtin_variables(self, task: Task, timestamp: datetime) -> dict[str, str]:
+        """Collect built-in variables that don't depend on working_dir.
+
+        These variables can be used in the working_dir field itself.
+
+        Args:
+            task: Task being executed
+            timestamp: Timestamp when task started execution
+
+        Returns:
+            Dictionary mapping built-in variable names to their string values
+
+        Raises:
+            ExecutionError: If any built-in variable fails to resolve
+        """
+        import os
+
+        builtin_vars = {}
+
+        # {{ tt.project_root }} - Absolute path to project root
+        builtin_vars['project_root'] = str(self.recipe.project_root.resolve())
+
+        # {{ tt.recipe_dir }} - Absolute path to directory containing the recipe file
+        builtin_vars['recipe_dir'] = str(self.recipe.recipe_path.parent.resolve())
+
+        # {{ tt.task_name }} - Name of currently executing task
+        builtin_vars['task_name'] = task.name
+
+        # {{ tt.timestamp }} - ISO8601 timestamp when task started execution
+        builtin_vars['timestamp'] = timestamp.strftime('%Y-%m-%dT%H:%M:%SZ')
+
+        # {{ tt.timestamp_unix }} - Unix epoch timestamp when task started
+        builtin_vars['timestamp_unix'] = str(int(timestamp.timestamp()))
+
+        # {{ tt.user_home }} - Current user's home directory (cross-platform)
+        try:
+            user_home = Path.home()
+            builtin_vars['user_home'] = str(user_home)
+        except Exception as e:
+            raise ExecutionError(
+                f"Failed to get user home directory for {{ tt.user_home }}: {e}"
+            )
+
+        # {{ tt.user_name }} - Current username (with fallback)
+        try:
+            user_name = os.getlogin()
+        except OSError:
+            # Fallback to environment variables if os.getlogin() fails
+            user_name = os.environ.get('USER') or os.environ.get('USERNAME') or 'unknown'
+        builtin_vars['user_name'] = user_name
+
+        return builtin_vars
+
+    def _collect_builtin_variables(self, task: Task, working_dir: Path, timestamp: datetime) -> dict[str, str]:
+        """Collect built-in variables for task execution.
+
+        Args:
+            task: Task being executed
+            working_dir: Resolved working directory for the task
+            timestamp: Timestamp when task started execution
+
+        Returns:
+            Dictionary mapping built-in variable names to their string values
+
+        Raises:
+            ExecutionError: If any built-in variable fails to resolve
+        """
+        # Get early builtin vars (those that don't depend on working_dir)
+        builtin_vars = self._collect_early_builtin_variables(task, timestamp)
+
+        # {{ tt.working_dir }} - Absolute path to task's effective working directory
+        # This is added after working_dir is resolved to avoid circular dependency
+        builtin_vars['working_dir'] = str(working_dir.resolve())
+
+        return builtin_vars
+
+    def _prepare_env_with_exports(self, exported_env_vars: dict[str, str] | None = None) -> dict[str, str]:
+        """Prepare environment with exported arguments.
+
+        Args:
+            exported_env_vars: Exported arguments to set as environment variables
+
+        Returns:
+            Environment dict with exported args merged
+
+        Raises:
+            ValueError: If an exported arg attempts to override a protected environment variable
+        """
+        env = os.environ.copy()
+        if exported_env_vars:
+            # Check for protected environment variable overrides
+            for key in exported_env_vars:
+                if key in self.PROTECTED_ENV_VARS:
+                    raise ValueError(
+                        f"Cannot override protected environment variable: {key}\n"
+                        f"Protected variables are: {', '.join(sorted(self.PROTECTED_ENV_VARS))}"
+                    )
+            env.update(exported_env_vars)
+        return env
+
     def _get_platform_default_environment(self) -> tuple[str, list[str]]:
         """Get default shell and args for current platform.
 
@@ -301,11 +413,47 @@ class Executor:
         Raises:
             ExecutionError: If task execution fails
         """
-        # Substitute arguments in command
-        cmd = self._substitute_args(task.cmd, args_dict)
+        # Capture timestamp at task start for consistency (in UTC)
+        task_start_time = datetime.now(timezone.utc)
+
+        # Parse task arguments to identify exported args
+        # Note: args_dict already has defaults applied by CLI (cli.py:413-424)
+        from tasktree.parser import parse_arg_spec
+        exported_args = set()
+        regular_args = {}
+        exported_env_vars = {}
+
+        for arg_spec in task.args:
+            parsed = parse_arg_spec(arg_spec)
+            if parsed.is_exported:
+                exported_args.add(parsed.name)
+                # Get value and convert to string for environment variable
+                # Value should always be in args_dict (CLI applies defaults)
+                if parsed.name in args_dict:
+                    exported_env_vars[parsed.name] = str(args_dict[parsed.name])
+            else:
+                if parsed.name in args_dict:
+                    regular_args[parsed.name] = args_dict[parsed.name]
+
+        # Collect early built-in variables (those that don't depend on working_dir)
+        # These can be used in the working_dir field itself
+        early_builtin_vars = self._collect_early_builtin_variables(task, task_start_time)
+
+        # Resolve working directory
+        # Validate that working_dir doesn't contain {{ tt.working_dir }} (circular dependency)
+        self._validate_no_working_dir_circular_ref(task.working_dir)
+        working_dir_str = self._substitute_builtin(task.working_dir, early_builtin_vars)
+        working_dir_str = self._substitute_args(working_dir_str, regular_args, exported_args)
+        working_dir_str = self._substitute_env(working_dir_str)
+        working_dir = self.recipe.project_root / working_dir_str
 
-        # Determine working directory
-        working_dir = self.recipe.project_root / task.working_dir
+        # Collect all built-in variables (including tt.working_dir now that it's resolved)
+        builtin_vars = self._collect_builtin_variables(task, working_dir, task_start_time)
+
+        # Substitute built-in variables, arguments, and environment variables in command
+        cmd = self._substitute_builtin(task.cmd, builtin_vars)
+        cmd = self._substitute_args(cmd, regular_args, exported_args)
+        cmd = self._substitute_env(cmd)
 
         # Check if task uses Docker environment
         env_name = self._get_effective_env_name(task)
@@ -319,22 +467,23 @@ class Executor:
         # Route to Docker execution or regular execution
         if env and env.dockerfile:
             # Docker execution path
-            self._run_task_in_docker(task, env, cmd, working_dir)
+            self._run_task_in_docker(task, env, cmd, working_dir, exported_env_vars)
         else:
             # Regular execution path
             shell, shell_args, preamble = self._resolve_environment(task)
 
             # Detect multi-line commands (ignore trailing newlines from YAML folded blocks)
             if "\n" in cmd.rstrip():
-                self._run_multiline_command(cmd, working_dir, task.name, shell, preamble)
+                self._run_multiline_command(cmd, working_dir, task.name, shell, preamble, exported_env_vars)
             else:
-                self._run_single_line_command(cmd, working_dir, task.name, shell, shell_args)
+                self._run_single_line_command(cmd, working_dir, task.name, shell, shell_args, exported_env_vars)
 
         # Update state
         self._update_state(task, args_dict)
 
     def _run_single_line_command(
-        self, cmd: str, working_dir: Path, task_name: str, shell: str, shell_args: list[str]
+        self, cmd: str, working_dir: Path, task_name: str, shell: str, shell_args: list[str],
+        exported_env_vars: dict[str, str] | None = None
     ) -> None:
         """Execute a single-line command via shell.
 
@@ -344,10 +493,14 @@ class Executor:
             task_name: Task name (for error messages)
             shell: Shell executable to use
             shell_args: Arguments to pass to shell
+            exported_env_vars: Exported arguments to set as environment variables
 
         Raises:
             ExecutionError: If command execution fails
         """
+        # Prepare environment with exported args
+        env = self._prepare_env_with_exports(exported_env_vars)
+
         try:
             # Build command: shell + args + cmd
             full_cmd = [shell] + shell_args + [cmd]
@@ -356,6 +509,7 @@ class Executor:
                 cwd=working_dir,
                 check=True,
                 capture_output=False,
+                env=env,
             )
         except subprocess.CalledProcessError as e:
             raise ExecutionError(
@@ -363,7 +517,8 @@ class Executor:
             )
 
     def _run_multiline_command(
-        self, cmd: str, working_dir: Path, task_name: str, shell: str, preamble: str
+        self, cmd: str, working_dir: Path, task_name: str, shell: str, preamble: str,
+        exported_env_vars: dict[str, str] | None = None
     ) -> None:
         """Execute a multi-line command via temporary script file.
 
@@ -373,10 +528,14 @@ class Executor:
             task_name: Task name (for error messages)
             shell: Shell to use for script execution
             preamble: Preamble text to prepend to script
+            exported_env_vars: Exported arguments to set as environment variables
 
         Raises:
             ExecutionError: If command execution fails
         """
+        # Prepare environment with exported args
+        env = self._prepare_env_with_exports(exported_env_vars)
+
         # Determine file extension based on platform
         is_windows = platform.system() == "Windows"
         script_ext = ".bat" if is_windows else ".sh"
@@ -417,6 +576,7 @@ class Executor:
                     cwd=working_dir,
                     check=True,
                     capture_output=False,
+                    env=env,
                 )
             except subprocess.CalledProcessError as e:
                 raise ExecutionError(
@@ -430,7 +590,8 @@ class Executor:
                 pass  # Ignore cleanup errors
 
     def _run_task_in_docker(
-        self, task: Task, env: Any, cmd: str, working_dir: Path
+        self, task: Task, env: Any, cmd: str, working_dir: Path,
+        exported_env_vars: dict[str, str] | None = None
     ) -> None:
         """Execute task inside Docker container.
 
@@ -439,6 +600,7 @@ class Executor:
             env: Docker environment configuration
             cmd: Command to execute
             working_dir: Host working directory
+            exported_env_vars: Exported arguments to set as environment variables
 
         Raises:
             ExecutionError: If Docker execution fails
@@ -448,10 +610,26 @@ class Executor:
             env.working_dir, task.working_dir
         )
 
+        # Validate and merge exported args with env vars (exported args take precedence)
+        docker_env_vars = env.env_vars.copy() if env.env_vars else {}
+        if exported_env_vars:
+            # Check for protected environment variable overrides
+            for key in exported_env_vars:
+                if key in self.PROTECTED_ENV_VARS:
+                    raise ValueError(
+                        f"Cannot override protected environment variable: {key}\n"
+                        f"Protected variables are: {', '.join(sorted(self.PROTECTED_ENV_VARS))}"
+                    )
+            docker_env_vars.update(exported_env_vars)
+
+        # Create modified environment with merged env vars using dataclass replace
+        from dataclasses import replace
+        modified_env = replace(env, env_vars=docker_env_vars)
+
         # Execute in container
         try:
             self.docker_manager.run_in_container(
-                env=env,
+                env=modified_env,
                 cmd=cmd,
                 working_dir=working_dir,
                 container_working_dir=container_working_dir,
@@ -459,21 +637,82 @@ class Executor:
         except docker_module.DockerError as e:
             raise ExecutionError(str(e)) from e
 
-    def _substitute_args(self, cmd: str, args_dict: dict[str, Any]) -> str:
-        """Substitute arguments in command string.
+    def _validate_no_working_dir_circular_ref(self, text: str) -> None:
+        """Validate that working_dir field does not contain {{ tt.working_dir }}.
+
+        Using {{ tt.working_dir }} in the working_dir field creates a circular dependency.
+
+        Args:
+            text: The working_dir field value to validate
+
+        Raises:
+            ExecutionError: If {{ tt.working_dir }} placeholder is found
+        """
+        import re
+        # Pattern to match {{ tt.working_dir }} specifically
+        pattern = re.compile(r'\{\{\s*tt\s*\.\s*working_dir\s*\}\}')
+
+        if pattern.search(text):
+            raise ExecutionError(
+                f"Cannot use {{{{ tt.working_dir }}}} in the 'working_dir' field.\n\n"
+                f"This creates a circular dependency (working_dir cannot reference itself).\n"
+                f"Other built-in variables like {{{{ tt.task_name }}}} or {{{{ tt.timestamp }}}} are allowed."
+            )
+
+    def _substitute_builtin(self, text: str, builtin_vars: dict[str, str]) -> str:
+        """Substitute {{ tt.name }} placeholders in text.
+
+        Built-in variables are resolved at execution time.
 
         Args:
-            cmd: Command template with {{arg}} placeholders
-            args_dict: Arguments to substitute
+            text: Text with {{ tt.name }} placeholders
+            builtin_vars: Built-in variable values
+
+        Returns:
+            Text with built-in variables substituted
+
+        Raises:
+            ValueError: If built-in variable is not defined
+        """
+        from tasktree.substitution import substitute_builtin_variables
+        return substitute_builtin_variables(text, builtin_vars)
+
+    def _substitute_args(self, cmd: str, args_dict: dict[str, Any], exported_args: set[str] | None = None) -> str:
+        """Substitute {{ arg.name }} placeholders in command string.
+
+        Variables are already substituted at parse time by the parser.
+        This only handles runtime argument substitution.
+
+        Args:
+            cmd: Command with {{ arg.name }} placeholders
+            args_dict: Argument values to substitute (only regular args)
+            exported_args: Set of argument names that are exported (not available for substitution)
 
         Returns:
             Command with arguments substituted
+
+        Raises:
+            ValueError: If an exported argument is used in template substitution
+        """
+        from tasktree.substitution import substitute_arguments
+        return substitute_arguments(cmd, args_dict, exported_args)
+
+    def _substitute_env(self, text: str) -> str:
+        """Substitute {{ env.NAME }} placeholders in text.
+
+        Environment variables are resolved at execution time from os.environ.
+
+        Args:
+            text: Text with {{ env.NAME }} placeholders
+
+        Returns:
+            Text with environment variables substituted
+
+        Raises:
+            ValueError: If environment variable is not set
         """
-        result = cmd
-        for key, value in args_dict.items():
-            placeholder = f"{{{{{key}}}}}"
-            result = result.replace(placeholder, str(value))
-        return result
+        from tasktree.substitution import substitute_environment
+        return substitute_environment(text)
 
     def _get_all_inputs(self, task: Task) -> list[str]:
         """Get all inputs for a task (explicit + implicit from dependencies).

tasktree/hasher.py

@@ -3,12 +3,46 @@ import json
 from typing import Any, Optional
 
 
-def hash_task(cmd: str, outputs: list[str], working_dir: str, args: list[str], env: str = "") -> str:
+def _arg_sort_key(arg: str | dict[str, Any]) -> str:
+    """Extract the sort key from an arg for deterministic hashing.
+
+    Args:
+        arg: Either a string arg or dict arg specification
+
+    Returns:
+        The argument name to use as a sort key
+    """
+    if isinstance(arg, dict):
+        # Dict args have exactly one key - the argument name
+        # This is validated by parse_arg_spec in parser.py
+        return next(iter(arg.keys()))
+    return arg
+
+
+def _normalize_choices_lists(args: list[str | dict[str, Any]]) ->  list[str | dict[str, Any]]:
+    normalized_args = []
+    for arg in args:
+        if isinstance(arg, dict):
+            # Deep copy and sort choices if present
+            normalized = {}
+            for key, value in arg.items():
+                if isinstance(value, dict) and 'choices' in value:
+                    normalized[key] = {**value, 'choices': sorted(value['choices'], key=str)}
+                else:
+                    normalized[key] = value
+            normalized_args.append(normalized)
+        else:
+            normalized_args.append(arg)
+
+    return normalized_args
+
+
+def hash_task(cmd: str, outputs: list[str], working_dir: str, args: list[str | dict[str, Any]], env: str = "") -> str:
     data = {
         "cmd": cmd,
         "outputs": sorted(outputs),
         "working_dir": working_dir,
-        "args": sorted(args),
+        "args": sorted(_normalize_choices_lists(args), key=_arg_sort_key),
         "env": env,
     }