cook-build 0.6.4__py3-none-any.whl

cook/__init__.py

@@ -0,0 +1,11 @@
+from .controller import Controller
+from .manager import create_task, Manager
+from .task import Task
+
+
+__all__ = [
+    "Controller",
+    "create_task",
+    "Manager",
+    "Task",
+]

cook/__main__.py

@@ -0,0 +1,392 @@
+import argparse
+import colorama
+from contextlib import closing
+from datetime import datetime
+import fnmatch
+import importlib.util
+import logging
+import os
+from pathlib import Path
+import re
+import sqlite3
+import sys
+import textwrap
+from typing import Iterable
+
+from .contexts import (
+    create_target_directories,
+    normalize_action,
+    normalize_dependencies,
+)
+from .controller import Controller, QUERIES
+from .manager import Manager
+from .task import Task
+from .util import FailedTaskError, format_datetime, format_timedelta
+
+
+LOGGER = logging.getLogger("cook")
+
+
+class NoMatchingTaskError(ValueError):
+    def __init__(
+        self, patterns: Iterable[re.Pattern | str], hidden_tasks_available: bool
+    ) -> None:
+        self.hidden_tasks_available = hidden_tasks_available
+        formatted_patterns = [f"`{pattern}`" for pattern in patterns]
+        if len(formatted_patterns) == 1:
+            message = f"found no tasks matching pattern {formatted_patterns[0]}"
+        else:
+            *formatted_patterns, last = formatted_patterns
+            message = (
+                "found no tasks matching patterns "
+                + ", ".join(formatted_patterns)
+                + (", or " if len(formatted_patterns) > 1 else " or ")
+                + last
+            )
+        if hidden_tasks_available:
+            message = message + "; use --all or -a to include hidden tasks"
+        super().__init__(message)
+
+
+class Args:
+    tasks: Iterable[re.Pattern | str]
+    re: bool
+    all: bool
+
+
+class Command:
+    """
+    Abstract base class for commands.
+    """
+
+    NAME: str | None = None
+    ALLOW_EMPTY_PATTERN: bool = False
+
+    def configure_parser(self, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument(
+            "--re",
+            "-r",
+            action="store_true",
+            help="use regular expressions for pattern matching instead of glob",
+        )
+        parser.add_argument(
+            "--all",
+            "-a",
+            action="store_true",
+            help="include tasks starting with `_` prefix",
+        )
+        parser.add_argument(
+            "tasks",
+            nargs="*" if self.ALLOW_EMPTY_PATTERN else "+",
+            help="task or tasks to execute as regular expressions or glob patterns",
+        )
+
+    def execute(self, controller: Controller, args: argparse.Namespace) -> None:
+        raise NotImplementedError
+
+    def discover_tasks(self, controller: Controller, args: Args) -> list[Task]:
+        task: Task
+        tasks: list[Task] = []
+
+        # Get tasks based on the pattern matching.
+        if not args.tasks:
+            tasks = list(controller.dependencies)
+        else:
+            for task in controller.dependencies:
+                if args.re:
+                    match = any(re.match(pattern, task.name) for pattern in args.tasks)
+                else:
+                    match = any(
+                        fnmatch.fnmatch(task.name, pattern)  # pyright: ignore[reportArgumentType]
+                        for pattern in args.tasks
+                    )
+                if match:
+                    tasks.append(task)
+
+        # Store whether any of the candidates are hidden by default.
+        has_hidden_task = any(task.name.startswith("_") for task in tasks)
+
+        # Filter out hidden tasks if desired unless the name is an exact match to a specified
+        # pattern.
+        if not args.all:
+            tasks = [
+                task
+                for task in tasks
+                if not task.name.startswith("_") or task.name in args.tasks
+            ]
+
+        if not tasks:
+            raise NoMatchingTaskError(args.tasks, not args.all and has_hidden_task)
+        return tasks
+
+
+class ExecArgs(Args):
+    jobs: int
+    dry_run: bool
+
+
+class ExecCommand(Command):
+    """
+    Execute one or more tasks.
+    """
+
+    NAME = "exec"
+
+    def configure_parser(self, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument(
+            "--jobs", "-j", help="number of concurrent jobs", type=int, default=1
+        )
+        parser.add_argument(
+            "--dry-run",
+            "-n",
+            help="show what \033[1mwould\033[0m be executed without running tasks",
+            action="store_true",
+            dest="dry_run",
+        )
+        super().configure_parser(parser)
+
+    def execute(self, controller: Controller, args: ExecArgs) -> None:  # pyright: ignore[reportIncompatibleMethodOverride]
+        tasks = self.discover_tasks(controller, args)
+        controller.execute(tasks, num_concurrent=args.jobs, dry_run=args.dry_run)
+
+
+class LsArgs(Args):
+    stale: bool
+    current: bool
+
+
+class LsCommand(Command):
+    """
+    List tasks.
+    """
+
+    NAME = "ls"
+    ALLOW_EMPTY_PATTERN = True
+
+    def configure_parser(self, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument(
+            "--stale", "-s", action="store_true", help="only show stale tasks"
+        )
+        parser.add_argument(
+            "--current", "-c", action="store_true", help="only show current tasks"
+        )
+        super().configure_parser(parser)
+
+    def execute(self, controller: Controller, args: LsArgs) -> None:  # pyright: ignore[reportIncompatibleMethodOverride]
+        tasks = self.discover_tasks(controller, args)
+        tasks = [
+            task.format(colorama.Fore.YELLOW if is_stale else colorama.Fore.GREEN)
+            for is_stale, task in zip(controller.is_stale(tasks), tasks)
+        ]
+        print("\n".join(tasks))
+
+    def discover_tasks(self, controller: Controller, args: LsArgs) -> list[Task]:  # pyright: ignore[reportIncompatibleMethodOverride]
+        if args.current and args.stale:
+            raise ValueError(
+                "only one of `--stale` and `--current` may be given at the same time"
+            )
+        tasks = super().discover_tasks(controller, args)
+        if args.stale:
+            return [
+                task for stale, task in zip(controller.is_stale(tasks), tasks) if stale
+            ]
+        elif args.current:
+            return [
+                task
+                for stale, task in zip(controller.is_stale(tasks), tasks)
+                if not stale
+            ]
+        return tasks
+
+
+class InfoCommand(LsCommand):
+    """
+    Display information about one or more tasks.
+    """
+
+    NAME = "info"
+
+    def execute(self, controller: Controller, args: LsArgs) -> None:
+        tasks = self.discover_tasks(controller, args)
+        stales = controller.is_stale(tasks)
+
+        stale_string = f"{colorama.Fore.YELLOW}stale{colorama.Fore.RESET}"
+        current_string = f"{colorama.Fore.GREEN}current{colorama.Fore.RESET}"
+        indent = "    "
+
+        task: Task
+        for stale, task in zip(stales, tasks):
+            # Show the status.
+            parts = [
+                f"status: {stale_string if stale else current_string}",
+            ]
+            # Show when the task last completed and failed.
+            last = controller.connection.execute(
+                "SELECT last_started, last_completed, last_failed FROM tasks WHERE name = :name",
+                {"name": task.name},
+            ).fetchone() or (None, None, None)
+            for key, value in zip(["started", "completed", "failed"], last):
+                if value is None:
+                    parts.append(f"last {key}: -")
+                    continue
+                parts.append(
+                    f"last {key}: {format_timedelta(datetime.now() - value)} ago "
+                    f"({format_datetime(value)})"
+                )
+            # Show dependencies and targets.
+            task_dependencies = list(
+                sorted(controller.dependencies.successors(task), key=lambda t: t.name)
+            )
+            task_dependencies = [
+                dep.format(colorama.Fore.YELLOW if is_stale else colorama.Fore.GREEN)
+                for is_stale, dep in zip(
+                    controller.is_stale(task_dependencies), task_dependencies
+                )
+            ]
+            items = [
+                ("dependencies", task.dependencies),
+                ("targets", task.targets),
+                ("task_dependencies", task_dependencies),
+            ]
+            for key, value in items:
+                value = "\n".join(map(str, value))
+                if value:
+                    parts.append(f"{key}:\n{textwrap.indent(value, indent)}")
+                else:
+                    parts.append(f"{key}: -")
+            parts.append(f"action: {task.action if task.action else '-'}")
+
+            parts = textwrap.indent("\n".join(parts), indent)
+            print(f"{task}\n{parts}")
+
+
+class ResetArgs(Args):
+    pass
+
+
+class ResetCommand(Command):
+    """
+    Reset the status of one or more tasks.
+    """
+
+    NAME = "reset"
+
+    def execute(self, controller: Controller, args: ResetArgs) -> None:  # pyright: ignore[reportIncompatibleMethodOverride]
+        tasks = self.discover_tasks(controller, args)
+        controller.reset(*tasks)
+
+
+class Formatter(logging.Formatter):
+    COLOR_BY_LEVEL = {
+        "DEBUG": colorama.Fore.MAGENTA,
+        "INFO": colorama.Fore.BLUE,
+        "WARNING": colorama.Fore.YELLOW,
+        "ERROR": colorama.Fore.RED,
+        "CRITICAL": colorama.Fore.WHITE + colorama.Back.RED,
+    }
+    RESET = colorama.Fore.RESET + colorama.Back.RESET
+
+    def format(self, record: logging.LogRecord) -> str:
+        color = self.COLOR_BY_LEVEL[record.levelname]
+        formatted = f"{color}{record.levelname}{self.RESET}: {record.getMessage()}"
+        if record.exc_info:
+            formatted = f"{formatted}\n{self.formatException(record.exc_info)}"
+        return formatted
+
+
+def __main__(cli_args: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser("cook")
+    parser.add_argument(
+        "--recipe",
+        help="file containing declarative recipe for tasks",
+        default="recipe.py",
+        type=Path,
+    )
+    parser.add_argument(
+        "--module", "-m", help="module containing declarative recipe for tasks"
+    )
+    parser.add_argument(
+        "--db", help="database for keeping track of assets", default=".cook"
+    )
+    parser.add_argument(
+        "--log-level",
+        help="log level",
+        default="info",
+        choices={"error", "warning", "info", "debug"},
+    )
+    subparsers = parser.add_subparsers()
+    subparsers.required = True
+
+    for command_cls in [ExecCommand, LsCommand, InfoCommand, ResetCommand]:
+        subparser = subparsers.add_parser(command_cls.NAME, help=command_cls.__doc__)
+        command = command_cls()
+        command.configure_parser(subparser)
+        subparser.set_defaults(command=command)
+
+    args = parser.parse_args(cli_args)
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(Formatter())
+    logging.basicConfig(level=args.log_level.upper(), handlers=[handler])
+
+    with Manager() as manager:
+        try:
+            manager.contexts.extend(
+                [
+                    create_target_directories(),
+                    normalize_action(),
+                    normalize_dependencies(),
+                ]
+            )
+            if args.module:
+                # Temporarily add the current working directory to the path.
+                try:
+                    sys.path.append(os.getcwd())
+                    importlib.import_module(args.module)
+                finally:
+                    sys.path.pop()
+            elif args.recipe.is_file():
+                # Parse the recipe.
+                spec = importlib.util.spec_from_file_location("recipe", args.recipe)
+                assert spec, f"Could not load spec for '{args.recipe}'."
+                recipe = importlib.util.module_from_spec(spec)
+                assert spec.loader, f"Could not load recipe '{args.recipe}'."
+                spec.loader.exec_module(recipe)
+            else:  # pragma: no cover
+                raise ValueError(
+                    "recipe file or module must be specified; default recipe.py not "
+                    "found"
+                )
+        except:  # noqa: E722
+            LOGGER.fatal("failed to load recipe", exc_info=sys.exc_info())
+            sys.exit(1)
+
+    with closing(
+        sqlite3.connect(args.db, detect_types=sqlite3.PARSE_DECLTYPES)
+    ) as connection:
+        _setup_schema(connection)
+        controller = Controller(manager.resolve_dependencies(), connection)
+        command: Command = args.command
+        try:
+            command.execute(controller, args)
+        except KeyboardInterrupt:  # pragma: no cover
+            LOGGER.warning("interrupted by user")
+        except NoMatchingTaskError as ex:
+            LOGGER.warning(ex)
+            sys.exit(1)
+        except FailedTaskError:
+            sys.exit(1)
+
+
+def _setup_schema(connection: sqlite3.Connection) -> None:
+    connection.executescript(QUERIES["schema"])
+    # Attempt to add the column which may not be present for cook versions <0.6.
+    try:
+        connection.execute("ALTER TABLE tasks ADD COLUMN last_started TIMESTAMP")
+    except sqlite3.OperationalError as ex:
+        if "duplicate column name" not in ex.args[0]:
+            raise  # pragma: no cover
+
+
+if __name__ == "__main__":
+    __main__()

cook/actions.py

@@ -0,0 +1,215 @@
+"""
+Actions
+-------
+
+Actions are performed when tasks are executed. Builtin actions include calling python functions
+using :class:`.FunctionAction`, running subprocesses using :class:`.SubprocessAction`, composing
+multiple actions using :class:`.CompositeAction`, and executing modules as scripts using
+:class:`.ModuleAction`.
+
+Custom actions can be implemented by inheriting from :class:`.Action` and implementing the
+:meth:`~.Action.execute` method which receives a :class:`~.task.Task`. The method should execute the
+action; its return value is ignored. For example, the following action waits for a specified time.
+
+.. doctest::
+
+    >>> from cook.actions import Action
+    >>> from cook.task import Task
+    >>> from time import sleep, time
+
+    >>> class SleepAction(Action):
+    ...     def __init__(self, delay: float) -> None:
+    ...         self.delay = delay
+    ...
+    ...     def execute(self, task: Task) -> None:
+    ...         start = time()
+    ...         sleep(self.delay)
+    ...         print(f"time: {time() - start:.3f}")
+
+    >>> action = SleepAction(0.1)
+    >>> action.execute(None)
+    time: 0.1...
+"""
+
+import hashlib
+import os
+import shlex
+import subprocess
+import sys
+from types import ModuleType
+from typing import Callable, TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from .task import Task
+    from .util import StopEvent
+
+
+class Action:
+    """
+    Action to perform when a task is executed in its own thread.
+    """
+
+    def execute(self, task: "Task", stop: "StopEvent | None" = None) -> None:
+        """
+        Execute the action.
+        """
+        raise NotImplementedError
+
+    @property
+    def hexdigest(self) -> str | None:
+        """
+        Optional digest to check if an action changed.
+        """
+        return None
+
+
+class FunctionAction(Action):
+    """
+    Action wrapping a python callable.
+
+    Args:
+        func: Function to call which must accept a :class:`~.task.Task` as its first argument.
+        *args: Additional positional arguments.
+        **kwargs: Keyword arguments.
+    """
+
+    def __init__(self, func: Callable, *args, **kwargs) -> None:
+        super().__init__()
+        self.func = func
+        self.args = args
+        self.kwargs = kwargs
+
+    def execute(self, task: "Task", stop: "StopEvent | None" = None) -> None:
+        self.func(task, *self.args, **self.kwargs)
+
+
+class SubprocessAction(Action):
+    """
+    Run a subprocess.
+
+    Args:
+        *args: Positional arguments for :class:`subprocess.Popen`.
+        **kwargs: Keyword arguments for :class:`subprocess.Popen`.
+
+    Example:
+
+        .. doctest::
+
+            >>> from cook.actions import SubprocessAction
+            >>> from pathlib import Path
+
+            >>> action = SubprocessAction(["touch", "hello.txt"])
+            >>> action.execute(None)
+            >>> Path("hello.txt").is_file()
+            True
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        self.args = args
+        self.kwargs = kwargs
+
+    def execute(self, task: "Task", stop: "StopEvent | None" = None) -> None:
+        # Repeatedly wait for the process to complete, checking the stop event after each poll.
+        interval = stop.interval if stop else None
+        process = subprocess.Popen(*self.args, **self.kwargs)
+        while True:
+            try:
+                returncode = process.wait(interval)
+                if returncode:
+                    raise subprocess.CalledProcessError(returncode, process.args)
+                return
+            except subprocess.TimeoutExpired:
+                if stop and stop.is_set():
+                    break
+
+        # Clean up the process by trying to terminate it and then killing it.
+        for method in [process.terminate, process.kill]:
+            method()
+            try:
+                returncode = process.wait(max(interval, 3) if interval else None)
+                if returncode:
+                    raise subprocess.CalledProcessError(returncode, process.args)
+                # The process managed to exit gracefully after the main loop. This is unlikely.
+                return  # pragma: no cover
+            except subprocess.TimeoutExpired:  # pragma: no cover
+                pass
+
+        # We couldn't kill the process. Also very unlikely.
+        raise subprocess.SubprocessError(
+            f"failed to shut down {process}"
+        )  # pragma: no cover
+
+    @property
+    def hexdigest(self) -> str:
+        hasher = hashlib.sha1()
+        (args,) = self.args
+        if isinstance(args, str):
+            hasher.update(args.encode())
+        else:
+            for arg in args:
+                hasher.update(arg.encode())
+        return hasher.hexdigest()
+
+    def __repr__(self) -> str:
+        args, *_ = self.args
+        if not isinstance(args, str):
+            args = " ".join(map(shlex.quote, args))
+        return f"{self.__class__.__name__}({repr(args)})"
+
+
+class CompositeAction(Action):
+    """
+    Execute multiple actions sequentially.
+
+    Args:
+        *actions: Actions to execute.
+    """
+
+    def __init__(self, *actions: Action) -> None:
+        self.actions = actions
+
+    def execute(self, task: "Task", stop: "StopEvent | None" = None) -> None:
+        for action in self.actions:
+            action.execute(task, stop)
+
+    @property
+    def hexdigest(self) -> str | None:
+        parts = []
+        for action in self.actions:
+            hexdigest = action.hexdigest
+            if hexdigest is None:
+                return None
+            parts.append(hexdigest)
+        return "".join(parts)
+
+
+class ModuleAction(SubprocessAction):
+    """
+    Execute a module as a script.
+
+    Args:
+        args: List comprising the module to execute as the first element and arguments for the
+            module as subsequent elements.
+        debug: Run the module using `pdb` (defaults to the :code:`COOK_DEBUG` environment variable
+            being set).
+        **kwargs: Keyword arguments for :class:`subprocess.Popen`.
+    """
+
+    def __init__(self, args: list, debug: bool | None = None, **kwargs) -> None:
+        if kwargs.get("shell"):
+            raise ValueError("shell execution is not supported by `ModuleAction`")
+        if not args:
+            raise ValueError("`args` must not be empty")
+        module: ModuleType
+        module, *args = args
+        if not isinstance(module, ModuleType):
+            raise TypeError("first element of `args` must be a module")
+
+        # Assemble the arguments.
+        args_ = [sys.executable, "-m"]
+        debug = "COOK_DEBUG" in os.environ if debug is None else debug
+        if debug:
+            args_.extend(["pdb", "-m"])
+        args_.extend([module.__name__, *map(str, args)])
+        super().__init__(args_, **kwargs)