cook-build 0.5.1__py3-none-any.whl → 0.6.0__py3-none-any.whl

cook/controller.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from datetime import datetime
 import hashlib
 import logging
@@ -7,7 +8,15 @@ from queue import Empty, Queue
 from sqlite3 import Connection
 import sys
 import threading
-from typing import Dict, List, Optional, Sequence, Set, Tuple, TYPE_CHECKING, Union
+from types import TracebackType
+from typing import (
+    cast,
+    Iterable,
+    Literal,
+    Sequence,
+    TYPE_CHECKING,
+    overload,
+)
 from . import util
 
 if TYPE_CHECKING:
@@ -22,7 +31,8 @@ QUERIES = {
         "name" TEXT PRIMARY KEY,
         "digest" TEXT NOT NULL,
         "last_completed" TIMESTAMP,
-        "last_failed" TIMESTAMP
+        "last_failed" TIMESTAMP,
+        "last_started" TIMESTAMP
     );
 
     -- Information about files so we can cache digests.
@@ -47,6 +57,11 @@ QUERIES = {
         VALUES (:name, '__failed__', :last_failed)
         ON CONFLICT ("name") DO UPDATE SET "digest" = '__failed__', last_failed = :last_failed
     """,
+    "upsert_task_started": """
+        INSERT INTO "tasks" ("name", "digest", "last_started")
+        VALUES (:name, '__pending__', :last_started)
+        ON CONFLICT ("name") DO UPDATE SET "digest" = '__pending__', last_started = :last_started
+    """,
     "upsert_file": """
         INSERT INTO "files" ("name", "digest", "last_digested")
         VALUES (:name, :digest, :last_digested)
@@ -56,24 +71,39 @@ QUERIES = {
         SELECT "digest", "last_digested"
         FROM "files"
         WHERE "name" = :name AND last_digested > :last_modified
-    """
+    """,
 }
 
 
+@dataclass
+class Event:
+    kind: Literal["start", "complete", "fail"]
+    task: "Task"
+    timestamp: datetime
+    exc_info: (
+        tuple[type[BaseException], BaseException, TracebackType]
+        | tuple[None, None, None]
+    )
+    digest: str | None
+
+
 class Controller:
     """
     Controller to manage dependencies and execute tasks.
     """
+
     def __init__(self, dependencies: nx.DiGraph, connection: Connection) -> None:
         self.dependencies = dependencies
         self.connection = connection
-        self._digest_cache: Dict[Path, Tuple[float, bytes]] = {}
+        self._digest_cache: dict[Path, tuple[float, bytes]] = {}
 
-    def resolve_stale_tasks(self, tasks: Optional[List["Task"]] = None) -> Set["Task"]:
+    def resolve_stale_tasks(self, tasks: list["Task"] | None = None) -> set["Task"]:
         self.is_stale(tasks or list(self.dependencies))
-        return {node for node, data in self.dependencies.nodes(True) if data.get("is_stale")}
+        return {
+            node for node, data in self.dependencies.nodes(True) if data.get("is_stale")
+        }
 
-    def _evaluate_task_hexdigest(self, task: "Task") -> str:
+    def _evaluate_task_hexdigest(self, task: "Task") -> str | None:
         """
         Evaluate the digest of a task by combining the digest of all its dependencies.
         """
@@ -94,7 +124,7 @@ class Controller:
             hasher.update(bytearray.fromhex(hexdigest))
         return hasher.hexdigest()
 
-    def _evaluate_path_hexdigest(self, path: Union[Path, str]) -> str:
+    def _evaluate_path_hexdigest(self, path: Path | str) -> str:
         """
         Get the digest of a file.
         """
@@ -102,10 +132,7 @@ class Controller:
         path = Path(path)
         stat = path.stat()
         name = str(path.resolve())
-        params = {
-            "name": name,
-            "last_modified": datetime.fromtimestamp(stat.st_mtime)
-        }
+        params = {"name": name, "last_modified": datetime.fromtimestamp(stat.st_mtime)}
         digest = self.connection.execute(QUERIES["select_file"], params).fetchone()
         if digest:
             return digest[0]
@@ -121,7 +148,13 @@ class Controller:
         self.connection.commit()
         return digest
 
-    def is_stale(self, task: Union["Task", Sequence["Task"]]) -> Union[bool, List[bool]]:
+    @overload
+    def is_stale(self, task: Sequence["Task"]) -> list[bool]: ...
+
+    @overload
+    def is_stale(self, task: "Task") -> bool: ...
+
+    def is_stale(self, task: "Task | Sequence[Task]") -> bool | list[bool]:
         """
         Determine if one or more tasks are stale.
 
@@ -156,16 +189,19 @@ class Controller:
         """
         # If there are no targets or the targets are missing, the task is stale.
         if not task.targets:
-            LOGGER.debug('%s is stale because it has no targets', task)
+            LOGGER.debug("%s is stale because it has no targets", task)
             return True
         for target in task.targets:
             if not target.is_file():
-                LOGGER.debug("%s is stale because its target `%s` is missing", task, target)
+                LOGGER.debug(
+                    "%s is stale because its target `%s` is missing", task, target
+                )
                 return True
 
         # If there is no digest in the database, the task is stale.
-        cached_digest = self.connection.execute("SELECT digest FROM tasks WHERE name = :name",
-                                                {"name": task.name}).fetchone()
+        cached_digest = self.connection.execute(
+            "SELECT digest FROM tasks WHERE name = :name", {"name": task.name}
+        ).fetchone()
         if cached_digest is None:
             LOGGER.debug("%s is stale because it does not have a hash entry", task)
             return True
@@ -176,16 +212,23 @@ class Controller:
             LOGGER.debug("%s is stale because one of its dependencies is missing", task)
 
         # If the digest has changed, the task is stale.
-        cached_digest, = cached_digest
+        (cached_digest,) = cached_digest
         if current_digest != cached_digest:
-            LOGGER.debug("%s is stale because one of its dependencies has changed (cached digest: "
-                         "%s, current digest: %s)", task, cached_digest, current_digest)
+            LOGGER.debug(
+                "%s is stale because one of its dependencies has changed (cached digest: "
+                "%s, current digest: %s)",
+                task,
+                cached_digest,
+                current_digest,
+            )
             return True
 
         LOGGER.debug("%s is up to date", task)
         return False
 
-    def execute(self, tasks: "Task", num_concurrent: int = 1, interval: float = 1) -> None:
+    def execute(
+        self, tasks: "Task | list[Task]", num_concurrent: int = 1, interval: float = 1
+    ) -> None:
         """
         Execute one or more tasks.
 
@@ -199,23 +242,30 @@ class Controller:
             return
 
         # Start the worker threads.
-        threads: List[threading.Thread] = []
+        threads: list[threading.Thread] = []
         input_queue = Queue()
-        output_queue = Queue()
+        output_queue = Queue[Event]()
         stop = util.StopEvent(interval)
         for i in range(num_concurrent):
-            thread = threading.Thread(target=self._target, name=f"cook-thread-{i}",
-                                      args=(stop, input_queue, output_queue), daemon=True)
+            thread = threading.Thread(
+                target=self._target,
+                name=f"cook-thread-{i}",
+                args=(stop, input_queue, output_queue),
+                daemon=True,
+            )
             thread.start()
             threads.append(thread)
 
         # Get the subgraph of stale nodes.
-        stale_nodes = [node for node, data in self.dependencies.nodes.data() if
-                       data.get("is_stale")]
-        dependencies: nx.DiGraph = self.dependencies.subgraph(stale_nodes).copy()
+        stale_nodes = [
+            node
+            for node, data in self.dependencies.nodes.data()
+            if data.get("is_stale")
+        ]
+        dependencies = cast(nx.DiGraph, self.dependencies.subgraph(stale_nodes).copy())
 
         # Initialize the input queue with leaf nodes.
-        for node, out_degree in dependencies.out_degree():
+        for node, out_degree in cast(Iterable, dependencies.out_degree()):
             if out_degree == 0:
                 input_queue.put((node, self._evaluate_task_hexdigest(node)))
 
@@ -223,48 +273,61 @@ class Controller:
             while dependencies.number_of_nodes():
                 # Try to get the next item in the queue, continuing if there's nothing available.
                 try:
-                    item: Optional[Tuple["Task", sys._OptExcInfo]] = \
-                        output_queue.get(timeout=interval)
+                    event = output_queue.get(timeout=interval)
                 except Empty:  # pragma: no cover
                     continue
 
-                assert item is not None, "output queue returned `None`; this is a bug"
+                assert event is not None, "output queue returned `None`; this is a bug"
 
                 # Unpack the results.
-                task, exc_info, digest = item
-                if exc_info:
+                if event.kind == "fail":
                     # Update the status in the database.
                     params = {
-                        "name": task.name,
-                        "last_failed": datetime.now(),
+                        "name": event.task.name,
+                        "last_failed": event.timestamp,
                     }
                     self.connection.execute(QUERIES["upsert_task_failed"], params)
                     self.connection.commit()
-                    raise util.FailedTaskError(exc_info[1], task=task) from exc_info[1]
-                else:
+                    ex = event.exc_info[1]
+                    raise util.FailedTaskError(ex, task=event.task) from ex
+                elif event.kind == "complete":
                     # Update the status in the database.
                     params = {
-                        "name": task.name,
-                        "digest": digest,
-                        "last_completed": datetime.now(),
+                        "name": event.task.name,
+                        "digest": event.digest,
+                        "last_completed": event.timestamp,
                     }
                     self.connection.execute(QUERIES["upsert_task_completed"], params)
                     self.connection.commit()
+                elif event.kind == "start":
+                    params = {
+                        "name": event.task.name,
+                        "last_started": event.timestamp,
+                    }
+                    self.connection.execute(QUERIES["upsert_task_started"], params)
+                    self.connection.commit()
+                    continue
+                else:
+                    raise ValueError(event)  # pragma: no cover
 
                 # Check if the stop event is set and abort if so.
                 if stop.is_set():
                     break
 
                 # Add tasks that are now leaf nodes to the tree.
-                predecessors = list(dependencies.predecessors(task))
-                dependencies.remove_node(task)
-                self.dependencies.add_node(task, is_stale=False)
-                for node, out_degree in dependencies.out_degree(predecessors):
+                predecessors = list(dependencies.predecessors(event.task))
+                dependencies.remove_node(event.task)
+                self.dependencies.add_node(event.task, is_stale=False)
+                for node, out_degree in cast(
+                    Iterable, dependencies.out_degree(predecessors)
+                ):
                     if out_degree == 0:
                         input_queue.put((node, self._evaluate_task_hexdigest(node)))
         finally:
             # Set the stop event and add "None" to the queue so the workers stop waiting.
-            LOGGER.debug("set stop event for threads: %s", [thread.name for thread in threads])
+            LOGGER.debug(
+                "set stop event for threads: %s", [thread.name for thread in threads]
+            )
             stop.set()
             for thread in threads:
                 input_queue.put((None, None))
@@ -275,13 +338,15 @@ class Controller:
                 if thread.is_alive():  # pragma: no cover
                     raise RuntimeError(f"thread {thread} failed to join")
 
-    def _target(self, stop: util.StopEvent, input_queue: Queue, output_queue: Queue) -> None:
+    def _target(
+        self, stop: util.StopEvent, input_queue: Queue, output_queue: Queue
+    ) -> None:
         LOGGER.debug(f"started thread `{threading.current_thread().name}`")
         while not stop.is_set():
             try:
                 task: "Task"
                 digest: str
-                task, digest = input_queue.get(stop.interval)
+                task, digest = input_queue.get(timeout=stop.interval)
             except Empty:  # pragma: no cover
                 # It's unlikely there's nothing on the queue, but let's handle it anyway.
                 continue
@@ -295,27 +360,63 @@ class Controller:
             start = datetime.now()
             try:
                 # Execute the task.
-                LOGGER.log(logging.DEBUG if task.name.startswith("_") else logging.INFO,
-                           "executing %s ...", task)
+                LOGGER.log(
+                    logging.DEBUG if task.name.startswith("_") else logging.INFO,
+                    "executing %s ...",
+                    task,
+                )
+                output_queue.put(
+                    Event(
+                        kind="start",
+                        task=task,
+                        digest=None,
+                        timestamp=start,
+                        exc_info=(None, None, None),
+                    )
+                )
                 task.execute(stop)
 
                 # Check that all targets were created.
                 for target in task.targets:
                     if not target.is_file():
-                        raise FileNotFoundError(f"task {task} did not create target {target}")
+                        raise FileNotFoundError(
+                            f"task {task} did not create target {target}"
+                        )
                     LOGGER.debug("%s created `%s`", task, target)
 
                 # Add the result to the output queue and report success.
-                output_queue.put((task, None, digest))
+                output_queue.put(
+                    Event(
+                        kind="complete",
+                        task=task,
+                        digest=digest,
+                        timestamp=datetime.now(),
+                        exc_info=(None, None, None),
+                    )
+                )
                 delta = util.format_timedelta(datetime.now() - start)
-                LOGGER.log(logging.DEBUG if task.name.startswith("_") else logging.INFO,
-                           "completed %s in %s", task, delta)
+                LOGGER.log(
+                    logging.DEBUG if task.name.startswith("_") else logging.INFO,
+                    "completed %s in %s",
+                    task,
+                    delta,
+                )
             except:  # noqa: E722
                 exc_info = sys.exc_info()
                 delta = util.format_timedelta(datetime.now() - start)
-                LOGGER.exception("failed to execute %s after %s", task, delta, exc_info=exc_info)
+                LOGGER.exception(
+                    "failed to execute %s after %s", task, delta, exc_info=exc_info
+                )
                 stop.set()
-                output_queue.put((task, sys.exc_info(), digest))
+                output_queue.put(
+                    Event(
+                        kind="fail",
+                        task=task,
+                        digest=digest,
+                        timestamp=datetime.now(),
+                        exc_info=sys.exc_info(),
+                    )
+                )
 
         # Put anything on the queue in case the parent is waiting.
         LOGGER.debug(f"exiting thread `{threading.current_thread().name}`")

cook/manager.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 import logging
 import networkx as nx
 from pathlib import Path
-from typing import Dict, List, Optional, Set, Tuple, TYPE_CHECKING
+from typing import TYPE_CHECKING
 from . import task as task_
 from . import util
 
@@ -20,11 +20,12 @@ class Manager:
     """
     Task manager that captures the relationship between tasks, targets, and dependencies.
     """
-    _INSTANCE: Optional[Manager] = None
 
-    def __init__(self, contexts: Optional[List["Context"]] = None) -> None:
-        self.contexts: List["Context"] = contexts or []
-        self.tasks: Dict[str, "Task"] = {}
+    _INSTANCE: Manager | None = None
+
+    def __init__(self, contexts: list["Context"] | None = None) -> None:
+        self.contexts: list["Context"] = contexts or []
+        self.tasks: dict[str, "Task"] = {}
 
     def __enter__(self) -> Manager:
         if Manager._INSTANCE:
@@ -62,8 +63,9 @@ class Manager:
             return task
         except:  # noqa: 722
             filename, lineno = util.get_location()
-            LOGGER.exception("failed to create task with name '%s' at %s:%d", name, filename,
-                             lineno)
+            LOGGER.exception(
+                "failed to create task with name '%s' at %s:%d", name, filename, lineno
+            )
             raise
 
     def resolve_dependencies(self) -> nx.DiGraph:
@@ -74,22 +76,28 @@ class Manager:
             Directed graph of dependencies. Edges point *from* a task *to* others it depends on.
         """
         # Run over all the targets and dependencies to explore connections between tasks.
-        task_by_target: Dict[Path, "Task"] = {}
-        tasks_by_file_dependency: Dict[Path, List["Task"]] = {}
-        dependencies: Dict["Task", Set["Task"]] = {}
+        task_by_target: dict[Path, "Task"] = {}
+        tasks_by_file_dependency: dict[Path, set["Task"]] = {}
+        dependencies: dict["Task", set["Task"]] = {}
         for task in self.tasks.values():
             if task.task_dependencies:
                 dependencies[task] = set(task.task_dependencies)
             for path in task.targets:
                 if path.is_symlink():
-                    LOGGER.warning("target %s of %s is a symlink which may lead to unexpected "
-                                   "behavior", path, task)
+                    LOGGER.warning(
+                        "target %s of %s is a symlink which may lead to unexpected "
+                        "behavior",
+                        path,
+                        task,
+                    )
                 path = path.resolve()
-                if (other := task_by_target.get(path)):
-                    raise ValueError(f"tasks {task} and {other} both have target {path}")
+                if other := task_by_target.get(path):
+                    raise ValueError(
+                        f"tasks {task} and {other} both have target {path}"
+                    )
                 task_by_target[path] = task
             for path in task.dependencies:
-                path = path.resolve()
+                path = Path(path).resolve()
                 tasks_by_file_dependency.setdefault(path, set()).add(task)
 
         # Build a directed graph of dependencies based on files produced and consumed by tasks.
@@ -107,7 +115,9 @@ class Manager:
 
         graph = nx.DiGraph()
         graph.add_nodes_from(self.tasks.values())
-        graph.add_edges_from((task, dep) for task, deps in dependencies.items() for dep in deps)
+        graph.add_edges_from(
+            (task, dep) for task, deps in dependencies.items() for dep in deps
+        )
 
         try:
             cycle = nx.find_cycle(graph)
@@ -118,16 +128,21 @@ class Manager:
         return graph
 
 
-def create_task(name: str, *, action: Optional["Action"] = None,
-                targets: Optional[List["Path"]] = None, dependencies: Optional[List["Path"]] = None,
-                task_dependencies: Optional[List["Task"]] = None,
-                location: Optional[Tuple[str, int]] = None) -> "Task":
+def create_task(
+    name: str,
+    *,
+    action: "Action | str | None" = None,
+    targets: list["Path"] | None = None,
+    dependencies: list["Path"] | None = None,
+    task_dependencies: list["Task"] | None = None,
+    location: tuple[str, int] | None = None,
+) -> "Task":
     """
     Create a new task.
 
     Args:
         name: Name of the new task.
-        action: Action to execute.
+        action: Action to execute or a string for shell commands.
         targets: Paths for files to be generated.
         dependencies: Paths to files on which this task depends.
         task_dependencies: Tasks which the new task explicitly depends on.
@@ -138,6 +153,10 @@ def create_task(name: str, *, action: Optional["Action"] = None,
         New task.
     """
     return Manager.get_instance().create_task(
-        name, action=action, targets=targets, dependencies=dependencies, location=location,
+        name,
+        action=action,
+        targets=targets,
+        dependencies=dependencies,
+        location=location,
         task_dependencies=task_dependencies,
     )

cook/task.py

@@ -1,8 +1,7 @@
 from __future__ import annotations
 import colorama
 from pathlib import Path
-import threading
-from typing import List, Optional, Tuple, TYPE_CHECKING
+from typing import TYPE_CHECKING
 from . import util
 
 
@@ -15,16 +14,17 @@ class Task:
     """
     Task to be executed.
     """
+
     def __init__(
-            self,
-            name: str,
-            *,
-            dependencies: Optional[List["PathOrStr"]] = None,
-            targets: Optional[List["PathOrStr"]] = None,
-            action: Optional[Action] = None,
-            task_dependencies: Optional[List[Task]] = None,
-            location: Optional[Tuple[str, int]] = None,
-            ) -> None:
+        self,
+        name: str,
+        *,
+        dependencies: list["PathOrStr"] | None = None,
+        targets: list["PathOrStr"] | None = None,
+        action: Action | None = None,
+        task_dependencies: list[Task] | None = None,
+        location: tuple[str, int] | None = None,
+    ) -> None:
         self.name = name
         self.dependencies = dependencies or []
         self.targets = [Path(path) for path in (targets or [])]
@@ -32,14 +32,14 @@ class Task:
         self.task_dependencies = task_dependencies or []
         self.location = location or util.get_location()
 
-    def execute(self, stop: Optional[threading.Event] = None) -> None:
+    def execute(self, stop: util.StopEvent | None = None) -> None:
         if self.action:
             self.action.execute(self, stop)
 
     def __hash__(self) -> int:
         return hash(self.name)
 
-    def format(self, color: str = None) -> str:
+    def format(self, color: str | None = None) -> str:
         name = self.name
         if color:
             name = f"{color}{name}{colorama.Fore.RESET}"

cook/util.py

@@ -7,26 +7,26 @@ import os
 from pathlib import Path
 import threading
 from time import time
-from typing import Tuple, TYPE_CHECKING, Union
+from typing import TYPE_CHECKING, Generator
 
 
 if TYPE_CHECKING:
     from .task import Task
 
 
-PathOrStr = Union[Path, str]
+PathOrStr = Path | str
 
 
-def evaluate_digest(path: PathOrStr, size=2 ** 16, hasher: str = "sha1") -> bytes:
-    hasher = hashlib.new(hasher)
+def evaluate_digest(path: PathOrStr, size=2**16, hasher: str = "sha1") -> bytes:
+    hasher_instance = hashlib.new(hasher)
     path = Path(path)
     with path.open("rb") as fp:
         while chunk := fp.read(size):
-            hasher.update(chunk)
-    return hasher.digest()
+            hasher_instance.update(chunk)
+    return hasher_instance.digest()
 
 
-def evaluate_hexdigest(path: PathOrStr, size=2 ** 16, hasher: str = "sha1") -> str:
+def evaluate_hexdigest(path: PathOrStr, size=2**16, hasher: str = "sha1") -> str:
     return evaluate_digest(path, size, hasher).hex()
 
 
@@ -43,6 +43,8 @@ class Timer:
 
     @property
     def duration(self):
+        assert self.start is not None, "Timer has not started yet."
+        assert self.end is not None, "Timer has not finished yet."
         return self.end - self.start
 
 
@@ -57,7 +59,7 @@ class FailedTaskError(Exception):
 
 
 @contextlib.contextmanager
-def working_directory(path: PathOrStr) -> Path:
+def working_directory(path: PathOrStr) -> Generator[Path]:
     path = Path(path)
     original = Path.cwd()
     try:
@@ -67,7 +69,7 @@ def working_directory(path: PathOrStr) -> Path:
         os.chdir(original)
 
 
-def get_location() -> Tuple[str, int]:
+def get_location() -> tuple[Path, int]:
     """
     Get the first location in the call stack which is not part of the Cook package.
 
@@ -75,8 +77,10 @@ def get_location() -> Tuple[str, int]:
         Location as a tuple :code:`(filename, lineno)`.
     """
     frame = inspect.currentframe()
+    assert frame is not None, "Could not fetch current frame."
     while frame.f_globals.get("__name__", "<unknown>").startswith("cook"):
         frame = frame.f_back
+        assert frame is not None, "Could not fetch parent frame."
     return Path(frame.f_code.co_filename).resolve(), frame.f_lineno
 
 
@@ -84,6 +88,7 @@ class StopEvent(threading.Event):
     """
     Event used for stopping execution with a polling interval.
     """
+
     def __init__(self, interval: float = 1) -> None:
         super().__init__()
         self.interval = interval