chorelib 0.1.0__py3-none-any.whl

chorelib/.ruff_cache/.gitignore

@@ -0,0 +1,2 @@
+# Automatically created by ruff.
+*

chorelib/.ruff_cache/CACHEDIR.TAG

@@ -0,0 +1 @@
+Signature: 8a477f597d28d172789f06886806bc55

chorelib/__init__.py

@@ -0,0 +1,48 @@
+"""Chorelib: An async-first Python build automation framework.
+
+Provides a decorator-based DSL for defining build rules and tasks with
+dependency management, parallel execution, and mtime-based rebuild detection.
+
+Typical usage::
+
+    import chorelib
+
+    @chorelib.rule("output.txt", depends="input.txt")
+    def build_output(target, depends, needs):
+        chorelib.shell(f"cp {depends[0]} {target}")
+
+    if __name__ == "__main__":
+        chorelib.Main().run()
+"""
+
+from .depmain import Main
+from .deprunner import schedule
+from .ruledef import RuleSet
+from .utils import command, message, shell
+
+__version__ = "0.1.0"
+
+# Default RuleSet instance used by the module-level decorators.
+# Users can use `@chorelib.rule(...)` and `@chorelib.task` directly
+# without creating a RuleSet manually.
+_default_rules = RuleSet()
+rule = _default_rules.rule
+task = _default_rules.task
+mtime = _default_rules.mtime
+
+# Global verbosity level controlling message output.
+# 0 = normal, 1 = verbose, 2 = debug messages, 3+ = logging debug.
+verbose: int = 0
+
+__all__ = [
+    "RuleSet",
+    "rule",
+    "task",
+    "mtime",
+    "Main",
+    "schedule",
+    "verbose",
+    "command",
+    "message",
+    "shell",
+]

chorelib/depgraph.py

@@ -0,0 +1,122 @@
+"""Dependency graph construction and cycle detection.
+
+Builds a directed acyclic graph (DAG) of build targets and their
+dependencies. Detects circular dependencies via depth-first search
+before any build execution begins.
+"""
+
+from typing import Any
+
+from .errors import RuleError
+from .ruledef import RuleBase, RuleSet, Task
+
+
+class BuildInfo:
+    """A node in the dependency graph representing a single build target.
+
+    Attributes:
+        target: The target name (file path or task name).
+        rule: The Rule or Task that produces this target.
+        depends: List of dependency target names that trigger rebuilds.
+        needs: List of order-only prerequisite target names.
+    """
+
+    def __init__(self, target: str, rule: RuleBase, depends: list[str], needs: list[str]) -> None:
+        self.rule = rule
+        self.target = target
+        self.depends = depends
+        self.needs = needs
+
+    def __repr__(self) -> str:
+        return f"<BuildInfo> {self.rule}:{self.target}:{self.depends}:{self.needs}"
+
+    def is_task(self) -> bool:
+        """Return True if this node represents a Task (always-execute)."""
+        return isinstance(self.rule, Task)
+
+    def run_builder(self) -> Any:
+        """Execute the builder function for this target."""
+        return self.rule.run_builder(target=self.target, depends=self.depends, needs=self.needs)
+
+
+class DepGraph:
+    """Dependency graph that manages BuildInfo nodes and detects cycles.
+
+    Nodes are registered by recursively resolving each target's dependencies
+    through the RuleSet. After registration, ``detectloop`` should be called
+    to verify the graph is acyclic.
+    """
+
+    def __init__(self) -> None:
+        self._nodes: dict[str, BuildInfo] = {}
+
+    def addtarget(self, ruleset: RuleSet, target: str) -> bool:
+        """Add a target and all its transitive dependencies to the graph.
+
+        Returns:
+            True if the target was newly registered, False if already present.
+        """
+        return self._register(ruleset, target)
+
+    def get(self, target: str) -> BuildInfo | None:
+        """Return the BuildInfo node for a target, or None if not registered."""
+        return self._nodes.get(target)
+
+    def _register(self, ruleset: RuleSet, target: str) -> bool:
+        """Recursively register a target and its dependencies.
+
+        Looks up the matching rule in the ruleset, creates a BuildInfo node,
+        and recurses into each dependency and need.
+        """
+        if target in self._nodes:
+            return False
+
+        ret = ruleset.select_rule(target)
+        if ret:
+            rule, deps, needs = ret
+            node = BuildInfo(target, rule, deps, needs)
+            self._nodes[target] = node
+
+            # Recursively register all dependencies
+            for dep in deps:
+                self._register(ruleset, dep)
+
+            # Recursively register all order-only prerequisites
+            for need in needs:
+                self._register(ruleset, need)
+
+        return True
+
+    def detectloop(self) -> None:
+        """Detect circular dependencies in the graph using DFS.
+
+        Raises:
+            RuleError: If a dependency cycle is found.
+        """
+        done: set[BuildInfo] = set()
+
+        def _walk(node: BuildInfo, seen: set[BuildInfo]) -> None:
+            """DFS traversal tracking the current path in ``seen``."""
+            if node in seen:
+                raise RuleError(f"Dependency cycle detected: {node}")
+
+            if node in done:
+                return
+
+            seen.add(node)
+            try:
+                for dep in node.depends:
+                    depnode = self._nodes.get(dep)
+                    if depnode:
+                        _walk(depnode, seen)
+                for need in node.needs:
+                    neednode = self._nodes.get(need)
+                    if neednode:
+                        _walk(neednode, seen)
+            finally:
+                seen.remove(node)
+            done.add(node)
+
+        for node in self._nodes.values():
+            seen: set[BuildInfo] = set()
+            _walk(node, seen)

chorelib/depmain.py

@@ -0,0 +1,209 @@
+"""CLI entry point for chorelib build scripts.
+
+Provides the ``Main`` class that parses command-line arguments, selects
+targets, and drives the async build execution.
+"""
+
+import argparse
+import asyncio
+import logging
+import os
+import sys
+import textwrap
+import traceback
+from types import ModuleType
+
+import chorelib
+from chorelib.utils import message
+
+from .deprunner import run
+from .ruledef import RuleSet
+
+logger = logging.getLogger(__name__)
+
+
+class Main:
+    """CLI driver for chorelib build scripts.
+
+    Parses command-line arguments (targets, verbosity, workers, rebuild
+    flags) and runs the async build process. Subclass this to add custom
+    arguments via ``add_arguments``.
+    """
+
+    def __init__(self, rules: RuleSet | None = None) -> None:
+        if rules is None:
+            rules = chorelib._default_rules
+        self.rules = rules
+        self.args: argparse.Namespace
+        self._load_args()
+
+    def _load_args(self):
+        self.parser = self._build_parser()
+        self.args = self.parse_args(self.parser)
+
+        chorelib.verbose = self.args.verbose
+        if self.args.verbose >= 3:
+            logging.basicConfig(level=logging.DEBUG, format="%(message)s")
+
+    def parse_args(self, parser: argparse.ArgumentParser) -> argparse.Namespace:
+        """Parse command-line arguments. Override to customize parsing."""
+        return parser.parse_args()
+
+    def _build_parser(self) -> argparse.ArgumentParser:
+        """Create and configure the ArgumentParser with standard options."""
+        default_target = self.rules.select_default_target()
+        target_names = [n for n in self.rules.get_target_names()]
+
+        parser = argparse.ArgumentParser(
+            formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False
+        )
+
+        parser.add_argument(
+            "-h",
+            "--help",
+            dest="showhelp",
+            action="store_true",
+            help="show this help message and exit",
+        )
+
+        parser.add_argument(
+            "-C",
+            "--directory",
+            dest="directory",
+            help="Change to DIRECTORY before performing any operations",
+        )
+
+        parser.add_argument(
+            "-w",
+            "--workers",
+            type=int,
+            default=1,
+            help="Allow up to N workers to run simultaneously (default: 1)",
+        )
+
+        parser.add_argument(
+            "-r", "--rebuild", dest="rebuild", action="store_true", help="Rebuild all"
+        )
+
+        parser.add_argument(
+            "-l", "--list-targets", dest="list", action="store_true", help="List targets"
+        )
+
+        parser.add_argument(
+            "-v",
+            dest="verbose",
+            action="count",
+            default=0,
+            help="Increase verbosity level (default: 0)",
+        )
+
+        parser.add_argument(
+            "-V",
+            "--version",
+            dest="version",
+            action="store_true",
+            default=False,
+            help="Show version",
+        )
+
+        targetdesc = "Build targets"
+        if target_names:
+            targetdesc += ":" + " ".join(f"[{n}]" for n in target_names)
+        if default_target:
+            targetdesc += f" (Default: {default_target})"
+
+        parser.add_argument("targets", nargs="*", help=targetdesc)
+
+        self.add_arguments(parser)
+        return parser
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        """Hook for subclasses to add custom command-line arguments."""
+        pass
+
+    def _get_version(self) -> str:
+        """Return the version string for --version output."""
+        progname = os.path.basename(sys.argv[0])
+        version = getattr(chorelib, "__version__", "unknown")
+        return f"{progname} (chorelib version {version})"
+
+    def _list_targets(self):
+        for rule in self.rules.rules:
+            print(rule.get_descr())
+
+    def _build_doc(self, module: ModuleType) -> str:
+        """Build the help text from the module docstring and rule documentation."""
+        moduledoc = (module.__doc__ or "").strip()
+        lines: list[str] = []
+        for title, doc in self.rules.get_docs():
+            text = f"{title}:"
+            if doc:
+                doc = doc.lstrip("\n")
+                doc = textwrap.indent(textwrap.dedent(doc).rstrip(), " " * 4)
+                text = f"{text}\n{doc}"
+            lines.append(text)
+        targetsdoc = ""
+        if lines:
+            targetsdoc = "\n\n".join(lines)
+
+        return "\n\n".join(s for s in [moduledoc, targetsdoc] if s)
+
+    def _get_command_module(self) -> ModuleType:
+        """Return the __main__ module (the user's build script)."""
+        return sys.modules["__main__"]
+
+    def build(self, targets: list[str]) -> None:
+        """Execute the async build for the given targets."""
+        asyncio.run(
+            run(self.rules, targets, num_workers=self.args.workers, rebuild=self.args.rebuild)
+        )
+
+    def get_default_targets(self) -> list[str]:
+        """Return the default target(s) when none are specified on the command line."""
+        default_target = self.rules.select_default_target()
+        if not default_target:
+            return []
+        return [str(default_target)]
+
+    def run(self) -> None:
+        """Main entry point: parse args, configure logging, and execute the build."""
+        if self.args.showhelp:
+            doc = self._build_doc(self._get_command_module())
+            self.parser.description = doc
+            self.parser.print_help()
+            sys.exit(0)
+            return
+
+        if self.args.version:
+            print(self._get_version())
+            sys.exit(0)
+            return
+
+        if self.args.list:
+            self._list_targets()
+            sys.exit(0)
+            return
+
+        targets: list[str] = self.args.targets
+        if not targets:
+            targets = self.get_default_targets()
+            message(f"No targets specified. Using default targets: {targets}", 2)
+            if not targets:
+                sys.exit("No default target defined.")
+                return
+
+        curdir = os.getcwd()
+        if self.args.directory:
+            os.chdir(self.args.directory)
+            logger.debug(f"Changed directory to {self.args.directory}")
+
+        try:
+            self.build(targets)
+        except Exception as e:
+            print("Error:", e, file=sys.stderr)
+            if self.args.verbose >= 3:
+                traceback.print_exc()
+            sys.exit(1)
+        finally:
+            if self.args.directory:
+                os.chdir(curdir)

chorelib/deprunner.py

@@ -0,0 +1,279 @@
+"""Asynchronous build runner with parallel execution support.
+
+Executes build targets based on the dependency graph, comparing mtimes
+to skip up-to-date targets. Builder functions run in a ThreadPoolExecutor
+for parallelism.
+"""
+
+import asyncio
+import errno
+import logging
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor
+from threading import get_ident, local
+from typing import Any
+
+from .depgraph import BuildInfo, DepGraph
+from .errors import RuleNotFoundError, TargetNotFoundError
+from .ruledef import MTimeFunc, RuleSet
+from .utils import message, to_timestamp
+
+logger = logging.getLogger(__name__)
+
+
+class Runner:
+    """Asynchronous build runner that manages building targets based on dependencies.
+
+    Coordinates the build process: resolves the dependency graph, compares
+    mtimes to determine what needs rebuilding, and executes builder functions
+    in parallel using a ThreadPoolExecutor.
+
+    Attributes:
+        rules: The RuleSet containing all rule, task, and mtime definitions.
+        graph: The DepGraph tracking registered targets.
+        build_tasks: Dict mapping target names to their asyncio Tasks.
+        skipped_targets: Set of targets that were up-to-date and skipped.
+        build_always: If True, rebuild all targets regardless of mtime.
+    """
+
+    def __init__(
+        self, rules: RuleSet, num_workers: int | None = 0, build_always: bool = False
+    ) -> None:
+        self.loop = asyncio.get_event_loop()
+        self.threadid = get_ident()
+        self.running = False
+        self.num_workers = num_workers
+        self.build_always = build_always
+
+        self.rules = rules
+        self.graph = DepGraph()
+        self.build_tasks: dict[str, asyncio.Task[Any]] = {}
+        self.skipped_targets: set[str] = set()
+
+        logger.debug(
+            f"Initializing Runner with num_workers={self.num_workers}, "
+            f"build_always={self.build_always}"
+        )
+
+        # Use a thread pool for parallel builds when multiple workers requested
+        self.executor: ThreadPoolExecutor | None = None
+        if self.num_workers is None or self.num_workers > 1:
+            self.executor = ThreadPoolExecutor(max_workers=self.num_workers)
+
+    async def _run_in_executor(self, func: Any, *args: Any, **kwargs: Any) -> Any:
+        """Run a function in the thread pool executor, or inline if single-threaded.
+
+        Sets up thread-local state so that ``schedule()`` can find the
+        active runner from within builder functions.
+        """
+
+        def wrapper() -> Any:
+            if getattr(_runner, "running", None):
+                raise RuntimeError(f"Nested calls to _run_in_executor are not allowed: {func}")
+
+            _runner.running = self
+            try:
+                logger.debug(f"Running {func} in executor")
+                return func(*args, **kwargs)
+            finally:
+                _runner.running = None
+
+        if self.executor:
+            loop = asyncio.get_running_loop()
+            return await loop.run_in_executor(self.executor, wrapper)
+        else:
+            return func(*args, **kwargs)
+
+    async def _get_mtime(self, mtimefunc: MTimeFunc, target: str) -> Any:
+        """Retrieve the mtime of a target using its mtime function.
+
+        Raises:
+            TargetNotFoundError: If the mtime function returns None
+                (target does not exist).
+        """
+        message(f"Running mtime function {mtimefunc} for target: {target}", 2)
+        ret = await self._run_in_executor(mtimefunc, target)
+        ret = to_timestamp(ret)
+        if ret is None:
+            raise TargetNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), target)
+        return ret
+
+    async def _run_build_deps(self, node: BuildInfo, build_always: bool) -> Any | None:
+        """Build dependencies of the node and return the latest timestamp among them."""
+        deps = [self._ensure_target(t, build_always) for t in node.depends]
+        needs = [self._ensure_target(t, False) for t in node.needs]
+        dep_timestamps: list[Any] = list(await asyncio.gather(*deps))
+        await asyncio.gather(*needs)
+        if dep_timestamps:
+            return max(dep_timestamps)
+        else:
+            # No dependencies
+            return None
+
+    async def _run_builder(self, node: BuildInfo) -> Any:
+        """Run the builder function for the node."""
+        logger.debug(f"Starting {node.run_builder} for `{node.target}`")
+        ts = await self._run_in_executor(node.run_builder)
+        timestamp = to_timestamp(ts)
+        if timestamp is None:
+            timestamp = time.time()
+        return timestamp
+
+    @staticmethod
+    def default_get_file_mtime(name: str) -> Any:
+        """default function to get file modification time on local filesystem."""
+        return os.path.getmtime(name)
+
+    async def _build_target(self, target: str, build_always: bool) -> Any:
+        """Build a single target and return its timestamp.
+
+        For tasks or forced rebuilds, the builder runs unconditionally.
+        For rules, the target's mtime is compared against its dependencies'
+        mtimes to decide whether a rebuild is needed.
+        """
+        message(f"Building target: {target}", 1)
+
+        if not self.running:
+            return None
+
+        node = self.graph.get(target)
+
+        # Tasks and forced rebuilds always execute the builder
+        if node and (build_always or node.is_task()):
+            await self._run_build_deps(node, build_always)
+            timestamp = await self._run_builder(node)
+            return timestamp
+
+        # Check the current mtime of the target
+        mtimefunc = self.rules.get_mtime_func(target)
+        timestamp_or_none: Any
+        try:
+            timestamp_or_none = await self._get_mtime(mtimefunc, target)
+        except (FileNotFoundError, TargetNotFoundError) as e:
+            message(f"File not found for target '{target}': {e!s}", 2)
+            if not node:
+                # No builder defined for this target
+                raise RuleNotFoundError(target) from e
+            timestamp_or_none = None
+
+        if node:
+            # Build dependencies and compare mtimes
+            dep_timestamp = await self._run_build_deps(node, build_always)
+            if (timestamp_or_none is None) or (
+                (dep_timestamp is not None) and (timestamp_or_none < dep_timestamp)
+            ):
+                # Target is missing or older than dependencies — rebuild
+                message(f"Start building '{target}'", 2)
+                timestamp_or_none = await self._run_builder(node)
+            else:
+                self.skipped_targets.add(target)
+                message(f"Skipping build for target '{target}'; up to date.", 1)
+
+        return timestamp_or_none
+
+    def _ensure_target(self, target: str, build_always: bool) -> None:
+        """Ensure that the target is built and return its timestamp."""
+        task = self.build_tasks.get(target)
+        if not task:
+            task = asyncio.create_task(self._build_target(target, build_always))
+            task.set_name(target)
+            self.build_tasks[target] = task
+        return task
+
+    def _addtargets(self, targets: list[str]) -> list[str]:
+        """Add targets to the build graph."""
+        added: list[str] = []
+        for target in targets:
+            if self.graph.addtarget(self.rules, target):
+                added.append(target)
+
+        if added:
+            self.graph.detectloop()
+
+        for target in added:
+            self._ensure_target(target, self.build_always)
+
+        return added
+
+    async def wait_until_done(self) -> None:
+        """Wait until all registered build tasks have completed."""
+        try:
+            while True:
+                tasks = [task for task in self.build_tasks.values() if not task.done()]
+                if not tasks:
+                    break
+                await asyncio.gather(*tasks)
+        except Exception:
+            self.runnning = False
+            tasks = [task for task in self.build_tasks.values()]
+            for task in tasks:
+                task.cancel()
+            raise
+
+    async def run(self, targets: list[str]) -> None:
+        """Run the build for the specified targets."""
+        self.running = True
+        _runner.running = self
+        self.skipped_targets = set()
+
+        # start building
+        self._addtargets(targets)
+        await self.wait_until_done()
+
+        for target in targets:
+            if target in self.skipped_targets:
+                message(f"{target!r} is up to date.")
+
+
+# Thread-local storage for tracking the active Runner instance.
+# Used by schedule() to find the runner from within builder functions.
+_runner = local()
+
+
+def schedule(targets: list[str]) -> None:
+    """Dynamically add new targets to the running build from within a builder.
+
+    This function can be called from a builder function to request that
+    additional targets be built. It is thread-safe and works from both
+    the event loop thread and worker threads.
+
+    Args:
+        targets: A list of target names to schedule for building.
+
+    Raises:
+        RuntimeError: If no build runner is currently active.
+    """
+    runner: Runner | None = getattr(_runner, "running", None)
+    if not runner:
+        raise RuntimeError("No build runner is running.")
+
+    if runner.threadid != get_ident():
+        # Called from a worker thread — schedule via the event loop
+        async def _schedule_targets() -> None:
+            runner._addtargets(targets)  # noqa: F841
+
+        fut = asyncio.run_coroutine_threadsafe(_schedule_targets(), runner.loop)
+        fut.result()
+    else:
+        # Called from the event loop thread directly
+        runner._addtargets(targets)
+
+
+async def run(
+    rules: RuleSet,
+    targets: list[str],
+    num_workers: int | None = 0,
+    rebuild: bool = False,
+) -> None:
+    """Run the build process for the specified targets.
+
+    Args:
+        rules: The RuleSet containing rule, task, and mtime definitions.
+        targets: A list of target names to build.
+        num_workers: Maximum number of parallel worker threads.
+            0 or 1 for single-threaded, None for unlimited.
+        rebuild: If True, rebuild all targets regardless of mtime.
+    """
+    runner = Runner(rules, num_workers=num_workers, build_always=rebuild)
+    await runner.run(targets)

chorelib/errors.py

@@ -0,0 +1,27 @@
+"""Exception classes for chorelib."""
+
+
+class RuleError(Exception):
+    """General error related to rule definitions or dependency cycles."""
+
+    pass
+
+
+class RuleNotFoundError(Exception):
+    """Raised when no rule is found to build a requested target."""
+
+    def __repr__(self) -> str:
+        return f"RuleNotFoundError: {self!s}"
+
+    def __str__(self) -> str:
+        return f"No rule found to build target '{self.args[0]}'"
+
+
+class TargetNotFoundError(Exception):
+    """Raised when a target file or resource does not exist and has no builder."""
+
+    def __repr__(self) -> str:
+        return f"TargetNotFoundError: {self!s}"
+
+    def __str__(self) -> str:
+        return f"Target '{self.args[0]}' not found"