invoke-toolkit 0.0.64__tar.gz → 0.0.65__tar.gz

{invoke_toolkit-0.0.64 → invoke_toolkit-0.0.65}/.gitignore

@@ -19,3 +19,6 @@ invoke.yaml
 *.md
 !docs/**/*.md
 .worktrees/
+
+# Ship the local_tasks discovery example (local_tasks.py is globally ignored)
+!tests/examples/local_tasks/local_tasks.py

{invoke_toolkit-0.0.64 → invoke_toolkit-0.0.65}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: invoke-toolkit
-Version: 0.0.64
+Version: 0.0.65
 Summary: Extended APIs for PyInvoke: composable task plugins, typed CLI args, Rich output, shell completion, and PEP 723 script support
 Project-URL: Documentation, https://github.com/D3f0/invoke-toolkit#readme
 Project-URL: Issues, https://github.com/D3f0/invoke-toolkit/issues

{invoke_toolkit-0.0.64 → invoke_toolkit-0.0.65}/src/invoke_toolkit/collections.py

@@ -1,6 +1,6 @@
 """Extended collection with package inspection"""
 
-import importlib
+import hashlib
 import importlib.util
 import pkgutil
 import sys
@@ -257,12 +257,16 @@ class ToolkitCollection(Collection):
             sys.path.insert(0, search_path_str)
 
         try:
-            spec = importlib.util.spec_from_file_location(
-                "local_tasks", local_tasks_file
+            module_name = (
+                "_invoke_toolkit_local_tasks_"
+                + hashlib.sha256(
+                    str(local_tasks_file.resolve()).encode("utf-8")
+                ).hexdigest()
             )
+            spec = importlib.util.spec_from_file_location(module_name, local_tasks_file)
             if spec and spec.loader:
                 local_tasks_module = importlib.util.module_from_spec(spec)
-                sys.modules["local_tasks"] = local_tasks_module
+                sys.modules[module_name] = local_tasks_module
                 spec.loader.exec_module(local_tasks_module)
 
                 # Create a collection from the local_tasks module

invoke_toolkit-0.0.65/src/invoke_toolkit/context/async_tools.py

@@ -0,0 +1,198 @@
+"""Async execution helpers for :class:`ToolkitContext`."""
+
+from __future__ import annotations
+
+import asyncio
+import asyncio.subprocess as async_subprocess
+import contextvars
+import os
+import sys
+from contextlib import contextmanager
+from typing import Any, Awaitable, Iterator
+
+from invoke.exceptions import CommandTimedOut, UnexpectedExit
+from invoke.runners import Result, default_encoding, normalize_hide
+
+
+_async_task_context: contextvars.ContextVar[bool] = contextvars.ContextVar(
+    "invoke_toolkit_async_task_context", default=False
+)
+
+
+@contextmanager
+def async_task_context() -> Iterator[None]:
+    """Mark synchronous context methods called from an async task."""
+    token = _async_task_context.set(True)
+    try:
+        yield
+    finally:
+        _async_task_context.reset(token)
+
+
+def in_async_task_context() -> bool:
+    """Return whether the current execution is an invoke-toolkit async task."""
+    return _async_task_context.get()
+
+
+class AsyncGatherScope:
+    """Schedule awaitables and await them together when the scope exits.
+
+    Submitted awaitables start immediately. ``results`` contains their return
+    values in submission order after a successful scope exit.
+    """
+
+    def __init__(self) -> None:
+        self._tasks: list[asyncio.Future[Any]] = []
+        self.results: tuple[Any, ...] = ()
+
+    async def __aenter__(self) -> "AsyncGatherScope":
+        return self
+
+    def __call__(self, awaitable: Awaitable[Any]) -> asyncio.Future[Any]:
+        """Schedule *awaitable* and return its task handle."""
+        task = asyncio.ensure_future(awaitable)
+        self._tasks.append(task)
+        return task
+
+    async def __aexit__(self, exc_type, exc_value, traceback) -> bool:
+        if exc_type is not None:
+            await self._cancel_remaining()
+            return False
+
+        try:
+            self.results = tuple(await asyncio.gather(*self._tasks))
+        except BaseException:
+            await self._cancel_remaining()
+            raise
+        return False
+
+    async def _cancel_remaining(self) -> None:
+        pending = [task for task in self._tasks if not task.done()]
+        for task in pending:
+            task.cancel()
+        if pending:
+            await asyncio.gather(*pending, return_exceptions=True)
+
+
+async def run_async_command(  # pylint: disable=too-many-branches,too-many-locals
+    context: Any, command: str, **kwargs: Any
+) -> Result:
+    """Run a shell command without blocking the current event loop."""
+    options = dict(context.config.run.items())
+    for key, value in kwargs.items():
+        if key not in options and key != "timeout":
+            raise TypeError(f"run_async() got an unexpected keyword argument '{key}'")
+        options[key] = value
+
+    timeout = options.pop("timeout", context.config.timeouts.command)
+    if options.get("asynchronous") or options.get("disown"):
+        raise ValueError("run_async() cannot use asynchronous or disown")
+    if options.get("pty"):
+        raise ValueError("run_async() does not support pty=True")
+    if options.get("watchers"):
+        raise ValueError("run_async() does not support stream watchers")
+    if options.get("echo_stdin"):
+        raise ValueError("run_async() does not support echo_stdin")
+    if options.get("in_stream") not in (None, False):
+        raise ValueError("run_async() only supports disabled stdin")
+
+    command = context._prefix_commands(command)  # pylint: disable=protected-access
+    shell = options.get("shell") or "bash"
+    encoding = options.get("encoding") or default_encoding()
+    hide = normalize_hide(
+        options.get("hide"), options.get("out_stream"), options.get("err_stream")
+    )
+    env_values = options.get("env") or {}
+    env = (
+        dict(env_values)
+        if options.get("replace_env")
+        else dict(os.environ, **env_values)
+    )
+
+    if options.get("echo"):
+        print(options.get("echo_format", "{command}").format(command=command))
+    if options.get("dry"):
+        return Result(
+            command=command,
+            shell=shell,
+            env=env,
+            encoding=encoding,
+            exited=0,
+            hide=hide,
+        )
+
+    process = await asyncio.create_subprocess_shell(
+        command,
+        executable=shell,
+        stdout=async_subprocess.PIPE,
+        stderr=async_subprocess.PIPE,
+        stdin=async_subprocess.DEVNULL,
+        env=env,
+    )
+    try:
+        communicate = process.communicate()
+        if timeout is None:
+            stdout, stderr = await communicate
+        else:
+            stdout, stderr = await asyncio.wait_for(communicate, timeout=timeout)
+    except asyncio.TimeoutError as exc:
+        await _terminate_process(process)
+        result = _result(command, shell, env, encoding, process, b"", b"", hide)
+        raise CommandTimedOut(result, timeout=timeout) from exc
+    except asyncio.CancelledError:
+        await _terminate_process(process)
+        raise
+
+    result = _result(command, shell, env, encoding, process, stdout, stderr, hide)
+    _write_streams(result, options)
+    if not result.ok and not options.get("warn"):
+        raise UnexpectedExit(result)
+    return result
+
+
+async def _terminate_process(process: async_subprocess.Process) -> None:
+    if process.returncode is not None:
+        return
+    process.terminate()
+    try:
+        await asyncio.wait_for(process.wait(), timeout=1)
+    except asyncio.TimeoutError:
+        process.kill()
+        await process.wait()
+
+
+def _result(
+    command: str,
+    shell: str,
+    env: dict[str, str],
+    encoding: str,
+    process: async_subprocess.Process,
+    stdout: bytes,
+    stderr: bytes,
+    hide: tuple[str, ...],
+) -> Result:
+    exited = process.returncode
+    if exited is None:
+        raise RuntimeError("Process completed without an exit code")
+    return Result(
+        stdout=stdout.decode(encoding, errors="replace"),
+        stderr=stderr.decode(encoding, errors="replace"),
+        command=command,
+        shell=shell,
+        env=env,
+        encoding=encoding,
+        exited=exited,
+        hide=hide,
+        pid=process.pid,
+    )
+
+
+def _write_streams(result: Result, options: dict[str, Any]) -> None:
+    out_stream = options.get("out_stream") or sys.stdout
+    err_stream = options.get("err_stream") or sys.stderr
+    if "stdout" not in result.hide and out_stream is not False:
+        out_stream.write(result.stdout)
+        out_stream.flush()
+    if "stderr" not in result.hide and err_stream is not False:
+        err_stream.write(result.stderr)
+        err_stream.flush()

{invoke_toolkit-0.0.64 → invoke_toolkit-0.0.65}/src/invoke_toolkit/context/context.py

@@ -8,6 +8,7 @@ from os import PathLike
 from typing import (
     TYPE_CHECKING,
     Any,
+    Awaitable,
     Callable,
     Generator,
     Iterator,
@@ -22,6 +23,7 @@ from typing import (
 
 import setproctitle
 from invoke.context import Context
+from invoke.runners import Result
 from invoke.util import debug
 from rich import inspect
 
@@ -29,7 +31,8 @@ from invoke_toolkit.config import ToolkitConfig
 from invoke_toolkit.config.status_helper import StatusHelper
 from invoke_toolkit.output.console import get_console
 
-from .types import BoundPrintProtocol, ContextRunProtocol
+from .types import BoundPrintProtocol
+from .async_tools import AsyncGatherScope, in_async_task_context, run_async_command
 
 if TYPE_CHECKING:
     from rich.console import Console
@@ -58,10 +61,8 @@ class ConfigProtocol(Protocol):
 
 
 class ToolkitContext(Context, ConfigProtocol):
-    """Type annotated override"""
+    """Type annotated override with async task support."""
 
-    run: ContextRunProtocol
-    _console: "Console"
     _config: ToolkitConfig
     _status_helper: StatusHelper
 
@@ -93,8 +94,22 @@ class ToolkitContext(Context, ConfigProtocol):
     @property
     def console(self) -> "Console":
         """A console instance to do rich output"""
-        console = get_console()
-        return console
+        return get_console()
+
+    def run(self, command: str, **kwargs: Any) -> Result | Awaitable[Result]:
+        """Run a command or return an awaitable when called by an async task."""
+        if in_async_task_context() and not kwargs.get("asynchronous", False):
+            return self.run_async(command, **kwargs)
+        runner = self.config.runners.local(self)
+        return self._run(runner, command, **kwargs)
+
+    async def run_async(self, command: str, **kwargs: Any) -> Result:
+        """Run a command without blocking the current asyncio event loop."""
+        return await run_async_command(self, command, **kwargs)
+
+    def gather(self) -> AsyncGatherScope:
+        """Return an async context manager for explicit concurrent work."""
+        return AsyncGatherScope()
 
     # @contextmanager
     @property

{invoke_toolkit-0.0.64 → invoke_toolkit-0.0.65}/src/invoke_toolkit/context/types.py

@@ -3,7 +3,7 @@ Type annotations for finding out what's there in ctx.attribute
 """
 
 import sys
-from typing import IO, Any, Optional, Union
+from typing import IO, Any, Awaitable, Optional, Union
 
 from invoke.runners import Result
 from invoke.watchers import StreamWatcher
@@ -11,6 +11,12 @@ from rich.console import JustifyMethod, OverflowMethod, Style
 from typing_extensions import Annotated, Protocol
 
 
+class AsyncContextRunProtocol(Protocol):
+    """Coroutine API for native asyncio command execution."""
+
+    def __call__(self, command: str, **kwargs: Any) -> Awaitable[Result]: ...
+
+
 class BoundPrintProtocol(Protocol):
     def __call__(
         self,

{invoke_toolkit-0.0.64 → invoke_toolkit-0.0.65}/src/invoke_toolkit/executor.py

@@ -2,8 +2,10 @@
 Custom executor class to for Syntax highlighted output
 """
 
+import inspect
+from contextlib import nullcontext
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
 from invoke.executor import Executor
 from invoke.parser import ParserContext, ParseResult
@@ -13,6 +15,7 @@ from invoke.util import debug
 
 from invoke_toolkit.collections import ToolkitCollection
 from invoke_toolkit.config import ToolkitConfig
+from invoke_toolkit.context.async_tools import async_task_context
 from invoke_toolkit.output import get_console
 from invoke_toolkit.tasks.tasks import ToolkitCall, ToolkitTask
 
@@ -50,110 +53,74 @@ class ToolkitExecutor(Executor):
         self.config = config if config is not None else ToolkitConfig()
         self.core = core if core is not None else ParseResult()
 
+    async def execute_async(
+        self, *tasks: Union[str, Tuple[str, Dict[str, Any]], ParserContext]
+    ) -> Dict["ToolkitTask", Any]:
+        """Execute task calls in order, awaiting coroutine task results."""
+        calls = self.normalize(tasks)
+        direct = list(calls)
+        expanded = self.expand_calls(calls)
+        try:
+            dedupe = self.config.tasks.dedupe
+        except AttributeError:
+            dedupe = True
+        calls = self.dedupe(expanded) if dedupe else expanded
+        results: Dict["ToolkitTask", Any] = {}
+        for call in calls:
+            autoprint = call in direct and call.autoprint
+            config = self.config
+            collection_config = self.collection.configuration(call.called_as)
+            config.load_collection(collection_config)
+            config.load_shell_env()
+            context = call.make_context(config, core_parse_result=self.core)
+            args = (context, *call.args)
+            context_manager = (
+                async_task_context()
+                if inspect.iscoroutinefunction(call.task.body)
+                else nullcontext()
+            )
+            with context_manager:
+                result = call.task(*args, **call.kwargs)
+                if inspect.isawaitable(result):
+                    result = await result
+            if autoprint:
+                if isinstance(result, (str, Path)):
+                    print(result)
+                else:
+                    get_console("out").print(result)
+            results[call.task] = result  # ty: ignore[invalid-assignment]
+        return results
+
     def execute(
         self, *tasks: Union[str, Tuple[str, Dict[str, Any]], ParserContext]
     ) -> Dict["ToolkitTask", "Result"]:
-        """
-        Execute one or more ``tasks`` in sequence.
-
-        :param tasks:
-            An all-purpose iterable of "tasks to execute", each member of which
-            may take one of the following forms:
-
-            **A string** naming a task from the Executor's `.Collection`. This
-            name may contain dotted syntax appropriate for calling namespaced
-            tasks, e.g. ``subcollection.taskname``. Such tasks are executed
-            without arguments.
-
-            **A two-tuple** whose first element is a task name string (as
-            above) and whose second element is a dict suitable for use as
-            ``**kwargs`` when calling the named task. E.g.::
-
-                [
-                    ('task1', {}),
-                    ('task2', {'arg1': 'val1'}),
-                    ...
-                ]
-
-            is equivalent, roughly, to::
-
-                task1()
-                task2(arg1='val1')
-
-            **A `.ParserContext`** instance, whose ``.name`` attribute is used
-            as the task name and whose ``.as_kwargs`` attribute is used as the
-            task kwargs (again following the above specifications).
-
-            .. note::
-                When called without any arguments at all (i.e. when ``*tasks``
-                is empty), the default task from ``self.collection`` is used
-                instead, if defined.
-
-        :returns:
-            A dict mapping task objects to their return values.
-
-            This dict may include pre- and post-tasks if any were executed. For
-            example, in a collection with a ``build`` task depending on another
-            task named ``setup``, executing ``build`` will result in a dict
-            with two keys, one for ``build`` and one for ``setup``.
-
-        .. versionadded:: 1.0
-        """
+        """Execute one or more tasks in sequence."""
         # Normalize input
-        debug("Examining top level tasks {!r}".format(list(tasks)))  # pylint: disable=W1202
+        debug("Examining top level tasks {!r}".format(list(tasks)))
         calls = self.normalize(tasks)
-        debug("Tasks (now Calls) with kwargs: {!r}".format(calls))  # pylint: disable=W1202
-        # Obtain copy of directly-given tasks since they should sometimes
-        # behave differently
         direct = list(calls)
-        # Expand pre/post tasks
-        # TODO: may make sense to bundle expansion & deduping now eh?
         expanded = self.expand_calls(calls)
-        # Get some good value for dedupe option, even if config doesn't have
-        # the tree we expect. (This is a concession to testing.)
         try:
             dedupe = self.config.tasks.dedupe
         except AttributeError:
             dedupe = True
-        # Dedupe across entire run now that we know about all calls in order
         calls = self.dedupe(expanded) if dedupe else expanded
-        # Execute
         results = {}
-        # TODO: maybe clone initial config here? Probably not necessary,
-        # especially given Executor is not designed to execute() >1 time at the
-        # moment...
         for call in calls:
             autoprint = call in direct and call.autoprint
-            debug("Executing {!r}".format(call))  # pylint: disable=W1202
-            # Hand in reference to our config, which will preserve user
-            # modifications across the lifetime of the session.
             config = self.config
-            # But make sure we reset its task-sensitive levels each time
-            # (collection & shell env)
-            # TODO: load_collection needs to be skipped if task is anonymous
-            # (Fabric 2 or other subclassing libs only)
-            collection_config = self.collection.configuration(call.called_as)  # type: ignore[attr-defined]
+            collection_config = self.collection.configuration(call.called_as)
             config.load_collection(collection_config)
             config.load_shell_env()
-            debug("Finished loading collection & shell env configs")
-            # Get final context from the Call (which will know how to generate
-            # an appropriate one; e.g. subclasses might use extra data from
-            # being parameterized), handing in this config for use there.
-            context = call.make_context(config, core_parse_result=self.core)  # type: ignore[attr-defined]
-            args = (context, *call.args)  # type: ignore[attr-defined]
-            result = call.task(*args, **call.kwargs)  # type: ignore[attr-defined]
+            context = call.make_context(config, core_parse_result=self.core)
+            args = (context, *call.args)
+            result = call.task(*args, **call.kwargs)
             if autoprint:
-                # NOTE: Long strings will get wrapped when using autoprint in a console
-                #       we will use print for strings, for the case of piping output
-                #       and any non string type will be formatted by the console
                 if isinstance(result, (str, Path)):
                     print(result)
                 else:
                     get_console("out").print(result)
-                # print(result)
-            # TODO: handle the non-dedupe case / the same-task-different-args
-            # case, wherein one task obj maps to >1 result.
-            results[call.task] = result  # type: ignore[attr-defined]
+            results[call.task] = result
         return results
 
     def normalize(
@@ -220,6 +187,9 @@ class ToolkitExecutor(Executor):
         """
         ret = []
         for call in calls:
+            if isinstance(call, (list, tuple)):
+                ret.extend(self.expand_calls(cast(List[ToolkitCall], call)))
+                continue
             # Normalize to Call (this method is sometimes called with pre/post
             # task lists, which may contain 'raw' Task objects)
             if isinstance(call, Task):

{invoke_toolkit-0.0.64 → invoke_toolkit-0.0.65}/src/invoke_toolkit/program/program.py

@@ -7,15 +7,16 @@ It allows three classes to be parametrized: Loader, Config and Executor
 __all__ = ["ToolkitProgram"]
 
 import ast
+import asyncio
 import inspect
 import os
 import re
 import sys
-from importlib import metadata
+from importlib import import_module, metadata
 from importlib.util import module_from_spec
 from logging import getLogger
 from pathlib import Path
-from typing import Dict, List, Literal, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, Iterator, List, Literal, Optional, Sequence, Tuple, Union
 
 from rich.table import Table
 
@@ -57,6 +58,15 @@ from invoke_toolkit.parser import ToolkitArgument
 EMPTY_COLLECTION_NAME = "_empty"
 
 
+def _task_bodies(items: Sequence[Any]) -> Iterator[Any]:
+    """Yield task bodies, recursively flattening grouped pre/post entries."""
+    for item in items:
+        if isinstance(item, (list, tuple)):
+            yield from _task_bodies(item)
+        elif hasattr(item, "body"):
+            yield item.body
+
+
 class ToolkitProgram(Program):
     """Invoke Toolkit program providing rich output, package versioning and other features"""
 
@@ -129,13 +139,12 @@ class ToolkitProgram(Program):
             self.parse_tasks()
             # End of parsing (typically bailout stuff like --list, --help)
             self.parse_cleanup()
-            # Update the earlier Config with new values from the parse step -
-            # runtime config file contents and flag-derived overrides (e.g. for
-            # run()'s echo, warn, etc options.)
+            # Apply runtime config and CLI-derived overrides before execution.
             self.update_config()
-            # Create an Executor, passing in the data resulting from the prior
-            # steps, then tell it to execute the tasks.
-            self.execute()
+            if self._contains_async_task():
+                asyncio.run(self._execute_async())
+            else:
+                self.execute()
         except (UnexpectedExit, Exit, ParseError) as e:
             debug("Received a possibly-skippable exception: {!r}".format(e))
             # Print error messages from parser, runner, etc if necessary;
@@ -162,6 +171,31 @@ class ToolkitProgram(Program):
         except KeyboardInterrupt:
             sys.exit(1)  # Same behavior as Python itself outside of REPL
 
+    def _contains_async_task(self) -> bool:
+        """Return whether the requested task graph contains coroutine work."""
+        for parser_context in self.tasks:
+            task = self.collection[parser_context.name]
+            calls = [task, *task.pre, *task.post]
+            if any(inspect.iscoroutinefunction(body) for body in _task_bodies(calls)):
+                return True
+        default = self.collection.default
+        return bool(
+            default and inspect.iscoroutinefunction(self.collection[default].body)
+        )
+
+    async def _execute_async(self) -> None:
+        """Execute parsed tasks through the async executor path."""
+        klass = self.executor_class
+        config_path = self.config.tasks.executor_class
+        if config_path is not None:
+            module_path, _, class_name = config_path.rpartition(".")
+            module = import_module(module_path)
+            klass = getattr(module, class_name)
+        executor = klass(self.collection, self.config, self.core)
+        if not hasattr(executor, "execute_async"):
+            raise TypeError("Async tasks require an executor with execute_async()")
+        await executor.execute_async(*self.tasks)  # ty: ignore[call-non-callable]
+
     def setup_consoles(self):
         """Pre-populate the console objects"""
         patterns = self.args["redact_pattern"].value

{invoke_toolkit-0.0.64 → invoke_toolkit-0.0.65}/src/invoke_toolkit/tasks/cache.py

@@ -9,6 +9,7 @@ Provides optional caching for task results with:
 """
 
 import hashlib
+import inspect
 import subprocess
 from dataclasses import dataclass, field
 from functools import wraps
@@ -171,6 +172,37 @@ def cached_task_wrapper(
     if not DISKCACHE_AVAILABLE:
         debug(f"diskcache not installed, caching disabled for {func_name}")
         return func
+    if inspect.iscoroutinefunction(func):
+
+        @wraps(func)
+        async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+            cache = get_cache()
+            if cache is None:
+                return await func(*args, **kwargs)
+            try:
+                cache_args = args[1:] if args else ()
+                key = make_cache_key(
+                    func_name=f"{config.key_prefix}{func_name}",
+                    args=cache_args,
+                    kwargs=kwargs,
+                    ignore_args=config.ignore_args,
+                )
+                result = cache.get(key, default=None)
+                if result is not None:
+                    debug(f"Cache HIT for {func_name} (key: {key[:50]}...)")
+                    return result
+                debug(f"Cache MISS for {func_name} (key: {key[:50]}...)")
+                result = await func(*args, **kwargs)
+                if result is not None:
+                    cache.set(key, result, expire=config.ttl)
+                return result
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                debug(f"Cache error for {func_name}: {exc}, running without cache")
+                return await func(*args, **kwargs)
+            finally:
+                cache.close()
+
+        return async_wrapper  # type: ignore[return-value]
 
     @wraps(func)
     def wrapper(*args: Any, **kwargs: Any) -> Any:
@@ -197,19 +229,12 @@ def cached_task_wrapper(
                 return result
 
             debug(f"Cache MISS for {func_name} (key: {key[:50]}...)")
-
-            # Execute function and cache result
             result = func(*args, **kwargs)
-
-            # Only cache non-None results
             if result is not None:
                 cache.set(key, result, expire=config.ttl)
-                debug(f"Cached result for {func_name} (ttl: {config.ttl})")
-
             return result
-        except Exception as e:  # pylint: disable=broad-exception-caught
-            # On any cache error, just run the function
-            debug(f"Cache error for {func_name}: {e}, running without cache")
+        except Exception as exc:  # pylint: disable=broad-exception-caught
+            debug(f"Cache error for {func_name}: {exc}, running without cache")
             return func(*args, **kwargs)
         finally:
             cache.close()