pyinfra 3.0b0__py2.py3-none-any.whl → 3.0b2__py2.py3-none-any.whl

pyinfra/api/__init__.py

@@ -11,6 +11,9 @@ from .config import Config  # noqa: F401 # pragma: no cover
 from .deploy import deploy  # noqa: F401 # pragma: no cover
 from .exceptions import DeployError  # noqa: F401 # pragma: no cover
 from .exceptions import (  # noqa: F401
+    FactError,
+    FactTypeError,
+    FactValueError,
     InventoryError,
     OperationError,
     OperationTypeError,

pyinfra/api/arguments.py

@@ -6,8 +6,10 @@ from typing import (
     Callable,
     Generic,
     Iterable,
+    List,
     Mapping,
     Optional,
+    Type,
     TypeVar,
     Union,
     cast,
@@ -169,7 +171,7 @@ class MetaArguments(TypedDict):
     name: str
     _ignore_errors: bool
     _continue_on_error: bool
-    _if: Callable[[], bool]
+    _if: Union[List[Callable[[], bool]], Callable[[], bool], None]
 
 
 meta_argument_meta: dict[str, ArgumentMeta] = {
@@ -190,8 +192,8 @@ meta_argument_meta: dict[str, ArgumentMeta] = {
         default=lambda _: False,
     ),
     "_if": ArgumentMeta(
-        "Only run this operation if this function returns True",
-        default=lambda _: None,
+        "Only run this operation if these functions return True",
+        default=lambda _: [],
     ),
 }
 
@@ -227,6 +229,11 @@ class AllArguments(ConnectorArguments, MetaArguments, ExecutionArguments):
     pass
 
 
+def all_global_arguments() -> List[tuple[str, Type]]:
+    """Return all global arguments and their types."""
+    return list(get_type_hints(AllArguments).items())
+
+
 all_argument_meta: dict[str, ArgumentMeta] = {
     **auth_argument_meta,
     **shell_argument_meta,
@@ -299,12 +306,12 @@ def pop_global_arguments(
     if context.ctx_config.isset():
         config = context.config
 
-    meta_kwargs = host.current_deploy_kwargs or {}
+    meta_kwargs: dict[str, Any] = host.current_deploy_kwargs or {}  # type: ignore[assignment]
 
     arguments: dict[str, Any] = {}
     found_keys: list[str] = []
 
-    for key, type_ in get_type_hints(AllArguments).items():
+    for key, type_ in all_global_arguments():
         if keys_to_check and key not in keys_to_check:
             continue
 

pyinfra/api/arguments_typed.py

@@ -1,6 +1,16 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Callable, Generator, Generic, Iterable, Mapping, Optional, Union
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Generator,
+    Generic,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Union,
+)
 
 from typing_extensions import ParamSpec, Protocol
 
@@ -22,6 +32,11 @@ class PyinfraOperation(Generic[P], Protocol):
     def __call__(
         self,
         #
+        # op args
+        # needs to be first
+        #
+        *args: P.args,
+        #
         # ConnectorArguments
         #
         # Auth
@@ -51,7 +66,7 @@ class PyinfraOperation(Generic[P], Protocol):
         name: Optional[str] = None,
         _ignore_errors: bool = False,
         _continue_on_error: bool = False,
-        _if: Optional[Callable[[], bool]] = None,
+        _if: Union[List[Callable[[], bool]], Callable[[], bool], None] = None,
         #
         # ExecutionArguments
         #
@@ -59,9 +74,7 @@ class PyinfraOperation(Generic[P], Protocol):
         _run_once: bool = False,
         _serial: bool = False,
         #
-        # The op itself
+        # op kwargs
         #
-        *args: P.args,
         **kwargs: P.kwargs,
-    ) -> "OperationMeta":
-        ...
+    ) -> "OperationMeta": ...

pyinfra/api/command.py

@@ -1,7 +1,9 @@
+from __future__ import annotations
+
 import shlex
 from inspect import getfullargspec
 from string import Formatter
-from typing import TYPE_CHECKING, Callable, Union
+from typing import IO, TYPE_CHECKING, Callable, Union
 
 import gevent
 from typing_extensions import Unpack
@@ -143,7 +145,7 @@ class StringCommand(PyinfraCommand):
 class FileUploadCommand(PyinfraCommand):
     def __init__(
         self,
-        src: str,
+        src: str | IO,
         dest: str,
         remote_temp_filename=None,
         **kwargs: Unpack[ConnectorArguments],
@@ -173,7 +175,7 @@ class FileDownloadCommand(PyinfraCommand):
     def __init__(
         self,
         src: str,
-        dest: str,
+        dest: str | IO,
         remote_temp_filename=None,
         **kwargs: Unpack[ConnectorArguments],
     ):

pyinfra/api/config.py

@@ -1,8 +1,14 @@
+try:
+    import importlib_metadata
+except ImportError:
+    import importlib.metadata as importlib_metadata  # type: ignore[no-redef]
 from os import path
-from typing import Optional
+from typing import Iterable, Optional, Set
 
-# TODO: move to importlib.resources
-from pkg_resources import Requirement, ResolutionError, parse_version, require
+from packaging.markers import Marker
+from packaging.requirements import Requirement
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
 
 from pyinfra import __version__, state
 
@@ -53,15 +59,113 @@ config_defaults = {key: value for key, value in ConfigDefaults.__dict__.items()
 def check_pyinfra_version(version: str):
     if not version:
         return
-    running_version = parse_version(__version__)
-    required_versions = Requirement.parse("pyinfra{0}".format(version))
+    running_version = Version(__version__)
+    required_versions = SpecifierSet(version)
 
-    if running_version not in required_versions:  # type: ignore[operator]
+    if running_version not in required_versions:
         raise PyinfraError(
             f"pyinfra version requirement not met (requires {version}, running {__version__})"
         )
 
 
+def _check_requirements(requirements: Iterable[str]) -> Set[Requirement]:
+    """
+    Check whether each of the given requirements and all their dependencies are
+    installed.
+
+    Or more precisely, this checks that each of the given *requirements* is
+    satisfied by some installed *distribution package*, and so on recursively
+    for each of the dependencies of those distribution packages. The terminology
+    here is as follows:
+
+    * A *distribution package* is essentially a thing that can be installed with
+      ``pip``, from an sdist or wheel or Git repo or so on.
+    * A *requirement* is the expectation that a distribution package satisfying
+      some constraint is installed.
+    * A *dependency* is a requirement specified by a distribution package (as
+      opposed to the requirements passed in to this function).
+
+    So what this function does is start from the given requirements, for each
+    one check that it is satisfied by some installed distribution package, and
+    if so recursively perform the same check on all the dependencies of that
+    distribution package. In short, it's traversing the graph of package
+    requirements. It stops whenever it finds a requirement that is not satisfied
+    (i.e. a required package that is not installed), or when it runs out of
+    requirements to check.
+
+    .. note::
+        This is basically equivalent to ``pkg_resources.require()`` except that
+        when ``require()`` succeeds, it will return the list of distribution
+        packages that satisfy the given requirements and their dependencies, and
+        when it fails, it will raise an exception. This function just returns
+        the requirements which were not satisfied instead.
+
+    :param requirements: The requirements to check for in the set of installed
+        packages (along with their dependencies).
+    :return: The set of requirements that were not satisfied, which will be
+        an empty set if all requirements (recursively) were satisfied.
+    """
+
+    # Based on pkg_resources.require() from setuptools. The implementation of
+    # hbutils.system.check_reqs() from the hbutils package was also helpful in
+    # clarifying what this is supposed to do.
+
+    reqs_to_check: Set[Requirement] = set(Requirement(r) for r in requirements)
+    reqs_satisfied: Set[Requirement] = set()
+    reqs_not_satisfied: Set[Requirement] = set()
+
+    while reqs_to_check:
+        req = reqs_to_check.pop()
+        assert req not in reqs_satisfied and req not in reqs_not_satisfied
+
+        # Check for an installed distribution package with the right name and version
+        try:
+            dist = importlib_metadata.distribution(req.name)
+        except importlib_metadata.PackageNotFoundError:
+            # No installed package with the right name
+            # This would raise a DistributionNotFound error from pkg_resources.require()
+            reqs_not_satisfied.add(req)
+            continue
+
+        if dist.version not in req.specifier:
+            # There is a distribution with the right name but wrong version
+            # This would raise a VersionConflict error from pkg_resources.require()
+            reqs_not_satisfied.add(req)
+            continue
+
+        reqs_satisfied.add(req)
+
+        # If the distribution package has dependencies of its own, go through
+        # those dependencies and for each one add it to the set to be checked if
+        # - it's unconditional (no marker)
+        # - or it's conditional and the condition is satisfied (the marker
+        #   evaluates to true) in the current environment
+        # Markers can check things like the Python version and system version
+        # etc., and/or they can check which extras of the distribution package
+        # were required. To facilitate checking extras we have to pass the extra
+        # in the environment when calling Marker.evaluate().
+        if dist.requires:
+            if req.extras:
+                extras_envs = [{"extra": extra} for extra in req.extras]
+
+                def evaluate_marker(marker: Marker) -> bool:
+                    return any(map(marker.evaluate, extras_envs))
+
+            else:
+
+                def evaluate_marker(marker: Marker) -> bool:
+                    return marker.evaluate()
+
+            for dist_req_str in dist.requires:
+                dist_req = Requirement(dist_req_str)
+                if dist_req in reqs_satisfied or dist_req in reqs_not_satisfied:
+                    continue
+                if (not dist_req.marker) or evaluate_marker(dist_req.marker):
+                    reqs_to_check.add(dist_req)
+
+    return reqs_not_satisfied
+
+
 def check_require_packages(requirements_config):
     if not requirements_config:
         return
@@ -72,14 +176,12 @@ def check_require_packages(requirements_config):
         with open(path.join(state.cwd or "", requirements_config), encoding="utf-8") as f:
             requirements = [line.split("#egg=")[-1] for line in f.read().splitlines()]
 
-    try:
-        require(requirements)
-    except ResolutionError as e:
+    requirements_not_met = _check_requirements(requirements)
+    if requirements_not_met:
         raise PyinfraError(
-            "Deploy requirements ({0}) not met: {1}".format(
-                requirements_config,
-                e,
-            ),
+            "Deploy requirements ({0}) not met: missing {1}".format(
+                requirements_config, ", ".join(str(r) for r in requirements_not_met)
+            )
         )
 
 

pyinfra/api/connectors.py

@@ -1,4 +1,7 @@
-import pkg_resources
+try:
+    from importlib_metadata import entry_points
+except ImportError:
+    from importlib.metadata import entry_points  # type: ignore[assignment]
 
 
 def _load_connector(entrypoint):
@@ -8,7 +11,7 @@ def _load_connector(entrypoint):
 def get_all_connectors():
     return {
         entrypoint.name: _load_connector(entrypoint)
-        for entrypoint in pkg_resources.iter_entry_points("pyinfra.connectors")
+        for entrypoint in entry_points(group="pyinfra.connectors")
     }
 
 

pyinfra/api/exceptions.py

@@ -10,6 +10,25 @@ class ConnectError(PyinfraError):
     """
 
 
+class FactError(PyinfraError):
+    """
+    Exception raised during fact gathering staging if a fact is unable to
+    generate output/change state.
+    """
+
+
+class FactTypeError(FactError, TypeError):
+    """
+    Exception raised when a fact is passed invalid argument types.
+    """
+
+
+class FactValueError(FactError, ValueError):
+    """
+    Exception raised when a fact is passed invalid argument values.
+    """
+
+
 class OperationError(PyinfraError):
     """
     Exception raised during fact gathering staging if an operation is unable to

pyinfra/api/facts.py

@@ -10,6 +10,7 @@ other host B while I operate on this host A).
 
 from __future__ import annotations
 
+import inspect
 import re
 from inspect import getcallargs
 from socket import error as socket_error, timeout as timeout_error
@@ -32,7 +33,7 @@ from paramiko import SSHException
 
 from pyinfra import logger
 from pyinfra.api import StringCommand
-from pyinfra.api.arguments import pop_global_arguments
+from pyinfra.api.arguments import all_global_arguments, pop_global_arguments
 from pyinfra.api.util import (
     get_kwargs_str,
     log_error_or_warning,
@@ -50,7 +51,7 @@ if TYPE_CHECKING:
     from pyinfra.api.host import Host
     from pyinfra.api.state import State
 
-SUDO_REGEX = r"^sudo: unknown user:"
+SUDO_REGEX = r"^sudo: unknown user"
 SU_REGEXES = (
     r"^su: user .+ does not exist",
     r"^su: unknown login",
@@ -76,6 +77,17 @@ class FactBase(Generic[T]):
         module_name = cls.__module__.replace("pyinfra.facts.", "")
         cls.name = f"{module_name}.{cls.__name__}"
 
+        # Check that fact's `command` method does not inadvertently take a global
+        # argument, most commonly `name`.
+        if hasattr(cls, "command") and callable(cls.command):
+            command_args = set(inspect.signature(cls.command).parameters.keys())
+            global_args = set([name for name, _ in all_global_arguments()])
+            command_global_args = command_args & global_args
+
+            if len(command_global_args) > 0:
+                names = ", ".join(command_global_args)
+                raise TypeError(f"{cls.name}'s arguments {names} are reserved for global arguments")
+
     @staticmethod
     def default() -> T:
         """
@@ -146,34 +158,25 @@ def _handle_fact_kwargs(state, host, cls, args, kwargs):
     args = args or []
     kwargs = kwargs or {}
 
-    # TODO: this is here to avoid popping stuff accidentally, this is horrible! Change the
-    # pop function to return the clean kwargs to avoid the indirect mutation.
-    kwargs = kwargs.copy()
+    # Start with a (shallow) copy of current operation kwargs if any
+    ctx_kwargs = (host.current_op_global_arguments or {}).copy()
+    # Update with the input kwargs (overrides)
+    ctx_kwargs.update(kwargs)
 
-    # Get the defaults *and* overrides by popping from kwargs, executor kwargs passed
-    # into get_fact override everything else (applied below).
-    override_kwargs, override_kwarg_keys = pop_global_arguments(
-        kwargs,
+    # Pop executor kwargs, pass remaining
+    global_kwargs, _ = pop_global_arguments(
+        ctx_kwargs,
         state=state,
         host=host,
-        keys_to_check=CONNECTOR_ARGUMENT_KEYS,
-    )
-
-    executor_kwargs = _get_executor_kwargs(
-        state,
-        host,
-        override_kwargs=override_kwargs,  # type: ignore[arg-type]
-        override_kwarg_keys=override_kwarg_keys,
     )
 
-    fact_kwargs = {}
+    fact_kwargs = {key: value for key, value in kwargs.items() if key not in global_kwargs}
 
-    if args or kwargs:
-        assert not isinstance(cls.command, str)
+    if args or fact_kwargs:
         # Merges args & kwargs into a single kwargs dictionary
-        fact_kwargs = getcallargs(cls().command, *args, **kwargs)
+        fact_kwargs = getcallargs(cls().command, *args, **fact_kwargs)
 
-    return fact_kwargs, executor_kwargs
+    return fact_kwargs, global_kwargs
 
 
 def get_facts(state: "State", *args, **kwargs):
@@ -241,7 +244,7 @@ def _get_fact(
     fact = cls()
     name = fact.name
 
-    fact_kwargs, executor_kwargs = _handle_fact_kwargs(state, host, cls, args, kwargs)
+    fact_kwargs, global_kwargs = _handle_fact_kwargs(state, host, cls, args, kwargs)
 
     kwargs_str = get_kwargs_str(fact_kwargs)
     logger.debug(
@@ -257,15 +260,9 @@ def _get_fact(
             raise_exceptions=True,
         )
 
-    ignore_errors = (
-        host.current_op_global_arguments["_ignore_errors"]
-        if host.in_op and host.current_op_global_arguments
-        else state.config.IGNORE_ERRORS
-    )
-
     # Facts can override the shell (winrm powershell vs cmd support)
     if fact.shell_executable:
-        executor_kwargs["_shell_executable"] = fact.shell_executable
+        global_kwargs["_shell_executable"] = fact.shell_executable
 
     command = _make_command(fact.command, fact_kwargs)
     requires_command = _make_command(fact.requires_command, fact_kwargs)
@@ -284,6 +281,10 @@ def _get_fact(
     status = False
     output = CommandOutput([])
 
+    executor_kwargs = {
+        key: value for key, value in global_kwargs.items() if key in CONNECTOR_ARGUMENT_KEYS
+    }
+
     try:
         status, output = host.run_shell_command(
             command,
@@ -295,7 +296,7 @@ def _get_fact(
         log_host_command_error(
             host,
             e,
-            timeout=executor_kwargs["_timeout"],
+            timeout=global_kwargs["_timeout"],
         )
 
     stdout_lines, stderr_lines = output.stdout_lines, output.stderr_lines
@@ -334,12 +335,12 @@ def _get_fact(
 
         log_error_or_warning(
             host,
-            ignore_errors,
+            global_kwargs["_ignore_errors"],
             description=("could not load fact: {0} {1}").format(name, get_kwargs_str(fact_kwargs)),
         )
 
     # Check we've not failed
-    if not status and not ignore_errors and apply_failed_hosts:
+    if apply_failed_hosts and not status and not global_kwargs["_ignore_errors"]:
         state.fail_hosts({host})
 
     return data

pyinfra/api/host.py

@@ -1,10 +1,22 @@
 from __future__ import annotations
 
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Any, Callable, Generator, Optional, Type, TypeVar, Union, overload
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Generator,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+    cast,
+    overload,
+)
 from uuid import uuid4
 
 import click
+from typing_extensions import Unpack
 
 from pyinfra import logger
 from pyinfra.connectors.base import BaseConnector
@@ -21,7 +33,9 @@ if TYPE_CHECKING:
     from pyinfra.api.state import State
 
 
-def extract_callable_datas(datas: list[Union[Callable[..., Any], Any]]) -> Generator[Any, Any, Any]:
+def extract_callable_datas(
+    datas: list[Union[Callable[..., Any], Any]],
+) -> Generator[Any, Any, Any]:
     for data in datas:
         # Support for dynamic data, ie @deploy wrapped data defaults where
         # the data is stored on the state temporarily.
@@ -98,11 +112,16 @@ class Host:
     current_op_hash: Optional[str] = None
     current_op_global_arguments: Optional["AllArguments"] = None
 
-    # Current context inside a @deploy function (op gen stage)
+    # Current context inside a @deploy function which become part of the op data
     in_deploy: bool = False
     current_deploy_name: Optional[str] = None
     current_deploy_kwargs = None
-    current_deploy_data = None
+
+    # @deploy decorator data is a bit different - we need to handle the case
+    # where we're evaluating an operation at runtime (current_op_) but also
+    # when ordering operations (current_) outside of an operation context.
+    current_op_deploy_data: Optional[dict[str, Any]] = None
+    current_deploy_data: Optional[dict[str, Any]] = None
 
     # Current context during operation execution
     executing_op_hash: Optional[str] = None
@@ -204,9 +223,7 @@ class Host:
         self.log(message_styled, log_func=log_func)
 
     def get_deploy_data(self):
-        if self.current_deploy_data:
-            return self.current_deploy_data
-        return {}
+        return self.current_op_deploy_data or self.current_deploy_data or {}
 
     def noop(self, description):
         """
@@ -216,8 +233,25 @@ class Host:
         handler = logger.info if self.state.print_noop_info else logger.debug
         handler("{0}noop: {1}".format(self.print_prefix, description))
 
+    def when(self, condition: Callable[[], bool]):
+        return self.deploy(
+            "",
+            cast("AllArguments", {"_if": [condition]}),
+            {},
+            in_deploy=False,
+        )
+
+    def arguments(self, **arguments: Unpack["AllArguments"]):
+        return self.deploy("", arguments, {}, in_deploy=False)
+
     @contextmanager
-    def deploy(self, name: str, kwargs, data, in_deploy: bool = True):
+    def deploy(
+        self,
+        name: str,
+        kwargs: Optional["AllArguments"],
+        data: Optional[dict],
+        in_deploy: bool = True,
+    ):
         """
         Wraps a group of operations as a deploy, this should not be used
         directly, instead use ``pyinfra.api.deploy.deploy``.
@@ -234,6 +268,13 @@ class Host:
         old_deploy_data = self.current_deploy_data
         self.in_deploy = in_deploy
 
+        # Combine any old _ifs with the new ones
+        if old_deploy_kwargs and kwargs:
+            old_ifs = old_deploy_kwargs["_if"]
+            new_ifs = kwargs["_if"]
+            if old_ifs and new_ifs:
+                kwargs["_if"] = old_ifs + new_ifs
+
         # Set the new values
         self.current_deploy_name = name
         self.current_deploy_kwargs = kwargs
@@ -297,12 +338,10 @@ class Host:
     T = TypeVar("T")
 
     @overload
-    def get_fact(self, name_or_cls: Type[FactBase[T]], *args, **kwargs) -> T:
-        ...
+    def get_fact(self, name_or_cls: Type[FactBase[T]], *args, **kwargs) -> T: ...
 
     @overload
-    def get_fact(self, name_or_cls: Type[ShortFactBase[T]], *args, **kwargs) -> T:
-        ...
+    def get_fact(self, name_or_cls: Type[ShortFactBase[T]], *args, **kwargs) -> T: ...
 
     def get_fact(self, name_or_cls, *args, **kwargs):
         """

pyinfra/api/inventory.py

@@ -37,6 +37,10 @@ class Inventory:
 
     state: "State"
 
+    @staticmethod
+    def empty():
+        return Inventory(([], {}))
+
     def __init__(self, names_data, override_data=None, **groups):
         # Setup basics
         self.groups = defaultdict(list)  # lists of Host objects