pyinfra 3.0b1__py2.py3-none-any.whl → 3.0b3__py2.py3-none-any.whl

pyinfra/api/arguments.py

@@ -9,6 +9,7 @@ from typing import (
     List,
     Mapping,
     Optional,
+    Type,
     TypeVar,
     Union,
     cast,
@@ -170,7 +171,7 @@ class MetaArguments(TypedDict):
     name: str
     _ignore_errors: bool
     _continue_on_error: bool
-    _if: List[Callable[[], bool]]
+    _if: Union[List[Callable[[], bool]], Callable[[], bool], None]
 
 
 meta_argument_meta: dict[str, ArgumentMeta] = {
@@ -191,7 +192,7 @@ meta_argument_meta: dict[str, ArgumentMeta] = {
         default=lambda _: False,
     ),
     "_if": ArgumentMeta(
-        "Only run this operation if these functions returns True",
+        "Only run this operation if these functions return True",
         default=lambda _: [],
     ),
 }
@@ -228,6 +229,11 @@ class AllArguments(ConnectorArguments, MetaArguments, ExecutionArguments):
     pass
 
 
+def all_global_arguments() -> List[tuple[str, Type]]:
+    """Return all global arguments and their types."""
+    return list(get_type_hints(AllArguments).items())
+
+
 all_argument_meta: dict[str, ArgumentMeta] = {
     **auth_argument_meta,
     **shell_argument_meta,
@@ -305,7 +311,7 @@ def pop_global_arguments(
     arguments: dict[str, Any] = {}
     found_keys: list[str] = []
 
-    for key, type_ in get_type_hints(AllArguments).items():
+    for key, type_ in all_global_arguments():
         if keys_to_check and key not in keys_to_check:
             continue
 

pyinfra/api/arguments_typed.py

@@ -32,6 +32,11 @@ class PyinfraOperation(Generic[P], Protocol):
     def __call__(
         self,
         #
+        # op args
+        # needs to be first
+        #
+        *args: P.args,
+        #
         # ConnectorArguments
         #
         # Auth
@@ -61,7 +66,7 @@ class PyinfraOperation(Generic[P], Protocol):
         name: Optional[str] = None,
         _ignore_errors: bool = False,
         _continue_on_error: bool = False,
-        _if: Optional[List[Callable[[], bool]]] = None,
+        _if: Union[List[Callable[[], bool]], Callable[[], bool], None] = None,
         #
         # ExecutionArguments
         #
@@ -69,9 +74,7 @@ class PyinfraOperation(Generic[P], Protocol):
         _run_once: bool = False,
         _serial: bool = False,
         #
-        # The op itself
+        # op kwargs
         #
-        *args: P.args,
         **kwargs: P.kwargs,
-    ) -> "OperationMeta":
-        ...
+    ) -> "OperationMeta": ...

pyinfra/api/command.py

@@ -1,7 +1,9 @@
+from __future__ import annotations
+
 import shlex
 from inspect import getfullargspec
 from string import Formatter
-from typing import TYPE_CHECKING, Callable, Union
+from typing import IO, TYPE_CHECKING, Callable, Union
 
 import gevent
 from typing_extensions import Unpack
@@ -143,7 +145,7 @@ class StringCommand(PyinfraCommand):
 class FileUploadCommand(PyinfraCommand):
     def __init__(
         self,
-        src: str,
+        src: str | IO,
         dest: str,
         remote_temp_filename=None,
         **kwargs: Unpack[ConnectorArguments],
@@ -173,7 +175,7 @@ class FileDownloadCommand(PyinfraCommand):
     def __init__(
         self,
         src: str,
-        dest: str,
+        dest: str | IO,
         remote_temp_filename=None,
         **kwargs: Unpack[ConnectorArguments],
     ):

pyinfra/api/config.py

@@ -1,8 +1,14 @@
+try:
+    import importlib_metadata
+except ImportError:
+    import importlib.metadata as importlib_metadata  # type: ignore[no-redef]
 from os import path
-from typing import Optional
+from typing import Iterable, Optional, Set
 
-# TODO: move to importlib.resources
-from pkg_resources import Requirement, ResolutionError, parse_version, require
+from packaging.markers import Marker
+from packaging.requirements import Requirement
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
 
 from pyinfra import __version__, state
 
@@ -53,15 +59,113 @@ config_defaults = {key: value for key, value in ConfigDefaults.__dict__.items()
 def check_pyinfra_version(version: str):
     if not version:
         return
-    running_version = parse_version(__version__)
-    required_versions = Requirement.parse("pyinfra{0}".format(version))
+    running_version = Version(__version__)
+    required_versions = SpecifierSet(version)
 
-    if running_version not in required_versions:  # type: ignore[operator]
+    if running_version not in required_versions:
         raise PyinfraError(
             f"pyinfra version requirement not met (requires {version}, running {__version__})"
         )
 
 
+def _check_requirements(requirements: Iterable[str]) -> Set[Requirement]:
+    """
+    Check whether each of the given requirements and all their dependencies are
+    installed.
+
+    Or more precisely, this checks that each of the given *requirements* is
+    satisfied by some installed *distribution package*, and so on recursively
+    for each of the dependencies of those distribution packages. The terminology
+    here is as follows:
+
+    * A *distribution package* is essentially a thing that can be installed with
+      ``pip``, from an sdist or wheel or Git repo or so on.
+    * A *requirement* is the expectation that a distribution package satisfying
+      some constraint is installed.
+    * A *dependency* is a requirement specified by a distribution package (as
+      opposed to the requirements passed in to this function).
+
+    So what this function does is start from the given requirements, for each
+    one check that it is satisfied by some installed distribution package, and
+    if so recursively perform the same check on all the dependencies of that
+    distribution package. In short, it's traversing the graph of package
+    requirements. It stops whenever it finds a requirement that is not satisfied
+    (i.e. a required package that is not installed), or when it runs out of
+    requirements to check.
+
+    .. note::
+        This is basically equivalent to ``pkg_resources.require()`` except that
+        when ``require()`` succeeds, it will return the list of distribution
+        packages that satisfy the given requirements and their dependencies, and
+        when it fails, it will raise an exception. This function just returns
+        the requirements which were not satisfied instead.
+
+    :param requirements: The requirements to check for in the set of installed
+        packages (along with their dependencies).
+    :return: The set of requirements that were not satisfied, which will be
+        an empty set if all requirements (recursively) were satisfied.
+    """
+
+    # Based on pkg_resources.require() from setuptools. The implementation of
+    # hbutils.system.check_reqs() from the hbutils package was also helpful in
+    # clarifying what this is supposed to do.
+
+    reqs_to_check: Set[Requirement] = set(Requirement(r) for r in requirements)
+    reqs_satisfied: Set[Requirement] = set()
+    reqs_not_satisfied: Set[Requirement] = set()
+
+    while reqs_to_check:
+        req = reqs_to_check.pop()
+        assert req not in reqs_satisfied and req not in reqs_not_satisfied
+
+        # Check for an installed distribution package with the right name and version
+        try:
+            dist = importlib_metadata.distribution(req.name)
+        except importlib_metadata.PackageNotFoundError:
+            # No installed package with the right name
+            # This would raise a DistributionNotFound error from pkg_resources.require()
+            reqs_not_satisfied.add(req)
+            continue
+
+        if dist.version not in req.specifier:
+            # There is a distribution with the right name but wrong version
+            # This would raise a VersionConflict error from pkg_resources.require()
+            reqs_not_satisfied.add(req)
+            continue
+
+        reqs_satisfied.add(req)
+
+        # If the distribution package has dependencies of its own, go through
+        # those dependencies and for each one add it to the set to be checked if
+        # - it's unconditional (no marker)
+        # - or it's conditional and the condition is satisfied (the marker
+        #   evaluates to true) in the current environment
+        # Markers can check things like the Python version and system version
+        # etc., and/or they can check which extras of the distribution package
+        # were required. To facilitate checking extras we have to pass the extra
+        # in the environment when calling Marker.evaluate().
+        if dist.requires:
+            if req.extras:
+                extras_envs = [{"extra": extra} for extra in req.extras]
+
+                def evaluate_marker(marker: Marker) -> bool:
+                    return any(map(marker.evaluate, extras_envs))
+
+            else:
+
+                def evaluate_marker(marker: Marker) -> bool:
+                    return marker.evaluate()
+
+            for dist_req_str in dist.requires:
+                dist_req = Requirement(dist_req_str)
+                if dist_req in reqs_satisfied or dist_req in reqs_not_satisfied:
+                    continue
+                if (not dist_req.marker) or evaluate_marker(dist_req.marker):
+                    reqs_to_check.add(dist_req)
+
+    return reqs_not_satisfied
+
+
 def check_require_packages(requirements_config):
     if not requirements_config:
         return
@@ -72,14 +176,12 @@ def check_require_packages(requirements_config):
         with open(path.join(state.cwd or "", requirements_config), encoding="utf-8") as f:
             requirements = [line.split("#egg=")[-1] for line in f.read().splitlines()]
 
-    try:
-        require(requirements)
-    except ResolutionError as e:
+    requirements_not_met = _check_requirements(requirements)
+    if requirements_not_met:
         raise PyinfraError(
-            "Deploy requirements ({0}) not met: {1}".format(
-                requirements_config,
-                e,
-            ),
+            "Deploy requirements ({0}) not met: missing {1}".format(
+                requirements_config, ", ".join(str(r) for r in requirements_not_met)
+            )
         )
 
 

pyinfra/api/connectors.py

@@ -1,4 +1,7 @@
-import pkg_resources
+try:
+    from importlib_metadata import entry_points
+except ImportError:
+    from importlib.metadata import entry_points  # type: ignore[assignment]
 
 
 def _load_connector(entrypoint):
@@ -8,7 +11,7 @@ def _load_connector(entrypoint):
 def get_all_connectors():
     return {
         entrypoint.name: _load_connector(entrypoint)
-        for entrypoint in pkg_resources.iter_entry_points("pyinfra.connectors")
+        for entrypoint in entry_points(group="pyinfra.connectors")
     }
 
 

pyinfra/api/facts.py

@@ -10,6 +10,7 @@ other host B while I operate on this host A).
 
 from __future__ import annotations
 
+import inspect
 import re
 from inspect import getcallargs
 from socket import error as socket_error, timeout as timeout_error
@@ -32,7 +33,7 @@ from paramiko import SSHException
 
 from pyinfra import logger
 from pyinfra.api import StringCommand
-from pyinfra.api.arguments import pop_global_arguments
+from pyinfra.api.arguments import all_global_arguments, pop_global_arguments
 from pyinfra.api.util import (
     get_kwargs_str,
     log_error_or_warning,
@@ -76,6 +77,17 @@ class FactBase(Generic[T]):
         module_name = cls.__module__.replace("pyinfra.facts.", "")
         cls.name = f"{module_name}.{cls.__name__}"
 
+        # Check that fact's `command` method does not inadvertently take a global
+        # argument, most commonly `name`.
+        if hasattr(cls, "command") and callable(cls.command):
+            command_args = set(inspect.signature(cls.command).parameters.keys())
+            global_args = set([name for name, _ in all_global_arguments()])
+            command_global_args = command_args & global_args
+
+            if len(command_global_args) > 0:
+                names = ", ".join(command_global_args)
+                raise TypeError(f"{cls.name}'s arguments {names} are reserved for global arguments")
+
     @staticmethod
     def default() -> T:
         """
@@ -146,34 +158,25 @@ def _handle_fact_kwargs(state, host, cls, args, kwargs):
     args = args or []
     kwargs = kwargs or {}
 
-    # TODO: this is here to avoid popping stuff accidentally, this is horrible! Change the
-    # pop function to return the clean kwargs to avoid the indirect mutation.
-    kwargs = kwargs.copy()
+    # Start with a (shallow) copy of current operation kwargs if any
+    ctx_kwargs = (host.current_op_global_arguments or {}).copy()
+    # Update with the input kwargs (overrides)
+    ctx_kwargs.update(kwargs)
 
-    # Get the defaults *and* overrides by popping from kwargs, executor kwargs passed
-    # into get_fact override everything else (applied below).
-    override_kwargs, override_kwarg_keys = pop_global_arguments(
-        kwargs,
+    # Pop executor kwargs, pass remaining
+    global_kwargs, _ = pop_global_arguments(
+        ctx_kwargs,
         state=state,
         host=host,
-        keys_to_check=CONNECTOR_ARGUMENT_KEYS,
-    )
-
-    executor_kwargs = _get_executor_kwargs(
-        state,
-        host,
-        override_kwargs=override_kwargs,  # type: ignore[arg-type]
-        override_kwarg_keys=override_kwarg_keys,
     )
 
-    fact_kwargs = {}
+    fact_kwargs = {key: value for key, value in kwargs.items() if key not in global_kwargs}
 
-    if args or kwargs:
-        assert not isinstance(cls.command, str)
+    if args or fact_kwargs:
         # Merges args & kwargs into a single kwargs dictionary
-        fact_kwargs = getcallargs(cls().command, *args, **kwargs)
+        fact_kwargs = getcallargs(cls().command, *args, **fact_kwargs)
 
-    return fact_kwargs, executor_kwargs
+    return fact_kwargs, global_kwargs
 
 
 def get_facts(state: "State", *args, **kwargs):
@@ -241,7 +244,7 @@ def _get_fact(
     fact = cls()
     name = fact.name
 
-    fact_kwargs, executor_kwargs = _handle_fact_kwargs(state, host, cls, args, kwargs)
+    fact_kwargs, global_kwargs = _handle_fact_kwargs(state, host, cls, args, kwargs)
 
     kwargs_str = get_kwargs_str(fact_kwargs)
     logger.debug(
@@ -257,15 +260,9 @@ def _get_fact(
             raise_exceptions=True,
         )
 
-    ignore_errors = (
-        host.current_op_global_arguments["_ignore_errors"]
-        if host.in_op and host.current_op_global_arguments
-        else state.config.IGNORE_ERRORS
-    )
-
     # Facts can override the shell (winrm powershell vs cmd support)
     if fact.shell_executable:
-        executor_kwargs["_shell_executable"] = fact.shell_executable
+        global_kwargs["_shell_executable"] = fact.shell_executable
 
     command = _make_command(fact.command, fact_kwargs)
     requires_command = _make_command(fact.requires_command, fact_kwargs)
@@ -284,6 +281,10 @@ def _get_fact(
     status = False
     output = CommandOutput([])
 
+    executor_kwargs = {
+        key: value for key, value in global_kwargs.items() if key in CONNECTOR_ARGUMENT_KEYS
+    }
+
     try:
         status, output = host.run_shell_command(
             command,
@@ -295,7 +296,7 @@ def _get_fact(
         log_host_command_error(
             host,
             e,
-            timeout=executor_kwargs["_timeout"],
+            timeout=global_kwargs["_timeout"],
         )
 
     stdout_lines, stderr_lines = output.stdout_lines, output.stderr_lines
@@ -334,12 +335,12 @@ def _get_fact(
 
         log_error_or_warning(
             host,
-            ignore_errors,
+            global_kwargs["_ignore_errors"],
             description=("could not load fact: {0} {1}").format(name, get_kwargs_str(fact_kwargs)),
         )
 
     # Check we've not failed
-    if not status and not ignore_errors and apply_failed_hosts:
+    if apply_failed_hosts and not status and not global_kwargs["_ignore_errors"]:
         state.fail_hosts({host})
 
     return data

pyinfra/api/host.py

@@ -33,7 +33,9 @@ if TYPE_CHECKING:
     from pyinfra.api.state import State
 
 
-def extract_callable_datas(datas: list[Union[Callable[..., Any], Any]]) -> Generator[Any, Any, Any]:
+def extract_callable_datas(
+    datas: list[Union[Callable[..., Any], Any]],
+) -> Generator[Any, Any, Any]:
     for data in datas:
         # Support for dynamic data, ie @deploy wrapped data defaults where
         # the data is stored on the state temporarily.
@@ -336,12 +338,10 @@ class Host:
     T = TypeVar("T")
 
     @overload
-    def get_fact(self, name_or_cls: Type[FactBase[T]], *args, **kwargs) -> T:
-        ...
+    def get_fact(self, name_or_cls: Type[FactBase[T]], *args, **kwargs) -> T: ...
 
     @overload
-    def get_fact(self, name_or_cls: Type[ShortFactBase[T]], *args, **kwargs) -> T:
-        ...
+    def get_fact(self, name_or_cls: Type[ShortFactBase[T]], *args, **kwargs) -> T: ...
 
     def get_fact(self, name_or_cls, *args, **kwargs):
         """

pyinfra/api/inventory.py

@@ -37,6 +37,10 @@ class Inventory:
 
     state: "State"
 
+    @staticmethod
+    def empty():
+        return Inventory(([], {}))
+
     def __init__(self, names_data, override_data=None, **groups):
         # Setup basics
         self.groups = defaultdict(list)  # lists of Host objects

pyinfra/api/operation.py

@@ -45,7 +45,7 @@ class OperationMeta:
 
     _combined_output: Optional["CommandOutput"] = None
     _commands: Optional[list[Any]] = None
-    _maybe_is_change: Optional[bool] = False
+    _maybe_is_change: Optional[bool] = None
     _success: Optional[bool] = None
 
     def __init__(self, hash, is_change: Optional[bool]):
@@ -87,6 +87,19 @@ class OperationMeta:
         if not self.is_complete():
             raise RuntimeError("Cannot evaluate operation result before execution")
 
+    @property
+    def will_change(self) -> bool:
+        if self._maybe_is_change is not None:
+            return self._maybe_is_change
+
+        op_data = context.state.get_op_data_for_host(context.host, self._hash)
+        cmd_gen = op_data.command_generator
+        for _ in cmd_gen():
+            self._maybe_is_change = True
+            return True
+        self._maybe_is_change = False
+        return False
+
     def _did_change(self) -> bool:
         return bool(self._success and len(self._commands or []) > 0)
 
@@ -98,8 +111,9 @@ class OperationMeta:
     def did_not_change(self):
         return context.host.when(lambda: not self._did_change())
 
-    def did_succeed(self) -> bool:
-        self._raise_if_not_complete()
+    def did_succeed(self, _raise_if_not_complete=True) -> bool:
+        if _raise_if_not_complete:
+            self._raise_if_not_complete()
         return self._success is True
 
     def did_error(self) -> bool:
@@ -111,15 +125,7 @@ class OperationMeta:
     def changed(self) -> bool:
         if self.is_complete():
             return self._did_change()
-
-        if self._maybe_is_change is not None:
-            return self._maybe_is_change
-
-        op_data = context.state.get_op_data_for_host(context.host, self._hash)
-        cmd_gen = op_data.command_generator
-        for _ in cmd_gen():
-            return True
-        return False
+        return self.will_change
 
     @property
     def stdout_lines(self) -> list[str]:
@@ -257,10 +263,12 @@ def _wrap_operation(func: Callable[P, Generator], _set_in_op: bool = True) -> Py
         # *would* be made based on the *current* remote state.
 
         def command_generator() -> Iterator[PyinfraCommand]:
-            # Check global _if_ argument function and do nothing if returns False
+            # Check global _if argument function and do nothing if returns False
             if state.is_executing:
                 _ifs = global_arguments.get("_if")
-                if _ifs and not all(_if() for _if in _ifs):
+                if isinstance(_ifs, list) and not all(_if() for _if in _ifs):
+                    return
+                elif callable(_ifs) and not _ifs():
                     return
 
             host.in_op = _set_in_op

pyinfra/api/util.py

@@ -109,16 +109,16 @@ def get_caller_frameinfo(frame_offset: int = 0):
 
 
 def get_operation_order_from_stack(state: "State"):
+
     stack_items = list(reversed(stack()))
 
+    i = 0
     # Find the *first* occurrence of our deploy file in the reversed stack
     if state.current_deploy_filename:
         for i, stack_item in enumerate(stack_items):
             frame = getframeinfo(stack_item[0])
             if frame.filename == state.current_deploy_filename:
                 break
-    else:
-        i = 0
 
     # Now generate a list of line numbers *following that file*
     line_numbers = []
@@ -139,7 +139,7 @@ def get_operation_order_from_stack(state: "State"):
     return line_numbers
 
 
-def get_template(filename_or_io: str):
+def get_template(filename_or_io: str | IO):
     """
     Gets a jinja2 ``Template`` object for the input filename or string, with caching
     based on the filename of the template, or the SHA1 of the input string.
@@ -301,19 +301,27 @@ def make_hash(obj):
             if isinstance(obj, int)
             # Constants - the values can change between hosts but we should still
             # group them under the same operation hash.
-            else "_PYINFRA_CONSTANT"
-            if obj in (True, False, None)
-            # Plain strings
-            else obj
-            if isinstance(obj, str)
-            # Objects with __name__s
-            else obj.__name__
-            if hasattr(obj, "__name__")
-            # Objects with names
-            else obj.name
-            if hasattr(obj, "name")
-            # Repr anything else
-            else repr(obj)
+            else (
+                "_PYINFRA_CONSTANT"
+                if obj in (True, False, None)
+                # Plain strings
+                else (
+                    obj
+                    if isinstance(obj, str)
+                    # Objects with __name__s
+                    else (
+                        obj.__name__
+                        if hasattr(obj, "__name__")
+                        # Objects with names
+                        else (
+                            obj.name
+                            if hasattr(obj, "name")
+                            # Repr anything else
+                            else repr(obj)
+                        )
+                    )
+                )
+            )
         )
 
     return sha1_hash(hash_string)

pyinfra/connectors/base.py

@@ -108,8 +108,7 @@ class BaseConnector(abc.ABC):
         print_output: bool,
         print_input: bool,
         **arguments: Unpack["ConnectorArguments"],
-    ) -> tuple[bool, "CommandOutput"]:
-        ...
+    ) -> tuple[bool, "CommandOutput"]: ...
 
     @abc.abstractmethod
     def put_file(
@@ -120,8 +119,7 @@ class BaseConnector(abc.ABC):
         print_output: bool = False,
         print_input: bool = False,
         **arguments: Unpack["ConnectorArguments"],
-    ) -> bool:
-        ...
+    ) -> bool: ...
 
     @abc.abstractmethod
     def get_file(
@@ -132,8 +130,7 @@ class BaseConnector(abc.ABC):
         print_output: bool = False,
         print_input: bool = False,
         **arguments: Unpack["ConnectorArguments"],
-    ) -> bool:
-        ...
+    ) -> bool: ...
 
     def check_can_rsync(self):
         raise NotImplementedError("This connector does not support rsync")

pyinfra/connectors/docker.py

@@ -265,17 +265,10 @@ class DockerConnector(BaseConnector):
             )
 
             # Load the temporary file and write it to our file or IO object
-            with open(temp_filename, encoding="utf-8") as temp_f:
+            with open(temp_filename, "rb") as temp_f:
                 with get_file_io(filename_or_io, "wb") as file_io:
                     data = temp_f.read()
-                    data_bytes: bytes
-
-                    if isinstance(data, str):
-                        data_bytes = data.encode()
-                    else:
-                        data_bytes = data
-
-                    file_io.write(data_bytes)
+                    file_io.write(data)
         finally:
             os.close(fd)
             os.remove(temp_filename)

pyinfra/connectors/local.py

@@ -1,5 +1,5 @@
 import os
-from distutils.spawn import find_executable
+from shutil import which
 from tempfile import mkstemp
 from typing import TYPE_CHECKING, Tuple
 
@@ -207,7 +207,7 @@ class LocalConnector(BaseConnector):
         return True
 
     def check_can_rsync(self):
-        if not find_executable("rsync"):
+        if not which("rsync"):
             raise NotImplementedError("The `rsync` binary is not available on this system.")
 
     def rsync(