pyinfra 2.9.1__py2.py3-none-any.whl → 3.0__py2.py3-none-any.whl

pyinfra/api/command.py

@@ -1,20 +1,23 @@
+from __future__ import annotations
+
 import shlex
 from inspect import getfullargspec
 from string import Formatter
-from typing import TYPE_CHECKING
+from typing import IO, TYPE_CHECKING, Callable, Union
 
 import gevent
+from typing_extensions import Unpack
 
 from pyinfra.context import ctx_config, ctx_host
 
-from .arguments import get_executor_kwarg_keys
+from .arguments import ConnectorArguments
 
 if TYPE_CHECKING:
     from pyinfra.api.host import Host
     from pyinfra.api.state import State
 
 
-def make_formatted_string_command(string: str, *args, **kwargs):
+def make_formatted_string_command(string: str, *args, **kwargs) -> "StringCommand":
     """
     Helper function that takes a shell command or script as a string, splits it
     using ``shlex.split`` and then formats each bit, returning a ``StringCommand``
@@ -50,39 +53,46 @@ class MaskString(str):
 
 
 class QuoteString:
-    def __init__(self, obj):
-        self.object = obj
+    obj: Union[str, "StringCommand"]
 
-    def __repr__(self):
-        return "QuoteString({0})".format(self.object)
+    def __init__(self, obj: Union[str, "StringCommand"]):
+        self.obj = obj
+
+    def __repr__(self) -> str:
+        return f"QuoteString({self.obj})"
 
 
 class PyinfraCommand:
-    def __init__(self, *args, **kwargs):
-        self.executor_kwargs = {
-            key: kwargs[key] for key in get_executor_kwarg_keys() if key in kwargs
-        }
+    connector_arguments: ConnectorArguments
 
-    def __eq__(self, other):
+    def __init__(self, **arguments: Unpack[ConnectorArguments]):
+        self.connector_arguments = arguments
+
+    def __eq__(self, other) -> bool:
         if isinstance(other, self.__class__) and repr(self) == repr(other):
             return True
         return False
 
-    def execute(self, state: "State", host: "Host", executor_kwargs):
+    def execute(self, state: "State", host: "Host", connector_arguments: ConnectorArguments):
         raise NotImplementedError
 
 
 class StringCommand(PyinfraCommand):
-    def __init__(self, *bits, **kwargs):
-        super().__init__(**kwargs)
+    def __init__(
+        self,
+        *bits,
+        _separator=" ",
+        **arguments: Unpack[ConnectorArguments],
+    ):
+        super().__init__(**arguments)
         self.bits = bits
-        self.separator = kwargs.pop("_separator", " ")
+        self.separator = _separator
 
-    def __str__(self):
+    def __str__(self) -> str:
         return self.get_masked_value()
 
-    def __repr__(self):
-        return "StringCommand({0})".format(self.get_masked_value())
+    def __repr__(self) -> str:
+        return f"StringCommand({self.get_masked_value()})"
 
     def _get_all_bits(self, bit_accessor):
         all_bits = []
@@ -91,7 +101,7 @@ class StringCommand(PyinfraCommand):
             quote = False
             if isinstance(bit, QuoteString):
                 quote = True
-                bit = bit.object
+                bit = bit.obj
 
             if isinstance(bit, StringCommand):
                 bit = bit_accessor(bit)
@@ -106,14 +116,14 @@ class StringCommand(PyinfraCommand):
 
         return all_bits
 
-    def get_raw_value(self):
+    def get_raw_value(self) -> str:
         return self.separator.join(
             self._get_all_bits(
                 lambda bit: bit.get_raw_value(),
             ),
         )
 
-    def get_masked_value(self):
+    def get_masked_value(self) -> str:
         return self.separator.join(
             [
                 "***" if isinstance(bit, MaskString) else bit
@@ -121,20 +131,25 @@ class StringCommand(PyinfraCommand):
             ],
         )
 
-    def execute(self, state: "State", host: "Host", executor_kwargs):
-        executor_kwargs.update(self.executor_kwargs)
+    def execute(self, state: "State", host: "Host", connector_arguments: ConnectorArguments):
+        connector_arguments.update(self.connector_arguments)
 
         return host.run_shell_command(
             self,
             print_output=state.print_output,
             print_input=state.print_input,
-            return_combined_output=True,
-            **executor_kwargs,
+            **connector_arguments,
         )
 
 
 class FileUploadCommand(PyinfraCommand):
-    def __init__(self, src: str, dest: str, remote_temp_filename=None, **kwargs):
+    def __init__(
+        self,
+        src: str | IO,
+        dest: str,
+        remote_temp_filename=None,
+        **kwargs: Unpack[ConnectorArguments],
+    ):
         super().__init__(**kwargs)
         self.src = src
         self.dest = dest
@@ -143,8 +158,8 @@ class FileUploadCommand(PyinfraCommand):
     def __repr__(self):
         return "FileUploadCommand({0}, {1})".format(self.src, self.dest)
 
-    def execute(self, state: "State", host: "Host", executor_kwargs):
-        executor_kwargs.update(self.executor_kwargs)
+    def execute(self, state: "State", host: "Host", connector_arguments: ConnectorArguments):
+        connector_arguments.update(self.connector_arguments)
 
         return host.put_file(
             self.src,
@@ -152,12 +167,18 @@ class FileUploadCommand(PyinfraCommand):
             remote_temp_filename=self.remote_temp_filename,
             print_output=state.print_output,
             print_input=state.print_input,
-            **executor_kwargs,
+            **connector_arguments,
         )
 
 
 class FileDownloadCommand(PyinfraCommand):
-    def __init__(self, src: str, dest: str, remote_temp_filename=None, **kwargs):
+    def __init__(
+        self,
+        src: str,
+        dest: str | IO,
+        remote_temp_filename=None,
+        **kwargs: Unpack[ConnectorArguments],
+    ):
         super().__init__(**kwargs)
         self.src = src
         self.dest = dest
@@ -166,8 +187,8 @@ class FileDownloadCommand(PyinfraCommand):
     def __repr__(self):
         return "FileDownloadCommand({0}, {1})".format(self.src, self.dest)
 
-    def execute(self, state: "State", host: "Host", executor_kwargs):
-        executor_kwargs.update(self.executor_kwargs)
+    def execute(self, state: "State", host: "Host", connector_arguments: ConnectorArguments):
+        connector_arguments.update(self.connector_arguments)
 
         return host.get_file(
             self.src,
@@ -175,12 +196,18 @@ class FileDownloadCommand(PyinfraCommand):
             remote_temp_filename=self.remote_temp_filename,
             print_output=state.print_output,
             print_input=state.print_input,
-            **executor_kwargs,
+            **connector_arguments,
         )
 
 
 class FunctionCommand(PyinfraCommand):
-    def __init__(self, function, args, func_kwargs, **kwargs):
+    def __init__(
+        self,
+        function: Callable,
+        args,
+        func_kwargs,
+        **kwargs: Unpack[ConnectorArguments],
+    ):
         super().__init__(**kwargs)
         self.function = function
         self.args = args
@@ -193,34 +220,22 @@ class FunctionCommand(PyinfraCommand):
             self.kwargs,
         )
 
-    def execute(self, state: "State", host: "Host", executor_kwargs):
+    def execute(self, state: "State", host: "Host", connector_arguments: ConnectorArguments):
         argspec = getfullargspec(self.function)
         if "state" in argspec.args and "host" in argspec.args:
             return self.function(state, host, *self.args, **self.kwargs)
 
-        # Note: we use try/except here to catch any exception inside the greenlet
-        # and bubble that back out as a return value before re-raising outside.
-        # This is because gevent dumps stack traces when raised within a greenlet
-        # and we want to handle that ourselves.
         def execute_function():
             with ctx_config.use(state.config.copy()):
                 with ctx_host.use(host):
-                    try:
-                        self.function(*self.args, **self.kwargs)
-                    except Exception as e:
-                        return e
-                    else:
-                        return True
+                    self.function(*self.args, **self.kwargs)
 
         greenlet = gevent.spawn(execute_function)
-        res = greenlet.get()
-        if isinstance(res, Exception):
-            raise res
-        return res
+        return greenlet.get()
 
 
 class RsyncCommand(PyinfraCommand):
-    def __init__(self, src: str, dest: str, flags, **kwargs):
+    def __init__(self, src: str, dest: str, flags, **kwargs: Unpack[ConnectorArguments]):
         super().__init__(**kwargs)
         self.src = src
         self.dest = dest
@@ -229,12 +244,12 @@ class RsyncCommand(PyinfraCommand):
     def __repr__(self):
         return "RsyncCommand({0}, {1}, {2})".format(self.src, self.dest, self.flags)
 
-    def execute(self, state: "State", host: "Host", executor_kwargs):
+    def execute(self, state: "State", host: "Host", connector_arguments: ConnectorArguments):
         return host.rsync(
             self.src,
             self.dest,
             self.flags,
             print_output=state.print_output,
             print_input=state.print_input,
-            **executor_kwargs,
+            **connector_arguments,
         )

pyinfra/api/config.py

@@ -1,6 +1,14 @@
+try:
+    import importlib_metadata
+except ImportError:
+    import importlib.metadata as importlib_metadata  # type: ignore[no-redef]
 from os import path
+from typing import Iterable, Optional, Set
 
-from pkg_resources import Requirement, ResolutionError, parse_version, require
+from packaging.markers import Marker
+from packaging.requirements import Requirement
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
 
 from pyinfra import __version__, state
 
@@ -9,37 +17,40 @@ from .exceptions import PyinfraError
 
 class ConfigDefaults:
     # % of hosts which have to fail for all operations to stop
-    FAIL_PERCENT = None
+    FAIL_PERCENT: Optional[int] = None
     # Seconds to timeout SSH connections
-    CONNECT_TIMEOUT = 10
-    # Temporary directory (on the remote side) to use for caching any files/downloads
-    TEMP_DIR = "/tmp"
+    CONNECT_TIMEOUT: int = 10
+    # Temporary directory (on the remote side) to use for caching any files/downloads, the default
+    # None value first tries to load the hosts' temporary directory configured via "TMPDIR" env
+    # variable, falling back to DEFAULT_TEMP_DIR if not set.
+    TEMP_DIR: Optional[str] = None
+    DEFAULT_TEMP_DIR: str = "/tmp"
     # Gevent pool size (defaults to #of target hosts)
-    PARALLEL = 0
+    PARALLEL: int = 0
     # Specify the required pyinfra version (using PEP 440 setuptools specifier)
-    REQUIRE_PYINFRA_VERSION = None
+    REQUIRE_PYINFRA_VERSION: Optional[str] = None
     # Specify any required packages (either using PEP 440 or a requirements file)
     # Note: this can also include pyinfra potentially replacing REQUIRE_PYINFRA_VERSION
-    REQUIRE_PACKAGES = None
+    REQUIRE_PACKAGES: Optional[str] = None
     # All these can be overridden inside individual operation calls:
     # Switch to this user (from ssh_user) using su before executing operations
-    SU_USER = None
-    USE_SU_LOGIN = False
-    SU_SHELL = None
-    PRESERVE_SU_ENV = False
+    SU_USER: Optional[str] = None
+    USE_SU_LOGIN: bool = False
+    SU_SHELL: bool = False
+    PRESERVE_SU_ENV: bool = False
     # Use sudo and optional user
-    SUDO = False
-    SUDO_USER = None
-    PRESERVE_SUDO_ENV = False
-    USE_SUDO_LOGIN = False
-    USE_SUDO_PASSWORD = False
+    SUDO: bool = False
+    SUDO_USER: Optional[str] = None
+    PRESERVE_SUDO_ENV: bool = False
+    USE_SUDO_LOGIN: bool = False
+    SUDO_PASSWORD: Optional[str] = None
     # Use doas and optional user
-    DOAS = False
-    DOAS_USER = None
+    DOAS: bool = False
+    DOAS_USER: Optional[str] = None
     # Only show errors but don't count as failure
-    IGNORE_ERRORS = False
+    IGNORE_ERRORS: bool = False
     # Shell to use to execute commands
-    SHELL = "sh"
+    SHELL: str = "sh"
 
 
 config_defaults = {key: value for key, value in ConfigDefaults.__dict__.items() if key.isupper()}
@@ -48,15 +59,113 @@ config_defaults = {key: value for key, value in ConfigDefaults.__dict__.items()
 def check_pyinfra_version(version: str):
     if not version:
         return
-    running_version = parse_version(__version__)
-    required_versions = Requirement.parse("pyinfra{0}".format(version))
+    running_version = Version(__version__)
+    required_versions = SpecifierSet(version)
 
-    if not required_versions.specifier.contains(running_version):
+    if running_version not in required_versions:
         raise PyinfraError(
             f"pyinfra version requirement not met (requires {version}, running {__version__})"
         )
 
 
+def _check_requirements(requirements: Iterable[str]) -> Set[Requirement]:
+    """
+    Check whether each of the given requirements and all their dependencies are
+    installed.
+
+    Or more precisely, this checks that each of the given *requirements* is
+    satisfied by some installed *distribution package*, and so on recursively
+    for each of the dependencies of those distribution packages. The terminology
+    here is as follows:
+
+    * A *distribution package* is essentially a thing that can be installed with
+      ``pip``, from an sdist or wheel or Git repo or so on.
+    * A *requirement* is the expectation that a distribution package satisfying
+      some constraint is installed.
+    * A *dependency* is a requirement specified by a distribution package (as
+      opposed to the requirements passed in to this function).
+
+    So what this function does is start from the given requirements, for each
+    one check that it is satisfied by some installed distribution package, and
+    if so recursively perform the same check on all the dependencies of that
+    distribution package. In short, it's traversing the graph of package
+    requirements. It stops whenever it finds a requirement that is not satisfied
+    (i.e. a required package that is not installed), or when it runs out of
+    requirements to check.
+
+    .. note::
+        This is basically equivalent to ``pkg_resources.require()`` except that
+        when ``require()`` succeeds, it will return the list of distribution
+        packages that satisfy the given requirements and their dependencies, and
+        when it fails, it will raise an exception. This function just returns
+        the requirements which were not satisfied instead.
+
+    :param requirements: The requirements to check for in the set of installed
+        packages (along with their dependencies).
+    :return: The set of requirements that were not satisfied, which will be
+        an empty set if all requirements (recursively) were satisfied.
+    """
+
+    # Based on pkg_resources.require() from setuptools. The implementation of
+    # hbutils.system.check_reqs() from the hbutils package was also helpful in
+    # clarifying what this is supposed to do.
+
+    reqs_to_check: Set[Requirement] = set(Requirement(r) for r in requirements)
+    reqs_satisfied: Set[Requirement] = set()
+    reqs_not_satisfied: Set[Requirement] = set()
+
+    while reqs_to_check:
+        req = reqs_to_check.pop()
+        assert req not in reqs_satisfied and req not in reqs_not_satisfied
+
+        # Check for an installed distribution package with the right name and version
+        try:
+            dist = importlib_metadata.distribution(req.name)
+        except importlib_metadata.PackageNotFoundError:
+            # No installed package with the right name
+            # This would raise a DistributionNotFound error from pkg_resources.require()
+            reqs_not_satisfied.add(req)
+            continue
+
+        if dist.version not in req.specifier:
+            # There is a distribution with the right name but wrong version
+            # This would raise a VersionConflict error from pkg_resources.require()
+            reqs_not_satisfied.add(req)
+            continue
+
+        reqs_satisfied.add(req)
+
+        # If the distribution package has dependencies of its own, go through
+        # those dependencies and for each one add it to the set to be checked if
+        # - it's unconditional (no marker)
+        # - or it's conditional and the condition is satisfied (the marker
+        #   evaluates to true) in the current environment
+        # Markers can check things like the Python version and system version
+        # etc., and/or they can check which extras of the distribution package
+        # were required. To facilitate checking extras we have to pass the extra
+        # in the environment when calling Marker.evaluate().
+        if dist.requires:
+            if req.extras:
+                extras_envs = [{"extra": extra} for extra in req.extras]
+
+                def evaluate_marker(marker: Marker) -> bool:
+                    return any(map(marker.evaluate, extras_envs))
+
+            else:
+
+                def evaluate_marker(marker: Marker) -> bool:
+                    return marker.evaluate()
+
+            for dist_req_str in dist.requires:
+                dist_req = Requirement(dist_req_str)
+                if dist_req in reqs_satisfied or dist_req in reqs_not_satisfied:
+                    continue
+                if (not dist_req.marker) or evaluate_marker(dist_req.marker):
+                    reqs_to_check.add(dist_req)
+
+    return reqs_not_satisfied
+
+
 def check_require_packages(requirements_config):
     if not requirements_config:
         return
@@ -64,17 +173,15 @@ def check_require_packages(requirements_config):
     if isinstance(requirements_config, (list, tuple)):
         requirements = requirements_config
     else:
-        with open(path.join(state.cwd, requirements_config), encoding="utf-8") as f:
+        with open(path.join(state.cwd or "", requirements_config), encoding="utf-8") as f:
             requirements = [line.split("#egg=")[-1] for line in f.read().splitlines()]
 
-    try:
-        require(requirements)
-    except ResolutionError as e:
+    requirements_not_met = _check_requirements(requirements)
+    if requirements_not_met:
         raise PyinfraError(
-            "Deploy requirements ({0}) not met: {1}".format(
-                requirements_config,
-                e,
-            ),
+            "Deploy requirements ({0}) not met: missing {1}".format(
+                requirements_config, ", ".join(str(r) for r in requirements_not_met)
+            )
         )
 
 

pyinfra/api/connect.py

@@ -36,7 +36,7 @@ def connect_all(state: "State"):
         # Raise any unexpected exception
         greenlet.get()
 
-        if host.connection:
+        if host.connected:
             state.activate_host(host)
         else:
             failed_hosts.add(host)

pyinfra/api/connectors.py

@@ -1,36 +1,17 @@
-import pkg_resources
-
-
-class BaseConnectorMeta:
-    handles_execution = False
-    keys_prefix = ""
-
-    class DataKeys:
-        pass
-
-    @classmethod
-    def keys(cls):
-        class Keys:
-            pass
-
-        for key in cls.DataKeys.__dict__:
-            if not key.startswith("_"):
-                setattr(Keys, key, f"{cls.keys_prefix}_{key}")
-
-        return Keys
+try:
+    from importlib_metadata import entry_points
+except ImportError:
+    from importlib.metadata import entry_points  # type: ignore[assignment]
 
 
 def _load_connector(entrypoint):
-    connector = entrypoint.load()
-    if not getattr(connector, "Meta", None):
-        connector.Meta = BaseConnectorMeta
-    return connector
+    return entrypoint.load()
 
 
 def get_all_connectors():
     return {
         entrypoint.name: _load_connector(entrypoint)
-        for entrypoint in pkg_resources.iter_entry_points("pyinfra.connectors")
+        for entrypoint in entry_points(group="pyinfra.connectors")
     }
 
 
@@ -38,7 +19,7 @@ def get_execution_connectors():
     return {
         connector: connector_mod
         for connector, connector_mod in get_all_connectors().items()
-        if connector_mod.Meta.handles_execution
+        if connector_mod.handles_execution
     }
 
 

pyinfra/api/deploy.py

@@ -5,31 +5,24 @@ creation (eg pyinfra-openstack).
 """
 
 from functools import wraps
-from typing import TYPE_CHECKING, Any, Callable, Union
+from typing import TYPE_CHECKING, Any, Callable, Optional, cast
+
+from typing_extensions import ParamSpec
 
 import pyinfra
-from pyinfra import context, logger
+from pyinfra import context
 from pyinfra.context import ctx_host, ctx_state
 
 from .arguments import pop_global_arguments
+from .arguments_typed import PyinfraOperation
 from .exceptions import PyinfraError
 from .host import Host
-from .util import get_args_kwargs_spec, get_call_location, memoize
+from .util import get_call_location
 
 if TYPE_CHECKING:
     from pyinfra.api.state import State
 
 
-@memoize
-def show_state_host_arguments_warning(call_location):
-    logger.warning(
-        (
-            "{0}:\n\tLegacy deploy function detected! Deploys should no longer define "
-            "`state` and `host` arguments."
-        ).format(call_location),
-    )
-
-
 def add_deploy(state: "State", deploy_func: Callable[..., Any], *args, **kwargs):
     """
     Prepare & add an deploy to pyinfra.state by executing it on all hosts.
@@ -58,61 +51,37 @@ def add_deploy(state: "State", deploy_func: Callable[..., Any], *args, **kwargs)
                 deploy_func(*args, **kwargs)
 
 
-def deploy(func_or_name: Union[Callable[..., Any], str], data_defaults=None, _call_location=None):
+P = ParamSpec("P")
+
+
+def deploy(name: Optional[str] = None, data_defaults=None):
     """
     Decorator that takes a deploy function (normally from a pyinfra_* package)
     and wraps any operations called inside with any deploy-wide kwargs/data.
     """
 
-    # If not decorating, return function with config attached
-    if isinstance(func_or_name, str):
-        name = func_or_name
-
-        def decorator(f):
-            f.deploy_name = name
-            if data_defaults:
-                f.deploy_data = data_defaults
-            return deploy(f, _call_location=get_call_location())
-
-        return decorator
+    def decorator(func: Callable[P, Any]) -> PyinfraOperation[P]:
+        func.deploy_name = name or func.__name__  # type: ignore[attr-defined]
+        if data_defaults:
+            func.deploy_data = data_defaults  # type: ignore[attr-defined]
+        return _wrap_deploy(func)
 
-    # Actually decorate!
-    func = func_or_name
+    return decorator
 
-    # Check whether an operation is "legacy" - ie contains state=None, host=None kwargs
-    # TODO: remove this in v3
-    is_legacy = False
-    args, kwargs = get_args_kwargs_spec(func)
-    if all(key in kwargs and kwargs[key] is None for key in ("state", "host")):
-        show_state_host_arguments_warning(_call_location or get_call_location())
-        is_legacy = True
-    func.is_legacy = is_legacy  # type: ignore
 
+def _wrap_deploy(func: Callable[P, Any]) -> PyinfraOperation[P]:
     @wraps(func)
-    def decorated_func(*args, **kwargs):
+    def decorated_func(*args: P.args, **kwargs: P.kwargs) -> Any:
         deploy_kwargs, _ = pop_global_arguments(kwargs)
 
-        # If this is a legacy operation function (ie - state & host arg kwargs), ensure that state
-        # and host are included as kwargs.
-        if func.is_legacy:
-            if "state" not in kwargs:
-                kwargs["state"] = context.state
-            if "host" not in kwargs:
-                kwargs["host"] = context.host
-        # If not legacy, pop off any state/host kwargs that may come from legacy @deploy functions
-        else:
-            kwargs.pop("state", None)
-            kwargs.pop("host", None)
-
-        # Name the deploy
-        deploy_name = getattr(func, "deploy_name", func.__name__)
         deploy_data = getattr(func, "deploy_data", None)
 
         with context.host.deploy(
-            name=deploy_name,
+            name=func.deploy_name,  # type: ignore[attr-defined]
             kwargs=deploy_kwargs,
             data=deploy_data,
         ):
             return func(*args, **kwargs)
 
-    return decorated_func
+    decorated_func._inner = func  # type: ignore[attr-defined]
+    return cast(PyinfraOperation[P], decorated_func)