pyinfra 3.0.dev0__py2.py3-none-any.whl → 3.0.1__py2.py3-none-any.whl

pyinfra/api/config.py

@@ -1,7 +1,14 @@
+try:
+    import importlib_metadata
+except ImportError:
+    import importlib.metadata as importlib_metadata  # type: ignore[no-redef]
 from os import path
+from typing import Iterable, Optional, Set
 
-# TODO: move to importlib.resources
-from pkg_resources import Requirement, ResolutionError, parse_version, require
+from packaging.markers import Marker
+from packaging.requirements import Requirement
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
 
 from pyinfra import __version__, state
 
@@ -10,37 +17,40 @@ from .exceptions import PyinfraError
 
 class ConfigDefaults:
     # % of hosts which have to fail for all operations to stop
-    FAIL_PERCENT = None
+    FAIL_PERCENT: Optional[int] = None
     # Seconds to timeout SSH connections
-    CONNECT_TIMEOUT = 10
-    # Temporary directory (on the remote side) to use for caching any files/downloads
-    TEMP_DIR = "/tmp"
+    CONNECT_TIMEOUT: int = 10
+    # Temporary directory (on the remote side) to use for caching any files/downloads, the default
+    # None value first tries to load the hosts' temporary directory configured via "TMPDIR" env
+    # variable, falling back to DEFAULT_TEMP_DIR if not set.
+    TEMP_DIR: Optional[str] = None
+    DEFAULT_TEMP_DIR: str = "/tmp"
     # Gevent pool size (defaults to #of target hosts)
-    PARALLEL = 0
+    PARALLEL: int = 0
     # Specify the required pyinfra version (using PEP 440 setuptools specifier)
-    REQUIRE_PYINFRA_VERSION = None
+    REQUIRE_PYINFRA_VERSION: Optional[str] = None
     # Specify any required packages (either using PEP 440 or a requirements file)
     # Note: this can also include pyinfra potentially replacing REQUIRE_PYINFRA_VERSION
-    REQUIRE_PACKAGES = None
+    REQUIRE_PACKAGES: Optional[str] = None
     # All these can be overridden inside individual operation calls:
     # Switch to this user (from ssh_user) using su before executing operations
-    SU_USER = None
-    USE_SU_LOGIN = False
-    SU_SHELL = None
-    PRESERVE_SU_ENV = False
+    SU_USER: Optional[str] = None
+    USE_SU_LOGIN: bool = False
+    SU_SHELL: bool = False
+    PRESERVE_SU_ENV: bool = False
     # Use sudo and optional user
-    SUDO = False
-    SUDO_USER = None
-    PRESERVE_SUDO_ENV = False
-    USE_SUDO_LOGIN = False
-    USE_SUDO_PASSWORD = False
+    SUDO: bool = False
+    SUDO_USER: Optional[str] = None
+    PRESERVE_SUDO_ENV: bool = False
+    USE_SUDO_LOGIN: bool = False
+    SUDO_PASSWORD: Optional[str] = None
     # Use doas and optional user
-    DOAS = False
-    DOAS_USER = None
+    DOAS: bool = False
+    DOAS_USER: Optional[str] = None
     # Only show errors but don't count as failure
-    IGNORE_ERRORS = False
+    IGNORE_ERRORS: bool = False
     # Shell to use to execute commands
-    SHELL = "sh"
+    SHELL: str = "sh"
 
 
 config_defaults = {key: value for key, value in ConfigDefaults.__dict__.items() if key.isupper()}
@@ -49,21 +59,113 @@ config_defaults = {key: value for key, value in ConfigDefaults.__dict__.items()
 def check_pyinfra_version(version: str):
     if not version:
         return
+    running_version = Version(__version__)
+    required_versions = SpecifierSet(version)
 
-    running_version = parse_version(__version__)
-    required_versions = Requirement.parse(
-        "pyinfra{0}".format(version),
-    )
-
-    if running_version not in required_versions:  # type: ignore[operator]
+    if running_version not in required_versions:
         raise PyinfraError(
-            ("pyinfra version requirement not met " "(requires {0}, running {1})").format(
-                version,
-                __version__,
-            ),
+            f"pyinfra version requirement not met (requires {version}, running {__version__})"
         )
 
 
+def _check_requirements(requirements: Iterable[str]) -> Set[Requirement]:
+    """
+    Check whether each of the given requirements and all their dependencies are
+    installed.
+
+    Or more precisely, this checks that each of the given *requirements* is
+    satisfied by some installed *distribution package*, and so on recursively
+    for each of the dependencies of those distribution packages. The terminology
+    here is as follows:
+
+    * A *distribution package* is essentially a thing that can be installed with
+      ``pip``, from an sdist or wheel or Git repo or so on.
+    * A *requirement* is the expectation that a distribution package satisfying
+      some constraint is installed.
+    * A *dependency* is a requirement specified by a distribution package (as
+      opposed to the requirements passed in to this function).
+
+    So what this function does is start from the given requirements, for each
+    one check that it is satisfied by some installed distribution package, and
+    if so recursively perform the same check on all the dependencies of that
+    distribution package. In short, it's traversing the graph of package
+    requirements. It stops whenever it finds a requirement that is not satisfied
+    (i.e. a required package that is not installed), or when it runs out of
+    requirements to check.
+
+    .. note::
+        This is basically equivalent to ``pkg_resources.require()`` except that
+        when ``require()`` succeeds, it will return the list of distribution
+        packages that satisfy the given requirements and their dependencies, and
+        when it fails, it will raise an exception. This function just returns
+        the requirements which were not satisfied instead.
+
+    :param requirements: The requirements to check for in the set of installed
+        packages (along with their dependencies).
+    :return: The set of requirements that were not satisfied, which will be
+        an empty set if all requirements (recursively) were satisfied.
+    """
+
+    # Based on pkg_resources.require() from setuptools. The implementation of
+    # hbutils.system.check_reqs() from the hbutils package was also helpful in
+    # clarifying what this is supposed to do.
+
+    reqs_to_check: Set[Requirement] = set(Requirement(r) for r in requirements)
+    reqs_satisfied: Set[Requirement] = set()
+    reqs_not_satisfied: Set[Requirement] = set()
+
+    while reqs_to_check:
+        req = reqs_to_check.pop()
+        assert req not in reqs_satisfied and req not in reqs_not_satisfied
+
+        # Check for an installed distribution package with the right name and version
+        try:
+            dist = importlib_metadata.distribution(req.name)
+        except importlib_metadata.PackageNotFoundError:
+            # No installed package with the right name
+            # This would raise a DistributionNotFound error from pkg_resources.require()
+            reqs_not_satisfied.add(req)
+            continue
+
+        if dist.version not in req.specifier:
+            # There is a distribution with the right name but wrong version
+            # This would raise a VersionConflict error from pkg_resources.require()
+            reqs_not_satisfied.add(req)
+            continue
+
+        reqs_satisfied.add(req)
+
+        # If the distribution package has dependencies of its own, go through
+        # those dependencies and for each one add it to the set to be checked if
+        # - it's unconditional (no marker)
+        # - or it's conditional and the condition is satisfied (the marker
+        #   evaluates to true) in the current environment
+        # Markers can check things like the Python version and system version
+        # etc., and/or they can check which extras of the distribution package
+        # were required. To facilitate checking extras we have to pass the extra
+        # in the environment when calling Marker.evaluate().
+        if dist.requires:
+            if req.extras:
+                extras_envs = [{"extra": extra} for extra in req.extras]
+
+                def evaluate_marker(marker: Marker) -> bool:
+                    return any(map(marker.evaluate, extras_envs))
+
+            else:
+
+                def evaluate_marker(marker: Marker) -> bool:
+                    return marker.evaluate()
+
+            for dist_req_str in dist.requires:
+                dist_req = Requirement(dist_req_str)
+                if dist_req in reqs_satisfied or dist_req in reqs_not_satisfied:
+                    continue
+                if (not dist_req.marker) or evaluate_marker(dist_req.marker):
+                    reqs_to_check.add(dist_req)
+
+    return reqs_not_satisfied
+
+
 def check_require_packages(requirements_config):
     if not requirements_config:
         return
@@ -74,14 +176,12 @@ def check_require_packages(requirements_config):
         with open(path.join(state.cwd or "", requirements_config), encoding="utf-8") as f:
             requirements = [line.split("#egg=")[-1] for line in f.read().splitlines()]
 
-    try:
-        require(requirements)
-    except ResolutionError as e:
+    requirements_not_met = _check_requirements(requirements)
+    if requirements_not_met:
         raise PyinfraError(
-            "Deploy requirements ({0}) not met: {1}".format(
-                requirements_config,
-                e,
-            ),
+            "Deploy requirements ({0}) not met: missing {1}".format(
+                requirements_config, ", ".join(str(r) for r in requirements_not_met)
+            )
         )
 
 

pyinfra/api/connectors.py

@@ -1,4 +1,7 @@
-import pkg_resources
+try:
+    from importlib_metadata import entry_points
+except ImportError:
+    from importlib.metadata import entry_points  # type: ignore[assignment]
 
 
 def _load_connector(entrypoint):
@@ -8,7 +11,7 @@ def _load_connector(entrypoint):
 def get_all_connectors():
     return {
         entrypoint.name: _load_connector(entrypoint)
-        for entrypoint in pkg_resources.iter_entry_points("pyinfra.connectors")
+        for entrypoint in entry_points(group="pyinfra.connectors")
     }
 
 

pyinfra/api/deploy.py

@@ -5,13 +5,16 @@ creation (eg pyinfra-openstack).
 """
 
 from functools import wraps
-from typing import TYPE_CHECKING, Any, Callable, Union
+from typing import TYPE_CHECKING, Any, Callable, Optional, cast
+
+from typing_extensions import ParamSpec
 
 import pyinfra
 from pyinfra import context
 from pyinfra.context import ctx_host, ctx_state
 
 from .arguments import pop_global_arguments
+from .arguments_typed import PyinfraOperation
 from .exceptions import PyinfraError
 from .host import Host
 from .util import get_call_location
@@ -48,40 +51,37 @@ def add_deploy(state: "State", deploy_func: Callable[..., Any], *args, **kwargs)
                 deploy_func(*args, **kwargs)
 
 
-def deploy(func_or_name: Union[Callable[..., Any], str], data_defaults=None, _call_location=None):
+P = ParamSpec("P")
+
+
+def deploy(name: Optional[str] = None, data_defaults=None):
     """
     Decorator that takes a deploy function (normally from a pyinfra_* package)
     and wraps any operations called inside with any deploy-wide kwargs/data.
     """
 
-    # If not decorating, return function with config attached
-    if isinstance(func_or_name, str):
-        name = func_or_name
-
-        def decorator(f):
-            f.deploy_name = name
-            if data_defaults:
-                f.deploy_data = data_defaults
-            return deploy(f, _call_location=get_call_location())
+    def decorator(func: Callable[P, Any]) -> PyinfraOperation[P]:
+        func.deploy_name = name or func.__name__  # type: ignore[attr-defined]
+        if data_defaults:
+            func.deploy_data = data_defaults  # type: ignore[attr-defined]
+        return _wrap_deploy(func)
 
-        return decorator
+    return decorator
 
-    # Actually decorate!
-    func = func_or_name
 
+def _wrap_deploy(func: Callable[P, Any]) -> PyinfraOperation[P]:
     @wraps(func)
-    def decorated_func(*args, **kwargs):
+    def decorated_func(*args: P.args, **kwargs: P.kwargs) -> Any:
         deploy_kwargs, _ = pop_global_arguments(kwargs)
 
-        # Name the deploy
-        deploy_name = getattr(func, "deploy_name", func.__name__)
         deploy_data = getattr(func, "deploy_data", None)
 
         with context.host.deploy(
-            name=deploy_name,
+            name=func.deploy_name,  # type: ignore[attr-defined]
             kwargs=deploy_kwargs,
             data=deploy_data,
         ):
             return func(*args, **kwargs)
 
-    return decorated_func
+    decorated_func._inner = func  # type: ignore[attr-defined]
+    return cast(PyinfraOperation[P], decorated_func)

pyinfra/api/exceptions.py

@@ -10,6 +10,25 @@ class ConnectError(PyinfraError):
     """
 
 
+class FactError(PyinfraError):
+    """
+    Exception raised during fact gathering staging if a fact is unable to
+    generate output/change state.
+    """
+
+
+class FactTypeError(FactError, TypeError):
+    """
+    Exception raised when a fact is passed invalid argument types.
+    """
+
+
+class FactValueError(FactError, ValueError):
+    """
+    Exception raised when a fact is passed invalid argument values.
+    """
+
+
 class OperationError(PyinfraError):
     """
     Exception raised during fact gathering staging if an operation is unable to
@@ -41,19 +60,31 @@ class InventoryError(PyinfraError):
     """
 
 
-class NoConnectorError(PyinfraError, TypeError):
+class NoConnectorError(PyinfraError, ValueError):
     """
     Raised when a requested connector is missing.
     """
 
 
-class NoHostError(PyinfraError, TypeError):
+class NoHostError(PyinfraError, KeyError):
     """
     Raised when an inventory is missing a host.
     """
 
 
-class NoGroupError(PyinfraError, TypeError):
+class NoGroupError(PyinfraError, KeyError):
+    """
+    Raised when an inventory is missing a group.
+    """
+
+
+class ConnectorDataTypeError(PyinfraError, TypeError):
+    """
+    Raised when host connector data has invalid types.
+    """
+
+
+class ArgumentTypeError(PyinfraError, TypeError):
     """
-    Raise when an inventory is missing a group.
+    Raised when global arguments are passed with invalid types.
     """

pyinfra/api/facts.py

@@ -10,10 +10,11 @@ other host B while I operate on this host A).
 
 from __future__ import annotations
 
+import inspect
 import re
 from inspect import getcallargs
 from socket import error as socket_error, timeout as timeout_error
-from typing import TYPE_CHECKING, Any, Callable, Iterable, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Generic, Iterable, Optional, Type, TypeVar, cast
 
 import click
 import gevent
@@ -21,68 +22,87 @@ from paramiko import SSHException
 
 from pyinfra import logger
 from pyinfra.api import StringCommand
-from pyinfra.api.arguments import pop_global_arguments
+from pyinfra.api.arguments import all_global_arguments, pop_global_arguments
 from pyinfra.api.util import (
     get_kwargs_str,
     log_error_or_warning,
     log_host_command_error,
-    make_hash,
     print_host_combined_output,
 )
 from pyinfra.connectors.util import CommandOutput
 from pyinfra.context import ctx_host, ctx_state
 from pyinfra.progress import progress_spinner
 
-from .arguments import get_connector_argument_keys
+from .arguments import CONNECTOR_ARGUMENT_KEYS
 
 if TYPE_CHECKING:
     from pyinfra.api.host import Host
     from pyinfra.api.state import State
 
-SUDO_REGEX = r"^sudo: unknown user:"
+SUDO_REGEX = r"^sudo: unknown user"
 SU_REGEXES = (
     r"^su: user .+ does not exist",
     r"^su: unknown login",
 )
 
 
-class FactNameMeta(type):
-    def __init__(cls, name: str, bases, attrs, **kwargs):
-        super().__init__(name, bases, attrs, **kwargs)
-        module_name = cls.__module__.replace("pyinfra.facts.", "")
-        cls.name = f"{module_name}.{cls.__name__}"
+T = TypeVar("T")
 
 
-class FactBase(metaclass=FactNameMeta):
+class FactBase(Generic[T]):
     name: str
 
     abstract: bool = True
 
-    shell_executable: Optional[str] = None
+    shell_executable: str | None = None
+
+    command: Callable[..., str | StringCommand]
 
-    requires_command: Optional[str] = None
+    def requires_command(self, *args, **kwargs) -> str | None:
+        return None
 
-    command: Union[str, Callable]
+    def __init_subclass__(cls) -> None:
+        super().__init_subclass__()
+        module_name = cls.__module__.replace("pyinfra.facts.", "")
+        cls.name = f"{module_name}.{cls.__name__}"
+
+        # Check that fact's `command` method does not inadvertently take a global
+        # argument, most commonly `name`.
+        if hasattr(cls, "command") and callable(cls.command):
+            command_args = set(inspect.signature(cls.command).parameters.keys())
+            global_args = set([name for name, _ in all_global_arguments()])
+            command_global_args = command_args & global_args
+
+            if len(command_global_args) > 0:
+                names = ", ".join(command_global_args)
+                raise TypeError(f"{cls.name}'s arguments {names} are reserved for global arguments")
 
     @staticmethod
-    def default():
+    def default() -> T:
         """
         Set the default attribute to be a type (eg list/dict).
         """
 
-    @staticmethod
-    def process(output):
-        return "\n".join(output)
+        return cast(T, None)
+
+    def process(self, output: Iterable[str]) -> T:
+        # NOTE: TypeVar does not support a default, so we have to cast this str -> T
+        return cast(T, "\n".join(output))
 
     def process_pipeline(self, args, output):
         return {arg: self.process([output[i]]) for i, arg in enumerate(args)}
 
 
-class ShortFactBase(metaclass=FactNameMeta):
-    fact: type[FactBase]
+class ShortFactBase(Generic[T]):
+    name: str
+    fact: Type[FactBase]
 
-    @staticmethod
-    def process_data(data):
+    def __init_subclass__(cls) -> None:
+        super().__init_subclass__()
+        module_name = cls.__module__.replace("pyinfra.facts.", "")
+        cls.name = f"{module_name}.{cls.__name__}"
+
+    def process_data(self, data):
         return data
 
 
@@ -98,64 +118,29 @@ def _make_command(command_attribute, host_args):
     return command_attribute
 
 
-def _get_executor_kwargs(
-    state: "State",
-    host: "Host",
-    override_kwargs: Optional[dict[str, Any]] = None,
-    override_kwarg_keys: Optional[list[str]] = None,
-):
-    if override_kwargs is None:
-        override_kwargs = {}
-    if override_kwarg_keys is None:
-        override_kwarg_keys = []
-
-    # Use the current operation global kwargs, or generate defaults
-    global_kwargs = host.current_op_global_arguments
-    if not global_kwargs:
-        global_kwargs, _ = pop_global_arguments({}, state, host)
-
-    # Apply any current op kwargs that *weren't* found in the overrides
-    override_kwargs.update(
-        {key: value for key, value in global_kwargs.items() if key not in override_kwarg_keys},
-    )
-
-    return {
-        key: value for key, value in override_kwargs.items() if key in get_connector_argument_keys()
-    }
-
-
 def _handle_fact_kwargs(state, host, cls, args, kwargs):
     args = args or []
     kwargs = kwargs or {}
 
-    # TODO: this is here to avoid popping stuff accidentally, this is horrible! Change the
-    # pop function to return the clean kwargs to avoid the indirect mutation.
-    kwargs = kwargs.copy()
+    # Start with a (shallow) copy of current operation kwargs if any
+    ctx_kwargs = (host.current_op_global_arguments or {}).copy()
+    # Update with the input kwargs (overrides)
+    ctx_kwargs.update(kwargs)
 
-    # Get the defaults *and* overrides by popping from kwargs, executor kwargs passed
-    # into get_fact override everything else (applied below).
-    override_kwargs, override_kwarg_keys = pop_global_arguments(
-        kwargs,
+    # Pop executor kwargs, pass remaining
+    global_kwargs, _ = pop_global_arguments(
+        ctx_kwargs,
         state=state,
         host=host,
-        keys_to_check=get_connector_argument_keys(),
-    )
-
-    executor_kwargs = _get_executor_kwargs(
-        state,
-        host,
-        override_kwargs=override_kwargs,  # type: ignore[arg-type]
-        override_kwarg_keys=override_kwarg_keys,
     )
 
-    fact_kwargs = {}
+    fact_kwargs = {key: value for key, value in kwargs.items() if key not in global_kwargs}
 
-    if args or kwargs:
-        assert not isinstance(cls.command, str)
+    if args or fact_kwargs:
         # Merges args & kwargs into a single kwargs dictionary
-        fact_kwargs = getcallargs(cls().command, *args, **kwargs)
+        fact_kwargs = getcallargs(cls().command, *args, **fact_kwargs)
 
-    return fact_kwargs, executor_kwargs
+    return fact_kwargs, global_kwargs
 
 
 def get_facts(state: "State", *args, **kwargs):
@@ -223,7 +208,7 @@ def _get_fact(
     fact = cls()
     name = fact.name
 
-    fact_kwargs, executor_kwargs = _handle_fact_kwargs(state, host, cls, args, kwargs)
+    fact_kwargs, global_kwargs = _handle_fact_kwargs(state, host, cls, args, kwargs)
 
     kwargs_str = get_kwargs_str(fact_kwargs)
     logger.debug(
@@ -239,15 +224,9 @@ def _get_fact(
             raise_exceptions=True,
         )
 
-    ignore_errors = (
-        host.current_op_global_arguments["_ignore_errors"]
-        if host.in_op and host.current_op_global_arguments
-        else state.config.IGNORE_ERRORS
-    )
-
     # Facts can override the shell (winrm powershell vs cmd support)
     if fact.shell_executable:
-        executor_kwargs["_shell_executable"] = fact.shell_executable
+        global_kwargs["_shell_executable"] = fact.shell_executable
 
     command = _make_command(fact.command, fact_kwargs)
     requires_command = _make_command(fact.requires_command, fact_kwargs)
@@ -266,6 +245,10 @@ def _get_fact(
     status = False
     output = CommandOutput([])
 
+    executor_kwargs = {
+        key: value for key, value in global_kwargs.items() if key in CONNECTOR_ARGUMENT_KEYS
+    }
+
     try:
         status, output = host.run_shell_command(
             command,
@@ -277,7 +260,7 @@ def _get_fact(
         log_host_command_error(
             host,
             e,
-            timeout=executor_kwargs["_timeout"],
+            timeout=global_kwargs.get("_timeout"),
         )
 
     stdout_lines, stderr_lines = output.stdout_lines, output.stderr_lines
@@ -316,24 +299,17 @@ def _get_fact(
 
         log_error_or_warning(
             host,
-            ignore_errors,
+            global_kwargs["_ignore_errors"],
             description=("could not load fact: {0} {1}").format(name, get_kwargs_str(fact_kwargs)),
         )
 
     # Check we've not failed
-    if not status and not ignore_errors and apply_failed_hosts:
+    if apply_failed_hosts and not status and not global_kwargs["_ignore_errors"]:
         state.fail_hosts({host})
 
     return data
 
 
-def _get_fact_hash(state: "State", host: "Host", cls, args, kwargs):
-    if issubclass(cls, ShortFactBase):
-        cls = cls.fact
-    fact_kwargs, executor_kwargs = _handle_fact_kwargs(state, host, cls, args, kwargs)
-    return make_hash((cls, fact_kwargs, executor_kwargs))
-
-
 def get_host_fact(
     state: "State",
     host: "Host",