pyinfra 3.0b0__tar.gz → 3.0b2__tar.gz

{pyinfra-3.0b0 → pyinfra-3.0b2}/CHANGELOG.md

@@ -1,4 +1,4 @@
-# v3.0.beta0
+# v3.0.beta2
 
 Welcome to pyinfra v3! This version is the biggest overhaul of pyinfra since it was created back in 2015. Most v2 deployment code should be automatically compatible, but as always be aware. Major changes:
 

{pyinfra-3.0b0/pyinfra.egg-info → pyinfra-3.0b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyinfra
-Version: 3.0b0
+Version: 3.0b2
 Summary: pyinfra automates/provisions/manages/deploys infrastructure.
 Home-page: https://pyinfra.com
 Author: Nick / Fizzadar
@@ -16,18 +16,75 @@ Classifier: Intended Audience :: Information Technology
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Systems Administration
 Classifier: Topic :: System :: Installation/Setup
 Classifier: Topic :: Utilities
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: gevent>=1.5
+Requires-Dist: paramiko<4,>=2.7
+Requires-Dist: click>2
+Requires-Dist: jinja2<4,>2
+Requires-Dist: python-dateutil<3,>2
+Requires-Dist: setuptools
+Requires-Dist: configparser
+Requires-Dist: pywinrm
+Requires-Dist: typeguard
+Requires-Dist: distro<2,>=1.6
+Requires-Dist: packaging>=16.1
+Requires-Dist: graphlib_backport; python_version < "3.9"
+Requires-Dist: typing-extensions; python_version < "3.11"
+Requires-Dist: importlib_metadata>=3.6; python_version < "3.10"
 Provides-Extra: test
+Requires-Dist: pytest==8.2.1; extra == "test"
+Requires-Dist: coverage==7.5.1; extra == "test"
+Requires-Dist: pytest-cov==5.0.0; extra == "test"
+Requires-Dist: black==24.4.2; extra == "test"
+Requires-Dist: isort==5.13.2; extra == "test"
+Requires-Dist: flake8==7.0.0; extra == "test"
+Requires-Dist: flake8-black==0.3.6; extra == "test"
+Requires-Dist: flake8-isort==6.1.1; extra == "test"
+Requires-Dist: mypy; extra == "test"
+Requires-Dist: types-cryptography; extra == "test"
+Requires-Dist: types-paramiko; extra == "test"
+Requires-Dist: types-python-dateutil; extra == "test"
+Requires-Dist: types-PyYAML; extra == "test"
+Requires-Dist: types-setuptools; extra == "test"
 Provides-Extra: docs
+Requires-Dist: pyinfra-guzzle_sphinx_theme==0.15; extra == "docs"
+Requires-Dist: myst-parser==2.0.0; extra == "docs"
+Requires-Dist: sphinx==6.2.1; extra == "docs"
 Provides-Extra: dev
-License-File: LICENSE.md
+Requires-Dist: pytest==8.2.1; extra == "dev"
+Requires-Dist: coverage==7.5.1; extra == "dev"
+Requires-Dist: pytest-cov==5.0.0; extra == "dev"
+Requires-Dist: black==24.4.2; extra == "dev"
+Requires-Dist: isort==5.13.2; extra == "dev"
+Requires-Dist: flake8==7.0.0; extra == "dev"
+Requires-Dist: flake8-black==0.3.6; extra == "dev"
+Requires-Dist: flake8-isort==6.1.1; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: types-cryptography; extra == "dev"
+Requires-Dist: types-paramiko; extra == "dev"
+Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
+Requires-Dist: pyinfra-guzzle_sphinx_theme==0.15; extra == "dev"
+Requires-Dist: myst-parser==2.0.0; extra == "dev"
+Requires-Dist: sphinx==6.2.1; extra == "dev"
+Requires-Dist: wheel; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: ipython; extra == "dev"
+Requires-Dist: ipdb; extra == "dev"
+Requires-Dist: ipdbplugin; extra == "dev"
+Requires-Dist: flake8-spellcheck==0.12.1; extra == "dev"
+Requires-Dist: redbaron; extra == "dev"
 
 <p align="center">
     <a href="https://pyinfra.com">
@@ -35,29 +92,27 @@ License-File: LICENSE.md
     </a>
 </p>
 
-<p align="center">
+<p>
     <strong>Note: this is the v3 branch, which is currently in beta. <a href="https://docs.pyinfra.com/en/next">See the docs for v3</a>. If needed the <a href="https://github.com/pyinfra-dev/pyinfra/tree/2.x/">2.x branch is here</a>, but is in bugfix only mode.</strong>
 </p>
 
-<p align="center">
-    <em>pyinfra automates infrastructure using Python. It’s fast and scales from one server to thousands. Great for ad-hoc command execution, service deployment, configuration management and more.</em>
+<p>
+    pyinfra turns Python code into shell commands and runs them on your servers. Execute ad-hoc commands and write declarative operations. Target SSH servers, local machine and Docker containers. Fast and scales from one server to thousands. Think <code>ansible</code> but Python instead of YAML, and a lot faster.
 </p>
 
 ---
 
-<p align="center">
-    <a href="https://docs.pyinfra.com"><strong>Documentation</strong></a> &rArr;
+<h3>
     <a href="https://docs.pyinfra.com/page/getting-started.html"><strong>Getting Started</strong></a> &bull;
-    <a href="https://docs.pyinfra.com/page/examples.html"><strong>Examples</strong></a> &bull;
+    <a href="https://github.com/pyinfra-dev/pyinfra-examples"><strong>Examples Repo</strong></a> &bull;
+    <a href="https://matrix.to/#/#pyinfra:matrix.org"><strong>Chat on Matrix</strong></a>
+</h3>
+<p>
+    <a href="https://docs.pyinfra.com"><strong>Documentation</strong></a> &bull;
     <a href="https://docs.pyinfra.com/page/support.html"><strong>Help & Support</strong></a> &bull;
     <a href="https://docs.pyinfra.com/page/contributing.html"><strong>Contributing</strong></a>
 </p>
 
-<p align="center">
-    Chat &rArr;
-    <a href="https://matrix.to/#/#pyinfra:matrix.org"><strong><code>#pyinfra</code> on Matrix</strong></a>
-</p>
-
 ---
 
 Why pyinfra? Design features include:

{pyinfra-3.0b0 → pyinfra-3.0b2}/README.md

@@ -4,29 +4,27 @@
     </a>
 </p>
 
-<p align="center">
+<p>
     <strong>Note: this is the v3 branch, which is currently in beta. <a href="https://docs.pyinfra.com/en/next">See the docs for v3</a>. If needed the <a href="https://github.com/pyinfra-dev/pyinfra/tree/2.x/">2.x branch is here</a>, but is in bugfix only mode.</strong>
 </p>
 
-<p align="center">
-    <em>pyinfra automates infrastructure using Python. It’s fast and scales from one server to thousands. Great for ad-hoc command execution, service deployment, configuration management and more.</em>
+<p>
+    pyinfra turns Python code into shell commands and runs them on your servers. Execute ad-hoc commands and write declarative operations. Target SSH servers, local machine and Docker containers. Fast and scales from one server to thousands. Think <code>ansible</code> but Python instead of YAML, and a lot faster.
 </p>
 
 ---
 
-<p align="center">
-    <a href="https://docs.pyinfra.com"><strong>Documentation</strong></a> &rArr;
+<h3>
     <a href="https://docs.pyinfra.com/page/getting-started.html"><strong>Getting Started</strong></a> &bull;
-    <a href="https://docs.pyinfra.com/page/examples.html"><strong>Examples</strong></a> &bull;
+    <a href="https://github.com/pyinfra-dev/pyinfra-examples"><strong>Examples Repo</strong></a> &bull;
+    <a href="https://matrix.to/#/#pyinfra:matrix.org"><strong>Chat on Matrix</strong></a>
+</h3>
+<p>
+    <a href="https://docs.pyinfra.com"><strong>Documentation</strong></a> &bull;
     <a href="https://docs.pyinfra.com/page/support.html"><strong>Help & Support</strong></a> &bull;
     <a href="https://docs.pyinfra.com/page/contributing.html"><strong>Contributing</strong></a>
 </p>
 
-<p align="center">
-    Chat &rArr;
-    <a href="https://matrix.to/#/#pyinfra:matrix.org"><strong><code>#pyinfra</code> on Matrix</strong></a>
-</p>
-
 ---
 
 Why pyinfra? Design features include:

{pyinfra-3.0b0 → pyinfra-3.0b2}/pyinfra/api/__init__.py

@@ -11,6 +11,9 @@ from .config import Config  # noqa: F401 # pragma: no cover
 from .deploy import deploy  # noqa: F401 # pragma: no cover
 from .exceptions import DeployError  # noqa: F401 # pragma: no cover
 from .exceptions import (  # noqa: F401
+    FactError,
+    FactTypeError,
+    FactValueError,
     InventoryError,
     OperationError,
     OperationTypeError,

{pyinfra-3.0b0 → pyinfra-3.0b2}/pyinfra/api/arguments.py

@@ -6,8 +6,10 @@ from typing import (
     Callable,
     Generic,
     Iterable,
+    List,
     Mapping,
     Optional,
+    Type,
     TypeVar,
     Union,
     cast,
@@ -169,7 +171,7 @@ class MetaArguments(TypedDict):
     name: str
     _ignore_errors: bool
     _continue_on_error: bool
-    _if: Callable[[], bool]
+    _if: Union[List[Callable[[], bool]], Callable[[], bool], None]
 
 
 meta_argument_meta: dict[str, ArgumentMeta] = {
@@ -190,8 +192,8 @@ meta_argument_meta: dict[str, ArgumentMeta] = {
         default=lambda _: False,
     ),
     "_if": ArgumentMeta(
-        "Only run this operation if this function returns True",
-        default=lambda _: None,
+        "Only run this operation if these functions return True",
+        default=lambda _: [],
     ),
 }
 
@@ -227,6 +229,11 @@ class AllArguments(ConnectorArguments, MetaArguments, ExecutionArguments):
     pass
 
 
+def all_global_arguments() -> List[tuple[str, Type]]:
+    """Return all global arguments and their types."""
+    return list(get_type_hints(AllArguments).items())
+
+
 all_argument_meta: dict[str, ArgumentMeta] = {
     **auth_argument_meta,
     **shell_argument_meta,
@@ -299,12 +306,12 @@ def pop_global_arguments(
     if context.ctx_config.isset():
         config = context.config
 
-    meta_kwargs = host.current_deploy_kwargs or {}
+    meta_kwargs: dict[str, Any] = host.current_deploy_kwargs or {}  # type: ignore[assignment]
 
     arguments: dict[str, Any] = {}
     found_keys: list[str] = []
 
-    for key, type_ in get_type_hints(AllArguments).items():
+    for key, type_ in all_global_arguments():
         if keys_to_check and key not in keys_to_check:
             continue
 

{pyinfra-3.0b0 → pyinfra-3.0b2}/pyinfra/api/arguments_typed.py

@@ -1,6 +1,16 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Callable, Generator, Generic, Iterable, Mapping, Optional, Union
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Generator,
+    Generic,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Union,
+)
 
 from typing_extensions import ParamSpec, Protocol
 
@@ -22,6 +32,11 @@ class PyinfraOperation(Generic[P], Protocol):
     def __call__(
         self,
         #
+        # op args
+        # needs to be first
+        #
+        *args: P.args,
+        #
         # ConnectorArguments
         #
         # Auth
@@ -51,7 +66,7 @@ class PyinfraOperation(Generic[P], Protocol):
         name: Optional[str] = None,
         _ignore_errors: bool = False,
         _continue_on_error: bool = False,
-        _if: Optional[Callable[[], bool]] = None,
+        _if: Union[List[Callable[[], bool]], Callable[[], bool], None] = None,
         #
         # ExecutionArguments
         #
@@ -59,9 +74,7 @@ class PyinfraOperation(Generic[P], Protocol):
         _run_once: bool = False,
         _serial: bool = False,
         #
-        # The op itself
+        # op kwargs
         #
-        *args: P.args,
         **kwargs: P.kwargs,
-    ) -> "OperationMeta":
-        ...
+    ) -> "OperationMeta": ...

{pyinfra-3.0b0 → pyinfra-3.0b2}/pyinfra/api/command.py

@@ -1,7 +1,9 @@
+from __future__ import annotations
+
 import shlex
 from inspect import getfullargspec
 from string import Formatter
-from typing import TYPE_CHECKING, Callable, Union
+from typing import IO, TYPE_CHECKING, Callable, Union
 
 import gevent
 from typing_extensions import Unpack
@@ -143,7 +145,7 @@ class StringCommand(PyinfraCommand):
 class FileUploadCommand(PyinfraCommand):
     def __init__(
         self,
-        src: str,
+        src: str | IO,
         dest: str,
         remote_temp_filename=None,
         **kwargs: Unpack[ConnectorArguments],
@@ -173,7 +175,7 @@ class FileDownloadCommand(PyinfraCommand):
     def __init__(
         self,
         src: str,
-        dest: str,
+        dest: str | IO,
         remote_temp_filename=None,
         **kwargs: Unpack[ConnectorArguments],
     ):

pyinfra-3.0b2/pyinfra/api/config.py

@@ -0,0 +1,231 @@
+try:
+    import importlib_metadata
+except ImportError:
+    import importlib.metadata as importlib_metadata  # type: ignore[no-redef]
+from os import path
+from typing import Iterable, Optional, Set
+
+from packaging.markers import Marker
+from packaging.requirements import Requirement
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
+
+from pyinfra import __version__, state
+
+from .exceptions import PyinfraError
+
+
+class ConfigDefaults:
+    # % of hosts which have to fail for all operations to stop
+    FAIL_PERCENT: Optional[int] = None
+    # Seconds to timeout SSH connections
+    CONNECT_TIMEOUT: int = 10
+    # Temporary directory (on the remote side) to use for caching any files/downloads, the default
+    # None value first tries to load the hosts' temporary directory configured via "TMPDIR" env
+    # variable, falling back to DEFAULT_TEMP_DIR if not set.
+    TEMP_DIR: Optional[str] = None
+    DEFAULT_TEMP_DIR: str = "/tmp"
+    # Gevent pool size (defaults to #of target hosts)
+    PARALLEL: int = 0
+    # Specify the required pyinfra version (using PEP 440 setuptools specifier)
+    REQUIRE_PYINFRA_VERSION: Optional[str] = None
+    # Specify any required packages (either using PEP 440 or a requirements file)
+    # Note: this can also include pyinfra potentially replacing REQUIRE_PYINFRA_VERSION
+    REQUIRE_PACKAGES: Optional[str] = None
+    # All these can be overridden inside individual operation calls:
+    # Switch to this user (from ssh_user) using su before executing operations
+    SU_USER: Optional[str] = None
+    USE_SU_LOGIN: bool = False
+    SU_SHELL: bool = False
+    PRESERVE_SU_ENV: bool = False
+    # Use sudo and optional user
+    SUDO: bool = False
+    SUDO_USER: Optional[str] = None
+    PRESERVE_SUDO_ENV: bool = False
+    USE_SUDO_LOGIN: bool = False
+    SUDO_PASSWORD: Optional[str] = None
+    # Use doas and optional user
+    DOAS: bool = False
+    DOAS_USER: Optional[str] = None
+    # Only show errors but don't count as failure
+    IGNORE_ERRORS: bool = False
+    # Shell to use to execute commands
+    SHELL: str = "sh"
+
+
+config_defaults = {key: value for key, value in ConfigDefaults.__dict__.items() if key.isupper()}
+
+
+def check_pyinfra_version(version: str):
+    if not version:
+        return
+    running_version = Version(__version__)
+    required_versions = SpecifierSet(version)
+
+    if running_version not in required_versions:
+        raise PyinfraError(
+            f"pyinfra version requirement not met (requires {version}, running {__version__})"
+        )
+
+
+def _check_requirements(requirements: Iterable[str]) -> Set[Requirement]:
+    """
+    Check whether each of the given requirements and all their dependencies are
+    installed.
+
+    Or more precisely, this checks that each of the given *requirements* is
+    satisfied by some installed *distribution package*, and so on recursively
+    for each of the dependencies of those distribution packages. The terminology
+    here is as follows:
+
+    * A *distribution package* is essentially a thing that can be installed with
+      ``pip``, from an sdist or wheel or Git repo or so on.
+    * A *requirement* is the expectation that a distribution package satisfying
+      some constraint is installed.
+    * A *dependency* is a requirement specified by a distribution package (as
+      opposed to the requirements passed in to this function).
+
+    So what this function does is start from the given requirements, for each
+    one check that it is satisfied by some installed distribution package, and
+    if so recursively perform the same check on all the dependencies of that
+    distribution package. In short, it's traversing the graph of package
+    requirements. It stops whenever it finds a requirement that is not satisfied
+    (i.e. a required package that is not installed), or when it runs out of
+    requirements to check.
+
+    .. note::
+        This is basically equivalent to ``pkg_resources.require()`` except that
+        when ``require()`` succeeds, it will return the list of distribution
+        packages that satisfy the given requirements and their dependencies, and
+        when it fails, it will raise an exception. This function just returns
+        the requirements which were not satisfied instead.
+
+    :param requirements: The requirements to check for in the set of installed
+        packages (along with their dependencies).
+    :return: The set of requirements that were not satisfied, which will be
+        an empty set if all requirements (recursively) were satisfied.
+    """
+
+    # Based on pkg_resources.require() from setuptools. The implementation of
+    # hbutils.system.check_reqs() from the hbutils package was also helpful in
+    # clarifying what this is supposed to do.
+
+    reqs_to_check: Set[Requirement] = set(Requirement(r) for r in requirements)
+    reqs_satisfied: Set[Requirement] = set()
+    reqs_not_satisfied: Set[Requirement] = set()
+
+    while reqs_to_check:
+        req = reqs_to_check.pop()
+        assert req not in reqs_satisfied and req not in reqs_not_satisfied
+
+        # Check for an installed distribution package with the right name and version
+        try:
+            dist = importlib_metadata.distribution(req.name)
+        except importlib_metadata.PackageNotFoundError:
+            # No installed package with the right name
+            # This would raise a DistributionNotFound error from pkg_resources.require()
+            reqs_not_satisfied.add(req)
+            continue
+
+        if dist.version not in req.specifier:
+            # There is a distribution with the right name but wrong version
+            # This would raise a VersionConflict error from pkg_resources.require()
+            reqs_not_satisfied.add(req)
+            continue
+
+        reqs_satisfied.add(req)
+
+        # If the distribution package has dependencies of its own, go through
+        # those dependencies and for each one add it to the set to be checked if
+        # - it's unconditional (no marker)
+        # - or it's conditional and the condition is satisfied (the marker
+        #   evaluates to true) in the current environment
+        # Markers can check things like the Python version and system version
+        # etc., and/or they can check which extras of the distribution package
+        # were required. To facilitate checking extras we have to pass the extra
+        # in the environment when calling Marker.evaluate().
+        if dist.requires:
+            if req.extras:
+                extras_envs = [{"extra": extra} for extra in req.extras]
+
+                def evaluate_marker(marker: Marker) -> bool:
+                    return any(map(marker.evaluate, extras_envs))
+
+            else:
+
+                def evaluate_marker(marker: Marker) -> bool:
+                    return marker.evaluate()
+
+            for dist_req_str in dist.requires:
+                dist_req = Requirement(dist_req_str)
+                if dist_req in reqs_satisfied or dist_req in reqs_not_satisfied:
+                    continue
+                if (not dist_req.marker) or evaluate_marker(dist_req.marker):
+                    reqs_to_check.add(dist_req)
+
+    return reqs_not_satisfied
+
+
+def check_require_packages(requirements_config):
+    if not requirements_config:
+        return
+
+    if isinstance(requirements_config, (list, tuple)):
+        requirements = requirements_config
+    else:
+        with open(path.join(state.cwd or "", requirements_config), encoding="utf-8") as f:
+            requirements = [line.split("#egg=")[-1] for line in f.read().splitlines()]
+
+    requirements_not_met = _check_requirements(requirements)
+    if requirements_not_met:
+        raise PyinfraError(
+            "Deploy requirements ({0}) not met: missing {1}".format(
+                requirements_config, ", ".join(str(r) for r in requirements_not_met)
+            )
+        )
+
+
+config_checkers = {
+    "REQUIRE_PYINFRA_VERSION": check_pyinfra_version,
+    "REQUIRE_PACKAGES": check_require_packages,
+}
+
+
+class Config(ConfigDefaults):
+    """
+    The default/base configuration options for a pyinfra deploy.
+    """
+
+    def __init__(self, **kwargs):
+        # Always apply some env
+        env = kwargs.pop("ENV", {})
+        self.ENV = env
+
+        config = config_defaults.copy()
+        config.update(kwargs)
+
+        for key, value in config.items():
+            setattr(self, key, value)
+
+    def __setattr__(self, key, value):
+        super().__setattr__(key, value)
+
+        checker = config_checkers.get(key)
+        if checker:
+            checker(value)
+
+    def get_current_state(self):
+        return [(key, getattr(self, key)) for key in config_defaults.keys()]
+
+    def set_current_state(self, config_state):
+        for key, value in config_state:
+            setattr(self, key, value)
+
+    def lock_current_state(self):
+        self._locked_config = self.get_current_state()
+
+    def reset_locked_state(self):
+        self.set_current_state(self._locked_config)
+
+    def copy(self) -> "Config":
+        return Config(**dict(self.get_current_state()))

{pyinfra-3.0b0 → pyinfra-3.0b2}/pyinfra/api/connectors.py

@@ -1,4 +1,7 @@
-import pkg_resources
+try:
+    from importlib_metadata import entry_points
+except ImportError:
+    from importlib.metadata import entry_points  # type: ignore[assignment]
 
 
 def _load_connector(entrypoint):
@@ -8,7 +11,7 @@ def _load_connector(entrypoint):
 def get_all_connectors():
     return {
         entrypoint.name: _load_connector(entrypoint)
-        for entrypoint in pkg_resources.iter_entry_points("pyinfra.connectors")
+        for entrypoint in entry_points(group="pyinfra.connectors")
     }
 
 

{pyinfra-3.0b0 → pyinfra-3.0b2}/pyinfra/api/exceptions.py

@@ -10,6 +10,25 @@ class ConnectError(PyinfraError):
     """
 
 
+class FactError(PyinfraError):
+    """
+    Exception raised during fact gathering staging if a fact is unable to
+    generate output/change state.
+    """
+
+
+class FactTypeError(FactError, TypeError):
+    """
+    Exception raised when a fact is passed invalid argument types.
+    """
+
+
+class FactValueError(FactError, ValueError):
+    """
+    Exception raised when a fact is passed invalid argument values.
+    """
+
+
 class OperationError(PyinfraError):
     """
     Exception raised during fact gathering staging if an operation is unable to