pyinfra 3.0b0__tar.gz → 3.0b1__tar.gz

{pyinfra-3.0b0 → pyinfra-3.0b1}/CHANGELOG.md

@@ -1,4 +1,4 @@
-# v3.0.beta0
+# v3.0.beta1
 
 Welcome to pyinfra v3! This version is the biggest overhaul of pyinfra since it was created back in 2015. Most v2 deployment code should be automatically compatible, but as always be aware. Major changes:
 

{pyinfra-3.0b0/pyinfra.egg-info → pyinfra-3.0b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyinfra
-Version: 3.0b0
+Version: 3.0b1
 Summary: pyinfra automates/provisions/manages/deploys infrastructure.
 Home-page: https://pyinfra.com
 Author: Nick / Fizzadar
@@ -35,17 +35,17 @@ License-File: LICENSE.md
     </a>
 </p>
 
-<p align="center">
+<p>
     <strong>Note: this is the v3 branch, which is currently in beta. <a href="https://docs.pyinfra.com/en/next">See the docs for v3</a>. If needed the <a href="https://github.com/pyinfra-dev/pyinfra/tree/2.x/">2.x branch is here</a>, but is in bugfix only mode.</strong>
 </p>
 
-<p align="center">
-    <em>pyinfra automates infrastructure using Python. It’s fast and scales from one server to thousands. Great for ad-hoc command execution, service deployment, configuration management and more.</em>
+<p>
+    pyinfra turns Python code into shell commands and runs them on your servers. Execute ad-hoc commands and write declarative operations. Target SSH servers, local machine and Docker containers. Fast and scales from one server to thousands. Think <code>ansible</code> but Python instead of YAML, and a lot faster.
 </p>
 
 ---
 
-<p align="center">
+<p>
     <a href="https://docs.pyinfra.com"><strong>Documentation</strong></a> &rArr;
     <a href="https://docs.pyinfra.com/page/getting-started.html"><strong>Getting Started</strong></a> &bull;
     <a href="https://docs.pyinfra.com/page/examples.html"><strong>Examples</strong></a> &bull;
@@ -53,7 +53,7 @@ License-File: LICENSE.md
     <a href="https://docs.pyinfra.com/page/contributing.html"><strong>Contributing</strong></a>
 </p>
 
-<p align="center">
+<p>
     Chat &rArr;
     <a href="https://matrix.to/#/#pyinfra:matrix.org"><strong><code>#pyinfra</code> on Matrix</strong></a>
 </p>

{pyinfra-3.0b0 → pyinfra-3.0b1}/README.md

@@ -4,17 +4,17 @@
     </a>
 </p>
 
-<p align="center">
+<p>
     <strong>Note: this is the v3 branch, which is currently in beta. <a href="https://docs.pyinfra.com/en/next">See the docs for v3</a>. If needed the <a href="https://github.com/pyinfra-dev/pyinfra/tree/2.x/">2.x branch is here</a>, but is in bugfix only mode.</strong>
 </p>
 
-<p align="center">
-    <em>pyinfra automates infrastructure using Python. It’s fast and scales from one server to thousands. Great for ad-hoc command execution, service deployment, configuration management and more.</em>
+<p>
+    pyinfra turns Python code into shell commands and runs them on your servers. Execute ad-hoc commands and write declarative operations. Target SSH servers, local machine and Docker containers. Fast and scales from one server to thousands. Think <code>ansible</code> but Python instead of YAML, and a lot faster.
 </p>
 
 ---
 
-<p align="center">
+<p>
     <a href="https://docs.pyinfra.com"><strong>Documentation</strong></a> &rArr;
     <a href="https://docs.pyinfra.com/page/getting-started.html"><strong>Getting Started</strong></a> &bull;
     <a href="https://docs.pyinfra.com/page/examples.html"><strong>Examples</strong></a> &bull;
@@ -22,7 +22,7 @@
     <a href="https://docs.pyinfra.com/page/contributing.html"><strong>Contributing</strong></a>
 </p>
 
-<p align="center">
+<p>
     Chat &rArr;
     <a href="https://matrix.to/#/#pyinfra:matrix.org"><strong><code>#pyinfra</code> on Matrix</strong></a>
 </p>

{pyinfra-3.0b0 → pyinfra-3.0b1}/pyinfra/api/__init__.py

@@ -11,6 +11,9 @@ from .config import Config  # noqa: F401 # pragma: no cover
 from .deploy import deploy  # noqa: F401 # pragma: no cover
 from .exceptions import DeployError  # noqa: F401 # pragma: no cover
 from .exceptions import (  # noqa: F401
+    FactError,
+    FactTypeError,
+    FactValueError,
     InventoryError,
     OperationError,
     OperationTypeError,

{pyinfra-3.0b0 → pyinfra-3.0b1}/pyinfra/api/arguments.py

@@ -6,6 +6,7 @@ from typing import (
     Callable,
     Generic,
     Iterable,
+    List,
     Mapping,
     Optional,
     TypeVar,
@@ -169,7 +170,7 @@ class MetaArguments(TypedDict):
     name: str
     _ignore_errors: bool
     _continue_on_error: bool
-    _if: Callable[[], bool]
+    _if: List[Callable[[], bool]]
 
 
 meta_argument_meta: dict[str, ArgumentMeta] = {
@@ -190,8 +191,8 @@ meta_argument_meta: dict[str, ArgumentMeta] = {
         default=lambda _: False,
     ),
     "_if": ArgumentMeta(
-        "Only run this operation if this function returns True",
-        default=lambda _: None,
+        "Only run this operation if these functions returns True",
+        default=lambda _: [],
     ),
 }
 
@@ -299,7 +300,7 @@ def pop_global_arguments(
     if context.ctx_config.isset():
         config = context.config
 
-    meta_kwargs = host.current_deploy_kwargs or {}
+    meta_kwargs: dict[str, Any] = host.current_deploy_kwargs or {}  # type: ignore[assignment]
 
     arguments: dict[str, Any] = {}
     found_keys: list[str] = []

{pyinfra-3.0b0 → pyinfra-3.0b1}/pyinfra/api/arguments_typed.py

@@ -1,6 +1,16 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Callable, Generator, Generic, Iterable, Mapping, Optional, Union
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Generator,
+    Generic,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Union,
+)
 
 from typing_extensions import ParamSpec, Protocol
 
@@ -51,7 +61,7 @@ class PyinfraOperation(Generic[P], Protocol):
         name: Optional[str] = None,
         _ignore_errors: bool = False,
         _continue_on_error: bool = False,
-        _if: Optional[Callable[[], bool]] = None,
+        _if: Optional[List[Callable[[], bool]]] = None,
         #
         # ExecutionArguments
         #

{pyinfra-3.0b0 → pyinfra-3.0b1}/pyinfra/api/exceptions.py

@@ -10,6 +10,25 @@ class ConnectError(PyinfraError):
     """
 
 
+class FactError(PyinfraError):
+    """
+    Exception raised during fact gathering staging if a fact is unable to
+    generate output/change state.
+    """
+
+
+class FactTypeError(FactError, TypeError):
+    """
+    Exception raised when a fact is passed invalid argument types.
+    """
+
+
+class FactValueError(FactError, ValueError):
+    """
+    Exception raised when a fact is passed invalid argument values.
+    """
+
+
 class OperationError(PyinfraError):
     """
     Exception raised during fact gathering staging if an operation is unable to

{pyinfra-3.0b0 → pyinfra-3.0b1}/pyinfra/api/facts.py

@@ -50,7 +50,7 @@ if TYPE_CHECKING:
     from pyinfra.api.host import Host
     from pyinfra.api.state import State
 
-SUDO_REGEX = r"^sudo: unknown user:"
+SUDO_REGEX = r"^sudo: unknown user"
 SU_REGEXES = (
     r"^su: user .+ does not exist",
     r"^su: unknown login",

{pyinfra-3.0b0 → pyinfra-3.0b1}/pyinfra/api/host.py

@@ -1,10 +1,22 @@
 from __future__ import annotations
 
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Any, Callable, Generator, Optional, Type, TypeVar, Union, overload
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Generator,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+    cast,
+    overload,
+)
 from uuid import uuid4
 
 import click
+from typing_extensions import Unpack
 
 from pyinfra import logger
 from pyinfra.connectors.base import BaseConnector
@@ -98,11 +110,16 @@ class Host:
     current_op_hash: Optional[str] = None
     current_op_global_arguments: Optional["AllArguments"] = None
 
-    # Current context inside a @deploy function (op gen stage)
+    # Current context inside a @deploy function which become part of the op data
     in_deploy: bool = False
     current_deploy_name: Optional[str] = None
     current_deploy_kwargs = None
-    current_deploy_data = None
+
+    # @deploy decorator data is a bit different - we need to handle the case
+    # where we're evaluating an operation at runtime (current_op_) but also
+    # when ordering operations (current_) outside of an operation context.
+    current_op_deploy_data: Optional[dict[str, Any]] = None
+    current_deploy_data: Optional[dict[str, Any]] = None
 
     # Current context during operation execution
     executing_op_hash: Optional[str] = None
@@ -204,9 +221,7 @@ class Host:
         self.log(message_styled, log_func=log_func)
 
     def get_deploy_data(self):
-        if self.current_deploy_data:
-            return self.current_deploy_data
-        return {}
+        return self.current_op_deploy_data or self.current_deploy_data or {}
 
     def noop(self, description):
         """
@@ -216,8 +231,25 @@ class Host:
         handler = logger.info if self.state.print_noop_info else logger.debug
         handler("{0}noop: {1}".format(self.print_prefix, description))
 
+    def when(self, condition: Callable[[], bool]):
+        return self.deploy(
+            "",
+            cast("AllArguments", {"_if": [condition]}),
+            {},
+            in_deploy=False,
+        )
+
+    def arguments(self, **arguments: Unpack["AllArguments"]):
+        return self.deploy("", arguments, {}, in_deploy=False)
+
     @contextmanager
-    def deploy(self, name: str, kwargs, data, in_deploy: bool = True):
+    def deploy(
+        self,
+        name: str,
+        kwargs: Optional["AllArguments"],
+        data: Optional[dict],
+        in_deploy: bool = True,
+    ):
         """
         Wraps a group of operations as a deploy, this should not be used
         directly, instead use ``pyinfra.api.deploy.deploy``.
@@ -234,6 +266,13 @@ class Host:
         old_deploy_data = self.current_deploy_data
         self.in_deploy = in_deploy
 
+        # Combine any old _ifs with the new ones
+        if old_deploy_kwargs and kwargs:
+            old_ifs = old_deploy_kwargs["_if"]
+            new_ifs = kwargs["_if"]
+            if old_ifs and new_ifs:
+                kwargs["_if"] = old_ifs + new_ifs
+
         # Set the new values
         self.current_deploy_name = name
         self.current_deploy_kwargs = kwargs

{pyinfra-3.0b0 → pyinfra-3.0b1}/pyinfra/api/operation.py

@@ -11,7 +11,7 @@ from functools import wraps
 from inspect import signature
 from io import StringIO
 from types import FunctionType
-from typing import Any, Callable, Generator, Iterator, Optional, cast
+from typing import TYPE_CHECKING, Any, Callable, Generator, Iterator, Optional, cast
 
 from typing_extensions import ParamSpec
 
@@ -36,16 +36,19 @@ from .util import (
 
 op_meta_default = object()
 
+if TYPE_CHECKING:
+    from pyinfra.connectors.util import CommandOutput
+
 
 class OperationMeta:
     _hash: str
 
-    _combined_output_lines = None
+    _combined_output: Optional["CommandOutput"] = None
     _commands: Optional[list[Any]] = None
-    _maybe_is_change: bool = False
+    _maybe_is_change: Optional[bool] = False
     _success: Optional[bool] = None
 
-    def __init__(self, hash, is_change=False):
+    def __init__(self, hash, is_change: Optional[bool]):
         self._hash = hash
         self._maybe_is_change = is_change
 
@@ -57,7 +60,7 @@ class OperationMeta:
         if self._commands is not None:
             return (
                 "OperationMeta(executed=True, "
-                f"changed={self.did_change()}, hash={self._hash}, commands={len(self._commands)})"
+                f"success={self.did_succeed}, hash={self._hash}, commands={len(self._commands)})"
             )
         return (
             "OperationMeta(executed=False, "
@@ -69,13 +72,13 @@ class OperationMeta:
         self,
         success: bool,
         commands: list[Any],
-        combined_output_lines,
+        combined_output: "CommandOutput",
     ) -> None:
         if self.is_complete():
             raise RuntimeError("Cannot complete an already complete operation")
         self._success = success
         self._commands = commands
-        self._combined_output_lines = combined_output_lines
+        self._combined_output = combined_output
 
     def is_complete(self) -> bool:
         return self._success is not None
@@ -84,10 +87,17 @@ class OperationMeta:
         if not self.is_complete():
             raise RuntimeError("Cannot evaluate operation result before execution")
 
-    def did_change(self) -> bool:
-        self._raise_if_not_complete()
+    def _did_change(self) -> bool:
         return bool(self._success and len(self._commands or []) > 0)
 
+    @property
+    def did_change(self):
+        return context.host.when(self._did_change)
+
+    @property
+    def did_not_change(self):
+        return context.host.when(lambda: not self._did_change())
+
     def did_succeed(self) -> bool:
         self._raise_if_not_complete()
         return self._success is True
@@ -96,19 +106,32 @@ class OperationMeta:
         self._raise_if_not_complete()
         return self._success is False
 
-    # Output lines
-    def _get_lines(self, types=("stdout", "stderr")):
-        self._raise_if_not_complete()
-        assert self._combined_output_lines is not None
-        return [line for type_, line in self._combined_output_lines if type_ in types]
+    # TODO: deprecated, remove in v4
+    @property
+    def changed(self) -> bool:
+        if self.is_complete():
+            return self._did_change()
+
+        if self._maybe_is_change is not None:
+            return self._maybe_is_change
+
+        op_data = context.state.get_op_data_for_host(context.host, self._hash)
+        cmd_gen = op_data.command_generator
+        for _ in cmd_gen():
+            return True
+        return False
 
     @property
-    def stdout_lines(self):
-        return self._get_lines(types=("stdout",))
+    def stdout_lines(self) -> list[str]:
+        self._raise_if_not_complete()
+        assert self._combined_output is not None
+        return self._combined_output.stdout_lines
 
     @property
-    def stderr_lines(self):
-        return self._get_lines(types=("stderr",))
+    def stderr_lines(self) -> list[str]:
+        self._raise_if_not_complete()
+        assert self._combined_output is not None
+        return self._combined_output.stderr_lines
 
     @property
     def stdout(self) -> str:
@@ -118,14 +141,6 @@ class OperationMeta:
     def stderr(self) -> str:
         return "\n".join(self.stderr_lines)
 
-    # TODO: deprecated, remove in v4
-    @property
-    def changed(self) -> int:
-        if not self.is_complete():
-            logger.warning("Checking changed before execution can give unexpected results")
-            return self._maybe_is_change
-        return self.did_change()
-
 
 def add_op(state: State, op_func, *args, **kwargs):
     """
@@ -164,6 +179,8 @@ P = ParamSpec("P")
 def operation(
     is_idempotent: bool = True,
     idempotent_notice: Optional[str] = None,
+    is_deprecated: bool = False,
+    deprecated_for: Optional[str] = None,
     _set_in_op: bool = True,
 ) -> Callable[[Callable[P, Generator]], PyinfraOperation[P]]:
     """
@@ -175,6 +192,8 @@ def operation(
     def decorator(f: Callable[P, Generator]) -> PyinfraOperation[P]:
         f.is_idempotent = is_idempotent  # type: ignore[attr-defined]
         f.idempotent_notice = idempotent_notice  # type: ignore[attr-defined]
+        f.is_deprecated = is_deprecated  # type: ignore[attr-defined]
+        f.deprecated_for = deprecated_for  # type: ignore[attr-defined]
         return _wrap_operation(f, _set_in_op=_set_in_op)
 
     return decorator
@@ -192,6 +211,15 @@ def _wrap_operation(func: Callable[P, Generator], _set_in_op: bool = True) -> Py
                 + "function to call the underlying operation."
             )
 
+        if func.is_deprecated:  # type: ignore[attr-defined]
+            if func.deprecated_for:  # type: ignore[attr-defined]
+                logger.warning(
+                    f"The {get_operation_name_from_func(func)} operation is "
+                    + f"deprecated, please use: {func.deprecated_for}",  # type: ignore[attr-defined] # noqa
+                )
+            else:
+                logger.warning(f"The {get_operation_name_from_func(func)} operation is deprecated")
+
         # Configure operation
         #
         # Get the meta kwargs (globals that apply to all hosts)
@@ -217,7 +245,12 @@ def _wrap_operation(func: Callable[P, Generator], _set_in_op: bool = True) -> Py
                     break
 
             if has_run:
-                return OperationMeta(op_hash)
+                return OperationMeta(op_hash, is_change=False)
+
+        # Grab a reference to any *current* deploy data as this may change when
+        # we later evaluate the operation at runtime.This means we put back the
+        # expected deploy data.
+        current_deploy_data = host.current_deploy_data
 
         # "Run" operation - here we make a generator that will yield out actual commands to execute
         # and, if we're diff-ing, we then iterate the generator now to determine if any changes
@@ -226,13 +259,14 @@ def _wrap_operation(func: Callable[P, Generator], _set_in_op: bool = True) -> Py
         def command_generator() -> Iterator[PyinfraCommand]:
             # Check global _if_ argument function and do nothing if returns False
             if state.is_executing:
-                _if = global_arguments.get("_if")
-                if _if and _if() is False:
+                _ifs = global_arguments.get("_if")
+                if _ifs and not all(_if() for _if in _ifs):
                     return
 
             host.in_op = _set_in_op
             host.current_op_hash = op_hash
             host.current_op_global_arguments = global_arguments
+            host.current_op_deploy_data = current_deploy_data
 
             try:
                 for command in func(*args, **kwargs):
@@ -243,10 +277,12 @@ def _wrap_operation(func: Callable[P, Generator], _set_in_op: bool = True) -> Py
                 host.in_op = False
                 host.current_op_hash = None
                 host.current_op_global_arguments = None
+                host.current_op_deploy_data = None
 
-        op_is_change = False
+        op_is_change = None
         if state.should_check_for_changes():
-            for command in command_generator():
+            op_is_change = False
+            for _ in command_generator():
                 op_is_change = True
                 break
         else:
@@ -279,6 +315,15 @@ def _wrap_operation(func: Callable[P, Generator], _set_in_op: bool = True) -> Py
     return cast(PyinfraOperation[P], decorated_func)
 
 
+def get_operation_name_from_func(func):
+    if func.__module__:
+        module_bits = func.__module__.split(".")
+        module_name = module_bits[-1]
+        return "{0}.{1}".format(module_name, func.__name__)
+    else:
+        return func.__name__
+
+
 def generate_operation_name(func, host, kwargs, global_arguments):
     # Generate an operation name if needed (Module/Operation format)
     name = global_arguments.get("name")
@@ -287,14 +332,7 @@ def generate_operation_name(func, host, kwargs, global_arguments):
         names = {name}
     else:
         add_args = True
-
-        if func.__module__:
-            module_bits = func.__module__.split(".")
-            module_name = module_bits[-1]
-            name = "{0}/{1}".format(module_name.title(), func.__name__.title())
-        else:
-            name = func.__name__
-
+        name = get_operation_name_from_func(func)
         names = {name}
 
     if host.current_deploy_name:

{pyinfra-3.0b0 → pyinfra-3.0b1}/pyinfra/api/operations.py

@@ -67,17 +67,16 @@ def _run_host_op(state: "State", host: "Host", op_hash: str) -> Optional[bool]:
     timeout = global_arguments.get("_timeout", 0)
 
     executor_kwarg_keys = CONNECTOR_ARGUMENT_KEYS
-    base_connector_arguments: ConnectorArguments = (
-        cast(  # https://github.com/python/mypy/issues/10371
-            ConnectorArguments,
-            {key: global_arguments[key] for key in executor_kwarg_keys if key in global_arguments},
-        )
+    # See: https://github.com/python/mypy/issues/10371
+    base_connector_arguments: ConnectorArguments = cast(
+        ConnectorArguments,
+        {key: global_arguments[key] for key in executor_kwarg_keys if key in global_arguments},  # type: ignore[literal-required] # noqa
     )
 
     did_error = False
     executed_commands = 0
     commands = []
-    all_combined_output_lines: list[OutputLine] = []
+    all_output_lines: list[OutputLine] = []
 
     for command in op_data.command_generator():
         commands.append(command)
@@ -103,19 +102,19 @@ def _run_host_op(state: "State", host: "Host", op_hash: str) -> Optional[bool]:
                 )
 
         elif isinstance(command, StringCommand):
-            combined_output_lines = CommandOutput([])
+            output_lines = CommandOutput([])
             try:
-                status, combined_output_lines = command.execute(
+                status, output_lines = command.execute(
                     state,
                     host,
                     connector_arguments,
                 )
             except (timeout_error, socket_error, SSHException) as e:
                 log_host_command_error(host, e, timeout=timeout)
-            all_combined_output_lines.extend(combined_output_lines)
+            all_output_lines.extend(output_lines)
             # If we failed and have not already printed the stderr, print it
             if status is False and not state.print_output:
-                print_host_combined_output(host, combined_output_lines)
+                print_host_combined_output(host, output_lines)
 
         else:
             try:
@@ -170,7 +169,7 @@ def _run_host_op(state: "State", host: "Host", op_hash: str) -> Optional[bool]:
     op_data.operation_meta.set_complete(
         op_success,
         commands,
-        all_combined_output_lines,
+        CommandOutput(all_output_lines),
     )
 
     return return_status

{pyinfra-3.0b0 → pyinfra-3.0b1}/pyinfra/api/state.py

@@ -344,10 +344,19 @@ class State:
     def get_results_for_host(self, host: "Host") -> StateHostResults:
         return self.results[host]
 
-    def get_op_data_for_host(self, host: "Host", op_hash: str):
+    def get_op_data_for_host(
+        self,
+        host: "Host",
+        op_hash: str,
+    ) -> StateOperationHostData:
         return self.ops[host][op_hash]
 
-    def set_op_data_for_host(self, host: "Host", op_hash: str, op_data: StateOperationHostData):
+    def set_op_data_for_host(
+        self,
+        host: "Host",
+        op_hash: str,
+        op_data: StateOperationHostData,
+    ):
         self.ops[host][op_hash] = op_data
 
     def activate_host(self, host: "Host"):

{pyinfra-3.0b0 → pyinfra-3.0b1}/pyinfra/connectors/base.py

@@ -84,7 +84,7 @@ class BaseConnector(abc.ABC):
 
     @staticmethod
     @abc.abstractmethod
-    def make_names_data(id: str) -> Iterator[tuple[str, dict, list[str]]]:
+    def make_names_data(name: str) -> Iterator[tuple[str, dict, list[str]]]:
         """
         Generates hosts/data/groups information for inventory. This allows a
         single connector reference to generate multiple target hosts.

{pyinfra-3.0b0 → pyinfra-3.0b1}/pyinfra/connectors/chroot.py

@@ -40,17 +40,17 @@ class ChrootConnector(BaseConnector):
         self.local = LocalConnector(state, host)
 
     @staticmethod
-    def make_names_data(directory: Optional[str] = None):
-        if not directory:
+    def make_names_data(name: Optional[str] = None):
+        if not name:
             raise InventoryError("No directory provided!")
 
         show_warning()
 
-        yield "@chroot/{0}".format(directory), {
-            "chroot_directory": "/{0}".format(directory.lstrip("/")),
+        yield "@chroot/{0}".format(name), {
+            "chroot_directory": "/{0}".format(name.lstrip("/")),
         }, ["@chroot"]
 
-    def connect(self):
+    def connect(self) -> None:
         self.local.connect()
 
         chroot_directory = self.host.data.chroot_directory
@@ -65,7 +65,6 @@ class ChrootConnector(BaseConnector):
             raise ConnectError(e.args[0])
 
         self.host.connector_data["chroot_directory"] = chroot_directory
-        return True
 
     def run_shell_command(
         self,