pyinfra 3.3__py2.py3-none-any.whl → 3.4__py2.py3-none-any.whl

pyinfra/api/arguments.py

@@ -18,14 +18,12 @@ from typing import (
 
 from typing_extensions import TypedDict
 
-from pyinfra import context
 from pyinfra.api.exceptions import ArgumentTypeError
-from pyinfra.api.state import State
 from pyinfra.api.util import raise_if_bad_type
+from pyinfra.context import ctx_config
 
 if TYPE_CHECKING:
-    from pyinfra.api.config import Config
-    from pyinfra.api.host import Host
+    from pyinfra.api import Config, Host, State
 
 T = TypeVar("T")
 default_sentinel = object()
@@ -292,10 +290,9 @@ __argument_docs__ = {
 
 
 def pop_global_arguments(
+    state: "State",
+    host: "Host",
     kwargs: dict[str, Any],
-    state: Optional["State"] = None,
-    host: Optional["Host"] = None,
-    keys_to_check=None,
 ) -> tuple[AllArguments, list[str]]:
     """
     Pop and return operation global keyword arguments, in preferred order:
@@ -306,22 +303,17 @@ def pop_global_arguments(
     + From the config variables
     """
 
-    state = state or context.state
-    host = host or context.host
-
     config = state.config
-    if context.ctx_config.isset():
-        config = context.config
+    if ctx_config.isset():
+        config = config
 
-    meta_kwargs: dict[str, Any] = host.current_deploy_kwargs or {}  # type: ignore[assignment]
+    cdkwargs = host.current_deploy_kwargs
+    meta_kwargs: dict[str, Any] = cdkwargs or {}  # type: ignore[assignment]
 
     arguments: dict[str, Any] = {}
     found_keys: list[str] = []
 
     for key, type_ in all_global_arguments():
-        if keys_to_check and key not in keys_to_check:
-            continue
-
         argument_meta = all_argument_meta[key]
         handler = argument_meta.handler
         default: Any = argument_meta.default(config)

pyinfra/api/deploy.py

@@ -82,7 +82,7 @@ def deploy(
 def _wrap_deploy(func: Callable[P, Any]) -> PyinfraOperation[P]:
     @wraps(func)
     def decorated_func(*args: P.args, **kwargs: P.kwargs) -> Any:
-        deploy_kwargs, _ = pop_global_arguments(kwargs)
+        deploy_kwargs, _ = pop_global_arguments(context.state, context.host, kwargs)
 
         deploy_data = getattr(func, "deploy_data", None)
 

pyinfra/api/facts.py

@@ -31,14 +31,12 @@ from pyinfra.api.util import (
     print_host_combined_output,
 )
 from pyinfra.connectors.util import CommandOutput
-from pyinfra.context import ctx_host, ctx_state
 from pyinfra.progress import progress_spinner
 
 from .arguments import CONNECTOR_ARGUMENT_KEYS
 
 if TYPE_CHECKING:
-    from pyinfra.api.host import Host
-    from pyinfra.api.state import State
+    from pyinfra.api import Host, State
 
 SUDO_REGEX = r"^sudo: unknown user"
 SU_REGEXES = (
@@ -121,21 +119,19 @@ def _make_command(command_attribute, host_args):
     return command_attribute
 
 
-def _handle_fact_kwargs(state, host, cls, args, kwargs):
+def _handle_fact_kwargs(state: "State", host: "Host", cls, args, kwargs):
     args = args or []
     kwargs = kwargs or {}
 
     # Start with a (shallow) copy of current operation kwargs if any
-    ctx_kwargs = (host.current_op_global_arguments or {}).copy()
+    ctx_kwargs: dict[str, Any] = (
+        cast(dict[str, Any], host.current_op_global_arguments) or {}
+    ).copy()
     # Update with the input kwargs (overrides)
     ctx_kwargs.update(kwargs)
 
     # Pop executor kwargs, pass remaining
-    global_kwargs, _ = pop_global_arguments(
-        ctx_kwargs,
-        state=state,
-        host=host,
-    )
+    global_kwargs, _ = pop_global_arguments(state, host, cast(dict[str, Any], ctx_kwargs))
 
     fact_kwargs = {key: value for key, value in kwargs.items() if key not in global_kwargs}
 
@@ -146,14 +142,12 @@ def _handle_fact_kwargs(state, host, cls, args, kwargs):
     return fact_kwargs, global_kwargs
 
 
-def get_facts(state: "State", *args, **kwargs):
-    def get_fact_with_context(state, host, *args, **kwargs):
-        with ctx_state.use(state):
-            with ctx_host.use(host):
-                return get_fact(state, host, *args, **kwargs)
+def get_facts(state, *args, **kwargs):
+    def get_host_fact(host, *args, **kwargs):
+        return get_fact(state, host, *args, **kwargs)
 
     greenlet_to_host = {
-        state.pool.spawn(get_fact_with_context, state, host, *args, **kwargs): host
+        state.pool.spawn(get_host_fact, host, *args, **kwargs): host
         for host in state.inventory.iter_active_hosts()
     }
 
@@ -311,13 +305,3 @@ def _get_fact(
         state.fail_hosts({host})
 
     return data
-
-
-def get_host_fact(
-    state: "State",
-    host: "Host",
-    cls,
-    args: Optional[Iterable] = None,
-    kwargs: Optional[dict] = None,
-) -> Any:
-    return get_fact(state, host, cls, args=args, kwargs=kwargs)

pyinfra/api/host.py

@@ -25,7 +25,7 @@ from pyinfra.connectors.util import CommandOutput, remove_any_sudo_askpass_file
 
 from .connectors import get_execution_connector
 from .exceptions import ConnectError
-from .facts import FactBase, ShortFactBase, get_host_fact
+from .facts import FactBase, ShortFactBase, get_fact
 from .util import memoize, sha1_hash
 
 if TYPE_CHECKING:
@@ -328,12 +328,18 @@ class Host:
 
         return temp_directory
 
-    def get_temp_filename(self, hash_key: Optional[str] = None, hash_filename: bool = True):
+    def get_temp_filename(
+        self,
+        hash_key: Optional[str] = None,
+        hash_filename: bool = True,
+        *,
+        temp_directory: Optional[str] = None,
+    ):
         """
         Generate a temporary filename for this deploy.
         """
 
-        temp_directory = self._get_temp_directory()
+        temp_directory = temp_directory or self._get_temp_directory()
 
         if not hash_key:
             hash_key = str(uuid4())
@@ -358,7 +364,7 @@ class Host:
         """
         Get a fact for this host, reading from the cache if present.
         """
-        return get_host_fact(self.state, self, name_or_cls, args=args, kwargs=kwargs)
+        return get_fact(self.state, self, name_or_cls, args=args, kwargs=kwargs)
 
     # Connector proxy
     #

pyinfra/api/inventory.py

@@ -36,6 +36,7 @@ class Inventory:
     """
 
     state: "State"
+    groups: dict[str, list[Host]]
 
     @staticmethod
     def empty():
@@ -181,7 +182,7 @@ class Inventory:
         """
         return len(self.state.activated_hosts)
 
-    def get_host(self, name: str, default=NoHostError):
+    def get_host(self, name: str, default=NoHostError) -> Host:
         """
         Get a single host by name.
         """
@@ -192,9 +193,10 @@ class Inventory:
         if default is NoHostError:
             raise NoHostError("No such host: {0}".format(name))
 
+        # TODO: remove default here?
         return default
 
-    def get_group(self, name: str, default=NoGroupError):
+    def get_group(self, name: str, default=NoGroupError) -> list[Host]:
         """
         Get a list of hosts belonging to a group.
         """
@@ -205,6 +207,7 @@ class Inventory:
         if default is NoGroupError:
             raise NoGroupError("No such group: {0}".format(name))
 
+        # TODO: remove default here?
         return default
 
     def get_data(self):

pyinfra/api/operation.py

@@ -232,7 +232,7 @@ def _wrap_operation(func: Callable[P, Generator], _set_in_op: bool = True) -> Py
         # Configure operation
         #
         # Get the meta kwargs (globals that apply to all hosts)
-        global_arguments, global_argument_keys = pop_global_arguments(kwargs)
+        global_arguments, global_argument_keys = pop_global_arguments(state, host, kwargs)
 
         names, add_args = generate_operation_name(func, host, kwargs, global_arguments)
         op_order, op_hash = solve_operation_consistency(names, state, host)

pyinfra/api/util.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+import hashlib
 from functools import wraps
-from hashlib import sha1
+from hashlib import md5, sha1, sha256
 from inspect import getframeinfo, stack
 from io import BytesIO, StringIO
 from os import getcwd, path, stat
@@ -341,7 +342,7 @@ class get_file_io:
     _close: bool = False
     _file_io: IO[Any]
 
-    def __init__(self, filename_or_io, mode="rb"):
+    def __init__(self, filename_or_io: str | IO, mode: str = "rb"):
         if not (
             # Check we can be read
             hasattr(filename_or_io, "read")
@@ -389,19 +390,32 @@ class get_file_io:
             return self.filename_or_io
 
 
-def get_file_sha1(filename_or_io):
+def get_file_md5(filename_or_io: str | IO):
+    return _get_file_digest(filename_or_io, md5())
+
+
+def get_file_sha1(filename_or_io: str | IO):
+    return _get_file_digest(filename_or_io, sha1())
+
+
+def get_file_sha256(filename_or_io: str | IO):
+    return _get_file_digest(filename_or_io, sha256())
+
+
+def _get_file_digest(filename_or_io: str | IO, hasher: hashlib._Hash):
     """
-    Calculates the SHA1 of a file or file object using a buffer to handle larger files.
+    Calculates the hash of a file or file object using a buffer to handle larger files.
     """
 
     file_data = get_file_io(filename_or_io)
     cache_key = file_data.cache_key
+    if cache_key:
+        cache_key = f"{cache_key}_{hasher.name}"
 
     if cache_key and cache_key in FILE_SHAS:
         return FILE_SHAS[cache_key]
 
     with file_data as file_io:
-        hasher = sha1()
         buff = file_io.read(BLOCKSIZE)
 
         while len(buff) > 0:
@@ -425,7 +439,7 @@ def get_path_permissions_mode(pathname: str):
     """
 
     mode_octal = oct(stat(pathname).st_mode)
-    return mode_octal[-3:]
+    return int(mode_octal[-3:])
 
 
 def raise_if_bad_type(

pyinfra/connectors/docker.py

@@ -33,29 +33,6 @@ connector_data_meta: dict[str, DataMeta] = {
 }
 
 
-def _find_start_docker_container(container_id) -> tuple[str, bool]:
-    docker_info = local.shell("docker container inspect {0}".format(container_id))
-    assert isinstance(docker_info, str)
-    docker_info = json.loads(docker_info)[0]
-    if docker_info["State"]["Running"] is False:
-        logger.info("Starting stopped container: {0}".format(container_id))
-        local.shell("docker container start {0}".format(container_id))
-        return container_id, False
-    return container_id, True
-
-
-def _start_docker_image(image_name):
-    try:
-        return local.shell(
-            "docker run -d {0} tail -f /dev/null".format(image_name),
-            splitlines=True,
-        )[
-            -1
-        ]  # last line is the container ID
-    except PyinfraError as e:
-        raise ConnectError(e.args[0])
-
-
 class DockerConnector(BaseConnector):
     """
     The Docker connector allows you to use pyinfra to create new Docker images or modify running
@@ -89,6 +66,9 @@ class DockerConnector(BaseConnector):
     writing deploys, operations or facts.
     """
 
+    # enable the use of other docker cli compatible tools like podman
+    docker_cmd = "docker"
+
     handles_execution = True
 
     data_cls = ConnectorData
@@ -104,29 +84,54 @@ class DockerConnector(BaseConnector):
         super().__init__(state, host)
         self.local = LocalConnector(state, host)
 
+    @override
     @staticmethod
     def make_names_data(name=None):
         if not name:
             raise InventoryError("No docker base ID provided!")
 
         yield (
-            "@docker/{0}".format(name),
+            f"@docker/{name}",
             {"docker_identifier": name},
             ["@docker"],
         )
 
+    # 2 helper functions
+    def _find_start_docker_container(self, container_id) -> tuple[str, bool]:
+        docker_info = local.shell(f"{self.docker_cmd} container inspect {container_id}")
+        assert isinstance(docker_info, str)
+        docker_info = json.loads(docker_info)[0]
+        if docker_info["State"]["Running"] is False:
+            logger.info(f"Starting stopped container: {container_id}")
+            local.shell(f"{self.docker_cmd} container start {container_id}")
+            return container_id, False
+        return container_id, True
+
+    def _start_docker_image(self, image_name):
+        try:
+            return local.shell(
+                f"{self.docker_cmd} run -d {image_name} tail -f /dev/null",
+                splitlines=True,
+            )[
+                -1
+            ]  # last line is the container ID
+        except PyinfraError as e:
+            raise ConnectError(e.args[0])
+
     @override
     def connect(self) -> None:
         self.local.connect()
 
         docker_identifier = self.data["docker_identifier"]
-        with progress_spinner({"prepare docker container"}):
+        with progress_spinner({f"prepare {self.docker_cmd} container"}):
             try:
-                self.container_id, was_running = _find_start_docker_container(docker_identifier)
+                self.container_id, was_running = self._find_start_docker_container(
+                    docker_identifier
+                )
                 if was_running:
                     self.no_stop = True
             except PyinfraError:
-                self.container_id = _start_docker_image(docker_identifier)
+                self.container_id = self._start_docker_image(docker_identifier)
 
     @override
     def disconnect(self) -> None:
@@ -134,26 +139,28 @@ class DockerConnector(BaseConnector):
 
         if self.no_stop:
             logger.info(
-                "{0}docker build complete, container left running: {1}".format(
+                "{0}{1} build complete, container left running: {2}".format(
                     self.host.print_prefix,
+                    self.docker_cmd,
                     click.style(container_id, bold=True),
                 ),
             )
             return
 
-        with progress_spinner({"docker commit"}):
-            image_id = local.shell("docker commit {0}".format(container_id), splitlines=True)[-1][
+        with progress_spinner({f"{self.docker_cmd} commit"}):
+            image_id = local.shell(f"{self.docker_cmd} commit {container_id}", splitlines=True)[-1][
                 7:19
             ]  # last line is the image ID, get sha256:[XXXXXXXXXX]...
 
-        with progress_spinner({"docker rm"}):
+        with progress_spinner({f"{self.docker_cmd} rm"}):
             local.shell(
-                "docker rm -f {0}".format(container_id),
+                f"{self.docker_cmd} rm -f {container_id}",
             )
 
         logger.info(
-            "{0}docker build complete, image ID: {1}".format(
+            "{0}{1} build complete, image ID: {2}".format(
                 self.host.print_prefix,
+                self.docker_cmd,
                 click.style(image_id, bold=True),
             ),
         )
@@ -175,7 +182,7 @@ class DockerConnector(BaseConnector):
 
         docker_flags = "-it" if local_arguments.get("_get_pty") else "-i"
         docker_command = StringCommand(
-            "docker",
+            self.docker_cmd,
             "exec",
             docker_flags,
             container_id,
@@ -202,7 +209,7 @@ class DockerConnector(BaseConnector):
         **kwargs,  # ignored (sudo/etc)
     ) -> bool:
         """
-        Upload a file/IO object to the target Docker container by copying it to a
+        Upload a file/IO object to the target container by copying it to a
         temporary location and then uploading it into the container using ``docker cp``.
         """
 
@@ -220,7 +227,7 @@ class DockerConnector(BaseConnector):
                     temp_f.write(data)
 
             docker_command = StringCommand(
-                "docker",
+                self.docker_cmd,
                 "cp",
                 temp_filename,
                 f"{self.container_id}:{remote_filename}",
@@ -260,7 +267,7 @@ class DockerConnector(BaseConnector):
         **kwargs,  # ignored (sudo/etc)
     ) -> bool:
         """
-        Download a file from the target Docker container by copying it to a temporary
+        Download a file from the target container by copying it to a temporary
         location and then reading that into our final file/IO object.
         """
 
@@ -268,7 +275,7 @@ class DockerConnector(BaseConnector):
 
         try:
             docker_command = StringCommand(
-                "docker",
+                self.docker_cmd,
                 "cp",
                 f"{self.container_id}:{remote_filename}",
                 temp_filename,
@@ -302,3 +309,75 @@ class DockerConnector(BaseConnector):
             )
 
         return status
+
+
+class PodmanConnector(DockerConnector):
+    """
+    The Podman connector allows you to use pyinfra to create new Podman images or modify running
+    Podman containers.
+
+    .. note::
+
+        The Podman connector allows pyinfra to target Podman containers as inventory and is
+        unrelated to the :doc:`../operations/docker` & :doc:`../facts/docker`.
+
+    You can pass either an image name or existing container ID:
+
+    + Image - will create a new container from the image, execute operations against it, save into \
+        a new Podman image and remove the container
+    + Existing container ID - will execute operations against the running container, leaving it \
+        running
+
+    .. code:: shell
+
+        # A Podman base image must be provided
+        pyinfra @podman/alpine:3.8 ...
+
+        # pyinfra can run on multiple Docker images in parallel
+        pyinfra @podman/alpine:3.8,@podman/ubuntu:bionic ...
+
+        # Execute against a running container
+        pyinfra @podman/2beb8c15a1b1 ...
+
+    The Podman connector is great for testing pyinfra operations locally, rather than connecting to
+    a remote host over SSH each time. This gives you a fast, local-first devloop to iterate on when
+    writing deploys, operations or facts.
+    """
+
+    docker_cmd = "podman"
+
+    @override
+    @staticmethod
+    def make_names_data(name=None):
+        if not name:
+            raise InventoryError("No podman base ID provided!")
+
+        yield (
+            f"@podman/{name}",
+            {"docker_identifier": name},
+            ["@podman"],
+        )
+
+    # Duplicate function definition to swap the docstring.
+    @override
+    def put_file(
+        self,
+        filename_or_io,
+        remote_filename,
+        remote_temp_filename=None,  # ignored
+        print_output=False,
+        print_input=False,
+        **kwargs,  # ignored (sudo/etc)
+    ) -> bool:
+        """
+        Upload a file/IO object to the target container by copying it to a
+        temporary location and then uploading it into the container using ``podman cp``.
+        """
+        return super().put_file(
+            filename_or_io,
+            remote_filename,
+            remote_temp_filename,  # ignored
+            print_output,
+            print_input,
+            **kwargs,  # ignored (sudo/etc)
+        )

pyinfra/connectors/dockerssh.py

@@ -50,6 +50,7 @@ class DockerSSHConnector(BaseConnector):
         super().__init__(state, host)
         self.ssh = SSHConnector(state, host)
 
+    @override
     @staticmethod
     def make_names_data(name):
         try:

pyinfra/connectors/local.py

@@ -38,6 +38,7 @@ class LocalConnector(BaseConnector):
 
     handles_execution = True
 
+    @override
     @staticmethod
     def make_names_data(name=None):
         if name is not None:

pyinfra/connectors/ssh.py

@@ -144,6 +144,7 @@ class SSHConnector(BaseConnector):
 
     client: Optional[SSHClient] = None
 
+    @override
     @staticmethod
     def make_names_data(name):
         yield "@ssh/{0}".format(name), {"ssh_hostname": name}, []

pyinfra/connectors/sshuserclient/client.py

@@ -158,6 +158,7 @@ class SSHClient(ParamikoClient):
             forward_agent,
             missing_host_key_policy,
             host_keys_file,
+            keep_alive,
         ) = self.parse_config(
             hostname,
             kwargs,
@@ -183,8 +184,6 @@ class SSHClient(ParamikoClient):
         if _pyinfra_ssh_forward_agent is not None:
             forward_agent = _pyinfra_ssh_forward_agent
 
-        keep_alive = config.get("keep_alive")
-
         if keep_alive:
             transport = self.get_transport()
             assert transport is not None, "No transport"
@@ -215,13 +214,14 @@ class SSHClient(ParamikoClient):
         cfg: dict = {"port": 22}
         cfg.update(initial_cfg or {})
 
+        keep_alive = 0
         forward_agent = False
         missing_host_key_policy = get_missing_host_key_policy(strict_host_key_checking)
         host_keys_file = path.expanduser("~/.ssh/known_hosts")  # OpenSSH default
 
         ssh_config = get_ssh_config(ssh_config_file)
         if not ssh_config:
-            return hostname, cfg, forward_agent, missing_host_key_policy, host_keys_file
+            return hostname, cfg, forward_agent, missing_host_key_policy, host_keys_file, keep_alive
 
         host_config = ssh_config.lookup(hostname)
         forward_agent = host_config.get("forwardagent") == "yes"
@@ -248,7 +248,7 @@ class SSHClient(ParamikoClient):
             cfg["port"] = int(host_config["port"])
 
         if "serveraliveinterval" in host_config:
-            cfg["keep_alive"] = int(host_config["serveraliveinterval"])
+            keep_alive = int(host_config["serveraliveinterval"])
 
         if "proxycommand" in host_config:
             cfg["sock"] = ProxyCommand(host_config["proxycommand"])
@@ -275,7 +275,7 @@ class SSHClient(ParamikoClient):
                 sock = c.gateway(hostname, cfg["port"], target, target_config["port"])
             cfg["sock"] = sock
 
-        return hostname, cfg, forward_agent, missing_host_key_policy, host_keys_file
+        return hostname, cfg, forward_agent, missing_host_key_policy, host_keys_file, keep_alive
 
     @staticmethod
     def derive_shorthand(ssh_config, host_string):

pyinfra/connectors/terraform.py

@@ -1,5 +1,7 @@
 import json
 
+from typing_extensions import override
+
 from pyinfra import local, logger
 from pyinfra.api.exceptions import InventoryError
 from pyinfra.api.util import memoize
@@ -76,6 +78,7 @@ class TerraformInventoryConnector(BaseConnector):
 
     """
 
+    @override
     @staticmethod
     def make_names_data(name=None):
         show_warning()

pyinfra/connectors/vagrant.py

@@ -3,6 +3,8 @@ from os import path
 from queue import Queue
 from threading import Thread
 
+from typing_extensions import override
+
 from pyinfra import local, logger
 from pyinfra.api.exceptions import InventoryError
 from pyinfra.api.util import memoize
@@ -131,6 +133,7 @@ class VagrantInventoryConnector(BaseConnector):
         pyinfra @vagrant/my-vm-name,@vagrant/another-vm-name ...
     """
 
+    @override
     @staticmethod
     def make_names_data(name=None):
         vagrant_ssh_info = get_vagrant_config(name)