pyinfra 3.5.1__py3-none-any.whl → 3.6__py3-none-any.whl

pyinfra/api/__init__.py

@@ -14,6 +14,7 @@ from .exceptions import (  # noqa: F401
     FactError,
     FactTypeError,
     FactValueError,
+    FactProcessError,
     InventoryError,
     OperationError,
     OperationTypeError,

pyinfra/api/arguments.py

@@ -77,6 +77,9 @@ class ConnectorArguments(TypedDict, total=False):
     _retry_delay: Union[int, float]
     _retry_until: Optional[Callable[[dict], bool]]
 
+    # Temp directory argument
+    _temp_dir: str
+
 
 def generate_env(config: "Config", value: dict) -> dict:
     env = config.ENV.copy()
@@ -163,6 +166,10 @@ shell_argument_meta: dict[str, ArgumentMeta] = {
         "String or buffer to send to the stdin of any commands.",
         default=lambda _: None,
     ),
+    "_temp_dir": ArgumentMeta(
+        "Temporary directory on the remote host for file operations.",
+        default=lambda config: config.TEMP_DIR,
+    ),
 }
 
 

pyinfra/api/exceptions.py

@@ -29,6 +29,12 @@ class FactValueError(FactError, ValueError):
     """
 
 
+class FactProcessError(FactError, RuntimeError):
+    """
+    Exception raised when the data gathered for a fact cannot be processed.
+    """
+
+
 class OperationError(PyinfraError):
     """
     Exception raised during fact gathering staging if an operation is unable to

pyinfra/api/facts.py

@@ -24,6 +24,7 @@ from typing_extensions import override
 from pyinfra import logger
 from pyinfra.api import StringCommand
 from pyinfra.api.arguments import all_global_arguments, pop_global_arguments
+from pyinfra.api.exceptions import FactProcessError
 from pyinfra.api.util import (
     get_kwargs_str,
     log_error_or_warning,
@@ -269,7 +270,22 @@ def _get_fact(
 
     if status:
         if stdout_lines:
-            data = fact.process(stdout_lines)
+            try:
+                data = fact.process(stdout_lines)
+            except FactProcessError as e:
+                log_error_or_warning(
+                    host,
+                    global_kwargs["_ignore_errors"],
+                    description=("could not process fact: {0} {1}").format(
+                        name, get_kwargs_str(fact_kwargs)
+                    ),
+                    exception=e,
+                )
+
+                # Check we've not failed
+                if apply_failed_hosts and not global_kwargs["_ignore_errors"]:
+                    state.fail_hosts({host})
+
     elif stderr_lines:
         # If we have error output and that error is sudo or su stating the user
         # does not exist, do not fail but instead return the default fact value.

pyinfra/api/host.py

@@ -328,6 +328,9 @@ class Host:
 
         return temp_directory
 
+    def get_temp_dir_config(self):
+        return self.state.config.TEMP_DIR or self.state.config.DEFAULT_TEMP_DIR
+
     def get_temp_filename(
         self,
         hash_key: Optional[str] = None,

pyinfra/api/metadata.py

@@ -0,0 +1,69 @@
+"""
+Support parsing pyinfra-metadata.toml
+
+Currently just parses plugins and their metadata.
+"""
+
+import tomllib
+from typing import Literal, get_args
+
+from pydantic import BaseModel, TypeAdapter, field_validator
+
+AllowedTagType = Literal[
+    "boot",
+    "containers",
+    "database",
+    "service-management",
+    "package-manager",
+    "python",
+    "ruby",
+    "javascript",
+    "configuration-management",
+    "security",
+    "storage",
+    "system",
+    "rust",
+    "version-control-system",
+]
+
+
+class Tag(BaseModel):
+    """Representation of a plugin tag."""
+
+    value: AllowedTagType
+
+    @field_validator("value", mode="before")
+    def _validate_value(cls, v) -> AllowedTagType:
+        allowed_tags = set(get_args(AllowedTagType))
+        if v not in allowed_tags:
+            raise ValueError(f"Invalid tag: {v}. Allowed: {allowed_tags}")
+        return v
+
+    @property
+    def title_case(self) -> str:
+        return " ".join([t.title() for t in self.value.split("-")])
+
+
+ALLOWED_TAGS = [Tag(value=tag) for tag in set(get_args(AllowedTagType))]
+
+
+class Plugin(BaseModel):
+    """Representation of a pyinfra plugin."""
+
+    name: str
+    # description: str # FUTURE we should grab these from doc strings
+    path: str
+    type: Literal["operation", "fact", "connector", "deploy"]
+    tags: list[Tag]
+
+    @field_validator("tags", mode="before")
+    def _wrap_tags(cls, v):
+        return [Tag(value=tag) if not isinstance(tag, Tag) else tag for tag in v]
+
+
+def parse_plugins(metadata_text: str) -> list[Plugin]:
+    """Given the contents of a pyinfra-metadata.toml parse out the plugins."""
+    pyinfra_metadata = tomllib.loads(metadata_text).get("pyinfra", None)
+    if not pyinfra_metadata:
+        raise ValueError("Missing [pyinfra.plugins] section in pyinfra-metadata.toml")
+    return TypeAdapter(list[Plugin]).validate_python(pyinfra_metadata["plugins"].values())

pyinfra/api/operations.py

@@ -194,11 +194,11 @@ def _run_host_op(state: "State", host: "Host", op_hash: str) -> Optional[bool]:
         host_results.ops += 1
         host_results.success_ops += 1
 
-        _status_log = "Success" if executed_commands > 0 else "No changes"
+        _status_text = "Success" if executed_commands > 0 else "No changes"
         if retry_attempt > 0:
-            _status_log = f"{_status_log} on retry {retry_attempt}"
+            _status_text = f"{_status_text} on retry {retry_attempt}"
 
-        _click_log_status = click.style(_status_log, "green")
+        _click_log_status = click.style(_status_text, "green" if executed_commands > 0 else "cyan")
         logger.info("{0}{1}".format(host.print_prefix, _click_log_status))
 
         state.trigger_callbacks("operation_host_success", host, op_hash, retry_attempt)

pyinfra/api/util.py

@@ -10,7 +10,7 @@ from socket import error as socket_error, timeout as timeout_error
 from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Type, Union
 
 import click
-from jinja2 import Environment, FileSystemLoader, StrictUndefined
+from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template
 from paramiko import SSHException
 from typeguard import TypeCheckError, check_type
 
@@ -26,7 +26,7 @@ if TYPE_CHECKING:
 BLOCKSIZE = 65536
 
 # Caches
-TEMPLATES: Dict[Any, Any] = {}
+TEMPLATES: Dict[str, Template] = {}
 FILE_SHAS: Dict[Any, Any] = {}
 
 PYINFRA_INSTALL_DIR = path.normpath(path.join(path.dirname(__file__), ".."))
@@ -139,7 +139,9 @@ def get_operation_order_from_stack(state: "State"):
     return line_numbers
 
 
-def get_template(filename_or_io: str | IO, jinja_env_kwargs: dict[str, Any] | None = None):
+def get_template(
+    filename_or_io: str | IO, jinja_env_kwargs: dict[str, Any] | None = None
+) -> Template:
     """
     Gets a jinja2 ``Template`` object for the input filename or string, with caching
     based on the filename of the template, or the SHA1 of the input string.
@@ -155,10 +157,11 @@ def get_template(filename_or_io: str | IO, jinja_env_kwargs: dict[str, Any] | No
     with file_data as file_io:
         template_string = file_io.read()
 
+    default_loader = FileSystemLoader(getcwd())
     template = Environment(
         undefined=StrictUndefined,
         keep_trailing_newline=True,
-        loader=FileSystemLoader(getcwd()),
+        loader=jinja_env_kwargs.pop("loader", default_loader),
         **jinja_env_kwargs,
     ).from_string(template_string)
 
@@ -219,7 +222,11 @@ def log_operation_start(
 
 
 def log_error_or_warning(
-    host: "Host", ignore_errors: bool, description: str = "", continue_on_error: bool = False
+    host: "Host",
+    ignore_errors: bool,
+    description: str = "",
+    continue_on_error: bool = False,
+    exception: Exception | None = None,
 ) -> None:
     log_func = logger.error
     log_color = "red"
@@ -234,6 +241,16 @@ def log_error_or_warning(
         if description:
             log_text = f"{log_text}: "
 
+    if exception:
+        exc = exception.__cause__ or exception
+        exc_text = "{0}: {1}".format(type(exc).__name__, exc)
+        log_func(
+            "{0}{1}".format(
+                host.print_prefix,
+                click.style(exc_text, log_color),
+            ),
+        )
+
     log_func(
         "{0}{1}{2}".format(
             host.print_prefix,

pyinfra/connectors/docker.py

@@ -26,10 +26,14 @@ if TYPE_CHECKING:
 
 class ConnectorData(TypedDict):
     docker_identifier: str
+    docker_platform: str
+    docker_architecture: str
 
 
 connector_data_meta: dict[str, DataMeta] = {
     "docker_identifier": DataMeta("ID of container or image to start from"),
+    "docker_platform": DataMeta("Platform to use for Docker image (e.g., linux/amd64)"),
+    "docker_architecture": DataMeta("Architecture to use for Docker image (e.g., amd64, arm64)"),
 }
 
 
@@ -108,9 +112,29 @@ class DockerConnector(BaseConnector):
         return container_id, True
 
     def _start_docker_image(self, image_name):
+        docker_cmd_parts = [
+            self.docker_cmd,
+            "run",
+            "-d",
+        ]
+
+        if self.data.get("docker_platform"):
+            docker_cmd_parts.extend(["--platform", self.data["docker_platform"]])
+        if self.data.get("docker_architecture"):
+            docker_cmd_parts.extend(["--arch", self.data["docker_architecture"]])
+
+        docker_cmd_parts.extend(
+            [
+                image_name,
+                "tail",
+                "-f",
+                "/dev/null",
+            ]
+        )
+
         try:
             return local.shell(
-                f"{self.docker_cmd} run -d {image_name} tail -f /dev/null",
+                " ".join(docker_cmd_parts),
                 splitlines=True,
             )[-1]  # last line is the container ID
         except PyinfraError as e:

pyinfra/connectors/ssh.py

@@ -9,6 +9,7 @@ from typing import IO, TYPE_CHECKING, Any, Iterable, Optional, Protocol, Tuple
 
 import click
 from paramiko import AuthenticationException, BadHostKeyException, SFTPClient, SSHException
+from paramiko.agent import Agent
 from typing_extensions import TypedDict, Unpack, override
 
 from pyinfra import logger
@@ -286,10 +287,64 @@ class SSHConnector(BaseConnector):
                 f"Host key for {e.hostname} does not match.",
             )
 
+        except SSHException as e:
+            if self._retry_paramiko_agent_keys(hostname, kwargs, e):
+                return
+            raise
+
     @override
     def disconnect(self) -> None:
         self.get_file_transfer_connection.cache.clear()
 
+    def _retry_paramiko_agent_keys(
+        self,
+        hostname: str,
+        kwargs: dict[str, Any],
+        error: SSHException,
+    ) -> bool:
+        # Workaround for Paramiko multi-key bug (paramiko/paramiko#1390).
+        if "no existing session" not in str(error).lower():
+            return False
+
+        if not kwargs.get("allow_agent"):
+            return False
+
+        try:
+            agent_keys = list(Agent().get_keys())
+        except Exception:
+            return False
+
+        if not agent_keys:
+            return False
+
+        # Skip the first agent key, since Paramiko already attempted it
+        attempt_keys = agent_keys[1:] if len(agent_keys) > 1 else agent_keys
+
+        for agent_key in attempt_keys:
+            if self.client is not None:
+                try:
+                    self.client.close()
+                except Exception:
+                    pass
+
+            self.client = SSHClient()
+
+            single_key_kwargs = dict(kwargs)
+            single_key_kwargs["allow_agent"] = False
+            single_key_kwargs["pkey"] = agent_key
+
+            try:
+                self.client.connect(hostname, **single_key_kwargs)
+                return True
+            except AuthenticationException:
+                continue
+            except SSHException as retry_error:
+                if "no existing session" in str(retry_error).lower():
+                    continue
+                raise retry_error
+
+        return False
+
     @override
     def run_shell_command(
         self,
@@ -342,8 +397,10 @@ class SSHConnector(BaseConnector):
                 get_pty=_get_pty,
             )
 
+            # Write any stdin and then close it
             if _stdin:
                 write_stdin(_stdin, stdin_buffer)
+            stdin_buffer.close()
 
             combined_output = read_output_buffers(
                 stdout_buffer,

pyinfra/connectors/util.py

@@ -22,17 +22,17 @@ if TYPE_CHECKING:
 
 
 SUDO_ASKPASS_ENV_VAR = "PYINFRA_SUDO_PASSWORD"
+
+
 SUDO_ASKPASS_COMMAND = r"""
-temp=$(mktemp "${{TMPDIR:=/tmp}}/pyinfra-sudo-askpass-XXXXXXXXXXXX")
+temp=$(mktemp "${{TMPDIR:={0}}}/pyinfra-sudo-askpass-XXXXXXXXXXXX")
 cat >"$temp"<<'__EOF__'
 #!/bin/sh
-printf '%s\n' "${0}"
+printf '%s\n' "${1}"
 __EOF__
 chmod 755 "$temp"
 echo "$temp"
-""".format(
-    SUDO_ASKPASS_ENV_VAR,
-)
+"""
 
 
 def run_local_process(
@@ -44,11 +44,14 @@ def run_local_process(
 ) -> tuple[int, "CommandOutput"]:
     process = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE)
 
-    if stdin:
-        write_stdin(stdin, process.stdin)
-
     assert process.stdout is not None
     assert process.stderr is not None
+    assert process.stdin is not None
+
+    # Write any stdin and then close it
+    if stdin:
+        write_stdin(stdin, process.stdin)
+    process.stdin.close()
 
     combined_output = read_output_buffers(
         process.stdout,
@@ -264,7 +267,9 @@ def extract_control_arguments(arguments: "ConnectorArguments") -> "ConnectorArgu
 def _ensure_sudo_askpass_set_for_host(host: "Host"):
     if host.connector_data.get("sudo_askpass_path"):
         return
-    _, output = host.run_shell_command(SUDO_ASKPASS_COMMAND)
+    _, output = host.run_shell_command(
+        SUDO_ASKPASS_COMMAND.format(host.get_temp_dir_config(), SUDO_ASKPASS_ENV_VAR)
+    )
     host.connector_data["sudo_askpass_path"] = shlex.quote(output.stdout_lines[0])
 
 
@@ -318,6 +323,8 @@ def make_unix_command(
     _retries=0,
     _retry_delay=0,
     _retry_until=None,
+    # Temp dir config (ignored in command generation, used for temp file path generation)
+    _temp_dir=None,
 ) -> StringCommand:
     """
     Builds a shell command with various kwargs.

pyinfra/facts/crontab.py

@@ -125,15 +125,15 @@ class Crontab(FactBase[CrontabFile]):
         # or CrontabFile.to_json()
         [
             {
-               command: "/path/to/command",
-               minute: "*",
-               hour: "*",
-               month: "*",
-               day_of_month: "*",
-               day_of_week: "*",
+                "command": "/path/to/command",
+                "minute": "*",
+                "hour": "*",
+                "month": "*",
+                "day_of_month": "*",
+                "day_of_week": "*",
             },
             {
-                "command": "echo another command
+                "command": "echo another command",
                 "special_time": "@daily",
             }
         ]

pyinfra/facts/files.py

@@ -664,8 +664,7 @@ class Block(FactBase):
 
 class FileContents(FactBase):
     """
-    Returns the contents of a file as a list of lines. Works with both sha1sum and sha1. Returns
-    ``None`` if the file doest not exist.
+    Returns the contents of a file as a list of lines. Returns ``None`` if the file does not exist.
     """
 
     @override

pyinfra/facts/npm.py

@@ -30,7 +30,7 @@ class NpmPackages(FactBase):
     @override
     def command(self, directory=None):
         if directory:
-            return ("cd {0} && npm list -g --depth=0").format(directory)
+            return ("! test -d {0} || (cd {0} && npm list -g --depth=0)").format(directory)
         return "npm list -g --depth=0"
 
     @override

pyinfra/facts/server.py

@@ -52,12 +52,28 @@ class Path(FactBase):
 
 class TmpDir(FactBase):
     """
-    Returns the temporary directory of the current server, if configured.
+    Returns the temporary directory of the current server.
+
+    According to POSIX standards, checks environment variables in this order:
+    1. TMPDIR (if set and accessible)
+    2. TMP (if set and accessible)
+    3. TEMP (if set and accessible)
+    4. Falls back to empty string
     """
 
     @override
     def command(self):
-        return "echo $TMPDIR"
+        return """
+if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ] && [ -w "$TMPDIR" ]; then
+    echo "$TMPDIR"
+elif [ -n "$TMP" ] && [ -d "$TMP" ] && [ -w "$TMP" ]; then
+    echo "$TMP"
+elif [ -n "$TEMP" ] && [ -d "$TEMP" ] && [ -w "$TEMP" ]; then
+    echo "$TEMP"
+else
+    echo ""
+fi
+        """.strip()
 
 
 class Hostname(FactBase):

pyinfra/operations/apk.py

@@ -1,5 +1,5 @@
 """
-Manage apk packages.
+Manage apk packages. (Alpine Linux)
 """
 
 from __future__ import annotations
@@ -64,6 +64,7 @@ def packages(
 
     .. code:: python
 
+        from pyinfra.operations import apk
         # Update package list and install packages
         apk.packages(
             name="Install Asterisk and Vim",

pyinfra/operations/apt.py

@@ -4,7 +4,7 @@ Manage apt packages and repositories.
 
 from __future__ import annotations
 
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 from urllib.parse import urlparse
 
 from pyinfra import host
@@ -67,6 +67,7 @@ def key(src: str | None = None, keyserver: str | None = None, keyid: str | list[
 
     .. code:: python
 
+        from pyinfra.operations import apt
         # Note: If using URL, wget is assumed to be installed.
         apt.key(
             name="Add the Docker apt gpg key",
@@ -309,12 +310,19 @@ def update(cache_time: int | None = None):
         # Ubuntu provides this handy file
         cache_info = host.get_fact(File, path=APT_UPDATE_FILENAME)
 
-        # Time on files is not tz-aware, and will be the same tz as the server's time,
-        # so we can safely remove the tzinfo from the Date fact before comparison.
-        host_cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta(seconds=cache_time)
-        if cache_info and cache_info["mtime"] and cache_info["mtime"] > host_cache_time:
-            host.noop("apt is already up to date")
-            return
+        if cache_info and cache_info["mtime"]:
+            # The fact Date contains the date of the server in its timezone.
+            # cache_info["mtime"] ignores the timezone and consider the timestamp as UTC.
+            # So let's do the same here for the server current Date : ignore the
+            # timezone and consider it as UTC to have correct comparison with
+            # cache_info["mtime].
+            host_utc_current_time = datetime.fromtimestamp(
+                host.get_fact(Date).timestamp(), timezone.utc
+            ).replace(tzinfo=None)
+            host_cache_time = host_utc_current_time - timedelta(seconds=cache_time)
+            if cache_info["mtime"] > host_cache_time:
+                host.noop("apt is already up to date")
+                return
 
     yield "apt-get update"
 

pyinfra/operations/brew.py

@@ -61,6 +61,7 @@ def packages(
 
     .. code:: python
 
+        from pyinfra.operations import brew
         # Update package list and install packages
         brew.packages(
             name='Install Vim and vimpager',

pyinfra/operations/crontab.py

@@ -54,6 +54,7 @@ def crontab(
 
     .. code:: python
 
+        from pyinfra.operations import crontab
         # simple example for a crontab
         crontab.crontab(
             name="Backup /etc weekly",
@@ -84,7 +85,9 @@ def crontab(
         ctb = ctb0
     name_comment = "# pyinfra-name={0}".format(cron_name)
 
-    existing_crontab = ctb.get_command(command=command, name=cron_name)
+    existing_crontab = ctb.get_command(
+        command=command if cron_name is None else None, name=cron_name
+    )
     existing_crontab_command = existing_crontab["command"] if existing_crontab else command
     existing_crontab_match = existing_crontab["command"] if existing_crontab else command
 

pyinfra/operations/dnf.py

@@ -25,9 +25,12 @@ def key(src: str):
 
     .. code:: python
 
+        from pyinfra import host
+        from pyinfra.operations import dnf
+        from pyinfra.facts.server import LinuxDistribution
         linux_id = host.get_fact(LinuxDistribution)["release_meta"].get("ID")
         dnf.key(
-            name="Add the Docker CentOS gpg key",
+            name="Add the Docker gpg key",
             src=f"https://download.docker.com/linux/{linux_id}/gpg",
         )