pyinfra 3.0.2__tar.gz → 3.1.1__tar.gz

{pyinfra-3.0.2 → pyinfra-3.1.1}/CHANGELOG.md

@@ -1,3 +1,44 @@
+# v3.1.1
+
+- Improve errors with 2.x style `@decorator` (vs `@decorator()`) functions
+- Document adding custom connectors (@simonhammes)
+- Add basic API example to docs (@pirate)
+- Fix sphinx warnings (@simonhammes)
+- Fix force & pull arguments in `git.worktree` operation
+- Fix `server.reboot` reconnection (@wackou)
+- Fix chroot/local connector non-utf file gets (@evoldstad)
+- Fix `AptSources` fact to parse components in order & with digits (@rsfzi)
+
+# v3.1
+
+Here's pyinfra 3.1 - a release primarily driven by contributors new and old - a HUGE THANK YOU to all of you who dedicate time to work on pushing pyinfra forward. New stuff:
+
+- Add `zfs` operations (`dataset`, `snapshot`, `volume`, `filesystem`) facts (`Pools`, `Datasets`, `Filesystems`, `Snapshots`, `Volumes`) (@taliaferro)
+- Add `flatpak` operations (`packages`) and facts (`FlatpakPackage`, `FlatpakPackages`) (@JustScreaMy)
+- Add `jinja_env_kwargs` argument to `files.template` operation (@DonDebonair)
+- Add using dictionaries as `@terraform` output (map from group -> hosts)
+- Add default `@terraform` output key - `pyinfra_inventory.value`, promote connector to beta
+- Add support for multiple keys in each `server.authorized_keys` file (@matthijskooijman)
+- Add print all dependency versions with `--support` flag (@kytta)
+
+Fixes:
+
+- Fix when `ssh_hostname` is set as override data, don't do inventory hostname check
+- Fix `apt.AptSources` parsing special characters (@CondensedTea)
+- Fix `server.reboot` connection detection (@bauen1 + @lemmi)
+- Fix systemd flagging of sockets running (@bauen1)
+- Fix mysql dump quoting (@simonhammes)
+- Fix tilde expansion in files facts (@simonhammes)
+- Fix host lookup check with SSH alias config (@simonhammes)
+- Fix crontab comparison (@andrew-d)
+
+Docs/internal tweaks:
+
+- Improve operations documentation (@bauen1)
+- Default to local machine if `user_name` set in systecmt (@bauen1)
+- Improve efficiency of Docker operations (@apecnascimento)
+- Shallow copy `host.data` data to mutation
+
 # v3.0.2
 
 - Fix `OperationMeta.did_change`: this is now a function as originally designed

{pyinfra-3.0.2/pyinfra.egg-info → pyinfra-3.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyinfra
-Version: 3.0.2
+Version: 3.1.1
 Summary: pyinfra automates/provisions/manages/deploys infrastructure.
 Home-page: https://pyinfra.com
 Author: Nick / Fizzadar

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/api/arguments.py

@@ -137,7 +137,7 @@ shell_argument_meta: dict[str, ArgumentMeta] = {
     ),
     "_chdir": ArgumentMeta(
         "Directory to switch to before executing the command.",
-        default=lambda _: "",
+        default=lambda _: None,
     ),
     "_env": ArgumentMeta(
         "Dictionary of environment variables to set.",

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/api/deploy.py

@@ -60,6 +60,14 @@ def deploy(name: Optional[str] = None, data_defaults=None):
     and wraps any operations called inside with any deploy-wide kwargs/data.
     """
 
+    if name and not isinstance(name, str):
+        raise PyinfraError(
+            (
+                "The `deploy` decorator must be called, ie `@deploy()`, "
+                "see: https://docs.pyinfra.com/en/3.x/compatibility.html#upgrading-pyinfra-from-2-x-3-x"  # noqa
+            )
+        )
+
     def decorator(func: Callable[P, Any]) -> PyinfraOperation[P]:
         func.deploy_name = name or func.__name__  # type: ignore[attr-defined]
         if data_defaults:

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/api/host.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from contextlib import contextmanager
+from copy import copy
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -41,7 +42,6 @@ def extract_callable_datas(
         # the data is stored on the state temporarily.
         if callable(data):
             data = data()
-
         yield data
 
 
@@ -66,7 +66,10 @@ class HostData:
     def __getattr__(self, key: str):
         for data in extract_callable_datas(self.datas):
             try:
-                return data[key]
+                # Take a shallow copy of the object here, we don't want modifications
+                # to host.data.<X> to stick, instead setting host.data.<Y> = is the
+                # correct way to achieve this (see __setattr__).
+                return copy(data[key])
             except KeyError:
                 pass
 
@@ -403,12 +406,13 @@ class Host:
         # Disconnect is an optional function for connectors if needed
         disconnect_func = getattr(self.connector, "disconnect", None)
         if disconnect_func:
-            return disconnect_func()
+            disconnect_func()
 
         # TODO: consider whether this should be here!
         remove_any_sudo_askpass_file(self)
 
         self.state.trigger_callbacks("host_disconnect", self)
+        self.connected = False
 
     def run_shell_command(self, *args, **kwargs) -> tuple[bool, CommandOutput]:
         """

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/api/util.py

@@ -139,12 +139,13 @@ def get_operation_order_from_stack(state: "State"):
     return line_numbers
 
 
-def get_template(filename_or_io: str | IO):
+def get_template(filename_or_io: str | IO, jinja_env_kwargs: dict[str, Any] | None = None):
     """
     Gets a jinja2 ``Template`` object for the input filename or string, with caching
     based on the filename of the template, or the SHA1 of the input string.
     """
-
+    if jinja_env_kwargs is None:
+        jinja_env_kwargs = {}
     file_data = get_file_io(filename_or_io, mode="r")
     cache_key = file_data.cache_key
 
@@ -158,6 +159,7 @@ def get_template(filename_or_io: str | IO):
         undefined=StrictUndefined,
         keep_trailing_newline=True,
         loader=FileSystemLoader(getcwd()),
+        **jinja_env_kwargs,
     ).from_string(template_string)
 
     if cache_key:

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/connectors/chroot.py

@@ -174,7 +174,7 @@ class ChrootConnector(BaseConnector):
             )
 
             # Load the temporary file and write it to our file or IO object
-            with open(temp_filename, encoding="utf-8") as temp_f:
+            with open(temp_filename, "rb") as temp_f:
                 with get_file_io(filename_or_io, "wb") as file_io:
                     data = temp_f.read()
                     data_bytes: bytes

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/connectors/local.py

@@ -184,7 +184,7 @@ class LocalConnector(BaseConnector):
                 raise IOError(output.stderr)
 
             # Load our file or IO object and write it to the temporary file
-            with open(temp_filename, encoding="utf-8") as temp_f:
+            with open(temp_filename, "rb") as temp_f:
                 with get_file_io(filename_or_io, "wb") as file_io:
                     data_bytes: bytes
 

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/connectors/ssh.py

@@ -264,6 +264,9 @@ class SSHConnector(BaseConnector):
                 f"Host key for {e.hostname} does not match.",
             )
 
+    def disconnect(self) -> None:
+        self.get_sftp_connection.cache.clear()
+
     def run_shell_command(
         self,
         command: StringCommand,
@@ -606,7 +609,7 @@ class SSHConnector(BaseConnector):
             rsync_flags=" ".join(flags),
             ssh_flags=" ".join(ssh_flags),
             remote_rsync_command=remote_rsync_command,
-            user=user,
+            user=user or "",
             hostname=hostname,
             src=src,
             dest=dest,

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/connectors/terraform.py

@@ -10,16 +10,15 @@ from .base import BaseConnector
 
 @memoize
 def show_warning():
-    logger.warning("The @terraform connector is in alpha!")
+    logger.warning("The @terraform connector is in beta!")
 
 
 def _flatten_dict_gen(d, parent_key, sep):
     for k, v in d.items():
         new_key = parent_key + sep + k if parent_key else k
+        yield new_key, v
         if isinstance(v, dict):
             yield from _flatten_dict(v, new_key, sep=sep).items()
-        else:
-            yield new_key, v
 
 
 def _flatten_dict(d: dict, parent_key: str = "", sep: str = "."):
@@ -82,7 +81,9 @@ class TerraformInventoryConnector(BaseConnector):
         show_warning()
 
         if not name:
-            name = ""
+            # This is the default which allows one to create a Terraform output
+            # "pyinfra" and directly call: pyinfra @terraform ...
+            name = "pyinfra_inventory.value"
 
         with progress_spinner({"fetch terraform output"}):
             tf_output_raw = local.shell("terraform output -json")
@@ -96,27 +97,36 @@ class TerraformInventoryConnector(BaseConnector):
             keys = "\n".join(f"   - {k}" for k in tf_output.keys())
             raise InventoryError(f"No Terraform output with key: `{name}`, valid keys:\n{keys}")
 
-        if not isinstance(tf_output_value, list):
+        if not isinstance(tf_output_value, (list, dict)):
             raise InventoryError(
                 "Invalid Terraform output type, should be `list`, got "
                 f"`{type(tf_output_value).__name__}`",
             )
 
-        for ssh_target in tf_output_value:
-            if isinstance(ssh_target, dict):
-                name = ssh_target.pop("name", ssh_target.get("ssh_hostname"))
-                if name is None:
-                    raise InventoryError(
-                        "Invalid Terraform list item, missing `name` or `ssh_hostname` keys",
-                    )
-                yield f"@terraform/{name}", ssh_target, ["@terraform"]
-
-            elif isinstance(ssh_target, str):
-                data = {"ssh_hostname": ssh_target}
-                yield f"@terraform/{ssh_target}", data, ["@terraform"]
+        if isinstance(tf_output_value, list):
+            tf_output_value = {
+                "all": tf_output_value,
+            }
 
-            else:
+        for group_name, hosts in tf_output_value.items():
+            if not isinstance(hosts, list):
                 raise InventoryError(
-                    "Invalid Terraform list item, should be `dict` or `str` got "
-                    f"`{type(ssh_target).__name__}`",
+                    "Invalid Terraform map value type, all values should be `list`, got "
+                    f"`{type(hosts).__name__}`",
                 )
+            for host in hosts:
+                if isinstance(host, dict):
+                    name = host.pop("name", host.get("ssh_hostname"))
+                    if name is None:
+                        raise InventoryError(
+                            "Invalid Terraform list item, missing `name` or `ssh_hostname` keys",
+                        )
+                    yield f"@terraform/{name}", host, ["@terraform", group_name]
+                elif isinstance(host, str):
+                    data = {"ssh_hostname": host}
+                    yield f"@terraform/{host}", data, ["@terraform", group_name]
+                else:
+                    raise InventoryError(
+                        "Invalid Terraform list item, should be `dict` or `str` got "
+                        f"`{type(host).__name__}`",
+                    )

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/facts/apt.py

@@ -9,7 +9,7 @@ from .util import make_cat_files_command
 
 
 def parse_apt_repo(name):
-    regex = r"^(deb(?:-src)?)(?:\s+\[([^\]]+)\])?\s+([^\s]+)\s+([a-z-]+)\s+([a-z-\s]*)$"
+    regex = r"^(deb(?:-src)?)(?:\s+\[([^\]]+)\])?\s+([^\s]+)\s+([^\s]+)\s+([a-z-\s\d]*)$"
 
     matches = re.match(regex, name)
 
@@ -32,7 +32,7 @@ def parse_apt_repo(name):
         "type": matches.group(1),
         "url": matches.group(3),
         "distribution": matches.group(4),
-        "components": set(matches.group(5).split()),
+        "components": list(matches.group(5).split()),
     }
 
 

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/facts/files.py

@@ -5,6 +5,7 @@ The files facts provide information about the filesystem and it's contents on th
 from __future__ import annotations
 
 import re
+import shlex
 import stat
 from datetime import datetime
 from typing import TYPE_CHECKING, List, Optional, Tuple, Union
@@ -109,13 +110,19 @@ class File(FactBase[Union[FileDict, Literal[False], None]]):
     type = "file"
 
     def command(self, path):
+        if path.startswith("~/"):
+            # Do not quote leading tilde to ensure that it gets properly expanded by the shell
+            path = f"~/{shlex.quote(path[2:])}"
+        else:
+            path = QuoteString(path)
+
         return make_formatted_string_command(
             (
                 # only stat if the path exists (file or symlink)
                 "! (test -e {0} || test -L {0} ) || "
                 "( {linux_stat_command} {0} 2> /dev/null || {bsd_stat_command} {0} )"
             ),
-            QuoteString(path),
+            path,
             linux_stat_command=LINUX_STAT_COMMAND,
             bsd_stat_command=BSD_STAT_COMMAND,
         )

pyinfra-3.1.1/pyinfra/facts/flatpak.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import re
+
+from pyinfra.api import FactBase
+
+
+class FlatpakBaseFact(FactBase):
+    abstract = True
+
+    def requires_command(self, *args, **kwargs) -> str:
+        return "flatpak"
+
+
+class FlatpakPackage(FlatpakBaseFact):
+    """
+    Returns information for an installed flatpak package
+
+    .. code:: python
+
+        {
+            "id": "org.signal.Signal",
+            "ref": "app/org.signal.Signal/x86_64/stable",
+            "version": "7.12.0"
+        }
+    """
+
+    default = dict
+    _regexes = {
+        "id": "^[ ]+ID:[ ]+(.*)$",
+        "ref": r"^[ ]+Ref:[ ]+(.*)$",
+        "version": r"^[ ]+Version:[ ]+([\w\d.-]+).*$",
+    }
+
+    def command(self, package):
+        return f"flatpak info {package}"
+
+    def process(self, output):
+        data = {}
+        for line in output:
+            for regex_name, regex in self._regexes.items():
+                matches = re.match(regex, line)
+                if matches:
+                    data[regex_name] = matches.group(1)
+
+        return data
+
+
+class FlatpakPackages(FlatpakBaseFact):
+    """
+    Returns a list of installed flatpak packages:
+
+    .. code:: python
+
+        [
+            "org.gnome.Platform",
+            "org.kde.Platform",
+            "org.kde.Sdk",
+            "org.libreoffice.LibreOffice",
+            "org.videolan.VLC"
+        ]
+    """
+
+    default = list
+
+    def command(self):
+        return "flatpak list --columns=application"
+
+    def process(self, output):
+        return [flatpak for flatpak in output[1:]]

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/facts/hardware.py

@@ -109,6 +109,7 @@ class NetworkDevices(FactBase):
     ``ipv6_addresses`` facts for easier-to-use shortcuts to get device addresses.
 
     .. code:: python
+
         "enp1s0": {
             "ether": "12:34:56:78:9A:BC",
             "mtu": 1500,

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/facts/selinux.py

@@ -58,7 +58,8 @@ class FileContext(FactBase):
 class FileContextMapping(FactBase):
     """
     Returns structured SELinux file context data for the specified target path prefix
-    using the same format as :ref:`selinux.FileContext`.  If there is no mapping, it returns ``{}``
+    using the same format as :ref:`facts:selinux.FileContext`.
+    If there is no mapping, it returns ``{}``
     Note: This fact requires root privileges.
     """
 
@@ -85,6 +86,7 @@ class SEPorts(FactBase):
     Note: This fact requires root privileges.
 
     .. code:: python
+
         {
             "tcp": { 22: "ssh_port_t", ...},
             "udp": { ...}

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/facts/systemd.py

@@ -32,6 +32,9 @@ def _make_systemctl_cmd(user_mode=False, machine=None, user_name=None):
             systemctl_cmd.append("--machine={1}@{0}".format(machine, user_name))
         else:
             systemctl_cmd.append("--machine={0}".format(machine))
+    elif user_name is not None:
+        # If only the user is given, assume that the connection should be made to the local machine
+        systemctl_cmd.append("--machine={0}@.host".format(user_name))
 
     return StringCommand(*systemctl_cmd)
 
@@ -58,7 +61,7 @@ class SystemdStatus(FactBase[Dict[str, bool]]):
     default = dict
 
     state_key = "SubState"
-    state_values = ["running", "waiting", "exited"]
+    state_values = ["running", "waiting", "exited", "listening"]
 
     def command(
         self,

pyinfra-3.1.1/pyinfra/facts/zfs.py

@@ -0,0 +1,57 @@
+"""
+Manage ZFS filesystems.
+"""
+
+from pyinfra.api import FactBase, ShortFactBase
+
+
+def _process_zfs_props_table(output):
+    datasets: dict = {}
+    for line in output:
+        dataset, property, value, source = tuple(line.split("\t"))
+        if dataset not in datasets:
+            datasets[dataset] = {}
+        datasets[dataset][property] = value
+    return datasets
+
+
+class Pools(FactBase):
+    def command(self):
+        return "zpool get -H all"
+
+    @staticmethod
+    def process(output):
+        return _process_zfs_props_table(output)
+
+
+class Datasets(FactBase):
+    def command(self):
+        return "zfs get -H all"
+
+    @staticmethod
+    def process(output):
+        return _process_zfs_props_table(output)
+
+
+class Filesystems(ShortFactBase):
+    fact = Datasets
+
+    @staticmethod
+    def process_data(data):
+        return {name: props for name, props in data.items() if props.get("type") == "filesystem"}
+
+
+class Snapshots(ShortFactBase):
+    fact = Datasets
+
+    @staticmethod
+    def process_data(data):
+        return {name: props for name, props in data.items() if props.get("type") == "snapshot"}
+
+
+class Volumes(ShortFactBase):
+    fact = Datasets
+
+    @staticmethod
+    def process_data(data):
+        return {name: props for name, props in data.items() if props.get("type") == "volume"}

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/operations/apt.py

@@ -53,7 +53,8 @@ def key(src: str | None = None, keyserver: str | None = None, keyid: str | list[
     .. warning::
         ``apt-key`` is deprecated in Debian, it is recommended NOT to use this
         operation and instead follow the instructions here:
-            https://wiki.debian.org/DebianRepository/UseThirdParty
+
+        https://wiki.debian.org/DebianRepository/UseThirdParty
 
     **Examples:**
 

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/operations/docker.py

@@ -4,7 +4,7 @@ Manager Docker Containers, Volumes and Networks
 
 from pyinfra import host
 from pyinfra.api import operation
-from pyinfra.facts.docker import DockerContainers, DockerNetworks, DockerVolumes
+from pyinfra.facts.docker import DockerContainer, DockerNetwork, DockerVolume
 
 from .util.docker import handle_docker
 
@@ -68,7 +68,7 @@ def container(
         )
     """
 
-    existent_container = [c for c in host.get_fact(DockerContainers) if container in c["Name"]]
+    existent_container = host.get_fact(DockerContainer, object_id=container)
 
     if force:
         if existent_container:
@@ -183,7 +183,7 @@ def volume(volume, driver="", labels=None, present=True):
         )
     """
 
-    existent_volume = [v for v in host.get_fact(DockerVolumes) if v["Name"] == volume]
+    existent_volume = host.get_fact(DockerVolume, object_id=volume)
 
     if present:
 
@@ -257,7 +257,7 @@ def network(
             present=True,
         )
     """
-    existent_network = [n for n in host.get_fact(DockerNetworks) if n["Name"] == network]
+    existent_network = host.get_fact(DockerNetwork, object_id=network)
 
     if present:
         if existent_network:

{pyinfra-3.0.2 → pyinfra-3.1.1}/pyinfra/operations/files.py

@@ -256,7 +256,7 @@ def line(
         change bits of lines, see ``files.replace``.
 
     Regex line escaping:
-        If matching special characters (eg a crontab line containing *), remember to escape
+        If matching special characters (eg a crontab line containing ``*``), remember to escape
         it first using Python's ``re.escape``.
 
     Backup:
@@ -523,7 +523,7 @@ def sync(
     + mode: permissions of the files
     + dir_mode: permissions of the directories
     + delete: delete remote files not present locally
-    + exclude: string or list/tuple of strings to match & exclude files (eg *.pyc)
+    + exclude: string or list/tuple of strings to match & exclude files (eg ``*.pyc``)
     + exclude_dir: string or list/tuple of strings to match & exclude directories (eg node_modules)
     + add_deploy_dir: interpret src as relative to deploy directory instead of current directory
 
@@ -915,7 +915,8 @@ def template(
     user: str | None = None,
     group: str | None = None,
     mode: str | None = None,
-    create_remote_dir=True,
+    create_remote_dir: bool = True,
+    jinja_env_kwargs: dict[str, Any] | None = None,
     **data,
 ):
     '''
@@ -927,12 +928,18 @@ def template(
     + group: group to own the files
     + mode: permissions of the files
     + create_remote_dir: create the remote directory if it doesn't exist
+    + jinja_env_kwargs: keyword arguments to be passed into the jinja Environment()
 
     ``create_remote_dir``:
         If the remote directory does not exist it will be created using the same
         user & group as passed to ``files.put``. The mode will *not* be copied over,
         if this is required call ``files.directory`` separately.
 
+    ``jinja_env_kwargs``:
+        To have more control over how jinja2 renders your template, you can pass
+        a dict with arguments that will be passed as keyword args to the jinja2
+        `Environment() <https://jinja.palletsprojects.com/en/3.0.x/api/#jinja2.Environment>`_.
+
     Notes:
        Common convention is to store templates in a "templates" directory and
        have a filename suffix with '.j2' (for jinja2).
@@ -1002,7 +1009,7 @@ def template(
 
     # Render and make file-like it's output
     try:
-        output = get_template(src).render(data)
+        output = get_template(src, jinja_env_kwargs).render(data)
     except (TemplateRuntimeError, TemplateSyntaxError, UndefinedError) as e:
         trace_frames = [
             frame

pyinfra-3.1.1/pyinfra/operations/flatpak.py

@@ -0,0 +1,79 @@
+"""
+Manage flatpak packages. See https://www.flatpak.org/
+"""
+
+from __future__ import annotations
+
+from pyinfra import host
+from pyinfra.api import operation
+from pyinfra.facts.flatpak import FlatpakPackages
+
+
+@operation()
+def packages(
+    packages: str | list[str] | None = None,
+    present=True,
+):
+    """
+    Install/remove a flatpak package
+
+    + packages: List of packages
+    + present: whether the package should be installed
+
+    **Examples:**
+
+    .. code:: python
+
+        # Install vlc flatpak
+        flatpak.package(
+            name="Install vlc",
+            packages="org.videolan.VLC",
+        )
+
+        # Install multiple flatpaks
+        flatpak.package(
+            name="Install vlc and kodi",
+            packages=["org.videolan.VLC", "tv.kodi.Kodi"],
+        )
+
+        # Remove vlc
+        flatpak.package(
+            name="Remove vlc",
+            packages="org.videolan.VLC",
+            present=False,
+        )
+    """
+
+    if packages is None:
+        return
+
+    if isinstance(packages, str):
+        packages = [packages]
+
+    flatpak_packages = host.get_fact(FlatpakPackages)
+
+    install_packages = []
+    remove_packages = []
+
+    for package in packages:
+        # it's installed
+        if package in flatpak_packages:
+            if not present:
+                # we don't want it
+                remove_packages.append(package)
+
+        # it's not installed
+        if package not in flatpak_packages:
+            # we want it
+            if present:
+                install_packages.append(package)
+
+            # we don't want it
+            else:
+                host.noop(f"flatpak package {package} is not installed")
+
+    if install_packages:
+        yield " ".join(["flatpak", "install", "--noninteractive"] + install_packages)
+
+    if remove_packages:
+        yield " ".join(["flatpak", "uninstall", "--noninteractive"] + remove_packages)