pyinfra 3.0.1__tar.gz → 3.1__tar.gz

{pyinfra-3.0.1 → pyinfra-3.1}/CHANGELOG.md

@@ -1,3 +1,39 @@
+# v3.1
+
+Here's pyinfra 3.1 - a release primarily driven by contributors new and old - a HUGE THANK YOU to all of you who dedicate time to work on pushing pyinfra forward. New stuff:
+
+- Add `zfs` operations (`dataset`, `snapshot`, `volume`, `filesystem`) facts (`Pools`, `Datasets`, `Filesystems`, `Snapshots`, `Volumes`) (@taliaferro)
+- Add `flatpak` operations (`packages`) and facts (`FlatpakPackage`, `FlatpakPackages`) (@JustScreaMy)
+- Add `jinja_env_kwargs` argument to `files.template` operation (@DonDebonair)
+- Add using dictionaries as `@terraform` output (map from group -> hosts)
+- Add default `@terraform` output key - `pyinfra_inventory.value`, promote connector to beta
+- Add support for multiple keys in each `server.authorized_keys` file (@matthijskooijman)
+- Add print all dependency versions with `--support` flag (@kytta)
+
+Fixes:
+
+- Fix when `ssh_hostname` is set as override data, don't do inventory hostname check
+- Fix `apt.AptSources` parsing special characters (@CondensedTea)
+- Fix `server.reboot` connection detection (@bauen1 + @lemmi)
+- Fix systemd flagging of sockets running (@bauen1)
+- Fix mysql dump quoting (@simonhammes)
+- Fix tilde expansion in files facts (@simonhammes)
+- Fix host lookup check with SSH alias config (@simonhammes)
+- Fix crontab comparison (@andrew-d)
+
+Docs/internal tweaks:
+
+- Improve operations documentation (@bauen1)
+- Default to local machine if `user_name` set in systecmt (@bauen1)
+- Improve efficiency of Docker operations (@apecnascimento)
+- Shallow copy `host.data` data to mutation
+
+# v3.0.2
+
+- Fix `OperationMeta.did_change`: this is now a function as originally designed
+- Add quick test for `host.when` context manager
+- Remove extra detected changes note when not relevant
+
 # v3.0.1
 
 - Switch to `command -v` not `which` in `server.Which` fact (@lemmi)

{pyinfra-3.0.1/pyinfra.egg-info → pyinfra-3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyinfra
-Version: 3.0.1
+Version: 3.1
 Summary: pyinfra automates/provisions/manages/deploys infrastructure.
 Home-page: https://pyinfra.com
 Author: Nick / Fizzadar

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/api/host.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from contextlib import contextmanager
+from copy import copy
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -41,7 +42,6 @@ def extract_callable_datas(
         # the data is stored on the state temporarily.
         if callable(data):
             data = data()
-
         yield data
 
 
@@ -66,7 +66,10 @@ class HostData:
     def __getattr__(self, key: str):
         for data in extract_callable_datas(self.datas):
             try:
-                return data[key]
+                # Take a shallow copy of the object here, we don't want modifications
+                # to host.data.<X> to stick, instead setting host.data.<Y> = is the
+                # correct way to achieve this (see __setattr__).
+                return copy(data[key])
             except KeyError:
                 pass
 

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/api/operation.py

@@ -60,7 +60,7 @@ class OperationMeta:
         if self._commands is not None:
             return (
                 "OperationMeta(executed=True, "
-                f"success={self.did_succeed}, hash={self._hash}, commands={len(self._commands)})"
+                f"success={self.did_succeed()}, hash={self._hash}, commands={len(self._commands)})"
             )
         return (
             "OperationMeta(executed=False, "
@@ -87,6 +87,12 @@ class OperationMeta:
         if not self.is_complete():
             raise RuntimeError("Cannot evaluate operation result before execution")
 
+    @property
+    def executed(self) -> bool:
+        if self._commands is None:
+            return False
+        return len(self._commands) > 0
+
     @property
     def will_change(self) -> bool:
         if self._maybe_is_change is not None:
@@ -100,16 +106,12 @@ class OperationMeta:
         self._maybe_is_change = False
         return False
 
-    def _did_change(self) -> bool:
+    def did_change(self) -> bool:
+        self._raise_if_not_complete()
         return bool(self._success and len(self._commands or []) > 0)
 
-    @property
-    def did_change(self):
-        return context.host.when(self._did_change)
-
-    @property
-    def did_not_change(self):
-        return context.host.when(lambda: not self._did_change())
+    def did_not_change(self) -> bool:
+        return not self.did_change()
 
     def did_succeed(self, _raise_if_not_complete=True) -> bool:
         if _raise_if_not_complete:
@@ -124,7 +126,7 @@ class OperationMeta:
     @property
     def changed(self) -> bool:
         if self.is_complete():
-            return self._did_change()
+            return self.did_change()
         return self.will_change
 
     @property

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/api/util.py

@@ -139,12 +139,13 @@ def get_operation_order_from_stack(state: "State"):
     return line_numbers
 
 
-def get_template(filename_or_io: str | IO):
+def get_template(filename_or_io: str | IO, jinja_env_kwargs: dict[str, Any] | None = None):
     """
     Gets a jinja2 ``Template`` object for the input filename or string, with caching
     based on the filename of the template, or the SHA1 of the input string.
     """
-
+    if jinja_env_kwargs is None:
+        jinja_env_kwargs = {}
     file_data = get_file_io(filename_or_io, mode="r")
     cache_key = file_data.cache_key
 
@@ -158,6 +159,7 @@ def get_template(filename_or_io: str | IO):
         undefined=StrictUndefined,
         keep_trailing_newline=True,
         loader=FileSystemLoader(getcwd()),
+        **jinja_env_kwargs,
     ).from_string(template_string)
 
     if cache_key:

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/connectors/ssh.py

@@ -606,7 +606,7 @@ class SSHConnector(BaseConnector):
             rsync_flags=" ".join(flags),
             ssh_flags=" ".join(ssh_flags),
             remote_rsync_command=remote_rsync_command,
-            user=user,
+            user=user or "",
             hostname=hostname,
             src=src,
             dest=dest,

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/connectors/terraform.py

@@ -10,16 +10,15 @@ from .base import BaseConnector
 
 @memoize
 def show_warning():
-    logger.warning("The @terraform connector is in alpha!")
+    logger.warning("The @terraform connector is in beta!")
 
 
 def _flatten_dict_gen(d, parent_key, sep):
     for k, v in d.items():
         new_key = parent_key + sep + k if parent_key else k
+        yield new_key, v
         if isinstance(v, dict):
             yield from _flatten_dict(v, new_key, sep=sep).items()
-        else:
-            yield new_key, v
 
 
 def _flatten_dict(d: dict, parent_key: str = "", sep: str = "."):
@@ -82,7 +81,9 @@ class TerraformInventoryConnector(BaseConnector):
         show_warning()
 
         if not name:
-            name = ""
+            # This is the default which allows one to create a Terraform output
+            # "pyinfra" and directly call: pyinfra @terraform ...
+            name = "pyinfra_inventory.value"
 
         with progress_spinner({"fetch terraform output"}):
             tf_output_raw = local.shell("terraform output -json")
@@ -96,27 +97,36 @@ class TerraformInventoryConnector(BaseConnector):
             keys = "\n".join(f"   - {k}" for k in tf_output.keys())
             raise InventoryError(f"No Terraform output with key: `{name}`, valid keys:\n{keys}")
 
-        if not isinstance(tf_output_value, list):
+        if not isinstance(tf_output_value, (list, dict)):
             raise InventoryError(
                 "Invalid Terraform output type, should be `list`, got "
                 f"`{type(tf_output_value).__name__}`",
             )
 
-        for ssh_target in tf_output_value:
-            if isinstance(ssh_target, dict):
-                name = ssh_target.pop("name", ssh_target.get("ssh_hostname"))
-                if name is None:
-                    raise InventoryError(
-                        "Invalid Terraform list item, missing `name` or `ssh_hostname` keys",
-                    )
-                yield f"@terraform/{name}", ssh_target, ["@terraform"]
-
-            elif isinstance(ssh_target, str):
-                data = {"ssh_hostname": ssh_target}
-                yield f"@terraform/{ssh_target}", data, ["@terraform"]
+        if isinstance(tf_output_value, list):
+            tf_output_value = {
+                "all": tf_output_value,
+            }
 
-            else:
+        for group_name, hosts in tf_output_value.items():
+            if not isinstance(hosts, list):
                 raise InventoryError(
-                    "Invalid Terraform list item, should be `dict` or `str` got "
-                    f"`{type(ssh_target).__name__}`",
+                    "Invalid Terraform map value type, all values should be `list`, got "
+                    f"`{type(hosts).__name__}`",
                 )
+            for host in hosts:
+                if isinstance(host, dict):
+                    name = host.pop("name", host.get("ssh_hostname"))
+                    if name is None:
+                        raise InventoryError(
+                            "Invalid Terraform list item, missing `name` or `ssh_hostname` keys",
+                        )
+                    yield f"@terraform/{name}", host, ["@terraform", group_name]
+                elif isinstance(host, str):
+                    data = {"ssh_hostname": host}
+                    yield f"@terraform/{host}", data, ["@terraform", group_name]
+                else:
+                    raise InventoryError(
+                        "Invalid Terraform list item, should be `dict` or `str` got "
+                        f"`{type(host).__name__}`",
+                    )

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/facts/apt.py

@@ -9,7 +9,7 @@ from .util import make_cat_files_command
 
 
 def parse_apt_repo(name):
-    regex = r"^(deb(?:-src)?)(?:\s+\[([^\]]+)\])?\s+([^\s]+)\s+([a-z-]+)\s+([a-z-\s]*)$"
+    regex = r"^(deb(?:-src)?)(?:\s+\[([^\]]+)\])?\s+([^\s]+)\s+([^\s]+)\s+([a-z-\s]*)$"
 
     matches = re.match(regex, name)
 

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/facts/files.py

@@ -5,6 +5,7 @@ The files facts provide information about the filesystem and it's contents on th
 from __future__ import annotations
 
 import re
+import shlex
 import stat
 from datetime import datetime
 from typing import TYPE_CHECKING, List, Optional, Tuple, Union
@@ -109,13 +110,19 @@ class File(FactBase[Union[FileDict, Literal[False], None]]):
     type = "file"
 
     def command(self, path):
+        if path.startswith("~/"):
+            # Do not quote leading tilde to ensure that it gets properly expanded by the shell
+            path = f"~/{shlex.quote(path[2:])}"
+        else:
+            path = QuoteString(path)
+
         return make_formatted_string_command(
             (
                 # only stat if the path exists (file or symlink)
                 "! (test -e {0} || test -L {0} ) || "
                 "( {linux_stat_command} {0} 2> /dev/null || {bsd_stat_command} {0} )"
             ),
-            QuoteString(path),
+            path,
             linux_stat_command=LINUX_STAT_COMMAND,
             bsd_stat_command=BSD_STAT_COMMAND,
         )

pyinfra-3.1/pyinfra/facts/flatpak.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import re
+
+from pyinfra.api import FactBase
+
+
+class FlatpakBaseFact(FactBase):
+    abstract = True
+
+    def requires_command(self, *args, **kwargs) -> str:
+        return "flatpak"
+
+
+class FlatpakPackage(FlatpakBaseFact):
+    """
+    Returns information for an installed flatpak package
+
+    .. code:: python
+
+        {
+            "id": "org.signal.Signal",
+            "ref": "app/org.signal.Signal/x86_64/stable",
+            "version": "7.12.0"
+        }
+    """
+
+    default = dict
+    _regexes = {
+        "id": "^[ ]+ID:[ ]+(.*)$",
+        "ref": r"^[ ]+Ref:[ ]+(.*)$",
+        "version": r"^[ ]+Version:[ ]+([\w\d.-]+).*$",
+    }
+
+    def command(self, package):
+        return f"flatpak info {package}"
+
+    def process(self, output):
+        data = {}
+        for line in output:
+            for regex_name, regex in self._regexes.items():
+                matches = re.match(regex, line)
+                if matches:
+                    data[regex_name] = matches.group(1)
+
+        return data
+
+
+class FlatpakPackages(FlatpakBaseFact):
+    """
+    Returns a list of installed flatpak packages:
+
+    .. code:: python
+
+        [
+            "org.gnome.Platform",
+            "org.kde.Platform",
+            "org.kde.Sdk",
+            "org.libreoffice.LibreOffice",
+            "org.videolan.VLC"
+        ]
+    """
+
+    default = list
+
+    def command(self):
+        return "flatpak list --columns=application"
+
+    def process(self, output):
+        return [flatpak for flatpak in output[1:]]

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/facts/systemd.py

@@ -32,6 +32,9 @@ def _make_systemctl_cmd(user_mode=False, machine=None, user_name=None):
             systemctl_cmd.append("--machine={1}@{0}".format(machine, user_name))
         else:
             systemctl_cmd.append("--machine={0}".format(machine))
+    elif user_name is not None:
+        # If only the user is given, assume that the connection should be made to the local machine
+        systemctl_cmd.append("--machine={0}@.host".format(user_name))
 
     return StringCommand(*systemctl_cmd)
 
@@ -58,7 +61,7 @@ class SystemdStatus(FactBase[Dict[str, bool]]):
     default = dict
 
     state_key = "SubState"
-    state_values = ["running", "waiting", "exited"]
+    state_values = ["running", "waiting", "exited", "listening"]
 
     def command(
         self,

pyinfra-3.1/pyinfra/facts/zfs.py

@@ -0,0 +1,57 @@
+"""
+Manage ZFS filesystems.
+"""
+
+from pyinfra.api import FactBase, ShortFactBase
+
+
+def _process_zfs_props_table(output):
+    datasets: dict = {}
+    for line in output:
+        dataset, property, value, source = tuple(line.split("\t"))
+        if dataset not in datasets:
+            datasets[dataset] = {}
+        datasets[dataset][property] = value
+    return datasets
+
+
+class Pools(FactBase):
+    def command(self):
+        return "zpool get -H all"
+
+    @staticmethod
+    def process(output):
+        return _process_zfs_props_table(output)
+
+
+class Datasets(FactBase):
+    def command(self):
+        return "zfs get -H all"
+
+    @staticmethod
+    def process(output):
+        return _process_zfs_props_table(output)
+
+
+class Filesystems(ShortFactBase):
+    fact = Datasets
+
+    @staticmethod
+    def process_data(data):
+        return {name: props for name, props in data.items() if props.get("type") == "filesystem"}
+
+
+class Snapshots(ShortFactBase):
+    fact = Datasets
+
+    @staticmethod
+    def process_data(data):
+        return {name: props for name, props in data.items() if props.get("type") == "snapshot"}
+
+
+class Volumes(ShortFactBase):
+    fact = Datasets
+
+    @staticmethod
+    def process_data(data):
+        return {name: props for name, props in data.items() if props.get("type") == "volume"}

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/operations/docker.py

@@ -4,7 +4,7 @@ Manager Docker Containers, Volumes and Networks
 
 from pyinfra import host
 from pyinfra.api import operation
-from pyinfra.facts.docker import DockerContainers, DockerNetworks, DockerVolumes
+from pyinfra.facts.docker import DockerContainer, DockerNetwork, DockerVolume
 
 from .util.docker import handle_docker
 
@@ -68,7 +68,7 @@ def container(
         )
     """
 
-    existent_container = [c for c in host.get_fact(DockerContainers) if container in c["Name"]]
+    existent_container = host.get_fact(DockerContainer, object_id=container)
 
     if force:
         if existent_container:
@@ -183,7 +183,7 @@ def volume(volume, driver="", labels=None, present=True):
         )
     """
 
-    existent_volume = [v for v in host.get_fact(DockerVolumes) if v["Name"] == volume]
+    existent_volume = host.get_fact(DockerVolume, object_id=volume)
 
     if present:
 
@@ -257,7 +257,7 @@ def network(
             present=True,
         )
     """
-    existent_network = [n for n in host.get_fact(DockerNetworks) if n["Name"] == network]
+    existent_network = host.get_fact(DockerNetwork, object_id=network)
 
     if present:
         if existent_network:

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/operations/files.py

@@ -915,7 +915,8 @@ def template(
     user: str | None = None,
     group: str | None = None,
     mode: str | None = None,
-    create_remote_dir=True,
+    create_remote_dir: bool = True,
+    jinja_env_kwargs: dict[str, Any] | None = None,
     **data,
 ):
     '''
@@ -927,12 +928,18 @@ def template(
     + group: group to own the files
     + mode: permissions of the files
     + create_remote_dir: create the remote directory if it doesn't exist
+    + jinja_env_kwargs: keyword arguments to be passed into the jinja Environment()
 
     ``create_remote_dir``:
         If the remote directory does not exist it will be created using the same
         user & group as passed to ``files.put``. The mode will *not* be copied over,
         if this is required call ``files.directory`` separately.
 
+    ``jinja_env_kwargs``:
+        To have more control over how jinja2 renders your template, you can pass
+        a dict with arguments that will be passed as keyword args to the jinja2
+        `Environment() <https://jinja.palletsprojects.com/en/3.0.x/api/#jinja2.Environment>`_.
+
     Notes:
        Common convention is to store templates in a "templates" directory and
        have a filename suffix with '.j2' (for jinja2).
@@ -1002,7 +1009,7 @@ def template(
 
     # Render and make file-like it's output
     try:
-        output = get_template(src).render(data)
+        output = get_template(src, jinja_env_kwargs).render(data)
     except (TemplateRuntimeError, TemplateSyntaxError, UndefinedError) as e:
         trace_frames = [
             frame

pyinfra-3.1/pyinfra/operations/flatpak.py

@@ -0,0 +1,79 @@
+"""
+Manage flatpak packages. See https://www.flatpak.org/
+"""
+
+from __future__ import annotations
+
+from pyinfra import host
+from pyinfra.api import operation
+from pyinfra.facts.flatpak import FlatpakPackages
+
+
+@operation()
+def packages(
+    packages: str | list[str] | None = None,
+    present=True,
+):
+    """
+    Install/remove a flatpak package
+
+    + packages: List of packages
+    + present: whether the package should be installed
+
+    **Examples:**
+
+    .. code:: python
+
+        # Install vlc flatpak
+        flatpak.package(
+            name="Install vlc",
+            packages="org.videolan.VLC",
+        )
+
+        # Install multiple flatpaks
+        flatpak.package(
+            name="Install vlc and kodi",
+            packages=["org.videolan.VLC", "tv.kodi.Kodi"],
+        )
+
+        # Remove vlc
+        flatpak.package(
+            name="Remove vlc",
+            packages="org.videolan.VLC",
+            present=False,
+        )
+    """
+
+    if packages is None:
+        return
+
+    if isinstance(packages, str):
+        packages = [packages]
+
+    flatpak_packages = host.get_fact(FlatpakPackages)
+
+    install_packages = []
+    remove_packages = []
+
+    for package in packages:
+        # it's installed
+        if package in flatpak_packages:
+            if not present:
+                # we don't want it
+                remove_packages.append(package)
+
+        # it's not installed
+        if package not in flatpak_packages:
+            # we want it
+            if present:
+                install_packages.append(package)
+
+            # we don't want it
+            else:
+                host.noop(f"flatpak package {package} is not installed")
+
+    if install_packages:
+        yield " ".join(["flatpak", "install", "--noninteractive"] + install_packages)
+
+    if remove_packages:
+        yield " ".join(["flatpak", "uninstall", "--noninteractive"] + remove_packages)

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/operations/mysql.py

@@ -554,7 +554,7 @@ def dump(
         )
     """
 
-    yield "{0} > {1}".format(
+    yield StringCommand(
         make_mysql_command(
             executable="mysqldump",
             database=database,
@@ -563,7 +563,8 @@ def dump(
             host=mysql_host,
             port=mysql_port,
         ),
-        dest,
+        ">",
+        QuoteString(dest),
     )
 
 
@@ -595,7 +596,7 @@ def load(
         )
     """
 
-    commands_bits = [
+    yield StringCommand(
         make_mysql_command(
             database=database,
             user=mysql_user,
@@ -605,5 +606,4 @@ def load(
         ),
         "<",
         QuoteString(src),
-    ]
-    yield StringCommand(*commands_bits)
+    )

{pyinfra-3.0.1 → pyinfra-3.1}/pyinfra/operations/postgres.py

@@ -16,7 +16,7 @@ See example/postgresql.py for detailed example
 from __future__ import annotations
 
 from pyinfra import host
-from pyinfra.api import MaskString, StringCommand, operation
+from pyinfra.api import MaskString, QuoteString, StringCommand, operation
 from pyinfra.facts.postgres import (
     PostgresDatabases,
     PostgresRoles,
@@ -302,7 +302,7 @@ def dump(
             port=psql_port,
         ),
         ">",
-        dest,
+        QuoteString(dest),
     )
 
 
@@ -345,5 +345,5 @@ def load(
             port=psql_port,
         ),
         "<",
-        src,
+        QuoteString(src),
     )