pyinfra 3.1.1__py2.py3-none-any.whl → 3.2__py2.py3-none-any.whl

pyinfra/facts/util/units.py

@@ -0,0 +1,30 @@
+# from https://stackoverflow.com/a/60708339, but with a few modifications
+from __future__ import annotations  # for | in type hints
+
+import re
+
+units = {
+    "B": 1,
+    "KB": 10**3,
+    "MB": 10**6,
+    "GB": 10**9,
+    "TB": 10**12,
+    "KIB": 2**10,
+    "MIB": 2**20,
+    "GIB": 2**30,
+    "TIB": 2**40,
+}
+
+
+def parse_human_readable_size(size: str) -> int:
+    size = size.upper()
+    if not re.match(r" ", size):
+        size = re.sub(r"([KMGT]?I?[B])", r" \1", size)
+    number, unit = [string.strip() for string in size.split()]
+    return int(float(number) * units[unit])
+
+
+def parse_size(size: str | int) -> int:
+    if isinstance(size, int):
+        return size
+    return parse_human_readable_size(size)

pyinfra/facts/zfs.py

@@ -1,5 +1,5 @@
 """
-Manage ZFS filesystems.
+Gather information about ZFS filesystems.
 """
 
 from pyinfra.api import FactBase, ShortFactBase
@@ -15,43 +15,46 @@ def _process_zfs_props_table(output):
     return datasets
 
 
-class Pools(FactBase):
+class ZfsPools(FactBase):
     def command(self):
         return "zpool get -H all"
 
-    @staticmethod
-    def process(output):
+    def process(self, output):
         return _process_zfs_props_table(output)
 
 
-class Datasets(FactBase):
+class ZfsDatasets(FactBase):
     def command(self):
         return "zfs get -H all"
 
-    @staticmethod
-    def process(output):
+    def process(self, output):
         return _process_zfs_props_table(output)
 
 
-class Filesystems(ShortFactBase):
-    fact = Datasets
+class ZfsFilesystems(ShortFactBase):
+    fact = ZfsDatasets
 
-    @staticmethod
-    def process_data(data):
+    def process_data(self, data):
         return {name: props for name, props in data.items() if props.get("type") == "filesystem"}
 
 
-class Snapshots(ShortFactBase):
-    fact = Datasets
+class ZfsSnapshots(ShortFactBase):
+    fact = ZfsDatasets
 
-    @staticmethod
-    def process_data(data):
+    def process_data(self, data):
         return {name: props for name, props in data.items() if props.get("type") == "snapshot"}
 
 
-class Volumes(ShortFactBase):
-    fact = Datasets
+class ZfsVolumes(ShortFactBase):
+    fact = ZfsDatasets
 
-    @staticmethod
-    def process_data(data):
+    def process_data(self, data):
         return {name: props for name, props in data.items() if props.get("type") == "volume"}
+
+
+# TODO: remove these in v4! Or flip the convention and remove all the other fact prefixes!
+Pools = ZfsPools
+Datasets = ZfsDatasets
+Filesystems = ZfsFilesystems
+Snapshots = ZfsSnapshots
+Volumes = ZfsVolumes

pyinfra/local.py

@@ -1,4 +1,5 @@
 from os import path
+from typing import Optional
 
 import click
 
@@ -10,7 +11,7 @@ from pyinfra.connectors.util import run_local_process
 from pyinfra.context import ctx_state
 
 
-def include(filename: str):
+def include(filename: str, data: Optional[dict] = None):
     """
     Executes a local python file within the ``pyinfra.state.cwd``
     directory.
@@ -33,7 +34,7 @@ def include(filename: str):
 
         from pyinfra_cli.util import exec_file
 
-        with host.deploy(path.relpath(filename, state.cwd), None, None, in_deploy=False):
+        with host.deploy(path.relpath(filename, state.cwd), None, data, in_deploy=False):
             exec_file(filename)
 
         # One potential solution to the above is to add local as an actual

pyinfra/operations/apt.py

@@ -9,7 +9,13 @@ from urllib.parse import urlparse
 
 from pyinfra import host
 from pyinfra.api import OperationError, operation
-from pyinfra.facts.apt import AptKeys, AptSources, parse_apt_repo
+from pyinfra.facts.apt import (
+    AptKeys,
+    AptSources,
+    SimulateOperationWillChange,
+    noninteractive_apt,
+    parse_apt_repo,
+)
 from pyinfra.facts.deb import DebPackage, DebPackages
 from pyinfra.facts.files import File
 from pyinfra.facts.gpg import GpgKey
@@ -21,21 +27,22 @@ from .util.packaging import ensure_packages
 APT_UPDATE_FILENAME = "/var/lib/apt/periodic/update-success-stamp"
 
 
-def noninteractive_apt(command: str, force=False):
-    args = ["DEBIAN_FRONTEND=noninteractive apt-get -y"]
-
-    if force:
-        args.append("--force-yes")
-
-    args.extend(
-        (
-            '-o Dpkg::Options::="--force-confdef"',
-            '-o Dpkg::Options::="--force-confold"',
-            command,
-        ),
-    )
-
-    return " ".join(args)
+def _simulate_then_perform(command: str):
+    changes = host.get_fact(SimulateOperationWillChange, command)
+
+    if not changes:
+        # Simulating apt-get command failed, so the actual
+        # operation will probably fail too:
+        yield noninteractive_apt(command)
+    elif (
+        changes["upgraded"] == 0
+        and changes["newly_installed"] == 0
+        and changes["removed"] == 0
+        and changes["not_upgraded"] == 0
+    ):
+        host.noop(f"{command} skipped, no changes would be performed")
+    else:
+        yield noninteractive_apt(command)
 
 
 @operation()
@@ -321,7 +328,7 @@ def update(cache_time: int | None = None):
 _update = update  # noqa: E305
 
 
-@operation(is_idempotent=False)
+@operation()
 def upgrade(auto_remove: bool = False):
     """
     Upgrades all apt packages.
@@ -349,13 +356,13 @@ def upgrade(auto_remove: bool = False):
     if auto_remove:
         command.append("--autoremove")
 
-    yield noninteractive_apt(" ".join(command))
+    yield from _simulate_then_perform(" ".join(command))
 
 
 _upgrade = upgrade  # noqa: E305 (for use below where update is a kwarg)
 
 
-@operation(is_idempotent=False)
+@operation()
 def dist_upgrade():
     """
     Updates all apt packages, employing dist-upgrade.
@@ -369,7 +376,7 @@ def dist_upgrade():
         )
     """
 
-    yield noninteractive_apt("dist-upgrade")
+    yield from _simulate_then_perform("dist-upgrade")
 
 
 @operation()

pyinfra/operations/crontab.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import shlex
+
+from pyinfra import host
+from pyinfra.api import StringCommand, operation
+from pyinfra.api.util import try_int
+from pyinfra.facts.crontab import Crontab, CrontabFile
+from pyinfra.operations.util.files import sed_replace
+
+
+@operation()
+def crontab(
+    command: str,
+    present=True,
+    user: str | None = None,
+    cron_name: str | None = None,
+    minute="*",
+    hour="*",
+    month="*",
+    day_of_week="*",
+    day_of_month="*",
+    special_time: str | None = None,
+    interpolate_variables=False,
+):
+    """
+    Add/remove/update crontab entries.
+
+    + command: the command for the cron
+    + present: whether this cron command should exist
+    + user: the user whose crontab to manage
+    + cron_name: name the cronjob so future changes to the command will overwrite
+    + modify_cron_name: modify the cron name
+    + minute: which minutes to execute the cron
+    + hour: which hours to execute the cron
+    + month: which months to execute the cron
+    + day_of_week: which day of the week to execute the cron
+    + day_of_month: which day of the month to execute the cron
+    + special_time: cron "nickname" time (@reboot, @daily, etc), overrides others
+    + interpolate_variables: whether to interpolate variables in ``command``
+
+    Cron commands:
+        Unless ``name`` is specified the command is used to identify crontab entries.
+        This means commands must be unique within a given users crontab. If you require
+        multiple identical commands, provide a different name argument for each.
+
+    Special times:
+        When provided, ``special_time`` will be used instead of any values passed in
+        for ``minute``/``hour``/``month``/``day_of_week``/``day_of_month``.
+
+    **Example:**
+
+    .. code:: python
+
+        # simple example for a crontab
+        crontab.crontab(
+            name="Backup /etc weekly",
+            command="/bin/tar cf /tmp/etc_bup.tar /etc",
+            name="backup_etc",
+            day_of_week=0,
+            hour=1,
+            minute=0,
+        )
+    """
+
+    def comma_sep(value):
+        if isinstance(value, (list, tuple)):
+            return ",".join("{0}".format(v) for v in value)
+        return value
+
+    minute = comma_sep(minute)
+    hour = comma_sep(hour)
+    month = comma_sep(month)
+    day_of_week = comma_sep(day_of_week)
+    day_of_month = comma_sep(day_of_month)
+
+    ctb0: CrontabFile | dict = host.get_fact(Crontab, user=user)
+    # facts from test are in dict
+    if isinstance(ctb0, dict):
+        ctb = CrontabFile(ctb0)
+    else:
+        ctb = ctb0
+    name_comment = "# pyinfra-name={0}".format(cron_name)
+
+    existing_crontab = ctb.get_command(command=command, name=cron_name)
+    existing_crontab_command = existing_crontab["command"] if existing_crontab else command
+    existing_crontab_match = existing_crontab["command"] if existing_crontab else command
+
+    exists = existing_crontab is not None
+    exists_name = existing_crontab is not None and name_comment in existing_crontab.get(
+        "comments", ""
+    )
+
+    edit_commands: list[str | StringCommand] = []
+    temp_filename = host.get_temp_filename()
+
+    if special_time:
+        new_crontab_line = "{0} {1}".format(special_time, command)
+    else:
+        new_crontab_line = "{minute} {hour} {day_of_month} {month} {day_of_week} {command}".format(
+            minute=minute,
+            hour=hour,
+            day_of_month=day_of_month,
+            month=month,
+            day_of_week=day_of_week,
+            command=command,
+        )
+
+    existing_crontab_match = ".*{0}.*".format(existing_crontab_match)
+
+    # Don't want the cron and it does exist? Remove the line
+    if not present and exists:
+        edit_commands.append(
+            sed_replace(
+                temp_filename,
+                existing_crontab_match,
+                "",
+                interpolate_variables=interpolate_variables,
+            ),
+        )
+
+    # Want the cron but it doesn't exist? Append the line
+    elif present and not exists:
+        print("present", present, "exists", exists)
+        if ctb:  # append a blank line if cron entries already exist
+            edit_commands.append("echo '' >> {0}".format(temp_filename))
+        if cron_name:
+            edit_commands.append(
+                "echo {0} >> {1}".format(
+                    shlex.quote(name_comment),
+                    temp_filename,
+                ),
+            )
+
+        edit_commands.append(
+            "echo {0} >> {1}".format(
+                shlex.quote(new_crontab_line),
+                temp_filename,
+            ),
+        )
+
+    # We have the cron and it exists, do it's details? If not, replace the line
+    elif present and exists:
+        assert existing_crontab is not None
+        if any(
+            (
+                exists_name != (cron_name is not None),
+                special_time != existing_crontab.get("special_time"),
+                try_int(minute) != existing_crontab.get("minute"),
+                try_int(hour) != existing_crontab.get("hour"),
+                try_int(month) != existing_crontab.get("month"),
+                try_int(day_of_week) != existing_crontab.get("day_of_week"),
+                try_int(day_of_month) != existing_crontab.get("day_of_month"),
+                existing_crontab_command != command,
+            ),
+        ):
+            if not exists_name and cron_name:
+                new_crontab_line = f"{name_comment}\n{new_crontab_line}"
+            edit_commands.append(
+                sed_replace(
+                    temp_filename,
+                    existing_crontab_match,
+                    new_crontab_line,
+                    interpolate_variables=interpolate_variables,
+                ),
+            )
+
+    if edit_commands:
+        crontab_args = []
+        if user:
+            crontab_args.append("-u {0}".format(user))
+
+        # List the crontab into a temporary file if it exists
+        if ctb:
+            yield "crontab -l {0} > {1}".format(" ".join(crontab_args), temp_filename)
+
+        # Now yield any edits
+        for edit_command in edit_commands:
+            yield edit_command
+
+        # Finally, use the tempfile to write a new crontab
+        yield "crontab {0} {1}".format(" ".join(crontab_args), temp_filename)
+    else:
+        host.noop(
+            "crontab {0} {1}".format(
+                command,
+                "exists" if present else "does not exist",
+            ),
+        )

pyinfra/operations/docker.py

@@ -1,5 +1,7 @@
 """
-Manager Docker Containers, Volumes and Networks
+Manager Docker containers, volumes and networks. These operations allow you to manage Docker from
+the view of the current inventory host. See the :doc:`../connectors/docker` to use Docker containers
+as inventory directly.
 """
 
 from pyinfra import host
@@ -30,9 +32,9 @@ def container(
     + networks: network list to attach on container
     + ports: port list to expose
     + volumes: volume list to map on container
-    + env_vars: environment varible list to inject on container
+    + env_vars: environment variable list to inject on container
     + pull_always: force image pull
-    + force: remove a contaner with same name and create a new one
+    + force: remove a container with same name and create a new one
     + present: whether the container should be up and running
     + start: start or stop the container
 
@@ -125,7 +127,7 @@ def image(image, present=True):
     Manage Docker images
 
     + image: Image and tag ex: nginx:alpine
-    + present: whether the Docker image should be exist
+    + present: whether the Docker image should exist
 
     **Examples:**
 
@@ -188,7 +190,7 @@ def volume(volume, driver="", labels=None, present=True):
     if present:
 
         if existent_volume:
-            host.noop("Volume alredy exist!")
+            host.noop("Volume already exists!")
             return
 
         yield handle_docker(
@@ -231,8 +233,8 @@ def network(
     """
     Manage docker networks
 
-    + network_name: Image name
-    + driver: Container image and tag ex: nginx:alpine
+    + network: Network name
+    + driver: Network driver ex: bridge or overlay
     + gateway: IPv4 or IPv6 Gateway for the master subnet
     + ip_range: Allocate container ip from a sub-range
     + ipam_driver: IP Address Management Driver
@@ -251,8 +253,7 @@ def network(
 
         # Create Docker network
         docker.network(
-            name="Create nginx network",
-            network_name="nginx",
+            network="nginx",
             attachable=True,
             present=True,
         )
@@ -261,7 +262,7 @@ def network(
 
     if present:
         if existent_network:
-            host.noop("Alredy exist a network with {0} name!".format(network))
+            host.noop("Network {0} already exists!".format(network))
             return
 
         yield handle_docker(
@@ -284,12 +285,12 @@ def network(
 
     else:
         if existent_network is None:
-            host.noop("Ther is not network with {0} name!".format(network))
+            host.noop("Network {0} does not exist!".format(network))
             return
 
         yield handle_docker(
             resource="network",
-            command="create",
+            command="remove",
             network=network,
         )
 

pyinfra/operations/files.py

@@ -51,6 +51,7 @@ from pyinfra.facts.files import (
     Md5File,
     Sha1File,
     Sha256File,
+    Sha384File,
 )
 from pyinfra.facts.server import Date, Which
 
@@ -67,6 +68,7 @@ def download(
     mode: str | None = None,
     cache_time: int | None = None,
     force=False,
+    sha384sum: str | None = None,
     sha256sum: str | None = None,
     sha1sum: str | None = None,
     md5sum: str | None = None,
@@ -84,6 +86,7 @@ def download(
     + mode: permissions of the files
     + cache_time: if the file exists already, re-download after this time (in seconds)
     + force: always download the file, even if it already exists
+    + sha384sum: sha384 hash to checksum the downloaded file against
     + sha256sum: sha256 hash to checksum the downloaded file against
     + sha1sum: sha1 hash to checksum the downloaded file against
     + md5sum: md5 hash to checksum the downloaded file against
@@ -135,6 +138,10 @@ def download(
             if sha256sum != host.get_fact(Sha256File, path=dest):
                 download = True
 
+        if sha384sum:
+            if sha384sum != host.get_fact(Sha384File, path=dest):
+                download = True
+
         if md5sum:
             if md5sum != host.get_fact(Md5File, path=dest):
                 download = True
@@ -211,6 +218,17 @@ def download(
                 QuoteString("SHA256 did not match!"),
             )
 
+        if sha384sum:
+            yield make_formatted_string_command(
+                (
+                    "(( sha384sum {0} 2> /dev/null || shasum -a 384 {0} ) "
+                    "| grep {1}) || ( echo {2} && exit 1 )"
+                ),
+                QuoteString(dest),
+                sha384sum,
+                QuoteString("SHA384 did not match!"),
+            )
+
         if md5sum:
             yield make_formatted_string_command(
                 (

pyinfra/operations/git.py

@@ -16,24 +16,40 @@ from .util.files import chown, unix_path_join
 
 
 @operation()
-def config(key: str, value: str, multi_value=False, repo: str | None = None):
+def config(key: str, value: str, multi_value=False, repo: str | None = None, system=False):
     """
-    Manage git config for a repository or globally.
+    Manage git config at repository, user or system level.
 
     + key: the key of the config to ensure
     + value: the value this key should have
     + multi_value: Add the value rather than set it for settings that can have multiple values
     + repo: specify the git repo path to edit local config (defaults to global)
+    + system: whether, when ``repo`` is unspecified, to work at system level (or default to global)
 
-    **Example:**
+    **Examples:**
 
     .. code:: python
 
         git.config(
-            name="Ensure user name is set for a repo",
+            name="Always prune specified repo",
+            key="fetch.prune",
+            value="true",
+            repo="/usr/local/src/pyinfra",
+        )
+
+        git.config(
+            name="Ensure user name is set for all repos of specified user",
             key="user.name",
             value="Anon E. Mouse",
-            repo="/usr/local/src/pyinfra",
+            _sudo=True,
+            _sudo_user="anon"
+        )
+
+        git.config(
+            name="Ensure same date format for all users",
+            key="log.date",
+            value="iso",
+            system=True
         )
 
     """
@@ -41,14 +57,14 @@ def config(key: str, value: str, multi_value=False, repo: str | None = None):
     existing_config = {}
 
     if not repo:
-        existing_config = host.get_fact(GitConfig)
+        existing_config = host.get_fact(GitConfig, system=system)
 
     # Only get the config if the repo exists at this stage
     elif host.get_fact(Directory, path=unix_path_join(repo, ".git")):
         existing_config = host.get_fact(GitConfig, repo=repo)
 
     if repo is None:
-        base_command = "git config --global"
+        base_command = "git config" + (" --system" if system else " --global")
     else:
         base_command = "cd {0} && git config --local".format(repo)
 

pyinfra/operations/opkg.py

@@ -0,0 +1,88 @@
+"""
+    Manage packages on OpenWrt using opkg
+        + ``update`` - update local copy of package information
+        + ``packages`` -  install and remove packages
+
+    see https://openwrt.org/docs/guide-user/additional-software/opkg
+    OpenWrt recommends against upgrading all packages  thus there is no ``opkg.upgrade`` function
+"""
+
+from typing import List, Union
+
+from pyinfra import host
+from pyinfra.api import StringCommand, operation
+from pyinfra.facts.opkg import OpkgPackages
+from pyinfra.operations.util.packaging import ensure_packages
+
+EQUALS = "="
+
+
+@operation(is_idempotent=False)
+def update():
+    """
+    Update the local opkg information.
+    """
+
+    yield StringCommand("opkg update")
+
+
+_update = update
+
+
+@operation()
+def packages(
+    packages: Union[str, List[str]] = "",
+    present: bool = True,
+    latest: bool = False,
+    update: bool = True,
+):
+    """
+    Add/remove/update opkg packages.
+
+    + packages: package or list of packages to that must/must not be present
+    + present: whether the package(s) should be installed (default True) or removed
+    + latest: whether to attempt to upgrade the specified package(s) (default False)
+    + update: run ``opkg update`` before installing packages (default True)
+
+    Not Supported:
+        Opkg does not support version pinning, i.e. ``<pkg>=<version>`` is not allowed
+        and will cause an exception.
+
+    **Examples:**
+
+    .. code:: python
+
+        # Ensure packages are installed∂ (will not force package upgrade)
+        opkg.packages(['asterisk', 'vim'], name="Install Asterisk and Vim")
+
+        # Install the latest versions of packages (always check)
+        opkg.packages(
+            'vim',
+            latest=True,
+            name="Ensure we have the latest version of Vim"
+        )
+    """
+    if str(packages) == "" or (
+        isinstance(packages, list) and (len(packages) < 1 or all(len(p) < 1 for p in packages))
+    ):
+        host.noop("empty or invalid package list provided to opkg.packages")
+        return
+
+    pkg_list = packages if isinstance(packages, list) else [packages]
+    have_equals = ",".join([pkg.split(EQUALS)[0] for pkg in pkg_list if EQUALS in pkg])
+    if len(have_equals) > 0:
+        raise ValueError(f"opkg does not support version pinning but found for: '{have_equals}'")
+
+    if update:
+        yield from _update._inner()
+
+    yield from ensure_packages(
+        host,
+        pkg_list,
+        host.get_fact(OpkgPackages),
+        present,
+        install_command="opkg install",
+        upgrade_command="opkg upgrade",
+        uninstall_command="opkg remove",
+        latest=latest,
+    )