pyinfra 3.3.1__py2.py3-none-any.whl → 3.4.1__py2.py3-none-any.whl

pyinfra/facts/git.py

@@ -23,6 +23,16 @@ class GitBranch(GitFactBase):
         return re.sub(r"(heads|tags)/", r"", "\n".join(output))
 
 
+class GitTag(GitFactBase):
+    @override
+    def command(self, repo) -> str:
+        return "! test -d {0} || (cd {0} && git tag)".format(repo)
+
+    @override
+    def process(self, output):
+        return output
+
+
 class GitConfig(GitFactBase):
     default = dict
 

pyinfra/facts/hardware.py

@@ -46,7 +46,7 @@ class Memory(FactBase):
             value, key = line.split(" ", 1)
 
             try:
-                value = int(value)
+                value = float(value)
             except ValueError:
                 continue
 

pyinfra/facts/opkg.py

@@ -81,6 +81,7 @@ class OpkgConf(FactBase):
         re.X,
     )
 
+    @override
     @staticmethod
     def default():
         return OpkgConfInfo({}, "", {}, {})

pyinfra/facts/server.py

@@ -1,16 +1,18 @@
 from __future__ import annotations
 
+import json
 import os
 import re
 import shutil
 from datetime import datetime
 from tempfile import mkdtemp
-from typing import Dict, List, Optional
+from typing import Dict, Iterable, List, Optional, Tuple
 
 from dateutil.parser import parse as parse_date
 from distro import distro
 from typing_extensions import TypedDict, override
 
+from pyinfra import host
 from pyinfra.api import FactBase, ShortFactBase
 from pyinfra.api.util import try_int
 from pyinfra.facts import crontab
@@ -205,40 +207,96 @@ class Mounts(FactBase[Dict[str, MountsDict]]):
     default = dict
 
     @override
-    def command(self):
-        return "mount"
+    def command(self) -> str:
+        self._kernel = host.get_fact(Kernel)
+
+        if self._kernel.strip() == "FreeBSD":
+            return "mount -p --libxo json"
+        else:
+            return "cat /proc/self/mountinfo"
 
     @override
     def process(self, output) -> dict[str, MountsDict]:
         devices: dict[str, MountsDict] = {}
 
-        for line in output:
-            is_map = False
-            if line.startswith("map "):
-                line = line[4:]
-                is_map = True
+        def unescape_octal(match: re.Match) -> str:
+            s = match.group(0)[1:]  # skip the backslash
+            return chr(int(s, base=8))
 
-            device, _, path, other_bits = line.split(" ", 3)
+        def replace_octal(s: str) -> str:
+            """
+            Unescape strings encoded by linux's string_escape_mem with ESCAPE_OCTAL flag.
+            """
+            return re.sub(r"\\[0-7]{3}", unescape_octal, s)
 
-            if is_map:
-                device = "map {0}".format(device)
+        if self._kernel == "FreeBSD":
+            full_output = "\n".join(output)
+            json_output = json.loads(full_output)
+            mount_fstab = json_output["mount"]["fstab"]
 
-            if other_bits.startswith("type"):
-                _, type_, options = other_bits.split(" ", 2)
-                options = options.strip("()").split(",")
-            else:
-                options = other_bits.strip("()").split(",")
-                type_ = options.pop(0)
+            for entry in mount_fstab:
+                path = entry["mntpoint"]
+                type_ = entry["fstype"]
+                device = entry["device"]
+                options = [option.strip() for option in entry["opts"].split(",")]
 
-            devices[path] = {
-                "device": device,
-                "type": type_,
-                "options": [option.strip() for option in options],
+                devices[path] = {"device": device, "type": type_, "options": options}
+
+            return devices
+
+        for line in output:
+            # ignore mount ID, parent ID, major:minor, root
+            _, _, _, _, mount_point, mount_options, line = line.split(sep=" ", maxsplit=6)
+
+            # ignore optional tags "shared", "master", "propagate_from" and "unbindable"
+            while True:
+                optional, line = line.split(sep=" ", maxsplit=1)
+                if optional == "-":
+                    break
+
+            fs_type, mount_source, super_options = line.split(sep=" ")
+
+            mount_options = mount_options.split(sep=",")
+
+            # escaped: mount_point, mount_source, super_options
+            # these strings can contain characters encoded in octal, e.g. '\054' for ','
+            mount_point = replace_octal(mount_point)
+            mount_source = replace_octal(mount_source)
+
+            # mount_options will override ro/rw and can be different than the super block options
+            # filter them, so they don't appear twice
+            super_options = [
+                replace_octal(opt)
+                for opt in super_options.split(sep=",")
+                if opt not in ["ro", "rw"]
+            ]
+
+            devices[mount_point] = {
+                "device": mount_source,
+                "type": fs_type,
+                "options": mount_options + super_options,
             }
 
         return devices
 
 
+class Port(FactBase[Tuple[str, int] | Tuple[None, None]]):
+    """
+    Returns the process occuping a port and its PID
+    """
+
+    @override
+    def command(self, port: int) -> str:
+        return f"ss -lptnH 'src :{port}'"
+
+    @override
+    def process(self, output: Iterable[str]) -> Tuple[str, int] | Tuple[None, None]:
+        for line in output:
+            proc, pid = line.split('"')[1], int(line.split("pid=")[1].split(",")[0])
+            return (proc, pid)
+        return None, None
+
+
 class KernelModules(FactBase):
     """
     Returns a dictionary of kernel module name -> info.
@@ -470,10 +528,10 @@ class Users(FactBase):
         for i in `cat /etc/passwd | cut -d: -f1`; do
             ENTRY=`grep ^$i: /etc/passwd`;
             LASTLOG=`(((lastlog -u $i || lastlogin $i) 2> /dev/null) | grep ^$i | tr -s ' ')`;
-            PASSWORD=`grep ^$i: /etc/shadow | cut -d: -f2`;
+            PASSWORD=`(grep ^$i: /etc/shadow || grep ^$i: /etc/master.passwd) 2> /dev/null | cut -d: -f2`;
             echo "$ENTRY|`id -gn $i`|`id -Gn $i`|$LASTLOG|$PASSWORD";
         done
-    """.strip()
+    """.strip()  # noqa
 
     default = dict
 

pyinfra/facts/systemd.py

@@ -64,7 +64,7 @@ class SystemdStatus(FactBase[Dict[str, bool]]):
     default = dict
 
     state_key = "SubState"
-    state_values = ["running", "waiting", "exited", "listening"]
+    state_values = ["running", "waiting", "exited", "listening", "mounted"]
 
     @override
     def command(

pyinfra/operations/crontab.py

@@ -2,11 +2,13 @@ from __future__ import annotations
 
 import shlex
 
-from pyinfra import host
-from pyinfra.api import StringCommand, operation
+from pyinfra import logger
+from pyinfra.api.command import StringCommand
+from pyinfra.api.operation import operation
 from pyinfra.api.util import try_int
+from pyinfra.context import host
 from pyinfra.facts.crontab import Crontab, CrontabFile
-from pyinfra.operations.util.files import sed_replace
+from pyinfra.operations.util.files import sed_delete, sed_replace
 
 
 @operation()
@@ -111,7 +113,7 @@ def crontab(
     # Don't want the cron and it does exist? Remove the line
     if not present and exists:
         edit_commands.append(
-            sed_replace(
+            sed_delete(
                 temp_filename,
                 existing_crontab_match,
                 "",
@@ -121,7 +123,7 @@ def crontab(
 
     # Want the cron but it doesn't exist? Append the line
     elif present and not exists:
-        print("present", present, "exists", exists)
+        logger.debug(f"present: {present}, exists: {exists}")
         if ctb:  # append a blank line if cron entries already exist
             edit_commands.append("echo '' >> {0}".format(temp_filename))
         if cron_name:

pyinfra/operations/docker.py

@@ -227,6 +227,7 @@ def network(
     ipam_driver="",
     subnet="",
     scope="",
+    aux_addresses=None,
     opts=None,
     ipam_opts=None,
     labels=None,
@@ -279,6 +280,7 @@ def network(
             ipam_driver=ipam_driver,
             subnet=subnet,
             scope=scope,
+            aux_addresses=aux_addresses,
             opts=opts,
             ipam_opts=ipam_opts,
             labels=labels,

pyinfra/operations/files.py

@@ -32,7 +32,9 @@ from pyinfra.api.command import make_formatted_string_command
 from pyinfra.api.util import (
     get_call_location,
     get_file_io,
+    get_file_md5,
     get_file_sha1,
+    get_file_sha256,
     get_path_permissions_mode,
     get_template,
     memoize,
@@ -56,7 +58,14 @@ from pyinfra.facts.files import (
 from pyinfra.facts.server import Date, Which
 
 from .util import files as file_utils
-from .util.files import adjust_regex, ensure_mode_int, get_timestamp, sed_replace, unix_path_join
+from .util.files import (
+    adjust_regex,
+    ensure_mode_int,
+    get_timestamp,
+    sed_delete,
+    sed_replace,
+    unix_path_join,
+)
 
 
 @operation()
@@ -75,6 +84,9 @@ def download(
     headers: dict[str, str] | None = None,
     insecure=False,
     proxy: str | None = None,
+    temp_dir: str | Path | None = None,
+    extra_curl_args: dict[str, str] | None = None,
+    extra_wget_args: dict[str, str] | None = None,
 ):
     """
     Download files from remote locations using ``curl`` or ``wget``.
@@ -93,6 +105,9 @@ def download(
     + headers: optional dictionary of headers to set for the HTTP request
     + insecure: disable SSL verification for the HTTP request
     + proxy: simple HTTP proxy through which we can download files, form `http://<yourproxy>:<port>`
+    + temp_dir: use this custom temporary directory during the download
+    + extra_curl_args: optional dictionary with custom arguments for curl
+    + extra_wget_args: optional dictionary with custom arguments for wget
 
     **Example:**
 
@@ -148,11 +163,21 @@ def download(
 
     # If we download, always do user/group/mode as SSH user may be different
     if download:
-        temp_file = host.get_temp_filename(dest)
+        temp_file = host.get_temp_filename(
+            dest, temp_directory=str(temp_dir) if temp_dir is not None else None
+        )
 
         curl_args: list[Union[str, StringCommand]] = ["-sSLf"]
         wget_args: list[Union[str, StringCommand]] = ["-q"]
 
+        if extra_curl_args:
+            for key, value in extra_curl_args.items():
+                curl_args.append(StringCommand(key, QuoteString(value)))
+
+        if extra_wget_args:
+            for key, value in extra_wget_args.items():
+                wget_args.append(StringCommand(key, QuoteString(value)))
+
         if proxy:
             curl_args.append(f"--proxy {proxy}")
             wget_args.append("-e use_proxy=yes")
@@ -423,9 +448,9 @@ def line(
         else:
             host.noop('line "{0}" exists in {1}'.format(replace or line, path))
 
-    # Line(s) exists and we want to remove them, replace with nothing
+    # Line(s) exists and we want to remove them
     elif present_lines and not present:
-        yield sed_replace(
+        yield sed_delete(
             path,
             match_line,
             "",
@@ -710,6 +735,21 @@ def _create_remote_dir(remote_filename, user, group):
         )
 
 
+def _file_equal(local_path: str | IO[Any] | None, remote_path: str) -> bool:
+    if local_path is None:
+        return False
+    for fact, get_sum in [
+        (Sha1File, get_file_sha1),
+        (Md5File, get_file_md5),
+        (Sha256File, get_file_sha256),
+    ]:
+        remote_sum = host.get_fact(fact, path=remote_path)
+        if remote_sum:
+            local_sum = get_sum(local_path)
+            return local_sum == remote_sum
+    return False
+
+
 @operation(
     # We don't (currently) cache the local state, so there's nothing we can
     # update to flag the local file as present.
@@ -766,12 +806,11 @@ def get(
 
     # Remote file exists - check if it matches our local
     else:
-        local_sum = get_file_sha1(dest)
-        remote_sum = host.get_fact(Sha1File, path=src)
-
-        # Check sha1sum, upload if needed
-        if local_sum != remote_sum:
+        # Check hash sum, download if needed
+        if not _file_equal(dest, src):
             yield FileDownloadCommand(src, dest, remote_temp_filename=host.get_temp_filename(dest))
+        else:
+            host.noop("file {0} has already been downloaded".format(dest))
 
 
 @operation()
@@ -805,7 +844,9 @@ def put(
 
     ``mode``:
         When set to ``True`` the permissions of the local file are applied to the
-        remote file after the upload is complete.
+        remote file after the upload is complete.  If set to an octal value with
+        digits for at least user, group, and other, either as an ``int`` or
+        ``str``, those permissions will be used.
 
     ``create_remote_dir``:
         If the remote directory does not exist it will be created using the same
@@ -816,6 +857,11 @@ def put(
         This operation is not suitable for large files as it may involve copying
         the file before uploading it.
 
+        Currently, if the mode argument is anything other than a ``bool`` or a full
+        octal permission set and the remote file exists, the operation will always
+        behave as if the remote file does not match the specified permissions and
+        requires a change.
+
     **Examples:**
 
     .. code:: python
@@ -837,7 +883,7 @@ def put(
     # Upload IO objects as-is
     if hasattr(src, "read"):
         local_file = src
-        local_sum = get_file_sha1(src)
+        local_sum_path = src
 
     # Assume string filename
     else:
@@ -849,9 +895,9 @@ def put(
         local_file = src
 
         if os.path.isfile(local_file):
-            local_sum = get_file_sha1(local_file)
+            local_sum_path = local_file
         elif assume_exists:
-            local_sum = None
+            local_sum_path = None
         else:
             raise IOError("No such file: {0}".format(local_file))
 
@@ -893,10 +939,7 @@ def put(
 
     # File exists, check sum and check user/group/mode if supplied
     else:
-        remote_sum = host.get_fact(Sha1File, path=dest)
-
-        # Check sha1sum, upload if needed
-        if local_sum != remote_sum:
+        if not _file_equal(local_sum_path, dest):
             yield FileUploadCommand(
                 local_file,
                 dest,
@@ -1612,12 +1655,12 @@ def block(
         "2>/dev/null || stat -f %Lp",
         q_path,
         ") $OUT && ",
-        '(chown $(stat -c "%U:%G"',
+        '(chown $(stat -c "%u:%g"',
         q_path,
-        "2>/dev/null) $OUT || ",
-        'chown -n $(stat -f "%u:%g"',
+        "2>/dev/null || ",
+        'stat -f "%u:%g"',
         q_path,
-        ') $OUT)  && mv "$OUT"',
+        '2>/dev/null ) $OUT) && mv "$OUT"',
         q_path,
     )
 

pyinfra/operations/flatpak.py

@@ -12,12 +12,14 @@ from pyinfra.facts.flatpak import FlatpakPackages
 @operation()
 def packages(
     packages: str | list[str] | None = None,
+    remote: str | None = None,
     present=True,
 ):
     """
     Install/remove a flatpak package
 
     + packages: List of packages
+    + remote: Source to install the application or runtime from
     + present: whether the package should be installed
 
     **Examples:**
@@ -30,6 +32,13 @@ def packages(
             packages="org.videolan.VLC",
         )
 
+        # Install vlc flatpak from flathub
+        flatpak.package(
+            name="Install vlc",
+            packages="org.videolan.VLC",
+            remote="flathub",
+        )
+
         # Install multiple flatpaks
         flatpak.package(
             name="Install vlc and kodi",
@@ -55,6 +64,12 @@ def packages(
     install_packages = []
     remove_packages = []
 
+    if remote is None:
+        remote = ""
+    else:
+        # ensure we have a space between the remote and packages
+        remote = remote.strip() + " "
+
     for package in packages:
         # it's installed
         if package in flatpak_packages:
@@ -73,7 +88,7 @@ def packages(
                 host.noop(f"flatpak package {package} is not installed")
 
     if install_packages:
-        yield " ".join(["flatpak", "install", "--noninteractive"] + install_packages)
+        yield f"flatpak install --noninteractive {remote}{' '.join(install_packages)}"
 
     if remove_packages:
-        yield " ".join(["flatpak", "uninstall", "--noninteractive"] + remove_packages)
+        yield f"flatpak uninstall --noninteractive {' '.join(remove_packages)}"

pyinfra/operations/git.py

@@ -9,7 +9,7 @@ import re
 from pyinfra import host
 from pyinfra.api import OperationError, operation
 from pyinfra.facts.files import Directory, File
-from pyinfra.facts.git import GitBranch, GitConfig, GitTrackingBranch
+from pyinfra.facts.git import GitBranch, GitConfig, GitTag, GitTrackingBranch
 
 from . import files, ssh
 from .util.files import chown, unix_path_join
@@ -144,10 +144,14 @@ def repo(
             git_commands.append("clone {0} .".format(src))
     # Ensuring existing repo
     else:
+        is_tag = False
         if branch and host.get_fact(GitBranch, repo=dest) != branch:
             git_commands.append("fetch")  # fetch to ensure we have the branch locally
             git_commands.append("checkout {0}".format(branch))
-        if pull:
+        if branch and branch in host.get_fact(GitTag, repo=dest):
+            git_commands.append("checkout {0}".format(branch))
+            is_tag = True
+        if pull and not is_tag:
             if rebase:
                 git_commands.append("pull --rebase")
             else:

pyinfra/operations/server.py

@@ -20,6 +20,7 @@ from pyinfra.facts.server import (
     Groups,
     Home,
     Hostname,
+    Kernel,
     KernelModules,
     Locales,
     Mounts,
@@ -335,7 +336,15 @@ def mount(
         mounted_options = mounts[path]["options"]
         needed_options = set(options) - set(mounted_options)
         if needed_options:
-            yield "mount -o remount,{0} {1}".format(options_string, path)
+            if host.get_fact(Kernel).strip() == "FreeBSD":
+                fs_type = mounts[path]["type"]
+                device = mounts[path]["device"]
+
+                yield "mount -o update,{options} -t {fs_type} {device} {path}".format(
+                    options=options_string, fs_type=fs_type, device=device, path=path
+                )
+            else:
+                yield "mount -o remount,{0} {1}".format(options_string, path)
 
     else:
         host.noop(
@@ -890,22 +899,29 @@ def user(
 
         if create_home:
             args.append("-m")
-        else:
+        elif os_type != "FreeBSD":
             args.append("-M")
 
-        if password:
+        if password and os_type != "FreeBSD":
             args.append("-p '{0}'".format(password))
 
         # Users are often added by other operations (package installs), so check
         # for the user at runtime before adding.
         add_user_command = "useradd"
+
         if os_type == "FreeBSD":
             add_user_command = "pw useradd"
-            yield "{0} -n {2} {1}".format(
-                add_user_command,
-                " ".join(args),
-                user,
-            )
+
+            if password:
+                yield "echo '{3}' | {0} -n {2} -H 0 {1}".format(
+                    add_user_command, " ".join(args), user, password
+                )
+            else:
+                yield "{0} -n {2} {1}".format(
+                    add_user_command,
+                    " ".join(args),
+                    user,
+                )
         else:
             yield "{0} {1} {2}".format(
                 add_user_command,
@@ -930,16 +946,22 @@ def user(
             args.append("-g {0}".format(group))
 
         # Check secondary groups, if defined
-        if groups and set(existing_user["groups"]) != set(groups):
+        if groups:
             if append:
-                args.append("-a")
-            args.append("-G {0}".format(",".join(groups)))
+                if not set(groups).issubset(existing_user["groups"]):
+                    args.append("-a")
+                    args.append("-G {0}".format(",".join(groups)))
+            elif set(existing_user["groups"]) != set(groups):
+                args.append("-G {0}".format(",".join(groups)))
 
         if comment and existing_user["comment"] != comment:
             args.append("-c '{0}'".format(comment))
 
         if password and existing_user["password"] != password:
-            args.append("-p '{0}'".format(password))
+            if os_type == "FreeBSD":
+                yield "echo '{0}' | pw usermod -n {1} -H 0".format(password, user)
+            else:
+                args.append("-p '{0}'".format(password))
 
         # Need to mod the user?
         if args:
@@ -947,18 +969,6 @@ def user(
                 yield "pw usermod -n {1} {0}".format(" ".join(args), user)
             else:
                 yield "usermod {0} {1}".format(" ".join(args), user)
-            if comment:
-                existing_user["comment"] = comment
-            if home:
-                existing_user["home"] = home
-            if shell:
-                existing_user["shell"] = shell
-            if group:
-                existing_user["group"] = group
-            if groups:
-                existing_user["groups"] = groups
-            if password:
-                existing_user["password"] = password
 
     # Ensure home directory ownership
     if ensure_home and home:

pyinfra/operations/util/docker.py

@@ -117,6 +117,7 @@ def _remove_volume(**kwargs):
 
 def _create_network(**kwargs):
     command = []
+    aux_addresses = kwargs["aux_addresses"] if kwargs["aux_addresses"] else {}
     opts = kwargs["opts"] if kwargs["opts"] else []
     ipam_opts = kwargs["ipam_opts"] if kwargs["ipam_opts"] else []
     labels = kwargs["labels"] if kwargs["labels"] else []
@@ -146,6 +147,9 @@ def _create_network(**kwargs):
     if kwargs["attachable"]:
         command.append("--attachable")
 
+    for host, address in aux_addresses.items():
+        command.append("--aux-address '{0}={1}'".format(host, address))
+
     for opt in opts:
         command.append("--opt {0}".format(opt))