pyinfra 3.3__py2.py3-none-any.whl → 3.4__py2.py3-none-any.whl

pyinfra/context.py

@@ -59,16 +59,22 @@ class ContextObject:
         if key in ("_container", "_base_cls"):
             return super().__setattr__(key, value)
 
-        if self._get_module() is None:
+        mod = self._get_module()
+        if mod is None:
             raise TypeError("Cannot assign to context base module")
-
-        return setattr(self._get_module(), key, value)
+        return setattr(mod, key, value)
 
     def __iter__(self):
-        return iter(self._get_module())
+        mod = self._get_module()
+        if mod is None:
+            raise ValueError("Context not set")
+        return iter(mod)
 
     def __len__(self):
-        return len(self._get_module())
+        mod = self._get_module()
+        if mod is None:
+            raise ValueError("Context not set")
+        return len(mod)
 
     @override
     def __eq__(self, other):
@@ -105,6 +111,9 @@ class ContextManager:
     @contextmanager
     def use(self, module):
         old_module = self.get()
+        if old_module is module:
+            yield  # if we're double-setting, nothing to do
+            return
         self.set(module)
         yield
         self.set(old_module)

pyinfra/facts/brew.py

@@ -47,6 +47,7 @@ class BrewVersion(FactBase):
     def requires_command(self) -> str:
         return "brew"
 
+    @override
     @staticmethod
     def default():
         return [0, 0, 0]

pyinfra/facts/docker.py

@@ -49,7 +49,9 @@ class DockerContainers(DockerFactBase):
 
     @override
     def command(self) -> str:
-        return "docker container inspect `docker ps -qa`"
+        return """
+        ids=$(docker ps -qa) && [ -n "$ids" ] && docker container inspect $ids || echo "[]"
+        """.strip()
 
 
 class DockerImages(DockerFactBase):
@@ -59,7 +61,9 @@ class DockerImages(DockerFactBase):
 
     @override
     def command(self) -> str:
-        return "docker image inspect `docker images -q`"
+        return """
+        ids=$(docker images -q) && [ -n "$ids" ] && docker image inspect $ids || echo "[]"
+        """.strip()
 
 
 class DockerNetworks(DockerFactBase):

pyinfra/facts/git.py

@@ -23,6 +23,16 @@ class GitBranch(GitFactBase):
         return re.sub(r"(heads|tags)/", r"", "\n".join(output))
 
 
+class GitTag(GitFactBase):
+    @override
+    def command(self, repo) -> str:
+        return "! test -d {0} || (cd {0} && git tag)".format(repo)
+
+    @override
+    def process(self, output):
+        return output
+
+
 class GitConfig(GitFactBase):
     default = dict
 

pyinfra/facts/hardware.py

@@ -46,7 +46,7 @@ class Memory(FactBase):
             value, key = line.split(" ", 1)
 
             try:
-                value = int(value)
+                value = float(value)
             except ValueError:
                 continue
 

pyinfra/facts/opkg.py

@@ -81,6 +81,7 @@ class OpkgConf(FactBase):
         re.X,
     )
 
+    @override
     @staticmethod
     def default():
         return OpkgConfInfo({}, "", {}, {})

pyinfra/facts/server.py

@@ -1,16 +1,18 @@
 from __future__ import annotations
 
+import json
 import os
 import re
 import shutil
 from datetime import datetime
 from tempfile import mkdtemp
-from typing import Dict, List, Optional
+from typing import Dict, Iterable, List, Optional, Tuple
 
 from dateutil.parser import parse as parse_date
 from distro import distro
 from typing_extensions import TypedDict, override
 
+from pyinfra import host
 from pyinfra.api import FactBase, ShortFactBase
 from pyinfra.api.util import try_int
 from pyinfra.facts import crontab
@@ -205,40 +207,96 @@ class Mounts(FactBase[Dict[str, MountsDict]]):
     default = dict
 
     @override
-    def command(self):
-        return "mount"
+    def command(self) -> str:
+        self._kernel = host.get_fact(Kernel)
+
+        if self._kernel.strip() == "FreeBSD":
+            return "mount -p --libxo json"
+        else:
+            return "cat /proc/self/mountinfo"
 
     @override
     def process(self, output) -> dict[str, MountsDict]:
         devices: dict[str, MountsDict] = {}
 
-        for line in output:
-            is_map = False
-            if line.startswith("map "):
-                line = line[4:]
-                is_map = True
+        def unescape_octal(match: re.Match) -> str:
+            s = match.group(0)[1:]  # skip the backslash
+            return chr(int(s, base=8))
 
-            device, _, path, other_bits = line.split(" ", 3)
+        def replace_octal(s: str) -> str:
+            """
+            Unescape strings encoded by linux's string_escape_mem with ESCAPE_OCTAL flag.
+            """
+            return re.sub(r"\\[0-7]{3}", unescape_octal, s)
 
-            if is_map:
-                device = "map {0}".format(device)
+        if self._kernel == "FreeBSD":
+            full_output = "\n".join(output)
+            json_output = json.loads(full_output)
+            mount_fstab = json_output["mount"]["fstab"]
 
-            if other_bits.startswith("type"):
-                _, type_, options = other_bits.split(" ", 2)
-                options = options.strip("()").split(",")
-            else:
-                options = other_bits.strip("()").split(",")
-                type_ = options.pop(0)
+            for entry in mount_fstab:
+                path = entry["mntpoint"]
+                type_ = entry["fstype"]
+                device = entry["device"]
+                options = [option.strip() for option in entry["opts"].split(",")]
 
-            devices[path] = {
-                "device": device,
-                "type": type_,
-                "options": [option.strip() for option in options],
+                devices[path] = {"device": device, "type": type_, "options": options}
+
+            return devices
+
+        for line in output:
+            # ignore mount ID, parent ID, major:minor, root
+            _, _, _, _, mount_point, mount_options, line = line.split(sep=" ", maxsplit=6)
+
+            # ignore optional tags "shared", "master", "propagate_from" and "unbindable"
+            while True:
+                optional, line = line.split(sep=" ", maxsplit=1)
+                if optional == "-":
+                    break
+
+            fs_type, mount_source, super_options = line.split(sep=" ")
+
+            mount_options = mount_options.split(sep=",")
+
+            # escaped: mount_point, mount_source, super_options
+            # these strings can contain characters encoded in octal, e.g. '\054' for ','
+            mount_point = replace_octal(mount_point)
+            mount_source = replace_octal(mount_source)
+
+            # mount_options will override ro/rw and can be different than the super block options
+            # filter them, so they don't appear twice
+            super_options = [
+                replace_octal(opt)
+                for opt in super_options.split(sep=",")
+                if opt not in ["ro", "rw"]
+            ]
+
+            devices[mount_point] = {
+                "device": mount_source,
+                "type": fs_type,
+                "options": mount_options + super_options,
             }
 
         return devices
 
 
+class Port(FactBase[Tuple[str, int] | Tuple[None, None]]):
+    """
+    Returns the process occuping a port and its PID
+    """
+
+    @override
+    def command(self, port: int) -> str:
+        return f"ss -lptnH 'src :{port}'"
+
+    @override
+    def process(self, output: Iterable[str]) -> Tuple[str, int] | Tuple[None, None]:
+        for line in output:
+            proc, pid = line.split('"')[1], int(line.split("pid=")[1].split(",")[0])
+            return (proc, pid)
+        return None, None
+
+
 class KernelModules(FactBase):
     """
     Returns a dictionary of kernel module name -> info.
@@ -470,10 +528,10 @@ class Users(FactBase):
         for i in `cat /etc/passwd | cut -d: -f1`; do
             ENTRY=`grep ^$i: /etc/passwd`;
             LASTLOG=`(((lastlog -u $i || lastlogin $i) 2> /dev/null) | grep ^$i | tr -s ' ')`;
-            PASSWORD=`grep ^$i: /etc/shadow | cut -d: -f2`;
+            PASSWORD=`(grep ^$i: /etc/shadow || grep ^$i: /etc/master.passwd) 2> /dev/null | cut -d: -f2`;
             echo "$ENTRY|`id -gn $i`|`id -Gn $i`|$LASTLOG|$PASSWORD";
         done
-    """.strip()
+    """.strip()  # noqa
 
     default = dict
 

pyinfra/facts/systemd.py

@@ -64,7 +64,7 @@ class SystemdStatus(FactBase[Dict[str, bool]]):
     default = dict
 
     state_key = "SubState"
-    state_values = ["running", "waiting", "exited", "listening"]
+    state_values = ["running", "waiting", "exited", "listening", "mounted"]
 
     @override
     def command(

pyinfra/operations/crontab.py

@@ -2,11 +2,13 @@ from __future__ import annotations
 
 import shlex
 
-from pyinfra import host
-from pyinfra.api import StringCommand, operation
+from pyinfra import logger
+from pyinfra.api.command import StringCommand
+from pyinfra.api.operation import operation
 from pyinfra.api.util import try_int
+from pyinfra.context import host
 from pyinfra.facts.crontab import Crontab, CrontabFile
-from pyinfra.operations.util.files import sed_replace
+from pyinfra.operations.util.files import sed_delete, sed_replace
 
 
 @operation()
@@ -111,7 +113,7 @@ def crontab(
     # Don't want the cron and it does exist? Remove the line
     if not present and exists:
         edit_commands.append(
-            sed_replace(
+            sed_delete(
                 temp_filename,
                 existing_crontab_match,
                 "",
@@ -121,7 +123,7 @@ def crontab(
 
     # Want the cron but it doesn't exist? Append the line
     elif present and not exists:
-        print("present", present, "exists", exists)
+        logger.debug(f"present: {present}, exists: {exists}")
         if ctb:  # append a blank line if cron entries already exist
             edit_commands.append("echo '' >> {0}".format(temp_filename))
         if cron_name:

pyinfra/operations/docker.py

@@ -227,6 +227,7 @@ def network(
     ipam_driver="",
     subnet="",
     scope="",
+    aux_addresses=None,
     opts=None,
     ipam_opts=None,
     labels=None,
@@ -279,6 +280,7 @@ def network(
             ipam_driver=ipam_driver,
             subnet=subnet,
             scope=scope,
+            aux_addresses=aux_addresses,
             opts=opts,
             ipam_opts=ipam_opts,
             labels=labels,

pyinfra/operations/files.py

@@ -32,7 +32,9 @@ from pyinfra.api.command import make_formatted_string_command
 from pyinfra.api.util import (
     get_call_location,
     get_file_io,
+    get_file_md5,
     get_file_sha1,
+    get_file_sha256,
     get_path_permissions_mode,
     get_template,
     memoize,
@@ -56,7 +58,14 @@ from pyinfra.facts.files import (
 from pyinfra.facts.server import Date, Which
 
 from .util import files as file_utils
-from .util.files import adjust_regex, ensure_mode_int, get_timestamp, sed_replace, unix_path_join
+from .util.files import (
+    adjust_regex,
+    ensure_mode_int,
+    get_timestamp,
+    sed_delete,
+    sed_replace,
+    unix_path_join,
+)
 
 
 @operation()
@@ -75,6 +84,9 @@ def download(
     headers: dict[str, str] | None = None,
     insecure=False,
     proxy: str | None = None,
+    temp_dir: str | Path | None = None,
+    extra_curl_args: dict[str, str] | None = None,
+    extra_wget_args: dict[str, str] | None = None,
 ):
     """
     Download files from remote locations using ``curl`` or ``wget``.
@@ -93,6 +105,9 @@ def download(
     + headers: optional dictionary of headers to set for the HTTP request
     + insecure: disable SSL verification for the HTTP request
     + proxy: simple HTTP proxy through which we can download files, form `http://<yourproxy>:<port>`
+    + temp_dir: use this custom temporary directory during the download
+    + extra_curl_args: optional dictionary with custom arguments for curl
+    + extra_wget_args: optional dictionary with custom arguments for wget
 
     **Example:**
 
@@ -148,11 +163,21 @@ def download(
 
     # If we download, always do user/group/mode as SSH user may be different
     if download:
-        temp_file = host.get_temp_filename(dest)
+        temp_file = host.get_temp_filename(
+            dest, temp_directory=str(temp_dir) if temp_dir is not None else None
+        )
 
         curl_args: list[Union[str, StringCommand]] = ["-sSLf"]
         wget_args: list[Union[str, StringCommand]] = ["-q"]
 
+        if extra_curl_args:
+            for key, value in extra_curl_args.items():
+                curl_args.append(StringCommand(key, QuoteString(value)))
+
+        if extra_wget_args:
+            for key, value in extra_wget_args.items():
+                wget_args.append(StringCommand(key, QuoteString(value)))
+
         if proxy:
             curl_args.append(f"--proxy {proxy}")
             wget_args.append("-e use_proxy=yes")
@@ -423,9 +448,9 @@ def line(
         else:
             host.noop('line "{0}" exists in {1}'.format(replace or line, path))
 
-    # Line(s) exists and we want to remove them, replace with nothing
+    # Line(s) exists and we want to remove them
     elif present_lines and not present:
-        yield sed_replace(
+        yield sed_delete(
             path,
             match_line,
             "",
@@ -710,6 +735,21 @@ def _create_remote_dir(remote_filename, user, group):
         )
 
 
+def _file_equal(local_path: str | IO[Any] | None, remote_path: str) -> bool:
+    if local_path is None:
+        return False
+    for fact, get_sum in [
+        (Sha1File, get_file_sha1),
+        (Md5File, get_file_md5),
+        (Sha256File, get_file_sha256),
+    ]:
+        remote_sum = host.get_fact(fact, path=remote_path)
+        if remote_sum:
+            local_sum = get_sum(local_path)
+            return local_sum == remote_sum
+    return False
+
+
 @operation(
     # We don't (currently) cache the local state, so there's nothing we can
     # update to flag the local file as present.
@@ -766,12 +806,11 @@ def get(
 
     # Remote file exists - check if it matches our local
     else:
-        local_sum = get_file_sha1(dest)
-        remote_sum = host.get_fact(Sha1File, path=src)
-
-        # Check sha1sum, upload if needed
-        if local_sum != remote_sum:
+        # Check hash sum, download if needed
+        if not _file_equal(dest, src):
             yield FileDownloadCommand(src, dest, remote_temp_filename=host.get_temp_filename(dest))
+        else:
+            host.noop("file {0} has already been downloaded".format(dest))
 
 
 @operation()
@@ -805,7 +844,9 @@ def put(
 
     ``mode``:
         When set to ``True`` the permissions of the local file are applied to the
-        remote file after the upload is complete.
+        remote file after the upload is complete.  If set to an octal value with
+        digits for at least user, group, and other, either as an ``int`` or
+        ``str``, those permissions will be used.
 
     ``create_remote_dir``:
         If the remote directory does not exist it will be created using the same
@@ -816,6 +857,11 @@ def put(
         This operation is not suitable for large files as it may involve copying
         the file before uploading it.
 
+        Currently, if the mode argument is anything other than a ``bool`` or a full
+        octal permission set and the remote file exists, the operation will always
+        behave as if the remote file does not match the specified permissions and
+        requires a change.
+
     **Examples:**
 
     .. code:: python
@@ -837,7 +883,7 @@ def put(
     # Upload IO objects as-is
     if hasattr(src, "read"):
         local_file = src
-        local_sum = get_file_sha1(src)
+        local_sum_path = src
 
     # Assume string filename
     else:
@@ -849,9 +895,9 @@ def put(
         local_file = src
 
         if os.path.isfile(local_file):
-            local_sum = get_file_sha1(local_file)
+            local_sum_path = local_file
         elif assume_exists:
-            local_sum = None
+            local_sum_path = None
         else:
             raise IOError("No such file: {0}".format(local_file))
 
@@ -893,10 +939,7 @@ def put(
 
     # File exists, check sum and check user/group/mode if supplied
     else:
-        remote_sum = host.get_fact(Sha1File, path=dest)
-
-        # Check sha1sum, upload if needed
-        if local_sum != remote_sum:
+        if not _file_equal(local_sum_path, dest):
             yield FileUploadCommand(
                 local_file,
                 dest,
@@ -1612,12 +1655,12 @@ def block(
         "2>/dev/null || stat -f %Lp",
         q_path,
         ") $OUT && ",
-        '(chown $(stat -c "%U:%G"',
+        '(chown $(stat -c "%u:%g"',
         q_path,
-        "2>/dev/null) $OUT || ",
-        'chown -n $(stat -f "%u:%g"',
+        "2>/dev/null || ",
+        'stat -f "%u:%g"',
         q_path,
-        ') $OUT)  && mv "$OUT"',
+        '2>/dev/null ) $OUT) && mv "$OUT"',
         q_path,
     )
 

pyinfra/operations/flatpak.py

@@ -12,12 +12,14 @@ from pyinfra.facts.flatpak import FlatpakPackages
 @operation()
 def packages(
     packages: str | list[str] | None = None,
+    remote: str | None = None,
     present=True,
 ):
     """
     Install/remove a flatpak package
 
     + packages: List of packages
+    + remote: Source to install the application or runtime from
     + present: whether the package should be installed
 
     **Examples:**
@@ -30,6 +32,13 @@ def packages(
             packages="org.videolan.VLC",
         )
 
+        # Install vlc flatpak from flathub
+        flatpak.package(
+            name="Install vlc",
+            packages="org.videolan.VLC",
+            remote="flathub",
+        )
+
         # Install multiple flatpaks
         flatpak.package(
             name="Install vlc and kodi",
@@ -55,6 +64,12 @@ def packages(
     install_packages = []
     remove_packages = []
 
+    if remote is None:
+        remote = ""
+    else:
+        # ensure we have a space between the remote and packages
+        remote = remote.strip() + " "
+
     for package in packages:
         # it's installed
         if package in flatpak_packages:
@@ -73,7 +88,7 @@ def packages(
                 host.noop(f"flatpak package {package} is not installed")
 
     if install_packages:
-        yield " ".join(["flatpak", "install", "--noninteractive"] + install_packages)
+        yield f"flatpak install --noninteractive {remote}{' '.join(install_packages)}"
 
     if remove_packages:
-        yield " ".join(["flatpak", "uninstall", "--noninteractive"] + remove_packages)
+        yield f"flatpak uninstall --noninteractive {' '.join(remove_packages)}"

pyinfra/operations/git.py

@@ -9,7 +9,7 @@ import re
 from pyinfra import host
 from pyinfra.api import OperationError, operation
 from pyinfra.facts.files import Directory, File
-from pyinfra.facts.git import GitBranch, GitConfig, GitTrackingBranch
+from pyinfra.facts.git import GitBranch, GitConfig, GitTag, GitTrackingBranch
 
 from . import files, ssh
 from .util.files import chown, unix_path_join
@@ -144,10 +144,14 @@ def repo(
             git_commands.append("clone {0} .".format(src))
     # Ensuring existing repo
     else:
+        is_tag = False
         if branch and host.get_fact(GitBranch, repo=dest) != branch:
             git_commands.append("fetch")  # fetch to ensure we have the branch locally
             git_commands.append("checkout {0}".format(branch))
-        if pull:
+        if branch and branch in host.get_fact(GitTag, repo=dest):
+            git_commands.append("checkout {0}".format(branch))
+            is_tag = True
+        if pull and not is_tag:
             if rebase:
                 git_commands.append("pull --rebase")
             else: