pyinfra 3.4__py2.py3-none-any.whl → 3.5__py2.py3-none-any.whl

pyinfra/operations/docker.py

@@ -4,25 +4,27 @@ the view of the current inventory host. See the :doc:`../connectors/docker` to u
 as inventory directly.
 """
 
+from __future__ import annotations
+
 from pyinfra import host
 from pyinfra.api import operation
-from pyinfra.facts.docker import DockerContainer, DockerNetwork, DockerVolume
+from pyinfra.facts.docker import DockerContainer, DockerNetwork, DockerPlugin, DockerVolume
 
 from .util.docker import ContainerSpec, handle_docker
 
 
 @operation()
 def container(
-    container,
-    image="",
-    ports=None,
-    networks=None,
-    volumes=None,
-    env_vars=None,
-    pull_always=False,
-    present=True,
-    force=False,
-    start=True,
+    container: str,
+    image: str = "",
+    ports: list[str] | None = None,
+    networks: list[str] | None = None,
+    volumes: list[str] | None = None,
+    env_vars: list[str] | None = None,
+    pull_always: bool = False,
+    present: bool = True,
+    force: bool = False,
+    start: bool = True,
 ):
     """
     Manage Docker containers
@@ -168,7 +170,7 @@ def image(image, present=True):
 
 
 @operation()
-def volume(volume, driver="", labels=None, present=True):
+def volume(volume: str, driver: str = "", labels: list[str] | None = None, present: bool = True):
     """
     Manage Docker volumes
 
@@ -220,20 +222,20 @@ def volume(volume, driver="", labels=None, present=True):
 
 @operation()
 def network(
-    network,
-    driver="",
-    gateway="",
-    ip_range="",
-    ipam_driver="",
-    subnet="",
-    scope="",
-    aux_addresses=None,
-    opts=None,
-    ipam_opts=None,
-    labels=None,
-    ingress=False,
-    attachable=False,
-    present=True,
+    network: str,
+    driver: str = "",
+    gateway: str = "",
+    ip_range: str = "",
+    ipam_driver: str = "",
+    subnet: str = "",
+    scope: str = "",
+    aux_addresses: dict[str, str] | None = None,
+    opts: list[str] | None = None,
+    ipam_opts: list[str] | None = None,
+    labels: list[str] | None = None,
+    ingress: bool = False,
+    attachable: bool = False,
+    present: bool = True,
 ):
     """
     Manage docker networks
@@ -245,6 +247,7 @@ def network(
     + ipam_driver: IP Address Management Driver
     + subnet: Subnet in CIDR format that represents a network segment
     + scope: Control the network's scope
+    + aux_addresses: named aux addresses for the network
     + opts: Set driver specific options
     + ipam_opts: Set IPAM driver specific options
     + labels: Label list to attach in the network
@@ -303,9 +306,9 @@ def network(
 
 @operation(is_idempotent=False)
 def prune(
-    all=False,
-    volumes=False,
-    filter="",
+    all: bool = False,
+    volumes: bool = False,
+    filter: str = "",
 ):
     """
     Execute a docker system prune.
@@ -344,3 +347,101 @@ def prune(
         volumes=volumes,
         filter=filter,
     )
+
+
+@operation()
+def plugin(
+    plugin: str,
+    alias: str | None = None,
+    present: bool = True,
+    enabled: bool = True,
+    plugin_options: dict[str, str] | None = None,
+):
+    """
+    Manage Docker plugins
+
+    + plugin: Plugin name
+    + alias: Alias for the plugin (optional)
+    + present: Whether the plugin should be installed
+    + enabled: Whether the plugin should be enabled
+    + plugin_options: Options to pass to the plugin
+
+    **Examples:**
+
+    .. code:: python
+
+        # Install and enable a Docker plugin
+        docker.plugin(
+            name="Install and enable a Docker plugin",
+            plugin="username/my-awesome-plugin:latest",
+            alias="my-plugin",
+            present=True,
+            enabled=True,
+            plugin_options={"option1": "value1", "option2": "value2"},
+        )
+    """
+    plugin_name = alias if alias else plugin
+    existent_plugin = host.get_fact(DockerPlugin, object_id=plugin_name)
+    if existent_plugin:
+        existent_plugin = existent_plugin[0]
+
+    if present:
+        if existent_plugin:
+            plugin_options_different = (
+                plugin_options and existent_plugin["Settings"]["Env"] != plugin_options
+            )
+            if plugin_options_different:
+                # Update options on existing plugin
+                if existent_plugin["Enabled"]:
+                    yield handle_docker(
+                        resource="plugin",
+                        command="disable",
+                        plugin=plugin_name,
+                    )
+                yield handle_docker(
+                    resource="plugin",
+                    command="set",
+                    plugin=plugin_name,
+                    enabled=enabled,
+                    existent_options=existent_plugin["Settings"]["Env"],
+                    required_options=plugin_options,
+                )
+                if enabled:
+                    yield handle_docker(
+                        resource="plugin",
+                        command="enable",
+                        plugin=plugin_name,
+                    )
+            else:
+                # Options are the same, check if enabled state is different
+                if existent_plugin["Enabled"] == enabled:
+                    host.noop(
+                        f"Plugin '{plugin_name}' is already installed with the same options "
+                        f"and {'enabled' if enabled else 'disabled'}."
+                    )
+                    return
+                else:
+                    command = "enable" if enabled else "disable"
+                    yield handle_docker(
+                        resource="plugin",
+                        command=command,
+                        plugin=plugin_name,
+                    )
+        else:
+            yield handle_docker(
+                resource="plugin",
+                command="install",
+                plugin=plugin,
+                alias=alias,
+                enabled=enabled,
+                plugin_options=plugin_options,
+            )
+    else:
+        if not existent_plugin:
+            host.noop(f"Plugin '{plugin_name}' is not installed.")
+            return
+        yield handle_docker(
+            resource="plugin",
+            command="remove",
+            plugin=plugin_name,
+        )

pyinfra/operations/files.py

@@ -8,12 +8,13 @@ import os
 import posixpath
 import sys
 import traceback
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 from fnmatch import fnmatch
 from io import StringIO
 from pathlib import Path
 from typing import IO, Any, Union
 
+import click
 from jinja2 import TemplateRuntimeError, TemplateSyntaxError, UndefinedError
 
 from pyinfra import host, logger, state
@@ -46,6 +47,7 @@ from pyinfra.facts.files import (
     Block,
     Directory,
     File,
+    FileContents,
     FindFiles,
     FindInFile,
     Flags,
@@ -59,8 +61,10 @@ from pyinfra.facts.server import Date, Which
 
 from .util import files as file_utils
 from .util.files import (
+    MetadataTimeField,
     adjust_regex,
     ensure_mode_int,
+    generate_color_diff,
     get_timestamp,
     sed_delete,
     sed_replace,
@@ -813,6 +817,56 @@ def get(
             host.noop("file {0} has already been downloaded".format(dest))
 
 
+def _canonicalize_timespec(field: MetadataTimeField, local_file, timespec):
+    if isinstance(timespec, datetime):
+        if not timespec.tzinfo:
+            # specify remote host timezone
+            timespec_with_tz = timespec.replace(tzinfo=host.get_fact(Date).tzinfo)
+            return timespec_with_tz
+        else:
+            return timespec
+    elif isinstance(timespec, bool) and timespec:
+        lf_ts = (
+            os.stat(local_file).st_atime
+            if field is MetadataTimeField.ATIME
+            else os.stat(local_file).st_mtime
+        )
+        return datetime.fromtimestamp(lf_ts, tz=timezone.utc)
+    else:
+        try:
+            isodatetime = datetime.fromisoformat(timespec)
+            if not isodatetime.tzinfo:
+                return isodatetime.replace(tzinfo=host.get_fact(Date).tzinfo)
+            else:
+                return isodatetime
+        except ValueError:
+            try:
+                timestamp = float(timespec)
+                return datetime.fromtimestamp(timestamp, tz=timezone.utc)
+            except ValueError:
+                # verify there is a remote file matching path in timesrc
+                ref_file = host.get_fact(File, path=timespec)
+                if ref_file:
+                    if field is MetadataTimeField.ATIME:
+                        assert ref_file["atime"] is not None
+                        return ref_file["atime"].replace(tzinfo=timezone.utc)
+                    else:
+                        assert ref_file["mtime"] is not None
+                        return ref_file["mtime"].replace(tzinfo=timezone.utc)
+                else:
+                    ValueError("Bad argument for `timesspec`: {0}".format(timespec))
+
+
+# returns True for a visible difference in the second field between the datetime values
+#   in the ref's TZ
+def _times_differ_in_s(ref, cand):
+    assert ref.tzinfo and cand.tzinfo
+    cand_in_ref_tz = cand.astimezone(ref.tzinfo)
+    return (abs((cand_in_ref_tz - ref).total_seconds()) >= 1.0) or (
+        ref.second != cand_in_ref_tz.second
+    )
+
+
 @operation()
 def put(
     src: str | IO[Any],
@@ -824,6 +878,8 @@ def put(
     create_remote_dir=True,
     force=False,
     assume_exists=False,
+    atime: datetime | float | int | str | bool | None = None,
+    mtime: datetime | float | int | str | bool | None = None,
 ):
     """
     Upload a local file, or file-like object, to the remote system.
@@ -837,6 +893,8 @@ def put(
     + create_remote_dir: create the remote directory if it doesn't exist
     + force: always upload the file, even if the remote copy matches
     + assume_exists: whether to assume the local file exists
+    + atime: value of atime the file should have, use ``True`` to match the local file
+    + mtime: value of mtime the file should have, use ``True`` to match the local file
 
     ``dest``:
         If this is a directory that already exists on the remote side, the local
@@ -853,7 +911,21 @@ def put(
         user & group as passed to ``files.put``. The mode will *not* be copied over,
         if this is required call ``files.directory`` separately.
 
-    Note:
+    ``atime`` and ``mtime``:
+        When set to values other than ``False`` or ``None``, the respective metadata
+        fields on the remote file will updated accordingly.  Timestamp values are
+        considered equivalent if the difference is less than one second and they have
+        the identical number in the seconds field.  If set to ``True`` the local
+        file is the source of the value.  Otherwise, these values can be provided as
+        ``datetime`` objects, POSIX timestamps, or strings that can be parsed into
+        either of these date and time specifications.  They can also be reference file
+        paths on the remote host, as with the ``-r`` argument to ``touch``.  If a
+        ``datetime`` argument has no ``tzinfo`` value (i.e., it is naive), it is
+        assumed to be in the remote host's local timezone.  There is no shortcut for
+        setting both ``atime` and ``mtime`` values with a single time specification,
+        unlike the native ``touch`` command.
+
+    Notes:
         This operation is not suitable for large files as it may involve copying
         the file before uploading it.
 
@@ -862,6 +934,12 @@ def put(
         behave as if the remote file does not match the specified permissions and
         requires a change.
 
+        If the ``atime`` argument is set for a given file, unless the remote
+        filesystem is mounted ``noatime`` or ``relatime``, multiple runs of this
+        operation will trigger the change detection for that file, since the act of
+        reading and checksumming the file will cause the host OS to update the file's
+        ``atime``.
+
     **Examples:**
 
     .. code:: python
@@ -925,6 +1003,16 @@ def put(
 
     # No remote file, always upload and user/group/mode if supplied
     if not remote_file or force:
+        if state.config.DIFF:
+            host.log(f"Will create {click.style(dest, bold=True)}", logger.info)
+
+            with get_file_io(src, "r") as f:
+                desired_lines = f.readlines()
+
+            for line in generate_color_diff([], desired_lines):
+                logger.info(f"  {line}")
+            logger.info("")
+
         yield FileUploadCommand(
             local_file,
             dest,
@@ -937,9 +1025,41 @@ def put(
         if mode:
             yield file_utils.chmod(dest, mode)
 
-    # File exists, check sum and check user/group/mode if supplied
+        # do mtime before atime to ensure atime setting isn't undone by mtime setting
+        if mtime:
+            yield file_utils.touch(
+                dest,
+                MetadataTimeField.MTIME,
+                _canonicalize_timespec(MetadataTimeField.MTIME, src, mtime),
+            )
+
+        if atime:
+            yield file_utils.touch(
+                dest,
+                MetadataTimeField.ATIME,
+                _canonicalize_timespec(MetadataTimeField.ATIME, src, atime),
+            )
+
+    # File exists, check sum and check user/group/mode/atime/mtime if supplied
     else:
         if not _file_equal(local_sum_path, dest):
+            if state.config.DIFF:
+                # Generate diff when contents change
+                current_contents = host.get_fact(FileContents, path=dest)
+                if current_contents:
+                    current_lines = [line + "\n" for line in current_contents]
+                else:
+                    current_lines = []
+
+                host.log(f"Will modify {click.style(dest, bold=True)}", logger.info)
+
+                with get_file_io(src, "r") as f:
+                    desired_lines = f.readlines()
+
+                for line in generate_color_diff(current_lines, desired_lines):
+                    logger.info(f"  {line}")
+                logger.info("")
+
             yield FileUploadCommand(
                 local_file,
                 dest,
@@ -952,6 +1072,20 @@ def put(
             if mode:
                 yield file_utils.chmod(dest, mode)
 
+            if mtime:
+                yield file_utils.touch(
+                    dest,
+                    MetadataTimeField.MTIME,
+                    _canonicalize_timespec(MetadataTimeField.MTIME, src, mtime),
+                )
+
+            if atime:
+                yield file_utils.touch(
+                    dest,
+                    MetadataTimeField.ATIME,
+                    _canonicalize_timespec(MetadataTimeField.ATIME, src, atime),
+                )
+
         else:
             changed = False
 
@@ -965,6 +1099,26 @@ def put(
                 yield file_utils.chown(dest, user, group)
                 changed = True
 
+            # Check mtime
+            if mtime:
+                canonical_mtime = _canonicalize_timespec(MetadataTimeField.MTIME, src, mtime)
+                assert remote_file["mtime"] is not None
+                if _times_differ_in_s(
+                    canonical_mtime, remote_file["mtime"].replace(tzinfo=timezone.utc)
+                ):
+                    yield file_utils.touch(dest, MetadataTimeField.MTIME, canonical_mtime)
+                    changed = True
+
+            # Check atime
+            if atime:
+                canonical_atime = _canonicalize_timespec(MetadataTimeField.ATIME, src, atime)
+                assert remote_file["atime"] is not None
+                if _times_differ_in_s(
+                    canonical_atime, remote_file["atime"].replace(tzinfo=timezone.utc)
+                ):
+                    yield file_utils.touch(dest, MetadataTimeField.ATIME, canonical_atime)
+                    changed = True
+
             if not changed:
                 host.noop("file {0} is already uploaded".format(dest))
 
@@ -1594,7 +1748,7 @@ def block(
             path="/etc/hosts",
             content="10.0.0.1 mars-one",
             before=True,
-            regex=".*localhost",
+            line=".*localhost",
         )
 
         # have two entries in /etc/host
@@ -1603,7 +1757,7 @@ def block(
             path="/etc/hosts",
             content="10.0.0.1 mars-one\\n10.0.0.2 mars-two",
             before=True,
-            regex=".*localhost",
+            line=".*localhost",
         )
 
         # remove marked entry from /etc/hosts
@@ -1618,7 +1772,7 @@ def block(
             name="add out of date warning to web page",
             path="/var/www/html/something.html",
             content= "<p>Warning: this page is out of date.</p>",
-            regex=".*<body>.*",
+            line=".*<body>.*",
             after=True
             marker="<!-- {mark} PYINFRA BLOCK -->",
         )
@@ -1665,6 +1819,7 @@ def block(
     )
 
     current = host.get_fact(Block, path=path, marker=marker, begin=begin, end=end)
+    # None means file didn't exist, empty list means marker was not found
     cmd = None
     if present:
         if not content:
@@ -1706,7 +1861,7 @@ def block(
             prog = (
                 'awk \'BEGIN {x=ARGV[2]; ARGV[2]=""} '
                 f"{print_after} f!=1 && /{regex}/ {{ print x; f=1}} "
-                f"END {{if (f==0) print ARGV[2] }} {print_before}'"
+                f"END {{if (f==0) print x }} {print_before}'"
             )
             cmd = StringCommand(
                 out_prep,

pyinfra/operations/git.py

@@ -148,7 +148,7 @@ def repo(
         if branch and host.get_fact(GitBranch, repo=dest) != branch:
             git_commands.append("fetch")  # fetch to ensure we have the branch locally
             git_commands.append("checkout {0}".format(branch))
-        if branch and branch in host.get_fact(GitTag, repo=dest):
+        if branch and branch in (host.get_fact(GitTag, repo=dest) or []):
             git_commands.append("checkout {0}".format(branch))
             is_tag = True
         if pull and not is_tag:

pyinfra/operations/openrc.py

@@ -48,10 +48,16 @@ def service(
         openrc_enabled = host.get_fact(OpenrcEnabled, runlevel=runlevel)
         is_enabled = openrc_enabled.get(service, False)
 
-        if enabled and not is_enabled:
-            yield "rc-update add {0}".format(service)
-            openrc_enabled[service] = True
-
-        if not enabled and is_enabled:
-            yield "rc-update del {0}".format(service)
-            openrc_enabled[service] = False
+        if enabled is True:
+            if not is_enabled:
+                yield "rc-update add {0} {1}".format(service, runlevel)
+                openrc_enabled[service] = True
+            else:
+                host.noop("service {0} is enabled".format(service))
+
+        if enabled is False:
+            if is_enabled:
+                yield "rc-update del {0} {1}".format(service, runlevel)
+                openrc_enabled[service] = False
+            else:
+                host.noop("service {0} is disabled".format(service))

pyinfra/operations/pip.py

@@ -11,7 +11,7 @@ from pyinfra.facts.files import File
 from pyinfra.facts.pip import PipPackages
 
 from . import files
-from .util.packaging import ensure_packages
+from .util.packaging import PkgInfo, ensure_packages
 
 
 @operation()
@@ -186,21 +186,20 @@ def packages(
 
     # Handle passed in packages
     if packages:
+        if isinstance(packages, str):
+            packages = [packages]
+        # PEP-0426 states that Python packages should be compared using lowercase, so lowercase the
+        # current packages. PkgInfo.from_pep508 takes care of the package name
         current_packages = host.get_fact(PipPackages, pip=pip)
-
-        # PEP-0426 states that Python packages should be compared using lowercase, so lowercase both
-        # the input packages and the fact packages before comparison.
-        packages = [pkg.lower() for pkg in packages]
         current_packages = {pkg.lower(): versions for pkg, versions in current_packages.items()}
 
         yield from ensure_packages(
             host,
-            packages,
+            list(filter(None, (PkgInfo.from_pep508(package) for package in packages))),
             current_packages,
             present,
             install_command=install_command,
             uninstall_command=uninstall_command,
             upgrade_command=upgrade_command,
-            version_join="==",
             latest=latest,
         )

pyinfra/operations/pipx.py

@@ -2,25 +2,27 @@
 Manage pipx (python) applications.
 """
 
+from typing import Optional, Union
+
 from pyinfra import host
 from pyinfra.api import operation
 from pyinfra.facts.pipx import PipxEnvironment, PipxPackages
 from pyinfra.facts.server import Path
 
-from .util.packaging import ensure_packages
+from .util.packaging import PkgInfo, ensure_packages
 
 
 @operation()
 def packages(
-    packages=None,
+    packages: Optional[Union[str, list[str]]] = None,
     present=True,
     latest=False,
-    extra_args=None,
+    extra_args: Optional[str] = None,
 ):
     """
     Install/remove/update pipx packages.
 
-    + packages: list of packages to ensure
+    + packages: list of packages (PEP-508 format) to ensure
     + present: whether the packages should be installed
     + latest: whether to upgrade packages without a specified version
     + extra_args: additional arguments to the pipx command
@@ -37,6 +39,9 @@ def packages(
             packages=["pyinfra"],
         )
     """
+    if packages is None:
+        host.noop("no package list provided to pipx.packages")
+        return
 
     prep_install_command = ["pipx", "install"]
 
@@ -47,19 +52,26 @@ def packages(
     uninstall_command = "pipx uninstall"
     upgrade_command = "pipx upgrade"
 
-    current_packages = host.get_fact(PipxPackages)
+    # PEP-0426 states that Python packages should be compared using lowercase, so lowercase the
+    # current packages.  PkgInfo.from_pep508 takes care of it for the package names
+    current_packages = {
+        pkg.lower(): version for pkg, version in host.get_fact(PipxPackages).items()
+    }
+    if isinstance(packages, str):
+        packages = [packages]
 
     # pipx support only one package name at a time
     for package in packages:
+        if (pkg_info := PkgInfo.from_pep508(package)) is None:
+            continue  # from_pep508 logged a warning
         yield from ensure_packages(
             host,
-            [package],
+            [pkg_info],
             current_packages,
             present,
             install_command=install_command,
             uninstall_command=uninstall_command,
             upgrade_command=upgrade_command,
-            version_join="==",
             latest=latest,
         )