python3-cyberfusion-common 2.10.11.3.3__py3-none-any.whl

cyberfusion/Common/Config.py

@@ -0,0 +1,42 @@
+"""Helper for handling config files."""
+
+import configparser
+import os
+from typing import Optional
+
+from cached_property import cached_property
+
+
+class CyberfusionConfig:
+    """Abstract ConfigParser implementation for use in scripts."""
+
+    def __init__(
+        self,
+        path: Optional[str] = None,
+    ) -> None:
+        """Set attributes."""
+        self._path = path
+
+    @property
+    def path(self) -> str:
+        """Set config file path."""
+        if not self._path:
+            self._path = os.path.join(
+                os.path.sep, "etc", "cyberfusion", "cyberfusion.cfg"
+            )
+
+        return self._path
+
+    @cached_property
+    def config(self) -> configparser.ConfigParser:
+        """Read config."""
+        config = configparser.ConfigParser()
+
+        with open(self.path, "r") as f:
+            config.read_file(f)
+
+        return config
+
+    def get(self, section: str, key: str) -> str:
+        """Retrieve config option."""
+        return self.config.get(section, key)

cyberfusion/Common/Filesystem.py

@@ -0,0 +1,79 @@
+"""Helper for handling filesystem."""
+
+import os
+from enum import Enum
+from pathlib import Path
+
+import psutil
+
+CEPH_NAME_ATTRIBUTE_RBYTES = "ceph.dir.rbytes"
+
+
+class FilesystemType(Enum):
+    """Filesystem types.
+
+    We don't expect any other filesystems than the ones in this Enum under normal
+    circumstances. This Enum prevents you from having to write filesystem types
+    by name manually; it's not a complete list of all possibilities.
+    """
+
+    EXT4 = "ext4"
+    EXT2 = "ext2"  # Usually for /boot
+    XFS = "xfs"
+    CEPH = "ceph"
+    OVERLAY = "overlay"  # Usually for Docker
+    APFS = "apfs"  # Development on macOS
+
+
+def get_filesystem(path: str) -> str:
+    """Get filesystem that absolute path is on.
+
+    Returns the earliest filesystem. E.g. if '/a/b/c' is passed, and both '/a' and
+    '/b' are mountpoints, '/b' is returned.
+
+    Does not resolve symlinks, so the filesystem of the symlink itself is looked
+    up.
+
+    Inspired by https://stackoverflow.com/a/4453715/3837431
+    """
+
+    # Ensure path is an absolute path. For relative paths, dirname() would return
+    # the wrong result
+
+    path = os.path.abspath(path)
+
+    # Work our way down the path by getting the directory that the current
+    # filesystem object is in
+
+    while not os.path.ismount(path):
+        path = os.path.dirname(path)
+
+    return path
+
+
+def get_filesystem_type(path: str) -> FilesystemType:
+    """Get type of filesystem."""
+
+    # If this yields no results (i.e. the path is missing), next() raises StopIteration
+
+    partition = next(
+        filter(lambda x: x.mountpoint == path, psutil.disk_partitions(all=True))
+    )
+
+    return FilesystemType(partition.fstype)
+
+
+def get_directory_size(path: str) -> int:
+    """Get size of directory."""
+
+    # CephFS is a special case, as CephFS has extended attributes to determine
+    # size
+
+    is_ceph = get_filesystem_type(get_filesystem(path)) == FilesystemType.CEPH
+
+    if is_ceph:
+        return int(os.getxattr(path, CEPH_NAME_ATTRIBUTE_RBYTES).decode("utf-8"))  # type: ignore[attr-defined]
+
+    # Loop through directory for filesystems without special implementation
+
+    return sum(f.stat().st_size for f in Path(path).rglob("*") if f.is_file())

cyberfusion/Common/FilesystemComparison.py

@@ -0,0 +1,185 @@
+"""Helper for comparing filesystem objects."""
+
+import filecmp
+import os
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+
+def _get_recursive_dircmps(dircmp: filecmp.dircmp) -> List[filecmp.dircmp]:
+    """Get dircmp object for every subdirectory.
+
+    Subdirectories are only returned for directories that are in the left and
+    right directories.
+    """
+    dircmps = []
+
+    dircmps.append(dircmp)
+
+    for sub_dircmp in dircmp.subdirs.values():
+        dircmps.extend(_get_recursive_dircmps(sub_dircmp))
+
+    return dircmps
+
+
+def get_different_files_in_directories(
+    left_directory_path: str, right_directory_path: str
+) -> List[Tuple[str, str]]:
+    """Get files with same name in both directories but with different contents or attributes.
+
+    Files which are present in subdirectories that only exist in either the left
+    or right directories are not returned. Use the 'get_directories_only_in_*_directory'
+    functions to get subdirectories that are in either the left or right directories.
+    This ensures that this function does not return individual files in whole new
+    directories.
+    """
+    dircmps = _get_recursive_dircmps(
+        filecmp.dircmp(left_directory_path, right_directory_path)
+    )
+
+    files = []
+
+    for dircmp in dircmps:
+        for diff_file in dircmp.diff_files:
+            absolute_path_left = os.path.join(dircmp.left, diff_file)
+            absolute_path_right = os.path.join(dircmp.right, diff_file)
+
+            files.append((absolute_path_left, absolute_path_right))
+
+    return files
+
+
+def get_files_only_in_left_directory(
+    left_directory_path: str, right_directory_path: str
+) -> List[str]:
+    """Get files which are only present in left directory.
+
+    Files which are present in subdirectories that only exist in either the left
+    or right directories are not returned. Use the 'get_directories_only_in_*_directory'
+    functions to get subdirectories that are in either the left or right directories.
+    This ensures that this function does not return individual files in whole new
+    directories.
+    """
+    dircmps = _get_recursive_dircmps(
+        filecmp.dircmp(left_directory_path, right_directory_path)
+    )
+
+    files = []
+
+    for dircmp in dircmps:
+        for left_only_file in dircmp.left_only:
+            absolute_path = os.path.join(dircmp.left, left_only_file)
+
+            if os.path.isdir(absolute_path):
+                continue
+
+            files.append(absolute_path)
+
+    return files
+
+
+def get_files_only_in_right_directory(
+    left_directory_path: str, right_directory_path: str
+) -> List[str]:
+    """Get files which are only present in right directory.
+
+    Files which are present in subdirectories that only exist in either the left
+    or right directories are not returned. Use the 'get_directories_only_in_*_directory'
+    functions to get subdirectories that are in either the left or right directories.
+    This ensures that this function does not return individual files in whole new
+    directories.
+    """
+    dircmps = _get_recursive_dircmps(
+        filecmp.dircmp(left_directory_path, right_directory_path)
+    )
+
+    files = []
+
+    for dircmp in dircmps:
+        for right_only_file in dircmp.right_only:
+            absolute_path = os.path.join(dircmp.right, right_only_file)
+
+            if os.path.isdir(absolute_path):
+                continue
+
+            files.append(absolute_path)
+
+    return files
+
+
+def get_directories_only_in_left_directory(
+    left_directory_path: str, right_directory_path: str
+) -> List[str]:
+    """Get directories which are only present in left directory."""
+    dircmps = _get_recursive_dircmps(
+        filecmp.dircmp(left_directory_path, right_directory_path)
+    )
+
+    directories = []
+
+    for dircmp in dircmps:
+        for left_only_file in dircmp.left_only:
+            absolute_path = os.path.join(dircmp.left, left_only_file)
+
+            if os.path.isfile(absolute_path):
+                continue
+
+            directories.append(absolute_path)
+
+    return directories
+
+
+def get_directories_only_in_right_directory(
+    left_directory_path: str, right_directory_path: str
+) -> List[str]:
+    """Get directories which are only present in right directory."""
+    dircmps = _get_recursive_dircmps(
+        filecmp.dircmp(left_directory_path, right_directory_path)
+    )
+
+    directories = []
+
+    for dircmp in dircmps:
+        for right_only_file in dircmp.right_only:
+            absolute_path = os.path.join(dircmp.right, right_only_file)
+
+            if os.path.isfile(absolute_path):
+                continue
+
+            directories.append(absolute_path)
+
+    return directories
+
+
+def get_nested_directory_structure(
+    paths_list: List[str],
+) -> Dict[str, Optional[Dict[str, Optional[dict]]]]:
+    """Convert list of paths to nested dict."""
+    tree: Dict[str, Optional[Dict[str, Optional[dict]]]] = {}
+
+    # Example of logic:
+    #
+    # >>> tree = {}
+    # >>> node = tree
+    # >>> node = node.setdefault('a', {})
+    # >>> node = node.setdefault('b', {})
+    # >>> node = tree
+    # >>> tree
+    # {'a': {'b': {}}}
+
+    for path in paths_list:
+        node = tree  # Reset to root level for new path, see docstring
+
+        parts = Path(path).parts
+
+        for i, element in enumerate(parts):
+            last = i == (len(parts) - 1)
+
+            if last:
+                node.setdefault(element, None)
+            else:
+                node = node.setdefault(  # type: ignore[assignment]  # mypy does not support nested types
+                    element, {}
+                )  # Next recursion will be at this level, see docstring
+
+    return tree

cyberfusion/Common/__init__.py

@@ -0,0 +1,156 @@
+"""Helper classes for Cyberfusion scripts."""
+
+import base64
+import os
+import secrets
+import shutil
+import string
+import uuid
+from datetime import datetime, timezone
+from hashlib import md5, sha1
+from socket import gethostname
+from typing import Optional
+
+import requests
+
+from cyberfusion.Common.exceptions import ExecutableNotFound
+
+CHAR_PREFIX_WILDCARD = "*"
+CHAR_LABEL = "."
+
+
+class EmailAddresses:
+    """Cyberfusion email addresses."""
+
+    SYSTEM_MESSAGES_CORE = "system-messages.core@cyberfusion.io"
+    SYSTEM_MESSAGES_INFRASTRUCTURE = "system-messages.infrastructure@cyberfusion.io"
+    SUPPORT = "support@cyberfusion.io"
+
+
+def find_executable(name: str) -> str:
+    """Find absolute path of executable.
+
+    Use this function when the executable must exist. This function raises an
+    exception if it does not.
+    """
+    path = shutil.which(name)
+
+    if path:
+        return path
+
+    raise ExecutableNotFound(name)
+
+
+def try_find_executable(name: str) -> Optional[str]:
+    """Find absolute path of executable.
+
+    Use this function when the executable may be absent. This function returns
+    None if it is.
+    """
+    return shutil.which(name)
+
+
+def download_from_url(url: str, *, root_directory: Optional[str] = None) -> str:
+    """Download from URL.
+
+    Large files are supported due to use of chunking and streaming.
+
+    Inspired by https://stackoverflow.com/a/16696317/19535769
+    """
+    if root_directory is None:
+        root_directory = os.path.join(os.path.sep, "tmp")
+
+    path = os.path.join(root_directory, generate_random_string())
+
+    # Create and set permissions
+
+    with open(path, "w"):
+        pass
+
+    os.chmod(path, 0o600)
+
+    # Download
+
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+
+        with open(path, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+    return path
+
+
+def get_tmp_file() -> str:
+    """Create tmp file and return path."""
+    path = os.path.join(os.path.sep, "tmp", str(uuid.uuid4()))
+
+    with open(path, "w"):
+        pass
+
+    os.chmod(path, 0o600)  # Do not allow regular users to view file contents
+
+    return path
+
+
+def get_hostname() -> str:
+    """Get hostname."""
+    return gethostname()
+
+
+def get_domain_is_wildcard(domain: str) -> bool:
+    """Determine if domain is wildcard."""
+    return domain.split(CHAR_LABEL)[0] == CHAR_PREFIX_WILDCARD
+
+
+def get_today_timestamp() -> float:
+    """Get UNIX timestamp from the first second of today."""
+    current_datetime = datetime.utcnow()
+
+    return (
+        datetime(
+            year=current_datetime.year,
+            month=current_datetime.month,
+            day=current_datetime.day,
+        )
+        .replace(tzinfo=timezone.utc)
+        .timestamp()
+    )
+
+
+def generate_random_string(length: int = 24) -> str:
+    """Generate random string."""
+
+    # Set allowed characters (ASCII + digits)
+
+    alphabet = string.ascii_letters + string.digits
+
+    # Return string using allowed characters
+
+    return "".join(secrets.choice(alphabet) for i in range(length))
+
+
+def hash_string_mariadb(string: str) -> str:
+    """Hash string with SHA-1 by MariaDB standards."""
+    return (
+        "*"
+        + sha1(sha1(string.encode("utf-8")).digest())  # noqa: S303
+        .hexdigest()
+        .upper()
+    )
+
+
+def convert_bytes_gib(size: int) -> float:
+    """Convert bytes to GiB."""
+    return size / (1024**3)
+
+
+def get_md5_hash(path: str) -> str:
+    """Get Base64 encoded 128-bit MD5 digest of file."""
+    hash_ = md5()
+
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(128 * hash_.block_size), b""):
+            hash_.update(chunk)
+
+    return base64.b64encode(hash_.digest()).decode()

cyberfusion/Common/exceptions/__init__.py

@@ -0,0 +1,10 @@
+"""Exceptions."""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class ExecutableNotFound(Exception):
+    """Raise exception when executable not found."""
+
+    name: str

python3_cyberfusion_common-2.10.11.3.3.dist-info/METADATA

@@ -0,0 +1,39 @@
+Metadata-Version: 2.1
+Name: python3-cyberfusion-common
+Version: 2.10.11.3.3
+Summary: Common utilities.
+Author-email: Cyberfusion <support@cyberfusion.io>
+Project-URL: Source, https://github.com/CyberfusionIO/python3-cyberfusion-common
+Description-Content-Type: text/markdown
+Requires-Dist: cached-property ==1.5.2
+Requires-Dist: psutil ==5.9.4
+Requires-Dist: requests ==2.28.1
+
+# python3-cyberfusion-common
+
+Common utilities.
+
+These utilities are mostly used by Cyberfusion libraries, but are also useful standalone.
+
+# Install
+
+## PyPI
+
+Run the following command to install the package from PyPI:
+
+    pip3 install python3-cyberfusion-common
+
+## Debian
+
+Run the following commands to build a Debian package:
+
+    mk-build-deps -i -t 'apt -o Debug::pkgProblemResolver=yes --no-install-recommends -y'
+    dpkg-buildpackage -us -uc
+
+# Configure
+
+No configuration is supported.
+
+# Usage
+
+See code.

python3_cyberfusion_common-2.10.11.3.3.dist-info/RECORD

@@ -0,0 +1,9 @@
+cyberfusion/Common/Config.py,sha256=RwIg8hAibvuep3ZI00DWqh6xD45TaWgS8YYMXNea54E,1049
+cyberfusion/Common/Filesystem.py,sha256=jjBnbOB75QDnEzAaLNEAR0ZIVw7osqe_7SGbOjnAeJQ,2215
+cyberfusion/Common/FilesystemComparison.py,sha256=Tb7d31hLS3WxCTTCGs75FFjVZXvvYAbEij1bnzG-ju0,5646
+cyberfusion/Common/__init__.py,sha256=HxM0gA8Y8BDwVTzqM16W3CnihBzBwgWbszVjjAYaUIg,3730
+cyberfusion/Common/exceptions/__init__.py,sha256=4V0jEBEQwm4uJPDNvgZ0-O5-kNi_hi4TF42icJT8sFo,171
+python3_cyberfusion_common-2.10.11.3.3.dist-info/METADATA,sha256=YaUGY3aQiDUVto7dfM8dlW3O0i85ettnB6bRfuCE2fA,895
+python3_cyberfusion_common-2.10.11.3.3.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
+python3_cyberfusion_common-2.10.11.3.3.dist-info/top_level.txt,sha256=ss011q9S6SL_KIIyq7iujFmIYa0grSjlnInO7cDkeag,12
+python3_cyberfusion_common-2.10.11.3.3.dist-info/RECORD,,

python3_cyberfusion_common-2.10.11.3.3.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (73.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

python3_cyberfusion_common-2.10.11.3.3.dist-info/top_level.txt

@@ -0,0 +1 @@
+cyberfusion