python3-cyberfusion-common 2.10.11.1__tar.gz

python3-cyberfusion-common-2.10.11.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Cyberfusion
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

python3-cyberfusion-common-2.10.11.1/PKG-INFO

@@ -0,0 +1,59 @@
+Metadata-Version: 2.1
+Name: python3-cyberfusion-common
+Version: 2.10.11.1
+Summary: Common utilities.
+Home-page: https://vcs.cyberfusion.nl/shared/python3-cyberfusion-common
+Author: William Edwards
+Author-email: wedwards@cyberfusion.nl
+License: MIT
+Keywords: cyberfusion,common
+Platform: linux
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# python3-cyberfusion-common
+
+Common utilities.
+
+# Install
+
+## PyPI
+
+Run the following command to install the package from PyPI:
+
+    pip3 install python3-cyberfusion-common
+
+## Generic
+
+Run the following command to create a source distribution:
+
+    python3 setup.py sdist
+
+## Debian
+
+Run the following commands to build a Debian package:
+
+    mk-build-deps -i -t 'apt -o Debug::pkgProblemResolver=yes --no-install-recommends -y'
+    dpkg-buildpackage -us -uc
+
+# Configure
+
+No configuration is supported.
+
+# Usage
+
+See code.
+
+# Tests
+
+Run tests with pytest:
+
+    pytest tests/
+
+The tests must be run from the project root.
+
+

python3-cyberfusion-common-2.10.11.1/README.md

@@ -0,0 +1,40 @@
+# python3-cyberfusion-common
+
+Common utilities.
+
+# Install
+
+## PyPI
+
+Run the following command to install the package from PyPI:
+
+    pip3 install python3-cyberfusion-common
+
+## Generic
+
+Run the following command to create a source distribution:
+
+    python3 setup.py sdist
+
+## Debian
+
+Run the following commands to build a Debian package:
+
+    mk-build-deps -i -t 'apt -o Debug::pkgProblemResolver=yes --no-install-recommends -y'
+    dpkg-buildpackage -us -uc
+
+# Configure
+
+No configuration is supported.
+
+# Usage
+
+See code.
+
+# Tests
+
+Run tests with pytest:
+
+    pytest tests/
+
+The tests must be run from the project root.

python3-cyberfusion-common-2.10.11.1/pyproject.toml

@@ -0,0 +1,22 @@
+[tool.isort]
+profile = "black"
+line_length = 79
+known_first_party = ["cyberfusion"]
+default_section = "THIRDPARTY"
+
+[tool.black]
+line-length = 79
+target-version = ["py37"]
+exclude = '''
+(
+  /(
+      \.eggs         # exclude a few common directories in the
+    | \.git          # root of the project
+    | \.hg
+    | \.mypy_cache
+    | \.tox
+    | \.venv
+    | venv
+  )/
+)
+'''

python3-cyberfusion-common-2.10.11.1/setup.cfg

@@ -0,0 +1,12 @@
+[aliases]
+test = pytest
+
+[tool:pytest]
+norecursedirs = .git build dist *.egg __pycache__ .cache
+testpaths = tests
+junit_suite_name = python3-cyberfusion-common
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

python3-cyberfusion-common-2.10.11.1/setup.py

@@ -0,0 +1,34 @@
+"""A setuptools based setup module."""
+
+from setuptools import setup
+
+with open("README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
+
+setup(
+    name="python3-cyberfusion-common",
+    version="2.10.11.1",
+    description="Common utilities.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    python_requires=">=3.9",
+    author="William Edwards",
+    author_email="wedwards@cyberfusion.nl",
+    url="https://vcs.cyberfusion.nl/shared/python3-cyberfusion-common",
+    platforms=["linux"],
+    packages=["cyberfusion.Common", "cyberfusion.Common.exceptions"],
+    package_dir={"": "src"},
+    data_files=[],
+    install_requires=[
+        "cached_property==1.5.2",
+        "psutil==5.8.0",
+        "requests==2.25.1",
+    ],
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    keywords=["cyberfusion", "common"],
+    license="MIT",
+)

python3-cyberfusion-common-2.10.11.1/src/cyberfusion/Common/Config.py

@@ -0,0 +1,41 @@
+"""Helper for handling config files."""
+
+import configparser
+import os
+from typing import Optional
+
+from cached_property import cached_property
+
+
+class CyberfusionConfig:
+    """Abstract ConfigParser implementation for use in scripts."""
+
+    def __init__(
+        self,
+        path: Optional[str] = None,
+    ) -> None:
+        """Set attributes."""
+        self._path = path
+
+    @property
+    def path(self) -> str:
+        """Set config file path."""
+        if not self._path:
+            self._path = os.path.join(
+                os.path.sep, "etc", "cyberfusion", "cyberfusion.cfg"
+            )
+
+        return self._path
+
+    @cached_property
+    def config(self) -> configparser.ConfigParser:
+        """Read config."""
+        config = configparser.ConfigParser()
+
+        config.read(self.path)
+
+        return config
+
+    def get(self, section: str, key: str) -> str:
+        """Retrieve config option."""
+        return self.config.get(section, key)

python3-cyberfusion-common-2.10.11.1/src/cyberfusion/Common/Filesystem.py

@@ -0,0 +1,81 @@
+"""Helper for handling filesystem."""
+
+import os
+from enum import Enum
+from pathlib import Path
+
+import psutil
+
+CEPH_NAME_ATTRIBUTE_RBYTES = "ceph.dir.rbytes"
+
+
+class FilesystemType(Enum):
+    """Filesystem types.
+
+    We don't expect any other filesystems than the ones in this Enum under normal
+    circumstances. This Enum prevents you from having to write filesystem types
+    by name manually; it's not a complete list of all possibilities.
+    """
+
+    EXT4 = "ext4"
+    EXT2 = "ext2"  # Usually for /boot
+    XFS = "xfs"
+    CEPH = "ceph"
+    OVERLAY = "overlay"  # Usually for Docker
+    APFS = "apfs"  # Development on macOS
+
+
+def get_filesystem(path: str) -> str:
+    """Get filesystem that absolute path is on.
+
+    Returns the earliest filesystem. E.g. if '/a/b/c' is passed, and both '/a' and
+    '/b' are mountpoints, '/b' is returned.
+
+    Does not resolve symlinks, so the filesystem of the symlink itself is looked
+    up.
+
+    Inspired by https://stackoverflow.com/a/4453715/3837431
+    """
+
+    # Ensure path is an absolute path. For relative paths, dirname() would return
+    # the wrong result
+
+    path = os.path.abspath(path)
+
+    # Work our way down the path by getting the directory that the current
+    # filesystem object is in
+
+    while not os.path.ismount(path):
+        path = os.path.dirname(path)
+
+    return path
+
+
+def get_filesystem_type(path: str) -> FilesystemType:
+    """Get type of filesystem."""
+
+    # If this yields no results (i.e. the path is missing), next() raises StopIteration
+
+    partition = next(
+        filter(
+            lambda x: x.mountpoint == path, psutil.disk_partitions(all=True)
+        )
+    )
+
+    return FilesystemType(partition.fstype)
+
+
+def get_directory_size(path: str) -> int:
+    """Get size of directory."""
+
+    # CephFS is a special case, as CephFS has extended attributes to determine
+    # size
+
+    is_ceph = get_filesystem_type(get_filesystem(path)) == FilesystemType.CEPH
+
+    if is_ceph:
+        return int(os.getxattr(path, CEPH_NAME_ATTRIBUTE_RBYTES).decode("utf-8"))  # type: ignore[attr-defined]
+
+    # Loop through directory for filesystems without special implementation
+
+    return sum(f.stat().st_size for f in Path(path).rglob("*") if f.is_file())

python3-cyberfusion-common-2.10.11.1/src/cyberfusion/Common/FilesystemComparison.py

@@ -0,0 +1,185 @@
+"""Helper for comparing filesystem objects."""
+
+import filecmp
+import os
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+
+def _get_recursive_dircmps(dircmp: filecmp.dircmp) -> List[filecmp.dircmp]:
+    """Get dircmp object for every subdirectory.
+
+    Subdirectories are only returned for directories that are in the left and
+    right directories.
+    """
+    dircmps = []
+
+    dircmps.append(dircmp)
+
+    for sub_dircmp in dircmp.subdirs.values():
+        dircmps.extend(_get_recursive_dircmps(sub_dircmp))
+
+    return dircmps
+
+
+def get_different_files_in_directories(
+    left_directory_path: str, right_directory_path: str
+) -> List[Tuple[str, str]]:
+    """Get files with same name in both directories but with different contents or attributes.
+
+    Files which are present in subdirectories that only exist in either the left
+    or right directories are not returned. Use the 'get_directories_only_in_*_directory'
+    functions to get subdirectories that are in either the left or right directories.
+    This ensures that this function does not return individual files in whole new
+    directories.
+    """
+    dircmps = _get_recursive_dircmps(
+        filecmp.dircmp(left_directory_path, right_directory_path)
+    )
+
+    files = []
+
+    for dircmp in dircmps:
+        for diff_file in dircmp.diff_files:
+            absolute_path_left = os.path.join(dircmp.left, diff_file)
+            absolute_path_right = os.path.join(dircmp.right, diff_file)
+
+            files.append((absolute_path_left, absolute_path_right))
+
+    return files
+
+
+def get_files_only_in_left_directory(
+    left_directory_path: str, right_directory_path: str
+) -> List[str]:
+    """Get files which are only present in left directory.
+
+    Files which are present in subdirectories that only exist in either the left
+    or right directories are not returned. Use the 'get_directories_only_in_*_directory'
+    functions to get subdirectories that are in either the left or right directories.
+    This ensures that this function does not return individual files in whole new
+    directories.
+    """
+    dircmps = _get_recursive_dircmps(
+        filecmp.dircmp(left_directory_path, right_directory_path)
+    )
+
+    files = []
+
+    for dircmp in dircmps:
+        for left_only_file in dircmp.left_only:
+            absolute_path = os.path.join(dircmp.left, left_only_file)
+
+            if os.path.isdir(absolute_path):
+                continue
+
+            files.append(absolute_path)
+
+    return files
+
+
+def get_files_only_in_right_directory(
+    left_directory_path: str, right_directory_path: str
+) -> List[str]:
+    """Get files which are only present in right directory.
+
+    Files which are present in subdirectories that only exist in either the left
+    or right directories are not returned. Use the 'get_directories_only_in_*_directory'
+    functions to get subdirectories that are in either the left or right directories.
+    This ensures that this function does not return individual files in whole new
+    directories.
+    """
+    dircmps = _get_recursive_dircmps(
+        filecmp.dircmp(left_directory_path, right_directory_path)
+    )
+
+    files = []
+
+    for dircmp in dircmps:
+        for right_only_file in dircmp.right_only:
+            absolute_path = os.path.join(dircmp.right, right_only_file)
+
+            if os.path.isdir(absolute_path):
+                continue
+
+            files.append(absolute_path)
+
+    return files
+
+
+def get_directories_only_in_left_directory(
+    left_directory_path: str, right_directory_path: str
+) -> List[str]:
+    """Get directories which are only present in left directory."""
+    dircmps = _get_recursive_dircmps(
+        filecmp.dircmp(left_directory_path, right_directory_path)
+    )
+
+    directories = []
+
+    for dircmp in dircmps:
+        for left_only_file in dircmp.left_only:
+            absolute_path = os.path.join(dircmp.left, left_only_file)
+
+            if os.path.isfile(absolute_path):
+                continue
+
+            directories.append(absolute_path)
+
+    return directories
+
+
+def get_directories_only_in_right_directory(
+    left_directory_path: str, right_directory_path: str
+) -> List[str]:
+    """Get directories which are only present in right directory."""
+    dircmps = _get_recursive_dircmps(
+        filecmp.dircmp(left_directory_path, right_directory_path)
+    )
+
+    directories = []
+
+    for dircmp in dircmps:
+        for right_only_file in dircmp.right_only:
+            absolute_path = os.path.join(dircmp.right, right_only_file)
+
+            if os.path.isfile(absolute_path):
+                continue
+
+            directories.append(absolute_path)
+
+    return directories
+
+
+def get_nested_directory_structure(
+    paths_list: List[str],
+) -> Dict[str, Optional[Dict[str, Optional[dict]]]]:
+    """Convert list of paths to nested dict."""
+    tree: Dict[str, Optional[Dict[str, Optional[dict]]]] = {}
+
+    # Example of logic:
+    #
+    # >>> tree = {}
+    # >>> node = tree
+    # >>> node = node.setdefault('a', {})
+    # >>> node = node.setdefault('b', {})
+    # >>> node = tree
+    # >>> tree
+    # {'a': {'b': {}}}
+
+    for path in paths_list:
+        node = tree  # Reset to root level for new path, see docstring
+
+        parts = Path(path).parts
+
+        for i, element in enumerate(parts):
+            last = i == (len(parts) - 1)
+
+            if last:
+                node.setdefault(element, None)
+            else:
+                node = node.setdefault(  # type: ignore[assignment]  # mypy does not support nested types
+                    element, {}
+                )  # Next recursion will be at this level, see docstring
+
+    return tree

python3-cyberfusion-common-2.10.11.1/src/cyberfusion/Common/__init__.py

@@ -0,0 +1,157 @@
+"""Helper classes for Cyberfusion scripts."""
+
+import base64
+import os
+import secrets
+import shutil
+import string
+import uuid
+from datetime import datetime, timezone
+from hashlib import md5, sha1
+from socket import gethostname
+from typing import Optional
+
+import requests
+
+from cyberfusion.Common.exceptions import ExecutableNotFound
+
+CHAR_PREFIX_WILDCARD = "*"
+CHAR_LABEL = "."
+
+
+class EmailAddresses:
+    """Cyberfusion email addresses."""
+
+    ENGINEERING = "engineering@cyberfusion.nl"
+    SUPPORT = "support@cyberfusion.nl"
+
+
+def find_executable(name: str) -> str:
+    """Find absolute path of executable.
+
+    Use this function when the executable must exist. This function raises an
+    exception if it does not.
+    """
+    path = shutil.which(name)
+
+    if path:
+        return path
+
+    raise ExecutableNotFound(name)
+
+
+def try_find_executable(name: str) -> Optional[str]:
+    """Find absolute path of executable.
+
+    Use this function when the executable may be absent. This function returns
+    None if it is.
+    """
+    return shutil.which(name)
+
+
+def download_from_url(
+    url: str, *, root_directory: Optional[str] = None
+) -> str:
+    """Download from URL.
+
+    Large files are supported due to use of chunking and streaming.
+
+    Inspired by https://stackoverflow.com/a/16696317/19535769
+    """
+    if root_directory is None:
+        root_directory = os.path.join(os.path.sep, "tmp")
+
+    path = os.path.join(root_directory, generate_random_string())
+
+    # Create and set permissions
+
+    with open(path, "w"):
+        pass
+
+    os.chmod(path, 0o600)
+
+    # Download
+
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+
+        with open(path, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+    return path
+
+
+def get_tmp_file() -> str:
+    """Create tmp file and return path."""
+    path = os.path.join(os.path.sep, "tmp", str(uuid.uuid4()))
+
+    with open(path, "w"):
+        pass
+
+    os.chmod(path, 0o600)  # Do not allow regular users to view file contents
+
+    return path
+
+
+def get_hostname() -> str:
+    """Get hostname."""
+    return gethostname()
+
+
+def get_domain_is_wildcard(domain: str) -> bool:
+    """Determine if domain is wildcard."""
+    return domain.split(CHAR_LABEL)[0] == CHAR_PREFIX_WILDCARD
+
+
+def get_today_timestamp() -> float:
+    """Get UNIX timestamp from the first second of today."""
+    current_datetime = datetime.utcnow()
+
+    return (
+        datetime(
+            year=current_datetime.year,
+            month=current_datetime.month,
+            day=current_datetime.day,
+        )
+        .replace(tzinfo=timezone.utc)
+        .timestamp()
+    )
+
+
+def generate_random_string(length: int = 24) -> str:
+    """Generate random string."""
+
+    # Set allowed characters (ASCII + digits)
+
+    alphabet = string.ascii_letters + string.digits
+
+    # Return string using allowed characters
+
+    return "".join(secrets.choice(alphabet) for i in range(length))
+
+
+def hash_string_mariadb(string: str) -> str:
+    """Hash string with SHA-1 by MariaDB standards."""
+    return (
+        "*"
+        + sha1(sha1(string.encode("utf-8")).digest())  # noqa: S303
+        .hexdigest()
+        .upper()
+    )
+
+
+def convert_bytes_gib(size: int) -> float:
+    """Convert bytes to GiB."""
+    return size / (1024**3)
+
+
+def get_md5_hash(path: str) -> str:
+    """Get Base64 encoded 128-bit MD5 digest of file."""
+    hash_ = md5()
+
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(128 * hash_.block_size), b""):
+            hash_.update(chunk)
+
+    return base64.b64encode(hash_.digest()).decode()

python3-cyberfusion-common-2.10.11.1/src/cyberfusion/Common/exceptions/__init__.py

@@ -0,0 +1,10 @@
+"""Exceptions."""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class ExecutableNotFound(Exception):
+    """Raise exception when executable not found."""
+
+    name: str

python3-cyberfusion-common-2.10.11.1/src/python3_cyberfusion_common.egg-info/PKG-INFO

@@ -0,0 +1,59 @@
+Metadata-Version: 2.1
+Name: python3-cyberfusion-common
+Version: 2.10.11.1
+Summary: Common utilities.
+Home-page: https://vcs.cyberfusion.nl/shared/python3-cyberfusion-common
+Author: William Edwards
+Author-email: wedwards@cyberfusion.nl
+License: MIT
+Keywords: cyberfusion,common
+Platform: linux
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# python3-cyberfusion-common
+
+Common utilities.
+
+# Install
+
+## PyPI
+
+Run the following command to install the package from PyPI:
+
+    pip3 install python3-cyberfusion-common
+
+## Generic
+
+Run the following command to create a source distribution:
+
+    python3 setup.py sdist
+
+## Debian
+
+Run the following commands to build a Debian package:
+
+    mk-build-deps -i -t 'apt -o Debug::pkgProblemResolver=yes --no-install-recommends -y'
+    dpkg-buildpackage -us -uc
+
+# Configure
+
+No configuration is supported.
+
+# Usage
+
+See code.
+
+# Tests
+
+Run tests with pytest:
+
+    pytest tests/
+
+The tests must be run from the project root.
+
+

python3-cyberfusion-common-2.10.11.1/src/python3_cyberfusion_common.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+LICENSE
+README.md
+pyproject.toml
+setup.cfg
+setup.py
+src/cyberfusion/Common/Config.py
+src/cyberfusion/Common/Filesystem.py
+src/cyberfusion/Common/FilesystemComparison.py
+src/cyberfusion/Common/__init__.py
+src/cyberfusion/Common/exceptions/__init__.py
+src/python3_cyberfusion_common.egg-info/PKG-INFO
+src/python3_cyberfusion_common.egg-info/SOURCES.txt
+src/python3_cyberfusion_common.egg-info/dependency_links.txt
+src/python3_cyberfusion_common.egg-info/requires.txt
+src/python3_cyberfusion_common.egg-info/top_level.txt

python3-cyberfusion-common-2.10.11.1/src/python3_cyberfusion_common.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

python3-cyberfusion-common-2.10.11.1/src/python3_cyberfusion_common.egg-info/requires.txt

@@ -0,0 +1,3 @@
+cached_property==1.5.2
+psutil==5.8.0
+requests==2.25.1

python3-cyberfusion-common-2.10.11.1/src/python3_cyberfusion_common.egg-info/top_level.txt

@@ -0,0 +1 @@
+cyberfusion