ceph-devstack 0.1.0__py3-none-any.whl

ceph_devstack/requirements.py

@@ -0,0 +1,277 @@
+import shlex
+import sys
+
+from pathlib import Path
+from packaging.version import parse as parse_version, Version
+from typing import List
+
+from ceph_devstack import config, logger
+from ceph_devstack.host import Host, host, local_host
+
+
+class Requirement:
+    host: Host = host
+    check_cmd: List[str]
+
+    async def evaluate(self) -> bool:
+        return await self.check()
+
+    async def check(self) -> bool:
+        proc = await self.host.arun(self.check_cmd)
+        return await proc.wait() == 0
+
+
+class FixableRequirement(Requirement):
+    fix_cmd: List[str]
+    suggest_msg: str
+
+    async def evaluate(self) -> bool:
+        if await self.check() is True:
+            return True
+        if config["args"].get("fix", False):
+            return await self.fix()
+        else:
+            await self.suggest()
+            return False
+
+    async def suggest(self):
+        if hasattr(self, "suggest_msg"):
+            logger.error(f"{self.suggest_msg}. Try: {shlex.join(self.fix_cmd)}")
+
+    async def fix(self) -> bool:
+        assert self.fix_cmd, "Attempted to fix without a fix command"
+        proc = await self.host.arun(self.fix_cmd)
+        return await proc.wait() == 0
+
+
+class LocalRequirement(Requirement):
+    host = local_host
+
+
+class PodmanPlatform(Requirement):
+    async def check(self):
+        result = False
+        try:
+            podman_info = await self.host.podman_info()
+        except FileNotFoundError:
+            logger.error("podman not found. Try: dnf install podman")
+            return False
+        try:
+            host_os = (
+                podman_info["host"].get("Os") or podman_info["host"]["os"]
+            ).lower()
+            if host_os == "linux":
+                result = True
+        except KeyError:
+            host_os = sys.platform.lower()
+            result = False
+            if sys.platform == "darwin":
+                logger.error(
+                    "The podman machine (VM) is not running. "
+                    "Try: podman machine init --now"
+                )
+            else:
+                logger.error(
+                    "Unknown error trying to query podman. Is podman installed?"
+                )
+            return result
+        if host_os != "linux":
+            logger.error("The platform '{host_os}' is not currently supported.")
+        return result
+
+
+class PodmanGraphDriver(Requirement):
+    async def check(self):
+        podman_info = await self.host.podman_info()
+        storage_conf_path = podman_info["store"]["configFile"]
+        graph_driver = podman_info["store"]["graphDriverName"]
+        if graph_driver == "overlay":
+            return True
+        else:
+            self.suggest_msg = (
+                f"The configured graph driver is '{graph_driver}'. "
+                f"It must be set to 'overlay' in {storage_conf_path}."
+            )
+            return False
+
+
+class KernelVersionForOverlay(Requirement):
+    async def check(self):
+        kernel_version = self.host.kernel_version()
+        version_for_overlay = Version("5.12")
+        if kernel_version < version_for_overlay:
+            self.suggest_msg = (
+                f"Kernel version ({kernel_version}) is too old to support native rootless "
+                f"overlayfs (needs {version_for_overlay})"
+            )
+            return False
+        return True
+
+
+class KernelVersionForCgroupV2(Requirement):
+    async def check(self):
+        version_for_cgroup = Version("4.15")
+        kernel_version = self.host.kernel_version()
+        if not kernel_version >= version_for_cgroup:
+            self.suggest_msg = (
+                f"Kernel version ({kernel_version}) is too old to support cgroup v2 "
+                f"(needs {version_for_cgroup})"
+            )
+        return False
+
+
+class CgroupV2(FixableRequirement):
+    suggest_msg = "cgroup v2 is not enabled"
+    fix_cmd = [
+        "sudo",
+        "grubby",
+        "--update-kernel=ALL",
+        "--args='systemd.unified_cgroup_hierarchy=1'",
+    ]
+
+    async def check(self):
+        podman_info = await self.host.podman_info()
+        return podman_info["host"]["cgroupVersion"] == "v2"
+
+
+class PodmanVersion(Requirement):
+    def __init__(self, version: str, msg: str = ""):
+        self.required_version = parse_version(version)
+        self.msg = msg
+
+    async def check(self):
+        podman_info = await self.host.podman_info()
+        podman_version = parse_version(podman_info["version"]["Version"])
+        if podman_version < self.required_version:
+            if self.msg:
+                logger.warning(self.msg)
+            return False
+        return True
+
+
+class PodmanRuntime(Requirement):
+    async def check(self):
+        podman_info = await self.host.podman_info()
+        storage_conf_path = podman_info["store"]["configFile"]
+        runtime = podman_info["host"]["ociRuntime"]["name"]
+        if runtime == "crun":
+            return True
+        else:
+            containers_conf_path = Path(storage_conf_path).parent / "containers.conf"
+            cmd = host.cmd(["podman", "system", "reset"])
+            logger.error(
+                f"The configured runtime is '{runtime}'. "
+                f"It must be set to 'crun' in {containers_conf_path}. "
+                f"Afterward, run '{cmd}'."
+            )
+            return False
+
+
+class SELinuxBoolean(FixableRequirement):
+    def __init__(self, boolean_name: str):
+        super().__init__()
+        self.boolean_name = boolean_name
+        self.fix_cmd = ["sudo", "setsebool", "-P", f"{self.boolean_name}=true"]
+        self.suggest_msg = f"SELinux boolean '{self.boolean_name}' must be enabled"
+
+    async def check(self):
+        return await self.host.check_selinux_bool(self.boolean_name)
+
+
+class SysctlValue(FixableRequirement):
+    def __init__(self, name: str, min_value: int):
+        super().__init__()
+        self.key = name
+        self.min_value = min_value
+        self.fix_cmd = ["sudo", "sysctl", f"{name}={min_value}"]
+
+    async def check(self):
+        current_value = await self.host.get_sysctl_value(self.key)
+        self.suggest_msg = f"sysctl setting {self.key} ({current_value}) is too low"
+        return current_value >= self.min_value
+
+
+class PodmanDNSPlugin(FixableRequirement):
+    suggest_msg = "Could not find the podman DNS plugin"
+
+    def __init__(self):
+        os_type = self.host.os_type()
+        if os_type == "centos":
+            dns_plugin_path = "/usr/libexec/cni/dnsname"
+            self.check_cmd = ["test", "-x", dns_plugin_path]
+            self.fix_cmd = ["sudo", "dnf", "install", "-y", dns_plugin_path]
+        elif os_type in ["ubuntu", "debian"]:
+            dns_plugin_path = "/usr/lib/cni/dnsname"
+            self.check_cmd = ["test", "-x", dns_plugin_path]
+            self.fix_cmd = [
+                "sudo",
+                "apt",
+                "install",
+                "-y",
+                "golang-github-containernetworking-plugin-dnsname",
+            ]
+
+
+class FuseOverlayfsPresence(FixableRequirement):
+    check_cmd = ["command", "-v", "fuse-overlayfs"]
+    suggest_msg = "Could not find fuse-overlayfs"
+    fix_cmd = ["sudo", "dnf", "install", "-y", "fuse-overlayfs"]
+
+
+class AppArmorProfile(FixableRequirement):
+    _profile_path = "/etc/apparmor.d/local/unix-chkpwd"
+    _profile_content = '"capability dac_override,"'
+    check_cmd = ["test", "-f", _profile_path]
+    suggest_msg = "Did not find required apparmor profile"
+    fix_cmd = [
+        "sudo",
+        "bash",
+        "-c",
+        f"echo -e {_profile_content} > {_profile_path} && systemctl reload apparmor",
+    ]
+
+
+async def check_requirements():
+    if not await PodmanPlatform().evaluate():
+        return False
+
+    result = True
+    # kernel and podman versions for native overlay filesystem
+    result = result and await PodmanGraphDriver().evaluate()
+    podman_overlay_version = "3.10"
+    podman_version_overlay = await PodmanVersion(
+        podman_overlay_version,
+        "Podman version is too old for rootless native overlayfs (needs {podman_overlay_version})",
+    ).evaluate()
+    needs_fuse = not (
+        await KernelVersionForOverlay().evaluate() and podman_version_overlay
+    )
+    # if not using native overlay, we need fuse-overlayfs
+    if needs_fuse:
+        result = result and await FuseOverlayfsPresence().evaluate()
+
+    # cgroup v2
+    if not await CgroupV2().evaluate():
+        result = result and await KernelVersionForCgroupV2().evaluate()
+
+    # runtime
+    result = result and await PodmanRuntime().evaluate()
+
+    # SELinux
+    if await host.selinux_enforcing():
+        result = result and await SELinuxBoolean("container_manage_cgroup").evaluate()
+        result = result and await SELinuxBoolean("container_use_devices").evaluate()
+
+    # AppArmor
+    if await host.apparmor_enabled():
+        result = result and await AppArmorProfile().evaluate()
+
+    # podman DNS plugin
+    if not await PodmanVersion("5.0").evaluate():
+        result = result and await PodmanDNSPlugin().evaluate()
+
+    # sysctl settings for OSD
+    result = result and await SysctlValue("fs.aio-max-nr", 2097152).evaluate()
+    result = result and await SysctlValue("kernel.pid_max", 4194304).evaluate()
+
+    return result

ceph_devstack/resources/__init__.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+import argparse
+import asyncio
+import json
+import os
+import subprocess
+
+from pathlib import Path
+from subprocess import CalledProcessError
+from typing import List, Dict, Set
+
+from ceph_devstack.host import host, local_host
+
+
+class DevStack:
+    def __init__(self, args: argparse.Namespace):
+        self.args = args
+        self.env: Dict[str, str] = {}
+
+    def choose_teuthology_branch(self):
+        branch = os.environ.get("TEUTHOLOGY_BRANCH", self.get_current_branch())
+        self.env["TEUTH_BRANCH"] = branch
+        return branch
+
+    def get_current_branch(self, repo_path: str = "."):
+        return subprocess.check_output(
+            ["git", "branch", "--show-current"],
+            cwd=repo_path,
+        ).decode()
+
+
+class PodmanResource:
+    cwd = "."
+    exists_cmd: List[str] = []
+    create_cmd: List[str] = []
+    remove_cmd: List[str] = []
+    start_cmd: List[str] = []
+    stop_cmd: List[str] = []
+    cmd_vars: List[str] = ["name"]
+    log: Dict[str, Set[str]] = {}
+    _name: str | None = None
+
+    def __init__(self, name: str = ""):
+        if name:
+            self._name = name
+
+    @property
+    def name(self) -> str:
+        if self._name is not None:
+            return self._name
+        return self.__class__.__name__.lower()
+
+    async def cmd(
+        self,
+        args: List[str],
+        check: bool = False,
+        force_local: bool = False,
+        stream_output: bool = False,
+    ) -> asyncio.subprocess.Process:
+        exec_host = local_host if force_local else host
+        proc = await exec_host.arun(
+            args,
+            cwd=Path(self.cwd),
+            stream_output=stream_output,
+        )
+        assert proc.stderr is not None
+        assert proc.stdout is not None
+        returncode = await proc.wait()
+        if check and returncode != 0:
+            # out = await proc.stderr.read()
+            # logger.error(out.decode())
+            raise CalledProcessError(cmd=args, returncode=returncode)
+        return proc
+
+    def format_cmd(self, args: List):
+        vars = {}
+        for k in self.cmd_vars:
+            v = getattr(self, k, None)
+            if v is not None:
+                if isinstance(v, Path):
+                    v = v.expanduser()
+                vars[k] = v
+        return [s.format(**vars) for s in args]
+
+    async def apply(self, action: str):
+        method = getattr(self, action, None)
+        if method is None:
+            return
+        await method()
+
+    async def inspect(self):
+        proc = await self.cmd(self.format_cmd(self.exists_cmd))
+        out, err = await proc.communicate()
+        return json.loads(out)
+        return json.loads(proc.stdout.read())
+        if proc.stdout is None:
+            return {}
+        return json.loads((await proc.stdout.read()).decode())
+
+    async def exists(self):
+        if not self.exists_cmd:
+            return False
+        proc = await self.cmd(self.format_cmd(self.exists_cmd), check=False)
+        return await proc.wait() == 0
+
+    async def create(self):
+        if not await self.exists():
+            await self.cmd(self.format_cmd(self.create_cmd), check=True)
+
+    async def remove(self):
+        await self.cmd(self.format_cmd(self.remove_cmd))
+
+    def __repr__(self):
+        param_str = "" if self._name is None else f'name="{self._name}"'
+        return f"{self.__class__.__name__}({param_str})"

ceph_devstack/resources/ceph/__init__.py

@@ -0,0 +1,266 @@
+import asyncio
+import contextlib
+import os
+import pathlib
+import tempfile
+
+from subprocess import CalledProcessError
+
+from ceph_devstack import config, logger
+from ceph_devstack.host import host
+from ceph_devstack.resources.misc import Secret, Network
+from ceph_devstack.resources.ceph.containers import (
+    Postgres,
+    Beanstalk,
+    Paddles,
+    Pulpito,
+    TestNode,
+    Teuthology,
+    Archive,
+)
+from ceph_devstack.resources.ceph.requirements import (
+    HasSudo,
+    LoopControlDeviceExists,
+    LoopControlDeviceWriteable,
+    SELinuxModule,
+)
+from ceph_devstack.resources.ceph.utils import get_most_recent_run, get_job_id
+from ceph_devstack.resources.ceph.exceptions import TooManyJobsFound
+
+
+class SSHKeyPair(Secret):
+    _name = "id_rsa"
+    cmd_vars = ["name", "privkey_path", "pubkey_path"]
+    privkey_path = "id_rsa"
+    pubkey_path = "id_rsa.pub"
+    exists_cmds = [
+        ["podman", "secret", "inspect", "{name}"],
+        ["podman", "secret", "inspect", "{name}.pub"],
+    ]
+    create_cmds = [
+        ["podman", "secret", "create", "{name}", "{privkey_path}"],
+        ["podman", "secret", "create", "{name}.pub", "{pubkey_path}"],
+    ]
+    remove_cmds = [
+        ["podman", "secret", "rm", "{name}"],
+        ["podman", "secret", "rm", "{name}.pub"],
+    ]
+
+    async def exists(self):
+        for exists_cmd in self.exists_cmds:
+            proc = await self.cmd(self.format_cmd(exists_cmd), check=False)
+            if await proc.wait():
+                return False
+        return True
+
+    async def create(self):
+        if await self.exists():
+            return
+        await self._get_ssh_keys()
+        for create_cmd in self.create_cmds:
+            await self.cmd(self.format_cmd(create_cmd), check=True)
+
+    async def remove(self):
+        for remove_cmd in self.remove_cmds:
+            await self.cmd(self.format_cmd(remove_cmd))
+
+    async def _get_ssh_keys(self):
+        privkey_path = os.environ.get("SSH_PRIVKEY_PATH")
+        self.pubkey_path = "/dev/null"
+        if not privkey_path:
+            temp_dir = tempfile.mkdtemp(
+                prefix="teuthology-ssh-key-",
+                dir="/tmp",
+            )
+            privkey_path = pathlib.Path(temp_dir) / self.__class__.privkey_path
+            await self.cmd(
+                ["ssh-keygen", "-t", "rsa", "-N", "", "-f", str(privkey_path)],
+                check=True,
+                force_local=True,
+            )
+        self.pubkey_path = f"{privkey_path}.pub"
+        self.privkey_path = privkey_path
+
+
+class CephDevStackNetwork(Network):
+    _name = "ceph-devstack"
+
+
+class CephDevStack:
+    networks = [CephDevStackNetwork]
+    secrets = [SSHKeyPair]
+
+    def __init__(self):
+        services = [
+            Postgres,
+            Paddles,
+            Beanstalk,
+            Pulpito,
+            Teuthology,
+            TestNode,
+            Archive,
+        ]
+        self.service_specs = {}
+        for service in services:
+            name = service.__name__.lower()
+            count = config["containers"][name].get("count", 1)
+            if count == 0:
+                continue
+            self.service_specs[name] = {
+                "obj": service,
+                "count": count,
+            }
+            if count == 1:
+                self.service_specs[name]["objects"] = [service()]
+            elif count > 1:
+                self.service_specs[name]["objects"] = [
+                    service(name=f"{name}_{i}") for i in range(count)
+                ]
+        if postgres_spec := self.service_specs.get("postgres"):
+            postgres_obj = postgres_spec["objects"][0]
+            paddles_obj = self.service_specs["paddles"]["objects"][0]
+            paddles_obj.env_vars["PADDLES_SQLALCHEMY_URL"] = (
+                postgres_obj.paddles_sqla_url
+            )
+
+    async def check_requirements(self):
+        result = True
+
+        result = has_sudo = await HasSudo().evaluate()
+        result = result and await LoopControlDeviceExists().evaluate()
+        result = result and await LoopControlDeviceWriteable().evaluate()
+
+        # Check for SELinux being enabled and Enforcing; then check for the
+        # presence of our module. If necessary, inform the user and instruct
+        # them how to build and install.
+        if has_sudo and await host.selinux_enforcing():
+            result = result and await SELinuxModule().evaluate()
+
+        for name, obj in config["containers"].items():
+            if (repo := obj.get("repo")) and not host.path_exists(repo):
+                result = False
+                logger.error(f"Repo for {name} not found at {repo}")
+        return result
+
+    async def apply(self, action):
+        return await getattr(self, action)()
+
+    async def pull(self):
+        logger.info("Pulling images...")
+        for spec in self.service_specs.values():
+            await spec["objects"][0].pull()
+
+    async def build(self):
+        logger.info("Building images...")
+        for spec in self.service_specs.values():
+            await spec["objects"][0].build()
+
+    async def create(self):
+        logger.info("Creating containers...")
+        await CephDevStackNetwork().create()
+        await SSHKeyPair().create()
+        containers = []
+        for spec in self.service_specs.values():
+            for object in spec["objects"]:
+                containers.append(object.create())
+        await asyncio.gather(*containers)
+
+    async def start(self):
+        await self.create()
+        logger.info("Starting containers...")
+        for spec in self.service_specs.values():
+            for object in spec["objects"]:
+                await object.start()
+        logger.info(
+            "All containers are running. To monitor teuthology, try running: podman "
+            "logs -f teuthology"
+        )
+        hostname = host.hostname()
+        logger.info(f"View test results at http://{hostname}:8081/")
+
+    async def stop(self):
+        logger.info("Stopping containers...")
+        containers = []
+        for spec in self.service_specs.values():
+            for object in spec["objects"]:
+                containers.append(object.stop())
+        await asyncio.gather(*containers)
+
+    async def remove(self):
+        logger.info("Removing containers...")
+        containers = []
+        for spec in self.service_specs.values():
+            for object in spec["objects"]:
+                containers.append(object.remove())
+        await asyncio.gather(*containers)
+        await CephDevStackNetwork().remove()
+        await SSHKeyPair().remove()
+
+    async def watch(self):
+        logger.info("Watching containers; will replace any that are stopped")
+        containers = []
+        for spec in self.service_specs.values():
+            if not spec["count"] > 0:
+                continue
+            for object in spec["objects"]:
+                containers.append(object)
+        logger.info(f"Watching {containers}")
+        while True:
+            try:
+                for container in containers:
+                    with contextlib.suppress(CalledProcessError):
+                        if not await container.exists():
+                            logger.info(
+                                f"Container {container.name} was removed; replacing"
+                            )
+                            await container.create()
+                            await container.start()
+                        elif not await container.is_running():
+                            logger.info(
+                                f"Container {container.name} stopped; restarting"
+                            )
+                            await container.start()
+            except KeyboardInterrupt:
+                break
+
+    async def wait(self, container_name: str):
+        for spec in self.service_specs.values():
+            for object in spec["objects"]:
+                if object.name == container_name:
+                    return await object.wait()
+        logger.error(f"Could not find container {container_name}")
+        return 1
+
+    async def logs(self, run_name: str = "", job_id: str = "", locate: bool = False):
+        try:
+            log_file = self.get_log_file(run_name, job_id)
+        except FileNotFoundError:
+            logger.error("No log file found")
+        except TooManyJobsFound as e:
+            msg = "Found too many jobs ({jobs}) for target run. Please pick a job id with -j option.".format(
+                jobs=", ".join(e.jobs)
+            )
+            logger.error(msg)
+        else:
+            if locate:
+                print(log_file)
+            else:
+                buffer_size = 8 * 1024
+                with open(log_file) as f:
+                    while chunk := f.read(buffer_size):
+                        print(chunk, end="")
+
+    def get_log_file(self, run_name: str = "", job_id: str = ""):
+        archive_dir = Teuthology().archive_dir.expanduser()
+
+        if not run_name:
+            run_name = get_most_recent_run(os.listdir(archive_dir))
+        run_dir = archive_dir.joinpath(run_name)
+
+        if not job_id:
+            job_id = get_job_id(os.listdir(run_dir))
+
+        log_file = run_dir.joinpath(job_id, "teuthology.log")
+        if not log_file.exists():
+            raise FileNotFoundError
+        return log_file