fm-weck 1.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

fm_weck/__init__.py

@@ -8,4 +8,4 @@
 from .config import Config  # noqa: F401
 from .image_mgr import ImageMgr  # noqa: F401
 
-__version__ = "1.1"
+__version__ = "1.1.2"

fm_weck/cli.py

@@ -24,6 +24,8 @@ from . import __version__
 from .engine import Engine, NoImageError
 from .serve import run_guided, run_manual, setup_fm_tool
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class ToolQualifier:
@@ -174,20 +176,17 @@ def parse(raw_args: list[str]) -> Tuple[Callable[[], None], Namespace]:
     def help_callback():
         parser.print_help()
 
-    try:
-        parser.parse_args(raw_args)
-    except SystemExit as e:
-        _, left_over = parser.parse_known_args(raw_args)
+    result, left_over = parser.parse_known_args(raw_args)
 
-        if not left_over:
-            # Some unrecoverable error occurred
-            raise e
+    if not left_over:
+        # Parsing went fine
+        return help_callback, result
 
-        # Find the first offending argument and insert "--" before it
-        # We do this to allow the user to pass arguments to the fm-tool without
-        # having to specify the pseudo argument "--"
-        idx = raw_args.index(left_over[0])
-        raw_args.insert(idx, "--")
+    # Find the first offending argument and insert "--" before it
+    # We do this to allow the user to pass arguments to the fm-tool without
+    # having to specify the pseudo argument "--"
+    idx = raw_args.index(left_over[0])
+    raw_args.insert(idx, "--")
 
     return help_callback, parser.parse_args(raw_args)
 
@@ -235,22 +234,23 @@ def set_log_level(loglevel: Optional[str], config: dict[str, Any]):
     level = "WARNING"
     level = loglevel.upper() if loglevel else config.get("logging", {}).get("level", level)
     logging.basicConfig(level=level)
+    logging.getLogger("httpcore").setLevel("WARNING")
 
 
 def main_run(args: argparse.Namespace):
     if not args.TOOL:
-        logging.error("No fm-tool given. Aborting...")
+        logger.error("No fm-tool given. Aborting...")
         return 1
     try:
         fm_data = resolve_tool(args.TOOL)
     except KeyError:
-        logging.error("Unknown tool %s", args.TOOL)
+        logger.error("Unknown tool %s", args.TOOL)
         return 1
 
     try:
         property_path = resolve_property(args.property) if args.property else None
     except KeyError:
-        logging.error("Unknown property %s", args.property)
+        logger.error("Unknown property %s", args.property)
         return 1
 
     run_guided(
@@ -267,12 +267,12 @@ def main_run(args: argparse.Namespace):
 
 def main_manual(args: argparse.Namespace):
     if not args.TOOL:
-        logging.error("No fm-tool given. Aborting...")
+        logger.error("No fm-tool given. Aborting...")
         return 1
     try:
         fm_data = resolve_tool(args.TOOL)
     except KeyError:
-        logging.error("Unknown tool %s", args.TOOL)
+        logger.error("Unknown tool %s", args.TOOL)
         return 1
 
     run_manual(
@@ -289,7 +289,7 @@ def main_install(args: argparse.Namespace):
         try:
             fm_data = resolve_tool(tool)
         except KeyError:
-            logging.error("Unknown tool %s. Skipping installation...", tool)
+            logger.error("Unknown tool %s. Skipping installation...", tool)
             continue
 
         setup_fm_tool(
@@ -306,7 +306,7 @@ def main_shell(args: argparse.Namespace):
         try:
             fm_data = resolve_tool(args.TOOL)
         except KeyError:
-            logging.error("Unknown tool %s", args.fm_data)
+            logger.error("Unknown tool %s", args.fm_data)
             return 1
         engine = Engine.from_config(fm_data, args.TOOL.version, Config())
     engine.interactive = True
@@ -339,7 +339,7 @@ printf '[defaults]\\nimage = "<your_image>"' > .weck
 
 Replace <your_image> with the image you want to use.
         """
-        logging.error(text)
+        logger.error(text)
         return
 
     text = """
@@ -353,7 +353,7 @@ image = "your_image"
 to your .weck file.
     """
 
-    logging.error(text, tool, config)
+    logger.error(text, tool, config)
 
 
 def cli(raw_args: list[str]):

fm_weck/config.py

@@ -4,11 +4,12 @@
 # SPDX-FileCopyrightText: 2024 Dirk Beyer <https://www.sosy-lab.org>
 #
 # SPDX-License-Identifier: Apache-2.0
+
 import os
 import shutil
 from functools import cache
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Callable, Iterable, Optional, Tuple, TypeVar
 
 import yaml
 from fm_tools.fmdata import FmData
@@ -31,8 +32,11 @@ BASE_CONFIG = """
 level = "INFO"
 
 [defaults]
+engine = "podman"
 """
 
+_T = TypeVar("_T")
+
 
 class Config(object):
     """
@@ -41,6 +45,7 @@ class Config(object):
 
     _instance = None
     _config_source = None
+    _source_path = None
 
     def __new__(cls):
         if cls._instance is None:
@@ -58,6 +63,7 @@ class Config(object):
 
             with config.open("rb") as f:
                 self._config = toml.load(f)
+                self._config_source = config.resolve()
                 return self._config
 
         for path in _SEARCH_ORDER:
@@ -76,7 +82,7 @@ class Config(object):
     def __getitem__(self, key: str) -> Any:
         return self._config[key]
 
-    def get(self, key: str, default: Any = None) -> Any:
+    def get(self, key: str, default: _T = None) -> _T:
         if self._config is not None:
             return self._config.get(key, default)
 
@@ -88,7 +94,25 @@ class Config(object):
     def from_defaults_or_none(self, key: str) -> Any:
         return self.defaults().get(key, None)
 
+    @staticmethod
+    def _handle_relative_paths(fn: Callable[[Any], Path]) -> Callable[[Any], Path]:
+        def wrapper(self, *args, **kwargs) -> Path:
+            """Makes sure relative Paths in the config are relative to the config file."""
+
+            path = fn(self, *args, **kwargs)
+
+            if not self._config_source:
+                return path
+
+            if path.is_absolute():
+                return path
+
+            return (self._config_source.parent / path).resolve()
+
+        return wrapper
+
     @property
+    @_handle_relative_paths
     def cache_location(self) -> Path:
         cache = Path.home() / ".cache" / "weck_cache"
         xdg_cache_home = os.environ.get("XDG_CACHE_HOME")
@@ -96,8 +120,16 @@ class Config(object):
         if xdg_cache_home:
             cache = Path(xdg_cache_home) / "weck_cache"
 
-        return Path(self.defaults().get("cache_location", cache))
-    
+        return Path(self.defaults().get("cache_location", cache.resolve()))
+
+    @_handle_relative_paths
+    def as_absolute_path(self, path: Path) -> Path:
+        return path
+
+    def mounts(self) -> Iterable[Tuple[Path, Path]]:
+        for local, container in self.get("mount", {}).items():
+            yield self.as_absolute_path(Path(local)), Path(container)
+
     def get_checksum_db(self) -> Path:
         return self.cache_location / ".checksums.dbm"
 

fm_weck/engine.py

@@ -7,90 +7,217 @@
 
 import logging
 import subprocess
-import uuid
 from abc import ABC, abstractmethod
-from functools import singledispatchmethod
+from functools import cached_property, singledispatchmethod
 from pathlib import Path
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 from fm_tools.fmdata import FmData, FmImageConfig
 
 from fm_weck.config import Config, parse_fm_data
 from fm_weck.image_mgr import ImageMgr
 
+logger = logging.getLogger(__name__)
+
 
 class NoImageError(Exception):
     pass
 
+
 class Engine(ABC):
     interactive: bool = False
     add_benchexec_capabilities: bool = False
     image: Optional[str] = None
 
-    @abstractmethod
-    def assemble_command(self, command: tuple[str, ...]) -> list[str]:
-        raise NotImplementedError
+    def __init__(self, image: Union[str, FmImageConfig]):
+        self.image = self._initialize_image(image)
+        self.extra_args = {}
+        self._engine = None
+
+    def get_workdir(self):
+        return Path("/home/cwd")
+
+    def mount(self, source: str, target: str):
+        self.extra_args["mounts"] = self.extra_args.get("mounts", []) + [
+            "-v",
+            f"{source}:{target}",
+        ]
 
     @abstractmethod
-    def mount(self, src: str, target: str):
+    def benchexec_capabilities(self):
         raise NotImplementedError
 
+    def base_command(self):
+        return [self._engine, "run"]
+
+    def interactive_command(self):
+        return ["-it"]
+
+    def setup_command(self):
+        return [
+            "--entrypoint",
+            '[""]',
+            "--cap-add",
+            "SYS_ADMIN",
+            "-v",
+            f"{Path.cwd().absolute()}:/home/cwd",
+            "-v",
+            f"{Config().cache_location}:/home/weck_cache",
+            "--workdir",
+            str(self.get_workdir()),
+            "--rm",
+        ]
+
+    def assemble_command(self, command: tuple[str, ...]) -> list[str]:
+        base = self.base_command()
+        if self.add_benchexec_capabilities:
+            base += self.benchexec_capabilities()
+
+        base += self.setup_command()
+
+        if self.interactive:
+            base += self.interactive_command()
+
+        for value in self.extra_args.values():
+            if isinstance(value, list) and not isinstance(value, str):
+                base += value
+            else:
+                base.append(value)
+
+        _command = self._prep_command(command)
+        return base + [self.image, *_command]
+
+    def _prep_command(self, command: tuple[str, ...]) -> tuple[str, ...]:
+        """We want to map absolute paths of the current working directory to the
+        working directory of the container."""
+
+        def _map_path(p: Union[str, Path]) -> Union[str, Path]:
+            if isinstance(p, Path):
+                if not p.is_absolute():
+                    return p
+                if p.is_relative_to(Path.cwd()):
+                    relative = p.relative_to(Path.cwd())
+                    return self.get_workdir() / relative
+                elif p.is_relative_to(Config().cache_location):
+                    relative = p.relative_to(Config().cache_location)
+                    return Path("/home/weck_cache") / relative
+                else:
+                    return p
+            mapped = _map_path(Path(p))
+            if Path(p) == mapped:
+                return p
+            else:
+                return mapped
+
+        return tuple(map(_map_path, command))
+
+    @singledispatchmethod
+    def _initialize_image(self, image: str) -> str:
+        logger.debug("Initializing image from string %s", image)
+        return image
+
+    @_initialize_image.register
+    def _from_fm_config(self, fm_config: FmImageConfig) -> str:
+        logger.debug("Initializing image from FmImageConfig: %s", fm_config)
+        return ImageMgr().prepare_image(self, fm_config)
+
     @staticmethod
     def extract_image(fm: Union[str, Path], version: str, config: dict) -> str:
         image = config.get("defaults", {}).get("image", None)
 
         return parse_fm_data(fm, version).get_images().with_fallback(image)
 
+    @staticmethod
+    def _base_engine_class(config: Config):
+        engine = config.defaults().get("engine", "").lower()
+
+        if engine == "docker":
+            return Docker
+        if engine == "podman":
+            return Podman
+
+        raise ValueError(f"Unknown engine {engine}")
+
     @singledispatchmethod
     @staticmethod
     def from_config(config: Config) -> "Engine":
-        engine = Podman(config.from_defaults_or_none("image"))
+        Base = Engine._base_engine_class(config)
+        engine = Base(config.from_defaults_or_none("image"))
         return Engine._prepare_engine(engine, config)
 
     @from_config.register
     @staticmethod
     def _(fm: Path, version: str, config: Config):
         image = Engine.extract_image(fm, version, config)
-        engine = Podman(image)
+        Base = Engine._base_engine_class(config)
+        engine = Base(image)
         return Engine._prepare_engine(engine, config)
 
     @from_config.register
     @staticmethod
     def _(fm: str, version: str, config: Config):
         image = Engine.extract_image(fm, version, config)
-        engine = Podman(image)
+        Base = Engine._base_engine_class(config)
+        engine = Base(image)
         return Engine._prepare_engine(engine, config)
 
     @from_config.register
     @staticmethod
     def _(fm: FmData, config: Config):
         image = fm.get_images().with_fallback(config.from_defaults_or_none("image"))
-        engine = Podman(image)
+        Base = Engine._base_engine_class(config)
+        engine = Base(image)
         return Engine._prepare_engine(engine, config)
 
     @staticmethod
-    def _prepare_engine(engine, config: dict) -> "Engine":
-        for src, target in config.get("mount", {}).items():
+    def _prepare_engine(engine, config: Config) -> "Engine":
+        for src, target in config.mounts():
             if not Path(src).exists():
-                logging.warning("Mount source %s does not exist. Ignoring it...", src)
+                logger.warning("Mount source %s does not exist. Ignoring it...", src)
                 continue
             engine.mount(src, target)
 
         return engine
 
     @abstractmethod
-    def image_from(self, containerfile: Path) -> "BuildCommand":
-        ...
+    def image_from(self, containerfile: Path) -> "BuildCommand": ...
 
     class BuildCommand(ABC):
-        @abstractmethod
-        def base_image(self, image: str): ...
+        build_args: List[str] = []
 
         @abstractmethod
-        def packages(self, packages: list[str]): ...
+        def __init__(self, containerfile: Path, **kwargs):
+            pass
 
-        @abstractmethod
-        def build(self): ...
+        def base_image(self, image: str):
+            self.build_args += ["--build-arg", f"BASE_IMAGE={image}"]
+            return self
+
+        def packages(self, packages: list[str]):
+            self.build_args += ["--build-arg", f"REQUIRED_PACKAGES={' '.join(packages)}"]
+            return self
+
+        def engine(self):
+            return [self._engine]
+
+        def build(self):
+            cmd = self.engine() + [
+                "build",
+                "-f",
+                self.containerfile,
+                *self.build_args,
+                ".",
+            ]
+
+            logging.debug("Running command: %s", cmd)
+
+            ret = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+            tag = ret.stdout.decode().splitlines()[-1].strip()
+            logger.info("Built image %s", tag)
+            logger.debug("Output of build image was:\n%s", ret.stdout.decode())
+
+            return tag
 
     @staticmethod
     def _run_process(command: tuple[str, ...] | list[str]):
@@ -102,84 +229,25 @@ class Engine(ABC):
             raise NoImageError("No image set for engine.")
 
         command = self.assemble_command(command)
-        logging.info("Running: %s", command)
+        logger.info("Running: %s", command)
         self._run_process(command)
 
-    def get_workdir(self):
-        return Path.cwd().absolute()
-
 
 class Podman(Engine):
     def __init__(self, image: Union[str, FmImageConfig]):
+        super().__init__(image)
         self._engine = "podman"
-        self.image = self._initialize_image(image)
-        self.extra_args = {}
-
-    @singledispatchmethod
-    def _initialize_image(self, image: str) -> str:
-        return image
-
-    @_initialize_image.register
-    def _from_fm_config(self, fm_config: FmImageConfig) -> str:
-        return ImageMgr().prepare_image(self, fm_config)
-
-    def mount_benchexec(self, benchexec_dir: str):
-        self.extra_args["benchexec"] = [
-            "-v",
-            f"{benchexec_dir.absolute()}:/benchexec",
-        ]
-
-    def mount(self, source: str, target: str):
-        self.extra_args["mounts"] = self.extra_args.get("mounts", []) + [
-            "-v",
-            f"{source}:{target}",
-        ]
 
     class PodmanBuildCommand(Engine.BuildCommand):
         def __init__(self, containerfile: Path):
             self.containerfile = containerfile
-            self.build_args = {}
-
-        def base_image(self, image: str):
-            self.build_args["--build-arg"] = f"BASE_IMAGE={image}"
-            return self
-
-        def packages(self, packages: list[str]):
-            self.build_args["--build-arg"] = f"REQUIRED_PACKAGES=\"{' '.join(packages)}\""
-            return self
-
-        def build(self):
-            tag_id = uuid.uuid4().hex
-            tag = f"fmweck/{tag_id}"
-
-            ret = subprocess.run(
-                [
-                    "podman",
-                    "build",
-                    "-t",
-                    tag,
-                    "-f",
-                    self.containerfile,
-                    *self.build_args,
-                    ".",
-                ],
-                check=True,
-            )
-
-            logging.debug("Build output: %s", ret.stdout)
-            logging.info("Built image %s", tag)
-
-            return tag
+            self._engine = "podman"
 
     def image_from(self, containerfile: Path):
         return self.PodmanBuildCommand(containerfile)
 
-    def get_workdir(self):
-        return Path("/home/cwd")
-
-    def assemble_command(self, command: tuple[str, ...]) -> list[str]:
-
-        benchexec_cap = [
+    def benchexec_capabilities(self):
+        return [
             "--annotation",
             "run.oci.keep_original_groups=1",
             "--security-opt",
@@ -190,17 +258,49 @@ class Podman(Engine):
             "/sys/fs/cgroup:/sys/fs/cgroup",
         ]
 
-        base = [
-            "podman",
-            "run",
-        ]
 
-        if self.add_benchexec_capabilities:
-            base += benchexec_cap
+class Docker(Engine):
+    def __init__(self, image: Union[str, FmImageConfig]):
+        super().__init__(image)
+        logger.debug("Image: %s", self.image)
+        self._engine = "docker"
+
+    class DockerBuildCommand(Engine.BuildCommand):
+        def __init__(self, containerfile: Path, needs_sudo: bool = False):
+            self.containerfile = containerfile
+            if needs_sudo:
+                self._engine = "sudo docker"
+            else:
+                self._engine = "docker"
+
+        def engine(self):
+            return self._engine.split(" ")
 
-        base += [
-            "--entrypoint", '[""]',
-            "--cap-add", "SYS_ADMIN",
+    def image_from(self, containerfile: Path):
+        return self.DockerBuildCommand(containerfile, needs_sudo=self._requires_sudo)
+
+    @cached_property
+    def _requires_sudo(self):
+        """Test if docker works without sudo."""
+        try:
+            subprocess.run(["docker", "info"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            logger.debug("Docker does not require sudo.")
+            return False
+        except subprocess.CalledProcessError:
+            logger.debug("Docker requires sudo.")
+            return True
+
+    def base_command(self):
+        if self._requires_sudo:
+            return ["sudo", "docker", "run"]
+        return ["docker", "run"]
+
+    def setup_command(self):
+        return [
+            "--entrypoint",
+            "/bin/sh",
+            "--cap-add",
+            "SYS_ADMIN",
             "-v",
             f"{Path.cwd().absolute()}:/home/cwd",
             "-v",
@@ -210,37 +310,13 @@ class Podman(Engine):
             "--rm",
         ]
 
-        if self.interactive:
-            base += ["-it"]
-
-        for value in self.extra_args.values():
-            if isinstance(value, list) and not isinstance(value, str):
-                base += value
-            else:
-                base.append(value)
-        _command = self._prep_command(command)
-        return base + [self.image, *_command]
-
-    def _prep_command(self, command: tuple[str, ...]) -> tuple[str, ...]:
-        """We want to map absolute paths of the current working directory to the
-        working directory of the container."""
-
-        def _map_path(p: Union[str, Path]) -> Union[str, Path]:
-            if isinstance(p, Path):
-                if not p.is_absolute():
-                    return p
-                if p.is_relative_to(Path.cwd()):
-                    relative = p.relative_to(Path.cwd())
-                    return self.get_workdir() / relative
-                elif p.is_relative_to(Config().cache_location):
-                    relative = p.relative_to(Config().cache_location)
-                    return Path("/home/weck_cache") / relative
-                else:
-                    return p
-            mapped = _map_path(Path(p))
-            if Path(p) == mapped:
-                return p
-            else:
-                return mapped
-
-        return tuple(map(_map_path, command))
+    def benchexec_capabilities(self):
+        return [
+            "--security-opt",
+            "seccomp=unconfined",
+            "--security-opt",
+            "apparmor=unconfined",
+            "--security-opt",
+            "label=disable",
+            "-v /sys/fs/cgroup:/sys/fs/cgroup",
+        ]

fm_weck/image_mgr.py

@@ -5,10 +5,12 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+import logging
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 from fm_tools.fmdata import FmImageConfig
+from yaspin import yaspin
 
 from fm_weck import Config
 
@@ -38,9 +40,15 @@ class ImageMgr(object):
         if image.base_images and not image.required_packages:
             return image.base_images[0]
 
+        logging.info(
+            "Building image from from base image %s with packages %s", image.base_images[0], image.required_packages
+        )
         image_cmd = engine.image_from(CONTAINERFILE)
-        image_cmd.packages(image.required_packages)
         image_cmd.base_image(image.base_images[0])
+        image_cmd.packages(image.required_packages)
+
+        with yaspin(text="Building image", color="cyan") as spinner:
+            tag = image_cmd.build()
+            spinner.ok("✅ ")
 
-        tag = image_cmd.build()
-        return "localhost/" + tag
+        return tag

fm_weck/resources/Containerfile

@@ -5,8 +5,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-ARG REQUIRED_PACKAGES
+
 ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
 RUN apt-get update
-RUN apt-get install -y ${REQUIRED_PACKAGES}
+ARG REQUIRED_PACKAGES
+RUN apt-get install -y ${REQUIRED_PACKAGES} && rm -rf /var/lib/apt/lists/*