qlever 0.2.5__py3-none-any.whl → 0.5.41__py3-none-any.whl

qlever/commands/warmup.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import subprocess
+
+from qlever.command import QleverCommand
+from qlever.log import log
+
+
+class WarmupCommand(QleverCommand):
+    """
+    Class for executing the `warmup` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return ("Execute WARMUP_CMD")
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
+        return {"server": ["port", "warmup_cmd"]}
+
+    def additional_arguments(self, subparser) -> None:
+        pass
+
+    def execute(self, args) -> bool:
+        # Show what the command is doing.
+        self.show(args.warmup_cmd, only_show=args.show)
+        if args.show:
+            return True
+
+        # Execute the command.
+        try:
+            subprocess.run(args.warmup_cmd, shell=True, check=True)
+        except Exception as e:
+            log.error(f"{e.output if hasattr(e, 'output') else e}")
+            return False
+        return True

qlever/config.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+import argparse
+import os
+import traceback
+from importlib.metadata import version
+from pathlib import Path
+
+import argcomplete
+from termcolor import colored
+
+from qlever import command_objects, engine_name, script_name
+from qlever.log import log, log_levels
+from qlever.qleverfile import Qleverfile
+
+
+# Simple exception class for configuration errors (the class need not do
+# anything, we just want a distinct exception type).
+class ConfigException(Exception):
+    def __init__(self, message):
+        stack = traceback.extract_stack()[-2]  # Caller's frame.
+        self.filename = stack.filename
+        self.lineno = stack.lineno
+        full_message = f"{message} [in {self.filename}:{self.lineno}]"
+        super().__init__(full_message)
+
+
+class QleverConfig:
+    """
+    Class that manages all config parameters, and overwrites them with the
+    settings from a Qleverfile.
+
+    IMPORTANT: An instance of this class is created for each execution of
+    the `qlever` script, in particular, each time the user triggers
+    autocompletion. Therefore, pay attention that no unnecessary work is done
+    before the call to `argcomplete.autocomplete(...)`. In particular, avoid
+    parsing the Qleverfile before that point, it's not needed for
+    autocompletion.
+    """
+
+    def add_subparser_for_command(self, subparsers, command_name,
+                                  command_object, all_qleverfile_args,
+                                  qleverfile_config=None):
+        """
+        Add subparser for the given command. Take the arguments from
+        `command_object.relevant_qleverfile_arguments()` and report an error if
+        one of them is not contained in `all_qleverfile_args`. Overwrite the
+        default values with the values from `qleverfile_config` if specified.
+        """
+
+        arg_names = command_object.relevant_qleverfile_arguments()
+
+        # Helper function that shows a detailed error messahe when an argument
+        # from `relevant_qleverfile_arguments` is not contained in
+        # `all_qleverfile_args`.
+        def argument_error(prefix):
+            log.info("")
+            log.error(f"{prefix} in `Qleverfile.all_arguments()` for command "
+                      f"`{command_name}`")
+            log.info("")
+            log.info(f"Value of `relevant_qleverfile_arguments` for "
+                     f"command `{command_name}`:")
+            log.info("")
+            log.info(f"{arg_names}")
+            log.info("")
+            exit(1)
+
+        # Add the subparser.
+        description = command_object.description()
+        subparser = subparsers.add_parser(command_name,
+                                          description=description,
+                                          help=description)
+
+        # Add the arguments relevant for the command.
+        for section in arg_names:
+            if section not in all_qleverfile_args:
+                argument_error(f"Section `{section}` not found")
+            for arg_name in arg_names[section]:
+                if arg_name not in all_qleverfile_args[section]:
+                    argument_error(f"Argument `{arg_name}` of section "
+                                   f"`{section}` not found")
+                args, kwargs = all_qleverfile_args[section][arg_name]
+                kwargs_copy = kwargs.copy()
+                # If `qleverfile_config` is given, add info about default
+                # values to the help string.
+                if qleverfile_config is not None:
+                    default_value = kwargs.get("default", None)
+                    qleverfile_value = qleverfile_config.get(
+                            section, arg_name, fallback=None)
+                    if qleverfile_value is not None:
+                        kwargs_copy["default"] = qleverfile_value
+                        kwargs_copy["required"] = False
+                        kwargs_copy["help"] += (f" [default, from Qleverfile:"
+                                                f" {qleverfile_value}]")
+                    else:
+                        kwargs_copy["help"] += f" [default: {default_value}]"
+                subparser.add_argument(*args, **kwargs_copy)
+
+        # Additional arguments that are shared by all commands.
+        command_object.additional_arguments(subparser)
+        subparser.add_argument("--show", action="store_true",
+                               default=False,
+                               help="Only show what would be executed"
+                                    ", but don't execute it")
+        subparser.add_argument("--log-level",
+                               choices=log_levels.keys(),
+                               default="INFO",
+                               help="Set the log level")
+
+    def parse_args(self):
+        # Determine whether we are in autocomplete mode or not.
+        autocomplete_mode = "COMP_LINE" in os.environ
+
+        # Check if the user has registered this script for argcomplete.
+        argcomplete_check_off = os.environ.get("QLEVER_ARGCOMPLETE_CHECK_OFF")
+        argcomplete_enabled = os.environ.get("QLEVER_ARGCOMPLETE_ENABLED")
+        if not argcomplete_enabled and not argcomplete_check_off:
+            log.info("")
+            log.warn(f"To enable autocompletion, run the following command, "
+                     f"and consider adding it to your `.bashrc` or `.zshrc`:"
+                     f"\n\n"
+                     f"eval \"$(register-python-argcomplete {script_name})\""
+                     f" && export QLEVER_ARGCOMPLETE_ENABLED=1")
+            log.info("")
+
+        # Create a temporary parser only to parse the `--qleverfile` option, in
+        # case it is given, and to determine whether a command was given that
+        # requires a Qleverfile. This is because in the actual parser below we
+        # want the values from the Qleverfile to be shown in the help strings,
+        # but only if this is actually necessary.
+        def add_qleverfile_option(parser):
+            parser.add_argument("--qleverfile", "-q", type=str,
+                                default="Qleverfile")
+        qleverfile_parser = argparse.ArgumentParser(add_help=False)
+        add_qleverfile_option(qleverfile_parser)
+        qleverfile_parser.add_argument("command", type=str, nargs="?")
+        qleverfile_args, _ = qleverfile_parser.parse_known_args()
+        qleverfile_path_name = qleverfile_args.qleverfile
+        # command = qleverfile_args.command
+        # should_have_qleverfile = command in command_objects \
+        #     and command_objects[command].should_have_qleverfile()
+
+        # Check if the Qleverfile exists and if we are using the default name.
+        # We need this again further down in the code, so remember it.
+        qleverfile_path = Path(qleverfile_path_name)
+        qleverfile_exists = qleverfile_path.is_file()
+        qleverfile_is_default = qleverfile_path_name \
+            == qleverfile_parser.get_default("qleverfile")
+        # If a Qleverfile with a non-default name was specified, but it does
+        # not exist, that's an error.
+        if not qleverfile_exists and not qleverfile_is_default:
+            raise ConfigException(f"Qleverfile with non-default name "
+                                  f"`{qleverfile_path_name}` specified, "
+                                  f"but it does not exist")
+        # If it exists and we are not in the autocompletion mode, parse it.
+        #
+        # IMPORTANT: No need to parse the Qleverfile in autocompletion mode and
+        # it would be unnecessarily expensive to do so.
+        #
+        # TODO: What if `command.should_have_qleverfile()` is `False`, should
+        # we then parse the Qleverfile or not.
+        if qleverfile_exists and not autocomplete_mode:
+            try:
+                qleverfile_config = Qleverfile.read(qleverfile_path)
+            except Exception as e:
+                log.info("")
+                log.error(f"Error parsing Qleverfile `{qleverfile_path}`"
+                          f": {e}")
+                log.info("")
+                exit(1)
+        else:
+            qleverfile_config = None
+
+        # Now the regular parser with commands and a subparser for each
+        # command. We have a dedicated class for each command. These classes
+        # are defined in the modules in `qlever/commands`. In `__init__.py`
+        # an object of each class is created and stored in `command_objects`.
+        parser = argparse.ArgumentParser(
+            description=colored(
+                f"This is the {script_name} command line tool, "
+                f"it's all you need to work with {engine_name}",
+                attrs=["bold"],
+            )
+        )
+        if script_name == "qlever":
+            parser.add_argument(
+                "--version",
+                action="version",
+                version=f"%(prog)s {version('qlever')}",
+            )
+        add_qleverfile_option(parser)
+        subparsers = parser.add_subparsers(dest='command')
+        subparsers.required = True
+        all_args = Qleverfile.all_arguments()
+        for command_name, command_object in command_objects.items():
+            self.add_subparser_for_command(
+                    subparsers, command_name, command_object,
+                    all_args, qleverfile_config)
+
+        # Enable autocompletion for the commands and their options.
+        #
+        # NOTE: All code executed before this line should be relatively cheap
+        # because it is executed whenever the user triggers the autocompletion.
+        argcomplete.autocomplete(parser, always_complete_options="long")
+
+        # If called without arguments, show the help message.
+        if len(os.sys.argv) == 1:
+            parser.print_help()
+            exit(0)
+
+        # Parse the command line arguments.
+        args = parser.parse_args()
+
+        # If the command says that we should have a Qleverfile, but we don't,
+        # issue a warning.
+        if command_objects[args.command].should_have_qleverfile():
+            if not qleverfile_exists:
+                log.warning(f"Invoking command `{args.command}` without a "
+                            "Qleverfile. You have to specify all required "
+                            "arguments on the command line. This is possible, "
+                            "but not recommended.")
+
+        return args

qlever/containerize.py

@@ -0,0 +1,167 @@
+# Copyright 2024, University of Freiburg,
+# Chair of Algorithms and Data Structures
+# Author: Hannah Bast <bast@cs.uni-freiburg.de>
+
+from __future__ import annotations
+
+import shlex
+import subprocess
+from typing import Optional
+
+from qlever.log import log
+from qlever.util import get_random_string, run_command
+
+
+class ContainerizeException(Exception):
+    pass
+
+
+class Containerize:
+    """
+    This class contains functions specific for running commands with various
+    container engines.
+    """
+
+    @staticmethod
+    def supported_systems() -> list[str]:
+        """
+        Return a list of the supported container systems. Make sure that they
+        are all indeed supported by `containerize_command` below.
+        """
+        return ["docker", "podman"]
+
+    @staticmethod
+    def containerize_command(
+        cmd: str,
+        container_system: str,
+        run_subcommand: str,
+        image_name: str,
+        container_name: str,
+        volumes: list[tuple[str, str]] = [],
+        ports: list[tuple[int, int]] = [],
+        working_directory: Optional[str] = None,
+        use_bash: bool = True,
+    ) -> str:
+        """
+        Get the command to run `cmd` with the given `container_system` and the
+        given options.
+        """
+
+        # Check that `container_system` is supported.
+        if container_system not in Containerize.supported_systems():
+            return ContainerizeException(
+                f'Invalid container system "{container_system}"'
+                f" (must be one of {Containerize.supported_systems()})"
+            )
+
+        # Set user and group ids. This is important so that the files created
+        # by the containerized command are owned by the user running the
+        # command.
+        if container_system == "docker":
+            user_option = " -u $(id -u):$(id -g)"
+        elif container_system == "podman":
+            user_option = " -u root"
+        else:
+            user_option = ""
+
+        # Options for mounting volumes, setting ports, and setting the working
+        # dir.
+        volume_options = "".join(
+            [
+                f' --mount type=bind,src="{v1}",target={v2}'
+                for v1, v2 in volumes
+            ]
+        )
+        port_options = "".join([f" -p {p1}:{p2}" for p1, p2 in ports])
+        working_directory_option = (
+            f" -w {working_directory}" if working_directory is not None else ""
+        )
+
+        # Construct the command that runs `cmd` with the given container
+        # system.
+        containerized_cmd = (
+            f"{container_system} {run_subcommand}"
+            f"{user_option}"
+            f" -v /etc/localtime:/etc/localtime:ro"
+            f"{volume_options}"
+            f"{port_options}"
+            f"{working_directory_option}"
+            f" --name {container_name}"
+            f" --init"
+        )
+        if use_bash:
+            containerized_cmd += (
+                f" --entrypoint bash {image_name} -c {shlex.quote(cmd)}"
+            )
+        else:
+            containerized_cmd += f" {image_name} {cmd}"
+        return containerized_cmd
+
+    @staticmethod
+    def is_running(container_system: str, container_name: str) -> bool:
+        # Note: the `{{{{` and `}}}}` result in `{{` and `}}`, respectively.
+        containers = (
+            run_command(
+                f'{container_system} ps --format="{{{{.Names}}}}"',
+                return_output=True,
+            )
+            .strip()
+            .splitlines()
+        )
+        return container_name in containers
+
+    @staticmethod
+    def stop_and_remove_container(
+        container_system: str, container_name: str
+    ) -> bool:
+        """
+        Stop the container with the given name using the given system. Return
+        `True` if a container with that name was found and stopped, `False`
+        otherwise.
+        """
+
+        # Check that `container_system` is supported.
+        if container_system not in Containerize.supported_systems():
+            return ContainerizeException(
+                f'Invalid container system "{container_system}"'
+                f" (must be one of {Containerize.supported_systems()})"
+            )
+
+        # Construct the command that stops the container.
+        stop_cmd = (
+            f"{container_system} stop {container_name} && "
+            f"{container_system} rm {container_name}"
+        )
+
+        # Run the command.
+        try:
+            subprocess.run(
+                stop_cmd,
+                shell=True,
+                check=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            return True
+        except Exception as e:
+            log.debug(f'Error running "{stop_cmd}": {e}')
+            return False
+
+    @staticmethod
+    def run_in_container(cmd: str, args) -> Optional[str]:
+        """
+        Run an arbitrary command in the qlever container and return its output.
+        """
+        if args.system in Containerize.supported_systems():
+            if not args.server_container:
+                args.server_container = get_random_string(20)
+            run_cmd = Containerize().containerize_command(
+                cmd,
+                args.system,
+                'run --rm -it --entrypoint "" ',
+                args.image,
+                args.server_container,
+                volumes=[("$(pwd)", "/index")],
+                working_directory="/index",
+            )
+            return run_command(run_cmd, return_output=True)

qlever/log.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import logging
+from contextlib import contextmanager
+
+from termcolor import colored
+
+
+class QleverLogFormatter(logging.Formatter):
+    """
+    Custom formatter for logging.
+    """
+
+    def format(self, record):
+        message = record.getMessage()
+        if record.levelno == logging.DEBUG:
+            return colored(f"{message}", "magenta")
+        elif record.levelno == logging.WARNING:
+            return colored(f"{message}", "yellow")
+        elif record.levelno in [logging.CRITICAL, logging.ERROR]:
+            return colored(f"{message}", "red")
+        else:
+            return message
+
+
+# Custom logger.
+log = logging.getLogger("qlever")
+log.setLevel(logging.INFO)
+handler = logging.StreamHandler()
+handler.setFormatter(QleverLogFormatter())
+log.addHandler(handler)
+log_levels = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL,
+    "NO_LOG": logging.CRITICAL + 1,
+}
+
+
+@contextmanager
+def mute_log(level=logging.ERROR):
+    """
+    Temporarily mute the log, simply works as follows:
+
+    with mute_log():
+       ...
+    """
+    original_level = log.getEffectiveLevel()
+    log.setLevel(level)
+    try:
+        yield
+    finally:
+        log.setLevel(original_level)

qlever/qlever_main.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# PYTHON_ARGCOMPLETE_OK
+
+# Copyright 2024, University of Freiburg,
+# Chair of Algorithms and Data Structures
+# Author: Hannah Bast <bast@cs.uni-freiburg.de>
+
+from __future__ import annotations
+
+import re
+import traceback
+
+from termcolor import colored
+
+from qlever import command_objects, script_name
+from qlever.config import ConfigException, QleverConfig
+from qlever.log import log, log_levels
+
+
+def main():
+    # Parse the command line arguments and read the Qleverfile.
+    try:
+        qlever_config = QleverConfig()
+        args = qlever_config.parse_args()
+    except ConfigException as e:
+        log.error(e)
+        log.info("")
+        log.info(traceback.format_exc())
+        exit(1)
+
+    # Execute the command.
+    command_object = command_objects[args.command]
+    log.setLevel(log_levels[args.log_level])
+    try:
+        log.info("")
+        log.info(colored(f"Command: {args.command}", attrs=["bold"]))
+        log.info("")
+        command_successful = command_object.execute(args)
+        log.info("")
+        if not command_successful:
+            exit(1)
+    except KeyboardInterrupt:
+        log.warn("\rCtrl-C pressed, exiting ...")
+        log.info("")
+        exit(1)
+    except Exception as e:
+        # Check if it's a certain kind of `AttributeError` and give a hint in
+        # that case.
+        log.debug(
+            "Command failed with exception, full traceback: "
+            f"{traceback.format_exc()}"
+        )
+        match_error = re.search(r"object has no attribute '(.+)'", str(e))
+        match_trace = re.search(
+            rf"({script_name}/commands/.+\.py)\", line (\d+)",
+            traceback.format_exc(),
+        )
+        if isinstance(e, AttributeError) and match_error and match_trace:
+            attribute = match_error.group(1)
+            trace_command = match_trace.group(1)
+            trace_line = match_trace.group(2)
+            log.error(f"{e} in `{trace_command}` at line {trace_line}")
+            log.info("")
+            log.info(
+                f"Likely cause: you used `args.{attribute}`, but it was "
+                f"neither defined in `relevant_qleverfile_arguments` "
+                f"nor in `additional_arguments`"
+            )
+            log.info("")
+            log.info(
+                f"If you did not implement `{trace_command}` yourself, "
+                f"please report this issue"
+            )
+            log.info("")
+        else:
+            log.error(f"An unexpected error occurred: {e}")
+            log.info("")
+            log.info(traceback.format_exc())
+        exit(1)