qlever 0.5.20__py3-none-any.whl → 0.5.22__py3-none-any.whl

qlever/commands/ui.py

@@ -1,12 +1,37 @@
 from __future__ import annotations
 
-import subprocess
 from os import environ
+from pathlib import Path
+
+import yaml
 
 from qlever.command import QleverCommand
 from qlever.containerize import Containerize
 from qlever.log import log
-from qlever.util import is_port_used
+from qlever.util import is_port_used, run_command
+
+
+# Return a YAML string for the given dictionary. Format values with
+# newlines using the "|" style.
+def dict_to_yaml(dictionary):
+    # Custom representer for yaml, which uses the "|" style only for
+    # multiline strings.
+    #
+    # NOTE: We replace all `\r\n` with `\n` because otherwise the `|` style
+    # does not work as expected.
+    class MultiLineDumper(yaml.Dumper):
+        def represent_scalar(self, tag, value, style=None):
+            value = value.replace("\r\n", "\n")
+            if isinstance(value, str) and "\n" in value:
+                style = "|"
+            return super().represent_scalar(tag, value, style)
+
+    # Dump as yaml.
+    return yaml.dump(
+        dictionary,
+        sort_keys=False,
+        Dumper=MultiLineDumper,
+    )
 
 
 class UiCommand(QleverCommand):
@@ -37,7 +62,30 @@ class UiCommand(QleverCommand):
         }
 
     def additional_arguments(self, subparser) -> None:
-        pass
+        subparser.add_argument(
+            "--ui-config-file",
+            default="Qleverfile-ui.yml",
+            help="Name of the config file for the QLever UI "
+            "(default: Qleverfile-ui.yml)",
+        )
+        subparser.add_argument(
+            "--ui-db-file",
+            help="Name of the database file for the QLever UI "
+            "(default: {name}.ui-db.sqlite3)",
+        )
+        subparser.add_argument(
+            "--no-pull-latest",
+            action="store_true",
+            default=False,
+            help="Do not pull the latest image for the QLever UI "
+            "(default: pull the latest image if image name contains '/')",
+        )
+        subparser.add_argument(
+            "--stop",
+            action="store_true",
+            default=False,
+            help="Stop the running container",
+        )
 
     def execute(self, args) -> bool:
         # If QLEVER_OVERRIDE_DISABLE_UI is set, this command is disabled.
@@ -59,57 +107,161 @@ class UiCommand(QleverCommand):
                 log.info("")
 
         # Construct commands and show them.
-        server_url = f"http://{args.host_name}:{args.port}"
+        pull_latest_image = "/" in args.ui_image and not args.no_pull_latest
+        ui_config_name = args.name
+        ui_db_file = args.ui_db_file or f"{args.name}.ui-db.sqlite3"
+        ui_db_file_from_image = "qleverui.sqlite3"
+        ui_config_file = args.ui_config_file
+        sparql_endpoint = f"http://{args.host_name}:{args.port}"
         ui_url = f"http://{args.host_name}:{args.ui_port}"
         pull_cmd = f"{args.ui_system} pull -q {args.ui_image}"
-        run_cmd = (
+        get_db_cmd = (
+            f"{args.ui_system} create "
+            f"--name {args.ui_container} "
+            f"{args.ui_image} "
+            f"&& {args.ui_system} cp "
+            f"{args.ui_container}:/app/db/{ui_db_file_from_image} {ui_db_file} "
+            f"&& {args.ui_system} rm -f {args.ui_container}"
+        )
+        start_ui_cmd = (
             f"{args.ui_system} run -d "
+            f"--volume $(pwd):/app/db "
+            f"--env QLEVERUI_DATABASE_URL=sqlite:////app/db/{ui_db_file} "
             f"--publish {args.ui_port}:7000 "
             f"--name {args.ui_container} "
             f"{args.ui_image}"
         )
-        exec_cmd = (
-            f"{args.ui_system} exec -it "
+        get_config_cmd = (
+            f"{args.ui_system} exec -i "
             f"{args.ui_container} "
-            f'bash -c "python manage.py configure '
-            f'{args.ui_config} {server_url}"'
+            f'bash -c "python manage.py config {ui_config_name}"'
         )
-        self.show(
-            "\n".join(
-                ["Stop running containers", pull_cmd, run_cmd, exec_cmd]
-            ),
-            only_show=args.show,
+        set_config_cmd = (
+            f"{args.ui_system} exec -i "
+            f"{args.ui_container} "
+            f'bash -c "python manage.py config {ui_config_name} '
+            f'/app/db/{ui_config_file} --hide-all-other-backends"'
         )
+        commands_to_show = []
+        if not args.stop:
+            if pull_latest_image:
+                commands_to_show.append(pull_cmd)
+            if not Path(ui_db_file).exists():
+                commands_to_show.append(get_db_cmd)
+            commands_to_show.append(start_ui_cmd)
+            if not Path(ui_config_file).exists():
+                commands_to_show.append(get_config_cmd)
+            else:
+                commands_to_show.append(set_config_cmd)
+        self.show("\n".join(commands_to_show), only_show=args.show)
         if qlever_is_running_in_container:
             return False
         if args.show:
             return True
 
         # Stop running containers.
+        was_found_and_stopped = False
         for container_system in Containerize.supported_systems():
-            Containerize.stop_and_remove_container(
+            was_found_and_stopped |= Containerize.stop_and_remove_container(
                 container_system, args.ui_container
             )
+        if was_found_and_stopped:
+            log.debug(f"Stopped and removed container `{args.ui_container}`")
+        else:
+            log.debug(f"No container with name `{args.ui_container}` found")
+        if args.stop:
+            return True
+
+        # Pull the latest image.
+        if pull_latest_image:
+            log.debug(f"Pulling image `{args.ui_image}` for QLever UI")
+            run_command(pull_cmd)
 
         # Check if the UI port is already being used.
         if is_port_used(args.ui_port):
             log.warning(
-                f"It looks like the specified port for the UI ({args.ui_port}) is already in use. You can set another port in the Qleverfile in the [ui] section with the UI_PORT variable."
+                f"It looks like port {args.ui_port} for the QLever UI "
+                f"is already in use. You can set another port in the "
+                f" Qleverfile in the [ui] section with the UI_PORT variable."
             )
 
-        # Try to start the QLever UI.
+        # Get the QLever UI database from the image, unless it already exists.
+        if Path(ui_db_file).exists():
+            log.debug(f"Found QLever UI database `{ui_db_file}`, reusing it")
+        else:
+            log.debug(f"Getting QLever UI database `{ui_db_file}` from image")
+            try:
+                run_command(get_db_cmd)
+            except Exception as e:
+                log.error(
+                    f"Failed to get {ui_db_file} from {args.ui_image} "
+                    f"({e})"
+                )
+                return False
+
+        # Start the QLever UI.
+        try:
+            log.debug(
+                f"Starting new container with name `{args.ui_container}`"
+            )
+            run_command(start_ui_cmd)
+        except Exception as e:
+            log.error(f"Failed to start container `{args.ui_container}` ({e})")
+            return False
+
+        # Check if config file with name `ui_config_file` exists. If not, try
+        # to obtain it via `get_config_cmd` and set it as default.
+        if Path(ui_config_file).exists():
+            log.info(f"Found config file `{ui_config_file}` and reusing it")
+        else:
+            try:
+                log.info(
+                    f"Get default config file `{ui_config_file}` from image "
+                    f"`{args.ui_image}` and set endpoint to `{sparql_endpoint}`"
+                )
+                config_yaml = run_command(get_config_cmd, return_output=True)
+                config_dict = yaml.safe_load(config_yaml)
+            except Exception as e:
+                log.error("")
+                log.error(
+                    f"An error occured while getting and parsing the "
+                    f"config file ({e})"
+                )
+                return False
+            try:
+                config_dict["config"]["backend"]["isDefault"] = True
+                config_dict["config"]["backend"]["baseUrl"] = sparql_endpoint
+                config_dict["config"]["backend"]["sortKey"] = 1
+                config_yaml = dict_to_yaml(config_dict)
+                with open(ui_config_file, "w") as f:
+                    f.write(config_yaml)
+            except Exception as e:
+                log.error("")
+                log.error(
+                    f"An error occured while modifying and writing the "
+                    f"config file ({e})"
+                )
+                return False
+
+        # Configure the QLever UI.
         try:
-            subprocess.run(pull_cmd, shell=True, stdout=subprocess.DEVNULL)
-            subprocess.run(run_cmd, shell=True, stdout=subprocess.DEVNULL)
-            subprocess.run(exec_cmd, shell=True, stdout=subprocess.DEVNULL)
-        except subprocess.CalledProcessError as e:
-            log.error(f"Failed to start the QLever UI ({e})")
+            run_command(set_config_cmd)
+        except Exception as e:
+            log.error(f"Failed to configure the QLever UI ({e})")
             return False
 
-        # Success.
+        # If we come this far, everything should work.
+        log.info("")
+        log.info(
+            f"The QLever UI should now be up at {ui_url}/{ui_config_name}"
+        )
+        log.info("")
+        log.debug(
+            "If you must, you can log in as QLever UI admin with "
+            'username and password "demo"'
+        )
         log.info(
-            f"The QLever UI should now be up at {ui_url} ..."
-            f"You can log in as QLever UI admin with username and "
-            f'password "demo"'
+            f"You can modify the config file at `{ui_config_file}` "
+            f"and then just run `qlever ui` again"
         )
         return True

qlever/config.py

@@ -9,7 +9,7 @@ from pathlib import Path
 import argcomplete
 from termcolor import colored
 
-from qlever import command_objects, script_name
+from qlever import command_objects, engine_name, script_name
 from qlever.log import log, log_levels
 from qlever.qleverfile import Qleverfile
 
@@ -176,11 +176,18 @@ class QleverConfig:
         # are defined in the modules in `qlever/commands`. In `__init__.py`
         # an object of each class is created and stored in `command_objects`.
         parser = argparse.ArgumentParser(
-                description=colored("This is the qlever command line tool, "
-                                    "it's all you need to work with QLever",
-                                    attrs=["bold"]))
-        parser.add_argument("--version", action="version",
-                            version=f"%(prog)s {version('qlever')}")
+            description=colored(
+                f"This is the {script_name} command line tool, "
+                f"it's all you need to work with {engine_name}",
+                attrs=["bold"],
+            )
+        )
+        if script_name == "qlever":
+            parser.add_argument(
+                "--version",
+                action="version",
+                version=f"%(prog)s {version('qlever')}",
+            )
         add_qleverfile_option(parser)
         subparsers = parser.add_subparsers(dest='command')
         subparsers.required = True

qlever/containerize.py

@@ -9,7 +9,7 @@ import subprocess
 from typing import Optional
 
 from qlever.log import log
-from qlever.util import run_command, get_random_string
+from qlever.util import get_random_string, run_command
 
 
 class ContainerizeException(Exception):
@@ -40,6 +40,7 @@ class Containerize:
         volumes: list[tuple[str, str]] = [],
         ports: list[tuple[int, int]] = [],
         working_directory: Optional[str] = None,
+        use_bash: bool = True,
     ) -> str:
         """
         Get the command to run `cmd` with the given `container_system` and the
@@ -80,11 +81,15 @@ class Containerize:
             f"{volume_options}"
             f"{port_options}"
             f"{working_directory_option}"
+            f" --name {container_name}"
             f" --init"
-            f" --entrypoint bash"
-            f" --name {container_name} {image_name}"
-            f" -c {shlex.quote(cmd)}"
         )
+        if use_bash:
+            containerized_cmd += (
+                f" --entrypoint bash {image_name} -c {shlex.quote(cmd)}"
+            )
+        else:
+            containerized_cmd += f" {image_name} {cmd}"
         return containerized_cmd
 
     @staticmethod

qlever/qlever_main.py

@@ -12,7 +12,7 @@ import traceback
 
 from termcolor import colored
 
-from qlever import command_objects
+from qlever import command_objects, script_name
 from qlever.config import ConfigException, QleverConfig
 from qlever.log import log, log_levels
 
@@ -35,9 +35,9 @@ def main():
         log.info("")
         log.info(colored(f"Command: {args.command}", attrs=["bold"]))
         log.info("")
-        commandWasSuccesful = command_object.execute(args)
+        command_successful = command_object.execute(args)
         log.info("")
-        if not commandWasSuccesful:
+        if not command_successful:
             exit(1)
     except KeyboardInterrupt:
         log.info("")
@@ -47,9 +47,14 @@ def main():
     except Exception as e:
         # Check if it's a certain kind of `AttributeError` and give a hint in
         # that case.
+        log.debug(
+            "Command failed with exception, full traceback: "
+            f"{traceback.format_exc()}"
+        )
         match_error = re.search(r"object has no attribute '(.+)'", str(e))
         match_trace = re.search(
-            r"(qlever/commands/.+\.py)\", line (\d+)", traceback.format_exc()
+            rf"({script_name}/commands/.+\.py)\", line (\d+)",
+            traceback.format_exc(),
         )
         if isinstance(e, AttributeError) and match_error and match_trace:
             attribute = match_error.group(1)

qlever/qleverfile.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
 import re
+import socket
 import subprocess
-from configparser import ConfigParser, ExtendedInterpolation
+from configparser import ConfigParser, ExtendedInterpolation, RawConfigParser
+from pathlib import Path
 
+from qlever import script_name
 from qlever.containerize import Containerize
 from qlever.log import log
 
@@ -242,6 +245,13 @@ class Qleverfile:
             default=8,
             help="The number of threads used for query processing",
         )
+        server_args["persist_updates"] = arg(
+            "--persist-updates",
+            action="store_true",
+            default=False,
+            help="Persist updates to the index (write updates to disk and "
+            "read them back in when restarting the server)",
+        )
         server_args["only_pso_and_pos_permutations"] = arg(
             "--only-pso-and-pos-permutations",
             action="store_true",
@@ -291,12 +301,12 @@ class Qleverfile:
         runtime_args["index_container"] = arg(
             "--index-container",
             type=str,
-            help="The name of the container used by `qlever index`",
+            help=f"The name of the container used by `{script_name} index`",
         )
         runtime_args["server_container"] = arg(
             "--server-container",
             type=str,
-            help="The name of the container used by `qlever start`",
+            help=f"The name of the container used by `{script_name} start`",
         )
 
         ui_args["ui_port"] = arg(
@@ -393,9 +403,9 @@ class Qleverfile:
             name = config["data"]["name"]
             runtime = config["runtime"]
             if "server_container" not in runtime:
-                runtime["server_container"] = f"qlever.server.{name}"
+                runtime["server_container"] = f"{script_name}.server.{name}"
             if "index_container" not in runtime:
-                runtime["index_container"] = f"qlever.index.{name}"
+                runtime["index_container"] = f"{script_name}.index.{name}"
             if "ui_container" not in config["ui"]:
                 config["ui"]["ui_container"] = f"qlever.ui.{name}"
             index = config["index"]
@@ -407,5 +417,49 @@ class Qleverfile:
         if index.get("text_index", "none") != "none":
             server["use_text_index"] = "yes"
 
+        # Add other non-trivial default values.
+        try:
+            config["server"]["host_name"] = socket.gethostname()
+        except Exception:
+            log.warning(
+                "Could not get the hostname, using `localhost` as default"
+            )
+            pass
+
         # Return the parsed Qleverfile with the added inherited values.
         return config
+
+    @staticmethod
+    def filter(
+        qleverfile_path: Path, options_included: dict[str, list[str]]
+    ) -> RawConfigParser:
+        """
+        Given a filter criteria (key: section_header, value: list[options]),
+        return a RawConfigParser object to create a new filtered Qleverfile
+        with only the specified sections and options (selects all options if
+        list[options] is empty). Mainly to be used by non-qlever scripts for
+        the setup-config command
+        """
+        # Read the Qleverfile.
+        config = RawConfigParser()
+        config.optionxform = str  # Preserve case sensitivity of keys
+        config.read(qleverfile_path)
+
+        filtered_config = RawConfigParser()
+        filtered_config.optionxform = str
+
+        for section, desired_fields in options_included.items():
+            if config.has_section(section):
+                filtered_config.add_section(section)
+
+                # If the list is empty, copy all fields
+                if not desired_fields:
+                    for field, value in config.items(section):
+                        filtered_config.set(section, field, value)
+                else:
+                    for desired_field in desired_fields:
+                        if config.has_option(section, desired_field):
+                            value = config.get(section, desired_field)
+                            filtered_config.set(section, desired_field, value)
+
+        return filtered_config

qlever/util.py

@@ -10,7 +10,9 @@ import string
 import subprocess
 from datetime import date, datetime
 from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
+
+import psutil
 
 from qlever.log import log
 
@@ -33,6 +35,7 @@ def run_command(
     cmd: str,
     return_output: bool = False,
     show_output: bool = False,
+    show_stderr: bool = False,
     use_popen: bool = False,
 ) -> Optional[str | subprocess.Popen]:
     """
@@ -50,7 +53,7 @@ def run_command(
         "shell": True,
         "text": True,
         "stdout": None if show_output else subprocess.PIPE,
-        "stderr": subprocess.PIPE,
+        "stderr": None if show_stderr else subprocess.PIPE,
     }
 
     # With `Popen`, the command runs in the current shell and a process object
@@ -72,8 +75,8 @@ def run_command(
             raise Exception(result.stderr.replace("\n", " ").strip())
         else:
             raise Exception(
-                f"Command failed with exit code {result.returncode}"
-                f" but nothing written to stderr"
+                f"Command failed with exit code {result.returncode}, "
+                f" nothing written to stderr"
             )
     # Optionally, return what was written to `stdout`.
     if return_output:
@@ -242,3 +245,85 @@ def format_size(bytes, suffix="B"):
         if bytes < factor:
             return f"{bytes:.2f} {unit}{suffix}"
         bytes /= factor
+
+
+def stop_process(proc: psutil.Process, pinfo: dict[str, Any]) -> bool:
+    """
+    Try to kill the given process, return True iff it was killed
+    successfully. The process_info is used for logging.
+    """
+    try:
+        proc.kill()
+        log.info(f"Killed process {pinfo['pid']}")
+        return True
+    except Exception as e:
+        log.error(
+            f"Could not kill process with PID "
+            f"{pinfo['pid']} ({e}) ... try to kill it "
+            f"manually"
+        )
+        log.info("")
+        show_process_info(proc, "", show_heading=True)
+        return False
+
+
+def stop_process_with_regex(cmdline_regex: str) -> list[bool] | None:
+    """
+    Given a cmdline_regex for a native process, try to kill the processes that
+    match the regex and return a list of their stopped status (bool).
+    Show the matched processes as log info.
+    """
+    stop_process_results = []
+    for proc in psutil.process_iter():
+        try:
+            pinfo = proc.as_dict(
+                attrs=[
+                    "pid",
+                    "username",
+                    "create_time",
+                    "memory_info",
+                    "cmdline",
+                ]
+            )
+            cmdline = " ".join(pinfo["cmdline"])
+        except Exception as e:
+            log.debug(f"Error getting process info: {e}")
+            return None
+        if re.search(cmdline_regex, cmdline):
+            log.info(
+                f"Found process {pinfo['pid']} from user "
+                f"{pinfo['username']} with command line: {cmdline}"
+            )
+            log.info("")
+            stop_process_results.append(stop_process(proc, pinfo))
+    return stop_process_results
+
+
+def binary_exists(binary: str, cmd_arg: str) -> bool:
+    """
+    When a command is run natively, check if the binary exists on the system
+    """
+    try:
+        run_command(f"{binary} --help")
+        return True
+    except Exception as e:
+        log.error(
+            f'Running "{binary}" failed, '
+            f"set `--{cmd_arg}` to a different binary or "
+            f"set `--system to a container system`"
+        )
+        log.info("")
+        log.info(f"The error message was: {e}")
+        return False
+
+
+def is_server_alive(url: str) -> bool:
+    """
+    Check if the server is already alive at the given endpoint url
+    """
+    check_server_cmd = f"curl -s {url}"
+    try:
+        run_command(check_server_cmd)
+        return True
+    except Exception:
+        return False

{qlever-0.5.20.dist-info → qlever-0.5.22.dist-info}/METADATA

@@ -1,11 +1,11 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: qlever
-Version: 0.5.20
-Summary: Script for using the QLever SPARQL engine.
+Version: 0.5.22
+Summary: Command-line tool for using the QLever graph database
 Author-email: Hannah Bast <bast@cs.uni-freiburg.de>
 License: Apache-2.0
 Project-URL: Github, https://github.com/ad-freiburg/qlever
-Keywords: SPARQL,RDF,Knowledge Graphs,Triple Store
+Keywords: Graph database,Triplestore,Knowledge graphs,SPARQL,RDF
 Classifier: Topic :: Database :: Database Engines/Servers
 Classifier: Topic :: Database :: Front-Ends
 Requires-Python: >=3.8
@@ -14,6 +14,8 @@ License-File: LICENSE
 Requires-Dist: psutil
 Requires-Dist: termcolor
 Requires-Dist: argcomplete
+Requires-Dist: pyyaml
+Dynamic: license-file
 
 # QLever