qlever 0.2.5__py3-none-any.whl → 0.5.41__py3-none-any.whl

qlever/commands/system_info.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import platform
+from importlib.metadata import version
+from pathlib import Path
+
+import psutil
+
+from qlever.command import QleverCommand
+from qlever.containerize import Containerize
+from qlever.log import log
+from qlever.util import format_size, run_command
+
+
+def show_heading(text: str) -> str:
+    log.info(text)
+    log.info("-" * len(text))
+    log.info("")
+
+
+def get_partition(dir: Path):
+    """
+    Returns the partition on which `dir` resides. May return None.
+    """
+    # The first partition that whose mountpoint is a parent of `dir` is
+    # returned. Sort the partitions by the length of the mountpoint to ensure
+    # that the result is correct. Assume there are partitions with mountpoint
+    # `/` and `/home`. This ensures that `/home/foo` is detected as being in
+    # the partition with mountpoint `/home`.
+    partitions = sorted(
+        psutil.disk_partitions(),
+        key=lambda part: len(part.mountpoint),
+        reverse=True,
+    )
+    for partition in partitions:
+        if dir.is_relative_to(partition.mountpoint):
+            return partition
+    return None
+
+
+class SystemInfoCommand(QleverCommand):
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Gather some system info to help with troubleshooting"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {"runtime": ["system", "image", "server_container"]}
+
+    def additional_arguments(self, subparser) -> None:
+        pass
+
+    def execute(self, args) -> bool:
+        # Say what the command is doing.
+        self.show(
+            "Show system information and Qleverfile", only_show=args.show
+        )
+        if args.show:
+            return True
+
+        # Show system information.
+        show_heading("System Information")
+        system = platform.system()
+        is_linux = system == "Linux"
+        is_mac = system == "Darwin"
+        is_windows = system == "Windows"
+        if is_windows:
+            log.warn("Only limited information is gathered on Windows.")
+        log.info(f"Version: {version('qlever')} (qlever --version)")
+        if is_linux:
+            info = platform.freedesktop_os_release()
+            log.info(f"OS: {platform.system()} ({info['PRETTY_NAME']})")
+        else:
+            log.info(f"OS: {platform.system()}")
+        log.info(f"Arch: {platform.machine()}")
+        log.info(f"Host: {platform.node()}")
+        psutil.virtual_memory().total / (1000**3)
+        memory_total = psutil.virtual_memory().total / (1024.0**3)
+        memory_available = psutil.virtual_memory().available / (1024.0**3)
+        log.info(
+            f"RAM: {memory_total:.1f} GB total, "
+            f"{memory_available:.1f} GB available"
+        )
+        num_cores = psutil.cpu_count(logical=False)
+        num_threads = psutil.cpu_count(logical=True)
+        cpu_freq = psutil.cpu_freq().max / 1000
+        log.info(
+            f"CPU: {num_cores} Cores, "
+            f"{num_threads} Threads @ {cpu_freq:.2f} GHz"
+        )
+
+        cwd = Path.cwd()
+        log.info(f"CWD: {cwd}")
+        # Free and total size of the partition on which the current working
+        # directory resides.
+        disk_usage = psutil.disk_usage(str(cwd))
+        partition = get_partition(cwd)
+        partition_description = f"{partition.device} @ {partition.mountpoint}"
+        fs_type = partition.fstype
+        fs_free = format_size(disk_usage.free)
+        fs_total = format_size(disk_usage.total)
+        log.info(
+            f"Disk space in {partition_description} is "
+            f"({fs_type}): {fs_free} free / {fs_total} total"
+        )
+        # User/Group on host and in container
+        if is_linux or is_mac:
+            user_info = run_command("id", return_output=True).strip()
+            log.info(f"User and group on host: {user_info}")
+        elif is_windows:
+            user_info = run_command("whoami /all", return_output=True).strip()
+            log.info(f"User and group on host: {user_info}")
+        if args.system in Containerize.supported_systems():
+            user_info = Containerize.run_in_container("id", args).strip()
+            log.info(f"User and group in container: {user_info}")
+
+        # Show Qleverfile.
+        log.info("")
+        show_heading("Contents of Qleverfile")
+        qleverfile = cwd / "Qleverfile"
+        if qleverfile.exists():
+            # TODO: output the effective qlever file using primites from #57
+            log.info(qleverfile.read_text())
+        else:
+            log.info("No Qleverfile found")
+        return True

qlever/commands/ui.py

@@ -0,0 +1,271 @@
+from __future__ import annotations
+
+from os import environ
+from pathlib import Path
+
+import yaml
+
+from qlever.command import QleverCommand
+from qlever.containerize import Containerize
+from qlever.log import log
+from qlever.util import is_port_used, run_command
+
+
+# Return a YAML string for the given dictionary. Format values with
+# newlines using the "|" style.
+def dict_to_yaml(dictionary: dict) -> str:
+    """
+    Custom representer for yaml, which uses the "|" style only for
+    multiline strings.
+
+    NOTE: We replace all `\r\n` with `\n` because otherwise the `|` style
+    does not work as expected.
+    """
+
+    class MultiLineDumper(yaml.SafeDumper):
+        def represent_scalar(self, tag, value, style=None):
+            value = value.replace("\r\n", "\n")
+            if isinstance(value, str) and "\n" in value:
+                style = "|"
+            return super().represent_scalar(tag, value, style)
+
+    # Dump as yaml.
+    return yaml.dump(
+        dictionary,
+        sort_keys=False,
+        allow_unicode=True,
+        Dumper=MultiLineDumper,
+    )
+
+
+class UiCommand(QleverCommand):
+    """
+    Class for launching the QLever UI web application.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Launch the QLever UI web application"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {
+            "data": ["name"],
+            "server": ["host_name", "port"],
+            "ui": [
+                "ui_port",
+                "ui_config",
+                "ui_system",
+                "ui_image",
+                "ui_container",
+            ],
+        }
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "--ui-config-file",
+            default="Qleverfile-ui.yml",
+            help="Name of the config file for the QLever UI "
+            "(default: Qleverfile-ui.yml)",
+        )
+        subparser.add_argument(
+            "--ui-db-file",
+            help="Name of the database file for the QLever UI "
+            "(default: {name}.ui-db.sqlite3)",
+        )
+        subparser.add_argument(
+            "--no-pull-latest",
+            action="store_true",
+            default=False,
+            help="Do not pull the latest image for the QLever UI "
+            "(default: pull the latest image if image name contains '/')",
+        )
+        subparser.add_argument(
+            "--stop",
+            action="store_true",
+            default=False,
+            help="Stop the running container",
+        )
+
+    def execute(self, args) -> bool:
+        # If QLEVER_OVERRIDE_DISABLE_UI is set, this command is disabled.
+        qlever_is_running_in_container = environ.get(
+            "QLEVER_IS_RUNNING_IN_CONTAINER"
+        )
+        if qlever_is_running_in_container:
+            log.error(
+                "The environment variable `QLEVER_OVERRIDE_DISABLE_UI` is set, "
+                "therefore `qlever ui` is not available (it should not be called "
+                "from inside a container)"
+            )
+            log.info("")
+            if not args.show:
+                log.info(
+                    "For your information, showing the commands that are "
+                    "executed when `qlever ui` is available:"
+                )
+                log.info("")
+
+        # Construct commands and show them.
+        pull_latest_image = "/" in args.ui_image and not args.no_pull_latest
+        ui_config_name = args.ui_config
+        ui_db_file = args.ui_db_file or f"{args.name}.ui-db.sqlite3"
+        ui_db_file_from_image = "qleverui.sqlite3"
+        ui_config_file = args.ui_config_file
+        sparql_endpoint = f"http://{args.host_name}:{args.port}"
+        ui_url = f"http://{args.host_name}:{args.ui_port}"
+        pull_cmd = f"{args.ui_system} pull -q {args.ui_image}"
+        get_db_cmd = (
+            f"{args.ui_system} create "
+            f"--name {args.ui_container} "
+            f"{args.ui_image} "
+            f"&& {args.ui_system} cp "
+            f"{args.ui_container}:/app/db/{ui_db_file_from_image} {ui_db_file} "
+            f"&& {args.ui_system} rm -f {args.ui_container}"
+        )
+        start_ui_cmd = (
+            f"{args.ui_system} run -d "
+            f"--volume $(pwd):/app/db "
+            f"--env QLEVERUI_DATABASE_URL=sqlite:////app/db/{ui_db_file} "
+            f"--publish {args.ui_port}:7000 "
+            f"--name {args.ui_container} "
+            f"{args.ui_image}"
+        )
+        get_config_cmd = (
+            f"{args.ui_system} exec -i "
+            f"{args.ui_container} "
+            f'bash -c "python manage.py config {ui_config_name}"'
+        )
+        set_config_cmd = (
+            f"{args.ui_system} exec -i "
+            f"{args.ui_container} "
+            f'bash -c "python manage.py config {ui_config_name} '
+            f'/app/db/{ui_config_file} --hide-all-other-backends"'
+        )
+        commands_to_show = []
+        if not args.stop:
+            if pull_latest_image:
+                commands_to_show.append(pull_cmd)
+            if not Path(ui_db_file).exists():
+                commands_to_show.append(get_db_cmd)
+            commands_to_show.append(start_ui_cmd)
+            if not Path(ui_config_file).exists():
+                commands_to_show.append(get_config_cmd)
+            else:
+                commands_to_show.append(set_config_cmd)
+        self.show("\n".join(commands_to_show), only_show=args.show)
+        if qlever_is_running_in_container:
+            return False
+        if args.show:
+            return True
+
+        # Stop running containers.
+        was_found_and_stopped = False
+        for container_system in Containerize.supported_systems():
+            was_found_and_stopped |= Containerize.stop_and_remove_container(
+                container_system, args.ui_container
+            )
+        if was_found_and_stopped:
+            log.debug(f"Stopped and removed container `{args.ui_container}`")
+        else:
+            log.debug(f"No container with name `{args.ui_container}` found")
+        if args.stop:
+            return True
+
+        # Pull the latest image.
+        if pull_latest_image:
+            log.debug(f"Pulling image `{args.ui_image}` for QLever UI")
+            run_command(pull_cmd)
+
+        # Check if the UI port is already being used.
+        if is_port_used(args.ui_port):
+            log.warning(
+                f"It looks like port {args.ui_port} for the QLever UI "
+                f"is already in use. You can set another port in the "
+                f" Qleverfile in the [ui] section with the UI_PORT variable."
+            )
+
+        # Get the QLever UI database from the image, unless it already exists.
+        if Path(ui_db_file).exists():
+            log.debug(f"Found QLever UI database `{ui_db_file}`, reusing it")
+        else:
+            log.debug(f"Getting QLever UI database `{ui_db_file}` from image")
+            try:
+                run_command(get_db_cmd)
+            except Exception as e:
+                log.error(
+                    f"Failed to get {ui_db_file} from {args.ui_image} "
+                    f"({e})"
+                )
+                return False
+
+        # Start the QLever UI.
+        try:
+            log.debug(
+                f"Starting new container with name `{args.ui_container}`"
+            )
+            run_command(start_ui_cmd)
+        except Exception as e:
+            log.error(f"Failed to start container `{args.ui_container}` ({e})")
+            return False
+
+        # Check if config file with name `ui_config_file` exists. If not, try
+        # to obtain it via `get_config_cmd` and set it as default.
+        if Path(ui_config_file).exists():
+            log.info(f"Found config file `{ui_config_file}` and reusing it")
+        else:
+            try:
+                log.info(
+                    f"Get default config file `{ui_config_file}` from image "
+                    f"`{args.ui_image}` and set endpoint to `{sparql_endpoint}`"
+                )
+                config_yaml = run_command(get_config_cmd, return_output=True)
+                config_dict = yaml.safe_load(config_yaml)
+            except Exception as e:
+                log.error("")
+                log.error(
+                    f"An error occured while getting and parsing the "
+                    f"config file ({e})"
+                )
+                return False
+            try:
+                config_dict["config"]["backend"]["isDefault"] = True
+                config_dict["config"]["backend"]["baseUrl"] = sparql_endpoint
+                config_dict["config"]["backend"]["sortKey"] = 1
+                config_yaml = dict_to_yaml(config_dict)
+                with open(ui_config_file, "w") as f:
+                    f.write(config_yaml)
+            except Exception as e:
+                log.error("")
+                log.error(
+                    f"An error occured while modifying and writing the "
+                    f"config file ({e})"
+                )
+                return False
+
+        # Configure the QLever UI.
+        try:
+            run_command(set_config_cmd)
+        except Exception as e:
+            log.error(f"Failed to configure the QLever UI ({e})")
+            return False
+
+        # If we come this far, everything should work.
+        log.info("")
+        log.info(
+            f"The QLever UI should now be up at {ui_url}/{ui_config_name}"
+        )
+        log.info("")
+        log.debug(
+            "If you must, you can log in as QLever UI admin with "
+            'username and password "demo"'
+        )
+        log.info(
+            f"You can modify the config file at `{ui_config_file}` "
+            f"and then just run `qlever ui` again"
+        )
+        return True

qlever/commands/update.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import shlex
+import time
+import traceback
+
+from qlever.command import QleverCommand
+from qlever.log import log
+from qlever.util import run_command
+
+
+class UpdateCommand(QleverCommand):
+    """
+    Class for executing a SPARQL UPDATE against a SPARQL endpoint.
+
+    The command accepts the update either directly on the command line or
+    via a file path provided with --update-file.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Send an update to a SPARQL endpoint"
+
+    def should_have_qleverfile(self) -> bool:
+        return False
+
+    def relevant_qleverfile_arguments(self) -> dict[str,list[str]]:
+        return {"server": ["host_name", "port", "access_token"]}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "update",
+            type=str,
+            nargs="?",
+            default=None,
+            help="SPARQL UPDATE to send (use --update-file to send from a file)",
+        )
+        subparser.add_argument(
+            "--update-file",
+            type=str,
+            help="Path to a file containing the SPARQL UPDATE to send",
+        )
+        subparser.add_argument(
+            "--sparql-endpoint", type=str, help="URL of the SPARQL endpoint"
+        )
+
+    def execute(self, args) -> bool:
+        sparql_endpoint = (
+            args.sparql_endpoint if args.sparql_endpoint else f"{args.host_name}:{args.port}"
+        )
+
+        curl_cmd = (
+            f"curl -s {sparql_endpoint} -X POST "
+            f"-H 'Authorization: Bearer {args.access_token}' "
+            f"-H 'Content-Type: application/sparql-update' "
+        )
+
+        if args.update:
+            curl_cmd += f"--data-binary {shlex.quote(args.update)}"
+        elif args.update_file:
+            curl_cmd += f"--data-binary @{shlex.quote(args.update_file)}"
+        else:
+            log.error("No SPARQL UPDATE provided. Pass it as an argument or via --update-file.")
+            return False
+
+        # Show and exit if requested
+        self.show(curl_cmd, only_show=args.show)
+        if args.show:
+            return True
+
+        # Execute update
+        try:
+            start_time = time.time()
+            run_command(curl_cmd)
+            time_msecs = round(1000 * (time.time() - start_time))
+            if args.log_level != "NO_LOG":
+                log.info("")
+                log.info(
+                    f"Update processing time (end-to-end): {time_msecs:,d} ms"
+                )
+        except Exception as e:
+            if args.log_level == "DEBUG":
+                traceback.print_exc()
+            log.error(e)
+            return False
+
+        return True
+