qlever 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

qlever/commands/setup_config.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import subprocess
+from os import environ
 from pathlib import Path
 
 from qlever.command import QleverCommand
@@ -15,9 +16,9 @@ class SetupConfigCommand(QleverCommand):
 
     def __init__(self):
         self.qleverfiles_path = Path(__file__).parent.parent / "Qleverfiles"
-        self.qleverfile_names = \
-            [p.name.split(".")[1]
-             for p in self.qleverfiles_path.glob("Qleverfile.*")]
+        self.qleverfile_names = [
+            p.name.split(".")[1] for p in self.qleverfiles_path.glob("Qleverfile.*")
+        ]
 
     def description(self) -> str:
         return "Get a pre-configured Qleverfile"
@@ -25,23 +26,38 @@ class SetupConfigCommand(QleverCommand):
     def should_have_qleverfile(self) -> bool:
         return False
 
-    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
+    def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
         return {}
 
     def additional_arguments(self, subparser) -> None:
         subparser.add_argument(
-                "config_name", type=str,
-                choices=self.qleverfile_names,
-                help="The name of the pre-configured Qleverfile to create")
+            "config_name",
+            type=str,
+            choices=self.qleverfile_names,
+            help="The name of the pre-configured Qleverfile to create",
+        )
 
     def execute(self, args) -> bool:
+        # Show a warning if `QLEVER_OVERRIDE_SYSTEM_NATIVE` is set.
+        qlever_is_running_in_container = environ.get("QLEVER_IS_RUNNING_IN_CONTAINER")
+        if qlever_is_running_in_container:
+            log.warning(
+                "The environment variable `QLEVER_IS_RUNNING_IN_CONTAINER` is set, "
+                "therefore the Qleverfile is modified to use `SYSTEM = native` "
+                "(since inside the container, QLever should run natively)"
+            )
+            log.info("")
         # Construct the command line and show it.
-        qleverfile_path = (self.qleverfiles_path
-                           / f"Qleverfile.{args.config_name}")
+        qleverfile_path = self.qleverfiles_path / f"Qleverfile.{args.config_name}"
         setup_config_cmd = (
-                f"cat {qleverfile_path}"
-                f" | sed -E 's/(^ACCESS_TOKEN.*)/\\1_{get_random_string(12)}/'"
-                f"> Qleverfile")
+            f"cat {qleverfile_path}"
+            f" | sed -E 's/(^ACCESS_TOKEN.*)/\\1_{get_random_string(12)}/'"
+        )
+        if qlever_is_running_in_container:
+            setup_config_cmd += (
+                " | sed -E 's/(^SYSTEM[[:space:]]*=[[:space:]]*).*/\\1native/'"
+            )
+        setup_config_cmd += "> Qleverfile"
         self.show(setup_config_cmd, only_show=args.show)
         if args.show:
             return False
@@ -51,31 +67,31 @@ class SetupConfigCommand(QleverCommand):
         if qleverfile_path.exists():
             log.error("`Qleverfile` already exists in current directory")
             log.info("")
-            log.info("If you want to create a new Qleverfile using "
-                     "`qlever setup-config`, delete the existing Qleverfile "
-                     "first")
+            log.info(
+                "If you want to create a new Qleverfile using "
+                "`qlever setup-config`, delete the existing Qleverfile "
+                "first"
+            )
             return False
 
         # Copy the Qleverfile to the current directory.
         try:
-            subprocess.run(setup_config_cmd, shell=True, check=True,
-                           stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
+            subprocess.run(
+                setup_config_cmd,
+                shell=True,
+                check=True,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+            )
         except Exception as e:
-            log.error(f"Could not copy \"{qleverfile_path}\""
-                      f" to current directory: {e}")
+            log.error(
+                f'Could not copy "{qleverfile_path}"' f" to current directory: {e}"
+            )
             return False
 
         # If we get here, everything went well.
-        log.info(f"Created Qleverfile for config \"{args.config_name}\""
-                 f" in current directory")
+        log.info(
+            f'Created Qleverfile for config "{args.config_name}"'
+            f" in current directory"
+        )
         return True
-
-        # if config_name == "default":
-        #     log.info("Since this is the default Qleverfile, you need to "
-        #              "edit it before you can continue")
-        #     log.info("")
-        #     log.info("Afterwards, run `qlever` without arguments to see "
-        #              "which actions are available")
-        # else:
-        #     show_available_action_names()
-        # log.info("")

qlever/commands/system_info.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import platform
+from importlib.metadata import version
+from pathlib import Path
+
+import psutil
+
+from qlever.command import QleverCommand
+from qlever.containerize import Containerize
+from qlever.log import log
+from qlever.util import format_size, run_command
+
+
+def show_heading(text: str) -> str:
+    log.info(text)
+    log.info("-" * len(text))
+    log.info("")
+
+
+def get_partition(dir: Path):
+    """
+    Returns the partition on which `dir` resides. May return None.
+    """
+    # The first partition that whose mountpoint is a parent of `dir` is
+    # returned. Sort the partitions by the length of the mountpoint to ensure
+    # that the result is correct. Assume there are partitions with mountpoint
+    # `/` and `/home`. This ensures that `/home/foo` is detected as being in
+    # the partition with mountpoint `/home`.
+    partitions = sorted(
+        psutil.disk_partitions(),
+        key=lambda part: len(part.mountpoint),
+        reverse=True,
+    )
+    for partition in partitions:
+        if dir.is_relative_to(partition.mountpoint):
+            return partition
+    return None
+
+
+class SystemInfoCommand(QleverCommand):
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Gather some system info to help with troubleshooting"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
+        return {"runtime": ["system", "image", "server_container"]}
+
+    def additional_arguments(self, subparser) -> None:
+        pass
+
+    def execute(self, args) -> bool:
+        # Say what the command is doing.
+        self.show("Show system information and Qleverfile", only_show=args.show)
+        if args.show:
+            return False
+
+        # Show system information.
+        show_heading("System Information")
+        system = platform.system()
+        is_linux = system == "Linux"
+        is_mac = system == "Darwin"
+        is_windows = system == "Windows"
+        if is_windows:
+            log.warn("Only limited information is gathered on Windows.")
+        log.info(f"Version: {version('qlever')} (qlever --version)")
+        if is_linux:
+            info = platform.freedesktop_os_release()
+            log.info(f"OS: {platform.system()} ({info['PRETTY_NAME']})")
+        else:
+            log.info(f"OS: {platform.system()}")
+        log.info(f"Arch: {platform.machine()}")
+        log.info(f"Host: {platform.node()}")
+        psutil.virtual_memory().total / (1000**3)
+        memory_total = psutil.virtual_memory().total / (1024.0**3)
+        memory_available = psutil.virtual_memory().available / (1024.0**3)
+        log.info(
+            f"RAM: {memory_total:.1f} GB total, " f"{memory_available:.1f} GB available"
+        )
+        num_cores = psutil.cpu_count(logical=False)
+        num_threads = psutil.cpu_count(logical=True)
+        cpu_freq = psutil.cpu_freq().max / 1000
+        log.info(
+            f"CPU: {num_cores} Cores, " f"{num_threads} Threads @ {cpu_freq:.2f} GHz"
+        )
+
+        cwd = Path.cwd()
+        log.info(f"CWD: {cwd}")
+        # Free and total size of the partition on which the current working
+        # directory resides.
+        disk_usage = psutil.disk_usage(str(cwd))
+        partition = get_partition(cwd)
+        partition_description = f"{partition.device} @ {partition.mountpoint}"
+        fs_type = partition.fstype
+        fs_free = format_size(disk_usage.free)
+        fs_total = format_size(disk_usage.total)
+        log.info(
+            f"Disk space in {partition_description} is "
+            f"({fs_type}): {fs_free} free / {fs_total} total"
+        )
+        # User/Group on host and in container
+        if is_linux or is_mac:
+            user_info = run_command("id", return_output=True).strip()
+            log.info(f"User and group on host: {user_info}")
+        elif is_windows:
+            user_info = run_command("whoami /all", return_output=True).strip()
+            log.info(f"User and group on host: {user_info}")
+        if args.system in Containerize.supported_systems():
+            user_info = Containerize.run_in_container("id", args).strip()
+            log.info(f"User and group in container: {user_info}")
+
+        # Show Qleverfile.
+        log.info("")
+        show_heading("Contents of Qleverfile")
+        qleverfile = cwd / "Qleverfile"
+        if qleverfile.exists():
+            # TODO: output the effective qlever file using primites from #57
+            log.info(qleverfile.read_text())
+        else:
+            log.info("No Qleverfile found")
+        return True

qlever/commands/ui.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import subprocess
+from os import environ
 
 from qlever.command import QleverCommand
 from qlever.containerize import Containerize
@@ -17,46 +18,70 @@ class UiCommand(QleverCommand):
         pass
 
     def description(self) -> str:
-        return ("Launch the QLever UI web application")
+        return "Launch the QLever UI web application"
 
     def should_have_qleverfile(self) -> bool:
         return True
 
-    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
-        return {"data": ["name"],
-                "server": ["host_name", "port"],
-                "ui": ["ui_port", "ui_config",
-                       "ui_system", "ui_image", "ui_container"]}
+    def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
+        return {
+            "data": ["name"],
+            "server": ["host_name", "port"],
+            "ui": ["ui_port", "ui_config", "ui_system", "ui_image", "ui_container"],
+        }
 
     def additional_arguments(self, subparser) -> None:
         pass
 
     def execute(self, args) -> bool:
+        # If QLEVER_OVERRIDE_DISABLE_UI is set, this command is disabled.
+        qlever_is_running_in_container = environ.get("QLEVER_IS_RUNNING_IN_CONTAINER")
+        if qlever_is_running_in_container:
+            log.error(
+                "The environment variable `QLEVER_OVERRIDE_DISABLE_UI` is set, "
+                "therefore `qlever ui` is not available (it should not be called "
+                "from inside a container)"
+            )
+            log.info("")
+            if not args.show:
+                log.info(
+                    "For your information, showing the commands that are "
+                    "executed when `qlever ui` is available:"
+                )
+                log.info("")
+
         # Construct commands and show them.
         server_url = f"http://{args.host_name}:{args.port}"
         ui_url = f"http://{args.host_name}:{args.ui_port}"
         pull_cmd = f"{args.ui_system} pull -q {args.ui_image}"
-        run_cmd = f"{args.ui_system} run -d " \
-                  f"--publish {args.ui_port}:7000 " \
-                  f"--name {args.ui_container} " \
-                  f"{args.ui_image}"
-        exec_cmd = f"{args.ui_system} exec -it " \
-                   f"{args.ui_container} " \
-                   f"bash -c \"python manage.py configure " \
-                   f"{args.ui_config} {server_url}\""
-        self.show("\n".join(["Stop running containers",
-                            pull_cmd, run_cmd, exec_cmd]), only_show=args.show)
-        if args.show:
+        run_cmd = (
+            f"{args.ui_system} run -d "
+            f"--publish {args.ui_port}:7000 "
+            f"--name {args.ui_container} "
+            f"{args.ui_image}"
+        )
+        exec_cmd = (
+            f"{args.ui_system} exec -it "
+            f"{args.ui_container} "
+            f'bash -c "python manage.py configure '
+            f'{args.ui_config} {server_url}"'
+        )
+        self.show(
+            "\n".join(["Stop running containers", pull_cmd, run_cmd, exec_cmd]),
+            only_show=args.show,
+        )
+        if qlever_is_running_in_container or args.show:
             return False
 
         # Stop running containers.
         for container_system in Containerize.supported_systems():
-            Containerize.stop_and_remove_container(
-                    container_system, args.ui_container)
+            Containerize.stop_and_remove_container(container_system, args.ui_container)
 
         # Check if the UI port is already being used.
         if is_port_used(args.ui_port):
-            log.warning(f"It looks like the specified port for the UI ({args.ui_port}) is already in use. You can set another port in the Qleverfile in the [ui] section with the UI_PORT variable.")
+            log.warning(
+                f"It looks like the specified port for the UI ({args.ui_port}) is already in use. You can set another port in the Qleverfile in the [ui] section with the UI_PORT variable."
+            )
 
         # Try to start the QLever UI.
         try:
@@ -68,7 +93,9 @@ class UiCommand(QleverCommand):
             return False
 
         # Success.
-        log.info(f"The QLever UI should now be up at {ui_url} ..."
-                 f"You can log in as QLever UI admin with username and "
-                 f"password \"demo\"")
+        log.info(
+            f"The QLever UI should now be up at {ui_url} ..."
+            f"You can log in as QLever UI admin with username and "
+            f'password "demo"'
+        )
         return True

qlever/containerize.py

@@ -9,7 +9,7 @@ import subprocess
 from typing import Optional
 
 from qlever.log import log
-from qlever.util import run_command
+from qlever.util import run_command, get_random_string
 
 
 class ContainerizeException(Exception):
@@ -31,12 +31,16 @@ class Containerize:
         return ["docker", "podman"]
 
     @staticmethod
-    def containerize_command(cmd: str, container_system: str,
-                             run_subcommand: str,
-                             image_name: str, container_name: str,
-                             volumes: list[tuple[str, str]] = [],
-                             ports: list[tuple[int, int]] = [],
-                             working_directory: Optional[str] = None) -> str:
+    def containerize_command(
+        cmd: str,
+        container_system: str,
+        run_subcommand: str,
+        image_name: str,
+        container_name: str,
+        volumes: list[tuple[str, str]] = [],
+        ports: list[tuple[int, int]] = [],
+        working_directory: Optional[str] = None,
+    ) -> str:
         """
         Get the command to run `cmd` with the given `container_system` and the
         given options.
@@ -45,8 +49,9 @@ class Containerize:
         # Check that `container_system` is supported.
         if container_system not in Containerize.supported_systems():
             return ContainerizeException(
-                    f"Invalid container system \"{container_system}\""
-                    f" (must be one of {Containerize.supported_systems()})")
+                f'Invalid container system "{container_system}"'
+                f" (must be one of {Containerize.supported_systems()})"
+            )
 
         # Set user and group ids. This is important so that the files created
         # by the containerized command are owned by the user running the
@@ -62,37 +67,40 @@ class Containerize:
         # dir.
         volume_options = "".join([f" -v {v1}:{v2}" for v1, v2 in volumes])
         port_options = "".join([f" -p {p1}:{p2}" for p1, p2 in ports])
-        working_directory_option = (f" -w {working_directory}"
-                                    if working_directory is not None else "")
+        working_directory_option = (
+            f" -w {working_directory}" if working_directory is not None else ""
+        )
 
         # Construct the command that runs `cmd` with the given container
         # system.
-        containerized_cmd = (f"{container_system} {run_subcommand}"
-                             f"{user_option}"
-                             f" -v /etc/localtime:/etc/localtime:ro"
-                             f"{volume_options}"
-                             f"{port_options}"
-                             f"{working_directory_option}"
-                             f" --init"
-                             f" --entrypoint bash"
-                             f" --name {container_name} {image_name}"
-                             f" -c {shlex.quote(cmd)}")
+        containerized_cmd = (
+            f"{container_system} {run_subcommand}"
+            f"{user_option}"
+            f" -v /etc/localtime:/etc/localtime:ro"
+            f"{volume_options}"
+            f"{port_options}"
+            f"{working_directory_option}"
+            f" --init"
+            f" --entrypoint bash"
+            f" --name {container_name} {image_name}"
+            f" -c {shlex.quote(cmd)}"
+        )
         return containerized_cmd
 
     @staticmethod
     def is_running(container_system: str, container_name: str) -> bool:
         # Note: the `{{{{` and `}}}}` result in `{{` and `}}`, respectively.
         containers = (
-            run_command(f"{container_system} ps --format=\"{{{{.Names}}}}\"",
-                        return_output=True)
+            run_command(
+                f'{container_system} ps --format="{{{{.Names}}}}"', return_output=True
+            )
             .strip()
             .splitlines()
         )
         return container_name in containers
 
     @staticmethod
-    def stop_and_remove_container(container_system: str,
-                                  container_name: str) -> bool:
+    def stop_and_remove_container(container_system: str, container_name: str) -> bool:
         """
         Stop the container with the given name using the given system. Return
         `True` if a container with that name was found and stopped, `False`
@@ -102,19 +110,45 @@ class Containerize:
         # Check that `container_system` is supported.
         if container_system not in Containerize.supported_systems():
             return ContainerizeException(
-                    f"Invalid container system \"{container_system}\""
-                    f" (must be one of {Containerize.supported_systems()})")
+                f'Invalid container system "{container_system}"'
+                f" (must be one of {Containerize.supported_systems()})"
+            )
 
         # Construct the command that stops the container.
-        stop_cmd = f"{container_system} stop {container_name} && " \
-                   f"{container_system} rm {container_name}"
+        stop_cmd = (
+            f"{container_system} stop {container_name} && "
+            f"{container_system} rm {container_name}"
+        )
 
         # Run the command.
         try:
-            subprocess.run(stop_cmd, shell=True, check=True,
-                           stdout=subprocess.DEVNULL,
-                           stderr=subprocess.DEVNULL)
+            subprocess.run(
+                stop_cmd,
+                shell=True,
+                check=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
             return True
         except Exception as e:
-            log.debug(f"Error running \"{stop_cmd}\": {e}")
+            log.debug(f'Error running "{stop_cmd}": {e}')
             return False
+
+    @staticmethod
+    def run_in_container(cmd: str, args) -> Optional[str]:
+        """
+        Run an arbitrary command in the qlever container and return its output.
+        """
+        if args.system in Containerize.supported_systems():
+            if not args.server_container:
+                args.server_container = get_random_string(20)
+            run_cmd = Containerize().containerize_command(
+                cmd,
+                args.system,
+                'run --rm -it --entrypoint "" ',
+                args.image,
+                args.server_container,
+                volumes=[("$(pwd)", "/index")],
+                working_directory="/index",
+            )
+            return run_command(run_cmd, return_output=True)

qlever/qleverfile.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import re
-import socket
 import subprocess
 from configparser import ConfigParser, ExtendedInterpolation
 
@@ -63,8 +62,16 @@ class Qleverfile:
                 help="A space-separated list of patterns that match "
                      "all the files of the dataset")
         index_args["cat_input_files"] = arg(
-                "--cat-input-files", type=str, required=True,
+                "--cat-input-files", type=str,
                 help="The command that produces the input")
+        index_args["multi_input_json"] = arg(
+                "--multi-input-json", type=str, default=None,
+                help="JSON to specify multiple input files, each with a "
+                "`cmd` (command that writes the triples to stdout), "
+                "`format` (format like for the `--format` option), "
+                "`graph` (name of the graph, use `-` for the default graph), "
+                "`parallel` (parallel parsing for large files, where all "
+                "prefix declaration are at the beginning)")
         index_args["settings_json"] = arg(
                 "--settings-json", type=str, default="{}",
                 help="The `.settings.json` file for the index")
@@ -106,7 +113,7 @@ class Qleverfile:
                 help="The binary for starting the server (this requires "
                      "that you have compiled QLever on your machine)")
         server_args["host_name"] = arg(
-                "--host-name", type=str, default=f"localhost",
+                "--host-name", type=str, default="localhost",
                 help="The name of the host on which the server listens for "
                      "requests")
         server_args["port"] = arg(