qlever 0.2.5__py3-none-any.whl → 0.5.41__py3-none-any.whl

qlever/commands/setup_config.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import subprocess
+from os import environ
+from pathlib import Path
+
+from qlever.command import QleverCommand
+from qlever.log import log
+from qlever.util import get_random_string
+
+
+class SetupConfigCommand(QleverCommand):
+    """
+    Class for executing the `setup-config` command.
+    """
+
+    def __init__(self):
+        self.qleverfiles_path = Path(__file__).parent.parent / "Qleverfiles"
+        self.qleverfile_names = [
+            p.name.split(".")[1] for p in self.qleverfiles_path.glob("Qleverfile.*")
+        ]
+
+    def description(self) -> str:
+        return "Get a pre-configured Qleverfile"
+
+    def should_have_qleverfile(self) -> bool:
+        return False
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "config_name",
+            type=str,
+            choices=self.qleverfile_names,
+            help="The name of the pre-configured Qleverfile to create",
+        )
+
+    def execute(self, args) -> bool:
+        # Show a warning if `QLEVER_OVERRIDE_SYSTEM_NATIVE` is set.
+        qlever_is_running_in_container = environ.get("QLEVER_IS_RUNNING_IN_CONTAINER")
+        if qlever_is_running_in_container:
+            log.warning(
+                "The environment variable `QLEVER_IS_RUNNING_IN_CONTAINER` is set, "
+                "therefore the Qleverfile is modified to use `SYSTEM = native` "
+                "(since inside the container, QLever should run natively)"
+            )
+            log.info("")
+        # Construct the command line and show it.
+        qleverfile_path = self.qleverfiles_path / f"Qleverfile.{args.config_name}"
+        setup_config_cmd = (
+            f"cat {qleverfile_path}"
+            f" | sed -E 's/(^ACCESS_TOKEN.*)/\\1_{get_random_string(12)}/'"
+        )
+        if qlever_is_running_in_container:
+            setup_config_cmd += (
+                " | sed -E 's/(^SYSTEM[[:space:]]*=[[:space:]]*).*/\\1native/'"
+            )
+        setup_config_cmd += "> Qleverfile"
+        self.show(setup_config_cmd, only_show=args.show)
+        if args.show:
+            return True
+
+        # If there is already a Qleverfile in the current directory, exit.
+        qleverfile_path = Path("Qleverfile")
+        if qleverfile_path.exists():
+            log.error("`Qleverfile` already exists in current directory")
+            log.info("")
+            log.info(
+                "If you want to create a new Qleverfile using "
+                "`qlever setup-config`, delete the existing Qleverfile "
+                "first"
+            )
+            return False
+
+        # Copy the Qleverfile to the current directory.
+        try:
+            subprocess.run(
+                setup_config_cmd,
+                shell=True,
+                check=True,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+            )
+        except Exception as e:
+            log.error(
+                f'Could not copy "{qleverfile_path}"' f" to current directory: {e}"
+            )
+            return False
+
+        # If we get here, everything went well.
+        log.info(
+            f'Created Qleverfile for config "{args.config_name}"'
+            f" in current directory"
+        )
+        return True

qlever/commands/start.py

@@ -0,0 +1,336 @@
+from __future__ import annotations
+
+import subprocess
+import time
+
+from qlever.command import QleverCommand
+from qlever.commands.cache_stats import CacheStatsCommand
+from qlever.commands.settings import SettingsCommand
+from qlever.commands.status import StatusCommand
+from qlever.commands.stop import StopCommand
+from qlever.commands.warmup import WarmupCommand
+from qlever.containerize import Containerize
+from qlever.log import log
+from qlever.qleverfile import Qleverfile
+from qlever.util import binary_exists, is_qlever_server_alive, run_command
+
+
+# Construct the command line based on the config file.
+def construct_command(args) -> str:
+    start_cmd = (
+        f"{args.server_binary}"
+        f" -i {args.name}"
+        f" -j {args.num_threads}"
+        f" -p {args.port}"
+        f" -m {args.memory_for_queries}"
+        f" -c {args.cache_max_size}"
+        f" -e {args.cache_max_size_single_entry}"
+        f" -k {args.cache_max_num_entries}"
+    )
+
+    if args.timeout:
+        start_cmd += f" -s {args.timeout}"
+    if args.access_token:
+        start_cmd += f" -a {args.access_token}"
+    if args.persist_updates:
+        start_cmd += " --persist-updates"
+    if args.only_pso_and_pos_permutations:
+        start_cmd += " --only-pso-and-pos-permutations"
+    if args.use_patterns == "no":
+        start_cmd += " --no-patterns"
+    if args.use_text_index == "yes":
+        start_cmd += " -t"
+    start_cmd += f" > {args.name}.server-log.txt 2>&1"
+    return start_cmd
+
+
+# Kill existing server on the same port. Trust that StopCommand() works?
+# Maybe return StopCommand().execute(args) and handle it with a try except?
+def kill_existing_server(args) -> bool:
+    args.cmdline_regex = f"^ServerMain.* -p {args.port}"
+    args.no_containers = True
+    if not StopCommand().execute(args):
+        log.error("Stopping the existing server failed")
+        return False
+    log.info("")
+    return True
+
+
+# Run the command in a container
+def wrap_command_in_container(args, start_cmd) -> str:
+    if not args.server_container:
+        args.server_container = f"qlever.server.{args.name}"
+    start_cmd = Containerize().containerize_command(
+        start_cmd,
+        args.system,
+        "run -d --restart=unless-stopped",
+        args.image,
+        args.server_container,
+        volumes=[("$(pwd)", "/index")],
+        ports=[(args.port, args.port)],
+        working_directory="/index",
+    )
+    return start_cmd
+
+
+# Set the index description.
+def set_index_description(access_arg, port, desc) -> bool:
+    curl_cmd = (
+        f"curl -Gs http://localhost:{port}/api"
+        f' --data-urlencode "index-description={desc}"'
+        f" {access_arg} > /dev/null"
+    )
+    log.debug(curl_cmd)
+    try:
+        run_command(curl_cmd)
+    except Exception as e:
+        log.error(f"Setting the index description failed ({e})")
+        return False
+    return True
+
+
+# Set the text description.
+def set_text_description(access_arg, port, text_desc) -> bool:
+    curl_cmd = (
+        f"curl -Gs http://localhost:{port}/api"
+        f' --data-urlencode "text-description={text_desc}"'
+        f" {access_arg} > /dev/null"
+    )
+    log.debug(curl_cmd)
+    try:
+        run_command(curl_cmd)
+    except Exception as e:
+        log.error(f"Setting the text description failed ({e})")
+        return False
+    return True
+
+
+class StartCommand(QleverCommand):
+    """
+    Class for executing the `start` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return (
+            "Start the QLever server (requires that you have built "
+            "an index with `qlever index` before)"
+        )
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {
+            "data": ["name", "description", "text_description"],
+            "server": [
+                "server_binary",
+                "host_name",
+                "port",
+                "access_token",
+                "memory_for_queries",
+                "cache_max_size",
+                "cache_max_size_single_entry",
+                "cache_max_num_entries",
+                "num_threads",
+                "timeout",
+                "persist_updates",
+                "only_pso_and_pos_permutations",
+                "use_patterns",
+                "use_text_index",
+                "warmup_cmd",
+            ],
+            "runtime": ["system", "image", "server_container"],
+        }
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "--kill-existing-with-same-port",
+            action="store_true",
+            default=False,
+            help="If a QLever server is already running "
+            "on the same port, kill it before "
+            "starting a new server",
+        )
+        subparser.add_argument(
+            "--no-warmup",
+            action="store_true",
+            default=False,
+            help="Do not execute the warmup command",
+        )
+        subparser.add_argument(
+            "--run-in-foreground",
+            action="store_true",
+            default=False,
+            help="Run the server in the foreground "
+            "(default: run in the background with `nohup`)",
+        )
+        subparser.add_argument(
+            "runtime_parameters",
+            nargs="*",
+            help="Space-separated list of runtime parameters to set "
+            "(in the form `key=value`) once the server is running",
+        ).completer = lambda **kwargs: [
+            f"{key}=" for key in Qleverfile.SERVER_RUNTIME_PARAMETERS
+        ]
+
+    def execute(self, args) -> bool:
+        # Set the endpoint URL.
+        args.endpoint_url = f"http://{args.host_name}:{args.port}"
+
+        # Kill existing server with the same name if so desired.
+        #
+        # TODO: This is currently disabled because I never used it once over
+        # the past weeks and it is not clear to me what the use case is.
+        if False:  # or args.kill_existing_with_same_name:
+            args.cmdline_regex = f"^ServerMain.* -i {args.name}"
+            args.no_containers = True
+            StopCommand().execute(args)
+            log.info("")
+
+        # Kill existing server on the same port if so desired.
+        if args.kill_existing_with_same_port:
+            if args.kill_existing_with_same_port and not kill_existing_server(
+                args
+            ):
+                return False
+
+        # Construct the command line based on the config file.
+        start_cmd = construct_command(args)
+
+        # Run the command in a container (if so desired). Otherwise run with
+        # `nohup` so that it keeps running after the shell is closed. With
+        # `--run-in-foreground`, run the server in the foreground.
+        if args.system in Containerize.supported_systems():
+            start_cmd = wrap_command_in_container(args, start_cmd)
+        elif args.run_in_foreground:
+            start_cmd = f"{start_cmd}"
+        else:
+            start_cmd = f"nohup {start_cmd} &"
+
+        # Show the command line.
+        self.show(start_cmd, only_show=args.show)
+        if args.show:
+            if args.runtime_parameters:
+                log.info("")
+                SettingsCommand().execute(args)
+            return True
+
+        # When running natively, check if the binary exists and works.
+        if args.system == "native":
+            if not binary_exists(args.server_binary, "server-binary"):
+                return False
+
+        # Check if a QLever server is already running on this port.
+        if is_qlever_server_alive(args.endpoint_url):
+            log.error(f"QLever server already running on {args.endpoint_url}")
+            log.info("")
+            log.info(
+                "To kill the existing server, use `qlever stop` "
+                "or `qlever start` with option "
+                "--kill-existing-with-same-port`"
+            )
+
+            # Show output of status command.
+            args.cmdline_regex = f"^ServerMain.* -p *{args.port}"
+            log.info("")
+            StatusCommand().execute(args)
+            return False
+
+        # Remove already existing container.
+        if (
+            args.system in Containerize.supported_systems()
+            and args.kill_existing_with_same_port
+        ):
+            try:
+                run_command(f"{args.system} rm -f {args.server_container}")
+            except Exception as e:
+                log.error(f"Removing existing container failed: {e}")
+                return False
+
+        # Check if another process is already listening.
+        # if self.net_connections_enabled:
+        #     if port in [conn.laddr.port for conn
+        #                 in psutil.net_connections()]:
+        #         log.error(f"Port {port} is already in use by another process"
+        #                   f" (use `lsof -i :{port}` to find out which one)")
+        #         return False
+
+        # Execute the command line.
+        try:
+            process = run_command(
+                start_cmd,
+                use_popen=args.run_in_foreground,
+            )
+        except Exception as e:
+            log.error(f"Starting the QLever server failed ({e})")
+            return False
+
+        # Tail the server log until the server is ready (note that the `exec`
+        # is important to make sure that the tail process is killed and not
+        # just the bash process).
+        if args.run_in_foreground:
+            log.info(
+                f"Follow {args.name}.server-log.txt as long as the server is"
+                f" running (Ctrl-C stops the server)"
+            )
+        else:
+            log.info(
+                f"Follow {args.name}.server-log.txt until the server is ready"
+                f" (Ctrl-C stops following the log, but NOT the server)"
+            )
+        log.info("")
+        tail_cmd = f"exec tail -f {args.name}.server-log.txt"
+        tail_proc = subprocess.Popen(tail_cmd, shell=True)
+        while not is_qlever_server_alive(args.endpoint_url):
+            time.sleep(1)
+
+        # Set the description for the index and text.
+        access_arg = f'--data-urlencode "access-token={args.access_token}"'
+        if args.description:
+            ret = set_index_description(
+                access_arg, args.port, args.description
+            )
+            if not ret:
+                return False
+        if args.text_description:
+            ret = set_text_description(
+                access_arg, args.port, args.text_description
+            )
+            if not ret:
+                return False
+
+        # Kill the tail process. NOTE: `tail_proc.kill()` does not work.
+        if not args.run_in_foreground:
+            tail_proc.terminate()
+
+        # Execute the warmup command.
+        if args.warmup_cmd and not args.no_warmup:
+            log.info("")
+            if not WarmupCommand().execute(args):
+                log.error("Warmup failed")
+                return False
+
+        # Show cache stats.
+        if not args.run_in_foreground:
+            log.info("")
+            args.detailed = False
+            args.sparql_endpoint = None
+            CacheStatsCommand().execute(args)
+
+        # Apply settings if any.
+        if args.runtime_parameters:
+            log.info("")
+            SettingsCommand().execute(args)
+
+        # With `--run-in-foreground`, wait until the server is stopped.
+        if args.run_in_foreground:
+            try:
+                process.wait()
+            except KeyboardInterrupt:
+                process.terminate()
+            tail_proc.terminate()
+
+        return True

qlever/commands/status.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import psutil
+
+from qlever.command import QleverCommand
+from qlever.util import show_process_info
+
+
+class StatusCommand(QleverCommand):
+    """
+    Class for executing the `status` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return ("Show QLever processes running on this machine")
+
+    def should_have_qleverfile(self) -> bool:
+        return False
+
+    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
+        return {}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument("--cmdline-regex",
+                               default="^(ServerMain|IndexBuilderMain)",
+                               help="Show only processes where the command "
+                                    "line matches this regex")
+
+    def execute(self, args) -> bool:
+        # Show action description.
+        self.show(f"Show all processes on this machine where "
+                  f"the command line matches {args.cmdline_regex}"
+                  f" using Python's psutil library", only_show=args.show)
+        if args.show:
+            return True
+
+        # Show the results as a table.
+        num_processes_found = 0
+        for proc in psutil.process_iter():
+            show_heading = num_processes_found == 0
+            process_shown = show_process_info(proc, args.cmdline_regex,
+                                              show_heading=show_heading)
+            if process_shown:
+                num_processes_found += 1
+        if num_processes_found == 0:
+            print("No processes found")
+        return True

qlever/commands/stop.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+from qlever.command import QleverCommand
+from qlever.commands.status import StatusCommand
+from qlever.containerize import Containerize
+from qlever.log import log
+from qlever.util import stop_process_with_regex
+
+
+def stop_container(server_container: str) -> bool:
+    """
+    Try to stop and remove container. return True iff it was stopped
+    successfully. Gives log info accordingly.
+    """
+    for container_system in Containerize.supported_systems():
+        if Containerize.stop_and_remove_container(
+            container_system, server_container
+        ):
+            log.info(
+                f"{container_system.capitalize()} container with "
+                f'name "{server_container}" stopped and removed'
+            )
+            return True
+    return False
+
+
+class StopCommand(QleverCommand):
+    """
+    Class for executing the `stop` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Stop QLever server for a given dataset or port"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
+        return {"data": ["name"],
+                "server": ["port"],
+                "runtime": ["server_container"]}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument("--cmdline-regex",
+                               default="ServerMain.* -i [^ ]*%%NAME%%",
+                               help="Show only processes where the command "
+                                    "line matches this regex")
+        subparser.add_argument("--no-containers", action="store_true",
+                               default=False,
+                               help="Do not look for containers, only for "
+                                    "native processes")
+
+    def execute(self, args) -> bool:
+        # Show action description.
+        cmdline_regex = args.cmdline_regex.replace("%%NAME%%", args.name)
+        description = f"Checking for processes matching \"{cmdline_regex}\""
+        if not args.no_containers:
+            description += (f" and for Docker container with name "
+                            f"\"{args.server_container}\"")
+        self.show(description, only_show=args.show)
+        if args.show:
+            return True
+
+        # First check if there is container running and if yes, stop and remove
+        # it (unless the user has specified `--no-containers`).
+        if not args.no_containers:
+            if stop_container(args.server_container):
+                return True
+
+        # Check if there is a process running on the server port using psutil.
+        # NOTE: On MacOS, some of the proc's returned by psutil.process_iter()
+        # no longer exist when we try to access them, so we just skip them.
+        stop_process_results = stop_process_with_regex(cmdline_regex)
+        if stop_process_results is None:
+            return False
+        if len(stop_process_results) > 0:
+            return all(stop_process_results)
+
+        # If no matching process found, show a message and the output of the
+        # status command.
+        message = "No matching process found" if args.no_containers else \
+            "No matching process or container found"
+        log.error(message)
+        args.cmdline_regex = "^ServerMain.* -i [^ ]*"
+        log.info("")
+        StatusCommand().execute(args)
+        return True