qlever 0.2.5__py3-none-any.whl → 0.5.41__py3-none-any.whl

qlever/commands/log.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import subprocess
+
+from qlever.command import QleverCommand
+from qlever.log import log
+
+
+class LogCommand(QleverCommand):
+    """
+    Class for executing the `log` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return ("Show the last lines of the server log file and follow it")
+
+    def should_have_qleverfile(self) -> bool:
+        return False
+
+    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
+        return {"data": ["name"]}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument("--tail-num-lines", type=int, default=20,
+                               help="Show this many of the last lines of the "
+                                    "log file")
+        subparser.add_argument("--from-beginning", action="store_true",
+                               default=False,
+                               help="Show all lines of the log file")
+        subparser.add_argument("--no-follow", action="store_true",
+                               default=False,
+                               help="Don't follow the log file")
+
+    def execute(self, args) -> bool:
+        # Construct the command and show it.
+        log_cmd = "tail"
+        if args.from_beginning:
+            log_cmd += " -n +1"
+        else:
+            log_cmd += f" -n {args.tail_num_lines}"
+        if not args.no_follow:
+            log_cmd += " -f"
+        log_file = f"{args.name}.server-log.txt"
+        log_cmd += f" {log_file}"
+        self.show(log_cmd, only_show=args.show)
+        if args.show:
+            return True
+
+        # Execute the command.
+        log.info(f"Follow log file {log_file}, press Ctrl-C to stop"
+                 f" following (will not stop the server)")
+        log.info("")
+        try:
+            subprocess.run(log_cmd, shell=True)
+            return True
+        except Exception as e:
+            log.error(e)
+            return False
+
+
+
+
+

qlever/commands/materialized_view.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import json
+import re
+import shlex
+import time
+
+from qlever.command import QleverCommand
+from qlever.log import log
+from qlever.util import (
+    run_command,
+)
+
+
+class MaterializedViewCommand(QleverCommand):
+    """
+    Class for executing the `materialized-view` command.
+    """
+
+    def __init__(self):
+        self.materialized_view_name_regex = r"^[A-Za-z0-9-]+$"
+        pass
+
+    def description(self) -> str:
+        return "Create a materialized view from the given query"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {
+            "data": ["name"],
+            "server": ["host_name", "port", "access_token"],
+        }
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "view_name",
+            type=str,
+            help="Name of the materialized view",
+        )
+        subparser.add_argument(
+            "view_query",
+            type=str,
+            help="SPARQL query from which to create the materialized view",
+        )
+        subparser.add_argument(
+            "--sparql-endpoint",
+            type=str,
+            help="URL of the SPARQL endpoint (default: <host_name>:<port>)",
+        )
+
+    def execute(self, args) -> bool:
+        # SPARQL endpoint to use.
+        sparql_endpoint = (
+            args.sparql_endpoint
+            if args.sparql_endpoint is not None
+            else f"{args.host_name}:{args.port}"
+        )
+
+        # Check that the name of the materialized view is valid.
+        if not re.match(self.materialized_view_name_regex, args.view_name):
+            log.error(
+                f"The name for the materialized view must match "
+                f"the regex {self.materialized_view_name_regex}"
+            )
+            return False
+
+        # Command for building the materialized view.
+        url = (
+            f"{sparql_endpoint}"
+            f"?cmd=write-materialized-view"
+            f"&view-name={args.view_name}"
+        )
+        materialized_view_cmd = (
+            f"curl -s {shlex.quote(url)} "
+            f"-H 'Authorization: Bearer {args.access_token}' "
+            f"-H 'Content-type: application/sparql-query' "
+            f"-d {shlex.quote(args.view_query)}"
+        )
+        self.show(materialized_view_cmd, only_show=args.show)
+        if args.show:
+            return True
+
+        # Run the command (and time it).
+        time_start = time.monotonic()
+        try:
+            log.info("Creating the materialized view ... "
+                     "(this may take a while, depending on the complexity "
+                     "of the query and the size of the result)")
+            log.info("")
+            result = run_command(materialized_view_cmd, return_output=True)
+        except Exception as e:
+            log.error(f"Creating the materialized view failed: {e}")
+            return False
+        time_end = time.monotonic()
+        duration_seconds = round(time_end - time_start)
+
+        # Try to parse the result (should be JSON).
+        try:
+            result_json = json.loads(result)
+            view_name = result_json.get("materialized-view-written")
+            log.info(
+                f"Materialized view '{view_name}' created successfully "
+                f"in {duration_seconds:,} seconds"
+            )
+        except Exception as e:
+            log.error(f'Failed to parse JSON from "{result}": {e}')
+
+        return True

qlever/commands/query.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import shlex
+import time
+import traceback
+
+from qlever.command import QleverCommand
+from qlever.log import log
+from qlever.util import run_command
+
+
+class QueryCommand(QleverCommand):
+    """
+    Class for executing the `query` command.
+    """
+
+    def __init__(self):
+        self.predefined_queries = {
+            "all-predicates": (
+                "SELECT (?p AS ?predicate) (COUNT(?p) AS ?count) "
+                "WHERE { ?s ?p ?o } "
+                "GROUP BY ?p ORDER BY DESC(?count)"
+            ),
+            "all-graphs": (
+                "SELECT ?g (COUNT(?g) AS ?count) "
+                "WHERE { GRAPH ?g { ?s ?p ?o } } "
+                "GROUP BY ?g ORDER BY DESC(?count)"
+            ),
+            "ten-random-triples": (
+                "SELECT * WHERE { ?s ?p ?o } ORDER BY RAND() LIMIT 10"
+            ),
+        }
+
+    def description(self) -> str:
+        return "Send a query to a SPARQL endpoint"
+
+    def should_have_qleverfile(self) -> bool:
+        return False
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {"server": ["host_name", "port", "access_token"]}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "query",
+            type=str,
+            nargs="?",
+            default="SELECT * WHERE { ?s ?p ?o } LIMIT 10",
+            help="SPARQL query to send",
+        )
+        subparser.add_argument(
+            "--predefined-query",
+            type=str,
+            choices=self.predefined_queries.keys(),
+            help="Use a predefined query",
+        )
+        subparser.add_argument(
+            "--pin-to-cache",
+            action="store_true",
+            default=False,
+            help="Pin the query to the cache",
+        )
+        subparser.add_argument(
+            "--sparql-endpoint", type=str, help="URL of the SPARQL endpoint"
+        )
+        subparser.add_argument(
+            "--accept",
+            type=str,
+            choices=[
+                "text/tab-separated-values",
+                "text/csv",
+                "application/sparql-results+json",
+                "application/sparql-results+xml",
+                "application/qlever-results+json",
+                "application/octet-stream",
+            ],
+            default="text/tab-separated-values",
+            help="Accept header for the SPARQL query",
+        )
+        subparser.add_argument(
+            "--no-time",
+            action="store_true",
+            default=False,
+            help="Do not print the (end-to-end) time taken",
+        )
+
+    def execute(self, args) -> bool:
+        # Use a predefined query if requested.
+        if args.predefined_query:
+            args.query = self.predefined_queries[args.predefined_query]
+
+        # When pinning to the cache, set `send=0` and request media type
+        # `application/qlever-results+json` so that we get the result size.
+        # Also, we need to provide the access token.
+        if args.pin_to_cache:
+            args.accept = "application/qlever-results+json"
+            curl_cmd_additions = (
+                f" --data pin-result=true --data send=0"
+                f" --data access-token="
+                f"{shlex.quote(args.access_token)}"
+                f" | jq .resultsize | numfmt --grouping"
+                f" | xargs -I {{}} printf"
+                f' "Result pinned to cache,'
+                f' number of rows: {{}}\\n"'
+            )
+        else:
+            curl_cmd_additions = ""
+
+        # Show what the command will do.
+        sparql_endpoint = (
+            args.sparql_endpoint
+            if args.sparql_endpoint
+            else f"{args.host_name}:{args.port}"
+        )
+        curl_cmd = (
+            f"curl -s {sparql_endpoint}"
+            f' -H "Accept: {args.accept}"'
+            f" --data-urlencode query={shlex.quote(args.query)}"
+            f"{curl_cmd_additions}"
+        )
+        self.show(curl_cmd, only_show=args.show)
+        if args.show:
+            return True
+
+        # Launch query.
+        try:
+            start_time = time.time()
+            run_command(curl_cmd, show_output=True)
+            time_msecs = round(1000 * (time.time() - start_time))
+            if not args.no_time and args.log_level != "NO_LOG":
+                log.info("")
+                log.info(
+                    f"Query processing time (end-to-end):"
+                    f" {time_msecs:,d} ms"
+                )
+        except Exception as e:
+            if args.log_level == "DEBUG":
+                traceback.print_exc()
+            log.error(e)
+            return False
+
+        return True

qlever/commands/rebuild_index.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+import subprocess
+import time
+from pathlib import Path
+
+from termcolor import colored
+
+from qlever.command import QleverCommand
+from qlever.log import log
+from qlever.util import (
+    run_command,
+)
+
+
+class RebuildIndexCommand(QleverCommand):
+    """
+    Class for executing the `rebuild-index` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Rebuild the index from the current data (including updates)"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {
+            "data": ["name"],
+            "server": ["host_name", "port", "access_token"],
+            "runtime": ["server_container"],
+        }
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "--index-dir",
+            type=str,
+            help="Directory for the new index (default: subdirectory "
+            "`rebuild.YYYY-MM-DDTHH:MM` of the current directory)",
+        )
+        subparser.add_argument(
+            "--index-name",
+            type=str,
+            help="Base name of the new index (default: use the same as the "
+            "current index)",
+        )
+        subparser.add_argument(
+            "--restart-when-finished",
+            action="store_true",
+            default=False,
+            help="When the rebuild is finished, stop the server with the old "
+            "index and start it again with the new index",
+        )
+
+    def execute(self, args) -> bool:
+        # Default values for arguments.
+        if args.index_name is None:
+            args.index_name = args.name
+        if args.index_dir is None:
+            timestamp = time.strftime("%Y-%m-%dT%H:%M", time.localtime())
+            args.index_dir = f"rebuild.{timestamp}"
+        if args.index_dir.endswith("/"):
+            args.index_dir = args.index_dir[:-1]
+
+        # Check that the index directory either does not exist or is empty.
+        index_path = Path(args.index_dir)
+        if index_path.exists() and any(index_path.iterdir()):
+            log.error(
+                f"The specified index directory '{args.index_dir}' already "
+                "exists and is not empty; please specify an empty or "
+                "non-existing directory"
+            )
+            return False
+
+        # Split `index_dir` into path and dir name. For example, if `index_dir`
+        # is `path/to/index`, then the path is `path/to` and the dir name
+        # is `index`.
+        #
+        # NOTE: We keep this separate because we can always create a
+        # subdirectory in the current directory (even when running in a
+        # container), but not necessarily a directory at an arbitrary path. If
+        # a path outside the current directory is desired, we move the index
+        # there after it has been built.
+        index_dir_path = str(Path(args.index_dir).parent)
+        index_dir_name = str(Path(args.index_dir).name)
+        log_file_name = f"{args.index_name}.rebuild-index-log.txt"
+
+        # Command for rebuilding the index.
+        mkdir_cmd = (
+            f"mkdir -p {index_dir_name} && "
+            f"> {index_dir_name}/{log_file_name} && "
+            f"cp -a Qleverfile {index_dir_name}"
+        )
+        rebuild_index_cmd = (
+            f"curl -s {args.host_name}:{args.port} "
+            f"-d cmd=rebuild-index "
+            f"-d index-name={index_dir_name}/{args.index_name} "
+            f"-d access-token={args.access_token}"
+        )
+        move_index_cmd = f"mv {index_dir_name} {index_dir_path}"
+        restart_server_cmd = (
+            f"cd {args.index_dir} && "
+            f"qlever start --kill-existing-with-same-port"
+        )
+
+        # Show the command lines.
+        cmds_to_show = [mkdir_cmd, rebuild_index_cmd]
+        if index_dir_path != ".":
+            cmds_to_show.append(move_index_cmd)
+        if args.restart_when_finished:
+            cmds_to_show.append(restart_server_cmd)
+        self.show("\n".join(cmds_to_show), only_show=args.show)
+        if args.show:
+            return True
+
+        # Create the index directory and the log file.
+        try:
+            run_command(mkdir_cmd)
+        except Exception as e:
+            log.error(f"Creating the index directory failed: {e}")
+            return False
+
+        # Show the server log while rebuilding the index.
+        #
+        # NOTE: This will only work satisfactorily when no other queries are
+        # being processed at the same time. It would be better if QLever
+        # logged the rebuild-index output to a separate log file.
+        tail_cmd = f"exec tail -n 0 -f {index_dir_name}/{log_file_name}"
+        tail_proc = subprocess.Popen(tail_cmd, shell=True)
+
+        # Run the index rebuild command (and time it).
+        try:
+            time_start = time.monotonic()
+            try:
+                run_command(rebuild_index_cmd, show_output=False)
+            except Exception as e:
+                log.error(f"Rebuilding the index failed: {e}")
+                return False
+            time_end = time.monotonic()
+            duration_seconds = round(time_end - time_start)
+            log.info("")
+            rebuild_done_msg = f"Rebuilt index in {duration_seconds:,} seconds"
+            if index_dir_path == ".":
+                rebuild_done_msg += (
+                    f", in the new directory '{args.index_dir}'"
+                )
+            log.info(rebuild_done_msg)
+        finally:
+            tail_proc.terminate()
+            tail_proc.wait()
+
+        # Move the new index to the specified directory, if needed.
+        if index_dir_path != ".":
+            try:
+                log.info(f"Moving the new index to {args.index_dir}")
+                run_command(move_index_cmd)
+            except Exception as e:
+                log.error(f"Moving the new index failed: {e}")
+                return False
+
+        # Restart the server with the new index, if requested.
+        if args.restart_when_finished:
+            try:
+                log.info("Restarting the server with the new index ...")
+                log.info("")
+                log.info(colored("Command: start", attrs=["bold"]))
+                log.info("")
+                run_command(restart_server_cmd, show_output=True)
+            except Exception as e:
+                log.error(f"Restarting the server failed: {e}")
+                return False
+
+        return True

qlever/commands/reset_updates.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import re
+
+from qlever.command import QleverCommand
+from qlever.log import log
+from qlever.util import run_command
+
+
+class ResetUpdatesCommand(QleverCommand):
+    """
+    Class for executing the `reset-updates` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Reset the updates on the server"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {"server": ["host_name", "port", "access_token"]}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "--sparql-endpoint",
+            help="URL of the QLever server, default is {host_name}:{port}",
+        )
+
+    def execute(self, args) -> bool:
+        reset_cmd = "curl -s"
+        if args.sparql_endpoint:
+            reset_cmd += f" {args.sparql_endpoint}"
+        else:
+            reset_cmd += f" {args.host_name}:{args.port}"
+        reset_cmd += f' --data-urlencode "cmd=clear-delta-triples" --data-urlencode "access-token={args.access_token}"'
+        self.show(reset_cmd, only_show=args.show)
+        if args.show:
+            return True
+
+        try:
+            reset_cmd += ' -w " %{http_code}"'
+            result = run_command(reset_cmd, return_output=True)
+            match = re.match(r"^(.*) (\d+)$", result, re.DOTALL)
+            if not match:
+                raise Exception(f"Unexpected output:\n{result}")
+            error_message = match.group(1).strip()
+            status_code = match.group(2)
+            if status_code != "200":
+                raise Exception(error_message)
+            message = "Updates reset successfully"
+            log.info(message)
+            return True
+        except Exception as e:
+            log.error(e)
+            return False

qlever/commands/settings.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import json
+
+from termcolor import colored
+
+from qlever.command import QleverCommand
+from qlever.log import log
+from qlever.qleverfile import Qleverfile
+from qlever.util import run_command
+
+
+class SettingsCommand(QleverCommand):
+    """
+    Class for executing the `settings` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Show or set server settings (after `qlever start`)"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {"server": ["port", "host_name", "access_token"]}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "runtime_parameters",
+            nargs="*",
+            help="Space-separated list of runtime parameters to set "
+            "in the form `key=value`; afterwards shows all settings, "
+            "with the changed ones highlighted",
+        ).completer = lambda **kwargs: [
+            f"{key}=" for key in Qleverfile.SERVER_RUNTIME_PARAMETERS
+        ]
+        subparser.add_argument(
+            "--endpoint_url",
+            type=str,
+            help="An arbitrary endpoint URL "
+            "(overriding the one in the Qleverfile)",
+        )
+
+    def execute(self, args) -> bool:
+        # Get endpoint URL from command line or Qleverfile.
+        if args.endpoint_url:
+            endpoint_url = args.endpoint_url
+        else:
+            endpoint_url = f"http://{args.host_name}:{args.port}"
+
+        # Construct the `curl` commands for setting and getting.
+        curl_cmds_setting = []
+        keys_set = set()
+        if args.runtime_parameters:
+            for key_value_pair in args.runtime_parameters:
+                try:
+                    key, value = key_value_pair.split("=")
+                except ValueError:
+                    log.error("Runtime parameter must be given as `key=value`")
+                    return False
+                curl_cmds_setting.append(
+                    f"curl -s {endpoint_url} -w %{{http_code}}"
+                    f' --data-urlencode "{key}={value}"'
+                    f' --data-urlencode "access-token={args.access_token}"'
+                )
+                keys_set.add(key)
+        curl_cmd_getting = (
+            f"curl -s {endpoint_url} -w %{{http_code}}"
+            f" --data-urlencode cmd=get-settings"
+        )
+        self.show(
+            "\n".join(curl_cmds_setting + [curl_cmd_getting]),
+            only_show=args.show,
+        )
+        if args.show:
+            return True
+
+        # Execute the `curl` commands for setting the key-value pairs if any.
+        for curl_cmd in curl_cmds_setting:
+            try:
+                curl_result = run_command(curl_cmd, return_output=True)
+                body, http_code = curl_result[:-3], curl_result[-3:]
+                if http_code != "200":
+                    raise Exception(body)
+            except Exception as e:
+                log.error(
+                    f"curl command for setting key-value pair failed: {e}"
+                )
+                return False
+
+        # Execute the `curl` commands for getting the settings.
+        try:
+            curl_result = run_command(curl_cmd_getting, return_output=True)
+            body, http_code = curl_result[:-3], curl_result[-3:]
+            if http_code != "200":
+                raise Exception(body)
+            settings_dict = json.loads(body)
+            if isinstance(settings_dict, list):
+                settings_dict = settings_dict[0]
+        except Exception as e:
+            log.error(f"curl command for getting settings failed: {e}")
+            return False
+        for key, value in settings_dict.items():
+            print(
+                colored(
+                    f"{key:<45}: {value}",
+                    "blue" if key in keys_set else None,
+                )
+            )
+
+        # That's it.
+        return True