qlever 0.2.5__py3-none-any.whl → 0.5.41__py3-none-any.whl

qlever/commands/cache_stats.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+
+from qlever.command import QleverCommand
+from qlever.log import log
+
+
+class CacheStatsCommand(QleverCommand):
+    """
+    Class for executing the `warmup` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Show how much of the cache is currently being used"
+
+    def should_have_qleverfile(self) -> bool:
+        return False
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {"server": ["host_name", "port"]}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "--sparql-endpoint",
+            help="URL of the SPARQL endpoint, default is {host_name}:{port}",
+        )
+        subparser.add_argument(
+            "--detailed",
+            action="store_true",
+            default=False,
+            help="Show detailed statistics and settings",
+        )
+
+    def execute(self, args) -> bool:
+        # Construct the two curl commands.
+        sparql_endpoint = (
+            args.sparql_endpoint
+            if args.sparql_endpoint
+            else f"{args.host_name}:{args.port}"
+        )
+        cache_stats_cmd = (
+            f'curl -s {sparql_endpoint} --data-urlencode "cmd=cache-stats"'
+        )
+        cache_settings_cmd = (
+            f'curl -s {sparql_endpoint} --data-urlencode "cmd=get-settings"'
+        )
+
+        # Show them.
+        self.show(
+            "\n".join([cache_stats_cmd, cache_settings_cmd]),
+            only_show=args.show,
+        )
+        if args.show:
+            return True
+
+        # Execute them.
+        try:
+            cache_stats = subprocess.check_output(cache_stats_cmd, shell=True)
+            cache_settings = subprocess.check_output(
+                cache_settings_cmd, shell=True
+            )
+            cache_stats_dict = json.loads(cache_stats)
+            cache_settings_dict = json.loads(cache_settings)
+            if isinstance(cache_settings_dict, list):
+                cache_settings_dict = cache_settings_dict[0]
+        except Exception as e:
+            log.error(f"Failed to get cache stats and settings: {e}")
+            return False
+
+        # Brief version.
+        if not args.detailed:
+            cache_size = cache_settings_dict["cache-max-size"]
+            if not cache_size.endswith(" GB"):
+                log.error(
+                    f"Cache size {cache_size} is not in GB, "
+                    f"QLever should return bytes instead"
+                )
+                return False
+            else:
+                cache_size = float(cache_size[:-3])
+            pinned_size = cache_stats_dict["cache-size-pinned"] / 1e9
+            non_pinned_size = cache_stats_dict["cache-size-unpinned"] / 1e9
+            cached_size = pinned_size + non_pinned_size
+            free_size = cache_size - cached_size
+            if cached_size == 0:
+                log.info(f"Cache is empty, all {cache_size:.1f} GB available")
+            else:
+                log.info(
+                    f"Pinned queries     : "
+                    f"{pinned_size:5.1f} GB of {cache_size:5.1f} GB"
+                    f"  [{pinned_size / cache_size:5.1%}]"
+                )
+                log.info(
+                    f"Non-pinned queries : "
+                    f"{non_pinned_size:5.1f} GB of {cache_size:5.1f} GB"
+                    f"  [{non_pinned_size / cache_size:5.1%}]"
+                )
+                log.info(
+                    f"FREE               : "
+                    f"{free_size:5.1f} GB of {cache_size:5.1f} GB"
+                    f"  [{1 - cached_size / cache_size:5.1%}]"
+                )
+            return True
+
+        # Complete version.
+        def show_dict_as_table(key_value_pairs):
+            max_key_len = max([len(key) for key, _ in key_value_pairs])
+            for key, value in key_value_pairs:
+                if isinstance(value, int) or re.match(r"^\d+$", value):
+                    value = "{:,}".format(int(value))
+                if re.match(r"^\d+\.\d+$", value):
+                    value = "{:.2f}".format(float(value))
+                log.info(f"{key.ljust(max_key_len)} : {value}")
+
+        show_dict_as_table(cache_stats_dict.items())
+        log.info("")
+        show_dict_as_table(cache_settings_dict.items())
+
+        return True

qlever/commands/clear_cache.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import re
+
+from qlever.command import QleverCommand
+from qlever.commands.cache_stats import CacheStatsCommand
+from qlever.log import log
+from qlever.util import run_command
+
+
+class ClearCacheCommand(QleverCommand):
+    """
+    Class for executing the `clear-cache` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Clear the query processing cache"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {"server": ["host_name", "port", "access_token"]}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "--sparql-endpoint",
+            help="URL of the QLever server, default is {host_name}:{port}",
+        )
+        subparser.add_argument(
+            "--complete",
+            action="store_true",
+            default=False,
+            help="Clear the cache completely, including the pinned queries",
+        )
+
+    def execute(self, args) -> bool:
+        # Determine SPARQL endpoint.
+        sparql_endpoint = (
+            args.sparql_endpoint
+            if args.sparql_endpoint
+            else (f"{args.host_name}:{args.port}")
+        )
+
+        # Construct command line and show it.
+        clear_cache_cmd = f"curl -s {sparql_endpoint} -d cmd=clear-cache"
+        if args.complete:
+            clear_cache_cmd += (
+                f"-complete"
+                f' --data-urlencode access-token="{args.access_token}"'
+            )
+        self.show(clear_cache_cmd, only_show=args.show)
+        if args.show:
+            return True
+
+        # Execute the command.
+        try:
+            clear_cache_cmd += ' -w " %{http_code}"'
+            result = run_command(clear_cache_cmd, return_output=True)
+            match = re.match(r"^(.*) (\d+)$", result, re.DOTALL)
+            if not match:
+                raise Exception(f"Unexpected output:\n{result}")
+            error_message = match.group(1).strip()
+            status_code = match.group(2)
+            if status_code != "200":
+                raise Exception(error_message)
+            message = "Cache cleared successfully"
+            if args.complete:
+                message += " (pinned and unpinned queries)"
+            else:
+                message += " (only unpinned queries)"
+            log.info(message)
+        except Exception as e:
+            log.error(e)
+            return False
+
+        # Show cache stats.
+        log.info("")
+        args.detailed = False
+        if not CacheStatsCommand().execute(args):
+            log.error(
+                "Clearing the cache was successful, but showing the "
+                "cache stats failed {e}"
+            )
+        return True

qlever/commands/extract_queries.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import re
+
+from qlever.command import QleverCommand
+from qlever.log import log
+
+
+class ExtractQueriesCommand(QleverCommand):
+    """
+    Class for executing the `extract-queries` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Extract all SPARQL queries from the server log"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {"data": ["name"]}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "--description-base",
+            type=str,
+            default="Log extract",
+            help="Base name for the query descriptions"
+            " (default: `Log extract`)",
+        )
+        subparser.add_argument(
+            "--log-file",
+            type=str,
+            help="Name of the log file to extract queries from"
+            " (default: `<name>.server-log.txt`)",
+        )
+        subparser.add_argument(
+            "--output-file",
+            type=str,
+            default="log-queries.txt",
+            help="Output file for the extracted queries (default: `log-queries.txt`)",
+        )
+        subparser.add_argument(
+            "--use-alive-check-tag-as-description-base",
+            action="store_true",
+            help="Use the tag from 'Alive check' messages"
+            " as the base for query descriptions (default: False)",
+        )
+
+    def execute(self, args) -> bool:
+        # Show what the command does.
+        if args.log_file is not None:
+            log_file_name = args.log_file
+        else:
+            log_file_name = f"{args.name}.server-log.txt"
+        self.show(
+            f"Extract SPARQL queries from `{log_file_name}`"
+            f" and write them to `{args.output_file}`",
+            only_show=args.show,
+        )
+        if args.show:
+            return True
+
+        # Regex for log entries of the form
+        # 2025-01-14 04:47:44.950 - INFO
+        log_line_regex = (
+            r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - [A-Z]+:"
+        )
+
+        # Read the log file line by line.
+        log_file = open(log_file_name, "r")
+        queries_file = open(args.output_file, "w")
+        query = None
+        description_base = args.description_base
+        description_base_count = {}
+        tsv_line_short_width = 150
+        for line in log_file:
+            # An "Alive check" message contains a tag, which we use as the base
+            # name of the query description.
+            if args.use_alive_check_tag_as_description_base:
+                alive_check_regex = r"Alive check with message \"(.*)\""
+                match = re.search(alive_check_regex, line)
+                if match:
+                    description_base = match.group(1)
+                    continue
+
+            # A new query in the log.
+            if "Processing the following SPARQL query" in line:
+                query = []
+                query_index = (
+                    description_base_count.get(description_base, 0) + 1
+                )
+                description_base_count[description_base] = query_index
+                continue
+            # If we have started a query: extend until we meet the next log
+            # line, then push the query. Remove comments.
+            if query is not None:
+                if not re.match(log_line_regex, line):
+                    if not re.match(r"^\s*#", line):
+                        line = re.sub(r" #.*", "", line)
+                        query.append(line)
+                else:
+                    query = re.sub(r"\s+", " ", "\n".join(query)).strip()
+                    description = f"{description_base}, Query #{query_index}"
+                    tsv_line = f"{description}\t{query}"
+                    tsv_line_short = (
+                        tsv_line
+                        if len(tsv_line) < tsv_line_short_width
+                        else tsv_line[:tsv_line_short_width] + "..."
+                    )
+                    log.info(tsv_line_short)
+                    print(tsv_line, file=queries_file)
+                    query = None
+
+        log_file.close()
+        queries_file.close()
+        return True

qlever/commands/get_data.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import shlex
+
+from qlever.command import QleverCommand
+from qlever.log import log
+from qlever.util import get_total_file_size, run_command
+
+
+class GetDataCommand(QleverCommand):
+    """
+    Class for executing the `get-data` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Get data using the GET_DATA_CMD in the Qleverfile"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
+        return {"data": ["name", "get_data_cmd"], "index": ["input_files"]}
+
+    def additional_arguments(self, subparser) -> None:
+        pass
+
+    def execute(self, args) -> bool:
+        # Construct the command line and show it.
+        self.show(args.get_data_cmd, only_show=args.show)
+        if args.show:
+            return True
+
+        # Execute the command line.
+        try:
+            run_command(args.get_data_cmd, show_output=True)
+        except Exception as e:
+            log.error(f"Problem executing \"{args.get_data_cmd}\": {e}")
+            return False
+
+        # Show the total file size in GB and return.
+        patterns = shlex.split(args.input_files)
+        total_file_size = get_total_file_size(patterns)
+        print(f"Download successful, total file size: "
+              f"{total_file_size:,} bytes")
+        return True