qlever 0.5.15__py3-none-any.whl → 0.5.18__py3-none-any.whl

qlever/commands/example_queries.py

@@ -17,14 +17,15 @@ from qlever.util import run_command, run_curl_command
 
 class ExampleQueriesCommand(QleverCommand):
     """
-    Class for executing the `warmup` command.
+    Class for running a given sequence of example queries and showing
+    their processing times and result sizes.
     """
 
     def __init__(self):
         pass
 
     def description(self) -> str:
-        return "Show how much of the cache is currently being used"
+        return "Run the given queries and show their processing times and result sizes"
 
     def should_have_qleverfile(self) -> bool:
         return False
@@ -51,13 +52,15 @@ class ExampleQueriesCommand(QleverCommand):
         subparser.add_argument(
             "--get-queries-cmd",
             type=str,
-            help="Command to get example queries as TSV " "(description, query)",
+            help="Command to get example queries as TSV "
+            "(description, query)",
         )
         subparser.add_argument(
             "--query-ids",
             type=str,
             default="1-$",
-            help="Query IDs as comma-separated list of " "ranges (e.g., 1-5,7,12-$)",
+            help="Query IDs as comma-separated list of "
+            "ranges (e.g., 1-5,7,12-$)",
         )
         subparser.add_argument(
             "--query-regex",
@@ -68,7 +71,7 @@ class ExampleQueriesCommand(QleverCommand):
         subparser.add_argument(
             "--download-or-count",
             choices=["download", "count"],
-            default="count",
+            default="download",
             help="Whether to download the full result "
             "or just compute the size of the result",
         )
@@ -88,10 +91,14 @@ class ExampleQueriesCommand(QleverCommand):
                 "text/tab-separated-values",
                 "text/csv",
                 "application/sparql-results+json",
+                "application/qlever-results+json",
                 "text/turtle",
+                "AUTO",
             ],
             default="application/sparql-results+json",
-            help="Accept header for the SPARQL query",
+            help="Accept header for the SPARQL query; AUTO means "
+            "`text/turtle` for CONSTRUCT AND DESCRIBE queries, "
+            "`application/sparql-results+json` for all others",
         )
         subparser.add_argument(
             "--clear-cache",
@@ -117,6 +124,13 @@ class ExampleQueriesCommand(QleverCommand):
             default=14,
             help="Width for printing the result size",
         )
+        subparser.add_argument(
+            "--add-query-type-to-description",
+            action="store_true",
+            default=False,
+            help="Add the query type (SELECT, ASK, CONSTRUCT, DESCRIBE, "
+            "UNKNOWN) to the description",
+        )
         subparser.add_argument(
             "--show-query",
             choices=["always", "never", "on-error"],
@@ -130,19 +144,35 @@ class ExampleQueriesCommand(QleverCommand):
             help="When showing the query, also show the prefixes",
         )
 
-    def pretty_print_query(self, query: str, show_prefixes: bool) -> None:
-        remove_prefixes_cmd = " | sed '/^PREFIX /Id'" if not show_prefixes else ""
+    def pretty_printed_query(self, query: str, show_prefixes: bool) -> str:
+        remove_prefixes_cmd = (
+            " | sed '/^PREFIX /Id'" if not show_prefixes else ""
+        )
         pretty_print_query_cmd = (
             f"echo {shlex.quote(query)}"
             f" | docker run -i --rm sparqling/sparql-formatter"
             f"{remove_prefixes_cmd} | grep -v '^$'"
         )
         try:
-            query_pp = run_command(pretty_print_query_cmd, return_output=True)
-            log.info(colored(query_pp.rstrip(), "cyan"))
-        except Exception as e:
-            log.error(f"Failed to pretty-print query: {e}")
-            log.info(colored(query.rstrip(), "cyan"))
+            query_pretty_printed = run_command(
+                pretty_print_query_cmd, return_output=True
+            )
+            return query_pretty_printed.rstrip()
+        except Exception:
+            log.error(
+                "Failed to pretty-print query, "
+                "returning original query: {e}"
+            )
+            return query.rstrip()
+
+    def sparql_query_type(self, query: str) -> str:
+        match = re.search(
+            r"(SELECT|ASK|CONSTRUCT|DESCRIBE)\s", query, re.IGNORECASE
+        )
+        if match:
+            return match.group(1).upper()
+        else:
+            return "UNKNOWN"
 
     def execute(self, args) -> bool:
         # We can't have both `--remove-offset-and-limit` and `--limit`.
@@ -150,8 +180,13 @@ class ExampleQueriesCommand(QleverCommand):
             log.error("Cannot have both --remove-offset-and-limit and --limit")
             return False
 
-        # If `args.accept` is `application/sparql-results+json`, we need `jq`.
-        if args.accept == "application/sparql-results+json":
+        # If `args.accept` is `application/sparql-results+json` or
+        # `application/qlever-results+json` or `AUTO`, we need `jq`.
+        if (
+            args.accept == "application/sparql-results+json"
+            or args.accept == "application/qlever-results+json"
+            or args.accept == "AUTO"
+        ):
             try:
                 subprocess.run(
                     "jq --version",
@@ -174,8 +209,9 @@ class ExampleQueriesCommand(QleverCommand):
             return False
 
         # Clear cache only works for QLever.
-        is_qlever = not args.sparql_endpoint or args.sparql_endpoint.startswith(
-            "https://qlever"
+        is_qlever = (
+            not args.sparql_endpoint
+            or args.sparql_endpoint.startswith("https://qlever")
         )
         if args.clear_cache == "yes" and not is_qlever:
             log.warning("Clearing the cache only works for QLever")
@@ -193,7 +229,9 @@ class ExampleQueriesCommand(QleverCommand):
         if args.query_regex:
             get_queries_cmd += f" | grep -Pi {shlex.quote(args.query_regex)}"
         sparql_endpoint = (
-            args.sparql_endpoint if args.sparql_endpoint else f"localhost:{args.port}"
+            args.sparql_endpoint
+            if args.sparql_endpoint
+            else f"localhost:{args.port}"
         )
         self.show(
             f"Obtain queries via: {get_queries_cmd}\n"
@@ -211,7 +249,9 @@ class ExampleQueriesCommand(QleverCommand):
 
         # Get the example queries.
         try:
-            example_query_lines = run_command(get_queries_cmd, return_output=True)
+            example_query_lines = run_command(
+                get_queries_cmd, return_output=True
+            )
             if len(example_query_lines) == 0:
                 log.error("No example queries matching the criteria found")
                 return False
@@ -220,6 +260,12 @@ class ExampleQueriesCommand(QleverCommand):
             log.error(f"Failed to get example queries: {e}")
             return False
 
+        # We want the width of the query description to be an uneven number (in
+        # case we have to truncated it, in which case we want to have a " ... "
+        # in the middle).
+        width_query_description_half = args.width_query_description // 2
+        width_query_description = 2 * width_query_description_half + 1
+
         # Launch the queries one after the other and for each print: the
         # description, the result size (number of rows), and the query
         # processing time (seconds).
@@ -227,13 +273,16 @@ class ExampleQueriesCommand(QleverCommand):
         result_sizes = []
         num_failed = 0
         for example_query_line in example_query_lines:
-            # Parse description and query.
+            # Parse description and query, and determine query type.
             description, query = example_query_line.split("\t")
             if len(query) == 0:
                 log.error("Could not parse description and query, line is:")
                 log.info("")
                 log.info(example_query_line)
                 return False
+            query_type = self.sparql_query_type(query)
+            if args.add_query_type_to_description or args.accept == "AUTO":
+                description = f"{description} [{query_type}]"
 
             # Clear the cache.
             if args.clear_cache == "yes":
@@ -267,7 +316,9 @@ class ExampleQueriesCommand(QleverCommand):
             # Count query.
             if args.download_or_count == "count":
                 # First find out if there is a FROM clause.
-                regex_from_clause = re.compile(r"\s*FROM\s+<[^>]+>\s*", re.IGNORECASE)
+                regex_from_clause = re.compile(
+                    r"\s*FROM\s+<[^>]+>\s*", re.IGNORECASE
+                )
                 match_from_clause = re.search(regex_from_clause, query)
                 from_clause = " "
                 if match_from_clause:
@@ -296,24 +347,39 @@ class ExampleQueriesCommand(QleverCommand):
             query = re.sub(r"\s*\.\s*\}", " }", query)
             if args.show_query == "always":
                 log.info("")
-                self.pretty_print_query(query, args.show_prefixes)
+                log.info(
+                    colored(
+                        self.pretty_printed_query(query, args.show_prefixes),
+                        "cyan",
+                    )
+                )
+
+            # Accept header. For "AUTO", use `text/turtle` for CONSTRUCT
+            # queries and `application/sparql-results+json` for all others.
+            accept_header = args.accept
+            if accept_header == "AUTO":
+                if query_type == "CONSTRUCT" or query_type == "DESCRIBE":
+                    accept_header = "text/turtle"
+                else:
+                    accept_header = "application/sparql-results+json"
 
             # Launch query.
             try:
                 curl_cmd = (
                     f"curl -s {sparql_endpoint}"
                     f' -w "HTTP code: %{{http_code}}\\n"'
-                    f' -H "Accept: {args.accept}"'
+                    f' -H "Accept: {accept_header}"'
                     f" --data-urlencode query={shlex.quote(query)}"
                 )
                 log.debug(curl_cmd)
                 result_file = (
-                    f"qlever.example_queries.result." f"{abs(hash(curl_cmd))}.tmp"
+                    f"qlever.example_queries.result."
+                    f"{abs(hash(curl_cmd))}.tmp"
                 )
                 start_time = time.time()
                 http_code = run_curl_command(
                     sparql_endpoint,
-                    headers={"Accept": args.accept},
+                    headers={"Accept": accept_header},
                     params={"query": query},
                     result_file=result_file,
                 ).strip()
@@ -323,7 +389,9 @@ class ExampleQueriesCommand(QleverCommand):
                 else:
                     error_msg = {
                         "short": f"HTTP code: {http_code}",
-                        "long": re.sub(r"\s+", " ", Path(result_file).read_text()),
+                        "long": re.sub(
+                            r"\s+", " ", Path(result_file).read_text()
+                        ),
                     }
             except Exception as e:
                 if args.log_level == "DEBUG":
@@ -336,8 +404,12 @@ class ExampleQueriesCommand(QleverCommand):
             # Get result size (via the command line, in order to avoid loading
             # a potentially large JSON file into Python, which is slow).
             if error_msg is None:
-                # CASE 0: Rhe result is empty despite a 200 HTTP code.
-                if Path(result_file).stat().st_size == 0:
+                # CASE 0: The result is empty despite a 200 HTTP code (not a
+                # problem for CONSTRUCT and DESCRIBE queries).
+                if Path(result_file).stat().st_size == 0 and (
+                    not query_type == "CONSTRUCT"
+                    and not query_type == "DESCRIBE"
+                ):
                     result_size = 0
                     error_msg = {
                         "short": "Empty result",
@@ -347,7 +419,7 @@ class ExampleQueriesCommand(QleverCommand):
 
                 # CASE 1: Just counting the size of the result (TSV or JSON).
                 elif args.download_or_count == "count":
-                    if args.accept == "text/tab-separated-values":
+                    if accept_header == "text/tab-separated-values":
                         result_size = run_command(
                             f"sed 1d {result_file}", return_output=True
                         )
@@ -370,21 +442,28 @@ class ExampleQueriesCommand(QleverCommand):
                 # CASE 2: Downloading the full result (TSV, CSV, Turtle, JSON).
                 else:
                     if (
-                        args.accept == "text/tab-separated-values"
-                        or args.accept == "text/csv"
+                        accept_header == "text/tab-separated-values"
+                        or accept_header == "text/csv"
                     ):
                         result_size = run_command(
                             f"sed 1d {result_file} | wc -l", return_output=True
                         )
-                    elif args.accept == "text/turtle":
+                    elif accept_header == "text/turtle":
                         result_size = run_command(
-                            f"sed '1d;/^@prefix/d;/^\\s*$/d' " f"{result_file} | wc -l",
+                            f"sed '1d;/^@prefix/d;/^\\s*$/d' "
+                            f"{result_file} | wc -l",
+                            return_output=True,
+                        )
+                    elif accept_header == "application/qlever-results+json":
+                        result_size = run_command(
+                            f'jq -r ".resultsize" {result_file}',
                             return_output=True,
                         )
                     else:
                         try:
                             result_size = run_command(
-                                f'jq -r ".results.bindings | length"' f" {result_file}",
+                                f'jq -r ".results.bindings | length"'
+                                f" {result_file}",
                                 return_output=True,
                             )
                         except Exception as e:
@@ -398,13 +477,16 @@ class ExampleQueriesCommand(QleverCommand):
                 Path(result_file).unlink(missing_ok=True)
 
             # Print description, time, result in tabular form.
-            if len(description) > args.width_query_description:
-                description = description[: args.width_query_description - 3]
-                description += "..."
+            if len(description) > width_query_description:
+                description = (
+                    description[: width_query_description_half - 2]
+                    + " ... "
+                    + description[-width_query_description_half + 2 :]
+                )
             if error_msg is None:
                 result_size = int(result_size)
                 log.info(
-                    f"{description:<{args.width_query_description}}  "
+                    f"{description:<{width_query_description}}  "
                     f"{time_seconds:6.2f} s  "
                     f"{result_size:>{args.width_result_size},}"
                 )
@@ -419,18 +501,28 @@ class ExampleQueriesCommand(QleverCommand):
                     and args.show_query != "on-error"
                 ):
                     error_msg["long"] = (
-                        error_msg["long"][: args.width_error_message - 3] + "..."
+                        error_msg["long"][: args.width_error_message - 3]
+                        + "..."
                     )
-                seperator_short_long = "\n" if args.show_query == "on-error" else "  "
+                seperator_short_long = (
+                    "\n" if args.show_query == "on-error" else "  "
+                )
                 log.info(
-                    f"{description:<{args.width_query_description}}    "
+                    f"{description:<{width_query_description}}    "
                     f"{colored('FAILED   ', 'red')}"
                     f"{colored(error_msg['short'], 'red'):>{args.width_result_size}}"
                     f"{seperator_short_long}"
                     f"{colored(error_msg['long'], 'red')}"
                 )
                 if args.show_query == "on-error":
-                    self.pretty_print_query(query, args.show_prefixes)
+                    log.info(
+                        colored(
+                            self.pretty_printed_query(
+                                query, args.show_prefixes
+                            ),
+                            "cyan",
+                        )
+                    )
                     log.info("")
 
         # Check that each query has a time and a result size, or it failed.
@@ -450,19 +542,19 @@ class ExampleQueriesCommand(QleverCommand):
             description = f"TOTAL   for {n} {query_or_queries}"
             log.info("")
             log.info(
-                f"{description:<{args.width_query_description}}  "
+                f"{description:<{width_query_description}}  "
                 f"{total_query_time:6.2f} s  "
                 f"{total_result_size:>14,}"
             )
             description = f"AVERAGE for {n} {query_or_queries}"
             log.info(
-                f"{description:<{args.width_query_description}}  "
+                f"{description:<{width_query_description}}  "
                 f"{average_query_time:6.2f} s  "
                 f"{average_result_size:>14,}"
             )
             description = f"MEDIAN  for {n} {query_or_queries}"
             log.info(
-                f"{description:<{args.width_query_description}}  "
+                f"{description:<{width_query_description}}  "
                 f"{median_query_time:6.2f} s  "
                 f"{median_result_size:>14,}"
             )
@@ -476,7 +568,7 @@ class ExampleQueriesCommand(QleverCommand):
                 num_failed_string += "  [all]"
             log.info(
                 colored(
-                    f"{description:<{args.width_query_description}}  "
+                    f"{description:<{width_query_description}}  "
                     f"{num_failed:>24}",
                     "red",
                 )

qlever/commands/extract_queries.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import re
+
+from qlever.command import QleverCommand
+from qlever.log import log
+
+
+class ExtractQueriesCommand(QleverCommand):
+    """
+    Class for executing the `extract-queries` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Extract all SPARQL queries from the server log"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
+        return {"data": ["name"]}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "--description-base",
+            type=str,
+            default="Log extract",
+            help="Base name for the query descriptions"
+            " (default: `Log extract`)",
+        )
+        subparser.add_argument(
+            "--log-file",
+            type=str,
+            help="Name of the log file to extract queries from"
+            " (default: `<name>.server-log.txt`)",
+        )
+        subparser.add_argument(
+            "--output-file",
+            type=str,
+            default="log-queries.txt",
+            help="Output file for the extracted queries (default: `log-queries.txt`)",
+        )
+
+    def execute(self, args) -> bool:
+        # Show what the command does.
+        if args.log_file is not None:
+            log_file_name = args.log_file
+        else:
+            log_file_name = f"{args.name}.server-log.txt"
+        self.show(
+            f"Extract SPARQL queries from `{log_file_name}`"
+            f" and write them to `{args.output_file}`",
+            only_show=args.show,
+        )
+        if args.show:
+            return True
+
+        # Regex for log entries of the form
+        # 2025-01-14 04:47:44.950 - INFO
+        log_line_regex = (
+            r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - [A-Z]+:"
+        )
+
+        # Read the log file line by line.
+        log_file = open(log_file_name, "r")
+        queries_file = open(args.output_file, "w")
+        query = None
+        description_base = args.description_base
+        description_base_count = {}
+        tsv_line_short_width = 150
+        for line in log_file:
+            # An "Alive check" message contains a tag, which we use as the base
+            # name of the query description.
+            alive_check_regex = r"Alive check with message \"(.*)\""
+            match = re.search(alive_check_regex, line)
+            if match:
+                description_base = match.group(1)
+                continue
+
+            # A new query in the log.
+            if "Processing the following SPARQL query" in line:
+                query = []
+                query_index = (
+                    description_base_count.get(description_base, 0) + 1
+                )
+                description_base_count[description_base] = query_index
+                continue
+            # If we have started a query: extend until we meet the next log
+            # line, then push the query. Remove comments.
+            if query is not None:
+                if not re.match(log_line_regex, line):
+                    if not re.match(r"^\s*#", line):
+                        line = re.sub(r" #.*", "", line)
+                        query.append(line)
+                else:
+                    query = re.sub(r"\s+", " ", "\n".join(query)).strip()
+                    description = f"{description_base}, Query #{query_index}"
+                    tsv_line = f"{description}\t{query}"
+                    tsv_line_short = (
+                        tsv_line
+                        if len(tsv_line) < tsv_line_short_width
+                        else tsv_line[:tsv_line_short_width] + "..."
+                    )
+                    log.info(tsv_line_short)
+                    print(tsv_line, file=queries_file)
+                    query = None
+
+        log_file.close()
+        queries_file.close()
+        return True

qlever/commands/index.py

@@ -2,13 +2,17 @@ from __future__ import annotations
 
 import glob
 import json
-import shlex
 import re
+import shlex
 
 from qlever.command import QleverCommand
 from qlever.containerize import Containerize
 from qlever.log import log
-from qlever.util import get_existing_index_files, get_total_file_size, run_command
+from qlever.util import (
+    get_existing_index_files,
+    get_total_file_size,
+    run_command,
+)
 
 
 class IndexCommand(QleverCommand):
@@ -36,9 +40,11 @@ class IndexCommand(QleverCommand):
                 "settings_json",
                 "index_binary",
                 "only_pso_and_pos_permutations",
+                "ulimit",
                 "use_patterns",
                 "text_index",
                 "stxxl_memory",
+                "parser_buffer_size",
             ],
             "runtime": ["system", "image", "index_container"],
         }
@@ -48,7 +54,7 @@ class IndexCommand(QleverCommand):
             "--overwrite-existing",
             action="store_true",
             default=False,
-            help="Overwrite an existing index, think twice before using.",
+            help="Overwrite an existing index, think twice before using this",
         )
 
     # Exception for invalid JSON.
@@ -76,7 +82,8 @@ class IndexCommand(QleverCommand):
         # Check that it is an array of length at least one.
         if not isinstance(input_specs, list):
             raise self.InvalidInputJson(
-                "`MULTI_INPUT_JSON` must be a JSON array", args.multi_input_json
+                "`MULTI_INPUT_JSON` must be a JSON array",
+                args.multi_input_json,
             )
         if len(input_specs) == 0:
             raise self.InvalidInputJson(
@@ -90,13 +97,15 @@ class IndexCommand(QleverCommand):
             # Check that `input_spec` is a dictionary.
             if not isinstance(input_spec, dict):
                 raise self.InvalidInputJson(
-                    f"Element {i} in `MULTI_INPUT_JSON` must be a JSON " "object",
+                    f"Element {i} in `MULTI_INPUT_JSON` must be a JSON "
+                    "object",
                     input_spec,
                 )
             # For each `input_spec`, we must have a command.
             if "cmd" not in input_spec:
                 raise self.InvalidInputJson(
-                    f"Element {i} in `MULTI_INPUT_JSON` must contain a " "key `cmd`",
+                    f"Element {i} in `MULTI_INPUT_JSON` must contain a "
+                    "key `cmd`",
                     input_spec,
                 )
             # If the command contains a `{}` placeholder, we need a `for-each`
@@ -147,7 +156,7 @@ class IndexCommand(QleverCommand):
                 raise self.InvalidInputJson(
                     f"Element {i} in `MULTI_INPUT_JSON` must only contain "
                     "the keys `format`, `graph`, and `parallel`. Contains "
-                    "extra keys {extra_keys}.",
+                    f"extra keys {extra_keys}.",
                     input_spec,
                 )
             # Add the command-line options for this input stream. We use
@@ -204,20 +213,31 @@ class IndexCommand(QleverCommand):
             index_cmd += " --only-pso-and-pos-permutations --no-patterns"
         if not args.use_patterns:
             index_cmd += " --no-patterns"
-        if args.text_index in ["from_text_records", "from_text_records_and_literals"]:
+        if args.text_index in [
+            "from_text_records",
+            "from_text_records_and_literals",
+        ]:
             index_cmd += (
-                f" -w {args.name}.wordsfile.tsv" f" -d {args.name}.docsfile.tsv"
+                f" -w {args.name}.wordsfile.tsv"
+                f" -d {args.name}.docsfile.tsv"
             )
-        if args.text_index in ["from_literals", "from_text_records_and_literals"]:
+        if args.text_index in [
+            "from_literals",
+            "from_text_records_and_literals",
+        ]:
             index_cmd += " --text-words-from-literals"
         if args.stxxl_memory:
             index_cmd += f" --stxxl-memory {args.stxxl_memory}"
+        if args.parser_buffer_size:
+            index_cmd += f" --parser-buffer-size {args.parser_buffer_size}"
         index_cmd += f" | tee {args.name}.index-log.txt"
 
         # If the total file size is larger than 10 GB, set ulimit (such that a
         # large number of open files is allowed).
         total_file_size = get_total_file_size(shlex.split(args.input_files))
-        if total_file_size > 1e10:
+        if args.ulimit is not None:
+            index_cmd = f"ulimit -Sn {args.ulimit}; {index_cmd}"
+        elif total_file_size > 1e10:
             index_cmd = f"ulimit -Sn 1048576; {index_cmd}"
 
         # Run the command in a container (if so desired).
@@ -234,7 +254,8 @@ class IndexCommand(QleverCommand):
 
         # Command for writing the settings JSON to a file.
         settings_json_cmd = (
-            f"echo {shlex.quote(args.settings_json)} " f"> {args.name}.settings.json"
+            f"echo {shlex.quote(args.settings_json)} "
+            f"> {args.name}.settings.json"
         )
 
         # Show the command line.
@@ -279,9 +300,15 @@ class IndexCommand(QleverCommand):
             return False
 
         # Remove already existing container.
-        if args.system in Containerize.supported_systems() and args.overwrite_existing:
+        if (
+            args.system in Containerize.supported_systems()
+            and args.overwrite_existing
+        ):
             if Containerize.is_running(args.system, args.index_container):
-                log.info("Another index process is running, trying to stop " "it ...")
+                log.info(
+                    "Another index process is running, trying to stop "
+                    "it ..."
+                )
                 log.info("")
                 try:
                     run_command(f"{args.system} rm -f {args.index_container}")