qlever 0.4.1__py3-none-any.whl → 0.4.2__py3-none-any.whl

qlever/commands/example_queries.py

@@ -4,13 +4,15 @@ import re
 import shlex
 import subprocess
 import time
+import traceback
+from pathlib import Path
 
 from termcolor import colored
 
 from qlever.command import QleverCommand
 from qlever.commands.clear_cache import ClearCacheCommand
 from qlever.log import log, mute_log
-from qlever.util import run_command
+from qlever.util import run_command, run_curl_command
 
 
 class ExampleQueriesCommand(QleverCommand):
@@ -57,12 +59,27 @@ class ExampleQueriesCommand(QleverCommand):
                                "or just compute the size of the result")
         subparser.add_argument("--limit", type=int,
                                help="Limit on the number of results")
+        subparser.add_argument("--accept", type=str,
+                               choices=["text/tab-separated-values",
+                                        "application/sparql-results+json"],
+                               default="text/tab-separated-values",
+                               help="Accept header for the SPARQL query")
         subparser.add_argument("--clear-cache",
                                choices=["yes", "no"],
                                default="yes",
                                help="Clear the cache before each query")
 
     def execute(self, args) -> bool:
+        # If `args.accept` is `application/sparql-results+json`, we need `jq`.
+        if args.accept == "application/sparql-results+json":
+            try:
+                subprocess.run("jq --version", shell=True, check=True,
+                               stdout=subprocess.DEVNULL,
+                               stderr=subprocess.DEVNULL)
+            except Exception as e:
+                log.error(f"Please install `jq` for {args.accept} ({e})")
+                return False
+
         # Handle shotcuts for SPARQL endpoint.
         if args.sparql_endpoint_preset in self.presets:
             args.sparql_endpoint = self.presets[args.sparql_endpoint_preset]
@@ -92,6 +109,7 @@ class ExampleQueriesCommand(QleverCommand):
                            else f"localhost:{args.port}")
         self.show(f"Obtain queries via: {get_queries_cmd}\n"
                   f"SPARQL endpoint: {sparql_endpoint}\n"
+                  f"Accept header: {args.accept}\n"
                   f"Clear cache before each query:"
                   f" {args.clear_cache.upper()}\n"
                   f"Download result for each query or just count:"
@@ -103,7 +121,8 @@ class ExampleQueriesCommand(QleverCommand):
 
         # Get the example queries.
         try:
-            example_query_lines = run_command(get_queries_cmd, return_output=True)
+            example_query_lines = run_command(get_queries_cmd,
+                                              return_output=True)
             if len(example_query_lines) == 0:
                 log.error("No example queries matching the criteria found")
                 return False
@@ -114,9 +133,10 @@ class ExampleQueriesCommand(QleverCommand):
 
         # Launch the queries one after the other and for each print: the
         # description, the result size, and the query processing time.
-        count = 0
         total_time_seconds = 0.0
         total_result_size = 0
+        count_succeeded = 0
+        count_failed = 0
         for example_query_line in example_query_lines:
             # Parse description and query.
             description, query = example_query_line.split("\t")
@@ -155,44 +175,93 @@ class ExampleQueriesCommand(QleverCommand):
                           + f" }} LIMIT {args.limit}"
 
             # Launch query.
-            query_cmd = (f"curl -sv {sparql_endpoint}"
-                         f" -H \"Accept: text/tab-separated-values\""
-                         f" --data-urlencode query={shlex.quote(query)}")
-            if args.download_or_count == "count":
-                query_cmd += " | sed 1d"
-            else:
-                query_cmd += " | sed 1d | wc -l"
             try:
-                log.debug(query_cmd)
+                curl_cmd = (f"curl -s {sparql_endpoint}"
+                            f" -w \"HTTP code: %{{http_code}}\\n\""
+                            f" -H \"Accept: {args.accept}\""
+                            f" --data-urlencode query={shlex.quote(query)}")
+                log.debug(curl_cmd)
+                result_file = (f"qlever.example_queries.result."
+                               f"{abs(hash(curl_cmd))}.tmp")
                 start_time = time.time()
-                result_size = run_command(query_cmd, return_output=True)
-                result_size = int(result_size.strip())
+                http_code = run_curl_command(sparql_endpoint,
+                                             headers={"Accept": args.accept},
+                                             params={"query": query},
+                                             result_file=result_file).strip()
+                if http_code != "200":
+                    raise Exception(f"HTTP code {http_code}"
+                                    f"  {Path(result_file).read_text()}")
                 time_seconds = time.time() - start_time
-                time_string = f"{time_seconds:.2f}"
-                result_string = f"{result_size:>14,}"
+                error_msg = None
             except Exception as e:
-                time_seconds = 0.0
-                time_string = "---"
-                result_size = 0
-                result_string = colored(f"        FAILED {e}", "red")
+                if args.log_level == "DEBUG":
+                    traceback.print_exc()
+                error_msg = re.sub(r"\s+", " ", str(e))
+
+            # Get result size (via the command line, in order to avoid loading
+            # a potentially large JSON file into Python, which is slow).
+            if error_msg is None:
+                try:
+                    if args.download_or_count == "count":
+                        if args.accept == "text/tab-separated-values":
+                            result_size = run_command(
+                                    f"sed 1d {result_file}",
+                                    return_output=True)
+                        else:
+                            result_size = run_command(
+                                    f"jq -r \".results.bindings[0]"
+                                    f" | to_entries[0].value.value"
+                                    f" | tonumber\" {result_file}",
+                                    return_output=True)
+                    else:
+                        if args.accept == "text/tab-separated-values":
+                            result_size = run_command(
+                                    f"sed 1d {result_file} | wc -l",
+                                    return_output=True)
+                        else:
+                            result_size = run_command(
+                                    f"jq -r \".results.bindings | length\""
+                                    f" {result_file}",
+                                    return_output=True)
+                    result_size = int(result_size)
+                except Exception as e:
+                    error_msg = str(e)
 
             # Print description, time, result in tabular form.
             if (len(description) > 60):
                 description = description[:57] + "..."
-            log.info(f"{description:<60}  {time_string:>6} s  "
-                     f"{result_string}")
-            count += 1
-            total_time_seconds += time_seconds
-            total_result_size += result_size
+            if error_msg is None:
+                log.info(f"{description:<60}  {time_seconds:6.2f} s  "
+                         f"{result_size:14,}")
+                count_succeeded += 1
+                total_time_seconds += time_seconds
+                total_result_size += result_size
+            else:
+                count_failed += 1
+                if (len(error_msg) > 60) and args.log_level != "DEBUG":
+                    error_msg = error_msg[:57] + "..."
+                log.error(f"{description:<60}    failed   "
+                          f"{colored(error_msg, 'red')}")
 
         # Print total time.
         log.info("")
-        description = (f"TOTAL   for {count} "
-                       f"{'query' if count == 1 else 'queries'}")
-        log.info(f"{description:<60}  {total_time_seconds:6.2f} s  "
-                 f"{total_result_size:>14,}")
-        description = (f"AVERAGE for {count} "
-                       f"{'query' if count == 1 else 'queries'}")
-        log.info(f"{description:<60}  {total_time_seconds / count:6.2f} s  "
-                 f"{round(total_result_size / count):>14,}")
+        if count_succeeded > 0:
+            query_or_queries = "query" if count_succeeded == 1 else "queries"
+            description = (f"TOTAL   for {count_succeeded} {query_or_queries}")
+            log.info(f"{description:<60}  "
+                     f"{total_time_seconds:6.2f} s  "
+                     f"{total_result_size:>14,}")
+            description = (f"AVERAGE for {count_succeeded} {query_or_queries}")
+            log.info(f"{description:<60}  "
+                     f"{total_time_seconds / count_succeeded:6.2f} s  "
+                     f"{round(total_result_size / count_succeeded):>14,}")
+        else:
+            if count_failed == 1:
+                log.info(colored("One query failed", "red"))
+            elif count_failed > 1:
+                log.info(colored("All queries failed", "red"))
+
+        # Return success (has nothing to do with how many queries failed).
+        if args.log_level != "DEBUG":
+            Path(result_file).unlink(missing_ok=True)
         return True

qlever/commands/index_stats.py

@@ -71,14 +71,17 @@ class IndexStatsCommand(QleverCommand):
 
         # Helper function that finds the next line matching the given `regex`,
         # starting from `current_line`, and extracts the time. Returns a tuple
-        # of the time and the regex match object. If a match is found,
-        # `current_line` is updated to the line after the match. Otherwise,
-        # `current_line` will be one beyond the last line, unless
-        # `line_is_optional` is true, in which case it will be the same as when
-        # the function was entered.
+        # of the time and the regex match object.
+        #
+        # If `update_current_line` is `False`, then `current_line` will not be
+        # updated by this call.
+        #
+        # Otherwise, and this is the default behavior, `current_line` will be
+        # updated to the line after the first match, or one beyond the last
+        # line if no match is found.
         current_line = 0
 
-        def find_next_line(regex, line_is_optional=False):
+        def find_next_line(regex, update_current_line=True):
             nonlocal lines
             nonlocal current_line
             current_line_backup = current_line
@@ -99,7 +102,7 @@ class IndexStatsCommand(QleverCommand):
                                   f"\"{timestamp_regex}\" from line "
                                   f" \"{line.rstrip()}\" ({e})")
             # If we get here, we did not find a matching line.
-            if line_is_optional:
+            if not update_current_line:
                 current_line = current_line_backup
             return None, None
 
@@ -110,24 +113,34 @@ class IndexStatsCommand(QleverCommand):
         convert_begin, _ = find_next_line(r"INFO:\s*Converting triples")
         perm_begin_and_info = []
         while True:
-            perm_begin, _ = find_next_line(r"INFO:\s*Creating a pair", True)
+            # Find the next line that starts a permutation.
+            #
+            # NOTE: Should work for the old and new format of the index log
+            # file (old format: "Creating a pair" + names of permutations in
+            # line "Writing meta data for ..."; new format: name of
+            # permutations already in line "Creating permutations ...").
+            perm_begin, _ = find_next_line(r"INFO:\s*Creating a pair",
+                                           update_current_line=False)
             if perm_begin is None:
+                perm_begin, perm_info = find_next_line(
+                    r"INFO:\s*Creating permutations ([A-Z]+ and [A-Z]+)",
+                    update_current_line=False)
+            else:
+                _, perm_info = find_next_line(
+                    r"INFO:\s*Writing meta data for ([A-Z]+ and [A-Z]+)",
+                    update_current_line=False)
+            if perm_info is None:
                 break
-            _, perm_info = find_next_line(r"INFO:\s*Writing meta data for"
-                                          r" ([A-Z]+ and [A-Z]+)", True)
-            # if perm_info is None:
-            #     break
             perm_begin_and_info.append((perm_begin, perm_info))
         convert_end = (perm_begin_and_info[0][0] if
                        len(perm_begin_and_info) > 0 else None)
         normal_end, _ = find_next_line(r"INFO:\s*Index build completed")
-        text_begin, _ = find_next_line(r"INFO:\s*Adding text index", True)
-        text_end, _ = find_next_line(r"INFO:\s*Text index build comp", True)
+        text_begin, _ = find_next_line(r"INFO:\s*Adding text index",
+                                       update_current_line=False)
+        text_end, _ = find_next_line(r"INFO:\s*Text index build comp",
+                                     update_current_line=False)
         if args.ignore_text_index:
             text_begin = text_end = None
-        # print("DEBUG:", len(perm_begin_and_info), perm_begin_and_info)
-        # print("DEBUG:", overall_begin)
-        # print("DEBUG:", normal_end)
 
         # Check whether at least the first phase is done.
         if overall_begin is None:

qlever/util.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-import secrets
 import re
+import secrets
 import shlex
 import shutil
 import string
@@ -31,7 +31,7 @@ def run_command(cmd: str, return_output: bool = False,
                 show_output: bool = False) -> Optional[str]:
     """
     Run the given command and throw an exception if the exit code is non-zero.
-    If `get_output` is `True`, return what the command wrote to `stdout`.
+    If `return_output` is `True`, return what the command wrote to `stdout`.
 
     NOTE: The `set -o pipefail` ensures that the exit code of the command is
     non-zero if any part of the pipeline fails (not just the last part).
@@ -68,6 +68,45 @@ def run_command(cmd: str, return_output: bool = False,
         return result.stdout
 
 
+def run_curl_command(url: str,
+                     headers: dict[str, str] = {},
+                     params: dict[str, str] = {},
+                     result_file: Optional[str] = None) -> str:
+    """
+    Run `curl` with the given `url`, `headers`, and `params`. If `result_file`
+    is `None`, return the output, otherwise, write the output to the given file
+    and return the HTTP code. If the `curl` command fails, throw an exception.
+
+    """
+    # Construct and run the `curl` command.
+    default_result_file = "/tmp/qlever.curl.result"
+    actual_result_file = result_file if result_file else default_result_file
+    curl_cmd = (f"curl -s -o \"{actual_result_file}\""
+                f" -w \"%{{http_code}}\n\" {url}"
+                + "".join([f" -H \"{key}: {value}\""
+                           for key, value in headers.items()])
+                + "".join([f" --data-urlencode {key}={shlex.quote(value)}"
+                           for key, value in params.items()]))
+    result = subprocess.run(curl_cmd, shell=True, text=True,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE)
+    # Case 1: An error occurred, raise an exception.
+    if result.returncode != 0:
+        if len(result.stderr) > 0:
+            raise Exception(result.stderr)
+        else:
+            raise Exception(f"curl command failed with exit code "
+                            f"{result.returncode}, stderr is empty")
+    # Case 2: Return output (read from `default_result_file`).
+    if result_file is None:
+        result_file_path = Path(default_result_file)
+        result = result_file_path.read_text()
+        result_file_path.unlink()
+        return result
+    # Case 3: Return HTTP code.
+    return result.stdout
+
+
 def is_qlever_server_alive(port: str) -> bool:
     """
     Helper function that checks if a QLever server is running on the given
@@ -82,30 +121,6 @@ def is_qlever_server_alive(port: str) -> bool:
     return exit_code == 0
 
 
-def get_curl_cmd_for_sparql_query(
-        query: str, port: int,
-        host: str = "localhost",
-        media_type: str = "application/sparql-results+qlever",
-        verbose: bool = False,
-        pinresult: bool = False,
-        access_token: Optional[str] = None,
-        send: Optional[int] = None) -> str:
-    """
-    Get curl command for given SPARQL query.
-    """
-    curl_cmd = (f"curl -s http://{host}:{port}"
-                f" -H \"Accept: {media_type}\" "
-                f" --data-urlencode query={shlex.quote(query)}")
-    if pinresult and access_token is not None:
-        curl_cmd += " --data-urlencode pinresult=true"
-        curl_cmd += f" --data-urlencode access_token={access_token}"
-    if send is not None:
-        curl_cmd += f" --data-urlencode send={send}"
-    if verbose:
-        curl_cmd += " --verbose"
-    return curl_cmd
-
-
 def get_existing_index_files(basename: str) -> list[str]:
     """
     Helper function that returns a list of all index files for `basename` in

{qlever-0.4.1.dist-info → qlever-0.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qlever
-Version: 0.4.1
+Version: 0.4.2
 Summary: Script for using the QLever SPARQL engine.
 Author-email: Hannah Bast <bast@cs.uni-freiburg.de>
 License: Apache License

{qlever-0.4.1.dist-info → qlever-0.4.2.dist-info}/RECORD

@@ -6,7 +6,7 @@ qlever/containerize.py,sha256=p8g3O3G8a_0XLzSTzl_e5t9dqjbCQ-ippoA8vI2Z9pI,4193
 qlever/log.py,sha256=k9Mq4hxQ_d2k0e-5ZVgcB2XIRhOsGMO9I3rIR7YQyDA,1376
 qlever/qlever_main.py,sha256=k8vIQYK7zqObFNet11iLf--nrLdPooL5amprmlySi4k,2300
 qlever/qleverfile.py,sha256=6Ll81xkzel_s2Ju9ZfBXUGlRfikaAzZM6Do-dTrdo3k,12934
-qlever/util.py,sha256=dwqtpY14P3ds_PYx5bgqus_nsx_BhPQzUSa0Z86ONdo,6236
+qlever/util.py,sha256=eepj0SY9JJOUQq5kvtoPnWfoLLV9fbw_sTEWKHet66E,7147
 qlever/Qleverfiles/Qleverfile.dblp,sha256=SFjBD20aOSWod4mEQnxHSDWdInoE_EFp2nyMw7ev7ZA,1167
 qlever/Qleverfiles/Qleverfile.dblp-plus,sha256=Dwd9pK1vPcelKfw6sA-IuyhbZ6yIxOh6_84JgPYnB9Q,1332
 qlever/Qleverfiles/Qleverfile.default,sha256=mljl6I1RCkpIWOqMQwjzPZIsarYQx1R0mIlc583KuqU,1869
@@ -28,10 +28,10 @@ qlever/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qlever/commands/add_text_index.py,sha256=dkqYtwgOhgnXiei_eyhBWYCtdAiQUEmjWoa3JMlMb4c,3641
 qlever/commands/cache_stats.py,sha256=6JjueQstAqc8dNfgY8TP2EitFMxdUvCwrcyd7KUEb2o,4157
 qlever/commands/clear_cache.py,sha256=AnE1MOoj1ZexxrRT8FGeBLlv8rtQIVV4DP8VBn5-X-s,2843
-qlever/commands/example_queries.py,sha256=3jlfHyL7pw1OSTuu3fY-23XaRAPIuEdNGW8QnIY2Va8,8644
+qlever/commands/example_queries.py,sha256=2rYTd35t0r7et0i-IBBcCpmVlYZya9kvwSI-gdTpNdE,12326
 qlever/commands/get_data.py,sha256=0fGuRLDB7YofHtpqk0ctq9_de_xeuliSmSZafGXAo1A,1470
 qlever/commands/index.py,sha256=lJhDnweknFZQm1czqPzNyz33EvbjIvOrS4j0wDaJ98o,5663
-qlever/commands/index_stats.py,sha256=ao7_ySyz8MAjUvCbEp3Kj30PsR5x3MBM3ohgEUWdALM,11083
+qlever/commands/index_stats.py,sha256=_BiUNBhmbYd9RPxrlm4HF0oENO6JmqnRiAkwkyOdN4U,11722
 qlever/commands/log.py,sha256=8Krt3MsTUDapYqVw1zUu5X15SF8mV97Uj0qKOWK8jXk,1861
 qlever/commands/setup_config.py,sha256=mFkEtCPZ6oeVfehjVLrcLttYcPDgtwXHrNIWWzvHOfo,2928
 qlever/commands/start.py,sha256=2rOtk3NmhEs28D5csL_a1BdjSWU9VkcH6AqYT0vdww0,9285
@@ -39,9 +39,9 @@ qlever/commands/status.py,sha256=5S6EdapZEwFKV9cQZtNYcZhMbAXAY-FP6ggjIhfX8ek,163
 qlever/commands/stop.py,sha256=TZs4bxKHvujlZAU8BZmFjA5eXSZNAa6EeNzvPpEZsuI,4139
 qlever/commands/ui.py,sha256=rV8u017WLbfz0zVT_c9GC4d9v1WWwrTM3kfGONbeCvQ,2499
 qlever/commands/warmup.py,sha256=WOZSxeV8U_F6pEEnAb6YybXLQMxZFTRJXs4BPHUhsmc,1030
-qlever-0.4.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-qlever-0.4.1.dist-info/METADATA,sha256=GkXf_oneu0Oe02UOPR8OvqVzxDNA-ljS6yPGLi2x_Bk,17076
-qlever-0.4.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qlever-0.4.1.dist-info/entry_points.txt,sha256=s0iWBHKRUzsJ7B6nVGiyMdOJtiOS84IJMSSxgbNU6LU,85
-qlever-0.4.1.dist-info/top_level.txt,sha256=kd3zsYqiFd0--Czh5XTVkfEq6XR-XgRFW35X0v0GT-c,7
-qlever-0.4.1.dist-info/RECORD,,
+qlever-0.4.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+qlever-0.4.2.dist-info/METADATA,sha256=tyLaWQtRaXbIaQkJ72mCcRpjxlusFHztHdAWedpZ1QE,17076
+qlever-0.4.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qlever-0.4.2.dist-info/entry_points.txt,sha256=s0iWBHKRUzsJ7B6nVGiyMdOJtiOS84IJMSSxgbNU6LU,85
+qlever-0.4.2.dist-info/top_level.txt,sha256=kd3zsYqiFd0--Czh5XTVkfEq6XR-XgRFW35X0v0GT-c,7
+qlever-0.4.2.dist-info/RECORD,,