qlever 0.5.12__py3-none-any.whl → 0.5.15__py3-none-any.whl

qlever/commands/index.py

@@ -99,13 +99,50 @@ class IndexCommand(QleverCommand):
                     f"Element {i} in `MULTI_INPUT_JSON` must contain a " "key `cmd`",
                     input_spec,
                 )
-            input_cmd = input_spec["cmd"]
+            # If the command contains a `{}` placeholder, we need a `for-each`
+            # key` specifying the pattern for the placeholder values, and vice
+            # versa.
+            if "{}" in input_spec["cmd"] and "for-each" not in input_spec:
+                raise self.InvalidInputJson(
+                    f"Element {i} in `MULTI_INPUT_JSON` must contain a "
+                    "key `for-each` if the command contains a placeholder "
+                    "`{}`",
+                    input_spec,
+                )
+            if "for-each" in input_spec and "{}" not in input_spec["cmd"]:
+                raise self.InvalidInputJson(
+                    f"Element {i} in `MULTI_INPUT_JSON` contains a "
+                    "key `for-each`, but the command does not contain a "
+                    "placeholder `{{}}`",
+                    input_spec,
+                )
+            # Get all commands. This is just the value of the `cmd` key if no
+            # `for-each` key is specified. Otherwise, we have a command for
+            # each file matching the pattern.
+            if "for-each" not in input_spec:
+                input_cmds = [input_spec["cmd"]]
+            else:
+                try:
+                    files = sorted(glob.glob(input_spec["for-each"]))
+                except Exception as e:
+                    raise self.InvalidInputJson(
+                        f"Element {i} in `MULTI_INPUT_JSON` contains an "
+                        f"invalid `for-each` pattern: {e}",
+                        input_spec,
+                    )
+                input_cmds = [input_spec["cmd"].format(file) for file in files]
             # The `format`, `graph`, and `parallel` keys are optional.
             input_format = input_spec.get("format", args.format)
             input_graph = input_spec.get("graph", "-")
             input_parallel = input_spec.get("parallel", "false")
             # There must not be any other keys.
-            extra_keys = input_spec.keys() - {"cmd", "format", "graph", "parallel"}
+            extra_keys = input_spec.keys() - {
+                "cmd",
+                "format",
+                "graph",
+                "parallel",
+                "for-each",
+            }
             if extra_keys:
                 raise self.InvalidInputJson(
                     f"Element {i} in `MULTI_INPUT_JSON` must only contain "
@@ -114,13 +151,16 @@ class IndexCommand(QleverCommand):
                     input_spec,
                 )
             # Add the command-line options for this input stream. We use
-            # process substitution `<(...)` as a convenient way to handle
-            # an input stream just like a file. This is not POSIX compliant,
-            # but supported by various shells, including bash and zsh.
-            input_options.append(
-                f"-f <({input_cmd}) -F {input_format} "
-                f'-g "{input_graph}" -p {input_parallel}'
-            )
+            # process substitution `<(...)` as a convenient way to handle an
+            # input stream just like a file. This is not POSIX compliant, but
+            # supported by various shells, including bash and zsh. If
+            # `for-each` is specified, add one command for each matching file.
+            for input_cmd in input_cmds:
+                input_option = f"-f <({input_cmd}) -g {input_graph}"
+                input_option += f" -F {input_format}"
+                if input_parallel == "true":
+                    input_option += " -p true"
+                input_options.append(input_option)
         # Return the concatenated command-line options.
         return " ".join(input_options)
 
@@ -200,7 +240,7 @@ class IndexCommand(QleverCommand):
         # Show the command line.
         self.show(f"{settings_json_cmd}\n{index_cmd}", only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # When running natively, check if the binary exists and works.
         if args.system == "native":

qlever/commands/index_stats.py

@@ -18,32 +18,45 @@ class IndexStatsCommand(QleverCommand):
         pass
 
     def description(self) -> str:
-        return ("Breakdown of the time and space used for the index build")
+        return "Breakdown of the time and space used for the index build"
 
     def should_have_qleverfile(self) -> bool:
         return False
 
-    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
+    def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
         return {"data": ["name"]}
 
     def additional_arguments(self, subparser) -> None:
-        subparser.add_argument("--only-time", action="store_true",
-                               default=False,
-                               help="Show only the time used")
-        subparser.add_argument("--only-space", action="store_true",
-                               default=False,
-                               help="Show only the space used")
-        subparser.add_argument("--ignore-text-index", action="store_true",
-                               default=False,
-                               help="Ignore the text index")
-        subparser.add_argument("--time-unit",
-                               choices=["s", "min", "h", "auto"],
-                               default="auto",
-                               help="The time unit")
-        subparser.add_argument("--size-unit",
-                               choices=["B", "MB", "GB", "TB", "auto"],
-                               default="auto",
-                               help="The size unit")
+        subparser.add_argument(
+            "--only-time",
+            action="store_true",
+            default=False,
+            help="Show only the time used",
+        )
+        subparser.add_argument(
+            "--only-space",
+            action="store_true",
+            default=False,
+            help="Show only the space used",
+        )
+        subparser.add_argument(
+            "--ignore-text-index",
+            action="store_true",
+            default=False,
+            help="Ignore the text index",
+        )
+        subparser.add_argument(
+            "--time-unit",
+            choices=["s", "min", "h", "auto"],
+            default="auto",
+            help="The time unit",
+        )
+        subparser.add_argument(
+            "--size-unit",
+            choices=["B", "MB", "GB", "TB", "auto"],
+            default="auto",
+            help="The size unit",
+        )
 
     def execute_time(self, args, log_file_name) -> bool:
         """
@@ -65,8 +78,9 @@ class IndexStatsCommand(QleverCommand):
                 with open(text_log_file_name, "r") as text_log_file:
                     lines.extend(text_log_file.readlines())
         except Exception as e:
-            log.error(f"Problem reading text index log file "
-                      f"{text_log_file_name}: {e}")
+            log.error(
+                f"Problem reading text index log file " f"{text_log_file_name}: {e}"
+            )
             return False
 
         # Helper function that finds the next line matching the given `regex`,
@@ -95,12 +109,14 @@ class IndexStatsCommand(QleverCommand):
                 if regex_match:
                     try:
                         return datetime.strptime(
-                                re.match(timestamp_regex, line).group(),
-                                timestamp_format), regex_match
+                            re.match(timestamp_regex, line).group(), timestamp_format
+                        ), regex_match
                     except Exception as e:
-                        log.error(f"Could not parse timestamp of form "
-                                  f"\"{timestamp_regex}\" from line "
-                                  f" \"{line.rstrip()}\" ({e})")
+                        log.error(
+                            f"Could not parse timestamp of form "
+                            f'"{timestamp_regex}" from line '
+                            f' "{line.rstrip()}" ({e})'
+                        )
             # If we get here, we did not find a matching line.
             if not update_current_line:
                 current_line = current_line_backup
@@ -119,26 +135,32 @@ class IndexStatsCommand(QleverCommand):
             # file (old format: "Creating a pair" + names of permutations in
             # line "Writing meta data for ..."; new format: name of
             # permutations already in line "Creating permutations ...").
-            perm_begin, _ = find_next_line(r"INFO:\s*Creating a pair",
-                                           update_current_line=False)
+            perm_begin, _ = find_next_line(
+                r"INFO:\s*Creating a pair", update_current_line=False
+            )
             if perm_begin is None:
                 perm_begin, perm_info = find_next_line(
                     r"INFO:\s*Creating permutations ([A-Z]+ and [A-Z]+)",
-                    update_current_line=False)
+                    update_current_line=False,
+                )
             else:
                 _, perm_info = find_next_line(
                     r"INFO:\s*Writing meta data for ([A-Z]+ and [A-Z]+)",
-                    update_current_line=False)
+                    update_current_line=False,
+                )
             if perm_info is None:
                 break
             perm_begin_and_info.append((perm_begin, perm_info))
-        convert_end = (perm_begin_and_info[0][0] if
-                       len(perm_begin_and_info) > 0 else None)
+        convert_end = (
+            perm_begin_and_info[0][0] if len(perm_begin_and_info) > 0 else None
+        )
         normal_end, _ = find_next_line(r"INFO:\s*Index build completed")
-        text_begin, _ = find_next_line(r"INFO:\s*Adding text index",
-                                       update_current_line=False)
-        text_end, _ = find_next_line(r"INFO:\s*Text index build comp",
-                                     update_current_line=False)
+        text_begin, _ = find_next_line(
+            r"INFO:\s*Adding text index", update_current_line=False
+        )
+        text_end, _ = find_next_line(
+            r"INFO:\s*Text index build comp", update_current_line=False
+        )
         if args.ignore_text_index:
             text_begin = text_end = None
 
@@ -147,9 +169,11 @@ class IndexStatsCommand(QleverCommand):
             log.error("Missing line that index build has started")
             return False
         if overall_begin and not merge_begin:
-            log.error("According to the log file, the index build "
-                      "has started, but is still in its first "
-                      "phase (parsing the input)")
+            log.error(
+                "According to the log file, the index build "
+                "has started, but is still in its first "
+                "phase (parsing the input)"
+            )
             return False
 
         # Helper function that shows the duration for a phase (if the start and
@@ -187,22 +211,24 @@ class IndexStatsCommand(QleverCommand):
         show_duration("Convert to global IDs", [(convert_begin, convert_end)])
         for i in range(len(perm_begin_and_info)):
             perm_begin, perm_info = perm_begin_and_info[i]
-            perm_end = perm_begin_and_info[i + 1][0] if i + 1 < len(
-                    perm_begin_and_info) else normal_end
-            perm_info_text = (perm_info.group(1).replace(" and ", " & ")
-                              if perm_info else f"#{i + 1}")
-            show_duration(f"Permutation {perm_info_text}",
-                          [(perm_begin, perm_end)])
+            perm_end = (
+                perm_begin_and_info[i + 1][0]
+                if i + 1 < len(perm_begin_and_info)
+                else normal_end
+            )
+            perm_info_text = (
+                perm_info.group(1).replace(" and ", " & ") if perm_info else f"#{i + 1}"
+            )
+            show_duration(f"Permutation {perm_info_text}", [(perm_begin, perm_end)])
         show_duration("Text index", [(text_begin, text_end)])
         if text_begin and text_end:
             log.info("")
-            show_duration("TOTAL time",
-                          [(overall_begin, normal_end),
-                           (text_begin, text_end)])
+            show_duration(
+                "TOTAL time", [(overall_begin, normal_end), (text_begin, text_end)]
+            )
         elif normal_end:
             log.info("")
-            show_duration("TOTAL time",
-                          [(overall_begin, normal_end)])
+            show_duration("TOTAL time", [(overall_begin, normal_end)])
         return True
 
     def execute_space(self, args) -> bool:
@@ -252,24 +278,29 @@ class IndexStatsCommand(QleverCommand):
         return True
 
     def execute(self, args) -> bool:
-        ret_value = args.show
+        return_value = True
 
         # The "time" part of the command.
         if not args.only_space:
             log_file_name = f"{args.name}.index-log.txt"
-            self.show(f"Breakdown of the time used for "
-                      f"building the index, based on the timestamps for key "
-                      f"lines in \"{log_file_name}\"", only_show=args.show)
+            self.show(
+                f"Breakdown of the time used for "
+                f"building the index, based on the timestamps for key "
+                f'lines in "{log_file_name}"',
+                only_show=args.show,
+            )
             if not args.show:
-                ret_value &= self.execute_time(args, log_file_name)
+                return_value &= self.execute_time(args, log_file_name)
             if not args.only_time:
                 log.info("")
 
         # The "space" part of the command.
         if not args.only_time:
-            self.show("Breakdown of the space used for building the index",
-                      only_show=args.show)
+            self.show(
+                "Breakdown of the space used for building the index",
+                only_show=args.show,
+            )
             if not args.show:
-                ret_value &= self.execute_space(args)
+                return_value &= self.execute_space(args)
 
-        return ret_value
+        return return_value

qlever/commands/log.py

@@ -47,10 +47,20 @@ class LogCommand(QleverCommand):
         log_cmd += f" {log_file}"
         self.show(log_cmd, only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # Execute the command.
         log.info(f"Follow log file {log_file}, press Ctrl-C to stop"
                  f" following (will not stop the server)")
         log.info("")
-        subprocess.run(log_cmd, shell=True)
+        try:
+            subprocess.run(log_cmd, shell=True)
+            return True
+        except Exception as e:
+            log.error(e)
+            return False
+
+
+
+
+

qlever/commands/query.py

@@ -18,42 +18,82 @@ class QueryCommand(QleverCommand):
         pass
 
     def description(self) -> str:
-        return ("Send a query to a SPARQL endpoint")
+        return "Send a query to a SPARQL endpoint"
 
     def should_have_qleverfile(self) -> bool:
         return False
 
-    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
-        return {"server": ["port"]}
+    def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
+        return {"server": ["port", "access_token"]}
 
     def additional_arguments(self, subparser) -> None:
-        subparser.add_argument("--query", type=str,
-                               default="SELECT * WHERE { ?s ?p ?o } LIMIT 10",
-                               help="SPARQL query to send")
-        subparser.add_argument("--sparql-endpoint", type=str,
-                               help="URL of the SPARQL endpoint")
-        subparser.add_argument("--accept", type=str,
-                               choices=["text/tab-separated-values",
-                                        "text/csv",
-                                        "application/sparql-results+json",
-                                        "application/sparql-results+xml",
-                                        "application/qlever-results+json"],
-                               default="text/tab-separated-values",
-                               help="Accept header for the SPARQL query")
-        subparser.add_argument("--no-time", action="store_true",
-                               default=False,
-                               help="Do not print the (end-to-end) time taken")
+        subparser.add_argument(
+            "query",
+            type=str,
+            nargs="?",
+            default="SELECT * WHERE { ?s ?p ?o } LIMIT 10",
+            help="SPARQL query to send",
+        )
+        subparser.add_argument(
+            "--pin-to-cache",
+            action="store_true",
+            default=False,
+            help="Pin the query to the cache",
+        )
+        subparser.add_argument(
+            "--sparql-endpoint", type=str, help="URL of the SPARQL endpoint"
+        )
+        subparser.add_argument(
+            "--accept",
+            type=str,
+            choices=[
+                "text/tab-separated-values",
+                "text/csv",
+                "application/sparql-results+json",
+                "application/sparql-results+xml",
+                "application/qlever-results+json",
+            ],
+            default="text/tab-separated-values",
+            help="Accept header for the SPARQL query",
+        )
+        subparser.add_argument(
+            "--no-time",
+            action="store_true",
+            default=False,
+            help="Do not print the (end-to-end) time taken",
+        )
 
     def execute(self, args) -> bool:
+        # When pinning to the cache, set `send=0` and request media type
+        # `application/qlever-results+json` so that we get the result size.
+        # Also, we need to provide the access token.
+        if args.pin_to_cache:
+            args.accept = "application/qlever-results+json"
+            curl_cmd_additions = (
+                f" --data pinresult=true --data send=0"
+                f" --data access-token="
+                f"{shlex.quote(args.access_token)}"
+                f" | jq .resultsize | numfmt --grouping"
+                f" | xargs -I {{}} printf"
+                f' "Result pinned to cache,'
+                f' number of rows: {{}}\\n"'
+            )
+        else:
+            curl_cmd_additions = ""
+
         # Show what the command will do.
-        sparql_endpoint = (args.sparql_endpoint if args.sparql_endpoint
-                           else f"localhost:{args.port}")
-        curl_cmd = (f"curl -s {sparql_endpoint}"
-                    f" -H \"Accept: {args.accept}\""
-                    f" --data-urlencode query={shlex.quote(args.query)}")
+        sparql_endpoint = (
+            args.sparql_endpoint if args.sparql_endpoint else f"localhost:{args.port}"
+        )
+        curl_cmd = (
+            f"curl -s {sparql_endpoint}"
+            f' -H "Accept: {args.accept}"'
+            f" --data-urlencode query={shlex.quote(args.query)}"
+            f"{curl_cmd_additions}"
+        )
         self.show(curl_cmd, only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # Launch query.
         try:
@@ -62,8 +102,7 @@ class QueryCommand(QleverCommand):
             time_msecs = round(1000 * (time.time() - start_time))
             if not args.no_time and args.log_level != "NO_LOG":
                 log.info("")
-                log.info(f"Query processing time (end-to-end):"
-                         f" {time_msecs:,d} ms")
+                log.info(f"Query processing time (end-to-end):" f" {time_msecs:,d} ms")
         except Exception as e:
             if args.log_level == "DEBUG":
                 traceback.print_exc()

qlever/commands/setup_config.py

@@ -60,7 +60,7 @@ class SetupConfigCommand(QleverCommand):
         setup_config_cmd += "> Qleverfile"
         self.show(setup_config_cmd, only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # If there is already a Qleverfile in the current directory, exit.
         qleverfile_path = Path("Qleverfile")

qlever/commands/start.py

@@ -72,7 +72,9 @@ class StartCommand(QleverCommand):
         if args.kill_existing_with_same_port:
             args.cmdline_regex = f"^ServerMain.* -p {args.port}"
             args.no_containers = True
-            StopCommand().execute(args)
+            if not StopCommand().execute(args):
+                log.error("Stopping the existing server failed")
+                return False
             log.info("")
 
         # Construct the command line based on the config file.
@@ -115,7 +117,7 @@ class StartCommand(QleverCommand):
         # Show the command line.
         self.show(start_cmd, only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # When running natively, check if the binary exists and works.
         if args.system == "native":
@@ -192,6 +194,7 @@ class StartCommand(QleverCommand):
                 run_command(curl_cmd)
             except Exception as e:
                 log.error(f"Setting the index description failed ({e})")
+                return False
         if args.text_description:
             text_desc = args.text_description
             curl_cmd = (f"curl -Gs http://localhost:{port}/api"
@@ -202,6 +205,7 @@ class StartCommand(QleverCommand):
                 run_command(curl_cmd)
             except Exception as e:
                 log.error(f"Setting the text description failed ({e})")
+                return False
 
         # Kill the tail process. NOTE: `tail_proc.kill()` does not work.
         tail_proc.terminate()
@@ -209,7 +213,9 @@ class StartCommand(QleverCommand):
         # Execute the warmup command.
         if args.warmup_cmd and not args.no_warmup:
             log.info("")
-            WarmupCommand().execute(args)
+            if not WarmupCommand().execute(args):
+                log.error("Warmup failed")
+                return False
 
         # Show cache stats.
         log.info("")

qlever/commands/status.py

@@ -35,7 +35,7 @@ class StatusCommand(QleverCommand):
                   f"the command line matches {args.cmdline_regex}"
                   f" using Python's psutil library", only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # Show the results as a table.
         num_processes_found = 0
@@ -47,3 +47,4 @@ class StatusCommand(QleverCommand):
                 num_processes_found += 1
         if num_processes_found == 0:
             print("No processes found")
+        return True

qlever/commands/stop.py

@@ -49,7 +49,7 @@ class StopCommand(QleverCommand):
                             f"\"{args.server_container}\"")
         self.show(description, only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # First check if there is container running and if yes, stop and remove
         # it (unless the user has specified `--no-containers`).
@@ -90,14 +90,12 @@ class StopCommand(QleverCommand):
                     return False
                 return True
 
-        # No matching process found.
+        # If no matching process found, show a message and the output of the
+        # status command.
         message = "No matching process found" if args.no_containers else \
                   "No matching process or container found"
         log.error(message)
-
-        # Show output of status command.
         args.cmdline_regex = "^ServerMain.* -i [^ ]*"
         log.info("")
         StatusCommand().execute(args)
-
-        return False
+        return True

qlever/commands/system_info.py

@@ -58,7 +58,7 @@ class SystemInfoCommand(QleverCommand):
         # Say what the command is doing.
         self.show("Show system information and Qleverfile", only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # Show system information.
         show_heading("System Information")

qlever/commands/ui.py

@@ -70,8 +70,10 @@ class UiCommand(QleverCommand):
             "\n".join(["Stop running containers", pull_cmd, run_cmd, exec_cmd]),
             only_show=args.show,
         )
-        if qlever_is_running_in_container or args.show:
+        if qlever_is_running_in_container:
             return False
+        if args.show:
+            return True
 
         # Stop running containers.
         for container_system in Containerize.supported_systems():

qlever/commands/warmup.py

@@ -30,7 +30,7 @@ class WarmupCommand(QleverCommand):
         # Show what the command is doing.
         self.show(args.warmup_cmd, only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # Execute the command.
         try:

qlever/qlever_main.py

@@ -35,31 +35,38 @@ def main():
         log.info("")
         log.info(colored(f"Command: {args.command}", attrs=["bold"]))
         log.info("")
-        command_object.execute(args)
+        commandWasSuccesful = command_object.execute(args)
         log.info("")
+        if not commandWasSuccesful:
+            exit(1)
     except KeyboardInterrupt:
         log.info("")
         log.info("Ctrl-C pressed, exiting ...")
         log.info("")
-        exit(0)
+        exit(1)
     except Exception as e:
         # Check if it's a certain kind of `AttributeError` and give a hint in
         # that case.
         match_error = re.search(r"object has no attribute '(.+)'", str(e))
-        match_trace = re.search(r"(qlever/commands/.+\.py)\", line (\d+)",
-                                traceback.format_exc())
+        match_trace = re.search(
+            r"(qlever/commands/.+\.py)\", line (\d+)", traceback.format_exc()
+        )
         if isinstance(e, AttributeError) and match_error and match_trace:
             attribute = match_error.group(1)
             trace_command = match_trace.group(1)
             trace_line = match_trace.group(2)
             log.error(f"{e} in `{trace_command}` at line {trace_line}")
             log.info("")
-            log.info(f"Likely cause: you used `args.{attribute}`, but it was "
-                     f"neither defined in `relevant_qleverfile_arguments` "
-                     f"nor in `additional_arguments`")
+            log.info(
+                f"Likely cause: you used `args.{attribute}`, but it was "
+                f"neither defined in `relevant_qleverfile_arguments` "
+                f"nor in `additional_arguments`"
+            )
             log.info("")
-            log.info(f"If you did not implement `{trace_command}` yourself, "
-                     f"please report this issue")
+            log.info(
+                f"If you did not implement `{trace_command}` yourself, "
+                f"please report this issue"
+            )
             log.info("")
         else:
             log.error(f"An unexpected error occurred: {e}")

{qlever-0.5.12.dist-info → qlever-0.5.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qlever
-Version: 0.5.12
+Version: 0.5.15
 Summary: Script for using the QLever SPARQL engine.
 Author-email: Hannah Bast <bast@cs.uni-freiburg.de>
 License: Apache-2.0