qlever 0.5.11__py3-none-any.whl → 0.5.15__py3-none-any.whl

qlever/commands/get_data.py

@@ -31,7 +31,7 @@ class GetDataCommand(QleverCommand):
         # Construct the command line and show it.
         self.show(args.get_data_cmd, only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # Execute the command line.
         try:

qlever/commands/index.py

@@ -3,12 +3,12 @@ from __future__ import annotations
 import glob
 import json
 import shlex
+import re
 
 from qlever.command import QleverCommand
 from qlever.containerize import Containerize
 from qlever.log import log
-from qlever.util import (get_existing_index_files, get_total_file_size,
-                         run_command)
+from qlever.util import get_existing_index_files, get_total_file_size, run_command
 
 
 class IndexCommand(QleverCommand):
@@ -20,24 +20,36 @@ class IndexCommand(QleverCommand):
         pass
 
     def description(self) -> str:
-        return ("Build the index for a given RDF dataset")
+        return "Build the index for a given RDF dataset"
 
     def should_have_qleverfile(self) -> bool:
         return True
 
-    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
-        return {"data": ["name", "format"],
-                "index": ["input_files", "cat_input_files", "multi_input_json",
-                          "parallel_parsing", "settings_json", "index_binary",
-                          "only_pso_and_pos_permutations", "use_patterns",
-                          "text_index", "stxxl_memory"],
-                "runtime": ["system", "image", "index_container"]}
+    def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
+        return {
+            "data": ["name", "format"],
+            "index": [
+                "input_files",
+                "cat_input_files",
+                "multi_input_json",
+                "parallel_parsing",
+                "settings_json",
+                "index_binary",
+                "only_pso_and_pos_permutations",
+                "use_patterns",
+                "text_index",
+                "stxxl_memory",
+            ],
+            "runtime": ["system", "image", "index_container"],
+        }
 
     def additional_arguments(self, subparser) -> None:
         subparser.add_argument(
-                "--overwrite-existing", action="store_true",
-                default=False,
-                help="Overwrite an existing index, think twice before using.")
+            "--overwrite-existing",
+            action="store_true",
+            default=False,
+            help="Overwrite an existing index, think twice before using.",
+        )
 
     # Exception for invalid JSON.
     class InvalidInputJson(Exception):
@@ -48,22 +60,29 @@ class IndexCommand(QleverCommand):
 
     # Helper function to get command line options from JSON.
     def get_input_options_for_json(self, args) -> str:
-        # Parse the JSON.
+        # Parse the JSON. If `args.multi_input_json` look like JSONL, turn
+        # it into a JSON array.
         try:
+            jsonl_line_regex = re.compile(r"^\s*\{.*\}\s*$")
+            jsonl_lines = args.multi_input_json.split("\n")
+            if all(re.match(jsonl_line_regex, line) for line in jsonl_lines):
+                args.multi_input_json = "[" + ", ".join(jsonl_lines) + "]"
             input_specs = json.loads(args.multi_input_json)
         except Exception as e:
             raise self.InvalidInputJson(
-                    f"Failed to parse `MULTI_INPUT_JSON` ({e})",
-                    args.multi_input_json)
+                f"Failed to parse `MULTI_INPUT_JSON` as either JSON or JSONL ({e})",
+                args.multi_input_json,
+            )
         # Check that it is an array of length at least one.
         if not isinstance(input_specs, list):
             raise self.InvalidInputJson(
-                    "`MULTI_INPUT_JSON` must be a JSON array",
-                    args.multi_input_json)
+                "`MULTI_INPUT_JSON` must be a JSON array", args.multi_input_json
+            )
         if len(input_specs) == 0:
             raise self.InvalidInputJson(
-                    "`MULTI_INPUT_JSON` must contain at least one element",
-                    args.multi_input_json)
+                "`MULTI_INPUT_JSON` must contain at least one element",
+                args.multi_input_json,
+            )
         # For each of the maps, construct the corresponding command-line
         # options to the index binary.
         input_options = []
@@ -71,35 +90,77 @@ class IndexCommand(QleverCommand):
             # Check that `input_spec` is a dictionary.
             if not isinstance(input_spec, dict):
                 raise self.InvalidInputJson(
-                        f"Element {i} in `MULTI_INPUT_JSON` must be a JSON "
-                        "object",
-                        input_spec)
+                    f"Element {i} in `MULTI_INPUT_JSON` must be a JSON " "object",
+                    input_spec,
+                )
             # For each `input_spec`, we must have a command.
             if "cmd" not in input_spec:
                 raise self.InvalidInputJson(
-                        f"Element {i} in `MULTI_INPUT_JSON` must contain a "
-                        "key `cmd`",
-                        input_spec)
-            input_cmd = input_spec["cmd"]
+                    f"Element {i} in `MULTI_INPUT_JSON` must contain a " "key `cmd`",
+                    input_spec,
+                )
+            # If the command contains a `{}` placeholder, we need a `for-each`
+            # key` specifying the pattern for the placeholder values, and vice
+            # versa.
+            if "{}" in input_spec["cmd"] and "for-each" not in input_spec:
+                raise self.InvalidInputJson(
+                    f"Element {i} in `MULTI_INPUT_JSON` must contain a "
+                    "key `for-each` if the command contains a placeholder "
+                    "`{}`",
+                    input_spec,
+                )
+            if "for-each" in input_spec and "{}" not in input_spec["cmd"]:
+                raise self.InvalidInputJson(
+                    f"Element {i} in `MULTI_INPUT_JSON` contains a "
+                    "key `for-each`, but the command does not contain a "
+                    "placeholder `{{}}`",
+                    input_spec,
+                )
+            # Get all commands. This is just the value of the `cmd` key if no
+            # `for-each` key is specified. Otherwise, we have a command for
+            # each file matching the pattern.
+            if "for-each" not in input_spec:
+                input_cmds = [input_spec["cmd"]]
+            else:
+                try:
+                    files = sorted(glob.glob(input_spec["for-each"]))
+                except Exception as e:
+                    raise self.InvalidInputJson(
+                        f"Element {i} in `MULTI_INPUT_JSON` contains an "
+                        f"invalid `for-each` pattern: {e}",
+                        input_spec,
+                    )
+                input_cmds = [input_spec["cmd"].format(file) for file in files]
             # The `format`, `graph`, and `parallel` keys are optional.
             input_format = input_spec.get("format", args.format)
             input_graph = input_spec.get("graph", "-")
             input_parallel = input_spec.get("parallel", "false")
             # There must not be any other keys.
-            extra_keys = input_spec.keys() - {"cmd", "format", "graph", "parallel"}
+            extra_keys = input_spec.keys() - {
+                "cmd",
+                "format",
+                "graph",
+                "parallel",
+                "for-each",
+            }
             if extra_keys:
                 raise self.InvalidInputJson(
-                        f"Element {i} in `MULTI_INPUT_JSON` must only contain "
-                        "the keys `format`, `graph`, and `parallel`. Contains "
-                        "extra keys {extra_keys}.",
-                        input_spec)
+                    f"Element {i} in `MULTI_INPUT_JSON` must only contain "
+                    "the keys `format`, `graph`, and `parallel`. Contains "
+                    "extra keys {extra_keys}.",
+                    input_spec,
+                )
             # Add the command-line options for this input stream. We use
-            # process substitution `<(...)` as a convenient way to handle
-            # an input stream just like a file. This is not POSIX compliant,
-            # but supported by various shells, including bash and zsh.
-            input_options.append(
-                    f"-f <({input_cmd}) -F {input_format} "
-                    f"-g \"{input_graph}\" -p {input_parallel}")
+            # process substitution `<(...)` as a convenient way to handle an
+            # input stream just like a file. This is not POSIX compliant, but
+            # supported by various shells, including bash and zsh. If
+            # `for-each` is specified, add one command for each matching file.
+            for input_cmd in input_cmds:
+                input_option = f"-f <({input_cmd}) -g {input_graph}"
+                input_option += f" -F {input_format}"
+                if input_parallel == "true":
+                    input_option += " -p true"
+                input_options.append(input_option)
         # Return the concatenated command-line options.
         return " ".join(input_options)
 
@@ -108,11 +169,13 @@ class IndexCommand(QleverCommand):
         # basename of the index, and the settings file). There are two ways
         # to specify the input: via a single stream or via multiple streams.
         if args.cat_input_files and not args.multi_input_json:
-            index_cmd = (f"{args.cat_input_files} | {args.index_binary}"
-                         f" -i {args.name} -s {args.name}.settings.json"
-                         f" -F {args.format} -f -")
+            index_cmd = (
+                f"{args.cat_input_files} | {args.index_binary}"
+                f" -i {args.name} -s {args.name}.settings.json"
+                f" -F {args.format} -f -"
+            )
             if args.parallel_parsing:
-                index_cmd += (f" -p {args.parallel_parsing}")
+                index_cmd += f" -p {args.parallel_parsing}"
         elif args.multi_input_json and not args.cat_input_files:
             try:
                 input_options = self.get_input_options_for_json(args)
@@ -121,13 +184,17 @@ class IndexCommand(QleverCommand):
                 log.info("")
                 log.info(e.additional_info)
                 return False
-            index_cmd = (f"{args.index_binary}"
-                         f" -i {args.name} -s {args.name}.settings.json"
-                         f" {input_options}")
+            index_cmd = (
+                f"{args.index_binary}"
+                f" -i {args.name} -s {args.name}.settings.json"
+                f" {input_options}"
+            )
         else:
-            log.error("Specify exactly one of `CAT_INPUT_FILES` (for a "
-                      "single input stream) or `MULTI_INPUT_JSON` (for "
-                      "multiple input streams)")
+            log.error(
+                "Specify exactly one of `CAT_INPUT_FILES` (for a "
+                "single input stream) or `MULTI_INPUT_JSON` (for "
+                "multiple input streams)"
+            )
             log.info("")
             log.info("See `qlever index --help` for more information")
             return False
@@ -137,12 +204,11 @@ class IndexCommand(QleverCommand):
             index_cmd += " --only-pso-and-pos-permutations --no-patterns"
         if not args.use_patterns:
             index_cmd += " --no-patterns"
-        if args.text_index in \
-                ["from_text_records", "from_text_records_and_literals"]:
-            index_cmd += (f" -w {args.name}.wordsfile.tsv"
-                          f" -d {args.name}.docsfile.tsv")
-        if args.text_index in \
-                ["from_literals", "from_text_records_and_literals"]:
+        if args.text_index in ["from_text_records", "from_text_records_and_literals"]:
+            index_cmd += (
+                f" -w {args.name}.wordsfile.tsv" f" -d {args.name}.docsfile.tsv"
+            )
+        if args.text_index in ["from_literals", "from_text_records_and_literals"]:
             index_cmd += " --text-words-from-literals"
         if args.stxxl_memory:
             index_cmd += f" --stxxl-memory {args.stxxl_memory}"
@@ -150,38 +216,42 @@ class IndexCommand(QleverCommand):
 
         # If the total file size is larger than 10 GB, set ulimit (such that a
         # large number of open files is allowed).
-        total_file_size = get_total_file_size(
-                shlex.split(args.input_files))
+        total_file_size = get_total_file_size(shlex.split(args.input_files))
         if total_file_size > 1e10:
             index_cmd = f"ulimit -Sn 1048576; {index_cmd}"
 
         # Run the command in a container (if so desired).
         if args.system in Containerize.supported_systems():
             index_cmd = Containerize().containerize_command(
-                    index_cmd,
-                    args.system, "run --rm",
-                    args.image,
-                    args.index_container,
-                    volumes=[("$(pwd)", "/index")],
-                    working_directory="/index")
+                index_cmd,
+                args.system,
+                "run --rm",
+                args.image,
+                args.index_container,
+                volumes=[("$(pwd)", "/index")],
+                working_directory="/index",
+            )
 
         # Command for writing the settings JSON to a file.
-        settings_json_cmd = (f"echo {shlex.quote(args.settings_json)} "
-                             f"> {args.name}.settings.json")
+        settings_json_cmd = (
+            f"echo {shlex.quote(args.settings_json)} " f"> {args.name}.settings.json"
+        )
 
         # Show the command line.
         self.show(f"{settings_json_cmd}\n{index_cmd}", only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # When running natively, check if the binary exists and works.
         if args.system == "native":
             try:
                 run_command(f"{args.index_binary} --help")
             except Exception as e:
-                log.error(f"Running \"{args.index_binary}\" failed, "
-                          f"set `--index-binary` to a different binary or "
-                          f"set `--system to a container system`")
+                log.error(
+                    f'Running "{args.index_binary}" failed, '
+                    f"set `--index-binary` to a different binary or "
+                    f"set `--system to a container system`"
+                )
                 log.info("")
                 log.info(f"The error message was: {e}")
                 return False
@@ -189,28 +259,29 @@ class IndexCommand(QleverCommand):
         # Check if all of the input files exist.
         for pattern in shlex.split(args.input_files):
             if len(glob.glob(pattern)) == 0:
-                log.error(f"No file matching \"{pattern}\" found")
+                log.error(f'No file matching "{pattern}" found')
                 log.info("")
-                log.info("Did you call `qlever get-data`? If you did, check "
-                         "GET_DATA_CMD and INPUT_FILES in the QLeverfile")
+                log.info(
+                    "Did you call `qlever get-data`? If you did, check "
+                    "GET_DATA_CMD and INPUT_FILES in the QLeverfile"
+                )
                 return False
 
         # Check if index files (name.index.*) already exist.
         existing_index_files = get_existing_index_files(args.name)
         if len(existing_index_files) > 0 and not args.overwrite_existing:
             log.error(
-                    f"Index files for basename \"{args.name}\" found, if you "
-                    f"want to overwrite them, use --overwrite-existing")
+                f'Index files for basename "{args.name}" found, if you '
+                f"want to overwrite them, use --overwrite-existing"
+            )
             log.info("")
             log.info(f"Index files found: {existing_index_files}")
             return False
 
         # Remove already existing container.
-        if args.system in Containerize.supported_systems() \
-                and args.overwrite_existing:
+        if args.system in Containerize.supported_systems() and args.overwrite_existing:
             if Containerize.is_running(args.system, args.index_container):
-                log.info("Another index process is running, trying to stop "
-                         "it ...")
+                log.info("Another index process is running, trying to stop " "it ...")
                 log.info("")
                 try:
                     run_command(f"{args.system} rm -f {args.index_container}")

qlever/commands/index_stats.py

@@ -18,32 +18,45 @@ class IndexStatsCommand(QleverCommand):
         pass
 
     def description(self) -> str:
-        return ("Breakdown of the time and space used for the index build")
+        return "Breakdown of the time and space used for the index build"
 
     def should_have_qleverfile(self) -> bool:
         return False
 
-    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
+    def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
         return {"data": ["name"]}
 
     def additional_arguments(self, subparser) -> None:
-        subparser.add_argument("--only-time", action="store_true",
-                               default=False,
-                               help="Show only the time used")
-        subparser.add_argument("--only-space", action="store_true",
-                               default=False,
-                               help="Show only the space used")
-        subparser.add_argument("--ignore-text-index", action="store_true",
-                               default=False,
-                               help="Ignore the text index")
-        subparser.add_argument("--time-unit",
-                               choices=["s", "min", "h", "auto"],
-                               default="auto",
-                               help="The time unit")
-        subparser.add_argument("--size-unit",
-                               choices=["B", "MB", "GB", "TB", "auto"],
-                               default="auto",
-                               help="The size unit")
+        subparser.add_argument(
+            "--only-time",
+            action="store_true",
+            default=False,
+            help="Show only the time used",
+        )
+        subparser.add_argument(
+            "--only-space",
+            action="store_true",
+            default=False,
+            help="Show only the space used",
+        )
+        subparser.add_argument(
+            "--ignore-text-index",
+            action="store_true",
+            default=False,
+            help="Ignore the text index",
+        )
+        subparser.add_argument(
+            "--time-unit",
+            choices=["s", "min", "h", "auto"],
+            default="auto",
+            help="The time unit",
+        )
+        subparser.add_argument(
+            "--size-unit",
+            choices=["B", "MB", "GB", "TB", "auto"],
+            default="auto",
+            help="The size unit",
+        )
 
     def execute_time(self, args, log_file_name) -> bool:
         """
@@ -65,8 +78,9 @@ class IndexStatsCommand(QleverCommand):
                 with open(text_log_file_name, "r") as text_log_file:
                     lines.extend(text_log_file.readlines())
         except Exception as e:
-            log.error(f"Problem reading text index log file "
-                      f"{text_log_file_name}: {e}")
+            log.error(
+                f"Problem reading text index log file " f"{text_log_file_name}: {e}"
+            )
             return False
 
         # Helper function that finds the next line matching the given `regex`,
@@ -95,12 +109,14 @@ class IndexStatsCommand(QleverCommand):
                 if regex_match:
                     try:
                         return datetime.strptime(
-                                re.match(timestamp_regex, line).group(),
-                                timestamp_format), regex_match
+                            re.match(timestamp_regex, line).group(), timestamp_format
+                        ), regex_match
                     except Exception as e:
-                        log.error(f"Could not parse timestamp of form "
-                                  f"\"{timestamp_regex}\" from line "
-                                  f" \"{line.rstrip()}\" ({e})")
+                        log.error(
+                            f"Could not parse timestamp of form "
+                            f'"{timestamp_regex}" from line '
+                            f' "{line.rstrip()}" ({e})'
+                        )
             # If we get here, we did not find a matching line.
             if not update_current_line:
                 current_line = current_line_backup
@@ -119,26 +135,32 @@ class IndexStatsCommand(QleverCommand):
             # file (old format: "Creating a pair" + names of permutations in
             # line "Writing meta data for ..."; new format: name of
             # permutations already in line "Creating permutations ...").
-            perm_begin, _ = find_next_line(r"INFO:\s*Creating a pair",
-                                           update_current_line=False)
+            perm_begin, _ = find_next_line(
+                r"INFO:\s*Creating a pair", update_current_line=False
+            )
             if perm_begin is None:
                 perm_begin, perm_info = find_next_line(
                     r"INFO:\s*Creating permutations ([A-Z]+ and [A-Z]+)",
-                    update_current_line=False)
+                    update_current_line=False,
+                )
             else:
                 _, perm_info = find_next_line(
                     r"INFO:\s*Writing meta data for ([A-Z]+ and [A-Z]+)",
-                    update_current_line=False)
+                    update_current_line=False,
+                )
             if perm_info is None:
                 break
             perm_begin_and_info.append((perm_begin, perm_info))
-        convert_end = (perm_begin_and_info[0][0] if
-                       len(perm_begin_and_info) > 0 else None)
+        convert_end = (
+            perm_begin_and_info[0][0] if len(perm_begin_and_info) > 0 else None
+        )
         normal_end, _ = find_next_line(r"INFO:\s*Index build completed")
-        text_begin, _ = find_next_line(r"INFO:\s*Adding text index",
-                                       update_current_line=False)
-        text_end, _ = find_next_line(r"INFO:\s*Text index build comp",
-                                     update_current_line=False)
+        text_begin, _ = find_next_line(
+            r"INFO:\s*Adding text index", update_current_line=False
+        )
+        text_end, _ = find_next_line(
+            r"INFO:\s*Text index build comp", update_current_line=False
+        )
         if args.ignore_text_index:
             text_begin = text_end = None
 
@@ -147,9 +169,11 @@ class IndexStatsCommand(QleverCommand):
             log.error("Missing line that index build has started")
             return False
         if overall_begin and not merge_begin:
-            log.error("According to the log file, the index build "
-                      "has started, but is still in its first "
-                      "phase (parsing the input)")
+            log.error(
+                "According to the log file, the index build "
+                "has started, but is still in its first "
+                "phase (parsing the input)"
+            )
             return False
 
         # Helper function that shows the duration for a phase (if the start and
@@ -187,22 +211,24 @@ class IndexStatsCommand(QleverCommand):
         show_duration("Convert to global IDs", [(convert_begin, convert_end)])
         for i in range(len(perm_begin_and_info)):
             perm_begin, perm_info = perm_begin_and_info[i]
-            perm_end = perm_begin_and_info[i + 1][0] if i + 1 < len(
-                    perm_begin_and_info) else normal_end
-            perm_info_text = (perm_info.group(1).replace(" and ", " & ")
-                              if perm_info else f"#{i + 1}")
-            show_duration(f"Permutation {perm_info_text}",
-                          [(perm_begin, perm_end)])
+            perm_end = (
+                perm_begin_and_info[i + 1][0]
+                if i + 1 < len(perm_begin_and_info)
+                else normal_end
+            )
+            perm_info_text = (
+                perm_info.group(1).replace(" and ", " & ") if perm_info else f"#{i + 1}"
+            )
+            show_duration(f"Permutation {perm_info_text}", [(perm_begin, perm_end)])
         show_duration("Text index", [(text_begin, text_end)])
         if text_begin and text_end:
             log.info("")
-            show_duration("TOTAL time",
-                          [(overall_begin, normal_end),
-                           (text_begin, text_end)])
+            show_duration(
+                "TOTAL time", [(overall_begin, normal_end), (text_begin, text_end)]
+            )
         elif normal_end:
             log.info("")
-            show_duration("TOTAL time",
-                          [(overall_begin, normal_end)])
+            show_duration("TOTAL time", [(overall_begin, normal_end)])
         return True
 
     def execute_space(self, args) -> bool:
@@ -252,24 +278,29 @@ class IndexStatsCommand(QleverCommand):
         return True
 
     def execute(self, args) -> bool:
-        ret_value = args.show
+        return_value = True
 
         # The "time" part of the command.
         if not args.only_space:
             log_file_name = f"{args.name}.index-log.txt"
-            self.show(f"Breakdown of the time used for "
-                      f"building the index, based on the timestamps for key "
-                      f"lines in \"{log_file_name}\"", only_show=args.show)
+            self.show(
+                f"Breakdown of the time used for "
+                f"building the index, based on the timestamps for key "
+                f'lines in "{log_file_name}"',
+                only_show=args.show,
+            )
             if not args.show:
-                ret_value &= self.execute_time(args, log_file_name)
+                return_value &= self.execute_time(args, log_file_name)
             if not args.only_time:
                 log.info("")
 
         # The "space" part of the command.
         if not args.only_time:
-            self.show("Breakdown of the space used for building the index",
-                      only_show=args.show)
+            self.show(
+                "Breakdown of the space used for building the index",
+                only_show=args.show,
+            )
             if not args.show:
-                ret_value &= self.execute_space(args)
+                return_value &= self.execute_space(args)
 
-        return ret_value
+        return return_value

qlever/commands/log.py

@@ -47,10 +47,20 @@ class LogCommand(QleverCommand):
         log_cmd += f" {log_file}"
         self.show(log_cmd, only_show=args.show)
         if args.show:
-            return False
+            return True
 
         # Execute the command.
         log.info(f"Follow log file {log_file}, press Ctrl-C to stop"
                  f" following (will not stop the server)")
         log.info("")
-        subprocess.run(log_cmd, shell=True)
+        try:
+            subprocess.run(log_cmd, shell=True)
+            return True
+        except Exception as e:
+            log.error(e)
+            return False
+
+
+
+
+