qlever 0.2.5__py3-none-any.whl → 0.5.41__py3-none-any.whl

qlever/commands/index.py

@@ -0,0 +1,333 @@
+from __future__ import annotations
+
+import glob
+import json
+import re
+import shlex
+
+from qlever.command import QleverCommand
+from qlever.containerize import Containerize
+from qlever.log import log
+from qlever.util import (
+    binary_exists,
+    get_existing_index_files,
+    get_total_file_size,
+    run_command,
+)
+
+
+class IndexCommand(QleverCommand):
+    """
+    Class for executing the `index` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Build the index for a given RDF dataset"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {
+            "data": ["name", "format"],
+            "index": [
+                "input_files",
+                "cat_input_files",
+                "encode_as_id",
+                "multi_input_json",
+                "parallel_parsing",
+                "settings_json",
+                "vocabulary_type",
+                "index_binary",
+                "only_pso_and_pos_permutations",
+                "ulimit",
+                "use_patterns",
+                "add_has_word_triples",
+                "text_index",
+                "stxxl_memory",
+                "parser_buffer_size",
+            ],
+            "runtime": ["system", "image", "index_container"],
+        }
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "--overwrite-existing",
+            action="store_true",
+            default=False,
+            help="Overwrite an existing index, think twice before using this",
+        )
+
+    # Exception for invalid JSON.
+    class InvalidInputJson(Exception):
+        def __init__(self, error_message, additional_info):
+            self.error_message = error_message
+            self.additional_info = additional_info
+            super().__init__()
+
+    # Helper function to get command line options from JSON.
+    def get_input_options_for_json(self, args) -> str:
+        # Parse the JSON. If `args.multi_input_json` look like JSONL, turn
+        # it into a JSON array.
+        try:
+            jsonl_line_regex = re.compile(r"^\s*\{.*\}\s*$")
+            jsonl_lines = args.multi_input_json.split("\n")
+            if all(re.match(jsonl_line_regex, line) for line in jsonl_lines):
+                args.multi_input_json = "[" + ", ".join(jsonl_lines) + "]"
+            input_specs = json.loads(args.multi_input_json)
+        except Exception as e:
+            raise self.InvalidInputJson(
+                f"Failed to parse `MULTI_INPUT_JSON` as either JSON or JSONL ({e})",
+                args.multi_input_json,
+            )
+        # Check that it is an array of length at least one.
+        if not isinstance(input_specs, list):
+            raise self.InvalidInputJson(
+                "`MULTI_INPUT_JSON` must be a JSON array",
+                args.multi_input_json,
+            )
+        if len(input_specs) == 0:
+            raise self.InvalidInputJson(
+                "`MULTI_INPUT_JSON` must contain at least one element",
+                args.multi_input_json,
+            )
+        # For each of the maps, construct the corresponding command-line
+        # options to the index binary.
+        input_options = []
+        for i, input_spec in enumerate(input_specs):
+            # Check that `input_spec` is a dictionary.
+            if not isinstance(input_spec, dict):
+                raise self.InvalidInputJson(
+                    f"Element {i} in `MULTI_INPUT_JSON` must be a JSON object",
+                    input_spec,
+                )
+            # For each `input_spec`, we must have a command.
+            if "cmd" not in input_spec:
+                raise self.InvalidInputJson(
+                    f"Element {i} in `MULTI_INPUT_JSON` must contain a "
+                    "key `cmd`",
+                    input_spec,
+                )
+            # If the command contains a `{}` placeholder, we need a `for-each`
+            # key` specifying the pattern for the placeholder values, and vice
+            # versa.
+            if "{}" in input_spec["cmd"] and "for-each" not in input_spec:
+                raise self.InvalidInputJson(
+                    f"Element {i} in `MULTI_INPUT_JSON` must contain a "
+                    "key `for-each` if the command contains a placeholder "
+                    "`{}`",
+                    input_spec,
+                )
+            if "for-each" in input_spec and "{}" not in input_spec["cmd"]:
+                raise self.InvalidInputJson(
+                    f"Element {i} in `MULTI_INPUT_JSON` contains a "
+                    "key `for-each`, but the command does not contain a "
+                    "placeholder `{{}}`",
+                    input_spec,
+                )
+            # Get all commands. This is just the value of the `cmd` key if no
+            # `for-each` key is specified. Otherwise, we have a command for
+            # each file matching the pattern.
+            if "for-each" not in input_spec:
+                input_cmds = [input_spec["cmd"]]
+            else:
+                try:
+                    files = sorted(glob.glob(input_spec["for-each"]))
+                except Exception as e:
+                    raise self.InvalidInputJson(
+                        f"Element {i} in `MULTI_INPUT_JSON` contains an "
+                        f"invalid `for-each` pattern: {e}",
+                        input_spec,
+                    )
+                input_cmds = [input_spec["cmd"].format(file) for file in files]
+            # The `format`, `graph`, and `parallel` keys are optional.
+            input_format = input_spec.get("format", args.format)
+            input_graph = input_spec.get("graph", "-")
+            input_parallel = input_spec.get("parallel", "false")
+            # There must not be any other keys.
+            extra_keys = input_spec.keys() - {
+                "cmd",
+                "format",
+                "graph",
+                "parallel",
+                "for-each",
+            }
+            if extra_keys:
+                raise self.InvalidInputJson(
+                    f"Element {i} in `MULTI_INPUT_JSON` must only contain "
+                    "the keys `format`, `graph`, and `parallel`. Contains "
+                    f"extra keys {extra_keys}.",
+                    input_spec,
+                )
+            # Add the command-line options for this input stream. We use
+            # process substitution `<(...)` as a convenient way to handle an
+            # input stream just like a file. This is not POSIX compliant, but
+            # supported by various shells, including bash and zsh. If
+            # `for-each` is specified, add one command for each matching file.
+            for input_cmd in input_cmds:
+                input_option = f"-f <({input_cmd}) -g {input_graph}"
+                input_option += f" -F {input_format}"
+                if input_parallel == "true":
+                    input_option += " -p true"
+                else:
+                    input_option += " -p false"
+                input_options.append(input_option)
+        # Return the concatenated command-line options.
+        return " ".join(input_options)
+
+    def execute(self, args) -> bool:
+        # The mandatory part of the command line (specifying the input, the
+        # basename of the index, and the settings file). There are two ways
+        # to specify the input: via a single stream or via multiple streams.
+        if args.cat_input_files and not args.multi_input_json:
+            index_cmd = (
+                f"{args.cat_input_files} | {args.index_binary}"
+                f" -i {args.name} -s {args.name}.settings.json"
+                f" --vocabulary-type {args.vocabulary_type}"
+                f" -F {args.format} -f -"
+            )
+            if args.parallel_parsing:
+                index_cmd += f" -p {args.parallel_parsing}"
+        elif args.multi_input_json and not args.cat_input_files:
+            try:
+                input_options = self.get_input_options_for_json(args)
+            except self.InvalidInputJson as e:
+                log.error(e.error_message)
+                log.info("")
+                log.info(e.additional_info)
+                return False
+            index_cmd = (
+                f"{args.index_binary}"
+                f" -i {args.name} -s {args.name}.settings.json"
+                f" --vocabulary-type {args.vocabulary_type}"
+                f" {input_options}"
+            )
+        else:
+            log.error(
+                "Specify exactly one of `CAT_INPUT_FILES` (for a "
+                "single input stream) or `MULTI_INPUT_JSON` (for "
+                "multiple input streams)"
+            )
+            log.info("")
+            log.info("See `qlever index --help` for more information")
+            return False
+
+        # Add remaining options.
+        if args.encode_as_id:
+            index_cmd += f" --encode-as-id {args.encode_as_id}"
+        if args.only_pso_and_pos_permutations:
+            index_cmd += " --only-pso-and-pos-permutations"
+        if args.use_patterns == "no":
+            index_cmd += " --no-patterns"
+        if args.add_has_word_triples:
+            index_cmd += " --add-has-word-triples"
+        if args.text_index in [
+            "from_text_records",
+            "from_text_records_and_literals",
+        ]:
+            index_cmd += (
+                f" -w {args.name}.wordsfile.tsv -d {args.name}.docsfile.tsv"
+            )
+        if args.text_index in [
+            "from_literals",
+            "from_text_records_and_literals",
+        ]:
+            index_cmd += " --text-words-from-literals"
+        if args.stxxl_memory:
+            index_cmd += f" --stxxl-memory {args.stxxl_memory}"
+        if args.parser_buffer_size:
+            index_cmd += f" --parser-buffer-size {args.parser_buffer_size}"
+        index_cmd += f" 2>&1 | tee {args.name}.index-log.txt"
+
+        # If the total file size is larger than 10 GB, set ulimit (such that a
+        # large number of open files is allowed).
+        total_file_size = get_total_file_size(shlex.split(args.input_files))
+        if args.ulimit is not None:
+            index_cmd = f"ulimit -Sn {args.ulimit} && {index_cmd}"
+        elif total_file_size > 1e10:
+            index_cmd = f"ulimit -Sn 500000 && {index_cmd}"
+
+        # Run the command in a container (if so desired).
+        if args.system in Containerize.supported_systems():
+            index_cmd = Containerize().containerize_command(
+                index_cmd,
+                args.system,
+                "run --rm",
+                args.image,
+                args.index_container,
+                volumes=[("$(pwd)", "/index")],
+                working_directory="/index",
+            )
+
+        # Command for writing the settings JSON to a file.
+        settings_json_cmd = (
+            f"echo {shlex.quote(args.settings_json)} "
+            f"> {args.name}.settings.json"
+        )
+
+        # Show the command line.
+        self.show(f"{settings_json_cmd}\n{index_cmd}", only_show=args.show)
+        if args.show:
+            return True
+
+        # When running natively, check if the binary exists and works.
+        if args.system == "native":
+            if not binary_exists(args.index_binary, "index-binary"):
+                return False
+
+        # Check if all of the input files exist.
+        for pattern in shlex.split(args.input_files):
+            if len(glob.glob(pattern)) == 0:
+                log.error(f'No file matching "{pattern}" found')
+                log.info("")
+                log.info(
+                    "Did you call `qlever get-data`? If you did, check "
+                    "GET_DATA_CMD and INPUT_FILES in the QLeverfile"
+                )
+                return False
+
+        # Check if index files (name.index.*) already exist.
+        existing_index_files = get_existing_index_files(args.name)
+        if len(existing_index_files) > 0 and not args.overwrite_existing:
+            log.error(
+                f'Index files for basename "{args.name}" found, if you '
+                f"want to overwrite them, use --overwrite-existing"
+            )
+            log.info("")
+            log.info(f"Index files found: {existing_index_files}")
+            return False
+
+        # Remove already existing container.
+        if (
+            args.system in Containerize.supported_systems()
+            and args.overwrite_existing
+        ):
+            if Containerize.is_running(args.system, args.index_container):
+                log.info(
+                    "Another index process is running, trying to stop it ..."
+                )
+                log.info("")
+                try:
+                    run_command(f"{args.system} rm -f {args.index_container}")
+                except Exception as e:
+                    log.error(f"Removing existing container failed: {e}")
+                    return False
+
+        # Write settings.json file.
+        try:
+            run_command(settings_json_cmd)
+        except Exception as e:
+            log.error(f"Writing the settings.json file failed: {e}")
+            return False
+
+        # Run the index command.
+        try:
+            run_command(index_cmd, show_output=True)
+        except Exception as e:
+            log.error(f"Building the index failed: {e}")
+            return False
+
+        return True

qlever/commands/index_stats.py

@@ -0,0 +1,306 @@
+from __future__ import annotations
+
+import re
+from datetime import datetime
+from pathlib import Path
+
+from qlever.command import QleverCommand
+from qlever.log import log
+from qlever.util import get_total_file_size
+
+
+class IndexStatsCommand(QleverCommand):
+    """
+    Class for executing the `index-stats` command.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Breakdown of the time and space used for the index build"
+
+    def should_have_qleverfile(self) -> bool:
+        return False
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {"data": ["name"]}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+            "--only-time",
+            action="store_true",
+            default=False,
+            help="Show only the time used",
+        )
+        subparser.add_argument(
+            "--only-space",
+            action="store_true",
+            default=False,
+            help="Show only the space used",
+        )
+        subparser.add_argument(
+            "--ignore-text-index",
+            action="store_true",
+            default=False,
+            help="Ignore the text index",
+        )
+        subparser.add_argument(
+            "--time-unit",
+            choices=["s", "min", "h", "auto"],
+            default="auto",
+            help="The time unit",
+        )
+        subparser.add_argument(
+            "--size-unit",
+            choices=["B", "MB", "GB", "TB", "auto"],
+            default="auto",
+            help="The size unit",
+        )
+
+    def execute_time(self, args, log_file_name) -> bool:
+        """
+        Part of `execute` that shows the time used.
+        """
+
+        # Read the content of `log_file_name` into a list of lines.
+        try:
+            with open(log_file_name, "r") as log_file:
+                lines = log_file.readlines()
+        except Exception as e:
+            log.error(f"Problem reading index log file {log_file_name}: {e}")
+            return False
+        # If there is a separate `add-text-index-log.txt` file, append those
+        # lines.
+        try:
+            text_log_file_name = f"{args.name}.text-index-log.txt"
+            if Path(text_log_file_name).exists():
+                with open(text_log_file_name, "r") as text_log_file:
+                    lines.extend(text_log_file.readlines())
+        except Exception as e:
+            log.error(
+                f"Problem reading text index log file " f"{text_log_file_name}: {e}"
+            )
+            return False
+
+        # Helper function that finds the next line matching the given `regex`,
+        # starting from `current_line`, and extracts the time. Returns a tuple
+        # of the time and the regex match object.
+        #
+        # If `update_current_line` is `False`, then `current_line` will not be
+        # updated by this call.
+        #
+        # Otherwise, and this is the default behavior, `current_line` will be
+        # updated to the line after the first match, or one beyond the last
+        # line if no match is found.
+        current_line = 0
+
+        def find_next_line(regex, update_current_line=True):
+            nonlocal lines
+            nonlocal current_line
+            current_line_backup = current_line
+            # Find starting from `current_line`.
+            while current_line < len(lines):
+                line = lines[current_line]
+                current_line += 1
+                timestamp_regex = r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}"
+                timestamp_format = "%Y-%m-%d %H:%M:%S"
+                regex_match = re.search(regex, line)
+                if regex_match:
+                    try:
+                        return datetime.strptime(
+                            re.match(timestamp_regex, line).group(), timestamp_format
+                        ), regex_match
+                    except Exception as e:
+                        log.error(
+                            f"Could not parse timestamp of form "
+                            f'"{timestamp_regex}" from line '
+                            f' "{line.rstrip()}" ({e})'
+                        )
+            # If we get here, we did not find a matching line.
+            if not update_current_line:
+                current_line = current_line_backup
+            return None, None
+
+        # Find the lines matching the key_lines_regex and extract the time
+        # information from them.
+        overall_begin, _ = find_next_line(r"INFO:\s*Processing")
+        merge_begin, _ = find_next_line(r"INFO:\s*Merging partial vocab")
+        convert_begin, _ = find_next_line(r"INFO:\s*Converting triples")
+        perm_begin_and_info = []
+        while True:
+            # Find the next line that starts a permutation.
+            #
+            # NOTE: Should work for the old and new format of the index log
+            # file (old format: "Creating a pair" + names of permutations in
+            # line "Writing meta data for ..."; new format: name of
+            # permutations already in line "Creating permutations ...").
+            perm_begin, _ = find_next_line(
+                r"INFO:\s*Creating a pair", update_current_line=False
+            )
+            if perm_begin is None:
+                perm_begin, perm_info = find_next_line(
+                    r"INFO:\s*Creating permutations ([A-Z]+ and [A-Z]+)",
+                    update_current_line=False,
+                )
+            else:
+                _, perm_info = find_next_line(
+                    r"INFO:\s*Writing meta data for ([A-Z]+ and [A-Z]+)",
+                    update_current_line=False,
+                )
+            if perm_info is None:
+                break
+            perm_begin_and_info.append((perm_begin, perm_info))
+        convert_end = (
+            perm_begin_and_info[0][0] if len(perm_begin_and_info) > 0 else None
+        )
+        normal_end, _ = find_next_line(r"INFO:\s*Index build completed")
+        text_begin, _ = find_next_line(
+            r"INFO:\s*Adding text index", update_current_line=False
+        )
+        text_end, _ = find_next_line(
+            r"INFO:\s*Text index build comp", update_current_line=False
+        )
+        if args.ignore_text_index:
+            text_begin = text_end = None
+
+        # Check whether at least the first phase is done.
+        if overall_begin is None:
+            log.error("Missing line that index build has started")
+            return False
+        if overall_begin and not merge_begin:
+            log.error(
+                "According to the log file, the index build "
+                "has started, but is still in its first "
+                "phase (parsing the input)"
+            )
+            return False
+
+        # Helper function that shows the duration for a phase (if the start and
+        # end timestamps are available).
+        def show_duration(heading, start_end_pairs):
+            nonlocal time_unit
+            num_start_end_pairs = 0
+            diff_seconds = 0
+            for start, end in start_end_pairs:
+                if start and end:
+                    diff_seconds += (end - start).total_seconds()
+                    num_start_end_pairs += 1
+            if num_start_end_pairs > 0:
+                if time_unit == "h":
+                    diff = diff_seconds / 3600
+                elif time_unit == "min":
+                    diff = diff_seconds / 60
+                else:
+                    diff = diff_seconds
+                log.info(f"{heading:<21} : {diff:>6.1f} {time_unit}")
+
+        # Get the times of the various phases (hours or minutes, depending on
+        # how long the first phase took).
+        time_unit = args.time_unit
+        if time_unit == "auto":
+            time_unit = "h"
+            if merge_begin and overall_begin:
+                parse_duration = (merge_begin - overall_begin).total_seconds()
+                if parse_duration < 200:
+                    time_unit = "s"
+                elif parse_duration < 3600:
+                    time_unit = "min"
+        show_duration("Parse input", [(overall_begin, merge_begin)])
+        show_duration("Build vocabularies", [(merge_begin, convert_begin)])
+        show_duration("Convert to global IDs", [(convert_begin, convert_end)])
+        for i in range(len(perm_begin_and_info)):
+            perm_begin, perm_info = perm_begin_and_info[i]
+            perm_end = (
+                perm_begin_and_info[i + 1][0]
+                if i + 1 < len(perm_begin_and_info)
+                else normal_end
+            )
+            perm_info_text = (
+                perm_info.group(1).replace(" and ", " & ") if perm_info else f"#{i + 1}"
+            )
+            show_duration(f"Permutation {perm_info_text}", [(perm_begin, perm_end)])
+        show_duration("Text index", [(text_begin, text_end)])
+        if text_begin and text_end:
+            log.info("")
+            show_duration(
+                "TOTAL time", [(overall_begin, normal_end), (text_begin, text_end)]
+            )
+        elif normal_end:
+            log.info("")
+            show_duration("TOTAL time", [(overall_begin, normal_end)])
+        return True
+
+    def execute_space(self, args) -> bool:
+        """
+        Part of `execute` that shows the space used.
+        """
+
+        # Get the sizes for the various groups of index files.
+        index_size = get_total_file_size([f"{args.name}.index.*"])
+        vocab_size = get_total_file_size([f"{args.name}.vocabulary.*"])
+        text_size = get_total_file_size([f"{args.name}.text.*"])
+        if args.ignore_text_index:
+            text_size = 0
+        total_size = index_size + vocab_size + text_size
+
+        # Determing the proper unit for the size.
+        size_unit = args.size_unit
+        if size_unit == "auto":
+            size_unit = "TB"
+            if total_size < 1e6:
+                size_unit = "B"
+            elif total_size < 1e9:
+                size_unit = "MB"
+            elif total_size < 1e12:
+                size_unit = "GB"
+
+        # Helper function for showing the size in a uniform way.
+        def show_size(heading, size):
+            nonlocal size_unit
+            if size_unit == "GB":
+                size /= 1e9
+            elif size_unit == "MB":
+                size /= 1e6
+            elif size_unit == "TB":
+                size /= 1e12
+            if size_unit == "B":
+                log.info(f"{heading:<21} :  {size:,} {size_unit}")
+            else:
+                log.info(f"{heading:<21} : {size:>6.1f} {size_unit}")
+
+        show_size("Files index.*", index_size)
+        show_size("Files vocabulary.*", vocab_size)
+        if text_size > 0:
+            show_size("Files text.*", text_size)
+        log.info("")
+        show_size("TOTAL size", total_size)
+        return True
+
+    def execute(self, args) -> bool:
+        return_value = True
+
+        # The "time" part of the command.
+        if not args.only_space:
+            log_file_name = f"{args.name}.index-log.txt"
+            self.show(
+                f"Breakdown of the time used for "
+                f"building the index, based on the timestamps for key "
+                f'lines in "{log_file_name}"',
+                only_show=args.show,
+            )
+            if not args.show:
+                return_value &= self.execute_time(args, log_file_name)
+            if not args.only_time:
+                log.info("")
+
+        # The "space" part of the command.
+        if not args.only_time:
+            self.show(
+                "Breakdown of the space used for building the index",
+                only_show=args.show,
+            )
+            if not args.show:
+                return_value &= self.execute_space(args)
+
+        return return_value