qlever 0.5.8__py3-none-any.whl → 0.5.10__py3-none-any.whl

qlever/qleverfile.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import re
-import socket
 import subprocess
 from configparser import ConfigParser, ExtendedInterpolation
 
@@ -41,162 +40,277 @@ class Qleverfile:
         ui_args = all_args["ui"] = {}
 
         data_args["name"] = arg(
-                "--name", type=str, required=True,
-                help="The name of the dataset")
+            "--name", type=str, required=True, help="The name of the dataset"
+        )
         data_args["get_data_cmd"] = arg(
-                "--get-data-cmd", type=str, required=True,
-                help="The command to get the data")
+            "--get-data-cmd",
+            type=str,
+            required=True,
+            help="The command to get the data",
+        )
         data_args["description"] = arg(
-                "--description", type=str, required=True,
-                help="A concise description of the dataset")
+            "--description",
+            type=str,
+            required=True,
+            help="A concise description of the dataset",
+        )
         data_args["text_description"] = arg(
-                "--text-description", type=str, default=None,
-                help="A concise description of the additional text data"
-                     " if any")
+            "--text-description",
+            type=str,
+            default=None,
+            help="A concise description of the additional text data" " if any",
+        )
         data_args["format"] = arg(
-                "--format", type=str, default="ttl",
-                choices=["ttl", "nt", "nq"],
-                help="The format of the data")
+            "--format",
+            type=str,
+            default="ttl",
+            choices=["ttl", "nt", "nq"],
+            help="The format of the data",
+        )
 
         index_args["input_files"] = arg(
-                "--input-files", type=str, required=True,
-                help="A space-separated list of patterns that match "
-                     "all the files of the dataset")
+            "--input-files",
+            type=str,
+            required=True,
+            help="A space-separated list of patterns that match "
+            "all the files of the dataset",
+        )
         index_args["cat_input_files"] = arg(
-                "--cat-input-files", type=str, required=True,
-                help="The command that produces the input")
+            "--cat-input-files", type=str, help="The command that produces the input"
+        )
+        index_args["multi_input_json"] = arg(
+            "--multi-input-json",
+            type=str,
+            default=None,
+            help="JSON to specify multiple input files, each with a "
+            "`cmd` (command that writes the triples to stdout), "
+            "`format` (format like for the `--format` option), "
+            "`graph` (name of the graph, use `-` for the default graph), "
+            "`parallel` (parallel parsing for large files, where all "
+            "prefix declaration are at the beginning)",
+        )
+        index_args["parallel_parsing"] = arg(
+            "--parallel-parsing",
+            type=str,
+            choices=["true", "false"],
+            help="Use parallel parsing (recommended for large files, "
+            "but it requires that all prefix declarations are at the "
+            "beginning of the file)",
+        )
         index_args["settings_json"] = arg(
-                "--settings-json", type=str, default="{}",
-                help="The `.settings.json` file for the index")
+            "--settings-json",
+            type=str,
+            default="{}",
+            help="The `.settings.json` file for the index",
+        )
         index_args["index_binary"] = arg(
-                "--index-binary", type=str, default="IndexBuilderMain",
-                help="The binary for building the index (this requires "
-                     "that you have compiled QLever on your machine)")
+            "--index-binary",
+            type=str,
+            default="IndexBuilderMain",
+            help="The binary for building the index (this requires "
+            "that you have compiled QLever on your machine)",
+        )
         index_args["stxxl_memory"] = arg(
-                "--stxxl-memory", type=str, default="5G",
-                help="The amount of memory to use for the index build "
-                     "(the name of the option has historical reasons)")
+            "--stxxl-memory",
+            type=str,
+            default="5G",
+            help="The amount of memory to use for the index build "
+            "(the name of the option has historical reasons)",
+        )
         index_args["only_pso_and_pos_permutations"] = arg(
-                "--only-pso-and-pos-permutations", action="store_true",
-                default=False,
-                help="Only create the PSO and POS permutations")
+            "--only-pso-and-pos-permutations",
+            action="store_true",
+            default=False,
+            help="Only create the PSO and POS permutations",
+        )
         index_args["use_patterns"] = arg(
-                "--use-patterns", action="store_true", default=True,
-                help="Precompute so-called patterns needed for fast processing"
-                     " of queries like SELECT ?p (COUNT(DISTINCT ?s) AS ?c) "
-                     "WHERE { ?s ?p [] ... } GROUP BY ?p")
+            "--use-patterns",
+            action="store_true",
+            default=True,
+            help="Precompute so-called patterns needed for fast processing"
+            " of queries like SELECT ?p (COUNT(DISTINCT ?s) AS ?c) "
+            "WHERE { ?s ?p [] ... } GROUP BY ?p",
+        )
         index_args["text_index"] = arg(
-                "--text-index",
-                choices=["none", "from_text_records", "from_literals",
-                         "from_text_records_and_literals"],
-                default="none",
-                help="Whether to also build an index for text search"
-                     "and for which texts")
+            "--text-index",
+            choices=[
+                "none",
+                "from_text_records",
+                "from_literals",
+                "from_text_records_and_literals",
+            ],
+            default="none",
+            help="Whether to also build an index for text search" "and for which texts",
+        )
         index_args["text_words_file"] = arg(
-                "--text-words-file", type=str, default=None,
-                help="File with the words for the text index (one line "
-                     "per word, format: `word or IRI\t0 or 1\tdoc id\t1`)")
+            "--text-words-file",
+            type=str,
+            default=None,
+            help="File with the words for the text index (one line "
+            "per word, format: `word or IRI\t0 or 1\tdoc id\t1`)",
+        )
         index_args["text_docs_file"] = arg(
-                "--text-docs-file", type=str, default=None,
-                help="File with the documents for the text index (one line "
-                     "per document, format: `id\tdocument text`)")
+            "--text-docs-file",
+            type=str,
+            default=None,
+            help="File with the documents for the text index (one line "
+            "per document, format: `id\tdocument text`)",
+        )
 
         server_args["server_binary"] = arg(
-                "--server-binary", type=str, default="ServerMain",
-                help="The binary for starting the server (this requires "
-                     "that you have compiled QLever on your machine)")
+            "--server-binary",
+            type=str,
+            default="ServerMain",
+            help="The binary for starting the server (this requires "
+            "that you have compiled QLever on your machine)",
+        )
         server_args["host_name"] = arg(
-                "--host-name", type=str, default=f"localhost",
-                help="The name of the host on which the server listens for "
-                     "requests")
+            "--host-name",
+            type=str,
+            default="localhost",
+            help="The name of the host on which the server listens for " "requests",
+        )
         server_args["port"] = arg(
-                "--port", type=int,
-                help="The port on which the server listens for requests")
+            "--port", type=int, help="The port on which the server listens for requests"
+        )
         server_args["access_token"] = arg(
-                "--access-token", type=str, default=None,
-                help="The access token for privileged operations")
+            "--access-token",
+            type=str,
+            default=None,
+            help="The access token for privileged operations",
+        )
         server_args["memory_for_queries"] = arg(
-                "--memory-for-queries", type=str, default="5G",
-                help="The maximal amount of memory used for query processing"
-                     " (if a query needs more than what is available, the "
-                     "query will not be processed)")
+            "--memory-for-queries",
+            type=str,
+            default="5G",
+            help="The maximal amount of memory used for query processing"
+            " (if a query needs more than what is available, the "
+            "query will not be processed)",
+        )
         server_args["cache_max_size"] = arg(
-                "--cache-max-size", type=str, default="2G",
-                help="The maximal amount of memory used for caching")
+            "--cache-max-size",
+            type=str,
+            default="2G",
+            help="The maximal amount of memory used for caching",
+        )
         server_args["cache_max_size_single_entry"] = arg(
-                "--cache-max-size-single-entry", type=str, default="1G",
-                help="The maximal amount of memory used for caching a single "
-                     "query result")
+            "--cache-max-size-single-entry",
+            type=str,
+            default="1G",
+            help="The maximal amount of memory used for caching a single "
+            "query result",
+        )
         server_args["cache_max_num_entries"] = arg(
-                "--cache-max-num-entries", type=int, default=200,
-                help="The maximal number of entries in the cache"
-                     " (the eviction policy when the cache is full is LRU)")
+            "--cache-max-num-entries",
+            type=int,
+            default=200,
+            help="The maximal number of entries in the cache"
+            " (the eviction policy when the cache is full is LRU)",
+        )
         server_args["timeout"] = arg(
-                "--timeout", type=str, default="30s",
-                help="The maximal time in seconds a query is allowed to run"
-                     " (can be increased per query with the URL parameters "
-                     "`timeout` and `access_token`)")
+            "--timeout",
+            type=str,
+            default="30s",
+            help="The maximal time in seconds a query is allowed to run"
+            " (can be increased per query with the URL parameters "
+            "`timeout` and `access_token`)",
+        )
         server_args["num_threads"] = arg(
-                "--num-threads", type=int, default=8,
-                help="The number of threads used for query processing")
+            "--num-threads",
+            type=int,
+            default=8,
+            help="The number of threads used for query processing",
+        )
         server_args["only_pso_and_pos_permutations"] = arg(
-                "--only-pso-and-pos-permutations", action="store_true",
-                default=False,
-                help="Only use the PSO and POS permutations (then each "
-                     "triple pattern must have a fixed predicate)")
+            "--only-pso-and-pos-permutations",
+            action="store_true",
+            default=False,
+            help="Only use the PSO and POS permutations (then each "
+            "triple pattern must have a fixed predicate)",
+        )
         server_args["use_patterns"] = arg(
-                "--use-patterns", action="store_true", default=True,
-                help="Use the patterns precomputed during the index build"
-                     " (see `qlever index --help` for their utility)")
+            "--use-patterns",
+            action="store_true",
+            default=True,
+            help="Use the patterns precomputed during the index build"
+            " (see `qlever index --help` for their utility)",
+        )
         server_args["use_text_index"] = arg(
-                "--use-text-index", choices=["yes", "no"], default="no",
-                help="Whether to use the text index (requires that one was "
-                "built, see `qlever index`)")
+            "--use-text-index",
+            choices=["yes", "no"],
+            default="no",
+            help="Whether to use the text index (requires that one was "
+            "built, see `qlever index`)",
+        )
         server_args["warmup_cmd"] = arg(
-                "--warmup-cmd", type=str,
-                help="Command executed after the server has started "
-                     " (executed as part of `qlever start` unless "
-                     " `--no-warmup` is specified, or with `qlever warmup`)")
+            "--warmup-cmd",
+            type=str,
+            help="Command executed after the server has started "
+            " (executed as part of `qlever start` unless "
+            " `--no-warmup` is specified, or with `qlever warmup`)",
+        )
 
         runtime_args["system"] = arg(
-                "--system", type=str,
-                choices=Containerize.supported_systems() + ["native"],
-                default="docker",
-                help=("Whether to run commands like `index` or `start` "
-                      "natively or in a container, and if in a container, "
-                      "which system to use"))
+            "--system",
+            type=str,
+            choices=Containerize.supported_systems() + ["native"],
+            default="docker",
+            help=(
+                "Whether to run commands like `index` or `start` "
+                "natively or in a container, and if in a container, "
+                "which system to use"
+            ),
+        )
         runtime_args["image"] = arg(
-                "--image", type=str,
-                default="docker.io/adfreiburg/qlever",
-                help="The name of the image when running in a container")
+            "--image",
+            type=str,
+            default="docker.io/adfreiburg/qlever",
+            help="The name of the image when running in a container",
+        )
         runtime_args["index_container"] = arg(
-                "--index-container", type=str,
-                help="The name of the container used by `qlever index`")
+            "--index-container",
+            type=str,
+            help="The name of the container used by `qlever index`",
+        )
         runtime_args["server_container"] = arg(
-                "--server-container", type=str,
-                help="The name of the container used by `qlever start`")
+            "--server-container",
+            type=str,
+            help="The name of the container used by `qlever start`",
+        )
 
         ui_args["ui_port"] = arg(
-                "--ui-port", type=int, default=8176,
-                help="The port of the Qlever UI when running `qlever ui`")
+            "--ui-port",
+            type=int,
+            default=8176,
+            help="The port of the Qlever UI when running `qlever ui`",
+        )
         ui_args["ui_config"] = arg(
-                "--ui-config", type=str, default="default",
-                help="The name of the backend configuration for the QLever UI"
-                     " (this determines AC queries and example queries)")
+            "--ui-config",
+            type=str,
+            default="default",
+            help="The name of the backend configuration for the QLever UI"
+            " (this determines AC queries and example queries)",
+        )
         ui_args["ui_system"] = arg(
-                "--ui-system", type=str,
-                choices=Containerize.supported_systems(),
-                default="docker",
-                help="Which container system to use for `qlever ui`"
-                     " (unlike for `qlever index` and `qlever start`, "
-                     " \"native\" is not yet supported here)")
+            "--ui-system",
+            type=str,
+            choices=Containerize.supported_systems(),
+            default="docker",
+            help="Which container system to use for `qlever ui`"
+            " (unlike for `qlever index` and `qlever start`, "
+            ' "native" is not yet supported here)',
+        )
         ui_args["ui_image"] = arg(
-                "--ui-image", type=str,
-                default="docker.io/adfreiburg/qlever-ui",
-                help="The name of the image used for `qlever ui`")
+            "--ui-image",
+            type=str,
+            default="docker.io/adfreiburg/qlever-ui",
+            help="The name of the image used for `qlever ui`",
+        )
         ui_args["ui_container"] = arg(
-                "--ui-container", type=str,
-                help="The name of the container used for `qlever ui`")
+            "--ui-container",
+            type=str,
+            help="The name of the container used for `qlever ui`",
+        )
 
         return all_args
 
@@ -214,8 +328,7 @@ class Qleverfile:
 
         # Read the Qleverfile.
         defaults = {"random": "83724324hztz", "version": "01.01.01"}
-        config = ConfigParser(interpolation=ExtendedInterpolation(),
-                              defaults=defaults)
+        config = ConfigParser(interpolation=ExtendedInterpolation(), defaults=defaults)
         try:
             config.read(qleverfile_path)
         except Exception as e:
@@ -230,13 +343,18 @@ class Qleverfile:
                 if match:
                     try:
                         value = subprocess.check_output(
-                                match.group(1), shell=True, text=True,
-                                stderr=subprocess.STDOUT).strip()
+                            match.group(1),
+                            shell=True,
+                            text=True,
+                            stderr=subprocess.STDOUT,
+                        ).strip()
                     except Exception as e:
                         log.info("")
-                        log.error(f"Error evaluating {value} for option "
-                                  f"{section}.{option.upper()} in "
-                                  f"{qleverfile_path}:")
+                        log.error(
+                            f"Error evaluating {value} for option "
+                            f"{section}.{option.upper()} in "
+                            f"{qleverfile_path}:"
+                        )
                         log.info("")
                         log.info(e.output if hasattr(e, "output") else e)
                         exit(1)
@@ -263,8 +381,8 @@ class Qleverfile:
             if "text_docs_file" not in index:
                 index["text_docs_file"] = f"{name}.docsfile.tsv"
             server = config["server"]
-            if index.get("text_index", "none") != "none":
-                server["use_text_index"] = "yes"
+        if index.get("text_index", "none") != "none":
+            server["use_text_index"] = "yes"
 
         # Return the parsed Qleverfile with the added inherited values.
         return config

qlever/util.py

@@ -29,8 +29,9 @@ def get_total_file_size(patterns: list[str]) -> int:
     return total_size
 
 
-def run_command(cmd: str, return_output: bool = False,
-                show_output: bool = False) -> Optional[str]:
+def run_command(
+    cmd: str, return_output: bool = False, show_output: bool = False
+) -> Optional[str]:
     """
     Run the given command and throw an exception if the exit code is non-zero.
     If `return_output` is `True`, return what the command wrote to `stdout`.
@@ -45,7 +46,7 @@ def run_command(cmd: str, return_output: bool = False,
         "shell": True,
         "text": True,
         "stdout": None if show_output else subprocess.PIPE,
-        "stderr": subprocess.PIPE
+        "stderr": subprocess.PIPE,
     }
     result = subprocess.run(f"set -o pipefail; {cmd}", **subprocess_args)
     # If the exit code is non-zero, throw an exception. If something was
@@ -63,17 +64,20 @@ def run_command(cmd: str, return_output: bool = False,
             raise Exception(result.stderr.replace("\n", " ").strip())
         else:
             raise Exception(
-                    f"Command failed with exit code {result.returncode}"
-                    f" but nothing written to stderr")
+                f"Command failed with exit code {result.returncode}"
+                f" but nothing written to stderr"
+            )
     # Optionally, return what was written to `stdout`.
     if return_output:
         return result.stdout
 
 
-def run_curl_command(url: str,
-                     headers: dict[str, str] = {},
-                     params: dict[str, str] = {},
-                     result_file: Optional[str] = None) -> str:
+def run_curl_command(
+    url: str,
+    headers: dict[str, str] = {},
+    params: dict[str, str] = {},
+    result_file: Optional[str] = None,
+) -> str:
     """
     Run `curl` with the given `url`, `headers`, and `params`. If `result_file`
     is `None`, return the output, otherwise, write the output to the given file
@@ -83,22 +87,29 @@ def run_curl_command(url: str,
     # Construct and run the `curl` command.
     default_result_file = "/tmp/qlever.curl.result"
     actual_result_file = result_file if result_file else default_result_file
-    curl_cmd = (f"curl -s -o \"{actual_result_file}\""
-                f" -w \"%{{http_code}}\n\" {url}"
-                + "".join([f" -H \"{key}: {value}\""
-                           for key, value in headers.items()])
-                + "".join([f" --data-urlencode {key}={shlex.quote(value)}"
-                           for key, value in params.items()]))
-    result = subprocess.run(curl_cmd, shell=True, text=True,
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE)
+    curl_cmd = (
+        f'curl -s -o "{actual_result_file}"'
+        f' -w "%{{http_code}}\n" {url}'
+        + "".join([f' -H "{key}: {value}"' for key, value in headers.items()])
+        + "".join(
+            [
+                f" --data-urlencode {key}={shlex.quote(value)}"
+                for key, value in params.items()
+            ]
+        )
+    )
+    result = subprocess.run(
+        curl_cmd, shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    )
     # Case 1: An error occurred, raise an exception.
     if result.returncode != 0:
         if len(result.stderr) > 0:
             raise Exception(result.stderr)
         else:
-            raise Exception(f"curl command failed with exit code "
-                            f"{result.returncode}, stderr is empty")
+            raise Exception(
+                f"curl command failed with exit code "
+                f"{result.returncode}, stderr is empty"
+            )
     # Case 2: Return output (read from `default_result_file`).
     if result_file is None:
         result_file_path = Path(default_result_file)
@@ -117,9 +128,9 @@ def is_qlever_server_alive(port: str) -> bool:
 
     message = "from the qlever script".replace(" ", "%20")
     curl_cmd = f"curl -s http://localhost:{port}/ping?msg={message}"
-    exit_code = subprocess.call(curl_cmd, shell=True,
-                                stdout=subprocess.DEVNULL,
-                                stderr=subprocess.DEVNULL)
+    exit_code = subprocess.call(
+        curl_cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+    )
     return exit_code == 0
 
 
@@ -152,15 +163,15 @@ def show_process_info(psutil_process, cmdline_regex, show_heading=True):
 
     try:
         pinfo = psutil_process.as_dict(
-                attrs=['pid', 'username', 'create_time',
-                       'memory_info', 'cmdline'])
+            attrs=["pid", "username", "create_time", "memory_info", "cmdline"]
+        )
         # Note: pinfo[`cmdline`] is `None` if the process is a zombie.
-        cmdline = " ".join(pinfo['cmdline'] or [])
+        cmdline = " ".join(pinfo["cmdline"] or [])
         if len(cmdline) == 0 or not re.search(cmdline_regex, cmdline):
             return False
-        pid = pinfo['pid']
-        user = pinfo['username'] if pinfo['username'] else ""
-        start_time = datetime.fromtimestamp(pinfo['create_time'])
+        pid = pinfo["pid"]
+        user = pinfo["username"] if pinfo["username"] else ""
+        start_time = datetime.fromtimestamp(pinfo["create_time"])
         if start_time.date() == date.today():
             start_time = start_time.strftime("%H:%M")
         else:
@@ -193,10 +204,24 @@ def is_port_used(port: int) -> bool:
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         # Ensure that the port is not blocked after the check.
         sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        sock.bind(('', port))
+        sock.bind(("", port))
         sock.close()
         return False
     except OSError as err:
         if err.errno != errno.EADDRINUSE:
             log.warning(f"Failed to determine if port is used: {err}")
         return True
+
+
+def format_size(bytes, suffix="B"):
+    """
+    Scale bytes to its proper format
+    e.g:
+        1253656 => '1.20MB'
+        1253656678 => '1.17GB'
+    """
+    factor = 1024
+    for unit in ["", "K", "M", "G", "T", "P"]:
+        if bytes < factor:
+            return f"{bytes:.2f} {unit}{suffix}"
+        bytes /= factor

{qlever-0.5.8.dist-info → qlever-0.5.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qlever
-Version: 0.5.8
+Version: 0.5.10
 Summary: Script for using the QLever SPARQL engine.
 Author-email: Hannah Bast <bast@cs.uni-freiburg.de>
 License: Apache-2.0