qlever 0.5.11__py3-none-any.whl → 0.5.12__py3-none-any.whl

qlever/Qleverfiles/Qleverfile.dblp

@@ -17,7 +17,7 @@ FORMAT       = ttl
 
 [index]
 INPUT_FILES      = *.gz
-MULTI_INPUT_JSON = $$(ls *.gz | awk 'BEGIN { printf "[ " } NR > 1 { printf ", " } { printf "{\"cmd\": \"zcat " $$0 "\"}" } END { printf "]" }')
+MULTI_INPUT_JSON = $$(ls *.gz | xargs -I {} echo '{ "cmd": "zcat {}" }')
 SETTINGS_JSON    = { "ascii-prefixes-only": false, "num-triples-per-batch": 5000000, "prefixes-external": [""] }
 
 [server]

qlever/Qleverfiles/Qleverfile.wikidata

@@ -16,8 +16,7 @@ GET_DATA_URL      = https://dumps.wikimedia.org/wikidatawiki/entities
 GET_DATA_CMD      = curl -LRC - -O ${GET_DATA_URL}/latest-all.ttl.bz2 -O ${GET_DATA_URL}/latest-lexemes.ttl.bz2 2>&1 | tee wikidata.download-log.txt && curl -sL ${GET_DATA_URL}/dcatap.rdf | docker run -i --rm -v $$(pwd):/data stain/jena riot --syntax=RDF/XML --output=NT /dev/stdin > dcatap.nt
 DATE_WIKIDATA     = $$(date -r latest-all.ttl.bz2 +%d.%m.%Y || echo "NO_DATE")
 DATE_WIKIPEDIA    = $$(date -r wikipedia-abstracts.nt +%d.%m.%Y || echo "NO_DATE")
-DESCRIPTION       = Full Wikidata dump from ${GET_DATA_URL} (latest-all.ttl.bz2 and latest-lexemes.ttl.bz2, version ${DATE_WIKIDATA}) + English Wikipeda abstracts (version ${DATE_WIKIPEDIA}, available via schema:description)
-TEXT_DESCRIPTION  = All English and German literals + all sentences from the English Wikipedia (version ${DATE_WIKIPEDIA}), use with FILTER KEYWORDS(...)
+DESCRIPTION       = Full Wikidata dump from ${GET_DATA_URL} (latest-all.ttl.bz2 and latest-lexemes.ttl.bz2, version ${DATE_WIKIDATA})
 
 [index]
 INPUT_FILES      = latest-all.ttl.bz2 latest-lexemes.ttl.bz2 dcatap.nt
@@ -26,7 +25,6 @@ MULTI_INPUT_JSON = [{ "cmd": "lbzcat -n 4 latest-all.ttl.bz2", "format": "ttl",
                     { "cmd": "cat dcatap.nt", "format": "nt", "parallel": "false" }]
 SETTINGS_JSON    = { "languages-internal": [], "prefixes-external": [""], "locale": { "language": "en", "country": "US", "ignore-punctuation": true }, "ascii-prefixes-only": true, "num-triples-per-batch": 5000000 }
 STXXL_MEMORY     = 10G
-TEXT_INDEX       = from_text_records
 
 [server]
 PORT                        = 7001

qlever/commands/index.py

@@ -3,12 +3,12 @@ from __future__ import annotations
 import glob
 import json
 import shlex
+import re
 
 from qlever.command import QleverCommand
 from qlever.containerize import Containerize
 from qlever.log import log
-from qlever.util import (get_existing_index_files, get_total_file_size,
-                         run_command)
+from qlever.util import get_existing_index_files, get_total_file_size, run_command
 
 
 class IndexCommand(QleverCommand):
@@ -20,24 +20,36 @@ class IndexCommand(QleverCommand):
         pass
 
     def description(self) -> str:
-        return ("Build the index for a given RDF dataset")
+        return "Build the index for a given RDF dataset"
 
     def should_have_qleverfile(self) -> bool:
         return True
 
-    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
-        return {"data": ["name", "format"],
-                "index": ["input_files", "cat_input_files", "multi_input_json",
-                          "parallel_parsing", "settings_json", "index_binary",
-                          "only_pso_and_pos_permutations", "use_patterns",
-                          "text_index", "stxxl_memory"],
-                "runtime": ["system", "image", "index_container"]}
+    def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
+        return {
+            "data": ["name", "format"],
+            "index": [
+                "input_files",
+                "cat_input_files",
+                "multi_input_json",
+                "parallel_parsing",
+                "settings_json",
+                "index_binary",
+                "only_pso_and_pos_permutations",
+                "use_patterns",
+                "text_index",
+                "stxxl_memory",
+            ],
+            "runtime": ["system", "image", "index_container"],
+        }
 
     def additional_arguments(self, subparser) -> None:
         subparser.add_argument(
-                "--overwrite-existing", action="store_true",
-                default=False,
-                help="Overwrite an existing index, think twice before using.")
+            "--overwrite-existing",
+            action="store_true",
+            default=False,
+            help="Overwrite an existing index, think twice before using.",
+        )
 
     # Exception for invalid JSON.
     class InvalidInputJson(Exception):
@@ -48,22 +60,29 @@ class IndexCommand(QleverCommand):
 
     # Helper function to get command line options from JSON.
     def get_input_options_for_json(self, args) -> str:
-        # Parse the JSON.
+        # Parse the JSON. If `args.multi_input_json` look like JSONL, turn
+        # it into a JSON array.
         try:
+            jsonl_line_regex = re.compile(r"^\s*\{.*\}\s*$")
+            jsonl_lines = args.multi_input_json.split("\n")
+            if all(re.match(jsonl_line_regex, line) for line in jsonl_lines):
+                args.multi_input_json = "[" + ", ".join(jsonl_lines) + "]"
             input_specs = json.loads(args.multi_input_json)
         except Exception as e:
             raise self.InvalidInputJson(
-                    f"Failed to parse `MULTI_INPUT_JSON` ({e})",
-                    args.multi_input_json)
+                f"Failed to parse `MULTI_INPUT_JSON` as either JSON or JSONL ({e})",
+                args.multi_input_json,
+            )
         # Check that it is an array of length at least one.
         if not isinstance(input_specs, list):
             raise self.InvalidInputJson(
-                    "`MULTI_INPUT_JSON` must be a JSON array",
-                    args.multi_input_json)
+                "`MULTI_INPUT_JSON` must be a JSON array", args.multi_input_json
+            )
         if len(input_specs) == 0:
             raise self.InvalidInputJson(
-                    "`MULTI_INPUT_JSON` must contain at least one element",
-                    args.multi_input_json)
+                "`MULTI_INPUT_JSON` must contain at least one element",
+                args.multi_input_json,
+            )
         # For each of the maps, construct the corresponding command-line
         # options to the index binary.
         input_options = []
@@ -71,15 +90,15 @@ class IndexCommand(QleverCommand):
             # Check that `input_spec` is a dictionary.
             if not isinstance(input_spec, dict):
                 raise self.InvalidInputJson(
-                        f"Element {i} in `MULTI_INPUT_JSON` must be a JSON "
-                        "object",
-                        input_spec)
+                    f"Element {i} in `MULTI_INPUT_JSON` must be a JSON " "object",
+                    input_spec,
+                )
             # For each `input_spec`, we must have a command.
             if "cmd" not in input_spec:
                 raise self.InvalidInputJson(
-                        f"Element {i} in `MULTI_INPUT_JSON` must contain a "
-                        "key `cmd`",
-                        input_spec)
+                    f"Element {i} in `MULTI_INPUT_JSON` must contain a " "key `cmd`",
+                    input_spec,
+                )
             input_cmd = input_spec["cmd"]
             # The `format`, `graph`, and `parallel` keys are optional.
             input_format = input_spec.get("format", args.format)
@@ -89,17 +108,19 @@ class IndexCommand(QleverCommand):
             extra_keys = input_spec.keys() - {"cmd", "format", "graph", "parallel"}
             if extra_keys:
                 raise self.InvalidInputJson(
-                        f"Element {i} in `MULTI_INPUT_JSON` must only contain "
-                        "the keys `format`, `graph`, and `parallel`. Contains "
-                        "extra keys {extra_keys}.",
-                        input_spec)
+                    f"Element {i} in `MULTI_INPUT_JSON` must only contain "
+                    "the keys `format`, `graph`, and `parallel`. Contains "
+                    "extra keys {extra_keys}.",
+                    input_spec,
+                )
             # Add the command-line options for this input stream. We use
             # process substitution `<(...)` as a convenient way to handle
             # an input stream just like a file. This is not POSIX compliant,
             # but supported by various shells, including bash and zsh.
             input_options.append(
-                    f"-f <({input_cmd}) -F {input_format} "
-                    f"-g \"{input_graph}\" -p {input_parallel}")
+                f"-f <({input_cmd}) -F {input_format} "
+                f'-g "{input_graph}" -p {input_parallel}'
+            )
         # Return the concatenated command-line options.
         return " ".join(input_options)
 
@@ -108,11 +129,13 @@ class IndexCommand(QleverCommand):
         # basename of the index, and the settings file). There are two ways
         # to specify the input: via a single stream or via multiple streams.
         if args.cat_input_files and not args.multi_input_json:
-            index_cmd = (f"{args.cat_input_files} | {args.index_binary}"
-                         f" -i {args.name} -s {args.name}.settings.json"
-                         f" -F {args.format} -f -")
+            index_cmd = (
+                f"{args.cat_input_files} | {args.index_binary}"
+                f" -i {args.name} -s {args.name}.settings.json"
+                f" -F {args.format} -f -"
+            )
             if args.parallel_parsing:
-                index_cmd += (f" -p {args.parallel_parsing}")
+                index_cmd += f" -p {args.parallel_parsing}"
         elif args.multi_input_json and not args.cat_input_files:
             try:
                 input_options = self.get_input_options_for_json(args)
@@ -121,13 +144,17 @@ class IndexCommand(QleverCommand):
                 log.info("")
                 log.info(e.additional_info)
                 return False
-            index_cmd = (f"{args.index_binary}"
-                         f" -i {args.name} -s {args.name}.settings.json"
-                         f" {input_options}")
+            index_cmd = (
+                f"{args.index_binary}"
+                f" -i {args.name} -s {args.name}.settings.json"
+                f" {input_options}"
+            )
         else:
-            log.error("Specify exactly one of `CAT_INPUT_FILES` (for a "
-                      "single input stream) or `MULTI_INPUT_JSON` (for "
-                      "multiple input streams)")
+            log.error(
+                "Specify exactly one of `CAT_INPUT_FILES` (for a "
+                "single input stream) or `MULTI_INPUT_JSON` (for "
+                "multiple input streams)"
+            )
             log.info("")
             log.info("See `qlever index --help` for more information")
             return False
@@ -137,12 +164,11 @@ class IndexCommand(QleverCommand):
             index_cmd += " --only-pso-and-pos-permutations --no-patterns"
         if not args.use_patterns:
             index_cmd += " --no-patterns"
-        if args.text_index in \
-                ["from_text_records", "from_text_records_and_literals"]:
-            index_cmd += (f" -w {args.name}.wordsfile.tsv"
-                          f" -d {args.name}.docsfile.tsv")
-        if args.text_index in \
-                ["from_literals", "from_text_records_and_literals"]:
+        if args.text_index in ["from_text_records", "from_text_records_and_literals"]:
+            index_cmd += (
+                f" -w {args.name}.wordsfile.tsv" f" -d {args.name}.docsfile.tsv"
+            )
+        if args.text_index in ["from_literals", "from_text_records_and_literals"]:
             index_cmd += " --text-words-from-literals"
         if args.stxxl_memory:
             index_cmd += f" --stxxl-memory {args.stxxl_memory}"
@@ -150,24 +176,26 @@ class IndexCommand(QleverCommand):
 
         # If the total file size is larger than 10 GB, set ulimit (such that a
         # large number of open files is allowed).
-        total_file_size = get_total_file_size(
-                shlex.split(args.input_files))
+        total_file_size = get_total_file_size(shlex.split(args.input_files))
         if total_file_size > 1e10:
             index_cmd = f"ulimit -Sn 1048576; {index_cmd}"
 
         # Run the command in a container (if so desired).
         if args.system in Containerize.supported_systems():
             index_cmd = Containerize().containerize_command(
-                    index_cmd,
-                    args.system, "run --rm",
-                    args.image,
-                    args.index_container,
-                    volumes=[("$(pwd)", "/index")],
-                    working_directory="/index")
+                index_cmd,
+                args.system,
+                "run --rm",
+                args.image,
+                args.index_container,
+                volumes=[("$(pwd)", "/index")],
+                working_directory="/index",
+            )
 
         # Command for writing the settings JSON to a file.
-        settings_json_cmd = (f"echo {shlex.quote(args.settings_json)} "
-                             f"> {args.name}.settings.json")
+        settings_json_cmd = (
+            f"echo {shlex.quote(args.settings_json)} " f"> {args.name}.settings.json"
+        )
 
         # Show the command line.
         self.show(f"{settings_json_cmd}\n{index_cmd}", only_show=args.show)
@@ -179,9 +207,11 @@ class IndexCommand(QleverCommand):
             try:
                 run_command(f"{args.index_binary} --help")
             except Exception as e:
-                log.error(f"Running \"{args.index_binary}\" failed, "
-                          f"set `--index-binary` to a different binary or "
-                          f"set `--system to a container system`")
+                log.error(
+                    f'Running "{args.index_binary}" failed, '
+                    f"set `--index-binary` to a different binary or "
+                    f"set `--system to a container system`"
+                )
                 log.info("")
                 log.info(f"The error message was: {e}")
                 return False
@@ -189,28 +219,29 @@ class IndexCommand(QleverCommand):
         # Check if all of the input files exist.
         for pattern in shlex.split(args.input_files):
             if len(glob.glob(pattern)) == 0:
-                log.error(f"No file matching \"{pattern}\" found")
+                log.error(f'No file matching "{pattern}" found')
                 log.info("")
-                log.info("Did you call `qlever get-data`? If you did, check "
-                         "GET_DATA_CMD and INPUT_FILES in the QLeverfile")
+                log.info(
+                    "Did you call `qlever get-data`? If you did, check "
+                    "GET_DATA_CMD and INPUT_FILES in the QLeverfile"
+                )
                 return False
 
         # Check if index files (name.index.*) already exist.
         existing_index_files = get_existing_index_files(args.name)
         if len(existing_index_files) > 0 and not args.overwrite_existing:
             log.error(
-                    f"Index files for basename \"{args.name}\" found, if you "
-                    f"want to overwrite them, use --overwrite-existing")
+                f'Index files for basename "{args.name}" found, if you '
+                f"want to overwrite them, use --overwrite-existing"
+            )
             log.info("")
             log.info(f"Index files found: {existing_index_files}")
             return False
 
         # Remove already existing container.
-        if args.system in Containerize.supported_systems() \
-                and args.overwrite_existing:
+        if args.system in Containerize.supported_systems() and args.overwrite_existing:
             if Containerize.is_running(args.system, args.index_container):
-                log.info("Another index process is running, trying to stop "
-                         "it ...")
+                log.info("Another index process is running, trying to stop " "it ...")
                 log.info("")
                 try:
                     run_command(f"{args.system} rm -f {args.index_container}")

{qlever-0.5.11.dist-info → qlever-0.5.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qlever
-Version: 0.5.11
+Version: 0.5.12
 Summary: Script for using the QLever SPARQL engine.
 Author-email: Hannah Bast <bast@cs.uni-freiburg.de>
 License: Apache-2.0

{qlever-0.5.11.dist-info → qlever-0.5.12.dist-info}/RECORD

@@ -7,7 +7,7 @@ qlever/qlever_main.py,sha256=tA_xqOs_FjvqlDIvKTprwuysfTwzsUjE7at26gRhCVA,2336
 qlever/qlever_old.py,sha256=X-JxmepFKYeFgSLLp0TRDNqXSxDwIbc8_0Xstiems8c,62026
 qlever/qleverfile.py,sha256=lygAjI5_wV_e-JoIGIqVTdd4yyvApzZiSlqSsmdlJpU,14529
 qlever/util.py,sha256=qLxBRyHPT2VTj0xcOCFcP6HV-Lm-g-64QpvOc4V0_a8,8029
-qlever/Qleverfiles/Qleverfile.dblp,sha256=wXZweNfYgEx-IGF1vIJMgqYUSWanQlzPj1EzUOGVuXA,1340
+qlever/Qleverfiles/Qleverfile.dblp,sha256=nbkjBb6ZlmRgCsVcsx8U0EwI8csf8G03Rn2E10uCxVk,1269
 qlever/Qleverfiles/Qleverfile.dblp-plus,sha256=TJHxp8I1P6JKJjbuAllEpB32-huuY1gH0FlenqPVJ5g,1334
 qlever/Qleverfiles/Qleverfile.dbpedia,sha256=aaNZZayE-zVePGSwPzXemkX__Ns8-kP_E7DNNKZPnqg,1160
 qlever/Qleverfiles/Qleverfile.default,sha256=Kj-J1Kkv8PWN7wuMdZU6DUUlEuBIcSNysJCE-R63we8,2407
@@ -24,7 +24,7 @@ qlever/Qleverfiles/Qleverfile.pubchem,sha256=YuDzWQmukSvL1opu7cf1KX9407_P21lmecY
 qlever/Qleverfiles/Qleverfile.scientists,sha256=9eZ2c6P9a3E3VHa3RR7LdOQbF4k3oyyrn56Z3u4LZYs,1164
 qlever/Qleverfiles/Qleverfile.uniprot,sha256=9kAKseomdUnIt7EAZge39g1MTuaLVaSW9JYLHzIMolM,2338
 qlever/Qleverfiles/Qleverfile.vvz,sha256=cLzm85erKoFCDllH5eFcSi35MdR6Tahj1MgtvGRxanM,922
-qlever/Qleverfiles/Qleverfile.wikidata,sha256=JyWB7haqulineO3aPEsXqx12OpantxfxueeofcvMjpk,2343
+qlever/Qleverfiles/Qleverfile.wikidata,sha256=zVUXF75XJyK1h-J-7EjFemzmkSyoPtng1mNY3U7S78M,2061
 qlever/Qleverfiles/Qleverfile.wikipathways,sha256=UFEVLrtOBiSQfibBN9xc2wDXrnWcnx5f8PY9khcE6bc,1983
 qlever/Qleverfiles/Qleverfile.yago-4,sha256=hAS_2ZmC1zxNsKXip7t1F_iqu3CC-6O7v6HZhuFbnWY,1819
 qlever/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -33,7 +33,7 @@ qlever/commands/cache_stats.py,sha256=6JjueQstAqc8dNfgY8TP2EitFMxdUvCwrcyd7KUEb2
 qlever/commands/clear_cache.py,sha256=AnE1MOoj1ZexxrRT8FGeBLlv8rtQIVV4DP8VBn5-X-s,2843
 qlever/commands/example_queries.py,sha256=rtMOQw7cJe0Aia_1O7UyKcxHbz7ln9BSZYWUQI9OFA8,16389
 qlever/commands/get_data.py,sha256=f9kjZI3TKad6JHSuXWNkeoajmW8h0Sx8ShvjauDCtNo,1412
-qlever/commands/index.py,sha256=G4SPxJ1PW8KsLYl4OBV4rOLKSo-O3aR6nTT_0K6zAgU,10376
+qlever/commands/index.py,sha256=y0HlFSTbRQSnXRz0fn3Mar9FqfnOtC2U1nu74N5IwCA,10842
 qlever/commands/index_stats.py,sha256=_BiUNBhmbYd9RPxrlm4HF0oENO6JmqnRiAkwkyOdN4U,11722
 qlever/commands/log.py,sha256=8Krt3MsTUDapYqVw1zUu5X15SF8mV97Uj0qKOWK8jXk,1861
 qlever/commands/query.py,sha256=_IDH-M8gKL_f1i5wzu0X452pZSUD0_qXl6bPXC85wX0,2750
@@ -44,9 +44,9 @@ qlever/commands/stop.py,sha256=TZs4bxKHvujlZAU8BZmFjA5eXSZNAa6EeNzvPpEZsuI,4139
 qlever/commands/system_info.py,sha256=SShsnEV7QROgdbABeJ6Wk4U_CNjqlYO1J5HrpNCVNMs,4615
 qlever/commands/ui.py,sha256=rZxIYHZr-PqgQKfvVwl8woDnTGR-sFc-f6cPjcORaOk,3611
 qlever/commands/warmup.py,sha256=WOZSxeV8U_F6pEEnAb6YybXLQMxZFTRJXs4BPHUhsmc,1030
-qlever-0.5.11.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-qlever-0.5.11.dist-info/METADATA,sha256=Cy1xk-ZJLRR1r5LPwvAFwzuvJZwdTpDW8puG57Un-0Y,4583
-qlever-0.5.11.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-qlever-0.5.11.dist-info/entry_points.txt,sha256=U_gbYYi0wwdsn884eb0XoOXfvhACOsxhlO330dZ9bi0,87
-qlever-0.5.11.dist-info/top_level.txt,sha256=kd3zsYqiFd0--Czh5XTVkfEq6XR-XgRFW35X0v0GT-c,7
-qlever-0.5.11.dist-info/RECORD,,
+qlever-0.5.12.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+qlever-0.5.12.dist-info/METADATA,sha256=mm1muAfcKHTS4pN-xS3D0P4P57Z450Acifk2BckhAgc,4583
+qlever-0.5.12.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+qlever-0.5.12.dist-info/entry_points.txt,sha256=U_gbYYi0wwdsn884eb0XoOXfvhACOsxhlO330dZ9bi0,87
+qlever-0.5.12.dist-info/top_level.txt,sha256=kd3zsYqiFd0--Czh5XTVkfEq6XR-XgRFW35X0v0GT-c,7
+qlever-0.5.12.dist-info/RECORD,,