qlever 0.5.15__py3-none-any.whl → 0.5.18__py3-none-any.whl

qlever/qleverfile.py

@@ -76,7 +76,9 @@ class Qleverfile:
             "all the files of the dataset",
         )
         index_args["cat_input_files"] = arg(
-            "--cat-input-files", type=str, help="The command that produces the input"
+            "--cat-input-files",
+            type=str,
+            help="The command that produces the input",
         )
         index_args["multi_input_json"] = arg(
             "--multi-input-json",
@@ -103,6 +105,14 @@ class Qleverfile:
             default="{}",
             help="The `.settings.json` file for the index",
         )
+        index_args["ulimit"] = arg(
+            "--ulimit",
+            type=int,
+            default=None,
+            help="Explicitly set the limit for the maximal number of open "
+            "files (default: 1048576 when the total size of the input files "
+            "is larger than 10 GB)",
+        )
         index_args["index_binary"] = arg(
             "--index-binary",
             type=str,
@@ -113,10 +123,17 @@ class Qleverfile:
         index_args["stxxl_memory"] = arg(
             "--stxxl-memory",
             type=str,
-            default="5G",
             help="The amount of memory to use for the index build "
             "(the name of the option has historical reasons)",
         )
+        index_args["parser_buffer_size"] = arg(
+            "--parser-buffer-size",
+            type=str,
+            help="Each parser thread reads the input in batches of this size, "
+            "and in parallel parsing, each batch that is not the last must be "
+            "large enough to contain the end of at least one statement "
+            "(default: 10M)",
+        )
         index_args["only_pso_and_pos_permutations"] = arg(
             "--only-pso-and-pos-permutations",
             action="store_true",
@@ -140,7 +157,8 @@ class Qleverfile:
                 "from_text_records_and_literals",
             ],
             default="none",
-            help="Whether to also build an index for text search" "and for which texts",
+            help="Whether to also build an index for text search"
+            "and for which texts",
         )
         index_args["text_words_file"] = arg(
             "--text-words-file",
@@ -168,10 +186,13 @@ class Qleverfile:
             "--host-name",
             type=str,
             default="localhost",
-            help="The name of the host on which the server listens for " "requests",
+            help="The name of the host on which the server listens for "
+            "requests",
         )
         server_args["port"] = arg(
-            "--port", type=int, help="The port on which the server listens for requests"
+            "--port",
+            type=int,
+            help="The port on which the server listens for requests",
         )
         server_args["access_token"] = arg(
             "--access-token",
@@ -328,7 +349,9 @@ class Qleverfile:
 
         # Read the Qleverfile.
         defaults = {"random": "83724324hztz", "version": "01.01.01"}
-        config = ConfigParser(interpolation=ExtendedInterpolation(), defaults=defaults)
+        config = ConfigParser(
+            interpolation=ExtendedInterpolation(), defaults=defaults
+        )
         try:
             config.read(qleverfile_path)
         except Exception as e:

qlever/util.py

@@ -3,9 +3,9 @@ from __future__ import annotations
 import errno
 import re
 import secrets
-import socket
 import shlex
 import shutil
+import socket
 import string
 import subprocess
 from datetime import date, datetime
@@ -30,8 +30,11 @@ def get_total_file_size(patterns: list[str]) -> int:
 
 
 def run_command(
-    cmd: str, return_output: bool = False, show_output: bool = False
-) -> Optional[str]:
+    cmd: str,
+    return_output: bool = False,
+    show_output: bool = False,
+    use_popen: bool = False,
+) -> Optional[str | subprocess.Popen]:
     """
     Run the given command and throw an exception if the exit code is non-zero.
     If `return_output` is `True`, return what the command wrote to `stdout`.
@@ -41,6 +44,7 @@ def run_command(
 
     TODO: Find the executable for `bash` in `__init__.py`.
     """
+
     subprocess_args = {
         "executable": shutil.which("bash"),
         "shell": True,
@@ -48,17 +52,21 @@ def run_command(
         "stdout": None if show_output else subprocess.PIPE,
         "stderr": subprocess.PIPE,
     }
+
+    # With `Popen`, the command runs in the current shell and a process object
+    # is returned (which can be used, e.g., to kill the process).
+    if use_popen:
+        if return_output:
+            raise Exception("Cannot return output if `use_popen` is `True`")
+        return subprocess.Popen(f"set -o pipefail; {cmd}", **subprocess_args)
+
+    # With `run`, the command runs in a subshell and the output is captured.
     result = subprocess.run(f"set -o pipefail; {cmd}", **subprocess_args)
+
     # If the exit code is non-zero, throw an exception. If something was
     # written to `stderr`, use that as the exception message. Otherwise, use a
     # generic message (which is also what `subprocess.run` does with
     # `check=True`).
-    # log.debug(f"Command `{cmd}` returned the following result")
-    # log.debug("")
-    # log.debug(f"exit code: {result.returncode}")
-    # log.debug(f"stdout: {result.stdout}")
-    # log.debug(f"stderr: {result.stderr}")
-    # log.debug("")
     if result.returncode != 0:
         if len(result.stderr) > 0:
             raise Exception(result.stderr.replace("\n", " ").strip())
@@ -99,7 +107,11 @@ def run_curl_command(
         )
     )
     result = subprocess.run(
-        curl_cmd, shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        curl_cmd,
+        shell=True,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
     )
     # Case 1: An error occurred, raise an exception.
     if result.returncode != 0:
@@ -120,18 +132,23 @@ def run_curl_command(
     return result.stdout
 
 
-def is_qlever_server_alive(port: str) -> bool:
+def is_qlever_server_alive(endpoint_url: str) -> bool:
     """
     Helper function that checks if a QLever server is running on the given
-    port.
+    endpoint. Return `True` if the server is alive, `False` otherwise.
     """
 
-    message = "from the qlever script".replace(" ", "%20")
-    curl_cmd = f"curl -s http://localhost:{port}/ping?msg={message}"
-    exit_code = subprocess.call(
-        curl_cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+    message = "from the `qlever` CLI"
+    curl_cmd = (
+        f"curl -s {endpoint_url}/ping"
+        f" --data-urlencode msg={shlex.quote(message)}"
     )
-    return exit_code == 0
+    log.debug(curl_cmd)
+    try:
+        run_command(curl_cmd)
+        return True
+    except Exception:
+        return False
 
 
 def get_existing_index_files(basename: str) -> list[str]:

{qlever-0.5.15.dist-info → qlever-0.5.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: qlever
-Version: 0.5.15
+Version: 0.5.18
 Summary: Script for using the QLever SPARQL engine.
 Author-email: Hannah Bast <bast@cs.uni-freiburg.de>
 License: Apache-2.0

{qlever-0.5.15.dist-info → qlever-0.5.18.dist-info}/RECORD

@@ -1,12 +1,12 @@
-qlever/__init__.py,sha256=7VKA8tp5iHZQyTXhDOcxUbloZ7WyxDnkruq0iJOzQcE,1403
-qlever/command.py,sha256=yOr0Uc8D8-AM7EjwDsVzbc3KNYjPH-FVOZhIHkqO588,2749
+qlever/__init__.py,sha256=GlDcYX0TzV-7nAFaBzPc1Y6cNwz0rsLJFNfLNXl9UiM,1410
+qlever/command.py,sha256=oxsMtu445OnoZ6kKRJCUbyAGbavWcu39gLor8hWuLjQ,2765
 qlever/config.py,sha256=qYPy-MQ7BwGrvKSazQWhs0lnlOFqm-d47mpZhc3fptc,10254
 qlever/containerize.py,sha256=bnHmjKIFEe-NcuAMRNnXAFjRVLcLnk9f5JspCCyhgt8,5210
-qlever/log.py,sha256=2O_RvFymnu_dB10ErBTAOsI8bgjORfdD0tE3USH-siM,1315
+qlever/log.py,sha256=WLscWV4fFF_w_uXSOfvWzhyzRM7t_61inE2ks3zf6Gw,1317
 qlever/qlever_main.py,sha256=-E4W8YdZ_teszGwXu6bQgBcH3y47TFJU8JLPIDwc_-g,2449
-qlever/qlever_old.py,sha256=X-JxmepFKYeFgSLLp0TRDNqXSxDwIbc8_0Xstiems8c,62026
-qlever/qleverfile.py,sha256=lygAjI5_wV_e-JoIGIqVTdd4yyvApzZiSlqSsmdlJpU,14529
-qlever/util.py,sha256=qLxBRyHPT2VTj0xcOCFcP6HV-Lm-g-64QpvOc4V0_a8,8029
+qlever/qlever_old.py,sha256=cY55BD3tCsxBmxKobKkI-VtjQBj6-4b3TLGs_9gXwQc,63388
+qlever/qleverfile.py,sha256=8J5s0zhVH97QaQtedYkfyvyps5B28nhaoNwRF1fROwY,15287
+qlever/util.py,sha256=xAK9GT8SgU3z65F1dFXazxsd60letqLQqQAZ81mdJSY,8374
 qlever/Qleverfiles/Qleverfile.dblp,sha256=RaTJrabloAdaHwSl5V7Ws9phWyM_oXPSeZpKodZXA6c,1256
 qlever/Qleverfiles/Qleverfile.dblp-plus,sha256=TJHxp8I1P6JKJjbuAllEpB32-huuY1gH0FlenqPVJ5g,1334
 qlever/Qleverfiles/Qleverfile.dbpedia,sha256=aaNZZayE-zVePGSwPzXemkX__Ns8-kP_E7DNNKZPnqg,1160
@@ -15,38 +15,40 @@ qlever/Qleverfiles/Qleverfile.dnb,sha256=43w_CVi00yf7FHdDvBtHHQR3yU1d-JCNnD_uxYZ
 qlever/Qleverfiles/Qleverfile.fbeasy,sha256=9dwCMltT0BIMN4LRmaZFp1a7aV0kh0nJ9XLiQb_NJNo,940
 qlever/Qleverfiles/Qleverfile.freebase,sha256=eFMOxeyuWVbb06Gv2-VFkuKE5tTyckddTDHdw5wbZN8,1028
 qlever/Qleverfiles/Qleverfile.imdb,sha256=1xUBFimgnEHKP_o6tlqwJvIVpEE4Zx6UK_JnnQsG7Ew,1638
-qlever/Qleverfiles/Qleverfile.ohm-planet,sha256=yG6NgW6x6sGTzmCi1wEAkQ-i9goTj2u3PQxvCUwjExs,2122
+qlever/Qleverfiles/Qleverfile.ohm-planet,sha256=xZJ0tyJmPGcD-naWV7BUBCj8ADxcKEN2AxvsrHuYuZs,2286
 qlever/Qleverfiles/Qleverfile.olympics,sha256=5w9BOFwEBhdSzPz-0LRxwhv-7Gj6xbF539HOXr3cqD0,1088
 qlever/Qleverfiles/Qleverfile.orkg,sha256=Uizz-RhlSeExgfckWztewa4l_v3zMN8IR7NaGYKrqt4,937
 qlever/Qleverfiles/Qleverfile.osm-country,sha256=Pb9o5H3b7wVlVRgdiPHWCvrnkIRS1YUhxOazbUmoKZE,1897
-qlever/Qleverfiles/Qleverfile.osm-planet,sha256=555aaMOIzAILvzwpCjucd_GE7TlYxmT2Qh8hME68stU,1418
+qlever/Qleverfiles/Qleverfile.osm-planet,sha256=-TN85LV6A8ukWhzkriV4RFuXsduUwDbU9xpg14MXcbA,1379
 qlever/Qleverfiles/Qleverfile.pubchem,sha256=ooSj2gqTzbGY_pMCvfL-MfE7Z0d5hQB4_EF5Pp2Mn6M,14465
 qlever/Qleverfiles/Qleverfile.scientists,sha256=9eZ2c6P9a3E3VHa3RR7LdOQbF4k3oyyrn56Z3u4LZYs,1164
-qlever/Qleverfiles/Qleverfile.uniprot,sha256=Yejez7uel9Vp8e4w-fI8-qLSqWyDUvjlq33gpebVC5k,6329
+qlever/Qleverfiles/Qleverfile.uniprot,sha256=Agih0rz1gRkfQjHNynLqBQ5OzljofopbOgArP6ijsNQ,6240
 qlever/Qleverfiles/Qleverfile.vvz,sha256=cLzm85erKoFCDllH5eFcSi35MdR6Tahj1MgtvGRxanM,922
 qlever/Qleverfiles/Qleverfile.wikidata,sha256=zVUXF75XJyK1h-J-7EjFemzmkSyoPtng1mNY3U7S78M,2061
 qlever/Qleverfiles/Qleverfile.wikipathways,sha256=UFEVLrtOBiSQfibBN9xc2wDXrnWcnx5f8PY9khcE6bc,1983
 qlever/Qleverfiles/Qleverfile.yago-4,sha256=hAS_2ZmC1zxNsKXip7t1F_iqu3CC-6O7v6HZhuFbnWY,1819
 qlever/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-qlever/commands/add_text_index.py,sha256=cvBhX5rooRAXsn1Oput6b0B10w-MGMfS4ZPAhDfm7OQ,3669
+qlever/commands/add_text_index.py,sha256=xJ49Iq1-CszXjHDvOpllqLw1J1kCxQl7H848XD1vEz0,3820
 qlever/commands/cache_stats.py,sha256=zGPLSWDNn7dwAAt2o-2ExqHmw1FeN8i6nEQbaaqF830,4156
 qlever/commands/clear_cache.py,sha256=n52ohE1EE4M_B4kId_v8tbAlW-BGwG1K-ZYQ9QgxIJU,2974
-qlever/commands/example_queries.py,sha256=g5s9nPDj9SmvLYF_sexESDJxl7YzM9cAbNqPVU5-wZg,19526
+qlever/commands/example_queries.py,sha256=l2brrZk1YWh1rkOSeCTgUueOZ7Jvu5Qncag-9B88jOI,22881
+qlever/commands/extract_queries.py,sha256=TZBmZLz_NknU1LKbl9nPmxdb82lsPeDhTWjIo81llvA,3942
 qlever/commands/get_data.py,sha256=nHOHMjv0tSLWJDOR0ba_LK-Bk-mcGnphb8hbqcVYFhE,1411
-qlever/commands/index.py,sha256=02o_-EFwxyCOY7Pl49O1tVvMQELWTb5tCwdmSlNgM8w,12777
+qlever/commands/index.py,sha256=W99AeBJNiN9pBVAVaayRAUEm6K58ucFqZvW6KpFBGxU,13327
 qlever/commands/index_stats.py,sha256=9EBo1Oq5PGjajrvWJNafJ-Wg_d90DaO5AGq9a5plSRM,11720
 qlever/commands/log.py,sha256=vLqkgtx1udnQqoUBMWB5G9rwr-l7UKrDpyFYSMuoXWw,1987
-qlever/commands/query.py,sha256=vu6vTeTqtKhuKbv_wzloHvbJFj_2GTHPIfDpQfrc8FE,3603
+qlever/commands/query.py,sha256=lqQR3wiDLAzxg3Da5Xim6gxkAeEexPJxldoTfB9U4H0,4588
+qlever/commands/settings.py,sha256=lyI_mdRMp3zv3RQ-tU2g2ZivfMD0r0R6ODf0VH9aVBk,3742
 qlever/commands/setup_config.py,sha256=wEy1LAunpOnqrUCbazMpt1u9HJCKgXJEMxF3zjh0jb0,3344
-qlever/commands/start.py,sha256=TdAulZqkTv9W0QBPwQqg-ixYI3iHMpZ1SHHvFRO_jbs,9524
+qlever/commands/start.py,sha256=gtYKB1sgc5SwY0DA3IMnowXMqXccHA2cOYZ71dgey5k,11589
 qlever/commands/status.py,sha256=TtnBqcdkF3zTDKft07zpVcIX7kFu7d_nOy9b6Ohh9vQ,1650
-qlever/commands/stop.py,sha256=AVafaoNb1JYmhulZ9_Bvq5gKcxBekb-S1QQRjBFo_-k,4160
-qlever/commands/system_info.py,sha256=Y3DtUJAlnh6uSvIVoToE10G7SStu-L5NbalIXvBB6D0,4614
-qlever/commands/ui.py,sha256=Ppo6YXA8JXNDPvC5D5duKGMPyhRvXfc1CaoSwjL-jBg,3644
+qlever/commands/stop.py,sha256=z25gWfLA9qIJ7F3zWZa0p0oVnt61kHcsB1evjgXhKX4,4557
+qlever/commands/system_info.py,sha256=I84EKgMO5J8pvsTDhkVKHzsRLtPajNg9KTQN5kWjqLU,4660
+qlever/commands/ui.py,sha256=T2Sl6w9tXNpZf-Zv3A9SLmarujszTIjGOYyPfyT4sX4,3821
 qlever/commands/warmup.py,sha256=kJHzS7HJo8pD2CphJuaXDj_CYP02YDo2DVM-pun3A80,1029
-qlever-0.5.15.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-qlever-0.5.15.dist-info/METADATA,sha256=A9M3q3eqWpFdUBOgKMNiPE8LnR5u78LCHJqCIZaQAfY,4583
-qlever-0.5.15.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-qlever-0.5.15.dist-info/entry_points.txt,sha256=U_gbYYi0wwdsn884eb0XoOXfvhACOsxhlO330dZ9bi0,87
-qlever-0.5.15.dist-info/top_level.txt,sha256=kd3zsYqiFd0--Czh5XTVkfEq6XR-XgRFW35X0v0GT-c,7
-qlever-0.5.15.dist-info/RECORD,,
+qlever-0.5.18.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+qlever-0.5.18.dist-info/METADATA,sha256=sOgFKONlHKZQkpaYYHV-GnMXRHGnXuhpqYDI7rKJmso,4583
+qlever-0.5.18.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+qlever-0.5.18.dist-info/entry_points.txt,sha256=U_gbYYi0wwdsn884eb0XoOXfvhACOsxhlO330dZ9bi0,87
+qlever-0.5.18.dist-info/top_level.txt,sha256=kd3zsYqiFd0--Czh5XTVkfEq6XR-XgRFW35X0v0GT-c,7
+qlever-0.5.18.dist-info/RECORD,,

{qlever-0.5.15.dist-info → qlever-0.5.18.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any