qlever 0.5.6__py3-none-any.whl → 0.5.7__py3-none-any.whl

qlever/Qleverfiles/Qleverfile.ohm-planet

@@ -1,37 +1,37 @@
 # Qleverfile for OHM Planet, use with https://github.com/ad-freiburg/qlever-control
 #
 # qlever get-data  # ~20 mins (download PBF, convert to TTL, add GeoSPARQL triples)
-# qlever index     # ~10 mins and ~5 GB RAM (on an AMD Ryzen 9 5900X)
+# qlever index     # ~20 mins and ~5 GB RAM (on an AMD Ryzen 9 5900X)
 # qlever start     # ~1 sec
 #
-# For `qlever get-data` to work, `osm2rdf` and `spatialjoin` must be installed
-# and included in the `PATH`.
+# For `qlever get-data` to work, `osm2rdf` must be installed and in the `PATH`.
 
 [data]
 NAME           = ohm-planet
 GET_DATA_URL   = https://planet.openhistoricalmap.org/planet
+CHECK_BINARIES = osm2rdf -h > /dev/null || (echo "osm2rdf not found, make sure that it's installed and in your PATH" && exit 1)
 GET_DATA_CMD_1 = curl -LRfC - -o ${NAME}.pbf $$(curl -s ${GET_DATA_URL}/state.txt) 2>&1 | tee ${NAME}.download-log.txt
-GET_DATA_CMD_2 = osm2rdf ${NAME}.pbf -o ${NAME}.ttl --source-dataset OHM --simplify-wkt 0 --write-ogc-geo-triples none 2>&1 | tee ${NAME}.osm2rdf-log.txt
-GET_DATA_CMD_3 = bzcat ${NAME}.ttl.bz2 | \grep "^osm2rdf" | sed -En 's/^osm2rdf(geom)?:(ohm_)?(node|rel|way)[a-z]*_([0-9]+) geo:asWKT "([^\"]+)".*/ohm\3:\4\t\5/p' | tee ${NAME}.spatialjoin-input.tsv | spatialjoin --contains " ogc:sfContains " --covers " ogc:sfCovers " --intersects " ogc:sfIntersects " --equals " ogc:sfEquals " --touches " ogc:sfTouches " --crosses " ogc:sfCrosses " --overlaps " ogc:sfOverlaps " --suffix $$' .\n' -o ${NAME}.spatialjoin-triples.ttl.bz2 2>&1 | tee ${NAME}.spatialjoin-log.txt && rm -f areas events lines points simplelines
-GET_DATA_CMD   = set -o pipefail && ${GET_DATA_CMD_1} && echo && ${GET_DATA_CMD_2} && echo && ${GET_DATA_CMD_3} && head -100 <(bzcat ${NAME}.ttl.bz2) | sed '/^@prefix/!d' > ${NAME}.prefix-definitions
+GET_DATA_CMD_2 = osm2rdf ${NAME}.pbf -o ${NAME}.ttl --source-dataset OHM --cache . --add-hascentroid 2>&1 | tee ${NAME}.osm2rdf-log.txt
+GET_DATA_CMD   = set -o pipefail && ${CHECK_BINARIES} && ${GET_DATA_CMD_1} && echo && ${GET_DATA_CMD_2}
 VERSION        = $$(date -r ${NAME}.pbf +%d.%m.%Y || echo "NO_DATE")
 DESCRIPTION    = OHM Planet, data from ${GET_DATA_URL} version ${VERSION} (with GeoSPARQL predicates ogc:sfContains and ogc:sfIntersects)
 
 [index]
-INPUT_FILES      = ${data:NAME}.prefix-definitions ${data:NAME}.spatialjoin-triples.ttl.bz2 ${data:NAME}.ttl.bz2
+INPUT_FILES      = ${data:NAME}.ttl.bz2
 CAT_INPUT_FILES  = bzcat -f ${INPUT_FILES}
 SETTINGS_JSON    = { "prefixes-external": [""], "ascii-prefixes-only": false, "parallel-parsing": true, "num-triples-per-batch": 5000000 }
 
 [server]
-PORT               = 7037
-ACCESS_TOKEN       = ${data:NAME}
-MEMORY_FOR_QUERIES = 10G
-CACHE_MAX_SIZE     = 5G
-WARMUP_CMD         = curl -s http://localhost:${PORT} -H "Accept: application/qlever-results+json" --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> SELECT ?subject ?geometry WHERE { ?subject geo:hasGeometry ?m . ?m geo:asWKT ?geometry } INTERNAL SORT BY ?subject" --data-urlencode "access-token=${server:ACCESS_TOKEN}" --data-urlencode "pinresult=true" --data-urlencode "send=0" | jq .resultsize | xargs printf "Result size: %'d\n"
-
+PORT                        = 7037
+ACCESS_TOKEN                = ${data:NAME}
+MEMORY_FOR_QUERIES          = 10G
+CACHE_MAX_SIZE              = 5G
+CACHE_MAX_SIZE_SINGLE_ENTRY = 4G
+WARMUP_CMD                  = curl -s https://qlever.cs.uni-freiburg.de/mapui-petri/query --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> PREFIX osm: <https://www.openstreetmap.org/> SELECT ?osm_id ?geometry WHERE { ?osm_id geo:hasGeometry/geo:asWKT ?geometry . ?osm_id rdf:type osm:node } LIMIT 1" --data-urlencode "backend=https://qlever.cs.uni-freiburg.de/api/${data:NAME}" > /dev/null
 
 [runtime]
-SYSTEM = native
+SYSTEM = docker
+IMAGE  = adfreiburg/qlever:latest
 
 [ui]
 CONFIG = ohm-planet

qlever/Qleverfiles/Qleverfile.osm-country

@@ -8,35 +8,41 @@
 # the link under GET_DATA_CMD exists (the names are usually the canonical
 # names). The time for osm2rdf is around the same as that for "qlever index".
 
-# Indexer settings
+# Dataset settings
+[data]
 CONTINENT         = europe
 COUNTRY           = switzerland
-DB                = osm-${COUNTRY}
-PBF               = ${DB}.pbf
-RDF_FILES         = "${DB}.ttl.bz2"
-CAT_FILES         = "bzcat ${RDF_FILES}"
+NAME              = osm-${COUNTRY}
+PBF               = ${NAME}.pbf
 WITH_TEXT         = false
-STXXL_MEMORY   = 10
-SETTINGS_JSON     = '{ "prefixes-external": [ "\"LINESTRING(", "\"MULTIPOLYGON(", "\"POLYGON(" ], "ascii-prefixes-only": false, "num-triples-per-batch": 1000000 }'
-GET_DATA_CMD      = "wget -nc -O ${PBF} https://download.geofabrik.de/${CONTINENT}/${COUNTRY}-latest.osm.pbf; rm -f ${DB}.*.bz2; ( time /local/data/osm2rdf/build/apps/osm2rdf ${PBF} -o ${DB}.ttl --cache . --write-geometric-relation-statistics ) 2>&1 | tee ${DB}.osm2rdf-log.txt; rm -f spatial-*"
-DESCRIPTION = "OSM ${COUNTRY^}, dump from $(ls -l --time-style=+%d.%m.%Y ${PBF} 2> /dev/null | cut -d' ' -f6) with ogc:contains"
+VERSION           = $$(ls -l --time-style=+%d.%m.%Y ${PBF} 2> /dev/null | cut -d' ' -f6)
+GET_DATA_CMD      = wget -nc -O ${PBF} https://download.geofabrik.de/${CONTINENT}/${COUNTRY}-latest.osm.pbf; rm -f ${NAME}.*.bz2; ( time osm2rdf ${PBF} -o ${NAME}.ttl --cache . ) 2>&1 | tee ${NAME}.osm2rdf-log.txt; rm -f spatial-*
+DESCRIPTION       = OSM ${COUNTRY}, dump from ${VERSION} with ogc:sfContains
+
+# Indexer settings
+[index]
+INPUT_FILES       = ${data:NAME}.ttl.bz2
+CAT_INPUT_FILES   = bzcat ${data:NAME}.ttl.bz2
+STXXL_MEMORY      = 10G
+SETTINGS_JSON     = { "prefixes-external": [ "\"LINESTRING(", "\"MULTIPOLYGON(", "\"POLYGON(" ], "ascii-prefixes-only": false, "num-triples-per-batch": 1000000 }
 
 # Server settings
-HOSTNAME                    = $(hostname -f)
-SERVER_PORT                 = 7025
-ACCESS_TOKEN                = ${DB}_%RANDOM%
+[server]
+HOSTNAME                    = localhost
+PORT                        = 7025
+ACCESS_TOKEN                = ${data:NAME}_%RANDOM%
 MEMORY_FOR_QUERIES          = 20G
 CACHE_MAX_SIZE              = 10G
 CACHE_MAX_SIZE_SINGLE_ENTRY = 5G
 CACHE_MAX_NUM_ENTRIES       = 100
+TIMEOUT                     = 100s
 
-# QLever binaries
-QLEVER_BIN_DIR          = %QLEVER_BIN_DIR%
-USE_DOCKER              = true
-QLEVER_DOCKER_IMAGE     = adfreiburg/qlever
-QLEVER_DOCKER_CONTAINER = qlever.${DB}
+# Runtime to use
+[runtime]
+SYSTEM = docker
+IMAGE = docker.io/adfreiburg/qlever:latest
 
-# QLever UI
-QLEVERUI_PORT   = 7000
-QLEVERUI_DIR    = qlever-ui
-QLEVERUI_CONFIG = osm
+# Qlever UI
+[ui]
+UI_PORT   = 7000
+UI_CONFIG = osm

qlever/commands/index.py

@@ -119,11 +119,14 @@ class IndexCommand(QleverCommand):
         # Remove already existing container.
         if args.system in Containerize.supported_systems() \
                 and args.overwrite_existing:
-            try:
-                run_command(f"{args.system} rm -f {args.index_container}")
-            except Exception as e:
-                log.error(f"Removing existing container failed: {e}")
-                return False
+            if Containerize.is_running(args.system, args.index_container):
+                log.info("Another index process is running, trying to stop it ...")
+                log.info("")
+                try:
+                    run_command(f"{args.system} rm -f {args.index_container}")
+                except Exception as e:
+                    log.error(f"Removing existing container failed: {e}")
+                    return False
 
         # Write settings.json file.
         try:

qlever/containerize.py

@@ -9,6 +9,7 @@ import subprocess
 from typing import Optional
 
 from qlever.log import log
+from qlever.util import run_command
 
 
 class ContainerizeException(Exception):
@@ -78,6 +79,17 @@ class Containerize:
                              f" -c {shlex.quote(cmd)}")
         return containerized_cmd
 
+    @staticmethod
+    def is_running(container_system: str, container_name: str) -> bool:
+        # Note: the `{{{{` and `}}}}` result in `{{` and `}}`, respectively.
+        containers = (
+            run_command(f"{container_system} ps --format=\"{{{{.Names}}}}\"",
+                        return_output=True)
+            .strip()
+            .splitlines()
+        )
+        return container_name in containers
+
     @staticmethod
     def stop_and_remove_container(container_system: str,
                                   container_name: str) -> bool:

qlever/util.py

@@ -186,9 +186,8 @@ def get_random_string(length: int) -> str:
 
 def is_port_used(port: int) -> bool:
     """
-    Try to bind to the port on all interfaces to check if the port is already
-    in use. If the port is already in use, `socket.bind` will raise an
-    `OSError` with errno EADDRINUSE.
+    Try to bind to the port on all interfaces to check if the port is already in use.
+    If the port is already in use, `socket.bind` will raise an `OSError` with errno EADDRINUSE.
     """
     try:
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -201,16 +200,3 @@ def is_port_used(port: int) -> bool:
         if err.errno != errno.EADDRINUSE:
             log.warning(f"Failed to determine if port is used: {err}")
         return True
-
-
-def check_if_installed(name: str, check_cmd: str) -> bool:
-    """
-    Helper function that checks if a given program is installed by running
-    the given command.
-    """
-    try:
-        run_command(check_cmd)
-        return True
-    except Exception as e:
-        log.error(f"{name} is not installed: {e}")
-        return False

{qlever-0.5.6.dist-info → qlever-0.5.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qlever
-Version: 0.5.6
+Version: 0.5.7
 Summary: Script for using the QLever SPARQL engine.
 Author-email: Hannah Bast <bast@cs.uni-freiburg.de>
 License: Apache-2.0

{qlever-0.5.6.dist-info → qlever-0.5.7.dist-info}/RECORD

@@ -2,12 +2,12 @@ qlever/__init__.py,sha256=7VKA8tp5iHZQyTXhDOcxUbloZ7WyxDnkruq0iJOzQcE,1403
 qlever/__main__.py,sha256=MqM37bEzQeJEGUXZvuLcilIvnObZiG2eTGIkfKGpdnw,62016
 qlever/command.py,sha256=yOr0Uc8D8-AM7EjwDsVzbc3KNYjPH-FVOZhIHkqO588,2749
 qlever/config.py,sha256=qYPy-MQ7BwGrvKSazQWhs0lnlOFqm-d47mpZhc3fptc,10254
-qlever/containerize.py,sha256=p8g3O3G8a_0XLzSTzl_e5t9dqjbCQ-ippoA8vI2Z9pI,4193
+qlever/containerize.py,sha256=HRtC5jrRRpcBmdrXK2LsEAKiGVq2dWH1Oa2Ps2nvAsE,4643
 qlever/log.py,sha256=2O_RvFymnu_dB10ErBTAOsI8bgjORfdD0tE3USH-siM,1315
 qlever/qlever_main.py,sha256=tA_xqOs_FjvqlDIvKTprwuysfTwzsUjE7at26gRhCVA,2336
 qlever/qlever_old.py,sha256=X-JxmepFKYeFgSLLp0TRDNqXSxDwIbc8_0Xstiems8c,62026
 qlever/qleverfile.py,sha256=D321zDnWi-ScCefbFGBydKKI7lzzr1CkohHW6KuwVw0,13106
-qlever/util.py,sha256=xNXxXTDfoDqTV0DKo5rKQpkdIwvi7JwfW7ySelvJaZ0,8185
+qlever/util.py,sha256=20NQJquSk_mSqvlK4k0OrSBqWrxKs5SgVshm5ucus5o,7847
 qlever/Qleverfiles/Qleverfile.dblp,sha256=Y6BqAG1GZg-OmEs0HM00yAQuY2TGnSzsOO1LLmGVn2Y,994
 qlever/Qleverfiles/Qleverfile.dblp-plus,sha256=Dwd9pK1vPcelKfw6sA-IuyhbZ6yIxOh6_84JgPYnB9Q,1332
 qlever/Qleverfiles/Qleverfile.dbpedia,sha256=aaNZZayE-zVePGSwPzXemkX__Ns8-kP_E7DNNKZPnqg,1160
@@ -16,9 +16,9 @@ qlever/Qleverfiles/Qleverfile.dnb,sha256=43w_CVi00yf7FHdDvBtHHQR3yU1d-JCNnD_uxYZ
 qlever/Qleverfiles/Qleverfile.fbeasy,sha256=jeztW4gFpWL_w1nCH5qGHeZyZv2lz_kG6f1G3r3DkJ4,974
 qlever/Qleverfiles/Qleverfile.freebase,sha256=k6PqYrtHTBr0EydObm1Hg9QWyAAM9fXkdcjhReDg0fM,1035
 qlever/Qleverfiles/Qleverfile.imdb,sha256=8F6tpEi0uXsMm6lofwjNkLq1X9IEUSoHK8YK_5YCf_g,1638
-qlever/Qleverfiles/Qleverfile.ohm-planet,sha256=Y_yUxdpWpUOSDo_zmVKj3caa8X-Wv-1KmacBMks_4QA,2650
+qlever/Qleverfiles/Qleverfile.ohm-planet,sha256=yG6NgW6x6sGTzmCi1wEAkQ-i9goTj2u3PQxvCUwjExs,2122
 qlever/Qleverfiles/Qleverfile.olympics,sha256=5w9BOFwEBhdSzPz-0LRxwhv-7Gj6xbF539HOXr3cqD0,1088
-qlever/Qleverfiles/Qleverfile.osm-country,sha256=UnlkckSXJDrknZORlU-Hdj_J82U4kStl1aRctCc5n6M,1953
+qlever/Qleverfiles/Qleverfile.osm-country,sha256=Pb9o5H3b7wVlVRgdiPHWCvrnkIRS1YUhxOazbUmoKZE,1897
 qlever/Qleverfiles/Qleverfile.osm-planet,sha256=2RilNix0fplN3GsNNyOu3GzmUss1Pq7586WKOFAQnSs,1400
 qlever/Qleverfiles/Qleverfile.pubchem,sha256=YuDzWQmukSvL1opu7cf1KX9407_P21lmecYZ9cdbuvA,5611
 qlever/Qleverfiles/Qleverfile.scientists,sha256=9eZ2c6P9a3E3VHa3RR7LdOQbF4k3oyyrn56Z3u4LZYs,1164
@@ -34,7 +34,7 @@ qlever/commands/cache_stats.py,sha256=6JjueQstAqc8dNfgY8TP2EitFMxdUvCwrcyd7KUEb2
 qlever/commands/clear_cache.py,sha256=AnE1MOoj1ZexxrRT8FGeBLlv8rtQIVV4DP8VBn5-X-s,2843
 qlever/commands/example_queries.py,sha256=5-0ln5EkuDcQYPqKKAOcLaTIStMzFhkAogaNedfRc_I,15271
 qlever/commands/get_data.py,sha256=f9kjZI3TKad6JHSuXWNkeoajmW8h0Sx8ShvjauDCtNo,1412
-qlever/commands/index.py,sha256=iJ1wM7qtlAuRP_x0CupLWIndLRub1GqHvlCbB9ZlyPw,5680
+qlever/commands/index.py,sha256=TwL6o16KZON1hbD8UnXMw_tduLXWcy7D4o6tbsLt1OE,5888
 qlever/commands/index_stats.py,sha256=_BiUNBhmbYd9RPxrlm4HF0oENO6JmqnRiAkwkyOdN4U,11722
 qlever/commands/log.py,sha256=8Krt3MsTUDapYqVw1zUu5X15SF8mV97Uj0qKOWK8jXk,1861
 qlever/commands/query.py,sha256=_IDH-M8gKL_f1i5wzu0X452pZSUD0_qXl6bPXC85wX0,2750
@@ -44,9 +44,9 @@ qlever/commands/status.py,sha256=5S6EdapZEwFKV9cQZtNYcZhMbAXAY-FP6ggjIhfX8ek,163
 qlever/commands/stop.py,sha256=TZs4bxKHvujlZAU8BZmFjA5eXSZNAa6EeNzvPpEZsuI,4139
 qlever/commands/ui.py,sha256=b7g7Mp6ZWevn8f1kwFr-WR4ZWMq42KEV4cGl2QS7M1E,2828
 qlever/commands/warmup.py,sha256=WOZSxeV8U_F6pEEnAb6YybXLQMxZFTRJXs4BPHUhsmc,1030
-qlever-0.5.6.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-qlever-0.5.6.dist-info/METADATA,sha256=FRJKEH385p07cxSLLKRHSgiND-PFwFtPnIYWQjVBv3M,4582
-qlever-0.5.6.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-qlever-0.5.6.dist-info/entry_points.txt,sha256=U_gbYYi0wwdsn884eb0XoOXfvhACOsxhlO330dZ9bi0,87
-qlever-0.5.6.dist-info/top_level.txt,sha256=kd3zsYqiFd0--Czh5XTVkfEq6XR-XgRFW35X0v0GT-c,7
-qlever-0.5.6.dist-info/RECORD,,
+qlever-0.5.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+qlever-0.5.7.dist-info/METADATA,sha256=4g8Lf7NVarNL99xbtFGOCRTPky--pHznE7WtwYgYjfk,4582
+qlever-0.5.7.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+qlever-0.5.7.dist-info/entry_points.txt,sha256=U_gbYYi0wwdsn884eb0XoOXfvhACOsxhlO330dZ9bi0,87
+qlever-0.5.7.dist-info/top_level.txt,sha256=kd3zsYqiFd0--Czh5XTVkfEq6XR-XgRFW35X0v0GT-c,7
+qlever-0.5.7.dist-info/RECORD,,

{qlever-0.5.6.dist-info → qlever-0.5.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.0.0)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any