qlever 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

qlever/Qleverfiles/Qleverfile.ohm-planet

@@ -1,37 +1,37 @@
 # Qleverfile for OHM Planet, use with https://github.com/ad-freiburg/qlever-control
 #
 # qlever get-data  # ~20 mins (download PBF, convert to TTL, add GeoSPARQL triples)
-# qlever index     # ~10 mins and ~5 GB RAM (on an AMD Ryzen 9 5900X)
+# qlever index     # ~20 mins and ~5 GB RAM (on an AMD Ryzen 9 5900X)
 # qlever start     # ~1 sec
 #
-# For `qlever get-data` to work, `osm2rdf` and `spatialjoin` must be installed
-# and included in the `PATH`.
+# For `qlever get-data` to work, `osm2rdf` must be installed and in the `PATH`.
 
 [data]
 NAME           = ohm-planet
 GET_DATA_URL   = https://planet.openhistoricalmap.org/planet
+CHECK_BINARIES = osm2rdf -h > /dev/null || (echo "osm2rdf not found, make sure that it's installed and in your PATH" && exit 1)
 GET_DATA_CMD_1 = curl -LRfC - -o ${NAME}.pbf $$(curl -s ${GET_DATA_URL}/state.txt) 2>&1 | tee ${NAME}.download-log.txt
-GET_DATA_CMD_2 = osm2rdf ${NAME}.pbf -o ${NAME}.ttl --source-dataset OHM --simplify-wkt 0 --write-ogc-geo-triples none 2>&1 | tee ${NAME}.osm2rdf-log.txt
-GET_DATA_CMD_3 = bzcat ${NAME}.ttl.bz2 | \grep "^osm2rdf" | sed -En 's/^osm2rdf(geom)?:(ohm_)?(node|rel|way)[a-z]*_([0-9]+) geo:asWKT "([^\"]+)".*/ohm\3:\4\t\5/p' | tee ${NAME}.spatialjoin-input.tsv | spatialjoin --contains " ogc:sfContains " --covers " ogc:sfCovers " --intersects " ogc:sfIntersects " --equals " ogc:sfEquals " --touches " ogc:sfTouches " --crosses " ogc:sfCrosses " --overlaps " ogc:sfOverlaps " --suffix $$' .\n' -o ${NAME}.spatialjoin-triples.ttl.bz2 2>&1 | tee ${NAME}.spatialjoin-log.txt && rm -f areas events lines points simplelines
-GET_DATA_CMD   = set -o pipefail && ${GET_DATA_CMD_1} && echo && ${GET_DATA_CMD_2} && echo && ${GET_DATA_CMD_3} && head -100 <(bzcat ${NAME}.ttl.bz2) | sed '/^@prefix/!d' > ${NAME}.prefix-definitions
+GET_DATA_CMD_2 = osm2rdf ${NAME}.pbf -o ${NAME}.ttl --source-dataset OHM --cache . --add-hascentroid 2>&1 | tee ${NAME}.osm2rdf-log.txt
+GET_DATA_CMD   = set -o pipefail && ${CHECK_BINARIES} && ${GET_DATA_CMD_1} && echo && ${GET_DATA_CMD_2}
 VERSION        = $$(date -r ${NAME}.pbf +%d.%m.%Y || echo "NO_DATE")
 DESCRIPTION    = OHM Planet, data from ${GET_DATA_URL} version ${VERSION} (with GeoSPARQL predicates ogc:sfContains and ogc:sfIntersects)
 
 [index]
-INPUT_FILES      = ${data:NAME}.prefix-definitions ${data:NAME}.spatialjoin-triples.ttl.bz2 ${data:NAME}.ttl.bz2
+INPUT_FILES      = ${data:NAME}.ttl.bz2
 CAT_INPUT_FILES  = bzcat -f ${INPUT_FILES}
 SETTINGS_JSON    = { "prefixes-external": [""], "ascii-prefixes-only": false, "parallel-parsing": true, "num-triples-per-batch": 5000000 }
 
 [server]
-PORT               = 7037
-ACCESS_TOKEN       = ${data:NAME}
-MEMORY_FOR_QUERIES = 10G
-CACHE_MAX_SIZE     = 5G
-WARMUP_CMD         = curl -s http://localhost:${PORT} -H "Accept: application/qlever-results+json" --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> SELECT ?subject ?geometry WHERE { ?subject geo:hasGeometry ?m . ?m geo:asWKT ?geometry } INTERNAL SORT BY ?subject" --data-urlencode "access-token=${server:ACCESS_TOKEN}" --data-urlencode "pinresult=true" --data-urlencode "send=0" | jq .resultsize | xargs printf "Result size: %'d\n"
-
+PORT                        = 7037
+ACCESS_TOKEN                = ${data:NAME}
+MEMORY_FOR_QUERIES          = 10G
+CACHE_MAX_SIZE              = 5G
+CACHE_MAX_SIZE_SINGLE_ENTRY = 4G
+WARMUP_CMD                  = curl -s https://qlever.cs.uni-freiburg.de/mapui-petri/query --data-urlencode "query=PREFIX geo: <http://www.opengis.net/ont/geosparql#> PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> PREFIX osm: <https://www.openstreetmap.org/> SELECT ?osm_id ?geometry WHERE { ?osm_id geo:hasGeometry/geo:asWKT ?geometry . ?osm_id rdf:type osm:node } LIMIT 1" --data-urlencode "backend=https://qlever.cs.uni-freiburg.de/api/${data:NAME}" > /dev/null
 
 [runtime]
-SYSTEM = native
+SYSTEM = docker
+IMAGE  = adfreiburg/qlever:latest
 
 [ui]
 CONFIG = ohm-planet

qlever/Qleverfiles/Qleverfile.osm-country

@@ -8,35 +8,41 @@
 # the link under GET_DATA_CMD exists (the names are usually the canonical
 # names). The time for osm2rdf is around the same as that for "qlever index".
 
-# Indexer settings
+# Dataset settings
+[data]
 CONTINENT         = europe
 COUNTRY           = switzerland
-DB                = osm-${COUNTRY}
-PBF               = ${DB}.pbf
-RDF_FILES         = "${DB}.ttl.bz2"
-CAT_FILES         = "bzcat ${RDF_FILES}"
+NAME              = osm-${COUNTRY}
+PBF               = ${NAME}.pbf
 WITH_TEXT         = false
-STXXL_MEMORY   = 10
-SETTINGS_JSON     = '{ "prefixes-external": [ "\"LINESTRING(", "\"MULTIPOLYGON(", "\"POLYGON(" ], "ascii-prefixes-only": false, "num-triples-per-batch": 1000000 }'
-GET_DATA_CMD      = "wget -nc -O ${PBF} https://download.geofabrik.de/${CONTINENT}/${COUNTRY}-latest.osm.pbf; rm -f ${DB}.*.bz2; ( time /local/data/osm2rdf/build/apps/osm2rdf ${PBF} -o ${DB}.ttl --cache . --write-geometric-relation-statistics ) 2>&1 | tee ${DB}.osm2rdf-log.txt; rm -f spatial-*"
-DESCRIPTION = "OSM ${COUNTRY^}, dump from $(ls -l --time-style=+%d.%m.%Y ${PBF} 2> /dev/null | cut -d' ' -f6) with ogc:contains"
+VERSION           = $$(ls -l --time-style=+%d.%m.%Y ${PBF} 2> /dev/null | cut -d' ' -f6)
+GET_DATA_CMD      = wget -nc -O ${PBF} https://download.geofabrik.de/${CONTINENT}/${COUNTRY}-latest.osm.pbf; rm -f ${NAME}.*.bz2; ( time osm2rdf ${PBF} -o ${NAME}.ttl --cache . ) 2>&1 | tee ${NAME}.osm2rdf-log.txt; rm -f spatial-*
+DESCRIPTION       = OSM ${COUNTRY}, dump from ${VERSION} with ogc:sfContains
+
+# Indexer settings
+[index]
+INPUT_FILES       = ${data:NAME}.ttl.bz2
+CAT_INPUT_FILES   = bzcat ${data:NAME}.ttl.bz2
+STXXL_MEMORY      = 10G
+SETTINGS_JSON     = { "prefixes-external": [ "\"LINESTRING(", "\"MULTIPOLYGON(", "\"POLYGON(" ], "ascii-prefixes-only": false, "num-triples-per-batch": 1000000 }
 
 # Server settings
-HOSTNAME                    = $(hostname -f)
-SERVER_PORT                 = 7025
-ACCESS_TOKEN                = ${DB}_%RANDOM%
+[server]
+HOSTNAME                    = localhost
+PORT                        = 7025
+ACCESS_TOKEN                = ${data:NAME}_%RANDOM%
 MEMORY_FOR_QUERIES          = 20G
 CACHE_MAX_SIZE              = 10G
 CACHE_MAX_SIZE_SINGLE_ENTRY = 5G
 CACHE_MAX_NUM_ENTRIES       = 100
+TIMEOUT                     = 100s
 
-# QLever binaries
-QLEVER_BIN_DIR          = %QLEVER_BIN_DIR%
-USE_DOCKER              = true
-QLEVER_DOCKER_IMAGE     = adfreiburg/qlever
-QLEVER_DOCKER_CONTAINER = qlever.${DB}
+# Runtime to use
+[runtime]
+SYSTEM = docker
+IMAGE = docker.io/adfreiburg/qlever:latest
 
-# QLever UI
-QLEVERUI_PORT   = 7000
-QLEVERUI_DIR    = qlever-ui
-QLEVERUI_CONFIG = osm
+# Qlever UI
+[ui]
+UI_PORT   = 7000
+UI_CONFIG = osm

qlever/Qleverfiles/Qleverfile.wikimedia-commons

@@ -0,0 +1,37 @@
+# Qleverfile for Wikimedia Commons, TODO: add to https://github.com/ad-freiburg/qlever-control
+#
+# qlever get-data  # takes ~3 hours to download .bz2 file of size ~40 GB
+# qlever index     # takes ~2 hours and ~40 GB RAM (on an AMD Ryzen 9 5900X)
+# qlever start     # starts the server (takes around 15 seconds)
+
+[data]
+NAME              = wikimedia-commons
+MAIN_RDF_FILE     = latest-mediainfo.ttl.gz
+DATA_URL_BASE     = https://dumps.wikimedia.org/other/wikibase/commonswiki
+GET_TTL_CMD       = wget -nc ${DATA_URL_BASE}/${MAIN_RDF_FILE}
+GET_PROPS_CMD     = curl -s https://qlever.cs.uni-freiburg.de/api/wikidata -H "Accept: text/turtle" -H "Content-type: application/sparql-query" --data "PREFIX wikibase: <http://wikiba.se/ontology#> CONSTRUCT { ?s ?p ?o } WHERE { VALUES ?p { wikibase:claim wikibase:directClaim wikibase:novalue wikibase:propertyType wikibase:qualifier wikibase:qualifierValue wikibase:reference wikibase:referenceValue wikibase:statementProperty wikibase:statementValue } ?s ?p ?o }" > properties.nt
+GET_LABELS_CMD    = curl -s https://qlever.cs.uni-freiburg.de/api/wikidata -H "Accept: text/turtle" -H "Content-type: application/sparql-query" --data "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> CONSTRUCT { ?subject rdfs:label ?label } WHERE { ?subject @en@rdfs:label ?label }" > labels.nt
+GET_DATA_CMD      = ${GET_TTL_CMD} && ${GET_PROPS_CMD} && ${GET_LABELS_CMD}
+INDEX_DESCRIPTION = Wikimedia Commons from ${DATA_URL_BASE}, version 09.11.2023 + Wikidata triples for rdfs:label and wikibase:claim etc.
+
+[index]
+INPUT_FILES     = ${data:MAIN_RDF_FILE} labels.nt properties.nt
+CAT_INPUT_FILES = zcat -f ${INPUT_FILES}
+WITH_TEXT_INDEX = from_literals
+STXXL_MEMORY_GB = 5
+SETTINGS_JSON   = { "languages-internal": [], "prefixes-external": [""], "locale": { "language": "en", "country": "US", "ignore-punctuation": true }, "ascii-prefixes-only": true, "num-triples-per-batch": 5000000 }
+
+[server]
+PORT                           = 7033
+ACCESS_TOKEN                   = ${data:NAME}_2511328747
+MEMORY_FOR_QUERIES_GB          = 20
+CACHE_MAX_SIZE_GB              = 10
+CACHE_MAX_SIZE_GB_SINGLE_ENTRY = 5
+
+[runtime]
+SYSTEM = native
+IMAGE  = docker.io/adfreiburg/qlever:latest
+
+[ui]
+PORT   = 7000
+CONFIG = wikimedia-commons