qlever 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

qlever/Qleverfiles/Qleverfile.dblp

@@ -1,20 +1,24 @@
-# Qleverfile for DBLP, use with https://github.com/ad-freiburg/qlever-control
+# Qleverfile for DBLP, use with QLever CLI (`pip install qlever`)
 #
-# qlever get-data  # takes ~3 mins (downloads .ttl.gz file of size ~3 GB)
-# qlever index     # takes ~4 mins (on an AMD Ryzen 9 5900X)
-# qlever start     # takes a few seconds
+# qlever get-data  # ~1 min, ~5 GB compressed, 1.3 B triples
+# qlever index     # ~30 min, ~20 GB RAM, ~25 GB index size on disk
+# qlever start     # ~3 s, adjust MEMORY_FOR_QUERIES as needed
+#
+# Measured on an AMD Ryzen 9 5950X with 128 GB RAM, and NVMe SSD (25.10.2024)
 
 [data]
 NAME         = dblp
-GET_DATA_URL = https://dblp.org/rdf/dblp.ttl.gz
-GET_DATA_CMD = curl -LRC - -O ${GET_DATA_URL} 2>&1 | tee ${data:NAME}.download-log.txt
+DATA_TARFILE = dblp_KG_with_associated_data.tar
+GET_DATA_URL = https://sparql.dblp.org/download/${DATA_TARFILE}
+GET_DATA_CMD = (curl -LROC - ${GET_DATA_URL} && tar -xf ${DATA_TARFILE}) 2>&1 | tee ${NAME}.download-log.txt && rm -f ${DATA_TARFILE}
 VERSION      = $$(date -r dblp.ttl.gz +"%d.%m.%Y %H:%M" || echo "NO_DATE")
-DESCRIPTION  = DBLP computer science bibliography, data from ${GET_DATA_URL} (version ${VERSION})
+DESCRIPTION  = DBLP computer science bibliography + citations from OpenCitations, data from ${GET_DATA_URL} (version ${VERSION})
+FORMAT       = ttl
 
 [index]
-INPUT_FILES     = dblp.ttl.gz
-CAT_INPUT_FILES = zcat ${INPUT_FILES}
-SETTINGS_JSON   = { "ascii-prefixes-only": false, "num-triples-per-batch": 1000000, "prefixes-external": [""] }
+INPUT_FILES      = *.gz
+MULTI_INPUT_JSON = $$(ls *.gz | awk 'BEGIN { printf "[ " } NR > 1 { printf ", " } { printf "{\"cmd\": \"zcat " $$0 "\"}" } END { printf "]" }')
+SETTINGS_JSON    = { "ascii-prefixes-only": false, "num-triples-per-batch": 5000000, "prefixes-external": [""] }
 
 [server]
 PORT               = 7015

qlever/Qleverfiles/Qleverfile.dblp-plus

@@ -9,12 +9,12 @@
 [data]
 NAME              = dblp-plus
 GET_DATA_CMD      = wget -nc -O dblp.ttl.gz https://dblp.org/rdf/dblp.ttl.gz
-INDEX_DESCRIPTION = Publication data from https://dblp.org, with affiliations from https://www.wikidata.org and citations from https://opencitations.net
+DESCRIPTION       = Publication data from https://dblp.org, with affiliations from https://www.wikidata.org and citations from https://opencitations.net
 TEXT_DESCRIPTION  = All literals, search with FILTER KEYWORDS(?text, "...")
 
 [index]
 INPUT_FILES       = dblp.ttl.gz affiliations.nt affiliations.additions.nt citations.nt
-CAT_INPUT_FILES   = zcat -f ${RDF_FILES}
+CAT_INPUT_FILES   = zcat -f ${INPUT_FILES}
 SETTINGS_JSON     = { "ascii-prefixes-only": false, "num-triples-per-batch": 1000000, "prefixes-external": [ "<https://w3id.org", "<https://doi.org", "<http://dx.doi.org" ] }
 TEXT_INDEX        = from_literals
 

qlever/Qleverfiles/Qleverfile.default

@@ -31,7 +31,7 @@ SETTINGS_JSON   = { "num-triples-per-batch": 1000000 }
 # URL parameter `access_token`. It should not be easily guessable, unless you
 # don't mind others to get privileged access to your server.
 [server]
-PORT         =
+PORT         = 8888
 ACCESS_TOKEN = 
 
 # Use SYSTEM = docker to run QLever inside a docker container; the Docker image

qlever/Qleverfiles/Qleverfile.fbeasy

@@ -13,13 +13,13 @@ TEXT_DESCRIPTION  = Sentences from Wikipedia that mention at least one Freebase
 
 [index]
 INPUT_FILES     = fbeasy.nt
-CAT_INPUT_FILES = cat ${RDF_FILES}
+CAT_INPUT_FILES = cat ${INPUT_FILES}
 SETTINGS_JSON   = { "ascii-prefixes-only": true, "num-triples-per-batch": 10000000 }
 
 [server]
-PORT                        = 7003
-ACCESS_TOKEN                = ${data:NAME}_12631403
-MEMORY_FOR_QUERIES          = 5G
+PORT               = 7003
+ACCESS_TOKEN       = ${data:NAME}
+MEMORY_FOR_QUERIES = 5G
 
 [runtime]
 SYSTEM = docker

qlever/Qleverfiles/Qleverfile.freebase

@@ -12,12 +12,12 @@ DESCRIPTION  = RDF data from ${DATA_URL}, latest (and final) version from 09.08.
 
 [index]
 INPUT_FILES     = freebase-rdf-latest.gz
-CAT_INPUT_FILES = zcat ${RDF_FILES}
+CAT_INPUT_FILES = zcat ${INPUT_FILES}
 SETTINGS_JSON   = { "languages-internal": [ "en" ], "prefixes-external": ["<"], "locale": { "language": "en", "country": "US", "ignore-punctuation": true }, "ascii-prefixes-only": false, "num-triples-per-batch": 10000000 }
 
 [server]
 PORT               = 7002
-ACCESS_TOKEN       = ${data:NAME}_12631403
+ACCESS_TOKEN       = ${data:NAME}
 MEMORY_FOR_QUERIES = 10G
 
 [runtime]

qlever/Qleverfiles/Qleverfile.imdb

@@ -28,7 +28,7 @@ ACCESS_TOKEN       = ${data:NAME}
 MEMORY_FOR_QUERIES = 5G
 
 [runtime]
-SYSTEM = native
+SYSTEM = docker
 IMAGE  = docker.io/adfreiburg/qlever:latest
 
 [ui]

qlever/Qleverfiles/Qleverfile.orkg

@@ -0,0 +1,30 @@
+# Qleverfile for ORKG, use with the QLever CLI (`pip install qlever`)
+#
+# qlever get-data  # Get the dataset
+# qlever index     # Build index data structures
+# qlever start     # Start the server
+
+[data]
+NAME         = orkg
+GET_DATA_URL = https://orkg.org/api/rdf/dump
+GET_DATA_CMD = curl -LR -o ${NAME}.ttl ${GET_DATA_URL} 2>&1 | tee ${NAME}.download-log.txt
+VERSION      = $$(date -r ${NAME}.ttl +%d.%m.%Y || echo "NO_DATE")
+DESCRIPTION  = The Open Research Knowledge Graph (ORKG) (data from ${GET_DATA_URL}, version ${VERSION})
+
+[index]
+INPUT_FILES     = ${data:NAME}.ttl
+CAT_INPUT_FILES = cat ${INPUT_FILES}
+SETTINGS_JSON   = { "ascii-prefixes-only": false, "num-triples-per-batch": 1000000, "prefixes-external": [""] }
+
+[server]
+PORT               = 7053
+ACCESS_TOKEN       = ${data:NAME}
+MEMORY_FOR_QUERIES = 10G
+CACHE_MAX_SIZE     = 5G
+
+[runtime]
+SYSTEM = docker
+IMAGE  = docker.io/adfreiburg/qlever:latest
+
+[ui]
+UI_CONFIG = orkg

qlever/Qleverfiles/Qleverfile.osm-planet

@@ -11,7 +11,7 @@
 NAME         = osm-planet
 DATA_URL     = https://osm2rdf.cs.uni-freiburg.de/ttl/planet.osm.ttl.bz2
 GET_DATA_CMD = curl --location --fail --continue-at - --remote-time --output ${NAME}.ttl.bz2 ${DATA_URL}
-VERSION      = $$(date -r ${NAME}.ttl.bz2 +"%d.%m.%Y")
+VERSION      = $$(date -r ${NAME}.ttl.bz2 +"%d.%m.%Y" || echo "NO_DATE")
 DESCRIPTION  = OSM Planet, data from ${DATA_URL} version ${VERSION} (complete OSM data, with GeoSPARQL predicates ogc:sfContains and ogc:sfIntersects)
 
 [index]

qlever/Qleverfiles/Qleverfile.vvz

@@ -14,13 +14,13 @@ TEXT_DESCRIPTION = All literals, search with FILTER KEYWORDS(?text, "...")
 
 [index]
 INPUT_FILES     = vvz.ttl
-CAT_INPUT_FILES = cat ${FILE_NAMES}
-SETTINGS_JSON   = { "ascii-prefixes-only": true, "num-triples-per-batch": 1000000 }
+CAT_INPUT_FILES = cat ${INPUT_FILES}
+SETTINGS_JSON   = { "ascii-prefixes-only": false, "num-triples-per-batch": 1000000 }
 TEXT_INDEX      = from_literals
 
 [server]
 PORT               = 7041
-ACCESS_TOKEN       = ${data:NAME}_8736426534
+ACCESS_TOKEN       = ${data:NAME}
 MEMORY_FOR_QUERIES = 10G
 
 [runtime]

qlever/Qleverfiles/Qleverfile.wikidata

@@ -1,33 +1,45 @@
-# Qleverfile for Wikidata, use with qlever script (`pip install qlever`)
+# Qleverfile for Wikidata, use with the QLever CLI (`pip install qlever`)
 #
-# qlever get-data  # downloads two .bz2 files of total size ~100 GB
-# qlever index     # takes ~4.5 hours and ~20 GB RAM (on an AMD Ryzen 9 5900X)
-# qlever start     # starts the server (takes a few seconds)
+# qlever get-data  # ~7 hours, ~110 GB (compressed), ~20 billion triples
+# qlever index     # ~5 hours, ~20 GB RAM, ~500 GB index size on disk
+# qlever start     # a few seconds, adjust MEMORY_FOR_QUERIES as needed
+#
+# Adding a text index takes an additional ~2 hours and ~50 GB of disk space
+#
+# Measured on an AMD Ryzen 9 5950X with 128 GB RAM, and NVMe SSD (18.10.2024)
 
 [DEFAULT]
 NAME = wikidata
 
 [data]
-GET_DATA_URL = https://dumps.wikimedia.org/wikidatawiki/entities
-GET_DATA_CMD = curl -LRC - --remote-name-all ${GET_DATA_URL}/latest-all.ttl.bz2 ${GET_DATA_URL}/latest-lexemes.ttl.bz2 2>&1
-VERSION      = $$(date -r latest-all.ttl.bz2 +%d.%m.%Y || echo "NO_DATE")
-DESCRIPTION  = Full Wikidata dump from ${GET_DATA_URL} (latest-all.ttl.bz2 and latest-lexemes.ttl.bz2, version ${VERSION})
+GET_DATA_URL      = https://dumps.wikimedia.org/wikidatawiki/entities
+GET_DATA_CMD      = curl -LROC - ${GET_DATA_URL}/latest-all.ttl.bz2 ${GET_DATA_URL}/latest-lexemes.ttl.bz2 2>&1 | tee wikidata.download-log.txt && curl -sL ${GET_DATA_URL}/dcatap.rdf | docker run -i --rm -v $$(pwd):/data stain/jena riot --syntax=RDF/XML --output=NT /dev/stdin > dcatap.nt
+DATE_WIKIDATA     = $$(date -r latest-all.ttl.bz2 +%d.%m.%Y || echo "NO_DATE")
+DATE_WIKIPEDIA    = $$(date -r wikipedia-abstracts.nt +%d.%m.%Y || echo "NO_DATE")
+DESCRIPTION       = Full Wikidata dump from ${GET_DATA_URL} (latest-all.ttl.bz2 and latest-lexemes.ttl.bz2, version ${DATE_WIKIDATA}) + English Wikipeda abstracts (version ${DATE_WIKIPEDIA}, available via schema:description)
+TEXT_DESCRIPTION  = All English and German literals + all sentences from the English Wikipedia (version ${DATE_WIKIPEDIA}), use with FILTER KEYWORDS(...)
 
 [index]
-INPUT_FILES     = latest-all.ttl.bz2 latest-lexemes.ttl.bz2
-CAT_INPUT_FILES = lbzcat -n 4 -f ${INPUT_FILES}
-SETTINGS_JSON   = { "languages-internal": [], "prefixes-external": [""], "locale": { "language": "en", "country": "US", "ignore-punctuation": true }, "ascii-prefixes-only": true, "num-triples-per-batch": 5000000 }
-STXXL_MEMORY    = 10G
+INPUT_FILES      = latest-all.ttl.bz2 latest-lexemes.ttl.bz2 wikipedia-abstracts.nt dcatap.nt
+MULTI_INPUT_JSON = [{ "cmd": "lbzcat -n 4 latest-all.ttl.bz2", "format": "ttl", "parallel": "true" },
+                    { "cmd": "lbzcat -n 1 latest-lexemes.ttl.bz2", "format": "ttl", "parallel": "false" },
+                    { "cmd": "cat wikipedia-abstracts.nt", "format": "nt", "parallel": "false" },
+                    { "cmd": "cat dcatap.nt", "format": "nt", "parallel": "false" }]
+SETTINGS_JSON    = { "languages-internal": [], "prefixes-external": [""], "locale": { "language": "en", "country": "US", "ignore-punctuation": true }, "ascii-prefixes-only": true, "num-triples-per-batch": 5000000 }
+STXXL_MEMORY     = 10G
+TEXT_INDEX       = from_text_records
 
 [server]
-PORT               = 7001
-ACCESS_TOKEN       = ${data:NAME}
-MEMORY_FOR_QUERIES = 20G
-CACHE_MAX_SIZE     = 10G
+PORT                        = 7001
+ACCESS_TOKEN                = ${data:NAME}_3fz47hfzrbf64b
+MEMORY_FOR_QUERIES          = 40G
+CACHE_MAX_SIZE              = 30G
+CACHE_MAX_SIZE_SINGLE_ENTRY = 5G
+TIMEOUT                     = 300s
 
 [runtime]
 SYSTEM = docker
-IMAGE  = docker.io/adfreiburg/qlever:latest
+IMAGE  = adfreiburg/qlever
 
 [ui]
 UI_CONFIG = wikidata

qlever/Qleverfiles/Qleverfile.yago-4

@@ -16,14 +16,14 @@ DESCRIPTION  = "Full dump from https://yago-knowledge.org/downloads/yago-4, vers
 
 [index]
 INPUT_FILES     = yago-wd-*.nt.gz
-CAT_INPUT_FILES = zcat ${FILE_NAMES}
+CAT_INPUT_FILES = zcat ${INPUT_FILES}
 SETTINGS_JSON   = { "languages-internal": ["en"], "locale": { "language": "en", "country": "US", "ignore-punctuation": true }, "ascii-prefixes-only": false, "num-triples-per-batch": 5000000 }
 STXXL_MEMORY    = 10G
 
 [server]
-PORT                        = 9004
-ACCESS_TOKEN                = ${DB}_2347348732
-MEMORY_FOR_QUERIES          = 30G
+PORT               = 9004
+ACCESS_TOKEN       = ${data:NAME}
+MEMORY_FOR_QUERIES = 30G
 
 [runtime]
 SYSTEM = docker