qlever 0.5.12__py3-none-any.whl → 0.5.17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of qlever might be problematic. Click here for more details.
- qlever/Qleverfiles/Qleverfile.dblp +1 -1
- qlever/Qleverfiles/Qleverfile.pubchem +102 -26
- qlever/Qleverfiles/Qleverfile.uniprot +48 -16
- qlever/commands/add_text_index.py +2 -1
- qlever/commands/cache_stats.py +1 -1
- qlever/commands/clear_cache.py +4 -2
- qlever/commands/example_queries.py +236 -75
- qlever/commands/extract_queries.py +113 -0
- qlever/commands/get_data.py +1 -1
- qlever/commands/index.py +51 -11
- qlever/commands/index_stats.py +90 -59
- qlever/commands/log.py +12 -2
- qlever/commands/query.py +66 -27
- qlever/commands/settings.py +110 -0
- qlever/commands/setup_config.py +1 -1
- qlever/commands/start.py +222 -105
- qlever/commands/status.py +2 -1
- qlever/commands/stop.py +43 -32
- qlever/commands/system_info.py +1 -1
- qlever/commands/ui.py +3 -1
- qlever/commands/warmup.py +1 -1
- qlever/qlever_main.py +16 -9
- qlever/util.py +34 -17
- {qlever-0.5.12.dist-info → qlever-0.5.17.dist-info}/METADATA +2 -2
- qlever-0.5.17.dist-info/RECORD +54 -0
- {qlever-0.5.12.dist-info → qlever-0.5.17.dist-info}/WHEEL +1 -1
- qlever-0.5.12.dist-info/RECORD +0 -52
- {qlever-0.5.12.dist-info → qlever-0.5.17.dist-info}/LICENSE +0 -0
- {qlever-0.5.12.dist-info → qlever-0.5.17.dist-info}/entry_points.txt +0 -0
- {qlever-0.5.12.dist-info → qlever-0.5.17.dist-info}/top_level.txt +0 -0
|
@@ -17,7 +17,7 @@ FORMAT = ttl
|
|
|
17
17
|
|
|
18
18
|
[index]
|
|
19
19
|
INPUT_FILES = *.gz
|
|
20
|
-
MULTI_INPUT_JSON =
|
|
20
|
+
MULTI_INPUT_JSON = { "cmd": "zcat {}", "for-each": "*.gz" }
|
|
21
21
|
SETTINGS_JSON = { "ascii-prefixes-only": false, "num-triples-per-batch": 5000000, "prefixes-external": [""] }
|
|
22
22
|
|
|
23
23
|
[server]
|
|
@@ -1,51 +1,127 @@
|
|
|
1
|
-
# Qleverfile for PubChem, use with
|
|
1
|
+
# Qleverfile for PubChem, use with the QLever CLI (`pip install qlever`)
|
|
2
2
|
#
|
|
3
|
-
#
|
|
4
|
-
#
|
|
5
|
-
# qlever get-data # ~2 hours, ~150 GB, ~19 billion triples
|
|
6
|
-
# qlever index # ~7 hours, ~20 GB RAM, ~400 GB disk space
|
|
3
|
+
# qlever get-data # ~2 hours, ~120 GB, ~19 billion triples
|
|
4
|
+
# qlever index # ~6 hours, ~20 GB RAM, ~350 GB disk space (for the index)
|
|
7
5
|
# qlever start # a few seconds
|
|
8
6
|
#
|
|
9
|
-
#
|
|
7
|
+
# Measured on an AMD Ryzen 9 7950X with 128 GB RAM, and NVMe SSD (17.12.2024)
|
|
8
|
+
#
|
|
9
|
+
# NOTE 1: `qlever get-data` does not only download the PubChem RDF data, but also
|
|
10
10
|
# a number of ontologies. These are very useful to obtain names for IRIs like
|
|
11
11
|
# `sio:SIO_000008` or `obo:IAO_0000412` (otherwise very hard to understand).
|
|
12
|
-
# The ontologies BAO and NDF-RT are
|
|
12
|
+
# The ontologies BAO and NDF-RT are occasionally updated; for latest versions,
|
|
13
13
|
# see the download links at https://bioportal.bioontology.org/ontologies/BAO
|
|
14
14
|
# and https://bioportal.bioontology.org/ontologies/NDF-RT .
|
|
15
|
-
#
|
|
16
|
-
# NOTE 2: Many of the TTL files have generic prefix definitions in the middle
|
|
17
|
-
# of the file, like @prefix ns23: <http://identifiers.org/biocyc/ARACYC:> .
|
|
18
|
-
# See https://github.com/ad-freiburg/qlever/issues/711#issuecomment-1197113953
|
|
19
|
-
# This is allowed by the standard, but unusual. For use with QLever, we
|
|
20
|
-
# therefore convert the TTL files to NT when downloading them.
|
|
21
15
|
#
|
|
22
|
-
# NOTE
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
# about such IRIs while indexing, but accept them anyway.
|
|
16
|
+
# NOTE 2: The `MULTI_INPUT_JSON` zcats selected files together in one input
|
|
17
|
+
# stream because there are too many files and the command line triggered by
|
|
18
|
+
# `qlever index` would be too long otherwise.
|
|
26
19
|
|
|
27
20
|
[data]
|
|
28
21
|
NAME = pubchem
|
|
29
22
|
GET_DATA_URL = ftp://ftp.ncbi.nlm.nih.gov/pubchem/RDF
|
|
23
|
+
ONTOLOGIES_DIR = RDF.ontologies
|
|
24
|
+
PUBCHEM_DIR = RDF.pubchem
|
|
25
|
+
ONTOLOGIES_CSV = ontologies.csv
|
|
30
26
|
CHECK_REQUIREMENTS = for CMD in docker parallel; do $$CMD --version >/dev/null 2>&1 || (echo "Requires \"$$CMD\", please install it"; false); done
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
GET_DATA_CMD = ${CHECK_REQUIREMENTS} && ${
|
|
27
|
+
GET_DATA_CMD_1 = mkdir -p ${ONTOLOGIES_DIR} && cd ${ONTOLOGIES_DIR} && cat ${ONTOLOGIES_CSV} | parallel --colsep "," 'FILE={2} && URL={3} && ERRFILE=$${FILE%.*}.jena-stderr; echo "Processing $$URL ($$FILE) ..." && curl -sLRo $$FILE $$URL && docker run --rm -v $$(pwd):/data stain/jena riot --output=NT /data/$$FILE 2> $$ERRFILE | gzip -c > $${FILE%.*}.nt.gz && rm -f $$FILE; if [ -s $$ERRFILE ]; then grep -q "ERROR *riot" $$ERRFILE && echo "riot ERRORs in $$FILE, check $$ERRFILE"; else rm $$ERRFILE; fi'
|
|
28
|
+
GET_DATA_CMD_2 = mkdir -p ${PUBCHEM_DIR} && wget -r -nv -nH --cut-dirs=2 --no-parent -P ${PUBCHEM_DIR} ${GET_DATA_URL}
|
|
29
|
+
GET_DATA_CMD = ${CHECK_REQUIREMENTS} && ${GET_DATA_CMD_1} 2>&1 | tee pubchem.get-data-log.txt; ${GET_DATA_CMD_2} 2>&1 | tee -a pubchem.get-data-log.txt
|
|
34
30
|
VERSION = $$(date -r void.ttl +%d.%m.%Y || echo "NO_DATE")
|
|
35
31
|
DESCRIPTION = PubChem RDF from ${GET_DATA_URL} (version ${VERSION}) + associated ontologies (bao, bfo, biopax-level3, chebi, cheminf, cito, dublin_core_terms, fabio, go, iao, ncit, obi, pr, ro, sio, skos, so, uo)
|
|
36
|
-
MAKE_ONTOLOGIES_CSV = $$(mkdir -p
|
|
32
|
+
MAKE_ONTOLOGIES_CSV = $$(mkdir -p ${ONTOLOGIES_DIR} && echo "BAO - BioAssay Ontology,bao.owl,https://data.bioontology.org/ontologies/BAO/submissions/56/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb\nBFO - Basic Formal Ontology,bfo.owl,http://purl.obolibrary.org/obo/bfo.owl\nBioPAX - biological pathway data,bp.owl,http://www.biopax.org/release/biopax-level3.owl\nCHEMINF - Chemical Information Ontology,cheminf.owl,http://purl.obolibrary.org/obo/cheminf.owl\nChEBI - Chemical Entities of Biological Interest,chebi.owl,http://purl.obolibrary.org/obo/chebi.owl\nCiTO,cito.nt,http://purl.org/spar/cito.nt\nDCMI Terms,dcterms.nt,https://www.dublincore.org/specifications/dublin-core/dcmi-terms/dublin_core_terms.nt\nFaBiO,fabio.nt,http://purl.org/spar/fabio.nt\nGO - Gene Ontology,go.owl,http://purl.obolibrary.org/obo/go.owl\nIAO - Information Artifact Ontology,iao.owl,http://purl.obolibrary.org/obo/iao.owl\nNCIt,ncit.owl,http://purl.obolibrary.org/obo/ncit.owl\nNDF-RT,ndfrt.owl,https://data.bioontology.org/ontologies/NDF-RT/submissions/1/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb\nOBI - Ontology for Biomedical Investigations,obi.owl,http://purl.obolibrary.org/obo/obi.owl\nOWL,owl.ttl,http://www.w3.org/2002/07/owl.ttl\nPDBo,pdbo.owl,http://rdf.wwpdb.org/schema/pdbx-v40.owl\nPR - PRotein Ontology (PRO),pr.owl,http://purl.obolibrary.org/obo/pr.owl\nRDF Schema,rdfs.ttl,https://www.w3.org/2000/01/rdf-schema.ttl\nRDF,rdf.ttl,http://www.w3.org/1999/02/22-rdf-syntax-ns.ttl\nRO - Relation Ontology,ro.owl,http://purl.obolibrary.org/obo/ro.owl\nSIO - Semanticscience Integrated Ontology,sio.owl,http://semanticscience.org/ontology/sio.owl\nSKOS,skos.rdf,http://www.w3.org/TR/skos-reference/skos.rdf\nSO - Sequence types and features ontology,so.owl,http://purl.obolibrary.org/obo/so.owl\nUO - Units of measurement ontology,uo.owl,http://purl.obolibrary.org/obo/uo.owl" > ${ONTOLOGIES_DIR}/${ONTOLOGIES_CSV})
|
|
37
33
|
|
|
38
34
|
[index]
|
|
39
|
-
INPUT_FILES
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
35
|
+
INPUT_FILES = ${data:ONTOLOGIES_DIR}/*.nt.gz ${data:PUBCHEM_DIR}/*/*.ttl.gz ${data:PUBCHEM_DIR}/*/*/*.ttl.gz
|
|
36
|
+
BASE_URL = http://rdf.ncbi.nlm.nih.gov/pubchem
|
|
37
|
+
MULTI_INPUT_JSON = [{ "cmd": "zcat ${data:ONTOLOGIES_DIR}/*.nt.gz", "graph": "${BASE_URL}/ruleset"},
|
|
38
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/anatomy", "for-each": "${data:PUBCHEM_DIR}/anatomy/*.ttl.gz" },
|
|
39
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/author", "for-each": "${data:PUBCHEM_DIR}/author/*.ttl.gz" },
|
|
40
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/bioassay", "for-each": "${data:PUBCHEM_DIR}/bioassay/*.ttl.gz" },
|
|
41
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/book", "for-each": "${data:PUBCHEM_DIR}/book/*.ttl.gz" },
|
|
42
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/cell", "for-each": "${data:PUBCHEM_DIR}/cell/*.ttl.gz" },
|
|
43
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/compound/general/*0.ttl.gz", "graph": "${BASE_URL}/compound" },
|
|
44
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/compound/general/*1.ttl.gz", "graph": "${BASE_URL}/compound" },
|
|
45
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/compound/general/*2.ttl.gz", "graph": "${BASE_URL}/compound" },
|
|
46
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/compound/general/*3.ttl.gz", "graph": "${BASE_URL}/compound" },
|
|
47
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/compound/general/*4.ttl.gz", "graph": "${BASE_URL}/compound" },
|
|
48
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/compound/general/*5.ttl.gz", "graph": "${BASE_URL}/compound" },
|
|
49
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/compound/general/*6.ttl.gz", "graph": "${BASE_URL}/compound" },
|
|
50
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/compound/general/*7.ttl.gz", "graph": "${BASE_URL}/compound" },
|
|
51
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/compound/general/*8.ttl.gz", "graph": "${BASE_URL}/compound" },
|
|
52
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/compound/general/*9.ttl.gz", "graph": "${BASE_URL}/compound" },
|
|
53
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/compound", "for-each": "${data:PUBCHEM_DIR}/compound/general/*[!0-9].ttl.gz" },
|
|
54
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/concept", "for-each": "${data:PUBCHEM_DIR}/concept/*.ttl.gz" },
|
|
55
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/conserveddomain", "for-each": "${data:PUBCHEM_DIR}/conserveddomain/*.ttl.gz" },
|
|
56
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/cooccurrence", "for-each": "${data:PUBCHEM_DIR}/cooccurrence/*.ttl.gz" },
|
|
57
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/descriptor/compound/*0.ttl.gz", "graph": "${BASE_URL}/descriptor" },
|
|
58
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/descriptor/compound/*1.ttl.gz", "graph": "${BASE_URL}/descriptor" },
|
|
59
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/descriptor/compound/*2.ttl.gz", "graph": "${BASE_URL}/descriptor" },
|
|
60
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/descriptor/compound/*3.ttl.gz", "graph": "${BASE_URL}/descriptor" },
|
|
61
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/descriptor/compound/*4.ttl.gz", "graph": "${BASE_URL}/descriptor" },
|
|
62
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/descriptor/compound/*5.ttl.gz", "graph": "${BASE_URL}/descriptor" },
|
|
63
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/descriptor/compound/*6.ttl.gz", "graph": "${BASE_URL}/descriptor" },
|
|
64
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/descriptor/compound/*7.ttl.gz", "graph": "${BASE_URL}/descriptor" },
|
|
65
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/descriptor/compound/*8.ttl.gz", "graph": "${BASE_URL}/descriptor" },
|
|
66
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/descriptor/compound/*9.ttl.gz", "graph": "${BASE_URL}/descriptor" },
|
|
67
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/descriptor", "for-each": "${data:PUBCHEM_DIR}/descriptor/compound/*[!0-9].ttl.gz" },
|
|
68
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/descriptor", "for-each": "${data:PUBCHEM_DIR}/descriptor/substance/*.ttl.gz" },
|
|
69
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/disease", "for-each": "${data:PUBCHEM_DIR}/disease/*.ttl.gz" },
|
|
70
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/endpoint", "for-each": "${data:PUBCHEM_DIR}/endpoint/*.ttl.gz" },
|
|
71
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/gene", "for-each": "${data:PUBCHEM_DIR}/gene/*.ttl.gz"},
|
|
72
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/grant", "for-each": "${data:PUBCHEM_DIR}/grant/*.ttl.gz" },
|
|
73
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/inchikey", "for-each": "${data:PUBCHEM_DIR}/inchikey/*.ttl.gz" },
|
|
74
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/journal", "for-each": "${data:PUBCHEM_DIR}/journal/*.ttl.gz" },
|
|
75
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/measuregroup", "for-each": "${data:PUBCHEM_DIR}/measuregroup/*.ttl.gz" },
|
|
76
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/organization", "for-each": "${data:PUBCHEM_DIR}/organization/*.ttl.gz" },
|
|
77
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/*0.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
78
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/*1.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
79
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/*2.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
80
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/*3.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
81
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/*4.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
82
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/*5.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
83
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/*6.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
84
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/*7.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
85
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/*8.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
86
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/*9.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
87
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/cpc/*.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
88
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/patent/ipc/*.ttl.gz", "graph": "${BASE_URL}/patent" },
|
|
89
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/patent", "for-each": "${data:PUBCHEM_DIR}/patent/*[!0-9].ttl.gz" },
|
|
90
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/pathway", "for-each": "${data:PUBCHEM_DIR}/pathway/*.ttl.gz" },
|
|
91
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/protein", "for-each": "${data:PUBCHEM_DIR}/protein/*.ttl.gz" },
|
|
92
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/reference/*0.ttl.gz", "graph": "${BASE_URL}/reference" },
|
|
93
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/reference/*1.ttl.gz", "graph": "${BASE_URL}/reference" },
|
|
94
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/reference/*2.ttl.gz", "graph": "${BASE_URL}/reference" },
|
|
95
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/reference/*3.ttl.gz", "graph": "${BASE_URL}/reference" },
|
|
96
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/reference/*4.ttl.gz", "graph": "${BASE_URL}/reference" },
|
|
97
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/reference/*5.ttl.gz", "graph": "${BASE_URL}/reference" },
|
|
98
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/reference/*6.ttl.gz", "graph": "${BASE_URL}/reference" },
|
|
99
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/reference/*7.ttl.gz", "graph": "${BASE_URL}/reference" },
|
|
100
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/reference/*8.ttl.gz", "graph": "${BASE_URL}/reference" },
|
|
101
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/reference/*9.ttl.gz", "graph": "${BASE_URL}/reference" },
|
|
102
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/reference", "for-each": "${data:PUBCHEM_DIR}/reference/*[!0-9].ttl.gz" },
|
|
103
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/source", "for-each": "${data:PUBCHEM_DIR}/source/*.ttl.gz" },
|
|
104
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/substance/*0.ttl.gz", "graph": "${BASE_URL}/substance" },
|
|
105
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/substance/*1.ttl.gz", "graph": "${BASE_URL}/substance" },
|
|
106
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/substance/*2.ttl.gz", "graph": "${BASE_URL}/substance" },
|
|
107
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/substance/*3.ttl.gz", "graph": "${BASE_URL}/substance" },
|
|
108
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/substance/*4.ttl.gz", "graph": "${BASE_URL}/substance" },
|
|
109
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/substance/*5.ttl.gz", "graph": "${BASE_URL}/substance" },
|
|
110
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/substance/*6.ttl.gz", "graph": "${BASE_URL}/substance" },
|
|
111
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/substance/*7.ttl.gz", "graph": "${BASE_URL}/substance" },
|
|
112
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/substance/*8.ttl.gz", "graph": "${BASE_URL}/substance" },
|
|
113
|
+
{ "cmd": "zcat ${data:PUBCHEM_DIR}/substance/*9.ttl.gz", "graph": "${BASE_URL}/substance" },
|
|
114
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/substance", "for-each": "${data:PUBCHEM_DIR}/substance/*[!0-9].ttl.gz" },
|
|
115
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/synonym", "for-each": "${data:PUBCHEM_DIR}/synonym/*.ttl.gz" },
|
|
116
|
+
{ "cmd": "zcat {}", "graph": "${BASE_URL}/taxonomy", "for-each": "${data:PUBCHEM_DIR}/taxonomy/*.ttl.gz" }]
|
|
117
|
+
SETTINGS_JSON = { "languages-internal": [], "prefixes-external": [""], "num-triples-per-batch": 10000000 }
|
|
118
|
+
STXXL_MEMORY = 20G
|
|
43
119
|
|
|
44
120
|
[server]
|
|
45
121
|
PORT = 7023
|
|
46
122
|
ACCESS_TOKEN = ${data:NAME}
|
|
47
123
|
MEMORY_FOR_QUERIES = 20G
|
|
48
|
-
TIMEOUT =
|
|
124
|
+
TIMEOUT = 600s
|
|
49
125
|
|
|
50
126
|
[runtime]
|
|
51
127
|
SYSTEM = docker
|
|
@@ -1,30 +1,62 @@
|
|
|
1
1
|
# Qleverfile for UniProt, use with https://github.com/ad-freiburg/qlever-control
|
|
2
2
|
#
|
|
3
|
-
# qlever get-data # takes ~ 30 hours and ~
|
|
4
|
-
# qlever index # takes ~ 40 hours and ~ 60 GB RAM (on an AMD Ryzen 9
|
|
3
|
+
# qlever get-data # takes ~ 30 hours and ~ 1.6 TB of disk (for the TTL files)
|
|
4
|
+
# qlever index # takes ~ 40 hours and ~ 60 GB RAM (on an AMD Ryzen 9 9950X)
|
|
5
5
|
# qlever start # starts the server (takes a few seconds)
|
|
6
6
|
#
|
|
7
|
-
# Install packages: sudo apt install -y libxml2-utils parallel xz-utils
|
|
7
|
+
# Install packages: sudo apt install -y libxml2-utils parallel xz-utils wget
|
|
8
8
|
# Install manually: Apache Jena binaries (https://dlcdn.apache.org/jena/binaries)
|
|
9
9
|
#
|
|
10
10
|
# Set DATE to the date of the latest release. Build on SSD (requires ~ 7 TB
|
|
11
|
-
# during build, ~ 3 TB after build).
|
|
12
|
-
# HDD without significant performance loss (when running the server).
|
|
11
|
+
# during build, ~ 3 TB after build).
|
|
13
12
|
|
|
14
13
|
[data]
|
|
15
|
-
NAME
|
|
16
|
-
DATE
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
14
|
+
NAME = uniprot
|
|
15
|
+
DATE = 2024-11-27
|
|
16
|
+
RDFXML_DIR = rdf.${DATE}
|
|
17
|
+
TTL_DIR = ttl.${DATE}
|
|
18
|
+
UNIPROT_URL = https://ftp.uniprot.org/pub/databases/uniprot/current_release/rdf
|
|
19
|
+
RHEA_URL = https://ftp.expasy.org/databases/rhea/rdf
|
|
20
|
+
EXAMPLES_URL = https://github.com/sib-swiss/sparql-examples
|
|
21
|
+
GET_EXAMPLES_CMD = mkdir -p ${TTL_DIR} && git clone ${EXAMPLES_URL} && (cd sparql-examples && ./convertToOneTurtle.sh -p uniprot && gzip examples_uniprot.ttl && mv -f examples_uniprot.ttl.gz ../${TTL_DIR} && cd .. && rm -rf sparql-examples)
|
|
22
|
+
GET_RDFXML_CMD = mkdir -p ${RDFXML_DIR} && (echo "${RHEA_URL}/chebi.owl.gz"; echo "${RHEA_URL}/rhea.rdf.gz"; curl -s ${UNIPROT_URL}/RELEASE.meta4 | sed "s/<metalink.*/<metalink>/" | xmllint --xpath "/metalink/files/file/url[@location=\"ch\"]/text()" -) | while read URL; do wget --no-verbose -P ${RDFXML_DIR} $$URL 2>&1 | tee -a uniprot.download-log; done
|
|
23
|
+
RDFXML2TTL_CMD = mkdir -p ${TTL_DIR} && for RDFXML in ${RDFXML_DIR}/*.{owl,owl.xz,rdf,rdf.xz}; do echo "xzcat -f $$RDFXML | rdfxml --output=ttl -q 2> ${TTL_DIR}/$$(basename $$RDFXML).stderr | gzip -c > ${TTL_DIR}/$$(basename $$RDFXML | sed 's/\(rdf\|rdf.xz\|owl\|owl.xz\)$$/ttl.gz/') && echo 'DONE converting $$RDFXML'"; done | parallel
|
|
24
|
+
GET_DATA_CMD = date > ${NAME}.get-data.begin-date && ${GET_EXAMPLES_CMD} && ${GET_RDFXML_CMD} && ${RDFXML2TTL_CMD} && date > ${NAME}.get-data.end-date
|
|
25
|
+
DESCRIPTION = Complete UniProt data from ${UNIPROT_URL}, with additional data from ${RHEA_URL} and ${EXAMPLES_URL}
|
|
22
26
|
|
|
23
27
|
[index]
|
|
24
|
-
INPUT_FILES
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
+
INPUT_FILES = ${data:TTL_DIR}/*.ttl.gz
|
|
29
|
+
MULTI_INPUT_JSON = [{ "cmd": "zcat {}", "graph": "http://sparql.uniprot.org/uniprot", "for-each": "${data:TTL_DIR}/uniprotkb_reviewed_*.ttl.gz" },
|
|
30
|
+
{ "cmd": "zcat {}", "graph": "http://sparql.uniprot.org/uniprot", "for-each": "${data:TTL_DIR}/uniprotkb_unreviewed_*.ttl.gz" },
|
|
31
|
+
{ "cmd": "zcat {}", "graph": "http://sparql.uniprot.org/uniparc", "for-each": "${data:TTL_DIR}/uniparc_*.ttl.gz" },
|
|
32
|
+
{ "cmd": "zcat {}", "graph": "http://sparql.uniprot.org/uniref", "for-each": "${data:TTL_DIR}/uniref*.ttl.gz" },
|
|
33
|
+
{ "cmd": "zcat {}", "graph": "http://sparql.uniprot.org/obsolete", "for-each": "${data:TTL_DIR}/uniprotkb_obsolete_*.ttl.gz" },
|
|
34
|
+
{ "cmd": "zcat ${data:TTL_DIR}/chebi.ttl.gz", "graph": "http://sparql.uniprot.org/chebi" },
|
|
35
|
+
{ "cmd": "zcat ${data:TTL_DIR}/citation_mapping.ttl.gz", "graph": "http://sparql.uniprot.org/citationmapping" },
|
|
36
|
+
{ "cmd": "zcat ${data:TTL_DIR}/citations.ttl.gz", "graph": "http://sparql.uniprot.org/citations" },
|
|
37
|
+
{ "cmd": "zcat ${data:TTL_DIR}/databases.ttl.gz", "graph": "http://sparql.uniprot.org/databases" },
|
|
38
|
+
{ "cmd": "zcat ${data:TTL_DIR}/diseases.ttl.gz", "graph": "http://sparql.uniprot.org/diseases" },
|
|
39
|
+
{ "cmd": "zcat ${data:TTL_DIR}/enzyme-hierarchy.ttl.gz", "graph": "http://sparql.uniprot.org/enzymes" },
|
|
40
|
+
{ "cmd": "zcat ${data:TTL_DIR}/enzyme.ttl.gz", "graph": "http://sparql.uniprot.org/enzymes" },
|
|
41
|
+
{ "cmd": "zcat ${data:TTL_DIR}/go-hierarchy.ttl.gz", "graph": "http://sparql.uniprot.org/go" },
|
|
42
|
+
{ "cmd": "zcat ${data:TTL_DIR}/go.ttl.gz", "graph": "http://sparql.uniprot.org/go" },
|
|
43
|
+
{ "cmd": "zcat ${data:TTL_DIR}/journals.ttl.gz", "graph": "http://sparql.uniprot.org/journal" },
|
|
44
|
+
{ "cmd": "zcat ${data:TTL_DIR}/keywords-hierarchy.ttl.gz", "graph": "http://sparql.uniprot.org/keywords" },
|
|
45
|
+
{ "cmd": "zcat ${data:TTL_DIR}/keywords.ttl.gz", "graph": "http://sparql.uniprot.org/keywords" },
|
|
46
|
+
{ "cmd": "zcat ${data:TTL_DIR}/locations-hierarchy.ttl.gz", "graph": "http://sparql.uniprot.org/locations" },
|
|
47
|
+
{ "cmd": "zcat ${data:TTL_DIR}/locations.ttl.gz", "graph": "http://sparql.uniprot.org/locations" },
|
|
48
|
+
{ "cmd": "zcat ${data:TTL_DIR}/pathways-hierarchy*.ttl.gz", "graph": "http://sparql.uniprot.org/pathways" },
|
|
49
|
+
{ "cmd": "zcat ${data:TTL_DIR}/pathways.ttl.gz", "graph": "http://sparql.uniprot.org/pathways" },
|
|
50
|
+
{ "cmd": "zcat ${data:TTL_DIR}/proteomes.ttl.gz", "graph": "http://sparql.uniprot.org/proteomes" },
|
|
51
|
+
{ "cmd": "zcat ${data:TTL_DIR}/taxonomy-hierarchy.ttl.gz", "graph": "http://sparql.uniprot.org/taxonomy" },
|
|
52
|
+
{ "cmd": "zcat ${data:TTL_DIR}/taxonomy.ttl.gz", "graph": "http://sparql.uniprot.org/taxonomy" },
|
|
53
|
+
{ "cmd": "zcat ${data:TTL_DIR}/tissues.ttl.gz", "graph": "http://sparql.uniprot.org/tissues" },
|
|
54
|
+
{ "cmd": "zcat ${data:TTL_DIR}/rhea.ttl.gz", "graph": "https://sparql.rhea-db.org/rhea" },
|
|
55
|
+
{ "cmd": "zcat ${data:TTL_DIR}/examples_uniprot.ttl.gz", "graph": "http://sparql.uniprot.org/.well-known/sparql-examples" },
|
|
56
|
+
{ "cmd": "zcat ${data:TTL_DIR}/core.ttl.gz", "graph": "http://purl.uniprot.org/core" },
|
|
57
|
+
{ "cmd": "zcat ${data:TTL_DIR}/void.ttl.gz", "graph": "http://rdfs.org/ns/void" }]
|
|
58
|
+
SETTINGS_JSON = { "languages-internal": [], "prefixes-external": [""], "locale": { "language": "en", "country": "US", "ignore-punctuation": true }, "ascii-prefixes-only": true, "num-triples-per-batch": 25000000 }
|
|
59
|
+
STXXL_MEMORY = 60G
|
|
28
60
|
|
|
29
61
|
[server]
|
|
30
62
|
PORT = 7018
|
|
@@ -64,7 +64,7 @@ class AddTextIndexCommand(QleverCommand):
|
|
|
64
64
|
# Show the command line.
|
|
65
65
|
self.show(add_text_index_cmd, only_show=args.show)
|
|
66
66
|
if args.show:
|
|
67
|
-
return
|
|
67
|
+
return True
|
|
68
68
|
|
|
69
69
|
# When running natively, check if the binary exists and works.
|
|
70
70
|
if args.system == "native":
|
|
@@ -74,6 +74,7 @@ class AddTextIndexCommand(QleverCommand):
|
|
|
74
74
|
log.error(f"Running \"{args.index_binary}\" failed ({e}), "
|
|
75
75
|
f"set `--index-binary` to a different binary or "
|
|
76
76
|
f"use `--container_system`")
|
|
77
|
+
return False
|
|
77
78
|
|
|
78
79
|
# Check if text index files already exist.
|
|
79
80
|
existing_text_index_files = get_existing_index_files(
|
qlever/commands/cache_stats.py
CHANGED
qlever/commands/clear_cache.py
CHANGED
|
@@ -48,7 +48,7 @@ class ClearCacheCommand(QleverCommand):
|
|
|
48
48
|
f"\"{args.access_token}\"")
|
|
49
49
|
self.show(clear_cache_cmd, only_show=args.show)
|
|
50
50
|
if args.show:
|
|
51
|
-
return
|
|
51
|
+
return True
|
|
52
52
|
|
|
53
53
|
# Execute the command.
|
|
54
54
|
try:
|
|
@@ -76,5 +76,7 @@ class ClearCacheCommand(QleverCommand):
|
|
|
76
76
|
# Show cache stats.
|
|
77
77
|
log.info("")
|
|
78
78
|
args.detailed = False
|
|
79
|
-
CacheStatsCommand().execute(args)
|
|
79
|
+
if not CacheStatsCommand().execute(args):
|
|
80
|
+
log.error("Clearing the cache was successful, but showing the "
|
|
81
|
+
"cache stats failed {e}")
|
|
80
82
|
return True
|