qlever 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

qlever/Qleverfiles/Qleverfile.yago-4

@@ -1,41 +1,37 @@
 # Qleverfile for YAGO 4, use with https://github.com/ad-freiburg/qlever-control
 #
-# qlever get-data  # downloads eight nt.gz file of size ~60 GB (as of 12.03.2020)
-# qlever index     # takes ~4.5 hours and ~10 GB RAM (on an AMD Ryzen 9 5900X)
-# qlever start     # starts the server (takes around one minute)
+# qlever get-data  # downloads 8 nt.gz file of size ~60 GB (as of 12.03.2020)
+# qlever index     # takes ~4 hours and ~10 GB RAM (on an AMD Ryzen 9 5900X)
+# qlever start     # starts the server
 
-# Indexer settings
-DB                = yago-4
-RDF_FILES         = "yago-wd-*.nt.gz"
-CAT_FILES         = "zcat ${RDF_FILES}"
-WITH_TEXT_INDEX   = false
-STXXL_MEMORY      = 10G
-SETTINGS_JSON     = '{ "languages-internal": ["en"], "locale": { "language": "en", "country": "US", "ignore-punctuation": true }, "ascii-prefixes-only": true, "num-triples-per-batch": 5000000 }'
+# NOTE concerning GET_DATA_CMD: The triples from wd-annotated-facts are
+# contained in wd-facts. The "full types" are the YAGO types, the "simple
+# types" are the schema.org types. They don't interfere with each other because
+# they have distinct prefixes.
 
-# Get the data
-#
-# NOTE: The triples from wd-annotated-facts are contained in wd-facts. The
-# "full types" are the YAGO types, the "simple types" are the schema.org types.
-# They don't interfere with each other because they have distinct prefixes.
-GET_DATA_CMD      = "wget -nc https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-class.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-facts.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-full-types.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-labels.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-sameAs.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-schema.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-shapes.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-simple-types.nt.gz"
+[data]
+NAME         = yago-4
+GET_DATA_CMD = curl --location --continue-at - --remote-name-all https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-class.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-facts.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-full-types.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-labels.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-sameAs.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-schema.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-shapes.nt.gz https://yago-knowledge.org/data/yago4/full/2020-02-24/yago-wd-simple-types.nt.gz
 INDEX_DESCRIPTION = "Full dump from https://yago-knowledge.org/downloads/yago-4, version 12.03.2020"
 
-# Server settings
-HOSTNAME                    = $(hostname -f)
-SERVER_PORT                 = 9004
+[index]
+FILE_NAMES      = yago-wd-*.nt.gz
+CAT_FILES       = zcat ${FILE_NAMES}
+WITH_TEXT_INDEX = false
+STXXL_MEMORY    = 10G
+SETTINGS_JSON   = { "languages-internal": ["en"], "locale": { "language": "en", "country": "US", "ignore-punctuation": true }, "ascii-prefixes-only": false, "num-triples-per-batch": 5000000 }
+
+[server]
+PORT                        = 9004
 ACCESS_TOKEN                = ${DB}_2347348732
 MEMORY_FOR_QUERIES          = 30G
 CACHE_MAX_SIZE              = 20G
 CACHE_MAX_SIZE_SINGLE_ENTRY = 5G
-CACHE_MAX_NUM_ENTRIES       = 100
 
-# QLever binaries
-QLEVER_BIN_DIR          = /local/data/qlever/qlever-code/build
-USE_DOCKER              = true
-QLEVER_DOCKER_IMAGE     = adfreiburg/qlever
-QLEVER_DOCKER_CONTAINER = qlever.${DB}
+[docker]
+USE_DOCKER = true
+IMAGE      = adfreiburg/qlever
 
-# QLever UI
-QLEVERUI_PORT   = 7000
-QLEVERUI_DIR    = qlever-ui
-QLEVERUI_CONFIG = yago-4
+[ui]
+PORT   = 7000
+CONFIG = yago-4

qlever/__init__.py

@@ -146,6 +146,11 @@ class Actions:
             "general": {
                 "log_level": "info",
                 "pid": "0",
+                "example_queries_url": (f"https://qlever.cs.uni-freiburg.de/"
+                                        f"api/examples/"
+                                        f"{self.config['ui']['config']}"),
+                "example_queries_limit": "10",
+                "example_queries_send": "0",
             },
             "index": {
                 "binary": "IndexBuilderMain",
@@ -1016,6 +1021,59 @@ class Actions:
             raise ActionException(f"Failed to get cache stats and settings: "
                                   f"{e}")
 
+    @track_action_rank
+    def action_clear_cache(self, only_show=False):
+        """
+        Action that clears the cache (unpinned entries only).
+        """
+
+        # Construct the curl command.
+        clear_cache_cmd = (f"curl -s {self.config['server']['url']} "
+                           f"--data-urlencode \"cmd=clear-cache\"")
+
+        # Show it.
+        self.show(clear_cache_cmd, only_show)
+        if only_show:
+            return
+
+        # Execute it.
+        try:
+            subprocess.run(clear_cache_cmd, shell=True,
+                           stdout=subprocess.DEVNULL)
+            print("Cache cleared (only unpinned entries)")
+            print()
+            self.action_cache_stats_and_settings(only_show)
+        except Exception as e:
+            raise ActionException(f"Failed to clear the cache: {e}")
+
+    @track_action_rank
+    def action_clear_cache_complete(self, only_show=False):
+        """
+        Action that clears the cache completely (both pinned and unpinned
+        entries).
+        """
+
+        # Construct the curl command.
+        access_token = self.config['server']['access_token']
+        clear_cache_cmd = (f"curl -s {self.config['server']['url']} "
+                           f"--data-urlencode \"cmd=clear-cache-complete\" "
+                           f"--data-urlencode \"access-token={access_token}\"")
+
+        # Show it.
+        self.show(clear_cache_cmd, only_show)
+        if only_show:
+            return
+
+        # Execute it.
+        try:
+            subprocess.run(clear_cache_cmd, shell=True,
+                           stdout=subprocess.DEVNULL)
+            print("Cache cleared (both pinned and unpinned entries)")
+            print()
+            self.action_cache_stats_and_settings(only_show)
+        except Exception as e:
+            raise ActionException(f"Failed to clear the cache: {e}")
+
     @track_action_rank
     def action_autocompletion_warmup(self, only_show=False):
         """
@@ -1064,6 +1122,8 @@ class Actions:
                        f"--data-urlencode access-token={access_token} "
                        f"--data-urlencode pinresult=true "
                        f"--data-urlencode send=0")
+            clear_cache_cmd = (f"curl -s {self.config['server']['url']} "
+                               f"--data-urlencode \"cmd=clear-cache\"")
             log.info(pin_cmd)
             # Launch query and show the `resultsize` of the JSON response.
             try:
@@ -1073,6 +1133,9 @@ class Actions:
                 if "exception" in json_result:
                     raise Exception(json_result["exception"])
                 log.info(f"Result size: {json_result['resultsize']:,}")
+                log.info(clear_cache_cmd)
+                subprocess.check_output(clear_cache_cmd, shell=True,
+                                        stderr=subprocess.DEVNULL)
             except Exception as e:
                 log.error(f"Query failed: {e}")
 
@@ -1083,14 +1146,18 @@ class Actions:
         """
 
         # Construct curl command to obtain the example queries.
-        config_name = self.config["ui"]["config"]
-        examples_url = f"{self.config['ui']['url']}/examples/{config_name}"
-        curl_cmd = f"curl -s {examples_url}"
+        config_general = self.config["general"]
+        example_queries_url = config_general["example_queries_url"]
+        example_queries_limit = int(config_general["example_queries_limit"])
+        example_queries_send = int(config_general["example_queries_send"])
+        curl_cmd = f"curl -s {example_queries_url}"
 
         # Show what the action does.
         self.show(f"Launch example queries obtained via: {curl_cmd}\n"
                   f"SPARQL endpoint: {self.config['server']['url']}\n"
-                  f"Clearing the cache before each query + using send=0",
+                  f"Clearing the cache before each query\n"
+                  f"Using send={example_queries_send} and limit="
+                  f"{example_queries_limit}",
                   only_show)
         if only_show:
             return
@@ -1114,7 +1181,7 @@ class Actions:
             query_cmd = (f"curl -s {self.config['server']['url']} "
                          f"-H \"Accept: application/qlever-results+json\" "
                          f"--data-urlencode query={shlex.quote(query)} "
-                         f"--data-urlencode send=0")
+                         f"--data-urlencode send={example_queries_send}")
             try:
                 subprocess.run(clear_cache_cmd, shell=True,
                                stdout=subprocess.DEVNULL,
@@ -1126,28 +1193,33 @@ class Actions:
                 if "exception" in json_result:
                     raise Exception(json_result["exception"])
                 result_size = int(json_result["resultsize"])
-                result_string = "{:,}".format(result_size)
+                result_string = f"{result_size:>14,}"
             except Exception as e:
                 time_seconds = 0.0
                 result_size = 0
-                result_string = f"{RED}{e}{NORMAL}"
+                result_string = (f"{RED}        FAILED{NORMAL}"
+                                 f" {RED}({e}){NORMAL}")
 
             # Print description, time, result in tabular form.
             log.debug(query)
             log.info(f"{description:<60}  {time_seconds:6.2f} s  "
-                     f"{result_string:>10}")
+                     f"{result_string}")
             count += 1
             total_time_seconds += time_seconds
             total_result_size += result_size
-            if count == 10:
+            if count == example_queries_limit:
                 break
 
         # Print total time.
         log.info("")
-        description = (f"TOTAL for {count} "
+        description = (f"TOTAL   for {count} "
                        f"{'query' if count == 1 else 'queries'}")
         log.info(f"{description:<60}  {total_time_seconds:6.2f} s  "
-                 f"{total_result_size:>10,}")
+                 f"{total_result_size:>14,}")
+        description = (f"AVERAGE for {count} "
+                       f"{'query' if count == 1 else 'queries'}")
+        log.info(f"{description:<60}  {total_time_seconds / count:6.2f} s  "
+                 f"{round(total_result_size / count):>14,}")
 
     @track_action_rank
     def action_memory_profile(self, only_show=False):

{qlever-0.3.0.dist-info → qlever-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qlever
-Version: 0.3.0
+Version: 0.3.1
 Summary: Script for using the QLever SPARQL engine.
 Author-email: Hannah Bast <bast@cs.uni-freiburg.de>
 License: Apache License

{qlever-0.3.0.dist-info → qlever-0.3.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-qlever/__init__.py,sha256=ax4oYkCN-Clf_h1kFs1lGnn7uLn-hIXsEId7ONcp1W8,58258
+qlever/__init__.py,sha256=55Dxjww9I-7XCHrjvzku79lPSdg6brRxvwwZUZg2c3Q,61344
 qlever/__main__.py,sha256=mmp__u0a-Dztg3BUlJ4LL98OVo_j1MLiDCKewKlwfMM,38
 qlever/Qleverfiles/Qleverfile.dblp,sha256=yuSDOhC3ypsXt2rAd4yjSzz0OQsZ2i6Pcq0Qh-stlBM,1199
 qlever/Qleverfiles/Qleverfile.dblp-plus,sha256=E9bW49ztinBlW7azcT51QPW2jQWBPk29SDZ31RuIOO4,1389
@@ -6,7 +6,6 @@ qlever/Qleverfiles/Qleverfile.default,sha256=1roS4FHGiSA7iXBcX2DH3zwE5Ozht6-A551
 qlever/Qleverfiles/Qleverfile.dnb,sha256=TNyLFuN_VJeNPlbHKAqLbSw1dgqUTBS5HbYJOJohRsI,1547
 qlever/Qleverfiles/Qleverfile.fbeasy,sha256=N_ctx6Ae7ISbrOtBthOso_5_SCZuMz7TxsaiTzFd-mc,1032
 qlever/Qleverfiles/Qleverfile.freebase,sha256=JbjW5JTpiCvaGVqjCq2CawPVNWStwwH5sptdK85Y2KM,1150
-qlever/Qleverfiles/Qleverfile.gnd,sha256=9TtYaaztetZ-V0xDE7FejkTFaH466WWmfI-R-IJkWE4,1049
 qlever/Qleverfiles/Qleverfile.imdb,sha256=o5LfmZZKZH7Ycu0qaxQF1L_4KVLhd5R2DIYw3bCJ7BM,1654
 qlever/Qleverfiles/Qleverfile.olympics,sha256=09yDlhbrr0kqImLQKqIe2NkS35aWGSJJUHyI1ollQBE,1055
 qlever/Qleverfiles/Qleverfile.osm-country,sha256=zYu4734aGeqwL2xBc9A4qHchYdyHyv6L5ny1XL7bA-A,1959
@@ -14,10 +13,10 @@ qlever/Qleverfiles/Qleverfile.pubchem,sha256=Wk5adgo4tIHbCg57cWwoOqIBBSN878wqjEW
 qlever/Qleverfiles/Qleverfile.scientists,sha256=oFhzURcRFciA27GZ-ux_hsDe0esBLobWHC6h_Vf2xy8,1735
 qlever/Qleverfiles/Qleverfile.uniprot,sha256=q5Nkr3diZ5oXdT0N72ONBEHBFZQCKYmEChAsjO4hnwE,2249
 qlever/Qleverfiles/Qleverfile.wikidata,sha256=qTb9UjiAIZVIxCIXVEvaNNMacN0c0x2cPWXChPQcgmw,1346
-qlever/Qleverfiles/Qleverfile.yago-4,sha256=iX6oUloi_opZYetfNJLmGoHd47VmCKiuUAhpwYJu4QU,2233
-qlever-0.3.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-qlever-0.3.0.dist-info/METADATA,sha256=9dOTtA06ifqEK8IaOqprN7zVx5wS-Shyj_PCqs7WEjw,16151
-qlever-0.3.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-qlever-0.3.0.dist-info/entry_points.txt,sha256=pWFGQafjVuvwADcae4_8QVFeqPbTynj5RabMsod5VOY,39
-qlever-0.3.0.dist-info/top_level.txt,sha256=kd3zsYqiFd0--Czh5XTVkfEq6XR-XgRFW35X0v0GT-c,7
-qlever-0.3.0.dist-info/RECORD,,
+qlever/Qleverfiles/Qleverfile.yago-4,sha256=ghAJRf6VhJfTOh8grdIekeFgmroaCVMTdRhKr7vYVes,1944
+qlever-0.3.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+qlever-0.3.1.dist-info/METADATA,sha256=diUXBn4bdAvOX3k3QEQYXfxQ748hdSM1FNCvB3l6JWg,16151
+qlever-0.3.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+qlever-0.3.1.dist-info/entry_points.txt,sha256=pWFGQafjVuvwADcae4_8QVFeqPbTynj5RabMsod5VOY,39
+qlever-0.3.1.dist-info/top_level.txt,sha256=kd3zsYqiFd0--Czh5XTVkfEq6XR-XgRFW35X0v0GT-c,7
+qlever-0.3.1.dist-info/RECORD,,

qlever/Qleverfiles/Qleverfile.gnd

@@ -1,31 +0,0 @@
-# Qleverfile for Olympics, use with https://github.com/ad-freiburg/qlever-control
-#
-# qlever get-data  # downloads .zip file of size 13 MB, uncompressed to 323 MB 
-# qlever index     # takes ~10 seconds and ~1 GB RAM (on an AMD Ryzen 9 5900X)
-# qlever start     # starts the server (instant)
-
-[data]
-NAME              = gnd
-BASE_URL          = https://github.com/wallscope/olympics-rdf
-GET_DATA_CMD      = curl -O https://data.dnb.de/opendata/authorities-gnd_lds.nt.gz
-INDEX_DESCRIPTION = GND authority data from https://data.dnb.de/opendata/authorities-gnd_lds.nt.gz
-# TEXT_DESCRIPTION  = All literals, search with FILTER CONTAINS(?var, "...")
-
-[index]
-FILE_NAMES    = authorities-gnd_lds.nt.gz
-CAT_FILES     = zcat ${FILE_NAMES}
-SETTINGS_JSON = { "ascii-prefixes-only": true, "num-triples-per-batch": 1000000 }
-
-[server]
-PORT                  = 7035
-ACCESS_TOKEN          = ${data:NAME}_7643543846
-MEMORY_FOR_QUERIES_GB = 5
-CACHE_MAX_SIZE_GB     = 2
-
-[docker]
-USE_DOCKER = true
-IMAGE      = adfreiburg/qlever
-
-[ui]
-PORT   = 7000
-CONFIG = olympics