qlever 0.2.13__py3-none-any.whl → 0.2.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of qlever might be problematic. Click here for more details.
- qlever/Qleverfiles/Qleverfile.dblp +1 -1
- qlever/Qleverfiles/Qleverfile.dblp-plus +1 -1
- qlever/Qleverfiles/Qleverfile.dnb +1 -1
- qlever/Qleverfiles/Qleverfile.fbeasy +1 -1
- qlever/Qleverfiles/Qleverfile.freebase +1 -1
- qlever/Qleverfiles/Qleverfile.gnd +1 -1
- qlever/Qleverfiles/Qleverfile.imdb +1 -1
- qlever/Qleverfiles/Qleverfile.olympics +1 -1
- qlever/Qleverfiles/Qleverfile.pubchem +1 -1
- qlever/__main__.py +1 -3
- {qlever-0.2.13.dist-info → qlever-0.2.14.dist-info}/METADATA +1 -1
- qlever-0.2.14.dist-info/RECORD +23 -0
- build/lib/build/lib/qlever/__init__.py +0 -1383
- build/lib/build/lib/qlever/__main__.py +0 -4
- build/lib/qlever/__init__.py +0 -1383
- build/lib/qlever/__main__.py +0 -4
- build/lib/src/qlever/__init__.py +0 -1383
- build/lib/src/qlever/__main__.py +0 -4
- qlever-0.2.13.dist-info/RECORD +0 -31
- src/qlever/__init__.py +0 -1383
- src/qlever/__main__.py +0 -4
- {qlever-0.2.13.dist-info → qlever-0.2.14.dist-info}/LICENSE +0 -0
- {qlever-0.2.13.dist-info → qlever-0.2.14.dist-info}/WHEEL +0 -0
- {qlever-0.2.13.dist-info → qlever-0.2.14.dist-info}/entry_points.txt +0 -0
- {qlever-0.2.13.dist-info → qlever-0.2.14.dist-info}/top_level.txt +0 -0
build/lib/src/qlever/__init__.py
DELETED
|
@@ -1,1383 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# PYTHON_ARGCOMPLETE_OK
|
|
3
|
-
|
|
4
|
-
# This is the `qlever` script (new version, written in Python). It serves as a
|
|
5
|
-
# convenient command-line tool for all things QLever. See the `README.md` file
|
|
6
|
-
# for how to use it.
|
|
7
|
-
|
|
8
|
-
from configparser import ConfigParser, ExtendedInterpolation
|
|
9
|
-
from datetime import datetime, date
|
|
10
|
-
import os
|
|
11
|
-
import glob
|
|
12
|
-
import inspect
|
|
13
|
-
import json
|
|
14
|
-
import logging
|
|
15
|
-
import psutil
|
|
16
|
-
import re
|
|
17
|
-
import shlex
|
|
18
|
-
import shutil
|
|
19
|
-
import socket
|
|
20
|
-
import subprocess
|
|
21
|
-
import sys
|
|
22
|
-
import time
|
|
23
|
-
from termcolor import colored
|
|
24
|
-
import traceback
|
|
25
|
-
|
|
26
|
-
BLUE = "\033[34m"
|
|
27
|
-
RED = "\033[31m"
|
|
28
|
-
BOLD = "\033[1m"
|
|
29
|
-
NORMAL = "\033[0m"
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
# Custom formatter for log messages.
|
|
33
|
-
class CustomFormatter(logging.Formatter):
|
|
34
|
-
def format(self, record):
|
|
35
|
-
message = record.getMessage()
|
|
36
|
-
if record.levelno == logging.DEBUG:
|
|
37
|
-
return colored(message, "magenta")
|
|
38
|
-
elif record.levelno == logging.WARNING:
|
|
39
|
-
return colored(message, "yellow")
|
|
40
|
-
elif record.levelno in [logging.CRITICAL, logging.ERROR]:
|
|
41
|
-
return colored(message, "red")
|
|
42
|
-
else:
|
|
43
|
-
return message
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
# Custom logger.
|
|
47
|
-
log = logging.getLogger("qlever")
|
|
48
|
-
log.setLevel(logging.INFO)
|
|
49
|
-
handler = logging.StreamHandler()
|
|
50
|
-
handler.setFormatter(CustomFormatter())
|
|
51
|
-
log.addHandler(handler)
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
# Helper function for tracking the order of the actions in class `Actions`.
|
|
55
|
-
def track_action_rank(method):
|
|
56
|
-
method.rank = track_action_rank.counter
|
|
57
|
-
track_action_rank.counter += 1
|
|
58
|
-
return method
|
|
59
|
-
track_action_rank.counter = 0 # noqa: E305
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
# Abort the script.
|
|
63
|
-
def abort_script(error_code=1):
|
|
64
|
-
log.info("")
|
|
65
|
-
sys.exit(error_code)
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
# Show the available config names.
|
|
69
|
-
def show_available_config_names():
|
|
70
|
-
# Get available config names from the Qleverfiles directory (which should
|
|
71
|
-
# be in the same directory as this script).
|
|
72
|
-
script_dir = os.path.dirname(__file__)
|
|
73
|
-
try:
|
|
74
|
-
qleverfiles_dir = os.path.join(script_dir, "Qleverfiles")
|
|
75
|
-
config_names = [qleverfile_name.split(".")[1] for
|
|
76
|
-
qleverfile_name in os.listdir(qleverfiles_dir)]
|
|
77
|
-
if not config_names:
|
|
78
|
-
raise Exception(f"Directory \"{qleverfiles_dir}\" exists, but "
|
|
79
|
-
f"contains no Qleverfiles")
|
|
80
|
-
except Exception as e:
|
|
81
|
-
log.error(f"Could not find any Qleverfiles in \"{qleverfiles_dir}\" "
|
|
82
|
-
f"({e})")
|
|
83
|
-
log.info("")
|
|
84
|
-
log.info("Check that you have fully downloaded or cloned "
|
|
85
|
-
"https://github.com/ad-freiburg/qlever-control, and "
|
|
86
|
-
"not just the script itself")
|
|
87
|
-
abort_script()
|
|
88
|
-
# Show available config names.
|
|
89
|
-
log.info(f"Available config names are: {', '.join(sorted(config_names))}")
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
# Show the available action names.
|
|
93
|
-
def show_available_action_names():
|
|
94
|
-
log.info("The qlever script takes a sequence of action names as "
|
|
95
|
-
"arguments, for example:")
|
|
96
|
-
log.info("")
|
|
97
|
-
log.info(f"{BLUE}qlever get-data index restart example-query ui {NORMAL}")
|
|
98
|
-
log.info("")
|
|
99
|
-
log.info(f"Available action names are: {', '.join(action_names)}")
|
|
100
|
-
log.info("")
|
|
101
|
-
log.info("To get autocompletion for these, run the following or "
|
|
102
|
-
"add it to your `.bashrc`:")
|
|
103
|
-
log.info("")
|
|
104
|
-
log.info(f"{BLUE}eval \"$(qlever setup-autocompletion)\"{NORMAL}")
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
# We want to distinguish between exception that we throw intentionally and all
|
|
108
|
-
# others.
|
|
109
|
-
class ActionException(Exception):
|
|
110
|
-
pass
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
# This class contains all the action :-)
|
|
114
|
-
class Actions:
|
|
115
|
-
|
|
116
|
-
def __init__(self):
|
|
117
|
-
self.config = ConfigParser(interpolation=ExtendedInterpolation())
|
|
118
|
-
# Check if the Qleverfile exists.
|
|
119
|
-
if not os.path.isfile("Qleverfile"):
|
|
120
|
-
log.setLevel(logging.INFO)
|
|
121
|
-
log.info("")
|
|
122
|
-
log.error("The qlever script needs a \"Qleverfile\" "
|
|
123
|
-
"in the current directory, but I could not find it")
|
|
124
|
-
log.info("")
|
|
125
|
-
log.info("Run `qlever setup-config <config name>` to create a "
|
|
126
|
-
"pre-filled Qleverfile")
|
|
127
|
-
log.info("")
|
|
128
|
-
show_available_config_names()
|
|
129
|
-
abort_script()
|
|
130
|
-
files_read = self.config.read("Qleverfile")
|
|
131
|
-
if not files_read:
|
|
132
|
-
log.error("ConfigParser could not read \"Qleverfile\"")
|
|
133
|
-
abort_script()
|
|
134
|
-
self.name = self.config['data']['name']
|
|
135
|
-
self.yes_values = ["1", "true", "yes"]
|
|
136
|
-
|
|
137
|
-
# Defaults for [server] that carry over from [index].
|
|
138
|
-
for option in ["with_text_index", "only_pso_and_pos_permutations",
|
|
139
|
-
"use_patterns"]:
|
|
140
|
-
if option in self.config['index'] and \
|
|
141
|
-
option not in self.config['server']:
|
|
142
|
-
self.config['server'][option] = \
|
|
143
|
-
self.config['index'][option]
|
|
144
|
-
|
|
145
|
-
# Default values for options that are not mandatory in the Qleverfile.
|
|
146
|
-
defaults = {
|
|
147
|
-
"general": {
|
|
148
|
-
"log_level": "info",
|
|
149
|
-
"pid": "0",
|
|
150
|
-
},
|
|
151
|
-
"index": {
|
|
152
|
-
"binary": "IndexBuilderMain",
|
|
153
|
-
"with_text_index": "false",
|
|
154
|
-
"only_pso_and_pos_permutations": "false",
|
|
155
|
-
"use_patterns": "true",
|
|
156
|
-
},
|
|
157
|
-
"server": {
|
|
158
|
-
"port": "7000",
|
|
159
|
-
"binary": "ServerMain",
|
|
160
|
-
"num_threads": "8",
|
|
161
|
-
"cache_max_size_gb": "5",
|
|
162
|
-
"cache_max_size_gb_single_entry": "1",
|
|
163
|
-
"cache_max_num_entries": "100",
|
|
164
|
-
"with_text_index": "false",
|
|
165
|
-
"only_pso_and_pos_permutations": "false",
|
|
166
|
-
"use_patterns": "true",
|
|
167
|
-
"url": f"http://localhost:{self.config['server']['port']}",
|
|
168
|
-
},
|
|
169
|
-
"docker": {
|
|
170
|
-
"image": "adfreiburg/qlever",
|
|
171
|
-
"container_server": f"qlever.server.{self.name}",
|
|
172
|
-
"container_indexer": f"qlever.indexer.{self.name}",
|
|
173
|
-
},
|
|
174
|
-
"ui": {
|
|
175
|
-
"port": "7000",
|
|
176
|
-
"image": "adfreiburg/qlever-ui",
|
|
177
|
-
"container": "qlever-ui",
|
|
178
|
-
"url": "https://qlever.cs.uni-freiburg.de/api",
|
|
179
|
-
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
for section in defaults:
|
|
183
|
-
# If the section does not exist, create it.
|
|
184
|
-
if not self.config.has_section(section):
|
|
185
|
-
self.config[section] = {}
|
|
186
|
-
# If an option does not exist, set it to the default value.
|
|
187
|
-
for option in defaults[section]:
|
|
188
|
-
if not self.config[section].get(option):
|
|
189
|
-
self.config[section][option] = defaults[section][option]
|
|
190
|
-
|
|
191
|
-
# If the log level was not explicitly set by the first command-line
|
|
192
|
-
# argument (see below), set it according to the Qleverfile.
|
|
193
|
-
if log.level == logging.NOTSET:
|
|
194
|
-
log_level = self.config['general']['log_level'].upper()
|
|
195
|
-
try:
|
|
196
|
-
log.setLevel(getattr(logging, log_level))
|
|
197
|
-
except AttributeError:
|
|
198
|
-
log.error(f"Invalid log level: \"{log_level}\"")
|
|
199
|
-
abort_script()
|
|
200
|
-
|
|
201
|
-
# Show some information (for testing purposes only).
|
|
202
|
-
log.debug(f"Parsed Qleverfile, sections are: "
|
|
203
|
-
f"{', '.join(self.config.sections())}")
|
|
204
|
-
|
|
205
|
-
# Check specifics of the installation.
|
|
206
|
-
self.check_installation()
|
|
207
|
-
|
|
208
|
-
def check_installation(self):
|
|
209
|
-
"""
|
|
210
|
-
Helper function that checks particulars of the installation and
|
|
211
|
-
remembers them so that all actions execute without errors.
|
|
212
|
-
"""
|
|
213
|
-
|
|
214
|
-
# Handle the case Systems like macOS do not allow
|
|
215
|
-
# psutil.net_connections().
|
|
216
|
-
try:
|
|
217
|
-
psutil.net_connections()
|
|
218
|
-
self.net_connections_enabled = True
|
|
219
|
-
except Exception as e:
|
|
220
|
-
self.net_connections_enabled = False
|
|
221
|
-
log.debug(f"Note: psutil.net_connections() failed ({e}),"
|
|
222
|
-
f" will not scan network connections for action"
|
|
223
|
-
f" \"start\"")
|
|
224
|
-
|
|
225
|
-
# Check whether docker is installed and works (on MacOS 12, docker
|
|
226
|
-
# hangs when installed without GUI, hence the timeout).
|
|
227
|
-
try:
|
|
228
|
-
completed_process = subprocess.run(
|
|
229
|
-
["docker", "info"], timeout=0.5,
|
|
230
|
-
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
231
|
-
if completed_process.returncode != 0:
|
|
232
|
-
raise Exception("docker info failed")
|
|
233
|
-
self.docker_enabled = True
|
|
234
|
-
except Exception:
|
|
235
|
-
self.docker_enabled = False
|
|
236
|
-
print("Note: `docker info` failed, therefore"
|
|
237
|
-
" docker.USE_DOCKER=true not supported")
|
|
238
|
-
|
|
239
|
-
def set_config(self, section, option, value):
|
|
240
|
-
"""
|
|
241
|
-
Helper function that sets a value in the config file (and throws an
|
|
242
|
-
exceptionon if the section or option does not exist).
|
|
243
|
-
"""
|
|
244
|
-
|
|
245
|
-
if not self.config.has_section(section):
|
|
246
|
-
log.error(f"Section [{section}] does not exist in Qleverfile")
|
|
247
|
-
abort_script()
|
|
248
|
-
if not self.config.has_option(section, option):
|
|
249
|
-
log.error(f"Option {option.upper()} does not exist in section "
|
|
250
|
-
f"[{section}] in Qleverfile")
|
|
251
|
-
abort_script()
|
|
252
|
-
self.config[section][option] = value
|
|
253
|
-
|
|
254
|
-
def get_total_file_size(self, paths):
|
|
255
|
-
"""
|
|
256
|
-
Helper function that gets the total size of all files in the given
|
|
257
|
-
paths in GB.
|
|
258
|
-
"""
|
|
259
|
-
|
|
260
|
-
total_size = 0
|
|
261
|
-
for path in paths:
|
|
262
|
-
for file in glob.glob(path):
|
|
263
|
-
total_size += os.path.getsize(file)
|
|
264
|
-
return total_size / 1e9
|
|
265
|
-
|
|
266
|
-
def alive_check(self, port):
|
|
267
|
-
"""
|
|
268
|
-
Helper function that checks if a QLever server is running on the given
|
|
269
|
-
port.
|
|
270
|
-
"""
|
|
271
|
-
|
|
272
|
-
message = "from the qlever script".replace(" ", "%20")
|
|
273
|
-
curl_cmd = f"curl -s http://localhost:{port}/ping?msg={message}"
|
|
274
|
-
exit_code = subprocess.call(curl_cmd, shell=True,
|
|
275
|
-
stdout=subprocess.DEVNULL,
|
|
276
|
-
stderr=subprocess.DEVNULL)
|
|
277
|
-
return exit_code == 0
|
|
278
|
-
|
|
279
|
-
def show_process_info(self, psutil_process,
|
|
280
|
-
cmdline_regex, show_heading=True):
|
|
281
|
-
"""
|
|
282
|
-
Helper function that shows information about a process if information
|
|
283
|
-
about the process can be retrieved and the command line matches the
|
|
284
|
-
given regex (in which case the function returns `True`). The heading is
|
|
285
|
-
only shown if `show_heading` is `True` and the function returns `True`.
|
|
286
|
-
"""
|
|
287
|
-
|
|
288
|
-
def show_table_line(pid, user, start_time, rss, cmdline):
|
|
289
|
-
log.info(f"{pid:<8} {user:<8} {start_time:>5} {rss:>5} {cmdline}")
|
|
290
|
-
try:
|
|
291
|
-
pinfo = psutil_process.as_dict(
|
|
292
|
-
attrs=['pid', 'username', 'create_time',
|
|
293
|
-
'memory_info', 'cmdline'])
|
|
294
|
-
cmdline = " ".join(pinfo['cmdline'])
|
|
295
|
-
if not re.search(cmdline_regex, cmdline):
|
|
296
|
-
return False
|
|
297
|
-
pid = pinfo['pid']
|
|
298
|
-
user = pinfo['username'] if pinfo['username'] else ""
|
|
299
|
-
start_time = datetime.fromtimestamp(pinfo['create_time'])
|
|
300
|
-
if start_time.date() == date.today():
|
|
301
|
-
start_time = start_time.strftime("%H:%M")
|
|
302
|
-
else:
|
|
303
|
-
start_time = start_time.strftime("%b%d")
|
|
304
|
-
rss = f"{pinfo['memory_info'].rss / 1e9:.0f}G"
|
|
305
|
-
if show_heading:
|
|
306
|
-
show_table_line("PID", "USER", "START", "RSS", "COMMAND")
|
|
307
|
-
show_table_line(pid, user, start_time, rss, cmdline)
|
|
308
|
-
return True
|
|
309
|
-
except Exception as e:
|
|
310
|
-
log.debug(f"Could not get process info: {e}")
|
|
311
|
-
return False
|
|
312
|
-
|
|
313
|
-
def show(self, action_description, only_show):
|
|
314
|
-
"""
|
|
315
|
-
Helper function that shows the command line or description of an
|
|
316
|
-
action, together with an explanation.
|
|
317
|
-
"""
|
|
318
|
-
|
|
319
|
-
log.info(f"{BLUE}{action_description}{NORMAL}")
|
|
320
|
-
log.info("")
|
|
321
|
-
if only_show:
|
|
322
|
-
log.info("You called \"qlever ... show\", therefore the action "
|
|
323
|
-
"is only shown, but not executed (omit the \"show\" to "
|
|
324
|
-
"execute it)")
|
|
325
|
-
|
|
326
|
-
@staticmethod
|
|
327
|
-
@track_action_rank
|
|
328
|
-
def action_setup_config(config_name="default"):
|
|
329
|
-
"""
|
|
330
|
-
Setup a pre-filled Qleverfile in the current directory.
|
|
331
|
-
"""
|
|
332
|
-
|
|
333
|
-
log.info(f"{BLUE}Creating a pre-filled Qleverfile{NORMAL}")
|
|
334
|
-
log.info("")
|
|
335
|
-
|
|
336
|
-
# If there is already a Qleverfile in the current directory, exit.
|
|
337
|
-
if os.path.isfile("Qleverfile"):
|
|
338
|
-
log.error("Qleverfile already exists in current directory")
|
|
339
|
-
log.info("")
|
|
340
|
-
log.info("If you want to create a new Qleverfile using "
|
|
341
|
-
"`qlever setup-config`, delete the existing Qleverfile "
|
|
342
|
-
"first")
|
|
343
|
-
abort_script()
|
|
344
|
-
|
|
345
|
-
# Get the directory of this script and copy the Qleverfile for `config`
|
|
346
|
-
# to the current directory.
|
|
347
|
-
script_dir = os.path.dirname(os.path.realpath(__file__))
|
|
348
|
-
qleverfile_path = os.path.join(script_dir,
|
|
349
|
-
f"Qleverfiles/Qleverfile.{config_name}")
|
|
350
|
-
if not os.path.isfile(qleverfile_path):
|
|
351
|
-
log.error(f"File \"{qleverfile_path}\" does not exist")
|
|
352
|
-
log.info("")
|
|
353
|
-
abort_script()
|
|
354
|
-
try:
|
|
355
|
-
shutil.copy(qleverfile_path, "Qleverfile")
|
|
356
|
-
except Exception as e:
|
|
357
|
-
log.error(f"Could not copy \"{qleverfile_path}\""
|
|
358
|
-
f" to current directory: {e}")
|
|
359
|
-
abort_script()
|
|
360
|
-
log.info(f"Created Qleverfile for config \"{config_name}\""
|
|
361
|
-
f" in current directory")
|
|
362
|
-
log.info("")
|
|
363
|
-
if config_name == "default":
|
|
364
|
-
log.info("Since this is the default Qleverfile, you need to "
|
|
365
|
-
"edit it before you can continue")
|
|
366
|
-
log.info("")
|
|
367
|
-
log.info("Afterwards, run `qlever` without arguments to see "
|
|
368
|
-
"which actions are available")
|
|
369
|
-
else:
|
|
370
|
-
log.info("If you are unsure how to continue, run `qlever`"
|
|
371
|
-
" without arguments to see the available actions")
|
|
372
|
-
log.info("")
|
|
373
|
-
|
|
374
|
-
@track_action_rank
|
|
375
|
-
def action_show_config(self, only_show=False):
|
|
376
|
-
"""
|
|
377
|
-
Action that shows the current configuration including the default
|
|
378
|
-
values for options that are not set explicitly in the Qleverfile.
|
|
379
|
-
"""
|
|
380
|
-
|
|
381
|
-
print(f"{BLUE}Showing the current configuration, including default"
|
|
382
|
-
f" values for options that are not set explicitly in the"
|
|
383
|
-
f" Qleverfile{NORMAL}")
|
|
384
|
-
for section in self.config.sections():
|
|
385
|
-
print()
|
|
386
|
-
print(f"[{section}]")
|
|
387
|
-
max_option_length = max([len(option) for option in
|
|
388
|
-
self.config[section]])
|
|
389
|
-
for option in self.config[section]:
|
|
390
|
-
print(f"{option.upper().ljust(max_option_length)} = "
|
|
391
|
-
f"{self.config[section][option]}")
|
|
392
|
-
|
|
393
|
-
print()
|
|
394
|
-
|
|
395
|
-
@track_action_rank
|
|
396
|
-
def action_get_data(self, only_show=False):
|
|
397
|
-
"""
|
|
398
|
-
Action that gets the data according to GET_DATA_CMD.
|
|
399
|
-
"""
|
|
400
|
-
|
|
401
|
-
# Construct the command line.
|
|
402
|
-
if not self.config['data']['get_data_cmd']:
|
|
403
|
-
log.error(f"{RED}No GET_DATA_CMD specified in Qleverfile")
|
|
404
|
-
return
|
|
405
|
-
cmdline = self.config['data']['get_data_cmd']
|
|
406
|
-
|
|
407
|
-
# Show it.
|
|
408
|
-
self.show(cmdline, only_show)
|
|
409
|
-
if only_show:
|
|
410
|
-
return
|
|
411
|
-
|
|
412
|
-
# Execute the command line.
|
|
413
|
-
os.system(cmdline)
|
|
414
|
-
total_file_size = self.get_total_file_size(
|
|
415
|
-
self.config['index']['file_names'].split())
|
|
416
|
-
print(f"Total file size: {total_file_size:.1f} GB")
|
|
417
|
-
|
|
418
|
-
@track_action_rank
|
|
419
|
-
def action_index(self, only_show=False):
|
|
420
|
-
"""
|
|
421
|
-
Action that builds a QLever index according to the settings in the
|
|
422
|
-
[index] section of the Qleverfile.
|
|
423
|
-
"""
|
|
424
|
-
|
|
425
|
-
# Construct the command line based on the config file.
|
|
426
|
-
index_config = self.config['index']
|
|
427
|
-
cmdline = (f"{index_config['cat_files']} | {index_config['binary']}"
|
|
428
|
-
f" -F ttl -f - -N"
|
|
429
|
-
f" -i {self.name}"
|
|
430
|
-
f" -s {self.name}.settings.json")
|
|
431
|
-
if index_config['only_pso_and_pos_permutations'] in self.yes_values:
|
|
432
|
-
cmdline += " --only-pso-and-pos-permutations --no-patterns"
|
|
433
|
-
if not index_config['use_patterns'] in self.yes_values:
|
|
434
|
-
cmdline += " --no-patterns"
|
|
435
|
-
if index_config['with_text_index'] in \
|
|
436
|
-
["from_text_records", "from_text_records_and_literals"]:
|
|
437
|
-
cmdline += (f" -w {self.name}.wordsfile.tsv"
|
|
438
|
-
f" -d {self.name}.docsfile.tsv")
|
|
439
|
-
if index_config['with_text_index'] in \
|
|
440
|
-
["from_literals", "from_text_records_and_literals"]:
|
|
441
|
-
cmdline += " --text-words-from-literals"
|
|
442
|
-
if 'stxxl_memory_gb' in index_config:
|
|
443
|
-
cmdline += f" --stxxl-memory-gb {index_config['stxxl_memory_gb']}"
|
|
444
|
-
cmdline += f" | tee {self.name}.index-log.txt"
|
|
445
|
-
|
|
446
|
-
# If the total file size is larger than 10 GB, set ulimit (such that a
|
|
447
|
-
# large number of open files is allowed).
|
|
448
|
-
total_file_size = self.get_total_file_size(
|
|
449
|
-
self.config['index']['file_names'].split())
|
|
450
|
-
if total_file_size > 10:
|
|
451
|
-
cmdline = f"ulimit -Sn 1048576; {cmdline}"
|
|
452
|
-
|
|
453
|
-
# If we are using Docker, run the command in a Docker container.
|
|
454
|
-
# Here is how the shell script does it:
|
|
455
|
-
if self.config['docker']['use_docker'] in self.yes_values:
|
|
456
|
-
docker_config = self.config['docker']
|
|
457
|
-
cmdline = (f"docker run -it --rm -u $(id -u):$(id -g)"
|
|
458
|
-
f" -v /etc/localtime:/etc/localtime:ro"
|
|
459
|
-
f" -v $(pwd):/index -w /index"
|
|
460
|
-
f" --entrypoint bash"
|
|
461
|
-
f" --name {docker_config['container_indexer']}"
|
|
462
|
-
f" {docker_config['image']}"
|
|
463
|
-
f" -c {shlex.quote(cmdline)}")
|
|
464
|
-
|
|
465
|
-
# Show the command line.
|
|
466
|
-
self.show(f"Write value of config variable index.SETTINGS_JSON to "
|
|
467
|
-
f"file {self.name}.settings.json\n"
|
|
468
|
-
f"{cmdline}", only_show)
|
|
469
|
-
if only_show:
|
|
470
|
-
return
|
|
471
|
-
|
|
472
|
-
# When docker.USE_DOCKER=false, check if the binary for building the
|
|
473
|
-
# index exists and works.
|
|
474
|
-
if self.config['docker']['use_docker'] not in self.yes_values:
|
|
475
|
-
try:
|
|
476
|
-
check_binary_cmd = f"{self.config['index']['binary']} --help"
|
|
477
|
-
subprocess.run(check_binary_cmd, shell=True, check=True,
|
|
478
|
-
stdout=subprocess.DEVNULL,
|
|
479
|
-
stderr=subprocess.DEVNULL)
|
|
480
|
-
except subprocess.CalledProcessError as e:
|
|
481
|
-
log.error(f"Running \"{check_binary_cmd}\" failed ({e}), "
|
|
482
|
-
f"set index.BINARY to a different binary or "
|
|
483
|
-
f"set docker.USE_DOCKER=true")
|
|
484
|
-
abort_script()
|
|
485
|
-
|
|
486
|
-
# Check if index files (name.index.*) already exist.
|
|
487
|
-
if glob.glob(f"{self.name}.index.*"):
|
|
488
|
-
raise ActionException(
|
|
489
|
-
f"Index files \"{self.name}.index.*\" already exist, "
|
|
490
|
-
f"please delete them if you want to rebuild the index")
|
|
491
|
-
|
|
492
|
-
# Write settings.json file and run the command.
|
|
493
|
-
with open(f"{self.name}.settings.json", "w") as f:
|
|
494
|
-
f.write(self.config['index']['settings_json'])
|
|
495
|
-
subprocess.run(cmdline, shell=True)
|
|
496
|
-
|
|
497
|
-
@track_action_rank
|
|
498
|
-
def action_remove_index(self, only_show=False):
|
|
499
|
-
"""
|
|
500
|
-
Action that removes the index files.
|
|
501
|
-
"""
|
|
502
|
-
|
|
503
|
-
# List of all the index files (not all of them need to be there).
|
|
504
|
-
index_fileglobs = (f"{self.name}.index.*",
|
|
505
|
-
f"{self.name}.patterns.*",
|
|
506
|
-
f"{self.name}.prefixes",
|
|
507
|
-
f"{self.name}.meta-data.json",
|
|
508
|
-
f"{self.name}.vocabulary.*")
|
|
509
|
-
|
|
510
|
-
# Show the command line.
|
|
511
|
-
self.show(f"Remove index files {', '.join(index_fileglobs)}",
|
|
512
|
-
only_show)
|
|
513
|
-
if only_show:
|
|
514
|
-
return
|
|
515
|
-
|
|
516
|
-
# Remove the index files.
|
|
517
|
-
files_removed = []
|
|
518
|
-
total_file_size = 0
|
|
519
|
-
for index_fileglob in index_fileglobs:
|
|
520
|
-
for filename in glob.glob(index_fileglob):
|
|
521
|
-
if os.path.isfile(filename):
|
|
522
|
-
total_file_size += os.path.getsize(filename)
|
|
523
|
-
os.remove(filename)
|
|
524
|
-
files_removed.append(filename)
|
|
525
|
-
if files_removed:
|
|
526
|
-
log.info(f"Removed the following index files of total size "
|
|
527
|
-
f"{total_file_size / 1e9:.1f} GB:")
|
|
528
|
-
log.info("")
|
|
529
|
-
log.info(", ".join(files_removed))
|
|
530
|
-
else:
|
|
531
|
-
log.info("None of the listed index files found, nothing removed")
|
|
532
|
-
|
|
533
|
-
@track_action_rank
|
|
534
|
-
def action_start(self, only_show=False):
|
|
535
|
-
"""
|
|
536
|
-
Action that starts the QLever server according to the settings in the
|
|
537
|
-
[server] section of the Qleverfile. If a server is already running, the
|
|
538
|
-
action reports that fact and does nothing.
|
|
539
|
-
"""
|
|
540
|
-
|
|
541
|
-
# Construct the command line based on the config file.
|
|
542
|
-
server_config = self.config['server']
|
|
543
|
-
cmdline = (f"{self.config['server']['binary']}"
|
|
544
|
-
f" -i {self.name}"
|
|
545
|
-
f" -j {server_config['num_threads']}"
|
|
546
|
-
f" -p {server_config['port']}"
|
|
547
|
-
f" -m {server_config['memory_for_queries_gb']}"
|
|
548
|
-
f" -c {server_config['cache_max_size_gb']}"
|
|
549
|
-
f" -e {server_config['cache_max_size_gb_single_entry']}"
|
|
550
|
-
f" -k {server_config['cache_max_num_entries']}")
|
|
551
|
-
if server_config['access_token']:
|
|
552
|
-
cmdline += f" -a {server_config['access_token']}"
|
|
553
|
-
if server_config['only_pso_and_pos_permutations'] in self.yes_values:
|
|
554
|
-
cmdline += " --only-pso-and-pos-permutations"
|
|
555
|
-
if not server_config['use_patterns'] in self.yes_values:
|
|
556
|
-
cmdline += " --no-patterns"
|
|
557
|
-
if server_config['with_text_index'] in \
|
|
558
|
-
["from_text_records",
|
|
559
|
-
"from_literals",
|
|
560
|
-
"from_text_records_and_literals"]:
|
|
561
|
-
cmdline += " -t"
|
|
562
|
-
cmdline += f" > {self.name}.server-log.txt 2>&1"
|
|
563
|
-
|
|
564
|
-
# If we are using Docker, run the command in a docker container.
|
|
565
|
-
if self.config['docker']['use_docker'] in self.yes_values:
|
|
566
|
-
docker_config = self.config['docker']
|
|
567
|
-
cmdline = (f"docker run -d --restart=unless-stopped"
|
|
568
|
-
f" -u $(id -u):$(id -g)"
|
|
569
|
-
f" -it -v /etc/localtime:/etc/localtime:ro"
|
|
570
|
-
f" -v $(pwd):/index"
|
|
571
|
-
f" -p {server_config['port']}:{server_config['port']}"
|
|
572
|
-
f" -w /index"
|
|
573
|
-
f" --entrypoint bash"
|
|
574
|
-
f" --name {docker_config['container_server']}"
|
|
575
|
-
f" {docker_config['image']}"
|
|
576
|
-
f" -c {shlex.quote(cmdline)}")
|
|
577
|
-
else:
|
|
578
|
-
cmdline = f"nohup {cmdline} &"
|
|
579
|
-
|
|
580
|
-
# Show the command line.
|
|
581
|
-
self.show(cmdline, only_show)
|
|
582
|
-
if only_show:
|
|
583
|
-
return
|
|
584
|
-
|
|
585
|
-
# When docker.USE_DOCKER=false, check if the binary for starting the
|
|
586
|
-
# server exists and works.
|
|
587
|
-
if self.config['docker']['use_docker'] not in self.yes_values:
|
|
588
|
-
try:
|
|
589
|
-
check_binary_cmd = f"{self.config['server']['binary']} --help"
|
|
590
|
-
subprocess.run(check_binary_cmd, shell=True, check=True,
|
|
591
|
-
stdout=subprocess.DEVNULL,
|
|
592
|
-
stderr=subprocess.DEVNULL)
|
|
593
|
-
except subprocess.CalledProcessError as e:
|
|
594
|
-
log.error(f"Running \"{check_binary_cmd}\" failed ({e}), "
|
|
595
|
-
f"set server.BINARY to a different binary or "
|
|
596
|
-
f"set docker.USE_DOCKER=true")
|
|
597
|
-
abort_script()
|
|
598
|
-
|
|
599
|
-
# Check if a QLever server is already running on this port.
|
|
600
|
-
port = server_config['port']
|
|
601
|
-
if self.alive_check(port):
|
|
602
|
-
raise ActionException(
|
|
603
|
-
f"QLever server already running on port {port}")
|
|
604
|
-
|
|
605
|
-
# Check if another process is already listening.
|
|
606
|
-
if self.net_connections_enabled:
|
|
607
|
-
if port in [conn.laddr.port for conn
|
|
608
|
-
in psutil.net_connections()]:
|
|
609
|
-
raise ActionException(
|
|
610
|
-
f"Port {port} is already in use by another process")
|
|
611
|
-
|
|
612
|
-
# Execute the command line.
|
|
613
|
-
subprocess.run(cmdline, shell=True,
|
|
614
|
-
stdout=subprocess.DEVNULL,
|
|
615
|
-
stderr=subprocess.DEVNULL)
|
|
616
|
-
|
|
617
|
-
# Tail the server log until the server is ready (note that the `exec`
|
|
618
|
-
# is important to make sure that the tail process is killed and not
|
|
619
|
-
# just the bash process).
|
|
620
|
-
log.info(f"Follow {self.name}.server-log.txt until the server is ready"
|
|
621
|
-
f" (Ctrl-C stops following the log, but not the server)")
|
|
622
|
-
log.info("")
|
|
623
|
-
tail_cmd = f"exec tail -f {self.name}.server-log.txt"
|
|
624
|
-
tail_proc = subprocess.Popen(tail_cmd, shell=True)
|
|
625
|
-
while not self.alive_check(port):
|
|
626
|
-
time.sleep(1)
|
|
627
|
-
|
|
628
|
-
# Set the access token if specified.
|
|
629
|
-
access_token = server_config['access_token']
|
|
630
|
-
access_arg = f"--data-urlencode \"access-token={access_token}\""
|
|
631
|
-
if "index_description" in self.config['data']:
|
|
632
|
-
desc = self.config['data']['index_description']
|
|
633
|
-
curl_cmd = (f"curl -Gs http://localhost:{port}/api"
|
|
634
|
-
f" --data-urlencode \"index-description={desc}\""
|
|
635
|
-
f" {access_arg} > /dev/null")
|
|
636
|
-
log.debug(curl_cmd)
|
|
637
|
-
subprocess.run(curl_cmd, shell=True)
|
|
638
|
-
if "text_description" in self.config['data']:
|
|
639
|
-
desc = self.config['data']['text_description']
|
|
640
|
-
curl_cmd = (f"curl -Gs http://localhost:{port}/api"
|
|
641
|
-
f" --data-urlencode \"text-description={desc}\""
|
|
642
|
-
f" {access_arg} > /dev/null")
|
|
643
|
-
log.debug(curl_cmd)
|
|
644
|
-
subprocess.run(curl_cmd, shell=True)
|
|
645
|
-
|
|
646
|
-
# Kill the tail process. NOTE: `tail_proc.kill()` does not work.
|
|
647
|
-
tail_proc.terminate()
|
|
648
|
-
|
|
649
|
-
@track_action_rank
|
|
650
|
-
def action_stop(self, only_show=False, fail_if_not_running=True):
|
|
651
|
-
"""
|
|
652
|
-
Action that stops the QLever server according to the settings in the
|
|
653
|
-
[server] section of the Qleverfile. If no server is running, the action
|
|
654
|
-
does nothing.
|
|
655
|
-
"""
|
|
656
|
-
|
|
657
|
-
# Show action description.
|
|
658
|
-
docker_container_name = self.config['docker']['container_server']
|
|
659
|
-
cmdline_regex = (f"ServerMain.* -i [^ ]*{self.name}")
|
|
660
|
-
self.show(f"Checking for process matching \"{cmdline_regex}\" "
|
|
661
|
-
f"and for Docker container with name "
|
|
662
|
-
f"\"{docker_container_name}\"", only_show)
|
|
663
|
-
if only_show:
|
|
664
|
-
return
|
|
665
|
-
|
|
666
|
-
# First check if there is docker container running.
|
|
667
|
-
if self.docker_enabled:
|
|
668
|
-
docker_cmd = (f"docker stop {docker_container_name} && "
|
|
669
|
-
f"docker rm {docker_container_name}")
|
|
670
|
-
try:
|
|
671
|
-
subprocess.run(docker_cmd, shell=True, check=True,
|
|
672
|
-
stdout=subprocess.DEVNULL,
|
|
673
|
-
stderr=subprocess.DEVNULL)
|
|
674
|
-
log.info(f"Docker container with name "
|
|
675
|
-
f"\"{docker_container_name}\" "
|
|
676
|
-
f"stopped and removed")
|
|
677
|
-
return
|
|
678
|
-
except Exception as e:
|
|
679
|
-
log.debug(f"Error running \"{docker_cmd}\": {e}")
|
|
680
|
-
|
|
681
|
-
# Check if there is a process running on the server port using psutil.
|
|
682
|
-
#
|
|
683
|
-
# NOTE: On MacOS, some of the proc's returned by psutil.process_iter()
|
|
684
|
-
# no longer exist when we try to access them, so we just skip them.
|
|
685
|
-
for proc in psutil.process_iter():
|
|
686
|
-
try:
|
|
687
|
-
pinfo = proc.as_dict(
|
|
688
|
-
attrs=['pid', 'username', 'create_time',
|
|
689
|
-
'memory_info', 'cmdline'])
|
|
690
|
-
cmdline = " ".join(pinfo['cmdline'])
|
|
691
|
-
except Exception as err:
|
|
692
|
-
log.debug(f"Error getting process info: {err}")
|
|
693
|
-
if re.match(cmdline_regex, cmdline):
|
|
694
|
-
log.info(f"Found process {pinfo['pid']} from user "
|
|
695
|
-
f"{pinfo['username']} with command line: {cmdline}")
|
|
696
|
-
print()
|
|
697
|
-
try:
|
|
698
|
-
proc.kill()
|
|
699
|
-
log.info(f"Killed process {pinfo['pid']}")
|
|
700
|
-
except Exception as e:
|
|
701
|
-
raise ActionException(
|
|
702
|
-
f"Could not kill process with PID "
|
|
703
|
-
f"{pinfo['pid']}: {e}")
|
|
704
|
-
return
|
|
705
|
-
|
|
706
|
-
# No matching process found.
|
|
707
|
-
message = "No matching process or Docker container found"
|
|
708
|
-
if fail_if_not_running:
|
|
709
|
-
raise ActionException(message)
|
|
710
|
-
else:
|
|
711
|
-
log.info(f"{message}, so nothing to stop")
|
|
712
|
-
|
|
713
|
-
@track_action_rank
|
|
714
|
-
def action_restart(self, only_show=False):
|
|
715
|
-
"""
|
|
716
|
-
Action that restarts the QLever server.
|
|
717
|
-
"""
|
|
718
|
-
|
|
719
|
-
# Show action description.
|
|
720
|
-
self.show("Stop running server if found, then start new server",
|
|
721
|
-
only_show)
|
|
722
|
-
if only_show:
|
|
723
|
-
return
|
|
724
|
-
|
|
725
|
-
# Do it.
|
|
726
|
-
self.action_stop(only_show=only_show, fail_if_not_running=False)
|
|
727
|
-
log.info("")
|
|
728
|
-
self.action_start()
|
|
729
|
-
|
|
730
|
-
@track_action_rank
|
|
731
|
-
def action_log(self, only_show=False):
|
|
732
|
-
"""
|
|
733
|
-
Action that shows the server log.
|
|
734
|
-
"""
|
|
735
|
-
|
|
736
|
-
# Show action description.
|
|
737
|
-
log_cmd = f"tail -f -n 50 {self.name}.server-log.txt"
|
|
738
|
-
self.show(log_cmd, only_show)
|
|
739
|
-
if only_show:
|
|
740
|
-
return
|
|
741
|
-
|
|
742
|
-
# Do it.
|
|
743
|
-
log.info(f"Follow {self.name}.server-log.txt (Ctrl-C stops"
|
|
744
|
-
f" following the log, but not the server)")
|
|
745
|
-
log.info("")
|
|
746
|
-
subprocess.run(log_cmd, shell=True)
|
|
747
|
-
|
|
748
|
-
@track_action_rank
|
|
749
|
-
def action_status(self, only_show=False):
|
|
750
|
-
"""
|
|
751
|
-
Action that shows all QLever processes running on this machine.
|
|
752
|
-
|
|
753
|
-
TODO: Also show the QLever-related docker containers.
|
|
754
|
-
"""
|
|
755
|
-
|
|
756
|
-
# Show action description.
|
|
757
|
-
cmdline_regex = "(ServerMain|IndexBuilderMain)"
|
|
758
|
-
# cmdline_regex = f"(ServerMain|IndexBuilderMain).*{self.name}"
|
|
759
|
-
self.show(f"{BLUE}Show all processes on this machine where "
|
|
760
|
-
f"the command line matches {cmdline_regex}"
|
|
761
|
-
f" using Python's psutil library", only_show)
|
|
762
|
-
if only_show:
|
|
763
|
-
return
|
|
764
|
-
|
|
765
|
-
# Show the results as a table.
|
|
766
|
-
num_processes_found = 0
|
|
767
|
-
for proc in psutil.process_iter():
|
|
768
|
-
show_heading = num_processes_found == 0
|
|
769
|
-
process_shown = self.show_process_info(proc, cmdline_regex,
|
|
770
|
-
show_heading=show_heading)
|
|
771
|
-
if process_shown:
|
|
772
|
-
num_processes_found += 1
|
|
773
|
-
if num_processes_found == 0:
|
|
774
|
-
print("No processes found")
|
|
775
|
-
|
|
776
|
-
@track_action_rank
|
|
777
|
-
def action_index_stats(self, only_show=False):
|
|
778
|
-
"""
|
|
779
|
-
Action that provides a breakdown of the time needed for building the
|
|
780
|
-
index, based on the log file of th index build.
|
|
781
|
-
"""
|
|
782
|
-
|
|
783
|
-
log_file_name = self.config['data']['name'] + ".index-log.txt"
|
|
784
|
-
log.info(f"{BLUE}Breakdown of the time for building the index, "
|
|
785
|
-
f"based on the timestamps for key lines in "
|
|
786
|
-
f"\"{log_file_name}{NORMAL}\"")
|
|
787
|
-
log.info("")
|
|
788
|
-
if only_show:
|
|
789
|
-
return
|
|
790
|
-
|
|
791
|
-
# Read the content of `log_file_name` into a list of lines.
|
|
792
|
-
try:
|
|
793
|
-
with open(log_file_name, "r") as f:
|
|
794
|
-
lines = f.readlines()
|
|
795
|
-
except Exception as e:
|
|
796
|
-
raise ActionException(f"Could not read log file {log_file_name}: "
|
|
797
|
-
f"{e}")
|
|
798
|
-
current_line = 0
|
|
799
|
-
|
|
800
|
-
# Helper lambda that finds the next line matching the given `regex`,
|
|
801
|
-
# starting from `current_line`, and extracts the time. Returns a tuple
|
|
802
|
-
# of the time and the regex match object. If a matchine is found,
|
|
803
|
-
# `current_line` is updated to the line after the match. Otherwise,
|
|
804
|
-
# `current_line` is not changed.
|
|
805
|
-
def find_next_line(regex, line_is_optional=False):
|
|
806
|
-
nonlocal lines
|
|
807
|
-
nonlocal current_line
|
|
808
|
-
current_line_backup = current_line
|
|
809
|
-
# Find starting from `current_line`.
|
|
810
|
-
while current_line < len(lines):
|
|
811
|
-
line = lines[current_line]
|
|
812
|
-
current_line += 1
|
|
813
|
-
timestamp_regex = r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}"
|
|
814
|
-
timestamp_format = "%Y-%m-%d %H:%M:%S"
|
|
815
|
-
regex_match = re.search(regex, line)
|
|
816
|
-
if regex_match:
|
|
817
|
-
try:
|
|
818
|
-
return datetime.strptime(
|
|
819
|
-
re.match(timestamp_regex, line).group(),
|
|
820
|
-
timestamp_format), regex_match
|
|
821
|
-
except Exception as e:
|
|
822
|
-
raise ActionException(
|
|
823
|
-
f"Could not parse timestamp of form "
|
|
824
|
-
f"\"{timestamp_regex}\" from line "
|
|
825
|
-
f" \"{line.rstrip()}\" ({e})")
|
|
826
|
-
# If we get here, we did not find a matching line.
|
|
827
|
-
if line_is_optional:
|
|
828
|
-
current_line = current_line_backup
|
|
829
|
-
return None, None
|
|
830
|
-
|
|
831
|
-
# Find the lines matching th key_lines_regex and extract the time
|
|
832
|
-
# information from them.
|
|
833
|
-
overall_begin, _ = find_next_line(r"INFO:\s*Processing")
|
|
834
|
-
merge_begin, _ = find_next_line(r"INFO:\s*Merging partial vocab")
|
|
835
|
-
convert_begin, _ = find_next_line(r"INFO:\s*Converting triples")
|
|
836
|
-
perm_begin_and_info = []
|
|
837
|
-
while True:
|
|
838
|
-
perm_begin, _ = find_next_line(r"INFO:\s*Creating a pair", True)
|
|
839
|
-
if perm_begin is None:
|
|
840
|
-
break
|
|
841
|
-
_, perm_info = find_next_line(r"INFO:\s*Writing meta data for"
|
|
842
|
-
r" ([A-Z]+ and [A-Z]+)", True)
|
|
843
|
-
if perm_info is None:
|
|
844
|
-
break
|
|
845
|
-
perm_begin_and_info.append((perm_begin, perm_info))
|
|
846
|
-
convert_end = (perm_begin_and_info[0][0] if
|
|
847
|
-
len(perm_begin_and_info) > 0 else None)
|
|
848
|
-
normal_end, _ = find_next_line(r"INFO:\s*Index build completed")
|
|
849
|
-
text_begin, _ = find_next_line(r"INFO:\s*Adding text index", True)
|
|
850
|
-
text_end, _ = find_next_line(r"INFO:\s*DocsDB done", True)
|
|
851
|
-
|
|
852
|
-
# Check whether at least the first phase is done.
|
|
853
|
-
if overall_begin is None:
|
|
854
|
-
raise ActionException("Missing line that index build has started")
|
|
855
|
-
if overall_begin and not merge_begin:
|
|
856
|
-
raise ActionException("According to the log file, the index build "
|
|
857
|
-
"has started, but is still in its first "
|
|
858
|
-
"phase (parsing the input)")
|
|
859
|
-
|
|
860
|
-
# Helper lambda that shows the duration for a phase (if the start and
|
|
861
|
-
# end timestamps are available).
|
|
862
|
-
def show_duration(heading, start_end_pairs):
|
|
863
|
-
nonlocal unit
|
|
864
|
-
num_start_end_pairs = 0
|
|
865
|
-
diff_seconds = 0
|
|
866
|
-
for start, end in start_end_pairs:
|
|
867
|
-
if start and end:
|
|
868
|
-
diff_seconds += (end - start).total_seconds()
|
|
869
|
-
num_start_end_pairs += 1
|
|
870
|
-
if num_start_end_pairs > 0:
|
|
871
|
-
if unit == "h":
|
|
872
|
-
diff = diff_seconds / 3600
|
|
873
|
-
elif unit == "min":
|
|
874
|
-
diff = diff_seconds / 60
|
|
875
|
-
else:
|
|
876
|
-
diff = diff_seconds
|
|
877
|
-
log.info(f"{heading:<23} : {diff:>5.1f} {unit}")
|
|
878
|
-
|
|
879
|
-
# Get the times of the various phases (hours or minutes, depending on
|
|
880
|
-
# how long the first phase took).
|
|
881
|
-
unit = "h"
|
|
882
|
-
if merge_begin and overall_begin:
|
|
883
|
-
parse_duration = (merge_begin - overall_begin).total_seconds()
|
|
884
|
-
if parse_duration < 200:
|
|
885
|
-
unit = "s"
|
|
886
|
-
elif parse_duration < 3600:
|
|
887
|
-
unit = "min"
|
|
888
|
-
show_duration("Parse input", [(overall_begin, merge_begin)])
|
|
889
|
-
show_duration("Build vocabularies", [(merge_begin, convert_begin)])
|
|
890
|
-
show_duration("Convert to global IDs", [(convert_begin, convert_end)])
|
|
891
|
-
for i in range(len(perm_begin_and_info)):
|
|
892
|
-
perm_begin, perm_info = perm_begin_and_info[i]
|
|
893
|
-
perm_end = perm_begin_and_info[i + 1][0] if i + 1 < len(
|
|
894
|
-
perm_begin_and_info) else normal_end
|
|
895
|
-
perm_info_text = (perm_info.group(1).replace(" and ", " & ")
|
|
896
|
-
if perm_info else f"#{i + 1}")
|
|
897
|
-
show_duration(f"Permutation {perm_info_text}",
|
|
898
|
-
[(perm_begin, perm_end)])
|
|
899
|
-
show_duration("Text index", [(text_begin, text_end)])
|
|
900
|
-
if text_begin and text_end:
|
|
901
|
-
log.info("")
|
|
902
|
-
show_duration("TOTAL index build time",
|
|
903
|
-
[(overall_begin, normal_end),
|
|
904
|
-
(text_begin, text_end)])
|
|
905
|
-
elif normal_end:
|
|
906
|
-
log.info("")
|
|
907
|
-
show_duration("TOTAL index build time",
|
|
908
|
-
[(overall_begin, normal_end)])
|
|
909
|
-
|
|
910
|
-
@track_action_rank
|
|
911
|
-
def action_test_query(self, only_show=False):
|
|
912
|
-
"""
|
|
913
|
-
Action that sends a simple test SPARQL query to the server.
|
|
914
|
-
"""
|
|
915
|
-
|
|
916
|
-
# Construct the curl command.
|
|
917
|
-
query = "SELECT * WHERE { ?s ?p ?o } LIMIT 10"
|
|
918
|
-
headers = ["Accept: text/tab-separated-values",
|
|
919
|
-
"Content-Type: application/sparql-query"]
|
|
920
|
-
curl_cmd = (f"curl -s {self.config['server']['url']} "
|
|
921
|
-
f"-H \"{headers[0]}\" -H \"{headers[1]}\" "
|
|
922
|
-
f"--data \"{query}\"")
|
|
923
|
-
|
|
924
|
-
# Show it.
|
|
925
|
-
self.show(curl_cmd, only_show)
|
|
926
|
-
if only_show:
|
|
927
|
-
return
|
|
928
|
-
|
|
929
|
-
# Execute it.
|
|
930
|
-
subprocess.run(curl_cmd, shell=True)
|
|
931
|
-
|
|
932
|
-
@track_action_rank
|
|
933
|
-
def action_ui(self, only_show=False):
|
|
934
|
-
"""
|
|
935
|
-
Action that starts the QLever UI with the server according to the
|
|
936
|
-
Qleverfile as backend.
|
|
937
|
-
"""
|
|
938
|
-
|
|
939
|
-
# Construct commands.
|
|
940
|
-
host_name = socket.getfqdn()
|
|
941
|
-
server_url = f"http://{host_name}:{self.config['server']['port']}"
|
|
942
|
-
docker_rm_cmd = f"docker rm -f {self.config['ui']['container']}"
|
|
943
|
-
docker_pull_cmd = f"docker pull {self.config['ui']['image']}"
|
|
944
|
-
docker_run_cmd = (f"docker run -d -p {self.config['ui']['port']}:7000 "
|
|
945
|
-
f"--name {self.config['ui']['container']} "
|
|
946
|
-
f"{self.config['ui']['image']} ")
|
|
947
|
-
docker_exec_cmd = (f"docker exec -it "
|
|
948
|
-
f"{self.config['ui']['container']} "
|
|
949
|
-
f"bash -c \"python manage.py configure "
|
|
950
|
-
f"{self.config['ui']['config']} "
|
|
951
|
-
f"{server_url}\"")
|
|
952
|
-
|
|
953
|
-
# Show them.
|
|
954
|
-
self.show("\n".join([docker_rm_cmd, docker_pull_cmd, docker_run_cmd,
|
|
955
|
-
docker_exec_cmd]), only_show)
|
|
956
|
-
if only_show:
|
|
957
|
-
return
|
|
958
|
-
|
|
959
|
-
# Execute them.
|
|
960
|
-
try:
|
|
961
|
-
subprocess.run(docker_rm_cmd, shell=True,
|
|
962
|
-
stdout=subprocess.DEVNULL)
|
|
963
|
-
subprocess.run(docker_pull_cmd, shell=True,
|
|
964
|
-
stdout=subprocess.DEVNULL)
|
|
965
|
-
subprocess.run(docker_run_cmd, shell=True,
|
|
966
|
-
stdout=subprocess.DEVNULL)
|
|
967
|
-
subprocess.run(docker_exec_cmd, shell=True,
|
|
968
|
-
stdout=subprocess.DEVNULL)
|
|
969
|
-
except subprocess.CalledProcessError as e:
|
|
970
|
-
raise ActionException(f"Failed to start the QLever UI {e}")
|
|
971
|
-
log.info(f"The QLever UI should now be up at "
|
|
972
|
-
f"http://{host_name}:{self.config['ui']['port']}")
|
|
973
|
-
log.info("You can log in as QLever UI admin with username and "
|
|
974
|
-
"passwort \"demo\"")
|
|
975
|
-
|
|
976
|
-
@track_action_rank
|
|
977
|
-
def action_cache_stats_and_settings(self, only_show=False):
|
|
978
|
-
"""
|
|
979
|
-
Action that shows the cache statistics and settings.
|
|
980
|
-
"""
|
|
981
|
-
|
|
982
|
-
# Construct the two curl commands.
|
|
983
|
-
cache_stats_cmd = (f"curl -s {self.config['server']['url']} "
|
|
984
|
-
f"--data-urlencode \"cmd=cache-stats\"")
|
|
985
|
-
cache_settings_cmd = (f"curl -s {self.config['server']['url']} "
|
|
986
|
-
f"--data-urlencode \"cmd=get-settings\"")
|
|
987
|
-
|
|
988
|
-
# Show them.
|
|
989
|
-
self.show("\n".join([cache_stats_cmd, cache_settings_cmd]), only_show)
|
|
990
|
-
if only_show:
|
|
991
|
-
return
|
|
992
|
-
|
|
993
|
-
# Execute them.
|
|
994
|
-
try:
|
|
995
|
-
cache_stats = subprocess.check_output(cache_stats_cmd, shell=True)
|
|
996
|
-
cache_settings = subprocess.check_output(cache_settings_cmd,
|
|
997
|
-
shell=True)
|
|
998
|
-
|
|
999
|
-
# Print the key-value pairs of the stats JSON in tabular form.
|
|
1000
|
-
def print_json_as_tabular(raw_json):
|
|
1001
|
-
key_value_pairs = json.loads(raw_json).items()
|
|
1002
|
-
max_key_len = max([len(key) for key, _ in key_value_pairs])
|
|
1003
|
-
for key, value in key_value_pairs:
|
|
1004
|
-
if isinstance(value, int) or re.match(r"^\d+$", value):
|
|
1005
|
-
value = "{:,}".format(int(value))
|
|
1006
|
-
if re.match(r"^\d+\.\d+$", value):
|
|
1007
|
-
value = "{:.2f}".format(float(value))
|
|
1008
|
-
log.info(f"{key.ljust(max_key_len)} : {value}")
|
|
1009
|
-
print_json_as_tabular(cache_stats)
|
|
1010
|
-
log.info("")
|
|
1011
|
-
print_json_as_tabular(cache_settings)
|
|
1012
|
-
except Exception as e:
|
|
1013
|
-
raise ActionException(f"Failed to get cache stats and settings: "
|
|
1014
|
-
f"{e}")
|
|
1015
|
-
|
|
1016
|
-
@track_action_rank
|
|
1017
|
-
def action_autocompletion_warmup(self, only_show=False):
|
|
1018
|
-
"""
|
|
1019
|
-
Action that pins the autocompletion queries from `ui.config` to the
|
|
1020
|
-
cache.
|
|
1021
|
-
"""
|
|
1022
|
-
|
|
1023
|
-
# Construct curl command to obtain the warmup queries.
|
|
1024
|
-
#
|
|
1025
|
-
# TODO: This is the access token expected by Django in views.py, where
|
|
1026
|
-
# it is currently set to dummy value. Find a sound yet simple mechanism
|
|
1027
|
-
# for this.
|
|
1028
|
-
access_token_ui = "top-secret"
|
|
1029
|
-
config_name = self.config["ui"]["config"]
|
|
1030
|
-
warmup_url = f"{self.config['ui']['url']}/warmup/{config_name}"
|
|
1031
|
-
curl_cmd = (f"curl -s {warmup_url}/queries?token={access_token_ui}")
|
|
1032
|
-
|
|
1033
|
-
# Show it.
|
|
1034
|
-
self.show(f"Pin warmup queries obtained via: {curl_cmd}", only_show)
|
|
1035
|
-
if only_show:
|
|
1036
|
-
return
|
|
1037
|
-
|
|
1038
|
-
# Get the queries.
|
|
1039
|
-
try:
|
|
1040
|
-
queries = subprocess.check_output(curl_cmd, shell=True)
|
|
1041
|
-
except subprocess.CalledProcessError as e:
|
|
1042
|
-
raise ActionException(f"Failed to get warmup queries ({e})")
|
|
1043
|
-
|
|
1044
|
-
# Iterate over them and pin them to the cache. Give a more generous
|
|
1045
|
-
# timeout (which requires an access token).
|
|
1046
|
-
header = "Accept: application/qlever-results+json"
|
|
1047
|
-
first = True
|
|
1048
|
-
timeout = "300s"
|
|
1049
|
-
access_token = self.config["server"]["access_token"]
|
|
1050
|
-
for description, query in [line.split("\t") for line in
|
|
1051
|
-
queries.decode("utf-8").split("\n")]:
|
|
1052
|
-
if first:
|
|
1053
|
-
first = False
|
|
1054
|
-
else:
|
|
1055
|
-
log.info("")
|
|
1056
|
-
log.info(f"{BOLD}Pin query: {description}{NORMAL}")
|
|
1057
|
-
pin_cmd = (f"curl -s {self.config['server']['url']}/api "
|
|
1058
|
-
f"-H \"{header}\" "
|
|
1059
|
-
f"--data-urlencode query={shlex.quote(query)} "
|
|
1060
|
-
f"--data-urlencode timeout={timeout} "
|
|
1061
|
-
f"--data-urlencode access-token={access_token} "
|
|
1062
|
-
f"--data-urlencode pinresult=true "
|
|
1063
|
-
f"--data-urlencode send=0")
|
|
1064
|
-
log.info(pin_cmd)
|
|
1065
|
-
# Launch query and show the `resultsize` of the JSON response.
|
|
1066
|
-
try:
|
|
1067
|
-
result = subprocess.check_output(pin_cmd, shell=True)
|
|
1068
|
-
json_result = json.loads(result.decode("utf-8"))
|
|
1069
|
-
# Check if the JSON has a key "exception".
|
|
1070
|
-
if "exception" in json_result:
|
|
1071
|
-
raise Exception(json_result["exception"])
|
|
1072
|
-
log.info(f"Result size: {json_result['resultsize']:,}")
|
|
1073
|
-
except Exception as e:
|
|
1074
|
-
log.error(f"Query failed: {e}")
|
|
1075
|
-
|
|
1076
|
-
@track_action_rank
|
|
1077
|
-
def action_example_queries(self, only_show=False):
|
|
1078
|
-
"""
|
|
1079
|
-
Action that shows the example queries from `ui.config`.
|
|
1080
|
-
"""
|
|
1081
|
-
|
|
1082
|
-
# Construct curl command to obtain the example queries.
|
|
1083
|
-
config_name = self.config["ui"]["config"]
|
|
1084
|
-
examples_url = f"{self.config['ui']['url']}/examples/{config_name}"
|
|
1085
|
-
curl_cmd = f"curl -s {examples_url}"
|
|
1086
|
-
|
|
1087
|
-
# Show what the action does.
|
|
1088
|
-
self.show(f"Launch example queries obtained via: {curl_cmd}\n"
|
|
1089
|
-
f"SPARQL endpoint: {self.config['server']['url']}\n"
|
|
1090
|
-
f"Clearing the cache before each query + using send=0",
|
|
1091
|
-
only_show)
|
|
1092
|
-
if only_show:
|
|
1093
|
-
return
|
|
1094
|
-
|
|
1095
|
-
# Get the queries.
|
|
1096
|
-
try:
|
|
1097
|
-
queries = subprocess.check_output(curl_cmd, shell=True)
|
|
1098
|
-
except subprocess.CalledProcessError as e:
|
|
1099
|
-
raise ActionException(f"Failed to get example queries ({e})")
|
|
1100
|
-
|
|
1101
|
-
# Launch the queries one after the other and for each print: the
|
|
1102
|
-
# description, the result size, and the query processing time.
|
|
1103
|
-
count = 0
|
|
1104
|
-
total_time_seconds = 0.0
|
|
1105
|
-
total_result_size = 0
|
|
1106
|
-
for description, query in [line.split("\t") for line in
|
|
1107
|
-
queries.decode("utf-8").split("\n")]:
|
|
1108
|
-
# Launch query and show the `resultsize` of the JSON response.
|
|
1109
|
-
clear_cache_cmd = (f"curl -s {self.config['server']['url']} "
|
|
1110
|
-
f"--data-urlencode cmd=clear-cache")
|
|
1111
|
-
query_cmd = (f"curl -s {self.config['server']['url']} "
|
|
1112
|
-
f"-H \"Accept: application/qlever-results+json\" "
|
|
1113
|
-
f"--data-urlencode query={shlex.quote(query)} "
|
|
1114
|
-
f"--data-urlencode send=0")
|
|
1115
|
-
try:
|
|
1116
|
-
subprocess.run(clear_cache_cmd, shell=True,
|
|
1117
|
-
stdout=subprocess.DEVNULL,
|
|
1118
|
-
stderr=subprocess.DEVNULL)
|
|
1119
|
-
start_time = time.time()
|
|
1120
|
-
result = subprocess.check_output(query_cmd, shell=True)
|
|
1121
|
-
time_seconds = time.time() - start_time
|
|
1122
|
-
json_result = json.loads(result.decode("utf-8"))
|
|
1123
|
-
if "exception" in json_result:
|
|
1124
|
-
raise Exception(json_result["exception"])
|
|
1125
|
-
result_size = int(json_result["resultsize"])
|
|
1126
|
-
result_string = "{:,}".format(result_size)
|
|
1127
|
-
except Exception as e:
|
|
1128
|
-
time_seconds = 0.0
|
|
1129
|
-
result_size = 0
|
|
1130
|
-
result_string = f"{RED}{e}{NORMAL}"
|
|
1131
|
-
|
|
1132
|
-
# Print description, time, result in tabular form.
|
|
1133
|
-
log.debug(query)
|
|
1134
|
-
log.info(f"{description:<60} {time_seconds:6.2f} s "
|
|
1135
|
-
f"{result_string:>10}")
|
|
1136
|
-
count += 1
|
|
1137
|
-
total_time_seconds += time_seconds
|
|
1138
|
-
total_result_size += result_size
|
|
1139
|
-
if count == 10:
|
|
1140
|
-
break
|
|
1141
|
-
|
|
1142
|
-
# Print total time.
|
|
1143
|
-
log.info("")
|
|
1144
|
-
description = (f"TOTAL for {count} "
|
|
1145
|
-
f"{'query' if count == 1 else 'queries'}")
|
|
1146
|
-
log.info(f"{description:<60} {total_time_seconds:6.2f} s "
|
|
1147
|
-
f"{total_result_size:>10,}")
|
|
1148
|
-
|
|
1149
|
-
@track_action_rank
|
|
1150
|
-
def action_memory_profile(self, only_show=False):
|
|
1151
|
-
"""
|
|
1152
|
-
Action that prints the memory usage of a process (specified via
|
|
1153
|
-
`general.PID`) to a file `<PID>.memory-usage.tsv`.
|
|
1154
|
-
"""
|
|
1155
|
-
|
|
1156
|
-
# Show what the action does.
|
|
1157
|
-
self.show("Poll memory usage of the given process every second "
|
|
1158
|
-
"and print it to a file", only_show)
|
|
1159
|
-
if only_show:
|
|
1160
|
-
return
|
|
1161
|
-
|
|
1162
|
-
# Show process information.
|
|
1163
|
-
if "pid" not in self.config["general"]:
|
|
1164
|
-
raise ActionException("PID must be specified via general.PID")
|
|
1165
|
-
try:
|
|
1166
|
-
pid = int(self.config["general"]["pid"])
|
|
1167
|
-
proc = psutil.Process(pid)
|
|
1168
|
-
except Exception as e:
|
|
1169
|
-
raise ActionException(f"Could not obtain information for process "
|
|
1170
|
-
f"with PID {pid} ({e})")
|
|
1171
|
-
self.show_process_info(proc, "", show_heading=True)
|
|
1172
|
-
log.info("")
|
|
1173
|
-
|
|
1174
|
-
# As long as the process exists, poll memory usage once per second and
|
|
1175
|
-
# print it to the screen as well as to a file `<PID>.memory-usage.tsv`.
|
|
1176
|
-
file = open(f"{pid}.memory-usage.tsv", "w")
|
|
1177
|
-
seconds = 0
|
|
1178
|
-
while proc.is_running():
|
|
1179
|
-
# Get memory usage in bytes and print as <timestamp>\t<size>, with
|
|
1180
|
-
# the timestand in the usual logger format (second precision).
|
|
1181
|
-
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
1182
|
-
memory_usage_gb = f"{proc.memory_info().rss / 1e9:.1f}"
|
|
1183
|
-
log.info(f"{timestamp}\t{memory_usage_gb}")
|
|
1184
|
-
file.write(f"{timestamp}\t{memory_usage_gb}\n")
|
|
1185
|
-
time.sleep(1)
|
|
1186
|
-
seconds += 1
|
|
1187
|
-
if seconds % 60 == 0:
|
|
1188
|
-
file.flush()
|
|
1189
|
-
file.close()
|
|
1190
|
-
|
|
1191
|
-
@track_action_rank
|
|
1192
|
-
def action_memory_profile_show(self, only_show=False):
|
|
1193
|
-
"""
|
|
1194
|
-
Action that shows a plot of the memory profile produce with action
|
|
1195
|
-
`memory_profile`.
|
|
1196
|
-
"""
|
|
1197
|
-
|
|
1198
|
-
# Construct gnuplot command.
|
|
1199
|
-
if "pid" not in self.config["general"]:
|
|
1200
|
-
raise ActionException("PID must be specified via general.PID")
|
|
1201
|
-
pid = int(self.config["general"]["pid"])
|
|
1202
|
-
gnuplot_script = (f"set datafile separator \"\t\"; "
|
|
1203
|
-
f"set xdata time; "
|
|
1204
|
-
f"set timefmt \"%Y-%m-%d %H:%M:%S\"; "
|
|
1205
|
-
f"set xlabel \"Time\"; "
|
|
1206
|
-
f"set ylabel \"Memory Usage\"; "
|
|
1207
|
-
f"set grid; "
|
|
1208
|
-
f"plot \"{pid}.memory-usage.tsv\" "
|
|
1209
|
-
f"using 1:2 with lines; "
|
|
1210
|
-
f"pause -1")
|
|
1211
|
-
gnuplot_cmd = f"gnuplot -e {shlex.quote(gnuplot_script)}"
|
|
1212
|
-
|
|
1213
|
-
# Show it.
|
|
1214
|
-
self.show(gnuplot_cmd, only_show)
|
|
1215
|
-
if only_show:
|
|
1216
|
-
return
|
|
1217
|
-
|
|
1218
|
-
# Launch gnuplot.
|
|
1219
|
-
try:
|
|
1220
|
-
subprocess.check_output(gnuplot_cmd, shell=True)
|
|
1221
|
-
except subprocess.CalledProcessError as e:
|
|
1222
|
-
raise ActionException(f"Failed to launch gnuplot ({e})")
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
def setup_autocompletion_cmd():
|
|
1226
|
-
"""
|
|
1227
|
-
Print the command for setting up autocompletion for the qlever.py script.
|
|
1228
|
-
|
|
1229
|
-
TODO: Currently works for bash only.
|
|
1230
|
-
"""
|
|
1231
|
-
|
|
1232
|
-
# Get methods that start wth "action_" from the Actions class, sorted by
|
|
1233
|
-
# their appearance in the class (see the `@track_action_rank` decorator).
|
|
1234
|
-
methods = inspect.getmembers(Actions, predicate=inspect.isfunction)
|
|
1235
|
-
methods = [m for m in methods if m[0].startswith("action_")]
|
|
1236
|
-
action_names = sorted([m[0] for m in methods],
|
|
1237
|
-
key=lambda m: getattr(Actions, m).rank)
|
|
1238
|
-
action_names = [_.replace("action_", "") for _ in action_names]
|
|
1239
|
-
action_names = [_.replace("_", "-") for _ in action_names]
|
|
1240
|
-
action_names = " ".join(action_names)
|
|
1241
|
-
|
|
1242
|
-
# Add config settings to the list of possible actions for autocompletion.
|
|
1243
|
-
action_names += " docker.USE_DOCKER=true docker.USE_DOCKER=false"
|
|
1244
|
-
action_names += " index.BINARY=IndexBuilderMain"
|
|
1245
|
-
action_names += " server.BINARY=ServerMain"
|
|
1246
|
-
|
|
1247
|
-
# Return multiline string with the command for setting up autocompletion.
|
|
1248
|
-
return f"""\
|
|
1249
|
-
_qlever_completion() {{
|
|
1250
|
-
local cur=${{COMP_WORDS[COMP_CWORD]}}
|
|
1251
|
-
COMPREPLY=( $(compgen -W "{action_names}" -- $cur) )
|
|
1252
|
-
}}
|
|
1253
|
-
complete -o nosort -F _qlever_completion qlever
|
|
1254
|
-
"""
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
# Get all action names.
|
|
1258
|
-
action_names = [_ for _ in dir(Actions) if _.startswith("action_")]
|
|
1259
|
-
action_names = [_.replace("action_", "") for _ in action_names]
|
|
1260
|
-
action_names = [_.replace("_", "-") for _ in action_names]
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
def main():
|
|
1264
|
-
# If the script is called without argument, say hello and provide some
|
|
1265
|
-
# help to get started.
|
|
1266
|
-
if len(sys.argv) == 1 or \
|
|
1267
|
-
(len(sys.argv) == 2 and sys.argv[1] == "help") or \
|
|
1268
|
-
(len(sys.argv) == 2 and sys.argv[1] == "--help") or \
|
|
1269
|
-
(len(sys.argv) == 2 and sys.argv[1] == "-h"):
|
|
1270
|
-
log.info("")
|
|
1271
|
-
log.info(f"{BOLD}Hello, I am the qlever script{NORMAL}")
|
|
1272
|
-
log.info("")
|
|
1273
|
-
if os.path.exists("Qleverfile"):
|
|
1274
|
-
log.info("I see that you already have a \"Qleverfile\" in the "
|
|
1275
|
-
"current directory, so you are ready to start")
|
|
1276
|
-
log.info("")
|
|
1277
|
-
show_available_action_names()
|
|
1278
|
-
else:
|
|
1279
|
-
log.info("You need a Qleverfile in the current directory, which "
|
|
1280
|
-
"you can create as follows:")
|
|
1281
|
-
log.info("")
|
|
1282
|
-
log.info(f"{BLUE}qlever setup-config <config name>{NORMAL}")
|
|
1283
|
-
log.info("")
|
|
1284
|
-
show_available_config_names()
|
|
1285
|
-
log.info("")
|
|
1286
|
-
log.info("If you omit <config name>, you get a default Qleverfile")
|
|
1287
|
-
log.info("")
|
|
1288
|
-
return
|
|
1289
|
-
|
|
1290
|
-
# If there is only argument `setup-autocompletion`, call the function
|
|
1291
|
-
# `Actions.setup_autocompletion()` above and exit.
|
|
1292
|
-
if len(sys.argv) == 2 and sys.argv[1] == "setup-autocompletion":
|
|
1293
|
-
log.setLevel(logging.ERROR)
|
|
1294
|
-
print(setup_autocompletion_cmd())
|
|
1295
|
-
sys.exit(0)
|
|
1296
|
-
|
|
1297
|
-
# If the first argument sets the log level, deal with that immediately (so
|
|
1298
|
-
# that it goes into effect before we do anything else). Otherwise, set the
|
|
1299
|
-
# log level to `NOTSET` (which will signal to the Actions class that it can
|
|
1300
|
-
# take the log level from the config file).
|
|
1301
|
-
log.setLevel(logging.NOTSET)
|
|
1302
|
-
if len(sys.argv) > 1:
|
|
1303
|
-
set_log_level_match = re.match(r"general.log_level=(\w+)",
|
|
1304
|
-
sys.argv[1], re.IGNORECASE)
|
|
1305
|
-
if set_log_level_match:
|
|
1306
|
-
log_level = set_log_level_match.group(1).upper()
|
|
1307
|
-
sys.argv = sys.argv[1:]
|
|
1308
|
-
try:
|
|
1309
|
-
log.setLevel(getattr(logging, log_level))
|
|
1310
|
-
log.debug("")
|
|
1311
|
-
log.debug(f"Log level set to {log_level}")
|
|
1312
|
-
log.debug("")
|
|
1313
|
-
except AttributeError:
|
|
1314
|
-
log.error(f"Invalid log level: \"{log_level}\"")
|
|
1315
|
-
abort_script()
|
|
1316
|
-
|
|
1317
|
-
# Helper function that executes an action.
|
|
1318
|
-
def execute_action(actions, action_name, **kwargs):
|
|
1319
|
-
log.info("")
|
|
1320
|
-
log.info(f"{BOLD}Action \"{action_name}\"{NORMAL}")
|
|
1321
|
-
log.info("")
|
|
1322
|
-
action = f"action_{action_name.replace('-', '_')}"
|
|
1323
|
-
try:
|
|
1324
|
-
getattr(actions, action)(**kwargs)
|
|
1325
|
-
except ActionException as err:
|
|
1326
|
-
print(f"{RED}{err}{NORMAL}")
|
|
1327
|
-
abort_script()
|
|
1328
|
-
except Exception as err:
|
|
1329
|
-
line = traceback.extract_tb(err.__traceback__)[-1].lineno
|
|
1330
|
-
print(f"{RED}Error in Python script (line {line}: {err})"
|
|
1331
|
-
f", stack trace follows:{NORMAL}")
|
|
1332
|
-
print()
|
|
1333
|
-
raise err
|
|
1334
|
-
|
|
1335
|
-
# If `setup-config` is among the command-line arguments, it must the first
|
|
1336
|
-
# one, followed by at most one more argument.
|
|
1337
|
-
if "setup-config" in sys.argv:
|
|
1338
|
-
if sys.argv.index("setup-config") > 1:
|
|
1339
|
-
log.setLevel(logging.ERROR)
|
|
1340
|
-
log.error("Action `setup-config` must be the first argument")
|
|
1341
|
-
abort_script()
|
|
1342
|
-
if len(sys.argv) > 3:
|
|
1343
|
-
log.setLevel(logging.ERROR)
|
|
1344
|
-
log.error("Action `setup-config` must be followed by at most one "
|
|
1345
|
-
"argument (the name of the desied configuration)")
|
|
1346
|
-
abort_script()
|
|
1347
|
-
log.setLevel(logging.INFO)
|
|
1348
|
-
config_name = sys.argv[2] if len(sys.argv) == 3 else "default"
|
|
1349
|
-
execute_action(Actions, "setup-config", config_name=config_name)
|
|
1350
|
-
return
|
|
1351
|
-
|
|
1352
|
-
actions = Actions()
|
|
1353
|
-
# log.info(f"Actions available are: {', '.join(action_names)}")
|
|
1354
|
-
# Show the log level as string.
|
|
1355
|
-
# log.info(f"Log level: {logging.getLevelName(log.getEffectiveLevel())}")
|
|
1356
|
-
|
|
1357
|
-
# Check if the last argument is "show" (if yes, remember it and remove it).
|
|
1358
|
-
only_show = True if len(sys.argv) > 1 and sys.argv[-1] == "show" else False
|
|
1359
|
-
if only_show:
|
|
1360
|
-
sys.argv = sys.argv[:-1]
|
|
1361
|
-
|
|
1362
|
-
# Initalize actions.
|
|
1363
|
-
# Execute the actions specified on the command line.
|
|
1364
|
-
for action_name in sys.argv[1:]:
|
|
1365
|
-
# If the action is of the form section.key=value, set the config value.
|
|
1366
|
-
set_config_match = re.match(r"(\w+)\.(\w+)=(.*)", action_name)
|
|
1367
|
-
if set_config_match:
|
|
1368
|
-
section, option, value = set_config_match.groups()
|
|
1369
|
-
log.info(f"Setting config value: {section}.{option}={value}")
|
|
1370
|
-
try:
|
|
1371
|
-
actions.set_config(section, option, value)
|
|
1372
|
-
except ValueError as err:
|
|
1373
|
-
log.error(err)
|
|
1374
|
-
abort_script()
|
|
1375
|
-
continue
|
|
1376
|
-
# If the action name does not exist, exit.
|
|
1377
|
-
if action_name not in action_names:
|
|
1378
|
-
log.error(f"Action \"{action_name}\" does not exist, available "
|
|
1379
|
-
f"actions are: {', '.join(action_names)}")
|
|
1380
|
-
abort_script()
|
|
1381
|
-
# Execute the action (or only show what would be executed).
|
|
1382
|
-
execute_action(actions, action_name, only_show=only_show)
|
|
1383
|
-
log.info("")
|