qlever 0.5.15__py3-none-any.whl → 0.5.17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of qlever might be problematic. Click here for more details.
- qlever/commands/example_queries.py +135 -44
- qlever/commands/extract_queries.py +113 -0
- qlever/commands/index.py +1 -1
- qlever/commands/settings.py +110 -0
- qlever/commands/start.py +215 -104
- qlever/commands/stop.py +39 -26
- qlever/util.py +34 -17
- {qlever-0.5.15.dist-info → qlever-0.5.17.dist-info}/METADATA +2 -2
- {qlever-0.5.15.dist-info → qlever-0.5.17.dist-info}/RECORD +13 -11
- {qlever-0.5.15.dist-info → qlever-0.5.17.dist-info}/WHEEL +1 -1
- {qlever-0.5.15.dist-info → qlever-0.5.17.dist-info}/LICENSE +0 -0
- {qlever-0.5.15.dist-info → qlever-0.5.17.dist-info}/entry_points.txt +0 -0
- {qlever-0.5.15.dist-info → qlever-0.5.17.dist-info}/top_level.txt +0 -0
|
@@ -51,13 +51,15 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
51
51
|
subparser.add_argument(
|
|
52
52
|
"--get-queries-cmd",
|
|
53
53
|
type=str,
|
|
54
|
-
help="Command to get example queries as TSV "
|
|
54
|
+
help="Command to get example queries as TSV "
|
|
55
|
+
"(description, query)",
|
|
55
56
|
)
|
|
56
57
|
subparser.add_argument(
|
|
57
58
|
"--query-ids",
|
|
58
59
|
type=str,
|
|
59
60
|
default="1-$",
|
|
60
|
-
help="Query IDs as comma-separated list of "
|
|
61
|
+
help="Query IDs as comma-separated list of "
|
|
62
|
+
"ranges (e.g., 1-5,7,12-$)",
|
|
61
63
|
)
|
|
62
64
|
subparser.add_argument(
|
|
63
65
|
"--query-regex",
|
|
@@ -68,7 +70,7 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
68
70
|
subparser.add_argument(
|
|
69
71
|
"--download-or-count",
|
|
70
72
|
choices=["download", "count"],
|
|
71
|
-
default="
|
|
73
|
+
default="download",
|
|
72
74
|
help="Whether to download the full result "
|
|
73
75
|
"or just compute the size of the result",
|
|
74
76
|
)
|
|
@@ -88,10 +90,14 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
88
90
|
"text/tab-separated-values",
|
|
89
91
|
"text/csv",
|
|
90
92
|
"application/sparql-results+json",
|
|
93
|
+
"application/qlever-results+json",
|
|
91
94
|
"text/turtle",
|
|
95
|
+
"AUTO",
|
|
92
96
|
],
|
|
93
97
|
default="application/sparql-results+json",
|
|
94
|
-
help="Accept header for the SPARQL query"
|
|
98
|
+
help="Accept header for the SPARQL query; AUTO means "
|
|
99
|
+
"`text/turtle` for CONSTRUCT AND DESCRIBE queries, "
|
|
100
|
+
"`application/sparql-results+json` for all others",
|
|
95
101
|
)
|
|
96
102
|
subparser.add_argument(
|
|
97
103
|
"--clear-cache",
|
|
@@ -117,6 +123,13 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
117
123
|
default=14,
|
|
118
124
|
help="Width for printing the result size",
|
|
119
125
|
)
|
|
126
|
+
subparser.add_argument(
|
|
127
|
+
"--add-query-type-to-description",
|
|
128
|
+
action="store_true",
|
|
129
|
+
default=False,
|
|
130
|
+
help="Add the query type (SELECT, ASK, CONSTRUCT, DESCRIBE, "
|
|
131
|
+
"UNKNOWN) to the description",
|
|
132
|
+
)
|
|
120
133
|
subparser.add_argument(
|
|
121
134
|
"--show-query",
|
|
122
135
|
choices=["always", "never", "on-error"],
|
|
@@ -130,19 +143,35 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
130
143
|
help="When showing the query, also show the prefixes",
|
|
131
144
|
)
|
|
132
145
|
|
|
133
|
-
def
|
|
134
|
-
remove_prefixes_cmd =
|
|
146
|
+
def pretty_printed_query(self, query: str, show_prefixes: bool) -> str:
|
|
147
|
+
remove_prefixes_cmd = (
|
|
148
|
+
" | sed '/^PREFIX /Id'" if not show_prefixes else ""
|
|
149
|
+
)
|
|
135
150
|
pretty_print_query_cmd = (
|
|
136
151
|
f"echo {shlex.quote(query)}"
|
|
137
152
|
f" | docker run -i --rm sparqling/sparql-formatter"
|
|
138
153
|
f"{remove_prefixes_cmd} | grep -v '^$'"
|
|
139
154
|
)
|
|
140
155
|
try:
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
156
|
+
query_pretty_printed = run_command(
|
|
157
|
+
pretty_print_query_cmd, return_output=True
|
|
158
|
+
)
|
|
159
|
+
return query_pretty_printed.rstrip()
|
|
160
|
+
except Exception:
|
|
161
|
+
log.error(
|
|
162
|
+
"Failed to pretty-print query, "
|
|
163
|
+
"returning original query: {e}"
|
|
164
|
+
)
|
|
165
|
+
return query.rstrip()
|
|
166
|
+
|
|
167
|
+
def sparql_query_type(self, query: str) -> str:
|
|
168
|
+
match = re.search(
|
|
169
|
+
r"(SELECT|ASK|CONSTRUCT|DESCRIBE)\s", query, re.IGNORECASE
|
|
170
|
+
)
|
|
171
|
+
if match:
|
|
172
|
+
return match.group(1).upper()
|
|
173
|
+
else:
|
|
174
|
+
return "UNKNOWN"
|
|
146
175
|
|
|
147
176
|
def execute(self, args) -> bool:
|
|
148
177
|
# We can't have both `--remove-offset-and-limit` and `--limit`.
|
|
@@ -150,8 +179,13 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
150
179
|
log.error("Cannot have both --remove-offset-and-limit and --limit")
|
|
151
180
|
return False
|
|
152
181
|
|
|
153
|
-
# If `args.accept` is `application/sparql-results+json
|
|
154
|
-
|
|
182
|
+
# If `args.accept` is `application/sparql-results+json` or
|
|
183
|
+
# `application/qlever-results+json` or `AUTO`, we need `jq`.
|
|
184
|
+
if (
|
|
185
|
+
args.accept == "application/sparql-results+json"
|
|
186
|
+
or args.accept == "application/qlever-results+json"
|
|
187
|
+
or args.accept == "AUTO"
|
|
188
|
+
):
|
|
155
189
|
try:
|
|
156
190
|
subprocess.run(
|
|
157
191
|
"jq --version",
|
|
@@ -174,8 +208,9 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
174
208
|
return False
|
|
175
209
|
|
|
176
210
|
# Clear cache only works for QLever.
|
|
177
|
-
is_qlever =
|
|
178
|
-
|
|
211
|
+
is_qlever = (
|
|
212
|
+
not args.sparql_endpoint
|
|
213
|
+
or args.sparql_endpoint.startswith("https://qlever")
|
|
179
214
|
)
|
|
180
215
|
if args.clear_cache == "yes" and not is_qlever:
|
|
181
216
|
log.warning("Clearing the cache only works for QLever")
|
|
@@ -193,7 +228,9 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
193
228
|
if args.query_regex:
|
|
194
229
|
get_queries_cmd += f" | grep -Pi {shlex.quote(args.query_regex)}"
|
|
195
230
|
sparql_endpoint = (
|
|
196
|
-
args.sparql_endpoint
|
|
231
|
+
args.sparql_endpoint
|
|
232
|
+
if args.sparql_endpoint
|
|
233
|
+
else f"localhost:{args.port}"
|
|
197
234
|
)
|
|
198
235
|
self.show(
|
|
199
236
|
f"Obtain queries via: {get_queries_cmd}\n"
|
|
@@ -211,7 +248,9 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
211
248
|
|
|
212
249
|
# Get the example queries.
|
|
213
250
|
try:
|
|
214
|
-
example_query_lines = run_command(
|
|
251
|
+
example_query_lines = run_command(
|
|
252
|
+
get_queries_cmd, return_output=True
|
|
253
|
+
)
|
|
215
254
|
if len(example_query_lines) == 0:
|
|
216
255
|
log.error("No example queries matching the criteria found")
|
|
217
256
|
return False
|
|
@@ -220,6 +259,12 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
220
259
|
log.error(f"Failed to get example queries: {e}")
|
|
221
260
|
return False
|
|
222
261
|
|
|
262
|
+
# We want the width of the query description to be an uneven number (in
|
|
263
|
+
# case we have to truncated it, in which case we want to have a " ... "
|
|
264
|
+
# in the middle).
|
|
265
|
+
width_query_description_half = args.width_query_description // 2
|
|
266
|
+
width_query_description = 2 * width_query_description_half + 1
|
|
267
|
+
|
|
223
268
|
# Launch the queries one after the other and for each print: the
|
|
224
269
|
# description, the result size (number of rows), and the query
|
|
225
270
|
# processing time (seconds).
|
|
@@ -227,13 +272,16 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
227
272
|
result_sizes = []
|
|
228
273
|
num_failed = 0
|
|
229
274
|
for example_query_line in example_query_lines:
|
|
230
|
-
# Parse description and query.
|
|
275
|
+
# Parse description and query, and determine query type.
|
|
231
276
|
description, query = example_query_line.split("\t")
|
|
232
277
|
if len(query) == 0:
|
|
233
278
|
log.error("Could not parse description and query, line is:")
|
|
234
279
|
log.info("")
|
|
235
280
|
log.info(example_query_line)
|
|
236
281
|
return False
|
|
282
|
+
query_type = self.sparql_query_type(query)
|
|
283
|
+
if args.add_query_type_to_description or args.accept == "AUTO":
|
|
284
|
+
description = f"{description} [{query_type}]"
|
|
237
285
|
|
|
238
286
|
# Clear the cache.
|
|
239
287
|
if args.clear_cache == "yes":
|
|
@@ -267,7 +315,9 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
267
315
|
# Count query.
|
|
268
316
|
if args.download_or_count == "count":
|
|
269
317
|
# First find out if there is a FROM clause.
|
|
270
|
-
regex_from_clause = re.compile(
|
|
318
|
+
regex_from_clause = re.compile(
|
|
319
|
+
r"\s*FROM\s+<[^>]+>\s*", re.IGNORECASE
|
|
320
|
+
)
|
|
271
321
|
match_from_clause = re.search(regex_from_clause, query)
|
|
272
322
|
from_clause = " "
|
|
273
323
|
if match_from_clause:
|
|
@@ -296,24 +346,39 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
296
346
|
query = re.sub(r"\s*\.\s*\}", " }", query)
|
|
297
347
|
if args.show_query == "always":
|
|
298
348
|
log.info("")
|
|
299
|
-
|
|
349
|
+
log.info(
|
|
350
|
+
colored(
|
|
351
|
+
self.pretty_printed_query(query, args.show_prefixes),
|
|
352
|
+
"cyan",
|
|
353
|
+
)
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
# Accept header. For "AUTO", use `text/turtle` for CONSTRUCT
|
|
357
|
+
# queries and `application/sparql-results+json` for all others.
|
|
358
|
+
accept_header = args.accept
|
|
359
|
+
if accept_header == "AUTO":
|
|
360
|
+
if query_type == "CONSTRUCT" or query_type == "DESCRIBE":
|
|
361
|
+
accept_header = "text/turtle"
|
|
362
|
+
else:
|
|
363
|
+
accept_header = "application/sparql-results+json"
|
|
300
364
|
|
|
301
365
|
# Launch query.
|
|
302
366
|
try:
|
|
303
367
|
curl_cmd = (
|
|
304
368
|
f"curl -s {sparql_endpoint}"
|
|
305
369
|
f' -w "HTTP code: %{{http_code}}\\n"'
|
|
306
|
-
f' -H "Accept: {
|
|
370
|
+
f' -H "Accept: {accept_header}"'
|
|
307
371
|
f" --data-urlencode query={shlex.quote(query)}"
|
|
308
372
|
)
|
|
309
373
|
log.debug(curl_cmd)
|
|
310
374
|
result_file = (
|
|
311
|
-
f"qlever.example_queries.result."
|
|
375
|
+
f"qlever.example_queries.result."
|
|
376
|
+
f"{abs(hash(curl_cmd))}.tmp"
|
|
312
377
|
)
|
|
313
378
|
start_time = time.time()
|
|
314
379
|
http_code = run_curl_command(
|
|
315
380
|
sparql_endpoint,
|
|
316
|
-
headers={"Accept":
|
|
381
|
+
headers={"Accept": accept_header},
|
|
317
382
|
params={"query": query},
|
|
318
383
|
result_file=result_file,
|
|
319
384
|
).strip()
|
|
@@ -323,7 +388,9 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
323
388
|
else:
|
|
324
389
|
error_msg = {
|
|
325
390
|
"short": f"HTTP code: {http_code}",
|
|
326
|
-
"long": re.sub(
|
|
391
|
+
"long": re.sub(
|
|
392
|
+
r"\s+", " ", Path(result_file).read_text()
|
|
393
|
+
),
|
|
327
394
|
}
|
|
328
395
|
except Exception as e:
|
|
329
396
|
if args.log_level == "DEBUG":
|
|
@@ -336,8 +403,12 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
336
403
|
# Get result size (via the command line, in order to avoid loading
|
|
337
404
|
# a potentially large JSON file into Python, which is slow).
|
|
338
405
|
if error_msg is None:
|
|
339
|
-
# CASE 0:
|
|
340
|
-
|
|
406
|
+
# CASE 0: The result is empty despite a 200 HTTP code (not a
|
|
407
|
+
# problem for CONSTRUCT and DESCRIBE queries).
|
|
408
|
+
if Path(result_file).stat().st_size == 0 and (
|
|
409
|
+
not query_type == "CONSTRUCT"
|
|
410
|
+
and not query_type == "DESCRIBE"
|
|
411
|
+
):
|
|
341
412
|
result_size = 0
|
|
342
413
|
error_msg = {
|
|
343
414
|
"short": "Empty result",
|
|
@@ -347,7 +418,7 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
347
418
|
|
|
348
419
|
# CASE 1: Just counting the size of the result (TSV or JSON).
|
|
349
420
|
elif args.download_or_count == "count":
|
|
350
|
-
if
|
|
421
|
+
if accept_header == "text/tab-separated-values":
|
|
351
422
|
result_size = run_command(
|
|
352
423
|
f"sed 1d {result_file}", return_output=True
|
|
353
424
|
)
|
|
@@ -370,21 +441,28 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
370
441
|
# CASE 2: Downloading the full result (TSV, CSV, Turtle, JSON).
|
|
371
442
|
else:
|
|
372
443
|
if (
|
|
373
|
-
|
|
374
|
-
or
|
|
444
|
+
accept_header == "text/tab-separated-values"
|
|
445
|
+
or accept_header == "text/csv"
|
|
375
446
|
):
|
|
376
447
|
result_size = run_command(
|
|
377
448
|
f"sed 1d {result_file} | wc -l", return_output=True
|
|
378
449
|
)
|
|
379
|
-
elif
|
|
450
|
+
elif accept_header == "text/turtle":
|
|
380
451
|
result_size = run_command(
|
|
381
|
-
f"sed '1d;/^@prefix/d;/^\\s*$/d' "
|
|
452
|
+
f"sed '1d;/^@prefix/d;/^\\s*$/d' "
|
|
453
|
+
f"{result_file} | wc -l",
|
|
454
|
+
return_output=True,
|
|
455
|
+
)
|
|
456
|
+
elif accept_header == "application/qlever-results+json":
|
|
457
|
+
result_size = run_command(
|
|
458
|
+
f'jq -r ".resultsize" {result_file}',
|
|
382
459
|
return_output=True,
|
|
383
460
|
)
|
|
384
461
|
else:
|
|
385
462
|
try:
|
|
386
463
|
result_size = run_command(
|
|
387
|
-
f'jq -r ".results.bindings | length"'
|
|
464
|
+
f'jq -r ".results.bindings | length"'
|
|
465
|
+
f" {result_file}",
|
|
388
466
|
return_output=True,
|
|
389
467
|
)
|
|
390
468
|
except Exception as e:
|
|
@@ -398,13 +476,16 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
398
476
|
Path(result_file).unlink(missing_ok=True)
|
|
399
477
|
|
|
400
478
|
# Print description, time, result in tabular form.
|
|
401
|
-
if len(description) >
|
|
402
|
-
description =
|
|
403
|
-
|
|
479
|
+
if len(description) > width_query_description:
|
|
480
|
+
description = (
|
|
481
|
+
description[: width_query_description_half - 2]
|
|
482
|
+
+ " ... "
|
|
483
|
+
+ description[-width_query_description_half + 2 :]
|
|
484
|
+
)
|
|
404
485
|
if error_msg is None:
|
|
405
486
|
result_size = int(result_size)
|
|
406
487
|
log.info(
|
|
407
|
-
f"{description:<{
|
|
488
|
+
f"{description:<{width_query_description}} "
|
|
408
489
|
f"{time_seconds:6.2f} s "
|
|
409
490
|
f"{result_size:>{args.width_result_size},}"
|
|
410
491
|
)
|
|
@@ -419,18 +500,28 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
419
500
|
and args.show_query != "on-error"
|
|
420
501
|
):
|
|
421
502
|
error_msg["long"] = (
|
|
422
|
-
error_msg["long"][: args.width_error_message - 3]
|
|
503
|
+
error_msg["long"][: args.width_error_message - 3]
|
|
504
|
+
+ "..."
|
|
423
505
|
)
|
|
424
|
-
seperator_short_long =
|
|
506
|
+
seperator_short_long = (
|
|
507
|
+
"\n" if args.show_query == "on-error" else " "
|
|
508
|
+
)
|
|
425
509
|
log.info(
|
|
426
|
-
f"{description:<{
|
|
510
|
+
f"{description:<{width_query_description}} "
|
|
427
511
|
f"{colored('FAILED ', 'red')}"
|
|
428
512
|
f"{colored(error_msg['short'], 'red'):>{args.width_result_size}}"
|
|
429
513
|
f"{seperator_short_long}"
|
|
430
514
|
f"{colored(error_msg['long'], 'red')}"
|
|
431
515
|
)
|
|
432
516
|
if args.show_query == "on-error":
|
|
433
|
-
|
|
517
|
+
log.info(
|
|
518
|
+
colored(
|
|
519
|
+
self.pretty_printed_query(
|
|
520
|
+
query, args.show_prefixes
|
|
521
|
+
),
|
|
522
|
+
"cyan",
|
|
523
|
+
)
|
|
524
|
+
)
|
|
434
525
|
log.info("")
|
|
435
526
|
|
|
436
527
|
# Check that each query has a time and a result size, or it failed.
|
|
@@ -450,19 +541,19 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
450
541
|
description = f"TOTAL for {n} {query_or_queries}"
|
|
451
542
|
log.info("")
|
|
452
543
|
log.info(
|
|
453
|
-
f"{description:<{
|
|
544
|
+
f"{description:<{width_query_description}} "
|
|
454
545
|
f"{total_query_time:6.2f} s "
|
|
455
546
|
f"{total_result_size:>14,}"
|
|
456
547
|
)
|
|
457
548
|
description = f"AVERAGE for {n} {query_or_queries}"
|
|
458
549
|
log.info(
|
|
459
|
-
f"{description:<{
|
|
550
|
+
f"{description:<{width_query_description}} "
|
|
460
551
|
f"{average_query_time:6.2f} s "
|
|
461
552
|
f"{average_result_size:>14,}"
|
|
462
553
|
)
|
|
463
554
|
description = f"MEDIAN for {n} {query_or_queries}"
|
|
464
555
|
log.info(
|
|
465
|
-
f"{description:<{
|
|
556
|
+
f"{description:<{width_query_description}} "
|
|
466
557
|
f"{median_query_time:6.2f} s "
|
|
467
558
|
f"{median_result_size:>14,}"
|
|
468
559
|
)
|
|
@@ -476,7 +567,7 @@ class ExampleQueriesCommand(QleverCommand):
|
|
|
476
567
|
num_failed_string += " [all]"
|
|
477
568
|
log.info(
|
|
478
569
|
colored(
|
|
479
|
-
f"{description:<{
|
|
570
|
+
f"{description:<{width_query_description}} "
|
|
480
571
|
f"{num_failed:>24}",
|
|
481
572
|
"red",
|
|
482
573
|
)
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from qlever.command import QleverCommand
|
|
6
|
+
from qlever.log import log
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ExtractQueriesCommand(QleverCommand):
|
|
10
|
+
"""
|
|
11
|
+
Class for executing the `extract-queries` command.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
def __init__(self):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
def description(self) -> str:
|
|
18
|
+
return "Extract all SPARQL queries from the server log"
|
|
19
|
+
|
|
20
|
+
def should_have_qleverfile(self) -> bool:
|
|
21
|
+
return True
|
|
22
|
+
|
|
23
|
+
def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
|
|
24
|
+
return {"data": ["name"]}
|
|
25
|
+
|
|
26
|
+
def additional_arguments(self, subparser) -> None:
|
|
27
|
+
subparser.add_argument(
|
|
28
|
+
"--description-base",
|
|
29
|
+
type=str,
|
|
30
|
+
default="Log extract",
|
|
31
|
+
help="Base name for the query descriptions"
|
|
32
|
+
" (default: `Log extract`)",
|
|
33
|
+
)
|
|
34
|
+
subparser.add_argument(
|
|
35
|
+
"--log-file",
|
|
36
|
+
type=str,
|
|
37
|
+
help="Name of the log file to extract queries from"
|
|
38
|
+
" (default: `<name>.server-log.txt`)",
|
|
39
|
+
)
|
|
40
|
+
subparser.add_argument(
|
|
41
|
+
"--output-file",
|
|
42
|
+
type=str,
|
|
43
|
+
default="log-queries.txt",
|
|
44
|
+
help="Output file for the extracted queries (default: `log-queries.txt`)",
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
def execute(self, args) -> bool:
|
|
48
|
+
# Show what the command does.
|
|
49
|
+
if args.log_file is not None:
|
|
50
|
+
log_file_name = args.log_file
|
|
51
|
+
else:
|
|
52
|
+
log_file_name = f"{args.name}.server-log.txt"
|
|
53
|
+
self.show(
|
|
54
|
+
f"Extract SPARQL queries from `{log_file_name}`"
|
|
55
|
+
f" and write them to `{args.output_file}`",
|
|
56
|
+
only_show=args.show,
|
|
57
|
+
)
|
|
58
|
+
if args.show:
|
|
59
|
+
return True
|
|
60
|
+
|
|
61
|
+
# Regex for log entries of the form
|
|
62
|
+
# 2025-01-14 04:47:44.950 - INFO
|
|
63
|
+
log_line_regex = (
|
|
64
|
+
r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - [A-Z]+:"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
# Read the log file line by line.
|
|
68
|
+
log_file = open(log_file_name, "r")
|
|
69
|
+
queries_file = open(args.output_file, "w")
|
|
70
|
+
query = None
|
|
71
|
+
description_base = args.description_base
|
|
72
|
+
description_base_count = {}
|
|
73
|
+
tsv_line_short_width = 150
|
|
74
|
+
for line in log_file:
|
|
75
|
+
# An "Alive check" message contains a tag, which we use as the base
|
|
76
|
+
# name of the query description.
|
|
77
|
+
alive_check_regex = r"Alive check with message \"(.*)\""
|
|
78
|
+
match = re.search(alive_check_regex, line)
|
|
79
|
+
if match:
|
|
80
|
+
description_base = match.group(1)
|
|
81
|
+
continue
|
|
82
|
+
|
|
83
|
+
# A new query in the log.
|
|
84
|
+
if "Processing the following SPARQL query" in line:
|
|
85
|
+
query = []
|
|
86
|
+
query_index = (
|
|
87
|
+
description_base_count.get(description_base, 0) + 1
|
|
88
|
+
)
|
|
89
|
+
description_base_count[description_base] = query_index
|
|
90
|
+
continue
|
|
91
|
+
# If we have started a query: extend until we meet the next log
|
|
92
|
+
# line, then push the query. Remove comments.
|
|
93
|
+
if query is not None:
|
|
94
|
+
if not re.match(log_line_regex, line):
|
|
95
|
+
if not re.match(r"^\s*#", line):
|
|
96
|
+
line = re.sub(r" #.*", "", line)
|
|
97
|
+
query.append(line)
|
|
98
|
+
else:
|
|
99
|
+
query = re.sub(r"\s+", " ", "\n".join(query)).strip()
|
|
100
|
+
description = f"{description_base}, Query #{query_index}"
|
|
101
|
+
tsv_line = f"{description}\t{query}"
|
|
102
|
+
tsv_line_short = (
|
|
103
|
+
tsv_line
|
|
104
|
+
if len(tsv_line) < tsv_line_short_width
|
|
105
|
+
else tsv_line[:tsv_line_short_width] + "..."
|
|
106
|
+
)
|
|
107
|
+
log.info(tsv_line_short)
|
|
108
|
+
print(tsv_line, file=queries_file)
|
|
109
|
+
query = None
|
|
110
|
+
|
|
111
|
+
log_file.close()
|
|
112
|
+
queries_file.close()
|
|
113
|
+
return True
|
qlever/commands/index.py
CHANGED
|
@@ -147,7 +147,7 @@ class IndexCommand(QleverCommand):
|
|
|
147
147
|
raise self.InvalidInputJson(
|
|
148
148
|
f"Element {i} in `MULTI_INPUT_JSON` must only contain "
|
|
149
149
|
"the keys `format`, `graph`, and `parallel`. Contains "
|
|
150
|
-
"extra keys {extra_keys}.",
|
|
150
|
+
f"extra keys {extra_keys}.",
|
|
151
151
|
input_spec,
|
|
152
152
|
)
|
|
153
153
|
# Add the command-line options for this input stream. We use
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
from termcolor import colored
|
|
6
|
+
|
|
7
|
+
from qlever.command import QleverCommand
|
|
8
|
+
from qlever.log import log
|
|
9
|
+
from qlever.util import run_command
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class SettingsCommand(QleverCommand):
|
|
13
|
+
"""
|
|
14
|
+
Class for executing the `settings` command.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self):
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
def description(self) -> str:
|
|
21
|
+
return "Show or set server settings (after `qlever start`)"
|
|
22
|
+
|
|
23
|
+
def should_have_qleverfile(self) -> bool:
|
|
24
|
+
return True
|
|
25
|
+
|
|
26
|
+
def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
|
|
27
|
+
return {"server": ["port", "host_name", "access_token"]}
|
|
28
|
+
|
|
29
|
+
def additional_arguments(self, subparser) -> None:
|
|
30
|
+
all_keys = [
|
|
31
|
+
"always-multiply-unions",
|
|
32
|
+
"cache-max-num-entries",
|
|
33
|
+
"cache-max-size",
|
|
34
|
+
"cache-max-size-single-entry",
|
|
35
|
+
"default-query-timeout",
|
|
36
|
+
"group-by-disable-index-scan-optimizations",
|
|
37
|
+
"group-by-hash-map-enabled",
|
|
38
|
+
"lazy-index-scan-max-size-materialization",
|
|
39
|
+
"lazy-index-scan-num-threads",
|
|
40
|
+
"lazy-index-scan-queue-size",
|
|
41
|
+
"lazy-result-max-cache-size",
|
|
42
|
+
"query-planning-budget",
|
|
43
|
+
"service-max-value-rows",
|
|
44
|
+
"sort-estimate-cancellation-factor",
|
|
45
|
+
"throw-on-unbound-variables",
|
|
46
|
+
"use-binsearch-transitive-path",
|
|
47
|
+
]
|
|
48
|
+
subparser.add_argument(
|
|
49
|
+
"runtime_parameter",
|
|
50
|
+
nargs="?",
|
|
51
|
+
help="Set the given runtime parameter (key=value)"
|
|
52
|
+
"; if no argument is given, show all settings",
|
|
53
|
+
).completer = lambda **kwargs: [f"{key}=" for key in all_keys]
|
|
54
|
+
subparser.add_argument(
|
|
55
|
+
"--endpoint_url",
|
|
56
|
+
type=str,
|
|
57
|
+
help="An arbitrary endpoint URL "
|
|
58
|
+
"(overriding the one in the Qleverfile)",
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
def execute(self, args) -> bool:
|
|
62
|
+
# Get endpoint URL from command line or Qleverfile.
|
|
63
|
+
if args.endpoint_url:
|
|
64
|
+
endpoint_url = args.endpoint_url
|
|
65
|
+
else:
|
|
66
|
+
endpoint_url = f"http://{args.host_name}:{args.port}"
|
|
67
|
+
|
|
68
|
+
# Construct the `curl` command for getting or setting.
|
|
69
|
+
if args.runtime_parameter:
|
|
70
|
+
try:
|
|
71
|
+
parameter_key, parameter_value = args.runtime_parameter.split(
|
|
72
|
+
"="
|
|
73
|
+
)
|
|
74
|
+
except ValueError:
|
|
75
|
+
log.error("Runtime parameter must be given as `key=value`")
|
|
76
|
+
return False
|
|
77
|
+
|
|
78
|
+
curl_cmd = (
|
|
79
|
+
f"curl -s {endpoint_url}"
|
|
80
|
+
f' --data-urlencode "{parameter_key}={parameter_value}"'
|
|
81
|
+
f' --data-urlencode "access-token={args.access_token}"'
|
|
82
|
+
)
|
|
83
|
+
else:
|
|
84
|
+
curl_cmd = (
|
|
85
|
+
f"curl -s {endpoint_url}" f" --data-urlencode cmd=get-settings"
|
|
86
|
+
)
|
|
87
|
+
parameter_key, parameter_value = None, None
|
|
88
|
+
self.show(curl_cmd, only_show=args.show)
|
|
89
|
+
if args.show:
|
|
90
|
+
return True
|
|
91
|
+
|
|
92
|
+
# Execute the `curl` command. Note that the `get-settings` command
|
|
93
|
+
# returns all settings in both scencarios (that is, also when setting a
|
|
94
|
+
# parameter).
|
|
95
|
+
try:
|
|
96
|
+
settings_json = run_command(curl_cmd, return_output=True)
|
|
97
|
+
settings_dict = json.loads(settings_json)
|
|
98
|
+
except Exception as e:
|
|
99
|
+
log.error(f"setting command failed: {e}")
|
|
100
|
+
return False
|
|
101
|
+
for key, value in settings_dict.items():
|
|
102
|
+
print(
|
|
103
|
+
colored(
|
|
104
|
+
f"{key:<45}: {value}",
|
|
105
|
+
"blue"
|
|
106
|
+
if parameter_key and key == parameter_key
|
|
107
|
+
else None,
|
|
108
|
+
)
|
|
109
|
+
)
|
|
110
|
+
return True
|
qlever/commands/start.py
CHANGED
|
@@ -13,6 +13,110 @@ from qlever.log import log
|
|
|
13
13
|
from qlever.util import is_qlever_server_alive, run_command
|
|
14
14
|
|
|
15
15
|
|
|
16
|
+
# Construct the command line based on the config file.
|
|
17
|
+
def construct_command(args) -> str:
|
|
18
|
+
start_cmd = (
|
|
19
|
+
f"{args.server_binary}"
|
|
20
|
+
f" -i {args.name}"
|
|
21
|
+
f" -j {args.num_threads}"
|
|
22
|
+
f" -p {args.port}"
|
|
23
|
+
f" -m {args.memory_for_queries}"
|
|
24
|
+
f" -c {args.cache_max_size}"
|
|
25
|
+
f" -e {args.cache_max_size_single_entry}"
|
|
26
|
+
f" -k {args.cache_max_num_entries}"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
if args.timeout:
|
|
30
|
+
start_cmd += f" -s {args.timeout}"
|
|
31
|
+
if args.access_token:
|
|
32
|
+
start_cmd += f" -a {args.access_token}"
|
|
33
|
+
if args.only_pso_and_pos_permutations:
|
|
34
|
+
start_cmd += " --only-pso-and-pos-permutations"
|
|
35
|
+
if not args.use_patterns:
|
|
36
|
+
start_cmd += " --no-patterns"
|
|
37
|
+
if args.use_text_index == "yes":
|
|
38
|
+
start_cmd += " -t"
|
|
39
|
+
start_cmd += f" > {args.name}.server-log.txt 2>&1"
|
|
40
|
+
return start_cmd
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# Kill existing server on the same port. Trust that StopCommand() works?
|
|
44
|
+
# Maybe return StopCommand().execute(args) and handle it with a try except?
|
|
45
|
+
def kill_existing_server(args) -> bool:
|
|
46
|
+
args.cmdline_regex = f"^ServerMain.* -p {args.port}"
|
|
47
|
+
args.no_containers = True
|
|
48
|
+
if not StopCommand().execute(args):
|
|
49
|
+
log.error("Stopping the existing server failed")
|
|
50
|
+
return False
|
|
51
|
+
log.info("")
|
|
52
|
+
return True
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# Run the command in a container
|
|
56
|
+
def wrap_command_in_container(args, start_cmd) -> str:
|
|
57
|
+
if not args.server_container:
|
|
58
|
+
args.server_container = f"qlever.server.{args.name}"
|
|
59
|
+
start_cmd = Containerize().containerize_command(
|
|
60
|
+
start_cmd,
|
|
61
|
+
args.system,
|
|
62
|
+
"run -d --restart=unless-stopped",
|
|
63
|
+
args.image,
|
|
64
|
+
args.server_container,
|
|
65
|
+
volumes=[("$(pwd)", "/index")],
|
|
66
|
+
ports=[(args.port, args.port)],
|
|
67
|
+
working_directory="/index",
|
|
68
|
+
)
|
|
69
|
+
return start_cmd
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
# When running natively, check if the binary exists and works.
|
|
73
|
+
def check_binary(binary) -> bool:
|
|
74
|
+
try:
|
|
75
|
+
run_command(f"{binary} --help")
|
|
76
|
+
return True
|
|
77
|
+
except Exception as e:
|
|
78
|
+
log.error(
|
|
79
|
+
f'Running "{binary}" failed, '
|
|
80
|
+
f"set `--server-binary` to a different binary or "
|
|
81
|
+
f"set `--system to a container system`"
|
|
82
|
+
)
|
|
83
|
+
log.info("")
|
|
84
|
+
log.info(f"The error message was: {e}")
|
|
85
|
+
return False
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# Set the index description.
|
|
89
|
+
def set_index_description(access_arg, port, desc) -> bool:
|
|
90
|
+
curl_cmd = (
|
|
91
|
+
f"curl -Gs http://localhost:{port}/api"
|
|
92
|
+
f' --data-urlencode "index-description={desc}"'
|
|
93
|
+
f" {access_arg} > /dev/null"
|
|
94
|
+
)
|
|
95
|
+
log.debug(curl_cmd)
|
|
96
|
+
try:
|
|
97
|
+
run_command(curl_cmd)
|
|
98
|
+
except Exception as e:
|
|
99
|
+
log.error(f"Setting the index description failed ({e})")
|
|
100
|
+
return False
|
|
101
|
+
return True
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# Set the text description.
|
|
105
|
+
def set_text_description(access_arg, port, text_desc) -> bool:
|
|
106
|
+
curl_cmd = (
|
|
107
|
+
f"curl -Gs http://localhost:{port}/api"
|
|
108
|
+
f' --data-urlencode "text-description={text_desc}"'
|
|
109
|
+
f" {access_arg} > /dev/null"
|
|
110
|
+
)
|
|
111
|
+
log.debug(curl_cmd)
|
|
112
|
+
try:
|
|
113
|
+
run_command(curl_cmd)
|
|
114
|
+
except Exception as e:
|
|
115
|
+
log.error(f"Setting the text description failed ({e})")
|
|
116
|
+
return False
|
|
117
|
+
return True
|
|
118
|
+
|
|
119
|
+
|
|
16
120
|
class StartCommand(QleverCommand):
|
|
17
121
|
"""
|
|
18
122
|
Class for executing the `start` command.
|
|
@@ -22,22 +126,35 @@ class StartCommand(QleverCommand):
|
|
|
22
126
|
pass
|
|
23
127
|
|
|
24
128
|
def description(self) -> str:
|
|
25
|
-
return (
|
|
26
|
-
|
|
129
|
+
return (
|
|
130
|
+
"Start the QLever server (requires that you have built "
|
|
131
|
+
"an index with `qlever index` before)"
|
|
132
|
+
)
|
|
27
133
|
|
|
28
134
|
def should_have_qleverfile(self) -> bool:
|
|
29
135
|
return True
|
|
30
136
|
|
|
31
|
-
def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
|
|
32
|
-
return {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
"
|
|
137
|
+
def relevant_qleverfile_arguments(self) -> dict[str : list[str]]:
|
|
138
|
+
return {
|
|
139
|
+
"data": ["name", "description", "text_description"],
|
|
140
|
+
"server": [
|
|
141
|
+
"server_binary",
|
|
142
|
+
"host_name",
|
|
143
|
+
"port",
|
|
144
|
+
"access_token",
|
|
145
|
+
"memory_for_queries",
|
|
146
|
+
"cache_max_size",
|
|
147
|
+
"cache_max_size_single_entry",
|
|
148
|
+
"cache_max_num_entries",
|
|
149
|
+
"num_threads",
|
|
150
|
+
"timeout",
|
|
151
|
+
"only_pso_and_pos_permutations",
|
|
152
|
+
"use_patterns",
|
|
153
|
+
"use_text_index",
|
|
154
|
+
"warmup_cmd",
|
|
155
|
+
],
|
|
156
|
+
"runtime": ["system", "image", "server_container"],
|
|
157
|
+
}
|
|
41
158
|
|
|
42
159
|
def additional_arguments(self, subparser) -> None:
|
|
43
160
|
# subparser.add_argument("--kill-existing-with-same-name",
|
|
@@ -46,16 +163,27 @@ class StartCommand(QleverCommand):
|
|
|
46
163
|
# help="If a QLever server is already running "
|
|
47
164
|
# "with the same name, kill it before "
|
|
48
165
|
# "starting a new server")
|
|
49
|
-
subparser.add_argument(
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
166
|
+
subparser.add_argument(
|
|
167
|
+
"--kill-existing-with-same-port",
|
|
168
|
+
action="store_true",
|
|
169
|
+
default=False,
|
|
170
|
+
help="If a QLever server is already running "
|
|
171
|
+
"on the same port, kill it before "
|
|
172
|
+
"starting a new server",
|
|
173
|
+
)
|
|
174
|
+
subparser.add_argument(
|
|
175
|
+
"--no-warmup",
|
|
176
|
+
action="store_true",
|
|
177
|
+
default=False,
|
|
178
|
+
help="Do not execute the warmup command",
|
|
179
|
+
)
|
|
180
|
+
subparser.add_argument(
|
|
181
|
+
"--run-in-foreground",
|
|
182
|
+
action="store_true",
|
|
183
|
+
default=False,
|
|
184
|
+
help="Run the server in the foreground "
|
|
185
|
+
"(default: run in the background with `nohup`)",
|
|
186
|
+
)
|
|
59
187
|
|
|
60
188
|
def execute(self, args) -> bool:
|
|
61
189
|
# Kill existing server with the same name if so desired.
|
|
@@ -70,47 +198,21 @@ class StartCommand(QleverCommand):
|
|
|
70
198
|
|
|
71
199
|
# Kill existing server on the same port if so desired.
|
|
72
200
|
if args.kill_existing_with_same_port:
|
|
73
|
-
args.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
log.error("Stopping the existing server failed")
|
|
201
|
+
if args.kill_existing_with_same_port and not kill_existing_server(
|
|
202
|
+
args
|
|
203
|
+
):
|
|
77
204
|
return False
|
|
78
|
-
log.info("")
|
|
79
205
|
|
|
80
206
|
# Construct the command line based on the config file.
|
|
81
|
-
start_cmd = (
|
|
82
|
-
f" -i {args.name}"
|
|
83
|
-
f" -j {args.num_threads}"
|
|
84
|
-
f" -p {args.port}"
|
|
85
|
-
f" -m {args.memory_for_queries}"
|
|
86
|
-
f" -c {args.cache_max_size}"
|
|
87
|
-
f" -e {args.cache_max_size_single_entry}"
|
|
88
|
-
f" -k {args.cache_max_num_entries}")
|
|
89
|
-
if args.timeout:
|
|
90
|
-
start_cmd += f" -s {args.timeout}"
|
|
91
|
-
if args.access_token:
|
|
92
|
-
start_cmd += f" -a {args.access_token}"
|
|
93
|
-
if args.only_pso_and_pos_permutations:
|
|
94
|
-
start_cmd += " --only-pso-and-pos-permutations"
|
|
95
|
-
if not args.use_patterns:
|
|
96
|
-
start_cmd += " --no-patterns"
|
|
97
|
-
if args.use_text_index == "yes":
|
|
98
|
-
start_cmd += " -t"
|
|
99
|
-
start_cmd += f" > {args.name}.server-log.txt 2>&1"
|
|
207
|
+
start_cmd = construct_command(args)
|
|
100
208
|
|
|
101
209
|
# Run the command in a container (if so desired). Otherwise run with
|
|
102
|
-
# `nohup` so that it keeps running after the shell is closed.
|
|
210
|
+
# `nohup` so that it keeps running after the shell is closed. With
|
|
211
|
+
# `--run-in-foreground`, run the server in the foreground.
|
|
103
212
|
if args.system in Containerize.supported_systems():
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
start_cmd =
|
|
107
|
-
start_cmd,
|
|
108
|
-
args.system, "run -d --restart=unless-stopped",
|
|
109
|
-
args.image,
|
|
110
|
-
args.server_container,
|
|
111
|
-
volumes=[("$(pwd)", "/index")],
|
|
112
|
-
ports=[(args.port, args.port)],
|
|
113
|
-
working_directory="/index")
|
|
213
|
+
start_cmd = wrap_command_in_container(args, start_cmd)
|
|
214
|
+
elif args.run_in_foreground:
|
|
215
|
+
start_cmd = f"{start_cmd}"
|
|
114
216
|
else:
|
|
115
217
|
start_cmd = f"nohup {start_cmd} &"
|
|
116
218
|
|
|
@@ -121,35 +223,32 @@ class StartCommand(QleverCommand):
|
|
|
121
223
|
|
|
122
224
|
# When running natively, check if the binary exists and works.
|
|
123
225
|
if args.system == "native":
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
except Exception as e:
|
|
127
|
-
log.error(f"Running \"{args.server_binary}\" failed, "
|
|
128
|
-
f"set `--server-binary` to a different binary or "
|
|
129
|
-
f"set `--system to a container system`")
|
|
130
|
-
log.info("")
|
|
131
|
-
log.info(f"The error message was: {e}")
|
|
226
|
+
ret = check_binary(args.server_binary)
|
|
227
|
+
if not ret:
|
|
132
228
|
return False
|
|
133
229
|
|
|
134
230
|
# Check if a QLever server is already running on this port.
|
|
135
|
-
|
|
136
|
-
if is_qlever_server_alive(
|
|
137
|
-
log.error(f"QLever server already running on
|
|
231
|
+
endpoint_url = f"http://localhost:{args.port}"
|
|
232
|
+
if is_qlever_server_alive(endpoint_url):
|
|
233
|
+
log.error(f"QLever server already running on {endpoint_url}")
|
|
138
234
|
log.info("")
|
|
139
|
-
log.info(
|
|
140
|
-
|
|
141
|
-
|
|
235
|
+
log.info(
|
|
236
|
+
"To kill the existing server, use `qlever stop` "
|
|
237
|
+
"or `qlever start` with option "
|
|
238
|
+
"--kill-existing-with-same-port`"
|
|
239
|
+
)
|
|
142
240
|
|
|
143
241
|
# Show output of status command.
|
|
144
|
-
args.cmdline_regex = f"^ServerMain.* -p *{port}"
|
|
242
|
+
args.cmdline_regex = f"^ServerMain.* -p *{args.port}"
|
|
145
243
|
log.info("")
|
|
146
244
|
StatusCommand().execute(args)
|
|
147
|
-
|
|
148
245
|
return False
|
|
149
246
|
|
|
150
247
|
# Remove already existing container.
|
|
151
|
-
if
|
|
152
|
-
|
|
248
|
+
if (
|
|
249
|
+
args.system in Containerize.supported_systems()
|
|
250
|
+
and args.kill_existing_with_same_port
|
|
251
|
+
):
|
|
153
252
|
try:
|
|
154
253
|
run_command(f"{args.system} rm -f {args.server_container}")
|
|
155
254
|
except Exception as e:
|
|
@@ -166,7 +265,10 @@ class StartCommand(QleverCommand):
|
|
|
166
265
|
|
|
167
266
|
# Execute the command line.
|
|
168
267
|
try:
|
|
169
|
-
run_command(
|
|
268
|
+
process = run_command(
|
|
269
|
+
start_cmd,
|
|
270
|
+
use_popen=args.run_in_foreground,
|
|
271
|
+
)
|
|
170
272
|
except Exception as e:
|
|
171
273
|
log.error(f"Starting the QLever server failed ({e})")
|
|
172
274
|
return False
|
|
@@ -174,41 +276,40 @@ class StartCommand(QleverCommand):
|
|
|
174
276
|
# Tail the server log until the server is ready (note that the `exec`
|
|
175
277
|
# is important to make sure that the tail process is killed and not
|
|
176
278
|
# just the bash process).
|
|
177
|
-
|
|
178
|
-
|
|
279
|
+
if args.run_in_foreground:
|
|
280
|
+
log.info(
|
|
281
|
+
f"Follow {args.name}.server-log.txt as long as the server is"
|
|
282
|
+
f" running (Ctrl-C stops the server)"
|
|
283
|
+
)
|
|
284
|
+
else:
|
|
285
|
+
log.info(
|
|
286
|
+
f"Follow {args.name}.server-log.txt until the server is ready"
|
|
287
|
+
f" (Ctrl-C stops following the log, but NOT the server)"
|
|
288
|
+
)
|
|
179
289
|
log.info("")
|
|
180
290
|
tail_cmd = f"exec tail -f {args.name}.server-log.txt"
|
|
181
291
|
tail_proc = subprocess.Popen(tail_cmd, shell=True)
|
|
182
|
-
while not is_qlever_server_alive(
|
|
292
|
+
while not is_qlever_server_alive(endpoint_url):
|
|
183
293
|
time.sleep(1)
|
|
184
294
|
|
|
185
|
-
# Set the
|
|
186
|
-
access_arg = f
|
|
295
|
+
# Set the description for the index and text.
|
|
296
|
+
access_arg = f'--data-urlencode "access-token={args.access_token}"'
|
|
187
297
|
if args.description:
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
log.debug(curl_cmd)
|
|
193
|
-
try:
|
|
194
|
-
run_command(curl_cmd)
|
|
195
|
-
except Exception as e:
|
|
196
|
-
log.error(f"Setting the index description failed ({e})")
|
|
298
|
+
ret = set_index_description(
|
|
299
|
+
access_arg, args.port, args.description
|
|
300
|
+
)
|
|
301
|
+
if not ret:
|
|
197
302
|
return False
|
|
198
303
|
if args.text_description:
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
log.debug(curl_cmd)
|
|
204
|
-
try:
|
|
205
|
-
run_command(curl_cmd)
|
|
206
|
-
except Exception as e:
|
|
207
|
-
log.error(f"Setting the text description failed ({e})")
|
|
304
|
+
ret = set_text_description(
|
|
305
|
+
access_arg, args.port, args.text_description
|
|
306
|
+
)
|
|
307
|
+
if not ret:
|
|
208
308
|
return False
|
|
209
309
|
|
|
210
310
|
# Kill the tail process. NOTE: `tail_proc.kill()` does not work.
|
|
211
|
-
|
|
311
|
+
if not args.run_in_foreground:
|
|
312
|
+
tail_proc.terminate()
|
|
212
313
|
|
|
213
314
|
# Execute the warmup command.
|
|
214
315
|
if args.warmup_cmd and not args.no_warmup:
|
|
@@ -218,8 +319,18 @@ class StartCommand(QleverCommand):
|
|
|
218
319
|
return False
|
|
219
320
|
|
|
220
321
|
# Show cache stats.
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
322
|
+
if not args.run_in_foreground:
|
|
323
|
+
log.info("")
|
|
324
|
+
args.detailed = False
|
|
325
|
+
args.server_url = None
|
|
326
|
+
CacheStatsCommand().execute(args)
|
|
327
|
+
|
|
328
|
+
# With `--run-in-foreground`, wait until the server is stopped.
|
|
329
|
+
if args.run_in_foreground:
|
|
330
|
+
try:
|
|
331
|
+
process.wait()
|
|
332
|
+
except KeyboardInterrupt:
|
|
333
|
+
process.terminate()
|
|
334
|
+
tail_proc.terminate()
|
|
335
|
+
|
|
225
336
|
return True
|
qlever/commands/stop.py
CHANGED
|
@@ -1,15 +1,40 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
-
|
|
3
2
|
import re
|
|
4
|
-
|
|
5
3
|
import psutil
|
|
6
|
-
|
|
7
4
|
from qlever.command import QleverCommand
|
|
8
5
|
from qlever.commands.status import StatusCommand
|
|
9
6
|
from qlever.containerize import Containerize
|
|
10
7
|
from qlever.log import log
|
|
11
8
|
from qlever.util import show_process_info
|
|
12
9
|
|
|
10
|
+
# try to kill the given process, return true iff it was killed successfully.
|
|
11
|
+
# the process_info is used for logging.
|
|
12
|
+
def stop_process(proc, pinfo):
|
|
13
|
+
try:
|
|
14
|
+
proc.kill()
|
|
15
|
+
log.info(f"Killed process {pinfo['pid']}")
|
|
16
|
+
return True
|
|
17
|
+
except Exception as e:
|
|
18
|
+
log.error(f"Could not kill process with PID "
|
|
19
|
+
f"{pinfo['pid']} ({e}) ... try to kill it "
|
|
20
|
+
f"manually")
|
|
21
|
+
log.info("")
|
|
22
|
+
show_process_info(proc, "", show_heading=True)
|
|
23
|
+
return False
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# try to stop and remove container. return True iff it was stopped
|
|
27
|
+
# successfully. Gives log info accordingly.
|
|
28
|
+
def stop_container(server_container):
|
|
29
|
+
for container_system in Containerize.supported_systems():
|
|
30
|
+
if Containerize.stop_and_remove_container(
|
|
31
|
+
container_system, server_container):
|
|
32
|
+
log.info(f"{container_system.capitalize()} container with "
|
|
33
|
+
f"name \"{server_container}\" stopped "
|
|
34
|
+
f" and removed")
|
|
35
|
+
return True
|
|
36
|
+
return False
|
|
37
|
+
|
|
13
38
|
|
|
14
39
|
class StopCommand(QleverCommand):
|
|
15
40
|
"""
|
|
@@ -20,7 +45,7 @@ class StopCommand(QleverCommand):
|
|
|
20
45
|
pass
|
|
21
46
|
|
|
22
47
|
def description(self) -> str:
|
|
23
|
-
return
|
|
48
|
+
return "Stop QLever server for a given datasedataset or port"
|
|
24
49
|
|
|
25
50
|
def should_have_qleverfile(self) -> bool:
|
|
26
51
|
return True
|
|
@@ -54,46 +79,34 @@ class StopCommand(QleverCommand):
|
|
|
54
79
|
# First check if there is container running and if yes, stop and remove
|
|
55
80
|
# it (unless the user has specified `--no-containers`).
|
|
56
81
|
if not args.no_containers:
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
container_system, args.server_container):
|
|
60
|
-
log.info(f"{container_system.capitalize()} container with "
|
|
61
|
-
f"name \"{args.server_container}\" stopped "
|
|
62
|
-
f" and removed")
|
|
63
|
-
return True
|
|
82
|
+
if stop_container(args.server_container):
|
|
83
|
+
return True
|
|
64
84
|
|
|
65
85
|
# Check if there is a process running on the server port using psutil.
|
|
66
|
-
#
|
|
67
86
|
# NOTE: On MacOS, some of the proc's returned by psutil.process_iter()
|
|
68
87
|
# no longer exist when we try to access them, so we just skip them.
|
|
88
|
+
stop_process_results = []
|
|
69
89
|
for proc in psutil.process_iter():
|
|
70
90
|
try:
|
|
71
91
|
pinfo = proc.as_dict(
|
|
72
|
-
|
|
73
|
-
|
|
92
|
+
attrs=['pid', 'username', 'create_time',
|
|
93
|
+
'memory_info', 'cmdline'])
|
|
74
94
|
cmdline = " ".join(pinfo['cmdline'])
|
|
75
95
|
except Exception as e:
|
|
76
96
|
log.debug(f"Error getting process info: {e}")
|
|
97
|
+
return False
|
|
77
98
|
if re.search(cmdline_regex, cmdline):
|
|
78
99
|
log.info(f"Found process {pinfo['pid']} from user "
|
|
79
100
|
f"{pinfo['username']} with command line: {cmdline}")
|
|
80
101
|
log.info("")
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
except Exception as e:
|
|
85
|
-
log.error(f"Could not kill process with PID "
|
|
86
|
-
f"{pinfo['pid']} ({e}) ... try to kill it "
|
|
87
|
-
f"manually")
|
|
88
|
-
log.info("")
|
|
89
|
-
show_process_info(proc, "", show_heading=True)
|
|
90
|
-
return False
|
|
91
|
-
return True
|
|
102
|
+
stop_process_results.append(stop_process(proc, pinfo))
|
|
103
|
+
if len(stop_process_results) > 0:
|
|
104
|
+
return all(stop_process_results)
|
|
92
105
|
|
|
93
106
|
# If no matching process found, show a message and the output of the
|
|
94
107
|
# status command.
|
|
95
108
|
message = "No matching process found" if args.no_containers else \
|
|
96
|
-
|
|
109
|
+
"No matching process or container found"
|
|
97
110
|
log.error(message)
|
|
98
111
|
args.cmdline_regex = "^ServerMain.* -i [^ ]*"
|
|
99
112
|
log.info("")
|
qlever/util.py
CHANGED
|
@@ -3,9 +3,9 @@ from __future__ import annotations
|
|
|
3
3
|
import errno
|
|
4
4
|
import re
|
|
5
5
|
import secrets
|
|
6
|
-
import socket
|
|
7
6
|
import shlex
|
|
8
7
|
import shutil
|
|
8
|
+
import socket
|
|
9
9
|
import string
|
|
10
10
|
import subprocess
|
|
11
11
|
from datetime import date, datetime
|
|
@@ -30,8 +30,11 @@ def get_total_file_size(patterns: list[str]) -> int:
|
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
def run_command(
|
|
33
|
-
cmd: str,
|
|
34
|
-
|
|
33
|
+
cmd: str,
|
|
34
|
+
return_output: bool = False,
|
|
35
|
+
show_output: bool = False,
|
|
36
|
+
use_popen: bool = False,
|
|
37
|
+
) -> Optional[str | subprocess.Popen]:
|
|
35
38
|
"""
|
|
36
39
|
Run the given command and throw an exception if the exit code is non-zero.
|
|
37
40
|
If `return_output` is `True`, return what the command wrote to `stdout`.
|
|
@@ -41,6 +44,7 @@ def run_command(
|
|
|
41
44
|
|
|
42
45
|
TODO: Find the executable for `bash` in `__init__.py`.
|
|
43
46
|
"""
|
|
47
|
+
|
|
44
48
|
subprocess_args = {
|
|
45
49
|
"executable": shutil.which("bash"),
|
|
46
50
|
"shell": True,
|
|
@@ -48,17 +52,21 @@ def run_command(
|
|
|
48
52
|
"stdout": None if show_output else subprocess.PIPE,
|
|
49
53
|
"stderr": subprocess.PIPE,
|
|
50
54
|
}
|
|
55
|
+
|
|
56
|
+
# With `Popen`, the command runs in the current shell and a process object
|
|
57
|
+
# is returned (which can be used, e.g., to kill the process).
|
|
58
|
+
if use_popen:
|
|
59
|
+
if return_output:
|
|
60
|
+
raise Exception("Cannot return output if `use_popen` is `True`")
|
|
61
|
+
return subprocess.Popen(f"set -o pipefail; {cmd}", **subprocess_args)
|
|
62
|
+
|
|
63
|
+
# With `run`, the command runs in a subshell and the output is captured.
|
|
51
64
|
result = subprocess.run(f"set -o pipefail; {cmd}", **subprocess_args)
|
|
65
|
+
|
|
52
66
|
# If the exit code is non-zero, throw an exception. If something was
|
|
53
67
|
# written to `stderr`, use that as the exception message. Otherwise, use a
|
|
54
68
|
# generic message (which is also what `subprocess.run` does with
|
|
55
69
|
# `check=True`).
|
|
56
|
-
# log.debug(f"Command `{cmd}` returned the following result")
|
|
57
|
-
# log.debug("")
|
|
58
|
-
# log.debug(f"exit code: {result.returncode}")
|
|
59
|
-
# log.debug(f"stdout: {result.stdout}")
|
|
60
|
-
# log.debug(f"stderr: {result.stderr}")
|
|
61
|
-
# log.debug("")
|
|
62
70
|
if result.returncode != 0:
|
|
63
71
|
if len(result.stderr) > 0:
|
|
64
72
|
raise Exception(result.stderr.replace("\n", " ").strip())
|
|
@@ -99,7 +107,11 @@ def run_curl_command(
|
|
|
99
107
|
)
|
|
100
108
|
)
|
|
101
109
|
result = subprocess.run(
|
|
102
|
-
curl_cmd,
|
|
110
|
+
curl_cmd,
|
|
111
|
+
shell=True,
|
|
112
|
+
text=True,
|
|
113
|
+
stdout=subprocess.PIPE,
|
|
114
|
+
stderr=subprocess.PIPE,
|
|
103
115
|
)
|
|
104
116
|
# Case 1: An error occurred, raise an exception.
|
|
105
117
|
if result.returncode != 0:
|
|
@@ -120,18 +132,23 @@ def run_curl_command(
|
|
|
120
132
|
return result.stdout
|
|
121
133
|
|
|
122
134
|
|
|
123
|
-
def is_qlever_server_alive(
|
|
135
|
+
def is_qlever_server_alive(endpoint_url: str) -> bool:
|
|
124
136
|
"""
|
|
125
137
|
Helper function that checks if a QLever server is running on the given
|
|
126
|
-
|
|
138
|
+
endpoint. Return `True` if the server is alive, `False` otherwise.
|
|
127
139
|
"""
|
|
128
140
|
|
|
129
|
-
message = "from the qlever
|
|
130
|
-
curl_cmd =
|
|
131
|
-
|
|
132
|
-
|
|
141
|
+
message = "from the `qlever` CLI"
|
|
142
|
+
curl_cmd = (
|
|
143
|
+
f"curl -s {endpoint_url}/ping"
|
|
144
|
+
f" --data-urlencode msg={shlex.quote(message)}"
|
|
133
145
|
)
|
|
134
|
-
|
|
146
|
+
log.debug(curl_cmd)
|
|
147
|
+
try:
|
|
148
|
+
run_command(curl_cmd)
|
|
149
|
+
return True
|
|
150
|
+
except Exception:
|
|
151
|
+
return False
|
|
135
152
|
|
|
136
153
|
|
|
137
154
|
def get_existing_index_files(basename: str) -> list[str]:
|
|
@@ -6,7 +6,7 @@ qlever/log.py,sha256=2O_RvFymnu_dB10ErBTAOsI8bgjORfdD0tE3USH-siM,1315
|
|
|
6
6
|
qlever/qlever_main.py,sha256=-E4W8YdZ_teszGwXu6bQgBcH3y47TFJU8JLPIDwc_-g,2449
|
|
7
7
|
qlever/qlever_old.py,sha256=X-JxmepFKYeFgSLLp0TRDNqXSxDwIbc8_0Xstiems8c,62026
|
|
8
8
|
qlever/qleverfile.py,sha256=lygAjI5_wV_e-JoIGIqVTdd4yyvApzZiSlqSsmdlJpU,14529
|
|
9
|
-
qlever/util.py,sha256=
|
|
9
|
+
qlever/util.py,sha256=xAK9GT8SgU3z65F1dFXazxsd60letqLQqQAZ81mdJSY,8374
|
|
10
10
|
qlever/Qleverfiles/Qleverfile.dblp,sha256=RaTJrabloAdaHwSl5V7Ws9phWyM_oXPSeZpKodZXA6c,1256
|
|
11
11
|
qlever/Qleverfiles/Qleverfile.dblp-plus,sha256=TJHxp8I1P6JKJjbuAllEpB32-huuY1gH0FlenqPVJ5g,1334
|
|
12
12
|
qlever/Qleverfiles/Qleverfile.dbpedia,sha256=aaNZZayE-zVePGSwPzXemkX__Ns8-kP_E7DNNKZPnqg,1160
|
|
@@ -31,22 +31,24 @@ qlever/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
31
31
|
qlever/commands/add_text_index.py,sha256=cvBhX5rooRAXsn1Oput6b0B10w-MGMfS4ZPAhDfm7OQ,3669
|
|
32
32
|
qlever/commands/cache_stats.py,sha256=zGPLSWDNn7dwAAt2o-2ExqHmw1FeN8i6nEQbaaqF830,4156
|
|
33
33
|
qlever/commands/clear_cache.py,sha256=n52ohE1EE4M_B4kId_v8tbAlW-BGwG1K-ZYQ9QgxIJU,2974
|
|
34
|
-
qlever/commands/example_queries.py,sha256=
|
|
34
|
+
qlever/commands/example_queries.py,sha256=8a7ViBkWP32UqC9-0vhmjo4mc6ZMfjnPlwzVthiHnPk,22792
|
|
35
|
+
qlever/commands/extract_queries.py,sha256=TZBmZLz_NknU1LKbl9nPmxdb82lsPeDhTWjIo81llvA,3942
|
|
35
36
|
qlever/commands/get_data.py,sha256=nHOHMjv0tSLWJDOR0ba_LK-Bk-mcGnphb8hbqcVYFhE,1411
|
|
36
|
-
qlever/commands/index.py,sha256=
|
|
37
|
+
qlever/commands/index.py,sha256=__vDlHnSkKNVqBAPZv2dzC5FYSI4QivOvjDHubFU35Q,12778
|
|
37
38
|
qlever/commands/index_stats.py,sha256=9EBo1Oq5PGjajrvWJNafJ-Wg_d90DaO5AGq9a5plSRM,11720
|
|
38
39
|
qlever/commands/log.py,sha256=vLqkgtx1udnQqoUBMWB5G9rwr-l7UKrDpyFYSMuoXWw,1987
|
|
39
40
|
qlever/commands/query.py,sha256=vu6vTeTqtKhuKbv_wzloHvbJFj_2GTHPIfDpQfrc8FE,3603
|
|
41
|
+
qlever/commands/settings.py,sha256=lyI_mdRMp3zv3RQ-tU2g2ZivfMD0r0R6ODf0VH9aVBk,3742
|
|
40
42
|
qlever/commands/setup_config.py,sha256=wEy1LAunpOnqrUCbazMpt1u9HJCKgXJEMxF3zjh0jb0,3344
|
|
41
|
-
qlever/commands/start.py,sha256=
|
|
43
|
+
qlever/commands/start.py,sha256=gtYKB1sgc5SwY0DA3IMnowXMqXccHA2cOYZ71dgey5k,11589
|
|
42
44
|
qlever/commands/status.py,sha256=TtnBqcdkF3zTDKft07zpVcIX7kFu7d_nOy9b6Ohh9vQ,1650
|
|
43
|
-
qlever/commands/stop.py,sha256=
|
|
45
|
+
qlever/commands/stop.py,sha256=z25gWfLA9qIJ7F3zWZa0p0oVnt61kHcsB1evjgXhKX4,4557
|
|
44
46
|
qlever/commands/system_info.py,sha256=Y3DtUJAlnh6uSvIVoToE10G7SStu-L5NbalIXvBB6D0,4614
|
|
45
47
|
qlever/commands/ui.py,sha256=Ppo6YXA8JXNDPvC5D5duKGMPyhRvXfc1CaoSwjL-jBg,3644
|
|
46
48
|
qlever/commands/warmup.py,sha256=kJHzS7HJo8pD2CphJuaXDj_CYP02YDo2DVM-pun3A80,1029
|
|
47
|
-
qlever-0.5.
|
|
48
|
-
qlever-0.5.
|
|
49
|
-
qlever-0.5.
|
|
50
|
-
qlever-0.5.
|
|
51
|
-
qlever-0.5.
|
|
52
|
-
qlever-0.5.
|
|
49
|
+
qlever-0.5.17.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
50
|
+
qlever-0.5.17.dist-info/METADATA,sha256=lQ-9nEtPrTwq2umXW5yylTGjB_zDHpvnF8yaP0dlyXo,4583
|
|
51
|
+
qlever-0.5.17.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
52
|
+
qlever-0.5.17.dist-info/entry_points.txt,sha256=U_gbYYi0wwdsn884eb0XoOXfvhACOsxhlO330dZ9bi0,87
|
|
53
|
+
qlever-0.5.17.dist-info/top_level.txt,sha256=kd3zsYqiFd0--Czh5XTVkfEq6XR-XgRFW35X0v0GT-c,7
|
|
54
|
+
qlever-0.5.17.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|