dbworkload 0.9.0__tar.gz → 0.9.2.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/PKG-INFO +3 -5
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/dbworkload/models/run.py +281 -189
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/dbworkload/utils/common.py +9 -0
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/pyproject.toml +8 -2
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/LICENSE +0 -0
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/README.md +0 -0
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/dbworkload/__init__.py +0 -0
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/dbworkload/cli/dep.py +0 -0
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/dbworkload/cli/main.py +0 -0
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/dbworkload/cli/util.py +0 -0
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/dbworkload/models/util.py +0 -0
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/dbworkload/templates/stub.j2 +0 -0
- {dbworkload-0.9.0 → dbworkload-0.9.2.dev1}/dbworkload/utils/simplefaker.py +0 -0
|
@@ -1,17 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: dbworkload
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.2.dev1
|
|
4
4
|
Summary: Workload framework
|
|
5
5
|
License: GPLv3+
|
|
6
6
|
Author: Fabio Ghirardello
|
|
7
|
-
Requires-Python: >=3.
|
|
7
|
+
Requires-Python: >=3.11,<4.0
|
|
8
8
|
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
9
9
|
Classifier: License :: Other/Proprietary License
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
15
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
@@ -35,6 +32,7 @@ Requires-Dist: pandas
|
|
|
35
32
|
Requires-Dist: plotext
|
|
36
33
|
Requires-Dist: plotly
|
|
37
34
|
Requires-Dist: prometheus-client
|
|
35
|
+
Requires-Dist: psutil (>=7.0.0,<8.0.0)
|
|
38
36
|
Requires-Dist: psycopg ; extra == "all" or extra == "postgres"
|
|
39
37
|
Requires-Dist: psycopg-binary ; extra == "all" or extra == "postgres"
|
|
40
38
|
Requires-Dist: pymongo ; extra == "all" or extra == "mongo"
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import errno
|
|
4
4
|
import logging
|
|
5
|
+
import math
|
|
5
6
|
import multiprocessing as mp
|
|
6
7
|
import os
|
|
7
8
|
import queue
|
|
@@ -19,6 +20,7 @@ from psutil import cpu_percent, virtual_memory
|
|
|
19
20
|
|
|
20
21
|
import dbworkload.utils.common
|
|
21
22
|
from dbworkload.cli.dep import ConnInfo
|
|
23
|
+
from dbworkload.utils.common import Action
|
|
22
24
|
|
|
23
25
|
# from cassandra.cluster import Cluster, ExecutionProfile, EXEC_PROFILE_DEFAULT, Session
|
|
24
26
|
# from cassandra.policies import (
|
|
@@ -34,11 +36,14 @@ MAX_RETRIES = 3
|
|
|
34
36
|
FREQUENCY = 10
|
|
35
37
|
STATS_BUFFER = 8
|
|
36
38
|
|
|
37
|
-
|
|
39
|
+
DBWORKLOAD_PIPE = "dbworkload.pipe"
|
|
38
40
|
|
|
39
41
|
logger = logging.getLogger("dbworkload")
|
|
40
42
|
|
|
41
|
-
|
|
43
|
+
sigterm_received = False
|
|
44
|
+
|
|
45
|
+
supervisors: dict[int, mp.Process] = {}
|
|
46
|
+
supervisor_queues: dict[int, mp.Queue] = {}
|
|
42
47
|
|
|
43
48
|
HEADERS: list = [
|
|
44
49
|
"elapsed",
|
|
@@ -98,30 +103,14 @@ def signal_handler(sig, frame):
|
|
|
98
103
|
frame (_type_):
|
|
99
104
|
"""
|
|
100
105
|
logger.info("KeyboardInterrupt signal detected. Stopping processes...")
|
|
101
|
-
global
|
|
102
|
-
|
|
106
|
+
global sigterm_received
|
|
107
|
+
|
|
108
|
+
# if a keyboardinterrupt event was already receive, just exit.
|
|
109
|
+
if sigterm_received:
|
|
103
110
|
logger.warning("Forcibly quitting. You're rude!")
|
|
104
111
|
sys.exit(1)
|
|
105
112
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
# send the poison pill to each proc.
|
|
109
|
-
# if dbworkload cannot graceful shutdown due
|
|
110
|
-
# to processes being still in the init phase
|
|
111
|
-
# when the pill is sent, a subsequent Ctrl+C will cause
|
|
112
|
-
# the pill to overflow the kill_q
|
|
113
|
-
# and raise the queue.Full exception, forcing to quit.
|
|
114
|
-
for q in queues.values():
|
|
115
|
-
try:
|
|
116
|
-
q.put("proc_end")
|
|
117
|
-
except queue.Full:
|
|
118
|
-
logger.error("Timed out")
|
|
119
|
-
sys.exit(1)
|
|
120
|
-
|
|
121
|
-
logger.debug("Sent poison pill to all procs")
|
|
122
|
-
|
|
123
|
-
if os.path.exists(FIFO):
|
|
124
|
-
os.remove(FIFO)
|
|
113
|
+
sigterm_received = True
|
|
125
114
|
|
|
126
115
|
|
|
127
116
|
def cycle(iterable, backwards=False):
|
|
@@ -141,7 +130,7 @@ def cycle(iterable, backwards=False):
|
|
|
141
130
|
# If a ramp time is specified, threads creation or destruction
|
|
142
131
|
# will be paced accordingly.
|
|
143
132
|
def launch_or_kill_workers(
|
|
144
|
-
|
|
133
|
+
supervisor_queues: list,
|
|
145
134
|
ramp_time: int,
|
|
146
135
|
cc_change: int,
|
|
147
136
|
proc_len: list,
|
|
@@ -156,11 +145,14 @@ def launch_or_kill_workers(
|
|
|
156
145
|
|
|
157
146
|
if cc_change > 0:
|
|
158
147
|
for _ in range(cc_change):
|
|
159
|
-
|
|
148
|
+
supervisor_queues[cycle(proc_len)].put(
|
|
160
149
|
(
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
150
|
+
Action.NEW_WORKER,
|
|
151
|
+
(
|
|
152
|
+
thread_id,
|
|
153
|
+
iterations_per_thread,
|
|
154
|
+
concurrency,
|
|
155
|
+
),
|
|
164
156
|
)
|
|
165
157
|
)
|
|
166
158
|
thread_id += 1
|
|
@@ -168,7 +160,7 @@ def launch_or_kill_workers(
|
|
|
168
160
|
|
|
169
161
|
if cc_change < 0:
|
|
170
162
|
for _ in range(abs(cc_change)):
|
|
171
|
-
|
|
163
|
+
supervisor_queues[cycle(proc_len, backwards=True)].put((Action.KILL_ONE,))
|
|
172
164
|
time.sleep(ramp_interval)
|
|
173
165
|
|
|
174
166
|
|
|
@@ -192,41 +184,46 @@ def run(
|
|
|
192
184
|
delay_stats: int,
|
|
193
185
|
log_level: str,
|
|
194
186
|
):
|
|
195
|
-
def gracefully_shutdown(
|
|
187
|
+
def gracefully_shutdown():
|
|
196
188
|
logger.debug("Gracefully shutting down...")
|
|
197
189
|
|
|
198
190
|
end_time = int(time.time())
|
|
199
|
-
|
|
191
|
+
_stats_received = stats_received
|
|
192
|
+
|
|
193
|
+
# notify all Supervisors to quit
|
|
194
|
+
for q in supervisor_queues.values():
|
|
195
|
+
q.put((Action.POISON_PILL,))
|
|
200
196
|
|
|
201
|
-
|
|
202
|
-
|
|
197
|
+
# wait for supervisors to quit and drain
|
|
198
|
+
# the to_main_q at the same time to avoid locking
|
|
199
|
+
for x in supervisors.values():
|
|
200
|
+
while x.is_alive():
|
|
203
201
|
try:
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
#
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
# break
|
|
202
|
+
msg = to_main_q.get(block=True, timeout=0.5)
|
|
203
|
+
if isinstance(msg, list):
|
|
204
|
+
_stats_received += 1
|
|
205
|
+
stats.add_tds(msg)
|
|
206
|
+
except queue.Empty:
|
|
207
|
+
pass
|
|
208
|
+
|
|
209
|
+
x.join()
|
|
210
|
+
|
|
211
|
+
# Catch all for loose stats, if any?
|
|
212
|
+
while True:
|
|
213
|
+
try:
|
|
214
|
+
msg = to_main_q.get(block=False)
|
|
215
|
+
if isinstance(msg, list):
|
|
216
|
+
_stats_received += 1
|
|
217
|
+
stats.add_tds(msg)
|
|
218
|
+
except queue.Empty:
|
|
219
|
+
break
|
|
220
|
+
|
|
221
|
+
cpu_util = cpu_percent()
|
|
222
|
+
vmem = virtual_memory().percent
|
|
223
|
+
if _stats_received != active_connections or cpu_util > 70 or vmem > 70:
|
|
224
|
+
logger.warning(
|
|
225
|
+
f"{_stats_received=}, expected={active_connections}. CPU Util={cpu_util}%, Memory={vmem}%"
|
|
226
|
+
)
|
|
230
227
|
|
|
231
228
|
# now that we have all stat reports, calculate the stats one last time.
|
|
232
229
|
report = stats.calculate_stats(active_connections, end_time - delay_stats)
|
|
@@ -314,6 +311,9 @@ def run(
|
|
|
314
311
|
sep="",
|
|
315
312
|
)
|
|
316
313
|
|
|
314
|
+
if os.path.exists(DBWORKLOAD_PIPE):
|
|
315
|
+
os.remove(DBWORKLOAD_PIPE)
|
|
316
|
+
|
|
317
317
|
sys.exit(0)
|
|
318
318
|
|
|
319
319
|
logger.setLevel(log_level)
|
|
@@ -347,10 +347,8 @@ def run(
|
|
|
347
347
|
|
|
348
348
|
to_main_q = mp.Queue()
|
|
349
349
|
|
|
350
|
-
global
|
|
350
|
+
global supervisor_queues
|
|
351
351
|
global supervisors
|
|
352
|
-
supervisors = {}
|
|
353
|
-
queues = {}
|
|
354
352
|
|
|
355
353
|
# start a separate thread for messages coming in via the pipe
|
|
356
354
|
# echo 5 > dbworkload.pipe # create 5 more connections
|
|
@@ -358,7 +356,7 @@ def run(
|
|
|
358
356
|
target=listen_to_pipe,
|
|
359
357
|
daemon=True,
|
|
360
358
|
args=(
|
|
361
|
-
|
|
359
|
+
supervisor_queues,
|
|
362
360
|
0,
|
|
363
361
|
procs,
|
|
364
362
|
None,
|
|
@@ -368,12 +366,12 @@ def run(
|
|
|
368
366
|
|
|
369
367
|
# launch supervisors in a dedicated OS process
|
|
370
368
|
for x in range(procs):
|
|
371
|
-
|
|
369
|
+
supervisor_queues[x] = mp.Queue()
|
|
372
370
|
supervisors[x] = mp.Process(
|
|
373
371
|
target=supervisor,
|
|
374
372
|
args=(
|
|
375
373
|
to_main_q,
|
|
376
|
-
|
|
374
|
+
supervisor_queues[x],
|
|
377
375
|
log_level,
|
|
378
376
|
conn_info,
|
|
379
377
|
driver,
|
|
@@ -402,7 +400,6 @@ def run(
|
|
|
402
400
|
current_proc = -1
|
|
403
401
|
current_cc = 0
|
|
404
402
|
thread_id = 0
|
|
405
|
-
pause_for_ramp_time = 0
|
|
406
403
|
|
|
407
404
|
iterations_per_thread = None
|
|
408
405
|
if iterations:
|
|
@@ -432,13 +429,17 @@ def run(
|
|
|
432
429
|
f"Starting schedule {i+1}/{len(schedule)}: {cc=}, {max_rate=}, {ramp_time=}, {dur=}"
|
|
433
430
|
)
|
|
434
431
|
|
|
432
|
+
pause_for_ramp_time = time.time() + 3 * FREQUENCY
|
|
433
|
+
exhaust_warning = 0
|
|
434
|
+
think_time = 0
|
|
435
|
+
|
|
435
436
|
# always make sure that a duration is specified, even if none was passed
|
|
436
437
|
# in which case it defaults to infinite
|
|
437
438
|
end_schedule_time = time.time() + dur if dur else float("inf")
|
|
438
439
|
|
|
439
440
|
# if max_rate was set instead of concurrency
|
|
440
441
|
# and current_cc = 0,
|
|
441
|
-
# start the workload with
|
|
442
|
+
# start the workload with 10 threads so that dbworkload
|
|
442
443
|
# has stats to measure on for adding/removing threads
|
|
443
444
|
# as part of the calculations for maintaining
|
|
444
445
|
# the desired max_rate
|
|
@@ -447,23 +448,23 @@ def run(
|
|
|
447
448
|
target=launch_or_kill_workers,
|
|
448
449
|
daemon=True,
|
|
449
450
|
args=(
|
|
450
|
-
|
|
451
|
+
supervisor_queues,
|
|
451
452
|
ramp_time,
|
|
452
|
-
|
|
453
|
+
10,
|
|
453
454
|
procs,
|
|
454
455
|
iterations_per_thread,
|
|
455
456
|
concurrency,
|
|
456
457
|
),
|
|
457
458
|
).start()
|
|
458
459
|
|
|
459
|
-
current_cc =
|
|
460
|
+
current_cc = 10
|
|
460
461
|
|
|
461
462
|
if not max_rate:
|
|
462
463
|
Thread(
|
|
463
464
|
target=launch_or_kill_workers,
|
|
464
465
|
daemon=True,
|
|
465
466
|
args=(
|
|
466
|
-
|
|
467
|
+
supervisor_queues,
|
|
467
468
|
ramp_time,
|
|
468
469
|
cc - current_cc,
|
|
469
470
|
procs,
|
|
@@ -474,7 +475,7 @@ def run(
|
|
|
474
475
|
|
|
475
476
|
current_cc = cc
|
|
476
477
|
|
|
477
|
-
|
|
478
|
+
task_done_threads = 0
|
|
478
479
|
|
|
479
480
|
# loop for the entire duration of the schedule's current line
|
|
480
481
|
while time.time() < end_schedule_time:
|
|
@@ -489,30 +490,24 @@ def run(
|
|
|
489
490
|
active_connections += 1
|
|
490
491
|
elif msg == "got_killed":
|
|
491
492
|
active_connections -= 1
|
|
492
|
-
elif msg == "proc_returned":
|
|
493
|
-
returned_procs += 1
|
|
494
|
-
logger.debug(f"Stopped processes: {returned_procs}/{procs} ")
|
|
495
493
|
elif msg == "task_done":
|
|
496
|
-
|
|
497
|
-
except queue.Empty:
|
|
498
|
-
pass
|
|
499
|
-
|
|
500
|
-
# check if all procs returned, then exit
|
|
501
|
-
if returned_procs >= procs or (
|
|
502
|
-
returned_threads > 0 and returned_threads >= active_connections
|
|
503
|
-
):
|
|
504
|
-
if msg == "task_done":
|
|
505
|
-
logger.info("Requested iteration/duration limit reached")
|
|
506
|
-
gracefully_shutdown()
|
|
507
|
-
elif msg == "proc_returned":
|
|
508
|
-
logger.debug("All procs returned")
|
|
509
|
-
gracefully_shutdown(by_keyinterrupt=True)
|
|
494
|
+
task_done_threads += 1
|
|
510
495
|
elif isinstance(msg, Exception):
|
|
511
|
-
logger.error(f"error_type={msg.__class__.__name__}, msg=
|
|
512
|
-
|
|
496
|
+
logger.error(f"error_type={msg.__class__.__name__}, {msg=}")
|
|
497
|
+
gracefully_shutdown()
|
|
513
498
|
else:
|
|
514
499
|
logger.error(f"unrecognized message: {msg}")
|
|
515
|
-
|
|
500
|
+
gracefully_shutdown()
|
|
501
|
+
|
|
502
|
+
except queue.Empty:
|
|
503
|
+
pass
|
|
504
|
+
|
|
505
|
+
if sigterm_received:
|
|
506
|
+
gracefully_shutdown()
|
|
507
|
+
|
|
508
|
+
if task_done_threads > 0 and task_done_threads >= active_connections:
|
|
509
|
+
logger.info("Requested iteration/duration limit reached")
|
|
510
|
+
gracefully_shutdown()
|
|
516
511
|
|
|
517
512
|
if time.time() >= report_time:
|
|
518
513
|
cpu_util = cpu_percent()
|
|
@@ -532,49 +527,131 @@ def run(
|
|
|
532
527
|
if max_rate and report:
|
|
533
528
|
current_rate = report[0][6] # __cycle__ period_ops/s
|
|
534
529
|
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
530
|
+
if time.time() > pause_for_ramp_time:
|
|
531
|
+
|
|
532
|
+
if current_rate < max_rate * 0.99:
|
|
533
|
+
if think_time > 0:
|
|
534
|
+
|
|
535
|
+
think_time = (
|
|
536
|
+
math.floor(
|
|
537
|
+
think_time * current_rate / max_rate * 1000
|
|
538
|
+
)
|
|
539
|
+
/ 1000
|
|
540
|
+
)
|
|
541
|
+
|
|
542
|
+
pause_for_ramp_time = time.time() + 3 * FREQUENCY
|
|
543
|
+
|
|
544
|
+
logger.info(
|
|
545
|
+
f"Calculating max_rate: {max_rate=} {current_rate=}, {current_cc=} {think_time=}"
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
else:
|
|
549
|
+
if exhaust_warning > 3:
|
|
550
|
+
logger.warning("Pointless to add any more threads")
|
|
551
|
+
else:
|
|
552
|
+
|
|
553
|
+
# approximate how many threads are needed to get
|
|
554
|
+
# to the desired max_rate given the current QPS rate
|
|
555
|
+
# and current thread count
|
|
556
|
+
# increase by 5%
|
|
557
|
+
extrapolated_cc = int(
|
|
558
|
+
max_rate / (current_rate / current_cc) * 1.05
|
|
559
|
+
)
|
|
560
|
+
|
|
561
|
+
exhaust_warning += 1
|
|
562
|
+
|
|
563
|
+
change = max(0, extrapolated_cc - current_cc)
|
|
564
|
+
|
|
565
|
+
Thread(
|
|
566
|
+
target=launch_or_kill_workers,
|
|
567
|
+
daemon=True,
|
|
568
|
+
args=(
|
|
569
|
+
supervisor_queues,
|
|
570
|
+
ramp_time,
|
|
571
|
+
change,
|
|
572
|
+
procs,
|
|
573
|
+
iterations_per_thread,
|
|
574
|
+
concurrency,
|
|
575
|
+
),
|
|
576
|
+
).start()
|
|
577
|
+
|
|
578
|
+
# give enough time for newly created threads to settle
|
|
579
|
+
# before new calculations are performed
|
|
580
|
+
pause_for_ramp_time = (
|
|
581
|
+
time.time() + ramp_time + 6 * FREQUENCY
|
|
582
|
+
)
|
|
583
|
+
|
|
584
|
+
logger.info(
|
|
585
|
+
f"Calculating max_rate: {max_rate=} {current_rate=}, {current_cc=} {change=}"
|
|
586
|
+
)
|
|
587
|
+
current_cc += change
|
|
588
|
+
|
|
589
|
+
# ramp_time is only considered for reaching the desired max_rate.
|
|
590
|
+
# For adjustments over time, we want the changes to happen immediately
|
|
591
|
+
# and not smoothed out over the initial ramp_time value
|
|
592
|
+
ramp_time = 0
|
|
593
|
+
|
|
594
|
+
for q in supervisor_queues.values():
|
|
595
|
+
q.put((Action.THINK_TIME, think_time))
|
|
596
|
+
|
|
597
|
+
elif current_rate * 0.99 > max_rate:
|
|
598
|
+
if think_time > 1:
|
|
599
|
+
# pointless to add more time, remove threads instead
|
|
600
|
+
|
|
601
|
+
# decrease count by 5%
|
|
602
|
+
change = int(current_cc * -0.05)
|
|
603
|
+
|
|
604
|
+
Thread(
|
|
605
|
+
target=launch_or_kill_workers,
|
|
606
|
+
daemon=True,
|
|
607
|
+
args=(
|
|
608
|
+
supervisor_queues,
|
|
609
|
+
ramp_time,
|
|
610
|
+
change,
|
|
611
|
+
procs,
|
|
612
|
+
iterations_per_thread,
|
|
613
|
+
concurrency,
|
|
614
|
+
),
|
|
615
|
+
).start()
|
|
616
|
+
|
|
617
|
+
# give enough time for newly created threads to settle
|
|
618
|
+
# before new calculations are performed
|
|
619
|
+
pause_for_ramp_time = (
|
|
620
|
+
time.time() + ramp_time + 6 * FREQUENCY
|
|
621
|
+
)
|
|
622
|
+
|
|
623
|
+
logger.info(
|
|
624
|
+
f"Calculating max_rate: {max_rate=} {current_rate=}, {current_cc=} {change=}"
|
|
625
|
+
)
|
|
626
|
+
current_cc += change
|
|
627
|
+
|
|
628
|
+
# ramp_time is only considered for reaching the desired max_rate.
|
|
629
|
+
# For adjustments over time, we want the changes to happen immediately
|
|
630
|
+
# and not smoothed out over the initial ramp_time value
|
|
631
|
+
ramp_time = 0
|
|
632
|
+
think_time = 0
|
|
633
|
+
|
|
634
|
+
else:
|
|
635
|
+
# add think_time to slow it down a bit
|
|
636
|
+
|
|
637
|
+
if think_time == 0:
|
|
638
|
+
|
|
639
|
+
think_time = round(
|
|
640
|
+
0.01 * current_rate / max_rate, 4
|
|
641
|
+
)
|
|
642
|
+
else:
|
|
643
|
+
think_time = round(
|
|
644
|
+
think_time * current_rate / max_rate, 4
|
|
645
|
+
)
|
|
646
|
+
|
|
647
|
+
pause_for_ramp_time = time.time() + 3 * FREQUENCY
|
|
648
|
+
|
|
649
|
+
logger.info(
|
|
650
|
+
f"Calculating max_rate: {max_rate=} {current_rate=}, {current_cc=} {think_time=}"
|
|
651
|
+
)
|
|
652
|
+
|
|
653
|
+
for q in supervisor_queues.values():
|
|
654
|
+
q.put((Action.THINK_TIME, think_time))
|
|
578
655
|
|
|
579
656
|
centroids = stats.get_centroids()
|
|
580
657
|
|
|
@@ -623,61 +700,63 @@ def supervisor(
|
|
|
623
700
|
offset: int,
|
|
624
701
|
id: int,
|
|
625
702
|
):
|
|
626
|
-
def gracefully_return(msg):
|
|
627
|
-
# wait for Threads to return before
|
|
628
|
-
# letting the Process MainThread return
|
|
629
|
-
# threading.enumerate()
|
|
630
|
-
for x in threads:
|
|
631
|
-
if x.is_alive():
|
|
632
|
-
from_proc_q.put("poison_pill")
|
|
633
|
-
|
|
634
|
-
for x in threads:
|
|
635
|
-
if x.is_alive():
|
|
636
|
-
x.join()
|
|
637
|
-
|
|
638
|
-
# send notification to MainThread
|
|
639
|
-
to_main_q.put(msg)
|
|
640
|
-
|
|
641
|
-
logger.debug(f"PROC-{id} terminated")
|
|
642
|
-
return
|
|
643
|
-
|
|
644
703
|
logger.setLevel(log_level)
|
|
645
|
-
logger.debug(f"
|
|
704
|
+
logger.debug(f"Supervisor-{id} started")
|
|
646
705
|
|
|
647
|
-
|
|
706
|
+
worker_threads: list[Thread] = []
|
|
648
707
|
from_proc_q = mp.Queue()
|
|
649
708
|
|
|
650
709
|
# capture KeyboardInterrupt and do nothing
|
|
651
710
|
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
652
711
|
|
|
712
|
+
global think_time
|
|
713
|
+
think_time = 0
|
|
714
|
+
|
|
653
715
|
while True:
|
|
654
716
|
msg = from_main_q.get(block=True)
|
|
655
717
|
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
718
|
+
match msg[0]:
|
|
719
|
+
case Action.POISON_PILL: # poison pill
|
|
720
|
+
logger.debug(f"Supervisor-{id} terminating...")
|
|
721
|
+
|
|
722
|
+
# wait for Threads to return before
|
|
723
|
+
# letting the Supervisor MainThread return
|
|
724
|
+
for x in worker_threads:
|
|
725
|
+
if x.is_alive():
|
|
726
|
+
from_proc_q.put(Action.POISON_PILL)
|
|
727
|
+
|
|
728
|
+
for x in worker_threads:
|
|
729
|
+
if x.is_alive():
|
|
730
|
+
x.join()
|
|
731
|
+
|
|
732
|
+
logger.debug(f"Supervisor-{id} terminated")
|
|
733
|
+
return
|
|
734
|
+
|
|
735
|
+
case Action.KILL_ONE: # kill_one
|
|
736
|
+
from_proc_q.put(Action.POISON_PILL)
|
|
737
|
+
|
|
738
|
+
case Action.THINK_TIME: # set think_time
|
|
739
|
+
think_time = msg[1]
|
|
740
|
+
|
|
741
|
+
case Action.NEW_WORKER: # add new worker
|
|
742
|
+
t = Thread(
|
|
743
|
+
target=worker,
|
|
744
|
+
daemon=True,
|
|
745
|
+
args=(
|
|
746
|
+
to_main_q,
|
|
747
|
+
from_proc_q,
|
|
748
|
+
log_level,
|
|
749
|
+
conn_info,
|
|
750
|
+
driver,
|
|
751
|
+
workload,
|
|
752
|
+
args,
|
|
753
|
+
conn_duration,
|
|
754
|
+
offset,
|
|
755
|
+
*msg[1],
|
|
756
|
+
),
|
|
757
|
+
)
|
|
758
|
+
t.start()
|
|
759
|
+
worker_threads.append(t)
|
|
681
760
|
|
|
682
761
|
|
|
683
762
|
def worker(
|
|
@@ -695,14 +774,13 @@ def worker(
|
|
|
695
774
|
concurrency: int = 0,
|
|
696
775
|
):
|
|
697
776
|
def gracefully_return(msg):
|
|
698
|
-
# send notification to MainThread
|
|
699
|
-
to_main_q.put(msg)
|
|
700
777
|
# send final stats
|
|
701
778
|
to_main_q.put(ws.get_tdigest_ndarray(), block=False)
|
|
702
779
|
|
|
703
|
-
|
|
780
|
+
# send notification to MainThread
|
|
781
|
+
to_main_q.put(msg)
|
|
704
782
|
|
|
705
|
-
|
|
783
|
+
logger.debug(f"Thread ID {id} returned")
|
|
706
784
|
|
|
707
785
|
logger.setLevel(log_level)
|
|
708
786
|
|
|
@@ -728,6 +806,15 @@ def worker(
|
|
|
728
806
|
to_main_q.put("init")
|
|
729
807
|
|
|
730
808
|
while True:
|
|
809
|
+
# listen for termination messages (poison pill)
|
|
810
|
+
try:
|
|
811
|
+
from_proc_q.get(block=False)
|
|
812
|
+
logger.debug("Poison pill received, terminating...")
|
|
813
|
+
gracefully_return("got_killed")
|
|
814
|
+
return
|
|
815
|
+
except queue.Empty:
|
|
816
|
+
pass
|
|
817
|
+
|
|
731
818
|
if conn_duration:
|
|
732
819
|
# reconnect every conn_duration +/- 20%
|
|
733
820
|
conn_endtime = time.time() + int(conn_duration * random.uniform(0.8, 1.2))
|
|
@@ -743,7 +830,6 @@ def worker(
|
|
|
743
830
|
run_init = False
|
|
744
831
|
|
|
745
832
|
if hasattr(w, "setup") and callable(w.setup):
|
|
746
|
-
logger.debug("Executing setup() function")
|
|
747
833
|
run_transaction(
|
|
748
834
|
conn,
|
|
749
835
|
lambda conn: w.setup(
|
|
@@ -763,8 +849,9 @@ def worker(
|
|
|
763
849
|
# listen for termination messages (poison pill)
|
|
764
850
|
try:
|
|
765
851
|
from_proc_q.get(block=False)
|
|
766
|
-
logger.debug("Poison pill received")
|
|
767
|
-
|
|
852
|
+
logger.debug("Poison pill received, terminating...")
|
|
853
|
+
gracefully_return("got_killed")
|
|
854
|
+
return
|
|
768
855
|
except queue.Empty:
|
|
769
856
|
pass
|
|
770
857
|
|
|
@@ -806,6 +893,10 @@ def worker(
|
|
|
806
893
|
|
|
807
894
|
ws.add_latency_measurement("__cycle__", time.time() - cycle_start)
|
|
808
895
|
|
|
896
|
+
if think_time > 0:
|
|
897
|
+
time.sleep(think_time)
|
|
898
|
+
ws.add_latency_measurement("__think_time__", think_time)
|
|
899
|
+
|
|
809
900
|
if to_main_q.full():
|
|
810
901
|
logger.error("=========== Q FULL!!!! ======================")
|
|
811
902
|
if time.time() >= stat_time:
|
|
@@ -853,13 +944,13 @@ def listen_to_pipe(queues, ramp_time, procs, iterations_per_thread, concurrency)
|
|
|
853
944
|
# https://stackoverflow.com/questions/39089776/python-read-named-pipe
|
|
854
945
|
|
|
855
946
|
try:
|
|
856
|
-
os.mkfifo(
|
|
947
|
+
os.mkfifo(DBWORKLOAD_PIPE)
|
|
857
948
|
except OSError as oe:
|
|
858
949
|
if oe.errno != errno.EEXIST:
|
|
859
950
|
raise
|
|
860
951
|
|
|
861
952
|
while True:
|
|
862
|
-
with open(
|
|
953
|
+
with open(DBWORKLOAD_PIPE) as fifo:
|
|
863
954
|
for line in fifo:
|
|
864
955
|
try:
|
|
865
956
|
t = int(line)
|
|
@@ -882,6 +973,7 @@ def listen_to_pipe(queues, ramp_time, procs, iterations_per_thread, concurrency)
|
|
|
882
973
|
|
|
883
974
|
|
|
884
975
|
def log_and_sleep(e: Exception):
|
|
976
|
+
raise e
|
|
885
977
|
logger.error(f"error_type={e.__class__.__name__}, msg={e}")
|
|
886
978
|
logger.info("Sleeping for %s seconds" % (DEFAULT_SLEEP))
|
|
887
979
|
time.sleep(DEFAULT_SLEEP)
|
|
@@ -48,10 +48,19 @@ NOT_NULL_MAX = 40
|
|
|
48
48
|
|
|
49
49
|
logger = logging.getLogger("dbworkload")
|
|
50
50
|
|
|
51
|
+
from enum import IntEnum
|
|
52
|
+
|
|
51
53
|
from prometheus_client.core import REGISTRY, HistogramMetricFamily
|
|
52
54
|
from prometheus_client.registry import Collector
|
|
53
55
|
|
|
54
56
|
|
|
57
|
+
class Action(IntEnum):
|
|
58
|
+
KILL_ONE = 7
|
|
59
|
+
THINK_TIME = 2
|
|
60
|
+
POISON_PILL = 9
|
|
61
|
+
NEW_WORKER = 1
|
|
62
|
+
|
|
63
|
+
|
|
55
64
|
class Stats:
|
|
56
65
|
"""Print workload stats
|
|
57
66
|
and export the stats as Prometheus endpoints
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "dbworkload"
|
|
3
|
-
version = "0.9.
|
|
3
|
+
version = "0.9.2.dev1"
|
|
4
4
|
description = "Workload framework"
|
|
5
5
|
authors = ["Fabio Ghirardello"]
|
|
6
6
|
license = "GPLv3+"
|
|
@@ -17,7 +17,7 @@ classifiers = [
|
|
|
17
17
|
dbworkload = 'dbworkload.cli.main:app'
|
|
18
18
|
|
|
19
19
|
[tool.poetry.dependencies]
|
|
20
|
-
python = "^3.
|
|
20
|
+
python = "^3.11"
|
|
21
21
|
pandas = "*"
|
|
22
22
|
tabulate = "*"
|
|
23
23
|
numpy = "*"
|
|
@@ -38,6 +38,7 @@ plotext = "*"
|
|
|
38
38
|
plotly = "*"
|
|
39
39
|
jinja2 = "*"
|
|
40
40
|
sqlparse = "*"
|
|
41
|
+
psutil = "^7.0.0"
|
|
41
42
|
|
|
42
43
|
[tool.poetry.extras]
|
|
43
44
|
all = ["psycopg", "psycopg-binary", "mysql-connector-python", "mariadb", "oracledb", "pyodbc", "pymongo", "cassandra-driver", "google-cloud-spanner"]
|
|
@@ -50,6 +51,11 @@ mongo = ["pymongo"]
|
|
|
50
51
|
cassandra = ["cassandra-driver"]
|
|
51
52
|
spanner = ["google-cloud-spanner"]
|
|
52
53
|
|
|
54
|
+
[tool.poetry.group.dev.dependencies]
|
|
55
|
+
mkdocs = "^1.6.1"
|
|
56
|
+
mkdocs-material = "^9.6.14"
|
|
57
|
+
mkdocs-click = "^0.9.0"
|
|
58
|
+
|
|
53
59
|
[build-system]
|
|
54
60
|
requires = ["poetry-core"]
|
|
55
61
|
build-backend = "poetry.core.masonry.api"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|