dbworkload 0.9.0__tar.gz → 0.9.2.dev1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,17 +1,14 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: dbworkload
3
- Version: 0.9.0
3
+ Version: 0.9.2.dev1
4
4
  Summary: Workload framework
5
5
  License: GPLv3+
6
6
  Author: Fabio Ghirardello
7
- Requires-Python: >=3.8,<4.0
7
+ Requires-Python: >=3.11,<4.0
8
8
  Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
9
9
  Classifier: License :: Other/Proprietary License
10
10
  Classifier: Operating System :: OS Independent
11
11
  Classifier: Programming Language :: Python :: 3
12
- Classifier: Programming Language :: Python :: 3.8
13
- Classifier: Programming Language :: Python :: 3.9
14
- Classifier: Programming Language :: Python :: 3.10
15
12
  Classifier: Programming Language :: Python :: 3.11
16
13
  Classifier: Programming Language :: Python :: 3.12
17
14
  Classifier: Programming Language :: Python :: 3.13
@@ -35,6 +32,7 @@ Requires-Dist: pandas
35
32
  Requires-Dist: plotext
36
33
  Requires-Dist: plotly
37
34
  Requires-Dist: prometheus-client
35
+ Requires-Dist: psutil (>=7.0.0,<8.0.0)
38
36
  Requires-Dist: psycopg ; extra == "all" or extra == "postgres"
39
37
  Requires-Dist: psycopg-binary ; extra == "all" or extra == "postgres"
40
38
  Requires-Dist: pymongo ; extra == "all" or extra == "mongo"
@@ -2,6 +2,7 @@
2
2
 
3
3
  import errno
4
4
  import logging
5
+ import math
5
6
  import multiprocessing as mp
6
7
  import os
7
8
  import queue
@@ -19,6 +20,7 @@ from psutil import cpu_percent, virtual_memory
19
20
 
20
21
  import dbworkload.utils.common
21
22
  from dbworkload.cli.dep import ConnInfo
23
+ from dbworkload.utils.common import Action
22
24
 
23
25
  # from cassandra.cluster import Cluster, ExecutionProfile, EXEC_PROFILE_DEFAULT, Session
24
26
  # from cassandra.policies import (
@@ -34,11 +36,14 @@ MAX_RETRIES = 3
34
36
  FREQUENCY = 10
35
37
  STATS_BUFFER = 8
36
38
 
37
- FIFO = "dbworkload.pipe"
39
+ DBWORKLOAD_PIPE = "dbworkload.pipe"
38
40
 
39
41
  logger = logging.getLogger("dbworkload")
40
42
 
41
- force_exit = False
43
+ sigterm_received = False
44
+
45
+ supervisors: dict[int, mp.Process] = {}
46
+ supervisor_queues: dict[int, mp.Queue] = {}
42
47
 
43
48
  HEADERS: list = [
44
49
  "elapsed",
@@ -98,30 +103,14 @@ def signal_handler(sig, frame):
98
103
  frame (_type_):
99
104
  """
100
105
  logger.info("KeyboardInterrupt signal detected. Stopping processes...")
101
- global force_exit
102
- if force_exit:
106
+ global sigterm_received
107
+
108
+ # if a keyboardinterrupt event was already receive, just exit.
109
+ if sigterm_received:
103
110
  logger.warning("Forcibly quitting. You're rude!")
104
111
  sys.exit(1)
105
112
 
106
- force_exit = True
107
-
108
- # send the poison pill to each proc.
109
- # if dbworkload cannot graceful shutdown due
110
- # to processes being still in the init phase
111
- # when the pill is sent, a subsequent Ctrl+C will cause
112
- # the pill to overflow the kill_q
113
- # and raise the queue.Full exception, forcing to quit.
114
- for q in queues.values():
115
- try:
116
- q.put("proc_end")
117
- except queue.Full:
118
- logger.error("Timed out")
119
- sys.exit(1)
120
-
121
- logger.debug("Sent poison pill to all procs")
122
-
123
- if os.path.exists(FIFO):
124
- os.remove(FIFO)
113
+ sigterm_received = True
125
114
 
126
115
 
127
116
  def cycle(iterable, backwards=False):
@@ -141,7 +130,7 @@ def cycle(iterable, backwards=False):
141
130
  # If a ramp time is specified, threads creation or destruction
142
131
  # will be paced accordingly.
143
132
  def launch_or_kill_workers(
144
- queues: list,
133
+ supervisor_queues: list,
145
134
  ramp_time: int,
146
135
  cc_change: int,
147
136
  proc_len: list,
@@ -156,11 +145,14 @@ def launch_or_kill_workers(
156
145
 
157
146
  if cc_change > 0:
158
147
  for _ in range(cc_change):
159
- queues[cycle(proc_len)].put(
148
+ supervisor_queues[cycle(proc_len)].put(
160
149
  (
161
- thread_id,
162
- iterations_per_thread,
163
- concurrency,
150
+ Action.NEW_WORKER,
151
+ (
152
+ thread_id,
153
+ iterations_per_thread,
154
+ concurrency,
155
+ ),
164
156
  )
165
157
  )
166
158
  thread_id += 1
@@ -168,7 +160,7 @@ def launch_or_kill_workers(
168
160
 
169
161
  if cc_change < 0:
170
162
  for _ in range(abs(cc_change)):
171
- queues[cycle(proc_len, backwards=True)].put("kill_one")
163
+ supervisor_queues[cycle(proc_len, backwards=True)].put((Action.KILL_ONE,))
172
164
  time.sleep(ramp_interval)
173
165
 
174
166
 
@@ -192,41 +184,46 @@ def run(
192
184
  delay_stats: int,
193
185
  log_level: str,
194
186
  ):
195
- def gracefully_shutdown(by_keyinterrupt: bool = False):
187
+ def gracefully_shutdown():
196
188
  logger.debug("Gracefully shutting down...")
197
189
 
198
190
  end_time = int(time.time())
199
- # _s = stats_received
191
+ _stats_received = stats_received
192
+
193
+ # notify all Supervisors to quit
194
+ for q in supervisor_queues.values():
195
+ q.put((Action.POISON_PILL,))
200
196
 
201
- if not by_keyinterrupt:
202
- for q in queues.values():
197
+ # wait for supervisors to quit and drain
198
+ # the to_main_q at the same time to avoid locking
199
+ for x in supervisors.values():
200
+ while x.is_alive():
203
201
  try:
204
- q.put("proc_end")
205
- except queue.Full:
206
- logger.error("Timed out")
207
- sys.exit(1)
208
-
209
- for x in supervisors.values():
210
- if x.is_alive():
211
- x.join()
212
-
213
- # Commenting the below as in theory there shouldn't be any stats that
214
- # comes in *after* the PROC returns. That is, all threads send stats
215
- # when all threads are returned, then the supervisor returns.
216
- # while True:
217
- # try:
218
- # msg = to_main_q.get(block=True, timeout=2.0)
219
- # if isinstance(msg, list):
220
- # _s += 1
221
- # stats.add_tds(msg)
222
- # if _s >= active_connections:
223
- # break
224
- # else:
225
- # logger.error("Timed out, quitting")
226
- # sys.exit(1)
227
-
228
- # except queue.Empty:
229
- # break
202
+ msg = to_main_q.get(block=True, timeout=0.5)
203
+ if isinstance(msg, list):
204
+ _stats_received += 1
205
+ stats.add_tds(msg)
206
+ except queue.Empty:
207
+ pass
208
+
209
+ x.join()
210
+
211
+ # Catch all for loose stats, if any?
212
+ while True:
213
+ try:
214
+ msg = to_main_q.get(block=False)
215
+ if isinstance(msg, list):
216
+ _stats_received += 1
217
+ stats.add_tds(msg)
218
+ except queue.Empty:
219
+ break
220
+
221
+ cpu_util = cpu_percent()
222
+ vmem = virtual_memory().percent
223
+ if _stats_received != active_connections or cpu_util > 70 or vmem > 70:
224
+ logger.warning(
225
+ f"{_stats_received=}, expected={active_connections}. CPU Util={cpu_util}%, Memory={vmem}%"
226
+ )
230
227
 
231
228
  # now that we have all stat reports, calculate the stats one last time.
232
229
  report = stats.calculate_stats(active_connections, end_time - delay_stats)
@@ -314,6 +311,9 @@ def run(
314
311
  sep="",
315
312
  )
316
313
 
314
+ if os.path.exists(DBWORKLOAD_PIPE):
315
+ os.remove(DBWORKLOAD_PIPE)
316
+
317
317
  sys.exit(0)
318
318
 
319
319
  logger.setLevel(log_level)
@@ -347,10 +347,8 @@ def run(
347
347
 
348
348
  to_main_q = mp.Queue()
349
349
 
350
- global queues
350
+ global supervisor_queues
351
351
  global supervisors
352
- supervisors = {}
353
- queues = {}
354
352
 
355
353
  # start a separate thread for messages coming in via the pipe
356
354
  # echo 5 > dbworkload.pipe # create 5 more connections
@@ -358,7 +356,7 @@ def run(
358
356
  target=listen_to_pipe,
359
357
  daemon=True,
360
358
  args=(
361
- queues,
359
+ supervisor_queues,
362
360
  0,
363
361
  procs,
364
362
  None,
@@ -368,12 +366,12 @@ def run(
368
366
 
369
367
  # launch supervisors in a dedicated OS process
370
368
  for x in range(procs):
371
- queues[x] = mp.Queue()
369
+ supervisor_queues[x] = mp.Queue()
372
370
  supervisors[x] = mp.Process(
373
371
  target=supervisor,
374
372
  args=(
375
373
  to_main_q,
376
- queues[x],
374
+ supervisor_queues[x],
377
375
  log_level,
378
376
  conn_info,
379
377
  driver,
@@ -402,7 +400,6 @@ def run(
402
400
  current_proc = -1
403
401
  current_cc = 0
404
402
  thread_id = 0
405
- pause_for_ramp_time = 0
406
403
 
407
404
  iterations_per_thread = None
408
405
  if iterations:
@@ -432,13 +429,17 @@ def run(
432
429
  f"Starting schedule {i+1}/{len(schedule)}: {cc=}, {max_rate=}, {ramp_time=}, {dur=}"
433
430
  )
434
431
 
432
+ pause_for_ramp_time = time.time() + 3 * FREQUENCY
433
+ exhaust_warning = 0
434
+ think_time = 0
435
+
435
436
  # always make sure that a duration is specified, even if none was passed
436
437
  # in which case it defaults to infinite
437
438
  end_schedule_time = time.time() + dur if dur else float("inf")
438
439
 
439
440
  # if max_rate was set instead of concurrency
440
441
  # and current_cc = 0,
441
- # start the workload with 1 thread so that dbworkload
442
+ # start the workload with 10 threads so that dbworkload
442
443
  # has stats to measure on for adding/removing threads
443
444
  # as part of the calculations for maintaining
444
445
  # the desired max_rate
@@ -447,23 +448,23 @@ def run(
447
448
  target=launch_or_kill_workers,
448
449
  daemon=True,
449
450
  args=(
450
- queues,
451
+ supervisor_queues,
451
452
  ramp_time,
452
- 1,
453
+ 10,
453
454
  procs,
454
455
  iterations_per_thread,
455
456
  concurrency,
456
457
  ),
457
458
  ).start()
458
459
 
459
- current_cc = 1
460
+ current_cc = 10
460
461
 
461
462
  if not max_rate:
462
463
  Thread(
463
464
  target=launch_or_kill_workers,
464
465
  daemon=True,
465
466
  args=(
466
- queues,
467
+ supervisor_queues,
467
468
  ramp_time,
468
469
  cc - current_cc,
469
470
  procs,
@@ -474,7 +475,7 @@ def run(
474
475
 
475
476
  current_cc = cc
476
477
 
477
- returned_threads = 0
478
+ task_done_threads = 0
478
479
 
479
480
  # loop for the entire duration of the schedule's current line
480
481
  while time.time() < end_schedule_time:
@@ -489,30 +490,24 @@ def run(
489
490
  active_connections += 1
490
491
  elif msg == "got_killed":
491
492
  active_connections -= 1
492
- elif msg == "proc_returned":
493
- returned_procs += 1
494
- logger.debug(f"Stopped processes: {returned_procs}/{procs} ")
495
493
  elif msg == "task_done":
496
- returned_threads += 1
497
- except queue.Empty:
498
- pass
499
-
500
- # check if all procs returned, then exit
501
- if returned_procs >= procs or (
502
- returned_threads > 0 and returned_threads >= active_connections
503
- ):
504
- if msg == "task_done":
505
- logger.info("Requested iteration/duration limit reached")
506
- gracefully_shutdown()
507
- elif msg == "proc_returned":
508
- logger.debug("All procs returned")
509
- gracefully_shutdown(by_keyinterrupt=True)
494
+ task_done_threads += 1
510
495
  elif isinstance(msg, Exception):
511
- logger.error(f"error_type={msg.__class__.__name__}, msg={msg}")
512
- sys.exit(1)
496
+ logger.error(f"error_type={msg.__class__.__name__}, {msg=}")
497
+ gracefully_shutdown()
513
498
  else:
514
499
  logger.error(f"unrecognized message: {msg}")
515
- sys.exit(1)
500
+ gracefully_shutdown()
501
+
502
+ except queue.Empty:
503
+ pass
504
+
505
+ if sigterm_received:
506
+ gracefully_shutdown()
507
+
508
+ if task_done_threads > 0 and task_done_threads >= active_connections:
509
+ logger.info("Requested iteration/duration limit reached")
510
+ gracefully_shutdown()
516
511
 
517
512
  if time.time() >= report_time:
518
513
  cpu_util = cpu_percent()
@@ -532,49 +527,131 @@ def run(
532
527
  if max_rate and report:
533
528
  current_rate = report[0][6] # __cycle__ period_ops/s
534
529
 
535
- # approximate how many threads are needed to get
536
- # to the desired max_rate given the current QPS rate
537
- # and current threads count
538
- extrapolated_cc = int(max_rate / (current_rate / current_cc))
539
-
540
- # adjust the thread count if there is a difference
541
- # between the current thread count and the calculated
542
- # thread count, but not if there is one such operation already
543
- # running, that is, not if there's an operation that is slow due
544
- # to a long ramp_time.
545
- if (
546
- extrapolated_cc - current_cc
547
- and time.time() >= pause_for_ramp_time
548
- ):
549
- Thread(
550
- target=launch_or_kill_workers,
551
- daemon=True,
552
- args=(
553
- queues,
554
- ramp_time,
555
- extrapolated_cc - current_cc,
556
- procs,
557
- iterations_per_thread,
558
- concurrency,
559
- ),
560
- ).start()
561
-
562
- # make sure we will not add/remove threads while the newly
563
- # created thread is still working
564
- pause_for_ramp_time = time.time() + ramp_time + 2 * FREQUENCY
565
-
566
- logger.warning(
567
- f"Calculating max_rate: desired max_rate: {max_rate}, "
568
- f"current_rate: {report[0][6]}, current_cc = {current_cc}, "
569
- f"extrapolated_cc = {extrapolated_cc}, "
570
- f"difference: {extrapolated_cc-current_cc}"
571
- )
572
- current_cc = extrapolated_cc
573
-
574
- # ramp_time is only considered for reaching the desired max_rate.
575
- # For adjustments over time, we want the changes to happen immediately
576
- # and not smoothed out over the initial ramp_time value
577
- ramp_time = 0
530
+ if time.time() > pause_for_ramp_time:
531
+
532
+ if current_rate < max_rate * 0.99:
533
+ if think_time > 0:
534
+
535
+ think_time = (
536
+ math.floor(
537
+ think_time * current_rate / max_rate * 1000
538
+ )
539
+ / 1000
540
+ )
541
+
542
+ pause_for_ramp_time = time.time() + 3 * FREQUENCY
543
+
544
+ logger.info(
545
+ f"Calculating max_rate: {max_rate=} {current_rate=}, {current_cc=} {think_time=}"
546
+ )
547
+
548
+ else:
549
+ if exhaust_warning > 3:
550
+ logger.warning("Pointless to add any more threads")
551
+ else:
552
+
553
+ # approximate how many threads are needed to get
554
+ # to the desired max_rate given the current QPS rate
555
+ # and current thread count
556
+ # increase by 5%
557
+ extrapolated_cc = int(
558
+ max_rate / (current_rate / current_cc) * 1.05
559
+ )
560
+
561
+ exhaust_warning += 1
562
+
563
+ change = max(0, extrapolated_cc - current_cc)
564
+
565
+ Thread(
566
+ target=launch_or_kill_workers,
567
+ daemon=True,
568
+ args=(
569
+ supervisor_queues,
570
+ ramp_time,
571
+ change,
572
+ procs,
573
+ iterations_per_thread,
574
+ concurrency,
575
+ ),
576
+ ).start()
577
+
578
+ # give enough time for newly created threads to settle
579
+ # before new calculations are performed
580
+ pause_for_ramp_time = (
581
+ time.time() + ramp_time + 6 * FREQUENCY
582
+ )
583
+
584
+ logger.info(
585
+ f"Calculating max_rate: {max_rate=} {current_rate=}, {current_cc=} {change=}"
586
+ )
587
+ current_cc += change
588
+
589
+ # ramp_time is only considered for reaching the desired max_rate.
590
+ # For adjustments over time, we want the changes to happen immediately
591
+ # and not smoothed out over the initial ramp_time value
592
+ ramp_time = 0
593
+
594
+ for q in supervisor_queues.values():
595
+ q.put((Action.THINK_TIME, think_time))
596
+
597
+ elif current_rate * 0.99 > max_rate:
598
+ if think_time > 1:
599
+ # pointless to add more time, remove threads instead
600
+
601
+ # decrease count by 5%
602
+ change = int(current_cc * -0.05)
603
+
604
+ Thread(
605
+ target=launch_or_kill_workers,
606
+ daemon=True,
607
+ args=(
608
+ supervisor_queues,
609
+ ramp_time,
610
+ change,
611
+ procs,
612
+ iterations_per_thread,
613
+ concurrency,
614
+ ),
615
+ ).start()
616
+
617
+ # give enough time for newly created threads to settle
618
+ # before new calculations are performed
619
+ pause_for_ramp_time = (
620
+ time.time() + ramp_time + 6 * FREQUENCY
621
+ )
622
+
623
+ logger.info(
624
+ f"Calculating max_rate: {max_rate=} {current_rate=}, {current_cc=} {change=}"
625
+ )
626
+ current_cc += change
627
+
628
+ # ramp_time is only considered for reaching the desired max_rate.
629
+ # For adjustments over time, we want the changes to happen immediately
630
+ # and not smoothed out over the initial ramp_time value
631
+ ramp_time = 0
632
+ think_time = 0
633
+
634
+ else:
635
+ # add think_time to slow it down a bit
636
+
637
+ if think_time == 0:
638
+
639
+ think_time = round(
640
+ 0.01 * current_rate / max_rate, 4
641
+ )
642
+ else:
643
+ think_time = round(
644
+ think_time * current_rate / max_rate, 4
645
+ )
646
+
647
+ pause_for_ramp_time = time.time() + 3 * FREQUENCY
648
+
649
+ logger.info(
650
+ f"Calculating max_rate: {max_rate=} {current_rate=}, {current_cc=} {think_time=}"
651
+ )
652
+
653
+ for q in supervisor_queues.values():
654
+ q.put((Action.THINK_TIME, think_time))
578
655
 
579
656
  centroids = stats.get_centroids()
580
657
 
@@ -623,61 +700,63 @@ def supervisor(
623
700
  offset: int,
624
701
  id: int,
625
702
  ):
626
- def gracefully_return(msg):
627
- # wait for Threads to return before
628
- # letting the Process MainThread return
629
- # threading.enumerate()
630
- for x in threads:
631
- if x.is_alive():
632
- from_proc_q.put("poison_pill")
633
-
634
- for x in threads:
635
- if x.is_alive():
636
- x.join()
637
-
638
- # send notification to MainThread
639
- to_main_q.put(msg)
640
-
641
- logger.debug(f"PROC-{id} terminated")
642
- return
643
-
644
703
  logger.setLevel(log_level)
645
- logger.debug(f"PROC-{id} started")
704
+ logger.debug(f"Supervisor-{id} started")
646
705
 
647
- threads: list[Thread] = []
706
+ worker_threads: list[Thread] = []
648
707
  from_proc_q = mp.Queue()
649
708
 
650
709
  # capture KeyboardInterrupt and do nothing
651
710
  signal.signal(signal.SIGINT, signal.SIG_IGN)
652
711
 
712
+ global think_time
713
+ think_time = 0
714
+
653
715
  while True:
654
716
  msg = from_main_q.get(block=True)
655
717
 
656
- if msg == "proc_end":
657
- logger.debug(f"PROC-{id} terminating...")
658
- gracefully_return("proc_returned")
659
- return
660
- elif msg == "kill_one":
661
- from_proc_q.put("poison_pill")
662
- elif isinstance(msg, tuple):
663
- t = Thread(
664
- target=worker,
665
- daemon=True,
666
- args=(
667
- to_main_q,
668
- from_proc_q,
669
- log_level,
670
- conn_info,
671
- driver,
672
- workload,
673
- args,
674
- conn_duration,
675
- offset,
676
- *msg,
677
- ),
678
- )
679
- t.start()
680
- threads.append(t)
718
+ match msg[0]:
719
+ case Action.POISON_PILL: # poison pill
720
+ logger.debug(f"Supervisor-{id} terminating...")
721
+
722
+ # wait for Threads to return before
723
+ # letting the Supervisor MainThread return
724
+ for x in worker_threads:
725
+ if x.is_alive():
726
+ from_proc_q.put(Action.POISON_PILL)
727
+
728
+ for x in worker_threads:
729
+ if x.is_alive():
730
+ x.join()
731
+
732
+ logger.debug(f"Supervisor-{id} terminated")
733
+ return
734
+
735
+ case Action.KILL_ONE: # kill_one
736
+ from_proc_q.put(Action.POISON_PILL)
737
+
738
+ case Action.THINK_TIME: # set think_time
739
+ think_time = msg[1]
740
+
741
+ case Action.NEW_WORKER: # add new worker
742
+ t = Thread(
743
+ target=worker,
744
+ daemon=True,
745
+ args=(
746
+ to_main_q,
747
+ from_proc_q,
748
+ log_level,
749
+ conn_info,
750
+ driver,
751
+ workload,
752
+ args,
753
+ conn_duration,
754
+ offset,
755
+ *msg[1],
756
+ ),
757
+ )
758
+ t.start()
759
+ worker_threads.append(t)
681
760
 
682
761
 
683
762
  def worker(
@@ -695,14 +774,13 @@ def worker(
695
774
  concurrency: int = 0,
696
775
  ):
697
776
  def gracefully_return(msg):
698
- # send notification to MainThread
699
- to_main_q.put(msg)
700
777
  # send final stats
701
778
  to_main_q.put(ws.get_tdigest_ndarray(), block=False)
702
779
 
703
- logger.debug(f"Thread ID {id} terminated")
780
+ # send notification to MainThread
781
+ to_main_q.put(msg)
704
782
 
705
- return
783
+ logger.debug(f"Thread ID {id} returned")
706
784
 
707
785
  logger.setLevel(log_level)
708
786
 
@@ -728,6 +806,15 @@ def worker(
728
806
  to_main_q.put("init")
729
807
 
730
808
  while True:
809
+ # listen for termination messages (poison pill)
810
+ try:
811
+ from_proc_q.get(block=False)
812
+ logger.debug("Poison pill received, terminating...")
813
+ gracefully_return("got_killed")
814
+ return
815
+ except queue.Empty:
816
+ pass
817
+
731
818
  if conn_duration:
732
819
  # reconnect every conn_duration +/- 20%
733
820
  conn_endtime = time.time() + int(conn_duration * random.uniform(0.8, 1.2))
@@ -743,7 +830,6 @@ def worker(
743
830
  run_init = False
744
831
 
745
832
  if hasattr(w, "setup") and callable(w.setup):
746
- logger.debug("Executing setup() function")
747
833
  run_transaction(
748
834
  conn,
749
835
  lambda conn: w.setup(
@@ -763,8 +849,9 @@ def worker(
763
849
  # listen for termination messages (poison pill)
764
850
  try:
765
851
  from_proc_q.get(block=False)
766
- logger.debug("Poison pill received")
767
- return gracefully_return("got_killed")
852
+ logger.debug("Poison pill received, terminating...")
853
+ gracefully_return("got_killed")
854
+ return
768
855
  except queue.Empty:
769
856
  pass
770
857
 
@@ -806,6 +893,10 @@ def worker(
806
893
 
807
894
  ws.add_latency_measurement("__cycle__", time.time() - cycle_start)
808
895
 
896
+ if think_time > 0:
897
+ time.sleep(think_time)
898
+ ws.add_latency_measurement("__think_time__", think_time)
899
+
809
900
  if to_main_q.full():
810
901
  logger.error("=========== Q FULL!!!! ======================")
811
902
  if time.time() >= stat_time:
@@ -853,13 +944,13 @@ def listen_to_pipe(queues, ramp_time, procs, iterations_per_thread, concurrency)
853
944
  # https://stackoverflow.com/questions/39089776/python-read-named-pipe
854
945
 
855
946
  try:
856
- os.mkfifo(FIFO)
947
+ os.mkfifo(DBWORKLOAD_PIPE)
857
948
  except OSError as oe:
858
949
  if oe.errno != errno.EEXIST:
859
950
  raise
860
951
 
861
952
  while True:
862
- with open(FIFO) as fifo:
953
+ with open(DBWORKLOAD_PIPE) as fifo:
863
954
  for line in fifo:
864
955
  try:
865
956
  t = int(line)
@@ -882,6 +973,7 @@ def listen_to_pipe(queues, ramp_time, procs, iterations_per_thread, concurrency)
882
973
 
883
974
 
884
975
  def log_and_sleep(e: Exception):
976
+ raise e
885
977
  logger.error(f"error_type={e.__class__.__name__}, msg={e}")
886
978
  logger.info("Sleeping for %s seconds" % (DEFAULT_SLEEP))
887
979
  time.sleep(DEFAULT_SLEEP)
@@ -48,10 +48,19 @@ NOT_NULL_MAX = 40
48
48
 
49
49
  logger = logging.getLogger("dbworkload")
50
50
 
51
+ from enum import IntEnum
52
+
51
53
  from prometheus_client.core import REGISTRY, HistogramMetricFamily
52
54
  from prometheus_client.registry import Collector
53
55
 
54
56
 
57
+ class Action(IntEnum):
58
+ KILL_ONE = 7
59
+ THINK_TIME = 2
60
+ POISON_PILL = 9
61
+ NEW_WORKER = 1
62
+
63
+
55
64
  class Stats:
56
65
  """Print workload stats
57
66
  and export the stats as Prometheus endpoints
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "dbworkload"
3
- version = "0.9.0"
3
+ version = "0.9.2.dev1"
4
4
  description = "Workload framework"
5
5
  authors = ["Fabio Ghirardello"]
6
6
  license = "GPLv3+"
@@ -17,7 +17,7 @@ classifiers = [
17
17
  dbworkload = 'dbworkload.cli.main:app'
18
18
 
19
19
  [tool.poetry.dependencies]
20
- python = "^3.8"
20
+ python = "^3.11"
21
21
  pandas = "*"
22
22
  tabulate = "*"
23
23
  numpy = "*"
@@ -38,6 +38,7 @@ plotext = "*"
38
38
  plotly = "*"
39
39
  jinja2 = "*"
40
40
  sqlparse = "*"
41
+ psutil = "^7.0.0"
41
42
 
42
43
  [tool.poetry.extras]
43
44
  all = ["psycopg", "psycopg-binary", "mysql-connector-python", "mariadb", "oracledb", "pyodbc", "pymongo", "cassandra-driver", "google-cloud-spanner"]
@@ -50,6 +51,11 @@ mongo = ["pymongo"]
50
51
  cassandra = ["cassandra-driver"]
51
52
  spanner = ["google-cloud-spanner"]
52
53
 
54
+ [tool.poetry.group.dev.dependencies]
55
+ mkdocs = "^1.6.1"
56
+ mkdocs-material = "^9.6.14"
57
+ mkdocs-click = "^0.9.0"
58
+
53
59
  [build-system]
54
60
  requires = ["poetry-core"]
55
61
  build-backend = "poetry.core.masonry.api"
File without changes
File without changes