dbworkload 0.8.3__tar.gz → 0.9.0__tar.gz

{dbworkload-0.8.3 → dbworkload-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbworkload
-Version: 0.8.3
+Version: 0.9.0
 Summary: Workload framework
 License: GPLv3+
 Author: Fabio Ghirardello

{dbworkload-0.8.3 → dbworkload-0.9.0}/dbworkload/cli/main.py

@@ -16,7 +16,6 @@ import yaml
 
 import dbworkload.cli.util
 import dbworkload.models.run
-import dbworkload.models.util
 import dbworkload.utils.common
 from dbworkload.cli.dep import EPILOG, ConnInfo, Param
 
@@ -153,6 +152,9 @@ def run(
         "--bins",
         help="comma separated list of ints defining the histogram bins.",
     ),
+    delay_stats: int = typer.Option(
+        0, "--delay-stats", help="Start collecting stats after the speciied seconds."
+    ),
     log_level: LogLevel = Param.LogLevel,
 ):
     logger.setLevel(log_level.upper())
@@ -259,6 +261,7 @@ def run(
         save,
         schedule,
         histogram_bins,
+        delay_stats,
         log_level.upper(),
     )
 

{dbworkload-0.8.3 → dbworkload-0.9.0}/dbworkload/models/run.py

@@ -15,6 +15,7 @@ from threading import Thread
 
 import numpy as np
 import tabulate
+from psutil import cpu_percent, virtual_memory
 
 import dbworkload.utils.common
 from dbworkload.cli.dep import ConnInfo
@@ -31,6 +32,7 @@ from dbworkload.cli.dep import ConnInfo
 DEFAULT_SLEEP = 3
 MAX_RETRIES = 3
 FREQUENCY = 10
+STATS_BUFFER = 8
 
 FIFO = "dbworkload.pipe"
 
@@ -102,7 +104,7 @@ def signal_handler(sig, frame):
         sys.exit(1)
 
     force_exit = True
-    
+
     # send the poison pill to each proc.
     # if dbworkload cannot graceful shutdown due
     # to processes being still in the init phase
@@ -187,11 +189,11 @@ def run(
     save: bool,
     schedule: list,
     histogram_bins: list,
+    delay_stats: int,
     log_level: str,
 ):
     def gracefully_shutdown(by_keyinterrupt: bool = False):
-
-        logger.info("Gracefully shutting down...")
+        logger.debug("Gracefully shutting down...")
 
         end_time = int(time.time())
         # _s = stats_received
@@ -208,7 +210,7 @@ def run(
                 if x.is_alive():
                     x.join()
 
-        # Commenting the below as in theory there shouldn't be any stats that 
+        # Commenting the below as in theory there shouldn't be any stats that
         # comes in *after* the PROC returns. That is, all threads send stats
         # when all threads are returned, then the supervisor returns.
         # while True:
@@ -227,7 +229,7 @@ def run(
         #         break
 
         # now that we have all stat reports, calculate the stats one last time.
-        report = stats.calculate_stats(active_connections, end_time)
+        report = stats.calculate_stats(active_connections, end_time - delay_stats)
         centroids = stats.get_centroids()
 
         if save:
@@ -249,7 +251,7 @@ def run(
 
         # the final stat report summarizes the entire test run
         final_stats_report = tabulate.tabulate(
-            stats.calculate_final_stats(active_connections, end_time),
+            stats.calculate_final_stats(active_connections, stats.endtime),
             FINAL_HEADERS,
             tablefmt="simple_outline",
             intfmt=",",
@@ -267,6 +269,7 @@ def run(
                 ["iterations", iterations],
                 ["ramp", ramp],
                 ["args", args],
+                ["delay_stats", delay_stats],
             ],
             headers=["Parameter", "Value"],
         )
@@ -384,10 +387,10 @@ def run(
         )
         supervisors[x].start()
 
-    # report time happens 2 seconds after the stats are received.
+    # report time happens STATS_BUFFER seconds after the stats are received.
     # we add this buffer to make sure we get all the stats reports
     # from each thread before we aggregate and display
-    report_time = start_time + FREQUENCY + 2
+    report_time = start_time + FREQUENCY + STATS_BUFFER + delay_stats
 
     returned_procs = 0
     active_connections = 0
@@ -426,7 +429,7 @@ def run(
             ramp_time = dur
 
         logger.info(
-            f"Starting schedule {i+1}/{len(schedule)}: cc={cc}, max_rate={max_rate}, ramp={ramp_time}, dur={dur}"
+            f"Starting schedule {i+1}/{len(schedule)}: {cc=}, {max_rate=}, {ramp_time=}, {dur=}"
         )
 
         # always make sure that a duration is specified, even if none was passed
@@ -488,7 +491,7 @@ def run(
                     active_connections -= 1
                 elif msg == "proc_returned":
                     returned_procs += 1
-                    logger.info(f"Stopped processes: {returned_procs}/{procs} ")
+                    logger.debug(f"Stopped processes: {returned_procs}/{procs} ")
                 elif msg == "task_done":
                     returned_threads += 1
             except queue.Empty:
@@ -512,11 +515,15 @@ def run(
                     sys.exit(1)
 
             if time.time() >= report_time:
-                # if stats_received != active_connections:
-                #     logger.warning("didn't receive all stats reports yet")
+                cpu_util = cpu_percent()
+                vmem = virtual_memory().percent
+                if stats_received != active_connections or cpu_util > 70 or vmem > 70:
+                    logger.warning(
+                        f"{stats_received=}, expected={active_connections}. CPU Util={cpu_util}%, Memory={vmem}%"
+                    )
 
-                # remove the 2 seconds added
-                endtime = int(time.time()) - 2
+                # remove the STATS_BUFFER seconds added
+                endtime = int(time.time() - delay_stats) - STATS_BUFFER
 
                 report = stats.calculate_stats(active_connections, endtime)
 
@@ -591,7 +598,7 @@ def run(
                 report_time += FREQUENCY
 
             # pause briefly to prevent the loop from overheating the CPU
-            time.sleep(.001)
+            time.sleep(0.001)
 
     gracefully_shutdown()
 

{dbworkload-0.8.3 → dbworkload-0.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dbworkload"
-version = "0.8.3"
+version = "0.9.0"
 description = "Workload framework"
 authors = ["Fabio Ghirardello"]
 license = "GPLv3+"