testrium 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- testrium-0.1.0/PKG-INFO +72 -0
- testrium-0.1.0/README.md +52 -0
- testrium-0.1.0/pyproject.toml +27 -0
- testrium-0.1.0/testrium/__init__.py +0 -0
- testrium-0.1.0/testrium/common/sql_pool.py +30 -0
- testrium-0.1.0/testrium/common/validate_configs.py +1 -0
- testrium-0.1.0/testrium/core.py +177 -0
- testrium-0.1.0/testrium/tools/benchmark.py +113 -0
- testrium-0.1.0/testrium/tools/dashborad.py +195 -0
- testrium-0.1.0/testrium/tools/test_results.db +0 -0
- testrium-0.1.0/testrium/utils.py +140 -0
testrium-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: testrium
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A modern, advanced testing framework.
|
|
5
|
+
Home-page: https://github.com/letalboy/testrium
|
|
6
|
+
License: MIT
|
|
7
|
+
Author: Cristian Camargo
|
|
8
|
+
Author-email: ccf@cdone.com.br
|
|
9
|
+
Requires-Python: >=3.10,<4.0
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Requires-Dist: pandas (>=2.2.2,<3.0.0)
|
|
16
|
+
Requires-Dist: toml (>=0.10.2,<0.11.0)
|
|
17
|
+
Project-URL: Documentation, https://github.com/letalboy/testrium#readme
|
|
18
|
+
Project-URL: Repository, https://github.com/letalboy/testrium
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
<div style="display: flex; align-items: center;">
|
|
22
|
+
<img src="https://github.com/letalboy/Spectrum/assets/63066865/cf9c60e7-9eba-474c-8cc1-b246e661fd5c" alt="Logo" width="200" height="200" style="margin-right: 20px;">
|
|
23
|
+
<div>
|
|
24
|
+
<h1>Spectrum</h1>
|
|
25
|
+
<p>Spectrum is a versatile library designed to test and analyze systems like Misny. It provides tools for data processing, monitoring, and ensuring system integrity, making it an essential part of your development workflow.</p>
|
|
26
|
+
</div>
|
|
27
|
+
</div>
|
|
28
|
+
<div>
|
|
29
|
+
<p>Key points:</p>
|
|
30
|
+
<ul>
|
|
31
|
+
<li>Better Organization Of Tests</li>
|
|
32
|
+
<li>Easy Test Of Async Or Paralelized Dependent Tests</li>
|
|
33
|
+
<li>Historic Performance Comparation and Evaluation</li>
|
|
34
|
+
<li>Allow Custom Behavior Though Callbacks Systems</li>
|
|
35
|
+
<li>Allow Manipulation Of The Test Flows</li>
|
|
36
|
+
<li>High COnfigurable And Expancible</li>
|
|
37
|
+
</ul>
|
|
38
|
+
</div>
|
|
39
|
+
|
|
40
|
+
To use is simple:
|
|
41
|
+
|
|
42
|
+
1. activate the poetry shell using:
|
|
43
|
+
|
|
44
|
+
```shell
|
|
45
|
+
poetry shell
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
2. go the the test folder and run:
|
|
49
|
+
|
|
50
|
+
```shell
|
|
51
|
+
poetry run spectrum
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
This should auto collect the test and run them automatically!
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
Base Idea:
|
|
59
|
+
- Read the folder of test folders
|
|
60
|
+
- Collect each one
|
|
61
|
+
- Run them based in the priority config file in the test toml if present
|
|
62
|
+
- Create a db to store tests results locally or if configured in the toml then send to a remote server
|
|
63
|
+
- Run the tests, save the results
|
|
64
|
+
- Compare the results witht he performance of the old tests to see if the test increase or decrese efficience.
|
|
65
|
+
- Create a way to evaluate the performance in multiple machines comparing the efficience without issues.
|
|
66
|
+
|
|
67
|
+
TODO:
|
|
68
|
+
- Create a mechanism to add events through a test
|
|
69
|
+
- Events will allow to indentify if we complete the required step
|
|
70
|
+
- Cross events, are events were we want to trace the process from one point to another and how much time it takes to
|
|
71
|
+
this will be able to be done using a structure that generate a event hash for each event so then we can trace it to the
|
|
72
|
+
other place.
|
testrium-0.1.0/README.md
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
<div style="display: flex; align-items: center;">
|
|
2
|
+
<img src="https://github.com/letalboy/Spectrum/assets/63066865/cf9c60e7-9eba-474c-8cc1-b246e661fd5c" alt="Logo" width="200" height="200" style="margin-right: 20px;">
|
|
3
|
+
<div>
|
|
4
|
+
<h1>Spectrum</h1>
|
|
5
|
+
<p>Spectrum is a versatile library designed to test and analyze systems like Misny. It provides tools for data processing, monitoring, and ensuring system integrity, making it an essential part of your development workflow.</p>
|
|
6
|
+
</div>
|
|
7
|
+
</div>
|
|
8
|
+
<div>
|
|
9
|
+
<p>Key points:</p>
|
|
10
|
+
<ul>
|
|
11
|
+
<li>Better Organization Of Tests</li>
|
|
12
|
+
<li>Easy Test Of Async Or Paralelized Dependent Tests</li>
|
|
13
|
+
<li>Historic Performance Comparation and Evaluation</li>
|
|
14
|
+
<li>Allow Custom Behavior Though Callbacks Systems</li>
|
|
15
|
+
<li>Allow Manipulation Of The Test Flows</li>
|
|
16
|
+
<li>High COnfigurable And Expancible</li>
|
|
17
|
+
</ul>
|
|
18
|
+
</div>
|
|
19
|
+
|
|
20
|
+
To use is simple:
|
|
21
|
+
|
|
22
|
+
1. activate the poetry shell using:
|
|
23
|
+
|
|
24
|
+
```shell
|
|
25
|
+
poetry shell
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
2. go the the test folder and run:
|
|
29
|
+
|
|
30
|
+
```shell
|
|
31
|
+
poetry run spectrum
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
This should auto collect the test and run them automatically!
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
Base Idea:
|
|
39
|
+
- Read the folder of test folders
|
|
40
|
+
- Collect each one
|
|
41
|
+
- Run them based in the priority config file in the test toml if present
|
|
42
|
+
- Create a db to store tests results locally or if configured in the toml then send to a remote server
|
|
43
|
+
- Run the tests, save the results
|
|
44
|
+
- Compare the results witht he performance of the old tests to see if the test increase or decrese efficience.
|
|
45
|
+
- Create a way to evaluate the performance in multiple machines comparing the efficience without issues.
|
|
46
|
+
|
|
47
|
+
TODO:
|
|
48
|
+
- Create a mechanism to add events through a test
|
|
49
|
+
- Events will allow to indentify if we complete the required step
|
|
50
|
+
- Cross events, are events were we want to trace the process from one point to another and how much time it takes to
|
|
51
|
+
this will be able to be done using a structure that generate a event hash for each event so then we can trace it to the
|
|
52
|
+
other place.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[tool.poetry]
|
|
2
|
+
name = "testrium"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "A modern, advanced testing framework."
|
|
5
|
+
authors = ["Cristian Camargo <ccf@cdone.com.br>"]
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
homepage = "https://github.com/letalboy/testrium"
|
|
8
|
+
repository = "https://github.com/letalboy/testrium"
|
|
9
|
+
license = "MIT"
|
|
10
|
+
|
|
11
|
+
[tool.poetry.dependencies]
|
|
12
|
+
python = "^3.10"
|
|
13
|
+
toml = "^0.10.2"
|
|
14
|
+
pandas = "^2.2.2"
|
|
15
|
+
|
|
16
|
+
[tool.poetry.group.dev.dependencies]
|
|
17
|
+
pytest = "^8.2.2"
|
|
18
|
+
|
|
19
|
+
[build-system]
|
|
20
|
+
requires = ["poetry-core"]
|
|
21
|
+
build-backend = "poetry.core.masonry.api"
|
|
22
|
+
|
|
23
|
+
[tool.poetry.scripts]
|
|
24
|
+
testrium = "testrium.core:main"
|
|
25
|
+
|
|
26
|
+
[tool.poetry.urls]
|
|
27
|
+
Documentation = "https://github.com/letalboy/testrium#readme"
|
|
File without changes
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import sqlite3
|
|
2
|
+
from queue import Queue
|
|
3
|
+
from threading import Lock
|
|
4
|
+
|
|
5
|
+
class SQLiteConnectionPool:
|
|
6
|
+
def __init__(self, max_connections:int, database_path:str):
|
|
7
|
+
self.data_base = database_path
|
|
8
|
+
self.max_connections = max_connections
|
|
9
|
+
self.connections = Queue(max_connections)
|
|
10
|
+
self.lock = Lock()
|
|
11
|
+
|
|
12
|
+
for i in range(max_connections):
|
|
13
|
+
connection = sqlite3.connect(self.data_base, check_same_thread=False)
|
|
14
|
+
self.connections.put(connection)
|
|
15
|
+
|
|
16
|
+
def get_connection(self):
|
|
17
|
+
with self.lock:
|
|
18
|
+
if self.connections.empty():
|
|
19
|
+
raise Exception("No available connections in the pool")
|
|
20
|
+
connection = self.connections.get()
|
|
21
|
+
return connection
|
|
22
|
+
|
|
23
|
+
def release_connection(self, connection):
|
|
24
|
+
with self.lock:
|
|
25
|
+
self.connections.put(connection)
|
|
26
|
+
|
|
27
|
+
def close_all_connections(self):
|
|
28
|
+
while not self.connections.empty():
|
|
29
|
+
connection = self.connections.get()
|
|
30
|
+
connection.close()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# TODO >>> This module have the goal to validate the configs toml and ensure it has the correct and desired keywords
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import time
|
|
3
|
+
import importlib.util
|
|
4
|
+
import toml
|
|
5
|
+
from colorama import init, Fore, Style
|
|
6
|
+
import shutil
|
|
7
|
+
from .utils import Events_Manager
|
|
8
|
+
|
|
9
|
+
# Initialize colorama
|
|
10
|
+
init(autoreset=True)
|
|
11
|
+
|
|
12
|
+
# Loads the config for each test:
|
|
13
|
+
def load_config(config_path):
|
|
14
|
+
print(f"{Fore.CYAN}Loading config from {config_path}")
|
|
15
|
+
with open(config_path, 'r') as file:
|
|
16
|
+
config = toml.load(file)
|
|
17
|
+
return config
|
|
18
|
+
|
|
19
|
+
# Loads the test functions:
|
|
20
|
+
def load_test_functions(dir_path):
|
|
21
|
+
test_functions = {}
|
|
22
|
+
for filename in os.listdir(dir_path):
|
|
23
|
+
if filename.startswith('_') or not filename.endswith('.py'):
|
|
24
|
+
continue
|
|
25
|
+
module_name = filename[:-3]
|
|
26
|
+
file_path = os.path.join(dir_path, filename)
|
|
27
|
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
|
28
|
+
module = importlib.util.module_from_spec(spec)
|
|
29
|
+
spec.loader.exec_module(module)
|
|
30
|
+
for attr in dir(module):
|
|
31
|
+
if attr == "Events_Manager":
|
|
32
|
+
continue
|
|
33
|
+
func = getattr(module, attr)
|
|
34
|
+
if callable(func) and not attr.startswith('_'):
|
|
35
|
+
test_functions[attr] = func
|
|
36
|
+
return test_functions
|
|
37
|
+
|
|
38
|
+
# Run setup of the test group
|
|
39
|
+
def run_setup(setup_path):
|
|
40
|
+
spec = importlib.util.spec_from_file_location("setup", setup_path)
|
|
41
|
+
setup_module = importlib.util.module_from_spec(spec)
|
|
42
|
+
spec.loader.exec_module(setup_module)
|
|
43
|
+
setup_module.main()
|
|
44
|
+
|
|
45
|
+
# Print the nice Pass or Fail banners
|
|
46
|
+
def print_banner(message, color=Fore.GREEN):
|
|
47
|
+
term_width = shutil.get_terminal_size().columns
|
|
48
|
+
message_length = len(message)
|
|
49
|
+
padding_length = (term_width - message_length) // 2
|
|
50
|
+
banner = "=" * padding_length + message + "=" * padding_length
|
|
51
|
+
|
|
52
|
+
# Ensure the banner fits exactly in the terminal width
|
|
53
|
+
if len(banner) < term_width:
|
|
54
|
+
banner += "=" * (term_width - len(banner))
|
|
55
|
+
|
|
56
|
+
print(color + banner)
|
|
57
|
+
|
|
58
|
+
def main():
|
|
59
|
+
base_dir = os.getcwd()
|
|
60
|
+
print(f"{Fore.GREEN}Current working directory: {base_dir}")
|
|
61
|
+
|
|
62
|
+
dir_names = os.listdir(base_dir)
|
|
63
|
+
valid_test_dirs = []
|
|
64
|
+
|
|
65
|
+
# -> Discover valid test folders with configs inside
|
|
66
|
+
for dir_name in dir_names:
|
|
67
|
+
dir_path = os.path.join(base_dir, dir_name)
|
|
68
|
+
|
|
69
|
+
if not os.path.isdir(dir_path):
|
|
70
|
+
print(f"{Fore.RED} File: {dir_name} is not a folder")
|
|
71
|
+
continue
|
|
72
|
+
|
|
73
|
+
setup_path = os.path.join(dir_path, 'setup.py')
|
|
74
|
+
config_path = os.path.join(dir_path, 'config.toml')
|
|
75
|
+
|
|
76
|
+
if not (os.path.exists(setup_path) and os.path.exists(config_path)):
|
|
77
|
+
print(f"{Fore.RED} File: {dir_name} invalid test")
|
|
78
|
+
continue
|
|
79
|
+
|
|
80
|
+
print(f"{Fore.MAGENTA}Found test directory: {dir_name}")
|
|
81
|
+
valid_test_dirs.append(dir_name)
|
|
82
|
+
|
|
83
|
+
tests_finded = len(valid_test_dirs)
|
|
84
|
+
|
|
85
|
+
# -> Early retun if not find any test:
|
|
86
|
+
if tests_finded > 0:
|
|
87
|
+
print(f"{Fore.GREEN} Find: {tests_finded:.0f} valid test groups!")
|
|
88
|
+
for test_group in valid_test_dirs:
|
|
89
|
+
print(f"{Fore.GREEN} - {test_group}")
|
|
90
|
+
else:
|
|
91
|
+
print(f"{Fore.RED} No valid test groups finded!")
|
|
92
|
+
return
|
|
93
|
+
|
|
94
|
+
# TODO >>> Find a way to log the milestones completed for each test and with this understand what they relate to
|
|
95
|
+
|
|
96
|
+
# -> Run the valid tests:
|
|
97
|
+
for dir_name in valid_test_dirs:
|
|
98
|
+
|
|
99
|
+
Events_Manager(Unit="", path=base_dir).drop_events_table()
|
|
100
|
+
|
|
101
|
+
dir_path = os.path.join(base_dir, dir_name)
|
|
102
|
+
setup_path = os.path.join(dir_path, 'setup.py')
|
|
103
|
+
|
|
104
|
+
#> Load test configs
|
|
105
|
+
config_path = os.path.join(dir_path, 'config.toml')
|
|
106
|
+
configs = load_config(config_path)
|
|
107
|
+
|
|
108
|
+
#> Load Units
|
|
109
|
+
confs = configs['Configs']
|
|
110
|
+
units = confs['units']
|
|
111
|
+
|
|
112
|
+
#> Load Units Predefinitions
|
|
113
|
+
total_units = 0
|
|
114
|
+
units_index = {}
|
|
115
|
+
for unit in units:
|
|
116
|
+
un = configs[f"{unit}"]
|
|
117
|
+
units_index[un["init"]] = {"name":f"{unit}", "events": f"{un['events']}"}
|
|
118
|
+
total_units += 1
|
|
119
|
+
|
|
120
|
+
print(f"Units loaded: {units_index}")
|
|
121
|
+
|
|
122
|
+
# TODO >>> Use the units order to setup the units one by one
|
|
123
|
+
# TODO >>> Create a meachanism to verify the events when they are required
|
|
124
|
+
|
|
125
|
+
print(f"{Fore.BLUE}Loading tests for {dir_name}...")
|
|
126
|
+
test_functions = load_test_functions(dir_path)
|
|
127
|
+
|
|
128
|
+
all_tests_passed = True
|
|
129
|
+
|
|
130
|
+
#> Run each test found in the test group
|
|
131
|
+
for test_name, test_func in test_functions.items():
|
|
132
|
+
print(f"{Fore.YELLOW}Running test: {test_name}")
|
|
133
|
+
start_time = time.time()
|
|
134
|
+
|
|
135
|
+
try:
|
|
136
|
+
if test_name == 'log_test_time':
|
|
137
|
+
test_func(dummy_function) # Pass a dummy function if required
|
|
138
|
+
elif test_name == 'verify_condition':
|
|
139
|
+
# TODO >>> Use this as a condition to verify if the requirements was completed for the test case
|
|
140
|
+
test_func(lambda: True) # Pass a lambda function if required
|
|
141
|
+
else:
|
|
142
|
+
test_func()
|
|
143
|
+
elapsed_time = time.time() - start_time
|
|
144
|
+
print(f"{Fore.GREEN}{test_name}: PASSED in {elapsed_time:.2f} seconds")
|
|
145
|
+
except Exception as e:
|
|
146
|
+
all_tests_passed = False
|
|
147
|
+
elapsed_time = time.time() - start_time
|
|
148
|
+
print(f"{Fore.RED}{test_name}: FAILED in {elapsed_time:.2f} seconds\nError: {e}")
|
|
149
|
+
|
|
150
|
+
#> Verify Events Completed By The Unit
|
|
151
|
+
for i in range(total_units):
|
|
152
|
+
unit = units_index[i]
|
|
153
|
+
unit_name = unit["name"]
|
|
154
|
+
if unit_name == "":
|
|
155
|
+
continue
|
|
156
|
+
unit_events = Events_Manager(Unit=unit_name, path=base_dir).List_Events()
|
|
157
|
+
for event in eval(unit["events"]):
|
|
158
|
+
if event not in unit_events:
|
|
159
|
+
print(f"event: [{event}] was not completed!")
|
|
160
|
+
all_tests_passed = False
|
|
161
|
+
|
|
162
|
+
if all_tests_passed:
|
|
163
|
+
print_banner(" PASS ", Fore.GREEN)
|
|
164
|
+
else:
|
|
165
|
+
print_banner(" FAILURE ", Fore.RED)
|
|
166
|
+
|
|
167
|
+
# Extra validation step that user migh want to define
|
|
168
|
+
def dummy_function():
|
|
169
|
+
"""
|
|
170
|
+
- This will allow to define a extra step in the verification
|
|
171
|
+
for example, read a file and validate if the test was a success.
|
|
172
|
+
see if the code did what it was suposed to do. etc..
|
|
173
|
+
"""
|
|
174
|
+
pass
|
|
175
|
+
|
|
176
|
+
if __name__ == "__main__":
|
|
177
|
+
main()
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import numpy as np
|
|
3
|
+
import os
|
|
4
|
+
import sqlite3
|
|
5
|
+
from tqdm import tqdm
|
|
6
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
7
|
+
|
|
8
|
+
def run_benchmark_multiple_times(benchmark_func, iterations=10, warmup=True):
|
|
9
|
+
if warmup:
|
|
10
|
+
benchmark_func()
|
|
11
|
+
|
|
12
|
+
times = []
|
|
13
|
+
for _ in tqdm(range(iterations), desc=benchmark_func.__name__, leave=False):
|
|
14
|
+
start_time = time.time()
|
|
15
|
+
benchmark_func()
|
|
16
|
+
end_time = time.time()
|
|
17
|
+
times.append(end_time - start_time)
|
|
18
|
+
|
|
19
|
+
return times # Return all times for visualization
|
|
20
|
+
|
|
21
|
+
def cpu_benchmark_single_core():
|
|
22
|
+
result = 0
|
|
23
|
+
for i in range(1, 1000000):
|
|
24
|
+
result += (i ** 0.5) ** 2
|
|
25
|
+
|
|
26
|
+
def cpu_benchmark_multicore(workers=4):
|
|
27
|
+
def worker_task(start, end):
|
|
28
|
+
result = 0
|
|
29
|
+
for i in range(start, end):
|
|
30
|
+
result += (i ** 0.5) ** 2
|
|
31
|
+
return result
|
|
32
|
+
|
|
33
|
+
chunk_size = 1000000 // workers
|
|
34
|
+
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
35
|
+
futures = [executor.submit(worker_task, i * chunk_size, (i + 1) * chunk_size) for i in range(workers)]
|
|
36
|
+
results = [future.result() for future in as_completed(futures)]
|
|
37
|
+
return sum(results)
|
|
38
|
+
|
|
39
|
+
def memory_benchmark():
|
|
40
|
+
array_size = 10000000
|
|
41
|
+
array = np.arange(array_size, dtype=np.float64)
|
|
42
|
+
array *= 2
|
|
43
|
+
|
|
44
|
+
def disk_benchmark():
|
|
45
|
+
with open('disk_benchmark.tmp', 'wb') as f:
|
|
46
|
+
f.write(os.urandom(500000000)) # Write 500MB
|
|
47
|
+
with open('disk_benchmark.tmp', 'rb') as f:
|
|
48
|
+
f.read()
|
|
49
|
+
os.remove('disk_benchmark.tmp')
|
|
50
|
+
|
|
51
|
+
def io_benchmark():
|
|
52
|
+
with open('io_benchmark.tmp', 'w') as f:
|
|
53
|
+
for _ in range(100000):
|
|
54
|
+
f.write('This is a test.\n')
|
|
55
|
+
os.remove('io_benchmark.tmp')
|
|
56
|
+
|
|
57
|
+
def peak_disk_write_benchmark(file_size_mb=500):
|
|
58
|
+
data = os.urandom(file_size_mb * 1024 * 1024) # Generate large data block
|
|
59
|
+
file_path = 'peak_disk_write.tmp'
|
|
60
|
+
|
|
61
|
+
start_time = time.time()
|
|
62
|
+
with open(file_path, 'wb') as f:
|
|
63
|
+
f.write(data)
|
|
64
|
+
f.flush() # Ensure all data is written
|
|
65
|
+
os.fsync(f.fileno()) # Flush OS buffer to disk
|
|
66
|
+
end_time = time.time()
|
|
67
|
+
|
|
68
|
+
os.remove(file_path) # Clean up
|
|
69
|
+
|
|
70
|
+
elapsed_time = end_time - start_time
|
|
71
|
+
throughput = file_size_mb / elapsed_time # MB/s
|
|
72
|
+
return throughput
|
|
73
|
+
|
|
74
|
+
def run_peak_disk_write_benchmark(iterations=5, file_size_mb=500):
|
|
75
|
+
throughputs = []
|
|
76
|
+
for _ in tqdm(range(iterations), desc="Peak Disk Write Benchmark", leave=False):
|
|
77
|
+
throughput = peak_disk_write_benchmark(file_size_mb)
|
|
78
|
+
throughputs.append(throughput)
|
|
79
|
+
return throughputs
|
|
80
|
+
|
|
81
|
+
def save_benchmark_results(machine_id, benchmark_name, times):
|
|
82
|
+
conn = sqlite3.connect('test_results.db')
|
|
83
|
+
cursor = conn.cursor()
|
|
84
|
+
cursor.execute('''
|
|
85
|
+
CREATE TABLE IF NOT EXISTS benchmark_samples (
|
|
86
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
87
|
+
machine_id TEXT,
|
|
88
|
+
benchmark_name TEXT,
|
|
89
|
+
time REAL
|
|
90
|
+
)
|
|
91
|
+
''')
|
|
92
|
+
for time in times:
|
|
93
|
+
cursor.execute('''
|
|
94
|
+
INSERT INTO benchmark_samples (machine_id, benchmark_name, time)
|
|
95
|
+
VALUES (?, ?, ?)
|
|
96
|
+
''', (machine_id, benchmark_name, time))
|
|
97
|
+
conn.commit()
|
|
98
|
+
conn.close()
|
|
99
|
+
|
|
100
|
+
# Example usage
|
|
101
|
+
cpu_times = run_benchmark_multiple_times(lambda: cpu_benchmark_multicore(workers=os.cpu_count()), iterations=150)
|
|
102
|
+
memory_times = run_benchmark_multiple_times(memory_benchmark, iterations=150)
|
|
103
|
+
disk_times = run_benchmark_multiple_times(disk_benchmark, iterations=150)
|
|
104
|
+
io_times = run_benchmark_multiple_times(io_benchmark, iterations=150)
|
|
105
|
+
peak_disk_write_throughputs = run_peak_disk_write_benchmark(iterations=150)
|
|
106
|
+
|
|
107
|
+
save_benchmark_results('machine_A', 'CPU', cpu_times)
|
|
108
|
+
save_benchmark_results('machine_A', 'Memory', memory_times)
|
|
109
|
+
save_benchmark_results('machine_A', 'Disk', disk_times)
|
|
110
|
+
save_benchmark_results('machine_A', 'IO', io_times)
|
|
111
|
+
save_benchmark_results('machine_A', 'PeakDiskWrite', peak_disk_write_throughputs)
|
|
112
|
+
|
|
113
|
+
print("Benchmarking complete. Results saved to the database.")
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
from dash import Dash, dcc, html
|
|
2
|
+
import dash_bootstrap_components as dbc
|
|
3
|
+
from dash.dependencies import Input, Output
|
|
4
|
+
import plotly.express as px
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import sqlite3
|
|
7
|
+
import numpy as np
|
|
8
|
+
from scipy.stats import gaussian_kde
|
|
9
|
+
|
|
10
|
+
# Normalization functions
|
|
11
|
+
def z_score_normalization(times):
|
|
12
|
+
mean = np.mean(times)
|
|
13
|
+
std = np.std(times)
|
|
14
|
+
if std == 0:
|
|
15
|
+
return times - mean # Avoid division by zero
|
|
16
|
+
return (times - mean) / std
|
|
17
|
+
|
|
18
|
+
def min_max_normalization(times):
|
|
19
|
+
min_time = np.min(times)
|
|
20
|
+
max_time = np.max(times)
|
|
21
|
+
if max_time == min_time:
|
|
22
|
+
return times - min_time # Avoid division by zero
|
|
23
|
+
return (times - min_time) / (max_time - min_time)
|
|
24
|
+
|
|
25
|
+
def exponential_moving_average(times, alpha=0.3):
|
|
26
|
+
ema_times = []
|
|
27
|
+
ema = times[0] # Start with the first value
|
|
28
|
+
for time in times:
|
|
29
|
+
ema = alpha * time + (1 - alpha) * ema
|
|
30
|
+
ema_times.append(ema)
|
|
31
|
+
return np.array(ema_times)
|
|
32
|
+
|
|
33
|
+
def log_transformation(times):
|
|
34
|
+
return np.log(times + 1) # Adding 1 to avoid log(0)
|
|
35
|
+
|
|
36
|
+
def select_normalization(times, benchmark_name):
|
|
37
|
+
# Apply specific normalization methods based on benchmark type
|
|
38
|
+
if 'Disk' in benchmark_name or 'PeakDiskWrite' in benchmark_name:
|
|
39
|
+
# Use EMA or log normalization for disk benchmarks
|
|
40
|
+
normalized_times = exponential_moving_average(times)
|
|
41
|
+
if np.std(normalized_times) > np.std(log_transformation(times)):
|
|
42
|
+
normalized_times = log_transformation(times)
|
|
43
|
+
best_method = 'EMA or log'
|
|
44
|
+
else:
|
|
45
|
+
# Use z-score normalization for other benchmarks
|
|
46
|
+
normalized_times = z_score_normalization(times)
|
|
47
|
+
best_method = 'z-score'
|
|
48
|
+
return best_method, normalized_times
|
|
49
|
+
|
|
50
|
+
def fetch_benchmark_data():
|
|
51
|
+
conn = sqlite3.connect('test_results.db')
|
|
52
|
+
df = pd.read_sql_query('SELECT * FROM benchmark_samples', conn)
|
|
53
|
+
conn.close()
|
|
54
|
+
return df
|
|
55
|
+
|
|
56
|
+
def compute_machine_score():
|
|
57
|
+
df = fetch_benchmark_data()
|
|
58
|
+
benchmarks = df['benchmark_name'].unique()
|
|
59
|
+
|
|
60
|
+
scores = {}
|
|
61
|
+
for benchmark in benchmarks:
|
|
62
|
+
times = df[df['benchmark_name'] == benchmark]['time'].values
|
|
63
|
+
if len(times) > 0:
|
|
64
|
+
best_method, normalized_times = select_normalization(times, benchmark)
|
|
65
|
+
scores[benchmark] = np.mean(normalized_times)
|
|
66
|
+
|
|
67
|
+
# Aggregate scores (simple average for now)
|
|
68
|
+
composite_score = np.mean(list(scores.values()))
|
|
69
|
+
|
|
70
|
+
# Apply normalization to the composite score for stability
|
|
71
|
+
composite_score_normalized = min_max_normalization(np.array([composite_score]))[0] # Using min-max normalization
|
|
72
|
+
|
|
73
|
+
# Debug output
|
|
74
|
+
print("Scores per benchmark:", scores)
|
|
75
|
+
print("Composite score:", composite_score)
|
|
76
|
+
print("Normalized composite score:", composite_score_normalized)
|
|
77
|
+
|
|
78
|
+
return composite_score_normalized
|
|
79
|
+
|
|
80
|
+
def analyze_clusters(times, num_clusters=3):
|
|
81
|
+
if len(times) > 0:
|
|
82
|
+
min_time = np.min(times)
|
|
83
|
+
max_time = np.max(times)
|
|
84
|
+
threshold = (max_time - min_time) / num_clusters
|
|
85
|
+
clusters = []
|
|
86
|
+
for i in range(num_clusters):
|
|
87
|
+
lower_bound = min_time + i * threshold
|
|
88
|
+
upper_bound = min_time + (i + 1) * threshold
|
|
89
|
+
cluster = times[(times >= lower_bound) & (times < upper_bound)]
|
|
90
|
+
clusters.append(cluster)
|
|
91
|
+
return clusters
|
|
92
|
+
return [times]
|
|
93
|
+
|
|
94
|
+
app = Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
|
|
95
|
+
|
|
96
|
+
app.layout = dbc.Container([
|
|
97
|
+
dbc.Row([
|
|
98
|
+
dbc.Col(html.H1("Benchmark Dashboard"), className="mb-2")
|
|
99
|
+
]),
|
|
100
|
+
dbc.Row([
|
|
101
|
+
dbc.Col([
|
|
102
|
+
dcc.Dropdown(
|
|
103
|
+
id='benchmark-dropdown',
|
|
104
|
+
options=[
|
|
105
|
+
{'label': 'CPU', 'value': 'CPU'},
|
|
106
|
+
{'label': 'Memory', 'value': 'Memory'},
|
|
107
|
+
{'label': 'Disk', 'value': 'Disk'},
|
|
108
|
+
{'label': 'IO', 'value': 'IO'},
|
|
109
|
+
{'label': 'Peak Disk Write', 'value': 'PeakDiskWrite'},
|
|
110
|
+
{'label': 'Machine Score', 'value': 'MachineScore'}
|
|
111
|
+
],
|
|
112
|
+
value='CPU',
|
|
113
|
+
clearable=False
|
|
114
|
+
)
|
|
115
|
+
], width=4),
|
|
116
|
+
dbc.Col([
|
|
117
|
+
dbc.Checkbox(
|
|
118
|
+
id='cluster-checkbox',
|
|
119
|
+
label='Show Clusters',
|
|
120
|
+
value=False
|
|
121
|
+
)
|
|
122
|
+
], width=2)
|
|
123
|
+
]),
|
|
124
|
+
dbc.Row([
|
|
125
|
+
dbc.Col([
|
|
126
|
+
dcc.Graph(id='benchmark-graph')
|
|
127
|
+
])
|
|
128
|
+
])
|
|
129
|
+
])
|
|
130
|
+
|
|
131
|
+
@app.callback(
|
|
132
|
+
Output('benchmark-graph', 'figure'),
|
|
133
|
+
[Input('benchmark-dropdown', 'value'), Input('cluster-checkbox', 'value')]
|
|
134
|
+
)
|
|
135
|
+
def update_graph(benchmark_name, show_clusters):
|
|
136
|
+
if benchmark_name == 'MachineScore':
|
|
137
|
+
score = compute_machine_score()
|
|
138
|
+
fig = px.bar(x=['Machine'], y=[score], labels={'x': 'Machine', 'y': 'Normalized Score'})
|
|
139
|
+
fig.update_layout(
|
|
140
|
+
title='Machine Composite Score',
|
|
141
|
+
yaxis_title='Normalized Score',
|
|
142
|
+
xaxis_title='Machine'
|
|
143
|
+
)
|
|
144
|
+
else:
|
|
145
|
+
df = fetch_benchmark_data()
|
|
146
|
+
filtered_df = df[df['benchmark_name'] == benchmark_name]
|
|
147
|
+
times = filtered_df['time'].values
|
|
148
|
+
|
|
149
|
+
if len(times) > 0: # Ensure times is not empty
|
|
150
|
+
if show_clusters:
|
|
151
|
+
clusters = analyze_clusters(times)
|
|
152
|
+
fig = px.histogram(x=times, nbins=30, histnorm='probability density') # Initialize with full data
|
|
153
|
+
for i, cluster in enumerate(clusters, 1):
|
|
154
|
+
if len(cluster) > 0:
|
|
155
|
+
best_method, normalized_times = select_normalization(cluster, benchmark_name)
|
|
156
|
+
kde = gaussian_kde(normalized_times)
|
|
157
|
+
x_range = np.linspace(min(normalized_times), max(normalized_times), 1000)
|
|
158
|
+
kde_values = kde(x_range)
|
|
159
|
+
|
|
160
|
+
# Add histogram and KDE for each cluster
|
|
161
|
+
fig.add_trace(px.histogram(x=normalized_times, nbins=30, histnorm='probability density').data[0])
|
|
162
|
+
fig.add_scatter(
|
|
163
|
+
x=x_range,
|
|
164
|
+
y=kde_values,
|
|
165
|
+
mode='lines',
|
|
166
|
+
name=f'Cluster {i} Density (Normalization: {best_method})'
|
|
167
|
+
)
|
|
168
|
+
else:
|
|
169
|
+
# Automatically select the best normalization method for the full dataset
|
|
170
|
+
best_method, normalized_times = select_normalization(times, benchmark_name)
|
|
171
|
+
kde = gaussian_kde(normalized_times)
|
|
172
|
+
x_range = np.linspace(min(normalized_times), max(normalized_times), 1000)
|
|
173
|
+
kde_values = kde(x_range)
|
|
174
|
+
|
|
175
|
+
# Create histogram with full dataset
|
|
176
|
+
fig = px.histogram(x=normalized_times, nbins=30, histnorm='probability density')
|
|
177
|
+
fig.add_scatter(
|
|
178
|
+
x=x_range,
|
|
179
|
+
y=kde_values,
|
|
180
|
+
mode='lines',
|
|
181
|
+
name='Density'
|
|
182
|
+
)
|
|
183
|
+
else:
|
|
184
|
+
fig = px.histogram() # Initialize an empty histogram
|
|
185
|
+
|
|
186
|
+
fig.update_layout(
|
|
187
|
+
title=f'{benchmark_name} Benchmark Results' + (' with Clusters' if show_clusters else ''),
|
|
188
|
+
xaxis_title='Normalized Time' if len(times) > 0 else 'Time (seconds)',
|
|
189
|
+
yaxis_title='Density'
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
return fig
|
|
193
|
+
|
|
194
|
+
if __name__ == '__main__':
|
|
195
|
+
app.run_server(debug=True)
|
|
Binary file
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import time
|
|
2
|
+
|
|
3
|
+
def log_test_time(func):
|
|
4
|
+
def wrapper(*args, **kwargs):
|
|
5
|
+
start_time = time.time()
|
|
6
|
+
result = func(*args, **kwargs)
|
|
7
|
+
elapsed_time = time.time() - start_time
|
|
8
|
+
print(f"{func.__name__} took {elapsed_time:.2f} seconds")
|
|
9
|
+
return result
|
|
10
|
+
return wrapper
|
|
11
|
+
|
|
12
|
+
def verify_condition(condition, message="Verification failed"):
|
|
13
|
+
if not condition:
|
|
14
|
+
raise AssertionError(message)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# TODO >>> Add a Events Manager that can save events locally
|
|
18
|
+
# Events are temporary milestones that we can define to a test
|
|
19
|
+
# This allows us to log events that we expect the test to complete.
|
|
20
|
+
|
|
21
|
+
from .common.sql_pool import SQLiteConnectionPool
|
|
22
|
+
import os
|
|
23
|
+
import pandas as pd
|
|
24
|
+
|
|
25
|
+
THIS_DIR = os.getcwd()
|
|
26
|
+
|
|
27
|
+
class Events_Manager:
|
|
28
|
+
|
|
29
|
+
def __init__(self, Unit:str, path:str):
|
|
30
|
+
"""
|
|
31
|
+
if unit == "*" select all units data
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
pool = SQLiteConnectionPool(3, os.path.join(THIS_DIR, "Data.db"))
|
|
35
|
+
self.connection = pool.get_connection()
|
|
36
|
+
|
|
37
|
+
self.Unit = Unit
|
|
38
|
+
|
|
39
|
+
cur = self.connection.cursor()
|
|
40
|
+
cur.execute('''CREATE TABLE IF NOT EXISTS Events (
|
|
41
|
+
ID INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
42
|
+
Unit TEXT,
|
|
43
|
+
StepCompleted TEXT,
|
|
44
|
+
EventType TEXT,
|
|
45
|
+
EventKey TEXT,
|
|
46
|
+
Time NUMBER
|
|
47
|
+
)''')
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def drop_events_table(self) -> None:
|
|
51
|
+
"""Drop the Events table from the database."""
|
|
52
|
+
|
|
53
|
+
try:
|
|
54
|
+
cur = self.connection.cursor()
|
|
55
|
+
|
|
56
|
+
sql_drop_table_query = """DROP TABLE IF EXISTS Events"""
|
|
57
|
+
cur.execute(sql_drop_table_query)
|
|
58
|
+
|
|
59
|
+
self.connection.commit()
|
|
60
|
+
print("Events table has been removed.")
|
|
61
|
+
|
|
62
|
+
except Exception as e:
|
|
63
|
+
print(f"Error occurred while removing the Events table. Error: {e}")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def List_Events(self) -> dict:
|
|
67
|
+
|
|
68
|
+
cur = self.connection.cursor()
|
|
69
|
+
|
|
70
|
+
if self.Unit == "*":
|
|
71
|
+
sqlite_select_query = """SELECT * FROM Events"""
|
|
72
|
+
cur.execute(sqlite_select_query)
|
|
73
|
+
|
|
74
|
+
else:
|
|
75
|
+
sqlite_select_query = """SELECT * FROM Events WHERE Unit = ?"""
|
|
76
|
+
Data = ((self.Unit, ))
|
|
77
|
+
cur.execute(sqlite_select_query, Data)
|
|
78
|
+
|
|
79
|
+
df = cur.fetchall()
|
|
80
|
+
df = pd.DataFrame(df, columns=['ID', 'Unit', 'StepCompleted', 'EventType', 'EventKey', 'Time'])
|
|
81
|
+
dict_df = df.to_dict()
|
|
82
|
+
|
|
83
|
+
return dict_df
|
|
84
|
+
|
|
85
|
+
def Set_Event (self, step:str, event_type:str="Default", **kwargs):
|
|
86
|
+
|
|
87
|
+
"""
|
|
88
|
+
Set a event
|
|
89
|
+
|
|
90
|
+
event_type: str - can be:
|
|
91
|
+
- Default
|
|
92
|
+
- Exception
|
|
93
|
+
- Send
|
|
94
|
+
- Receive
|
|
95
|
+
|
|
96
|
+
if event_type is Send and Receive it needs a predefined `event_key:str` kwarg,
|
|
97
|
+
the event key is a str and need to contain at least 16 digits that need to be random,
|
|
98
|
+
to generate a valid key pair and have 100% sure that this isn't registered you can use the
|
|
99
|
+
helper `gen_valid_event_key.py` at Myscelium/tests/Logs/gen_valid_event_key.py
|
|
100
|
+
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
if self.Unit == "*":
|
|
104
|
+
raise ValueError("Can't Set Event To Generalized Unit: '*'")
|
|
105
|
+
|
|
106
|
+
if event_type in ["Default", "Exception", "Send", "Receive"]:
|
|
107
|
+
pass
|
|
108
|
+
else:
|
|
109
|
+
raise ValueError("Event type can only be one of those: 'Default, Exception, Send, Receive'")
|
|
110
|
+
|
|
111
|
+
events = pd.DataFrame.from_dict(self.List_Events())
|
|
112
|
+
|
|
113
|
+
for i in events.index:
|
|
114
|
+
if events.loc[i, "StepCompleted"] == step:
|
|
115
|
+
return
|
|
116
|
+
else:
|
|
117
|
+
continue
|
|
118
|
+
|
|
119
|
+
cur = self.connection.cursor()
|
|
120
|
+
|
|
121
|
+
ts = time.time()
|
|
122
|
+
|
|
123
|
+
event_key = ""
|
|
124
|
+
|
|
125
|
+
if event_type == "Send" or event_type == "Receive":
|
|
126
|
+
|
|
127
|
+
if not ("event_key" in kwargs):
|
|
128
|
+
raise "You need to to specify a event code to Send an Receive event_types"
|
|
129
|
+
else:
|
|
130
|
+
pass
|
|
131
|
+
|
|
132
|
+
event_key = kwargs["event_key"]
|
|
133
|
+
|
|
134
|
+
Data = ((self.Unit, step, event_type, event_key, ts))
|
|
135
|
+
|
|
136
|
+
sqlite_insert_with_param = """INSERT INTO Events (Unit, StepCompleted, EventType, EventKey, Time) VALUES (?, ?, ?, ?, ?);"""
|
|
137
|
+
cur.execute(sqlite_insert_with_param, Data)
|
|
138
|
+
self.connection.commit()
|
|
139
|
+
|
|
140
|
+
return
|