testrium 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,72 @@
1
+ Metadata-Version: 2.1
2
+ Name: testrium
3
+ Version: 0.1.0
4
+ Summary: A modern, advanced testing framework.
5
+ Home-page: https://github.com/letalboy/testrium
6
+ License: MIT
7
+ Author: Cristian Camargo
8
+ Author-email: ccf@cdone.com.br
9
+ Requires-Python: >=3.10,<4.0
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Requires-Dist: pandas (>=2.2.2,<3.0.0)
16
+ Requires-Dist: toml (>=0.10.2,<0.11.0)
17
+ Project-URL: Documentation, https://github.com/letalboy/testrium#readme
18
+ Project-URL: Repository, https://github.com/letalboy/testrium
19
+ Description-Content-Type: text/markdown
20
+
21
+ <div style="display: flex; align-items: center;">
22
+ <img src="https://github.com/letalboy/Spectrum/assets/63066865/cf9c60e7-9eba-474c-8cc1-b246e661fd5c" alt="Logo" width="200" height="200" style="margin-right: 20px;">
23
+ <div>
24
+ <h1>Spectrum</h1>
25
+ <p>Spectrum is a versatile library designed to test and analyze systems like Misny. It provides tools for data processing, monitoring, and ensuring system integrity, making it an essential part of your development workflow.</p>
26
+ </div>
27
+ </div>
28
+ <div>
29
+ <p>Key points:</p>
30
+ <ul>
31
+ <li>Better Organization Of Tests</li>
32
+ <li>Easy Test Of Async Or Paralelized Dependent Tests</li>
33
+ <li>Historic Performance Comparation and Evaluation</li>
34
+ <li>Allow Custom Behavior Though Callbacks Systems</li>
35
+ <li>Allow Manipulation Of The Test Flows</li>
36
+ <li>High COnfigurable And Expancible</li>
37
+ </ul>
38
+ </div>
39
+
40
+ To use is simple:
41
+
42
+ 1. activate the poetry shell using:
43
+
44
+ ```shell
45
+ poetry shell
46
+ ```
47
+
48
+ 2. go the the test folder and run:
49
+
50
+ ```shell
51
+ poetry run spectrum
52
+ ```
53
+
54
+ This should auto collect the test and run them automatically!
55
+
56
+ ---
57
+
58
+ Base Idea:
59
+ - Read the folder of test folders
60
+ - Collect each one
61
+ - Run them based in the priority config file in the test toml if present
62
+ - Create a db to store tests results locally or if configured in the toml then send to a remote server
63
+ - Run the tests, save the results
64
+ - Compare the results witht he performance of the old tests to see if the test increase or decrese efficience.
65
+ - Create a way to evaluate the performance in multiple machines comparing the efficience without issues.
66
+
67
+ TODO:
68
+ - Create a mechanism to add events through a test
69
+ - Events will allow to indentify if we complete the required step
70
+ - Cross events, are events were we want to trace the process from one point to another and how much time it takes to
71
+ this will be able to be done using a structure that generate a event hash for each event so then we can trace it to the
72
+ other place.
@@ -0,0 +1,52 @@
1
+ <div style="display: flex; align-items: center;">
2
+ <img src="https://github.com/letalboy/Spectrum/assets/63066865/cf9c60e7-9eba-474c-8cc1-b246e661fd5c" alt="Logo" width="200" height="200" style="margin-right: 20px;">
3
+ <div>
4
+ <h1>Spectrum</h1>
5
+ <p>Spectrum is a versatile library designed to test and analyze systems like Misny. It provides tools for data processing, monitoring, and ensuring system integrity, making it an essential part of your development workflow.</p>
6
+ </div>
7
+ </div>
8
+ <div>
9
+ <p>Key points:</p>
10
+ <ul>
11
+ <li>Better Organization Of Tests</li>
12
+ <li>Easy Test Of Async Or Paralelized Dependent Tests</li>
13
+ <li>Historic Performance Comparation and Evaluation</li>
14
+ <li>Allow Custom Behavior Though Callbacks Systems</li>
15
+ <li>Allow Manipulation Of The Test Flows</li>
16
+ <li>High COnfigurable And Expancible</li>
17
+ </ul>
18
+ </div>
19
+
20
+ To use is simple:
21
+
22
+ 1. activate the poetry shell using:
23
+
24
+ ```shell
25
+ poetry shell
26
+ ```
27
+
28
+ 2. go the the test folder and run:
29
+
30
+ ```shell
31
+ poetry run spectrum
32
+ ```
33
+
34
+ This should auto collect the test and run them automatically!
35
+
36
+ ---
37
+
38
+ Base Idea:
39
+ - Read the folder of test folders
40
+ - Collect each one
41
+ - Run them based in the priority config file in the test toml if present
42
+ - Create a db to store tests results locally or if configured in the toml then send to a remote server
43
+ - Run the tests, save the results
44
+ - Compare the results witht he performance of the old tests to see if the test increase or decrese efficience.
45
+ - Create a way to evaluate the performance in multiple machines comparing the efficience without issues.
46
+
47
+ TODO:
48
+ - Create a mechanism to add events through a test
49
+ - Events will allow to indentify if we complete the required step
50
+ - Cross events, are events were we want to trace the process from one point to another and how much time it takes to
51
+ this will be able to be done using a structure that generate a event hash for each event so then we can trace it to the
52
+ other place.
@@ -0,0 +1,27 @@
1
+ [tool.poetry]
2
+ name = "testrium"
3
+ version = "0.1.0"
4
+ description = "A modern, advanced testing framework."
5
+ authors = ["Cristian Camargo <ccf@cdone.com.br>"]
6
+ readme = "README.md"
7
+ homepage = "https://github.com/letalboy/testrium"
8
+ repository = "https://github.com/letalboy/testrium"
9
+ license = "MIT"
10
+
11
+ [tool.poetry.dependencies]
12
+ python = "^3.10"
13
+ toml = "^0.10.2"
14
+ pandas = "^2.2.2"
15
+
16
+ [tool.poetry.group.dev.dependencies]
17
+ pytest = "^8.2.2"
18
+
19
+ [build-system]
20
+ requires = ["poetry-core"]
21
+ build-backend = "poetry.core.masonry.api"
22
+
23
+ [tool.poetry.scripts]
24
+ testrium = "testrium.core:main"
25
+
26
+ [tool.poetry.urls]
27
+ Documentation = "https://github.com/letalboy/testrium#readme"
File without changes
@@ -0,0 +1,30 @@
1
+ import sqlite3
2
+ from queue import Queue
3
+ from threading import Lock
4
+
5
+ class SQLiteConnectionPool:
6
+ def __init__(self, max_connections:int, database_path:str):
7
+ self.data_base = database_path
8
+ self.max_connections = max_connections
9
+ self.connections = Queue(max_connections)
10
+ self.lock = Lock()
11
+
12
+ for i in range(max_connections):
13
+ connection = sqlite3.connect(self.data_base, check_same_thread=False)
14
+ self.connections.put(connection)
15
+
16
+ def get_connection(self):
17
+ with self.lock:
18
+ if self.connections.empty():
19
+ raise Exception("No available connections in the pool")
20
+ connection = self.connections.get()
21
+ return connection
22
+
23
+ def release_connection(self, connection):
24
+ with self.lock:
25
+ self.connections.put(connection)
26
+
27
+ def close_all_connections(self):
28
+ while not self.connections.empty():
29
+ connection = self.connections.get()
30
+ connection.close()
@@ -0,0 +1 @@
1
+ # TODO >>> This module have the goal to validate the configs toml and ensure it has the correct and desired keywords
@@ -0,0 +1,177 @@
1
+ import os
2
+ import time
3
+ import importlib.util
4
+ import toml
5
+ from colorama import init, Fore, Style
6
+ import shutil
7
+ from .utils import Events_Manager
8
+
9
+ # Initialize colorama
10
+ init(autoreset=True)
11
+
12
+ # Loads the config for each test:
13
+ def load_config(config_path):
14
+ print(f"{Fore.CYAN}Loading config from {config_path}")
15
+ with open(config_path, 'r') as file:
16
+ config = toml.load(file)
17
+ return config
18
+
19
+ # Loads the test functions:
20
+ def load_test_functions(dir_path):
21
+ test_functions = {}
22
+ for filename in os.listdir(dir_path):
23
+ if filename.startswith('_') or not filename.endswith('.py'):
24
+ continue
25
+ module_name = filename[:-3]
26
+ file_path = os.path.join(dir_path, filename)
27
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
28
+ module = importlib.util.module_from_spec(spec)
29
+ spec.loader.exec_module(module)
30
+ for attr in dir(module):
31
+ if attr == "Events_Manager":
32
+ continue
33
+ func = getattr(module, attr)
34
+ if callable(func) and not attr.startswith('_'):
35
+ test_functions[attr] = func
36
+ return test_functions
37
+
38
+ # Run setup of the test group
39
+ def run_setup(setup_path):
40
+ spec = importlib.util.spec_from_file_location("setup", setup_path)
41
+ setup_module = importlib.util.module_from_spec(spec)
42
+ spec.loader.exec_module(setup_module)
43
+ setup_module.main()
44
+
45
+ # Print the nice Pass or Fail banners
46
+ def print_banner(message, color=Fore.GREEN):
47
+ term_width = shutil.get_terminal_size().columns
48
+ message_length = len(message)
49
+ padding_length = (term_width - message_length) // 2
50
+ banner = "=" * padding_length + message + "=" * padding_length
51
+
52
+ # Ensure the banner fits exactly in the terminal width
53
+ if len(banner) < term_width:
54
+ banner += "=" * (term_width - len(banner))
55
+
56
+ print(color + banner)
57
+
58
+ def main():
59
+ base_dir = os.getcwd()
60
+ print(f"{Fore.GREEN}Current working directory: {base_dir}")
61
+
62
+ dir_names = os.listdir(base_dir)
63
+ valid_test_dirs = []
64
+
65
+ # -> Discover valid test folders with configs inside
66
+ for dir_name in dir_names:
67
+ dir_path = os.path.join(base_dir, dir_name)
68
+
69
+ if not os.path.isdir(dir_path):
70
+ print(f"{Fore.RED} File: {dir_name} is not a folder")
71
+ continue
72
+
73
+ setup_path = os.path.join(dir_path, 'setup.py')
74
+ config_path = os.path.join(dir_path, 'config.toml')
75
+
76
+ if not (os.path.exists(setup_path) and os.path.exists(config_path)):
77
+ print(f"{Fore.RED} File: {dir_name} invalid test")
78
+ continue
79
+
80
+ print(f"{Fore.MAGENTA}Found test directory: {dir_name}")
81
+ valid_test_dirs.append(dir_name)
82
+
83
+ tests_finded = len(valid_test_dirs)
84
+
85
+ # -> Early retun if not find any test:
86
+ if tests_finded > 0:
87
+ print(f"{Fore.GREEN} Find: {tests_finded:.0f} valid test groups!")
88
+ for test_group in valid_test_dirs:
89
+ print(f"{Fore.GREEN} - {test_group}")
90
+ else:
91
+ print(f"{Fore.RED} No valid test groups finded!")
92
+ return
93
+
94
+ # TODO >>> Find a way to log the milestones completed for each test and with this understand what they relate to
95
+
96
+ # -> Run the valid tests:
97
+ for dir_name in valid_test_dirs:
98
+
99
+ Events_Manager(Unit="", path=base_dir).drop_events_table()
100
+
101
+ dir_path = os.path.join(base_dir, dir_name)
102
+ setup_path = os.path.join(dir_path, 'setup.py')
103
+
104
+ #> Load test configs
105
+ config_path = os.path.join(dir_path, 'config.toml')
106
+ configs = load_config(config_path)
107
+
108
+ #> Load Units
109
+ confs = configs['Configs']
110
+ units = confs['units']
111
+
112
+ #> Load Units Predefinitions
113
+ total_units = 0
114
+ units_index = {}
115
+ for unit in units:
116
+ un = configs[f"{unit}"]
117
+ units_index[un["init"]] = {"name":f"{unit}", "events": f"{un['events']}"}
118
+ total_units += 1
119
+
120
+ print(f"Units loaded: {units_index}")
121
+
122
+ # TODO >>> Use the units order to setup the units one by one
123
+ # TODO >>> Create a meachanism to verify the events when they are required
124
+
125
+ print(f"{Fore.BLUE}Loading tests for {dir_name}...")
126
+ test_functions = load_test_functions(dir_path)
127
+
128
+ all_tests_passed = True
129
+
130
+ #> Run each test found in the test group
131
+ for test_name, test_func in test_functions.items():
132
+ print(f"{Fore.YELLOW}Running test: {test_name}")
133
+ start_time = time.time()
134
+
135
+ try:
136
+ if test_name == 'log_test_time':
137
+ test_func(dummy_function) # Pass a dummy function if required
138
+ elif test_name == 'verify_condition':
139
+ # TODO >>> Use this as a condition to verify if the requirements was completed for the test case
140
+ test_func(lambda: True) # Pass a lambda function if required
141
+ else:
142
+ test_func()
143
+ elapsed_time = time.time() - start_time
144
+ print(f"{Fore.GREEN}{test_name}: PASSED in {elapsed_time:.2f} seconds")
145
+ except Exception as e:
146
+ all_tests_passed = False
147
+ elapsed_time = time.time() - start_time
148
+ print(f"{Fore.RED}{test_name}: FAILED in {elapsed_time:.2f} seconds\nError: {e}")
149
+
150
+ #> Verify Events Completed By The Unit
151
+ for i in range(total_units):
152
+ unit = units_index[i]
153
+ unit_name = unit["name"]
154
+ if unit_name == "":
155
+ continue
156
+ unit_events = Events_Manager(Unit=unit_name, path=base_dir).List_Events()
157
+ for event in eval(unit["events"]):
158
+ if event not in unit_events:
159
+ print(f"event: [{event}] was not completed!")
160
+ all_tests_passed = False
161
+
162
+ if all_tests_passed:
163
+ print_banner(" PASS ", Fore.GREEN)
164
+ else:
165
+ print_banner(" FAILURE ", Fore.RED)
166
+
167
+ # Extra validation step that user migh want to define
168
+ def dummy_function():
169
+ """
170
+ - This will allow to define a extra step in the verification
171
+ for example, read a file and validate if the test was a success.
172
+ see if the code did what it was suposed to do. etc..
173
+ """
174
+ pass
175
+
176
+ if __name__ == "__main__":
177
+ main()
@@ -0,0 +1,113 @@
1
+ import time
2
+ import numpy as np
3
+ import os
4
+ import sqlite3
5
+ from tqdm import tqdm
6
+ from concurrent.futures import ThreadPoolExecutor, as_completed
7
+
8
+ def run_benchmark_multiple_times(benchmark_func, iterations=10, warmup=True):
9
+ if warmup:
10
+ benchmark_func()
11
+
12
+ times = []
13
+ for _ in tqdm(range(iterations), desc=benchmark_func.__name__, leave=False):
14
+ start_time = time.time()
15
+ benchmark_func()
16
+ end_time = time.time()
17
+ times.append(end_time - start_time)
18
+
19
+ return times # Return all times for visualization
20
+
21
+ def cpu_benchmark_single_core():
22
+ result = 0
23
+ for i in range(1, 1000000):
24
+ result += (i ** 0.5) ** 2
25
+
26
+ def cpu_benchmark_multicore(workers=4):
27
+ def worker_task(start, end):
28
+ result = 0
29
+ for i in range(start, end):
30
+ result += (i ** 0.5) ** 2
31
+ return result
32
+
33
+ chunk_size = 1000000 // workers
34
+ with ThreadPoolExecutor(max_workers=workers) as executor:
35
+ futures = [executor.submit(worker_task, i * chunk_size, (i + 1) * chunk_size) for i in range(workers)]
36
+ results = [future.result() for future in as_completed(futures)]
37
+ return sum(results)
38
+
39
+ def memory_benchmark():
40
+ array_size = 10000000
41
+ array = np.arange(array_size, dtype=np.float64)
42
+ array *= 2
43
+
44
+ def disk_benchmark():
45
+ with open('disk_benchmark.tmp', 'wb') as f:
46
+ f.write(os.urandom(500000000)) # Write 500MB
47
+ with open('disk_benchmark.tmp', 'rb') as f:
48
+ f.read()
49
+ os.remove('disk_benchmark.tmp')
50
+
51
+ def io_benchmark():
52
+ with open('io_benchmark.tmp', 'w') as f:
53
+ for _ in range(100000):
54
+ f.write('This is a test.\n')
55
+ os.remove('io_benchmark.tmp')
56
+
57
+ def peak_disk_write_benchmark(file_size_mb=500):
58
+ data = os.urandom(file_size_mb * 1024 * 1024) # Generate large data block
59
+ file_path = 'peak_disk_write.tmp'
60
+
61
+ start_time = time.time()
62
+ with open(file_path, 'wb') as f:
63
+ f.write(data)
64
+ f.flush() # Ensure all data is written
65
+ os.fsync(f.fileno()) # Flush OS buffer to disk
66
+ end_time = time.time()
67
+
68
+ os.remove(file_path) # Clean up
69
+
70
+ elapsed_time = end_time - start_time
71
+ throughput = file_size_mb / elapsed_time # MB/s
72
+ return throughput
73
+
74
+ def run_peak_disk_write_benchmark(iterations=5, file_size_mb=500):
75
+ throughputs = []
76
+ for _ in tqdm(range(iterations), desc="Peak Disk Write Benchmark", leave=False):
77
+ throughput = peak_disk_write_benchmark(file_size_mb)
78
+ throughputs.append(throughput)
79
+ return throughputs
80
+
81
+ def save_benchmark_results(machine_id, benchmark_name, times):
82
+ conn = sqlite3.connect('test_results.db')
83
+ cursor = conn.cursor()
84
+ cursor.execute('''
85
+ CREATE TABLE IF NOT EXISTS benchmark_samples (
86
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
87
+ machine_id TEXT,
88
+ benchmark_name TEXT,
89
+ time REAL
90
+ )
91
+ ''')
92
+ for time in times:
93
+ cursor.execute('''
94
+ INSERT INTO benchmark_samples (machine_id, benchmark_name, time)
95
+ VALUES (?, ?, ?)
96
+ ''', (machine_id, benchmark_name, time))
97
+ conn.commit()
98
+ conn.close()
99
+
100
+ # Example usage
101
+ cpu_times = run_benchmark_multiple_times(lambda: cpu_benchmark_multicore(workers=os.cpu_count()), iterations=150)
102
+ memory_times = run_benchmark_multiple_times(memory_benchmark, iterations=150)
103
+ disk_times = run_benchmark_multiple_times(disk_benchmark, iterations=150)
104
+ io_times = run_benchmark_multiple_times(io_benchmark, iterations=150)
105
+ peak_disk_write_throughputs = run_peak_disk_write_benchmark(iterations=150)
106
+
107
+ save_benchmark_results('machine_A', 'CPU', cpu_times)
108
+ save_benchmark_results('machine_A', 'Memory', memory_times)
109
+ save_benchmark_results('machine_A', 'Disk', disk_times)
110
+ save_benchmark_results('machine_A', 'IO', io_times)
111
+ save_benchmark_results('machine_A', 'PeakDiskWrite', peak_disk_write_throughputs)
112
+
113
+ print("Benchmarking complete. Results saved to the database.")
@@ -0,0 +1,195 @@
1
+ from dash import Dash, dcc, html
2
+ import dash_bootstrap_components as dbc
3
+ from dash.dependencies import Input, Output
4
+ import plotly.express as px
5
+ import pandas as pd
6
+ import sqlite3
7
+ import numpy as np
8
+ from scipy.stats import gaussian_kde
9
+
10
+ # Normalization functions
11
+ def z_score_normalization(times):
12
+ mean = np.mean(times)
13
+ std = np.std(times)
14
+ if std == 0:
15
+ return times - mean # Avoid division by zero
16
+ return (times - mean) / std
17
+
18
+ def min_max_normalization(times):
19
+ min_time = np.min(times)
20
+ max_time = np.max(times)
21
+ if max_time == min_time:
22
+ return times - min_time # Avoid division by zero
23
+ return (times - min_time) / (max_time - min_time)
24
+
25
+ def exponential_moving_average(times, alpha=0.3):
26
+ ema_times = []
27
+ ema = times[0] # Start with the first value
28
+ for time in times:
29
+ ema = alpha * time + (1 - alpha) * ema
30
+ ema_times.append(ema)
31
+ return np.array(ema_times)
32
+
33
+ def log_transformation(times):
34
+ return np.log(times + 1) # Adding 1 to avoid log(0)
35
+
36
+ def select_normalization(times, benchmark_name):
37
+ # Apply specific normalization methods based on benchmark type
38
+ if 'Disk' in benchmark_name or 'PeakDiskWrite' in benchmark_name:
39
+ # Use EMA or log normalization for disk benchmarks
40
+ normalized_times = exponential_moving_average(times)
41
+ if np.std(normalized_times) > np.std(log_transformation(times)):
42
+ normalized_times = log_transformation(times)
43
+ best_method = 'EMA or log'
44
+ else:
45
+ # Use z-score normalization for other benchmarks
46
+ normalized_times = z_score_normalization(times)
47
+ best_method = 'z-score'
48
+ return best_method, normalized_times
49
+
50
+ def fetch_benchmark_data():
51
+ conn = sqlite3.connect('test_results.db')
52
+ df = pd.read_sql_query('SELECT * FROM benchmark_samples', conn)
53
+ conn.close()
54
+ return df
55
+
56
+ def compute_machine_score():
57
+ df = fetch_benchmark_data()
58
+ benchmarks = df['benchmark_name'].unique()
59
+
60
+ scores = {}
61
+ for benchmark in benchmarks:
62
+ times = df[df['benchmark_name'] == benchmark]['time'].values
63
+ if len(times) > 0:
64
+ best_method, normalized_times = select_normalization(times, benchmark)
65
+ scores[benchmark] = np.mean(normalized_times)
66
+
67
+ # Aggregate scores (simple average for now)
68
+ composite_score = np.mean(list(scores.values()))
69
+
70
+ # Apply normalization to the composite score for stability
71
+ composite_score_normalized = min_max_normalization(np.array([composite_score]))[0] # Using min-max normalization
72
+
73
+ # Debug output
74
+ print("Scores per benchmark:", scores)
75
+ print("Composite score:", composite_score)
76
+ print("Normalized composite score:", composite_score_normalized)
77
+
78
+ return composite_score_normalized
79
+
80
+ def analyze_clusters(times, num_clusters=3):
81
+ if len(times) > 0:
82
+ min_time = np.min(times)
83
+ max_time = np.max(times)
84
+ threshold = (max_time - min_time) / num_clusters
85
+ clusters = []
86
+ for i in range(num_clusters):
87
+ lower_bound = min_time + i * threshold
88
+ upper_bound = min_time + (i + 1) * threshold
89
+ cluster = times[(times >= lower_bound) & (times < upper_bound)]
90
+ clusters.append(cluster)
91
+ return clusters
92
+ return [times]
93
+
94
+ app = Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
95
+
96
+ app.layout = dbc.Container([
97
+ dbc.Row([
98
+ dbc.Col(html.H1("Benchmark Dashboard"), className="mb-2")
99
+ ]),
100
+ dbc.Row([
101
+ dbc.Col([
102
+ dcc.Dropdown(
103
+ id='benchmark-dropdown',
104
+ options=[
105
+ {'label': 'CPU', 'value': 'CPU'},
106
+ {'label': 'Memory', 'value': 'Memory'},
107
+ {'label': 'Disk', 'value': 'Disk'},
108
+ {'label': 'IO', 'value': 'IO'},
109
+ {'label': 'Peak Disk Write', 'value': 'PeakDiskWrite'},
110
+ {'label': 'Machine Score', 'value': 'MachineScore'}
111
+ ],
112
+ value='CPU',
113
+ clearable=False
114
+ )
115
+ ], width=4),
116
+ dbc.Col([
117
+ dbc.Checkbox(
118
+ id='cluster-checkbox',
119
+ label='Show Clusters',
120
+ value=False
121
+ )
122
+ ], width=2)
123
+ ]),
124
+ dbc.Row([
125
+ dbc.Col([
126
+ dcc.Graph(id='benchmark-graph')
127
+ ])
128
+ ])
129
+ ])
130
+
131
+ @app.callback(
132
+ Output('benchmark-graph', 'figure'),
133
+ [Input('benchmark-dropdown', 'value'), Input('cluster-checkbox', 'value')]
134
+ )
135
+ def update_graph(benchmark_name, show_clusters):
136
+ if benchmark_name == 'MachineScore':
137
+ score = compute_machine_score()
138
+ fig = px.bar(x=['Machine'], y=[score], labels={'x': 'Machine', 'y': 'Normalized Score'})
139
+ fig.update_layout(
140
+ title='Machine Composite Score',
141
+ yaxis_title='Normalized Score',
142
+ xaxis_title='Machine'
143
+ )
144
+ else:
145
+ df = fetch_benchmark_data()
146
+ filtered_df = df[df['benchmark_name'] == benchmark_name]
147
+ times = filtered_df['time'].values
148
+
149
+ if len(times) > 0: # Ensure times is not empty
150
+ if show_clusters:
151
+ clusters = analyze_clusters(times)
152
+ fig = px.histogram(x=times, nbins=30, histnorm='probability density') # Initialize with full data
153
+ for i, cluster in enumerate(clusters, 1):
154
+ if len(cluster) > 0:
155
+ best_method, normalized_times = select_normalization(cluster, benchmark_name)
156
+ kde = gaussian_kde(normalized_times)
157
+ x_range = np.linspace(min(normalized_times), max(normalized_times), 1000)
158
+ kde_values = kde(x_range)
159
+
160
+ # Add histogram and KDE for each cluster
161
+ fig.add_trace(px.histogram(x=normalized_times, nbins=30, histnorm='probability density').data[0])
162
+ fig.add_scatter(
163
+ x=x_range,
164
+ y=kde_values,
165
+ mode='lines',
166
+ name=f'Cluster {i} Density (Normalization: {best_method})'
167
+ )
168
+ else:
169
+ # Automatically select the best normalization method for the full dataset
170
+ best_method, normalized_times = select_normalization(times, benchmark_name)
171
+ kde = gaussian_kde(normalized_times)
172
+ x_range = np.linspace(min(normalized_times), max(normalized_times), 1000)
173
+ kde_values = kde(x_range)
174
+
175
+ # Create histogram with full dataset
176
+ fig = px.histogram(x=normalized_times, nbins=30, histnorm='probability density')
177
+ fig.add_scatter(
178
+ x=x_range,
179
+ y=kde_values,
180
+ mode='lines',
181
+ name='Density'
182
+ )
183
+ else:
184
+ fig = px.histogram() # Initialize an empty histogram
185
+
186
+ fig.update_layout(
187
+ title=f'{benchmark_name} Benchmark Results' + (' with Clusters' if show_clusters else ''),
188
+ xaxis_title='Normalized Time' if len(times) > 0 else 'Time (seconds)',
189
+ yaxis_title='Density'
190
+ )
191
+
192
+ return fig
193
+
194
+ if __name__ == '__main__':
195
+ app.run_server(debug=True)
@@ -0,0 +1,140 @@
1
+ import time
2
+
3
+ def log_test_time(func):
4
+ def wrapper(*args, **kwargs):
5
+ start_time = time.time()
6
+ result = func(*args, **kwargs)
7
+ elapsed_time = time.time() - start_time
8
+ print(f"{func.__name__} took {elapsed_time:.2f} seconds")
9
+ return result
10
+ return wrapper
11
+
12
+ def verify_condition(condition, message="Verification failed"):
13
+ if not condition:
14
+ raise AssertionError(message)
15
+
16
+
17
+ # TODO >>> Add a Events Manager that can save events locally
18
+ # Events are temporary milestones that we can define to a test
19
+ # This allows us to log events that we expect the test to complete.
20
+
21
+ from .common.sql_pool import SQLiteConnectionPool
22
+ import os
23
+ import pandas as pd
24
+
25
+ THIS_DIR = os.getcwd()
26
+
27
+ class Events_Manager:
28
+
29
+ def __init__(self, Unit:str, path:str):
30
+ """
31
+ if unit == "*" select all units data
32
+ """
33
+
34
+ pool = SQLiteConnectionPool(3, os.path.join(THIS_DIR, "Data.db"))
35
+ self.connection = pool.get_connection()
36
+
37
+ self.Unit = Unit
38
+
39
+ cur = self.connection.cursor()
40
+ cur.execute('''CREATE TABLE IF NOT EXISTS Events (
41
+ ID INTEGER PRIMARY KEY AUTOINCREMENT,
42
+ Unit TEXT,
43
+ StepCompleted TEXT,
44
+ EventType TEXT,
45
+ EventKey TEXT,
46
+ Time NUMBER
47
+ )''')
48
+
49
+
50
+ def drop_events_table(self) -> None:
51
+ """Drop the Events table from the database."""
52
+
53
+ try:
54
+ cur = self.connection.cursor()
55
+
56
+ sql_drop_table_query = """DROP TABLE IF EXISTS Events"""
57
+ cur.execute(sql_drop_table_query)
58
+
59
+ self.connection.commit()
60
+ print("Events table has been removed.")
61
+
62
+ except Exception as e:
63
+ print(f"Error occurred while removing the Events table. Error: {e}")
64
+
65
+
66
+ def List_Events(self) -> dict:
67
+
68
+ cur = self.connection.cursor()
69
+
70
+ if self.Unit == "*":
71
+ sqlite_select_query = """SELECT * FROM Events"""
72
+ cur.execute(sqlite_select_query)
73
+
74
+ else:
75
+ sqlite_select_query = """SELECT * FROM Events WHERE Unit = ?"""
76
+ Data = ((self.Unit, ))
77
+ cur.execute(sqlite_select_query, Data)
78
+
79
+ df = cur.fetchall()
80
+ df = pd.DataFrame(df, columns=['ID', 'Unit', 'StepCompleted', 'EventType', 'EventKey', 'Time'])
81
+ dict_df = df.to_dict()
82
+
83
+ return dict_df
84
+
85
+ def Set_Event (self, step:str, event_type:str="Default", **kwargs):
86
+
87
+ """
88
+ Set a event
89
+
90
+ event_type: str - can be:
91
+ - Default
92
+ - Exception
93
+ - Send
94
+ - Receive
95
+
96
+ if event_type is Send and Receive it needs a predefined `event_key:str` kwarg,
97
+ the event key is a str and need to contain at least 16 digits that need to be random,
98
+ to generate a valid key pair and have 100% sure that this isn't registered you can use the
99
+ helper `gen_valid_event_key.py` at Myscelium/tests/Logs/gen_valid_event_key.py
100
+
101
+ """
102
+
103
+ if self.Unit == "*":
104
+ raise ValueError("Can't Set Event To Generalized Unit: '*'")
105
+
106
+ if event_type in ["Default", "Exception", "Send", "Receive"]:
107
+ pass
108
+ else:
109
+ raise ValueError("Event type can only be one of those: 'Default, Exception, Send, Receive'")
110
+
111
+ events = pd.DataFrame.from_dict(self.List_Events())
112
+
113
+ for i in events.index:
114
+ if events.loc[i, "StepCompleted"] == step:
115
+ return
116
+ else:
117
+ continue
118
+
119
+ cur = self.connection.cursor()
120
+
121
+ ts = time.time()
122
+
123
+ event_key = ""
124
+
125
+ if event_type == "Send" or event_type == "Receive":
126
+
127
+ if not ("event_key" in kwargs):
128
+ raise "You need to to specify a event code to Send an Receive event_types"
129
+ else:
130
+ pass
131
+
132
+ event_key = kwargs["event_key"]
133
+
134
+ Data = ((self.Unit, step, event_type, event_key, ts))
135
+
136
+ sqlite_insert_with_param = """INSERT INTO Events (Unit, StepCompleted, EventType, EventKey, Time) VALUES (?, ?, ?, ?, ?);"""
137
+ cur.execute(sqlite_insert_with_param, Data)
138
+ self.connection.commit()
139
+
140
+ return