atomicshop 2.8.1__py3-none-any.whl → 2.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/print_api.py +2 -0
- atomicshop/system_resource_monitor.py +298 -0
- atomicshop/system_resources.py +168 -6
- atomicshop/wrappers/psutilw/disks.py +146 -25
- {atomicshop-2.8.1.dist-info → atomicshop-2.9.1.dist-info}/METADATA +1 -1
- {atomicshop-2.8.1.dist-info → atomicshop-2.9.1.dist-info}/RECORD +10 -9
- {atomicshop-2.8.1.dist-info → atomicshop-2.9.1.dist-info}/LICENSE.txt +0 -0
- {atomicshop-2.8.1.dist-info → atomicshop-2.9.1.dist-info}/WHEEL +0 -0
- {atomicshop-2.8.1.dist-info → atomicshop-2.9.1.dist-info}/top_level.txt +0 -0
atomicshop/__init__.py
CHANGED
atomicshop/print_api.py
CHANGED
|
@@ -39,6 +39,8 @@ def print_api(
|
|
|
39
39
|
|
|
40
40
|
Then, in the function, you can use it like this:
|
|
41
41
|
print_api(message, **print_kwargs)
|
|
42
|
+
# If 'print_kwargs' is 'None'.
|
|
43
|
+
print_api(message, **(print_kwargs or {}))
|
|
42
44
|
print_api(message, print_kwargs=print_kwargs)
|
|
43
45
|
print_api(message, print_kwargs={'logger': self.logger})
|
|
44
46
|
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
from typing import Union
|
|
2
|
+
import multiprocessing
|
|
3
|
+
|
|
4
|
+
from .print_api import print_api
|
|
5
|
+
from . import system_resources
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def run_check_system_resources(
|
|
9
|
+
interval, get_cpu, get_memory, get_disk_io_bytes, get_disk_files_count, get_disk_used_percent,
|
|
10
|
+
calculate_maximum_changed_disk_io, maximum_disk_io, shared_results, queue=None):
|
|
11
|
+
"""
|
|
12
|
+
Continuously update the system resources in the shared results dictionary.
|
|
13
|
+
This function runs in a separate process.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
while True:
|
|
17
|
+
# Get the results of the system resources check function and store them in temporary results dictionary.
|
|
18
|
+
results = system_resources.check_system_resources(
|
|
19
|
+
interval=interval, get_cpu=get_cpu, get_memory=get_memory,
|
|
20
|
+
get_disk_io_bytes=get_disk_io_bytes, get_disk_files_count=get_disk_files_count,
|
|
21
|
+
get_disk_used_percent=get_disk_used_percent)
|
|
22
|
+
|
|
23
|
+
if calculate_maximum_changed_disk_io:
|
|
24
|
+
if results['disk_io_read'] > maximum_disk_io['read_bytes_per_sec']:
|
|
25
|
+
maximum_disk_io['read_bytes_per_sec'] = results['disk_io_read']
|
|
26
|
+
if results['disk_io_write'] > maximum_disk_io['write_bytes_per_sec']:
|
|
27
|
+
maximum_disk_io['write_bytes_per_sec'] = results['disk_io_write']
|
|
28
|
+
if results['disk_files_count_read'] > maximum_disk_io['read_files_count_per_sec']:
|
|
29
|
+
maximum_disk_io['read_files_count_per_sec'] = results['disk_files_count_read']
|
|
30
|
+
if results['disk_files_count_write'] > maximum_disk_io['write_files_count_per_sec']:
|
|
31
|
+
maximum_disk_io['write_files_count_per_sec'] = results['disk_files_count_write']
|
|
32
|
+
results['maximum_disk_io'] = maximum_disk_io
|
|
33
|
+
|
|
34
|
+
# Update the shared results dictionary with the temporary results dictionary.
|
|
35
|
+
# This is done in separate steps to avoid overwriting the special 'multiprocessing.Manager.dict' object.
|
|
36
|
+
# So we update the shared results dictionary with the temporary results dictionary.
|
|
37
|
+
shared_results.update(results)
|
|
38
|
+
|
|
39
|
+
if queue is not None:
|
|
40
|
+
queue.put(results)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class SystemResourceMonitor:
|
|
44
|
+
"""
|
|
45
|
+
A class to monitor system resources in a separate process.
|
|
46
|
+
"""
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
interval: float = 1,
|
|
50
|
+
get_cpu: bool = True,
|
|
51
|
+
get_memory: bool = True,
|
|
52
|
+
get_disk_io_bytes: bool = True,
|
|
53
|
+
get_disk_files_count: bool = True,
|
|
54
|
+
get_disk_used_percent: bool = True,
|
|
55
|
+
calculate_maximum_changed_disk_io: bool = False,
|
|
56
|
+
use_queue: bool = False
|
|
57
|
+
):
|
|
58
|
+
"""
|
|
59
|
+
Initialize the system resource monitor.
|
|
60
|
+
:param interval: float, the interval in seconds to check the system resources.
|
|
61
|
+
Default is 1 second.
|
|
62
|
+
:param get_cpu: bool, get the CPU usage.
|
|
63
|
+
:param get_memory: bool, get the memory usage.
|
|
64
|
+
:param get_disk_io_bytes: bool, get the disk I/O utilization of bytes.
|
|
65
|
+
:param get_disk_files_count: bool, get the disk files count in the interval.
|
|
66
|
+
:param get_disk_used_percent: bool, get the disk used percentage.
|
|
67
|
+
:param calculate_maximum_changed_disk_io: bool, calculate the maximum changed disk I/O. This includes the
|
|
68
|
+
maximum changed disk I/O read and write in bytes/s and the maximum changed disk files count.
|
|
69
|
+
:param use_queue: bool, use queue to store results.
|
|
70
|
+
If you need ot get the queue, you can access it through the 'queue' attribute:
|
|
71
|
+
SystemResourceMonitor.queue
|
|
72
|
+
|
|
73
|
+
Example:
|
|
74
|
+
system_resource_monitor = SystemResourceMonitor()
|
|
75
|
+
your_queue = system_resource_monitor.queue
|
|
76
|
+
|
|
77
|
+
while True:
|
|
78
|
+
if not your_queue.empty():
|
|
79
|
+
results = your_queue.get()
|
|
80
|
+
print(results)
|
|
81
|
+
|
|
82
|
+
================
|
|
83
|
+
|
|
84
|
+
Usage Example with queue:
|
|
85
|
+
system_resource_monitor = SystemResourceMonitor(use_queue=True)
|
|
86
|
+
system_resource_monitor.start()
|
|
87
|
+
queue = system_resource_monitor.queue
|
|
88
|
+
while True:
|
|
89
|
+
if not queue.empty():
|
|
90
|
+
results = queue.get()
|
|
91
|
+
print(results)
|
|
92
|
+
|
|
93
|
+
================
|
|
94
|
+
|
|
95
|
+
Usage Example without queue:
|
|
96
|
+
interval = 1
|
|
97
|
+
system_resource_monitor = SystemResourceMonitor(interval=interval, use_queue=False)
|
|
98
|
+
system_resource_monitor.start()
|
|
99
|
+
while True:
|
|
100
|
+
time.sleep(interval)
|
|
101
|
+
results = system_resource_monitor.get_latest_results()
|
|
102
|
+
print(results)
|
|
103
|
+
"""
|
|
104
|
+
# Store parameters as instance attributes
|
|
105
|
+
self.interval: float = interval
|
|
106
|
+
self.get_cpu: bool = get_cpu
|
|
107
|
+
self.get_memory: bool = get_memory
|
|
108
|
+
self.get_disk_io_bytes: bool = get_disk_io_bytes
|
|
109
|
+
self.get_disk_files_count: bool = get_disk_files_count
|
|
110
|
+
self.get_disk_used_percent: bool = get_disk_used_percent
|
|
111
|
+
self.calculate_maximum_changed_disk_io: bool = calculate_maximum_changed_disk_io
|
|
112
|
+
|
|
113
|
+
self.manager = multiprocessing.Manager()
|
|
114
|
+
self.shared_results = self.manager.dict()
|
|
115
|
+
self.process = None
|
|
116
|
+
self.maximum_disk_io: dict = {
|
|
117
|
+
'read_bytes_per_sec': 0,
|
|
118
|
+
'write_bytes_per_sec': 0,
|
|
119
|
+
'read_files_count_per_sec': 0,
|
|
120
|
+
'write_files_count_per_sec': 0
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if use_queue:
|
|
124
|
+
self.queue = multiprocessing.Queue()
|
|
125
|
+
else:
|
|
126
|
+
self.queue = None
|
|
127
|
+
|
|
128
|
+
def start(self, print_kwargs: dict = None):
|
|
129
|
+
"""
|
|
130
|
+
Start the monitoring process.
|
|
131
|
+
:param print_kwargs:
|
|
132
|
+
:return:
|
|
133
|
+
"""
|
|
134
|
+
if print_kwargs is None:
|
|
135
|
+
print_kwargs = {}
|
|
136
|
+
|
|
137
|
+
if self.process is None or not self.process.is_alive():
|
|
138
|
+
self.process = multiprocessing.Process(target=run_check_system_resources, args=(
|
|
139
|
+
self.interval, self.get_cpu, self.get_memory, self.get_disk_io_bytes, self.get_disk_files_count,
|
|
140
|
+
self.get_disk_used_percent, self.calculate_maximum_changed_disk_io, self.maximum_disk_io,
|
|
141
|
+
self.shared_results, self.queue))
|
|
142
|
+
self.process.start()
|
|
143
|
+
else:
|
|
144
|
+
print_api("Monitoring process is already running.", color='yellow', **print_kwargs)
|
|
145
|
+
|
|
146
|
+
def get_latest_results(self) -> dict:
|
|
147
|
+
"""
|
|
148
|
+
Retrieve the latest results from the shared results dictionary.
|
|
149
|
+
"""
|
|
150
|
+
return dict(self.shared_results)
|
|
151
|
+
|
|
152
|
+
def stop(self):
|
|
153
|
+
"""
|
|
154
|
+
Stop the monitoring process.
|
|
155
|
+
"""
|
|
156
|
+
if self.process is not None:
|
|
157
|
+
self.process.terminate()
|
|
158
|
+
self.process.join()
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# === END OF SYSTEM RESOURCE MONITOR. ==================================================================================
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
SYSTEM_RESOURCES_MONITOR: Union[SystemResourceMonitor, None] = None
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def start_monitoring(
|
|
168
|
+
interval: float = 1,
|
|
169
|
+
get_cpu: bool = True,
|
|
170
|
+
get_memory: bool = True,
|
|
171
|
+
get_disk_io_bytes: bool = True,
|
|
172
|
+
get_disk_files_count: bool = True,
|
|
173
|
+
get_disk_used_percent: bool = True,
|
|
174
|
+
calculate_maximum_changed_disk_io: bool = False,
|
|
175
|
+
use_queue: bool = False,
|
|
176
|
+
print_kwargs: dict = None
|
|
177
|
+
):
|
|
178
|
+
"""
|
|
179
|
+
Start monitoring system resources.
|
|
180
|
+
:param interval: float, interval in seconds.
|
|
181
|
+
:param get_cpu: bool, get CPU usage.
|
|
182
|
+
:param get_memory: bool, get memory usage.
|
|
183
|
+
:param get_disk_io_bytes: bool, get TOTAL disk I/O utilization in bytes/s.
|
|
184
|
+
:param get_disk_files_count: bool, get TOTAL disk files count.
|
|
185
|
+
:param get_disk_used_percent: bool, get TOTAL disk used percentage.
|
|
186
|
+
:param calculate_maximum_changed_disk_io: bool, calculate the maximum changed disk I/O. This includes the
|
|
187
|
+
maximum changed disk I/O read and write in bytes/s and the maximum changed disk files count.
|
|
188
|
+
:param use_queue: bool, use queue to store results.
|
|
189
|
+
Usage Example:
|
|
190
|
+
system_resources.start_monitoring(use_queue=True)
|
|
191
|
+
queue = system_resources.get_monitoring_queue()
|
|
192
|
+
while True:
|
|
193
|
+
if not queue.empty():
|
|
194
|
+
results = queue.get()
|
|
195
|
+
print(results)
|
|
196
|
+
|
|
197
|
+
:param print_kwargs: dict, print kwargs.
|
|
198
|
+
:return:
|
|
199
|
+
"""
|
|
200
|
+
|
|
201
|
+
# if print_kwargs is None:
|
|
202
|
+
# print_kwargs = {}
|
|
203
|
+
|
|
204
|
+
global SYSTEM_RESOURCES_MONITOR
|
|
205
|
+
|
|
206
|
+
if not SYSTEM_RESOURCES_MONITOR:
|
|
207
|
+
SYSTEM_RESOURCES_MONITOR = SystemResourceMonitor(
|
|
208
|
+
interval=interval,
|
|
209
|
+
get_cpu=get_cpu,
|
|
210
|
+
get_memory=get_memory,
|
|
211
|
+
get_disk_io_bytes=get_disk_io_bytes,
|
|
212
|
+
get_disk_files_count=get_disk_files_count,
|
|
213
|
+
get_disk_used_percent=get_disk_used_percent,
|
|
214
|
+
calculate_maximum_changed_disk_io=calculate_maximum_changed_disk_io,
|
|
215
|
+
use_queue=use_queue
|
|
216
|
+
)
|
|
217
|
+
SYSTEM_RESOURCES_MONITOR.start()
|
|
218
|
+
else:
|
|
219
|
+
print_api("System resources monitoring is already running.", color='yellow', **(print_kwargs or {}))
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def stop_monitoring():
|
|
223
|
+
"""
|
|
224
|
+
Stop monitoring system resources.
|
|
225
|
+
:return: None
|
|
226
|
+
"""
|
|
227
|
+
global SYSTEM_RESOURCES_MONITOR
|
|
228
|
+
if SYSTEM_RESOURCES_MONITOR is not None:
|
|
229
|
+
SYSTEM_RESOURCES_MONITOR.stop()
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def get_monitoring_instance() -> SystemResourceMonitor:
|
|
233
|
+
"""
|
|
234
|
+
Get the system resources monitoring instance.
|
|
235
|
+
:return: SystemResourceMonitor
|
|
236
|
+
"""
|
|
237
|
+
global SYSTEM_RESOURCES_MONITOR
|
|
238
|
+
return SYSTEM_RESOURCES_MONITOR
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def get_result():
|
|
242
|
+
"""
|
|
243
|
+
Get system resources monitoring result.
|
|
244
|
+
|
|
245
|
+
Usage Example:
|
|
246
|
+
system_resources.start_monitoring()
|
|
247
|
+
|
|
248
|
+
while True:
|
|
249
|
+
time.sleep(1)
|
|
250
|
+
result = system_resources.get_result()
|
|
251
|
+
|
|
252
|
+
if result:
|
|
253
|
+
print(
|
|
254
|
+
f"{str(result['cpu_usage'])} | {str(result['memory_usage'])} | "
|
|
255
|
+
f"{str(result['disk_io_read'])} | {str(result['disk_io_write'])} | "
|
|
256
|
+
f"{str(result['disk_used_percent'])}"
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
:return: dict
|
|
260
|
+
"""
|
|
261
|
+
global SYSTEM_RESOURCES_MONITOR
|
|
262
|
+
if SYSTEM_RESOURCES_MONITOR is not None:
|
|
263
|
+
return SYSTEM_RESOURCES_MONITOR.get_latest_results()
|
|
264
|
+
else:
|
|
265
|
+
raise RuntimeError("System resources monitoring is not running.")
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def get_result_by_queue():
|
|
269
|
+
"""
|
|
270
|
+
Get system resources monitoring result by queue.
|
|
271
|
+
|
|
272
|
+
Usage Example:
|
|
273
|
+
system_resources.start_system_resources_monitoring()
|
|
274
|
+
|
|
275
|
+
while True:
|
|
276
|
+
result = system_resources.get_result_by_queue()
|
|
277
|
+
print(result)
|
|
278
|
+
|
|
279
|
+
:return: dict
|
|
280
|
+
"""
|
|
281
|
+
global SYSTEM_RESOURCES_MONITOR
|
|
282
|
+
if SYSTEM_RESOURCES_MONITOR is not None:
|
|
283
|
+
if not SYSTEM_RESOURCES_MONITOR.queue.empty():
|
|
284
|
+
return SYSTEM_RESOURCES_MONITOR.queue.get()
|
|
285
|
+
else:
|
|
286
|
+
raise RuntimeError("System resources monitoring is not running.")
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def get_monitoring_queue() -> Union[multiprocessing.Queue, None]:
|
|
290
|
+
"""
|
|
291
|
+
Get the monitoring queue.
|
|
292
|
+
:return: multiprocessing.Queue
|
|
293
|
+
"""
|
|
294
|
+
global SYSTEM_RESOURCES_MONITOR
|
|
295
|
+
if SYSTEM_RESOURCES_MONITOR is not None:
|
|
296
|
+
return SYSTEM_RESOURCES_MONITOR.queue
|
|
297
|
+
else:
|
|
298
|
+
raise RuntimeError("System resources monitoring is not running.")
|
atomicshop/system_resources.py
CHANGED
|
@@ -1,15 +1,86 @@
|
|
|
1
|
-
import
|
|
1
|
+
import os
|
|
2
2
|
import time
|
|
3
|
+
import tempfile
|
|
4
|
+
import shutil
|
|
5
|
+
import threading
|
|
3
6
|
|
|
4
7
|
from .print_api import print_api
|
|
5
8
|
from .wrappers.psutilw import cpus, memories, disks
|
|
9
|
+
from . import system_resource_monitor
|
|
6
10
|
|
|
7
11
|
|
|
8
|
-
def check_system_resources(
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
12
|
+
def check_system_resources(
|
|
13
|
+
interval: float = 1,
|
|
14
|
+
get_cpu: bool = True,
|
|
15
|
+
get_memory: bool = True,
|
|
16
|
+
get_disk_io_bytes: bool = True,
|
|
17
|
+
get_disk_files_count: bool = False,
|
|
18
|
+
get_disk_used_percent: bool = True
|
|
19
|
+
):
|
|
20
|
+
"""
|
|
21
|
+
Check system resources.
|
|
22
|
+
:param interval: float, interval in seconds.
|
|
23
|
+
:param get_cpu: bool, get CPU usage.
|
|
24
|
+
:param get_memory: bool, get memory usage.
|
|
25
|
+
:param get_disk_io_bytes: bool, get TOTAL disk I/O utilization in bytes/s.
|
|
26
|
+
:param get_disk_files_count: bool, get TOTAL disk files count.
|
|
27
|
+
:param get_disk_used_percent: bool, get TOTAL disk used percentage.
|
|
28
|
+
:return:
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
threads: list = []
|
|
32
|
+
result: dict = {
|
|
33
|
+
'cpu_usage': None,
|
|
34
|
+
'memory_usage': None,
|
|
35
|
+
'disk_io_write': None,
|
|
36
|
+
'disk_io_read': None,
|
|
37
|
+
'disk_files_count_read': None,
|
|
38
|
+
'disk_files_count_write': None,
|
|
39
|
+
'disk_used_percent': None
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
def set_cpu_usage():
|
|
43
|
+
result['cpu_usage'] = cpus.get_cpu_usage(interval=interval)
|
|
44
|
+
|
|
45
|
+
def set_memory_usage():
|
|
46
|
+
result['memory_usage'] = memories.get_memory_usage()
|
|
47
|
+
|
|
48
|
+
def set_disk_io_bytes_change():
|
|
49
|
+
aggregated_disk_io_utilization = (
|
|
50
|
+
disks.get_disk_io(interval=interval, aggregated=True, io_change_bytes=True))['aggregated']
|
|
51
|
+
result['disk_io_read'] = aggregated_disk_io_utilization['read_change_per_sec']
|
|
52
|
+
result['disk_io_write'] = aggregated_disk_io_utilization['write_change_per_sec']
|
|
53
|
+
|
|
54
|
+
def set_disk_files_count():
|
|
55
|
+
aggregated_disk_files_count = (
|
|
56
|
+
disks.get_disk_io(interval=interval, aggregated=True, io_file_count=True))['aggregated']
|
|
57
|
+
result['disk_files_count_read'] = aggregated_disk_files_count['read_file_count_per_sec']
|
|
58
|
+
result['disk_files_count_write'] = aggregated_disk_files_count['write_file_count_per_sec']
|
|
59
|
+
|
|
60
|
+
def set_disk_used_percent():
|
|
61
|
+
result['disk_used_percent'] = disks.get_disk_usage()['aggregated'].percent
|
|
62
|
+
|
|
63
|
+
# Create threads for each system resource check.
|
|
64
|
+
if get_cpu:
|
|
65
|
+
threads.append(threading.Thread(target=set_cpu_usage))
|
|
66
|
+
if get_memory:
|
|
67
|
+
threads.append(threading.Thread(target=set_memory_usage))
|
|
68
|
+
if get_disk_io_bytes:
|
|
69
|
+
threads.append(threading.Thread(target=set_disk_io_bytes_change))
|
|
70
|
+
if get_disk_files_count:
|
|
71
|
+
threads.append(threading.Thread(target=set_disk_files_count))
|
|
72
|
+
if get_disk_used_percent:
|
|
73
|
+
threads.append(threading.Thread(target=set_disk_used_percent))
|
|
74
|
+
|
|
75
|
+
# Start threads.
|
|
76
|
+
for thread in threads:
|
|
77
|
+
thread.start()
|
|
78
|
+
|
|
79
|
+
# Wait for all threads to complete.
|
|
80
|
+
for thread in threads:
|
|
81
|
+
thread.join()
|
|
82
|
+
|
|
83
|
+
return result
|
|
13
84
|
|
|
14
85
|
|
|
15
86
|
def wait_for_resource_availability(cpu_percent_max: int = 80, memory_percent_max: int = 80, wait_time: float = 5):
|
|
@@ -26,3 +97,94 @@ def wait_for_resource_availability(cpu_percent_max: int = 80, memory_percent_max
|
|
|
26
97
|
break
|
|
27
98
|
print_api(f"Waiting for resources to be available... CPU: {cpu}%, Memory: {memory}%", color='yellow')
|
|
28
99
|
time.sleep(wait_time) # Wait for 'wait_time' seconds before checking again
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_disk_speed_with_monitoring(
|
|
103
|
+
file_settings: list[dict],
|
|
104
|
+
remove_file_after_each_copy: bool = False,
|
|
105
|
+
target_directory=None,
|
|
106
|
+
read_full_file: bool = False,
|
|
107
|
+
monitoring: bool = True,
|
|
108
|
+
print_kwargs: dict = None
|
|
109
|
+
):
|
|
110
|
+
"""
|
|
111
|
+
Generates files and performs write and read operations in the specified target directory,
|
|
112
|
+
while monitoring disk I/O speeds in a separate thread. Returns the maximum read and write rates,
|
|
113
|
+
and the total operation time.
|
|
114
|
+
|
|
115
|
+
:param file_settings: list of dicts, of file settings. Each dict will contain:
|
|
116
|
+
'size': int, size of each file in bytes.
|
|
117
|
+
'count': int, number of files to generate and copy.
|
|
118
|
+
|
|
119
|
+
Example:
|
|
120
|
+
file_setting = [
|
|
121
|
+
{'size': 100000000, 'count': 100},
|
|
122
|
+
{'size': 500000000, 'count': 50}
|
|
123
|
+
]
|
|
124
|
+
|
|
125
|
+
This will generate 100 files of 100 MB each, and 50 files of 500 MB each.
|
|
126
|
+
:param remove_file_after_each_copy: Whether to remove the file after copying to target directory.
|
|
127
|
+
:param target_directory: Directory where files will be copied. Uses a temporary directory if None.
|
|
128
|
+
:param read_full_file: Whether to read the full file after copying, or read in chunks.
|
|
129
|
+
:param monitoring: Whether to skip monitoring disk I/O speeds.
|
|
130
|
+
:return: A tuple containing the total operation time in seconds and maximum_io_changes.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
max_io_changes: dict = {}
|
|
134
|
+
|
|
135
|
+
if monitoring:
|
|
136
|
+
system_resource_monitor.start_monitoring(
|
|
137
|
+
interval=1, get_cpu=False, get_memory=False, get_disk_io_bytes=True, get_disk_used_percent=False,
|
|
138
|
+
get_disk_files_count=True, calculate_maximum_changed_disk_io=True, use_queue=True)
|
|
139
|
+
|
|
140
|
+
if target_directory is None:
|
|
141
|
+
target_directory = tempfile.mkdtemp()
|
|
142
|
+
else:
|
|
143
|
+
os.makedirs(target_directory, exist_ok=True)
|
|
144
|
+
|
|
145
|
+
source_directory = os.path.join(target_directory, 'source')
|
|
146
|
+
dest_directory = os.path.join(target_directory, 'dest')
|
|
147
|
+
os.makedirs(source_directory, exist_ok=True)
|
|
148
|
+
os.makedirs(dest_directory, exist_ok=True)
|
|
149
|
+
|
|
150
|
+
overall_start_time = time.time() # Start timing the entire operation
|
|
151
|
+
|
|
152
|
+
for file_setting in file_settings:
|
|
153
|
+
for i in range(file_setting['count']):
|
|
154
|
+
# Generate file in source directory
|
|
155
|
+
src_file_path = os.path.join(source_directory, f"tempfile_{i}")
|
|
156
|
+
with open(src_file_path, "wb") as file:
|
|
157
|
+
file.write(os.urandom(file_setting['size']))
|
|
158
|
+
|
|
159
|
+
# Measure write speed.
|
|
160
|
+
shutil.copy(src_file_path, dest_directory)
|
|
161
|
+
|
|
162
|
+
target_file_path = os.path.join(dest_directory, os.path.basename(src_file_path))
|
|
163
|
+
print_api(f"Copied: {target_file_path}", **(print_kwargs or {}))
|
|
164
|
+
|
|
165
|
+
# Measure read speed.
|
|
166
|
+
with open(target_file_path, "rb") as file:
|
|
167
|
+
if read_full_file:
|
|
168
|
+
file.read()
|
|
169
|
+
else:
|
|
170
|
+
while file.read(1024 * 1024): # Read in chunks of 1 MB
|
|
171
|
+
pass
|
|
172
|
+
|
|
173
|
+
if remove_file_after_each_copy:
|
|
174
|
+
os.remove(target_file_path)
|
|
175
|
+
os.remove(src_file_path)
|
|
176
|
+
|
|
177
|
+
overall_end_time = time.time()
|
|
178
|
+
total_execution_time = overall_end_time - overall_start_time
|
|
179
|
+
print_api(f"Total execution time: {total_execution_time}", **(print_kwargs or {}))
|
|
180
|
+
|
|
181
|
+
# Cleanup. Remove all created files and directories.
|
|
182
|
+
shutil.rmtree(source_directory)
|
|
183
|
+
shutil.rmtree(dest_directory)
|
|
184
|
+
|
|
185
|
+
if monitoring:
|
|
186
|
+
# Stop the I/O monitoring.
|
|
187
|
+
max_io_changes = system_resource_monitor.get_result()['maximum_disk_io']
|
|
188
|
+
system_resource_monitor.stop_monitoring()
|
|
189
|
+
|
|
190
|
+
return total_execution_time, max_io_changes
|
|
@@ -1,31 +1,74 @@
|
|
|
1
|
-
import psutil
|
|
2
1
|
import time
|
|
3
2
|
|
|
3
|
+
import psutil
|
|
4
|
+
|
|
4
5
|
|
|
5
|
-
def
|
|
6
|
+
def get_disk_io(
|
|
6
7
|
interval: float = 1,
|
|
7
8
|
disk_list: list = None,
|
|
8
9
|
aggregated: bool = True,
|
|
9
|
-
separated: bool = False
|
|
10
|
+
separated: bool = False,
|
|
11
|
+
io_change_bytes: bool = False,
|
|
12
|
+
io_file_count: bool = False,
|
|
13
|
+
io_busy_time: bool = False,
|
|
10
14
|
) -> dict:
|
|
11
15
|
"""
|
|
12
16
|
Get disk utilization based on disk I/O changes, allowing for both aggregated and separated values.
|
|
17
|
+
Windows: because 'psutil.disk_io_counters' before using this function, you may need to execute the following
|
|
18
|
+
command in the command prompt:
|
|
19
|
+
diskperf -y
|
|
13
20
|
|
|
14
21
|
:param interval: Sampling interval in seconds to measure I/O changes.
|
|
15
22
|
:param disk_list: List of disks to measure. If None, measure all disks. Affects only when separated is True.
|
|
16
23
|
:param aggregated: Boolean indicating whether to return aggregated utilization.
|
|
17
24
|
:param separated: Boolean indicating whether to return separate utilizations for each disk.
|
|
25
|
+
:param io_change_bytes: Boolean indicating whether to return I/O change in bytes. Returned dictionary:
|
|
26
|
+
{
|
|
27
|
+
'read_change_bytes': int,
|
|
28
|
+
'write_change_bytes': int,
|
|
29
|
+
'read_change_per_sec': float,
|
|
30
|
+
'write_change_per_sec': float
|
|
31
|
+
}
|
|
32
|
+
:param io_file_count: Boolean indicating whether to return I/O file count. Returned dictionary:
|
|
33
|
+
{
|
|
34
|
+
'read_file_count': int,
|
|
35
|
+
'write_file_count': int,
|
|
36
|
+
'read_file_count_per_sec': float,
|
|
37
|
+
'write_file_count_per_sec': float
|
|
38
|
+
}
|
|
39
|
+
:param io_busy_time: Boolean indicating whether to return I/O busy time.
|
|
40
|
+
!!! For some reason on Windows it gets the count of files read or written and not the time in ms.
|
|
41
|
+
Returned dictionary:
|
|
42
|
+
{
|
|
43
|
+
'read_time_ms': int,
|
|
44
|
+
'write_time_ms': int,
|
|
45
|
+
'read_time_per_sec': float,
|
|
46
|
+
'write_time_per_sec': float,
|
|
47
|
+
'busy_time': int,
|
|
48
|
+
'busy_time_per_sec': float,
|
|
49
|
+
'busy_time_percent': float
|
|
50
|
+
}
|
|
18
51
|
:return: Disk utilization data.
|
|
19
52
|
"""
|
|
20
53
|
|
|
54
|
+
if not aggregated and not separated:
|
|
55
|
+
raise ValueError('At least one of aggregated or separated must be True.')
|
|
56
|
+
|
|
57
|
+
if not io_change_bytes and not io_file_count and not io_busy_time:
|
|
58
|
+
raise ValueError('At least one of io_change_bytes, io_file_count, or io_busy_time must be True.')
|
|
59
|
+
|
|
21
60
|
io_start_aggregated = None
|
|
22
61
|
io_end_aggregated = None
|
|
23
62
|
|
|
24
|
-
|
|
63
|
+
if separated:
|
|
64
|
+
io_start_separated = psutil.disk_io_counters(perdisk=True)
|
|
25
65
|
if aggregated:
|
|
26
66
|
io_start_aggregated = psutil.disk_io_counters(perdisk=False)
|
|
67
|
+
|
|
27
68
|
time.sleep(interval)
|
|
28
|
-
|
|
69
|
+
|
|
70
|
+
if separated:
|
|
71
|
+
io_end_separated = psutil.disk_io_counters(perdisk=True)
|
|
29
72
|
if aggregated:
|
|
30
73
|
io_end_aggregated = psutil.disk_io_counters(perdisk=False)
|
|
31
74
|
|
|
@@ -33,29 +76,107 @@ def get_disk_io_utilization(
|
|
|
33
76
|
if separated:
|
|
34
77
|
for disk in io_start_separated.keys():
|
|
35
78
|
if disk_list is None or disk in disk_list:
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
79
|
+
io_change[disk] = {}
|
|
80
|
+
if io_change_bytes:
|
|
81
|
+
read_change = io_end_separated[disk].read_bytes - io_start_separated[disk].read_bytes
|
|
82
|
+
write_change = io_end_separated[disk].write_bytes - io_start_separated[disk].write_bytes
|
|
83
|
+
read_change_per_sec = read_change / interval
|
|
84
|
+
write_change_per_sec = write_change / interval
|
|
85
|
+
io_change[disk].update({
|
|
86
|
+
'read_change_bytes': read_change,
|
|
87
|
+
'write_change_bytes': write_change,
|
|
88
|
+
'read_change_per_sec': read_change_per_sec,
|
|
89
|
+
'write_change_per_sec': write_change_per_sec,
|
|
90
|
+
})
|
|
91
|
+
if io_file_count:
|
|
92
|
+
read_count = io_end_separated[disk].read_count - io_start_separated[disk].read_count
|
|
93
|
+
write_count = io_end_separated[disk].write_count - io_start_separated[disk].write_count
|
|
94
|
+
read_count_per_sec = read_count / interval
|
|
95
|
+
write_count_per_sec = write_count / interval
|
|
96
|
+
io_change[disk].update({
|
|
97
|
+
'read_file_count': read_count,
|
|
98
|
+
'write_file_count': write_count,
|
|
99
|
+
'read_file_count_per_sec': read_count_per_sec,
|
|
100
|
+
'write_file_count_per_sec': write_count_per_sec,
|
|
101
|
+
})
|
|
102
|
+
if io_busy_time:
|
|
103
|
+
read_time = io_end_separated[disk].read_time - io_start_separated[disk].read_time
|
|
104
|
+
write_time = io_end_separated[disk].write_time - io_start_separated[disk].write_time
|
|
105
|
+
read_time_per_sec = read_time / interval
|
|
106
|
+
write_time_per_sec = write_time / interval
|
|
107
|
+
io_change[disk].update({
|
|
108
|
+
'read_time_ms': read_time,
|
|
109
|
+
'write_time_ms': write_time,
|
|
110
|
+
'read_time_per_sec': read_time_per_sec,
|
|
111
|
+
'write_time_per_sec': write_time_per_sec,
|
|
112
|
+
'busy_time': read_time + write_time,
|
|
113
|
+
'busy_time_per_sec': read_time_per_sec + write_time_per_sec,
|
|
114
|
+
'busy_time_percent': (read_time + write_time) / interval
|
|
115
|
+
})
|
|
46
116
|
|
|
47
117
|
if aggregated:
|
|
48
118
|
if not io_start_aggregated or not io_end_aggregated:
|
|
49
119
|
raise ValueError('Aggregated disk I/O counters are not available.')
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
'
|
|
59
|
-
|
|
120
|
+
|
|
121
|
+
io_change['aggregated'] = {}
|
|
122
|
+
|
|
123
|
+
if io_change_bytes:
|
|
124
|
+
total_read_change = io_end_aggregated.read_bytes - io_start_aggregated.read_bytes
|
|
125
|
+
total_write_change = io_end_aggregated.write_bytes - io_start_aggregated.write_bytes
|
|
126
|
+
total_read_change_per_sec = total_read_change / interval
|
|
127
|
+
total_write_change_per_sec = total_write_change / interval
|
|
128
|
+
io_change['aggregated'] = {
|
|
129
|
+
'read_change_bytes': total_read_change,
|
|
130
|
+
'write_change_bytes': total_write_change,
|
|
131
|
+
'read_change_per_sec': total_read_change_per_sec,
|
|
132
|
+
'write_change_per_sec': total_write_change_per_sec,
|
|
133
|
+
}
|
|
134
|
+
if io_file_count:
|
|
135
|
+
total_read_count = io_end_aggregated.read_count - io_start_aggregated.read_count
|
|
136
|
+
total_write_count = io_end_aggregated.write_count - io_start_aggregated.write_count
|
|
137
|
+
total_read_count_per_sec = total_read_count / interval
|
|
138
|
+
total_write_count_per_sec = total_write_count / interval
|
|
139
|
+
io_change['aggregated'].update({
|
|
140
|
+
'read_file_count': total_read_count,
|
|
141
|
+
'write_file_count': total_write_count,
|
|
142
|
+
'read_file_count_per_sec': total_read_count_per_sec,
|
|
143
|
+
'write_file_count_per_sec': total_write_count_per_sec,
|
|
144
|
+
})
|
|
145
|
+
if io_busy_time:
|
|
146
|
+
total_read_time = io_end_aggregated.read_time - io_start_aggregated.read_time
|
|
147
|
+
total_write_time = io_end_aggregated.write_time - io_start_aggregated.write_time
|
|
148
|
+
total_read_time_per_sec = total_read_time / interval
|
|
149
|
+
total_write_time_per_sec = total_write_time / interval
|
|
150
|
+
io_change['aggregated'].update({
|
|
151
|
+
'read_time_ms': total_read_time,
|
|
152
|
+
'write_time_ms': total_write_time,
|
|
153
|
+
'read_time_per_sec': total_read_time_per_sec,
|
|
154
|
+
'write_time_per_sec': total_write_time_per_sec,
|
|
155
|
+
'busy_time': total_read_time + total_write_time,
|
|
156
|
+
'busy_time_per_sec': total_read_time_per_sec + total_write_time_per_sec,
|
|
157
|
+
'busy_time_percent': (total_read_time + total_write_time) / interval
|
|
158
|
+
})
|
|
60
159
|
|
|
61
160
|
return io_change
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def get_disk_usage(disk_list: list = None) -> dict:
|
|
164
|
+
"""
|
|
165
|
+
Get the usage statistics of disks.
|
|
166
|
+
|
|
167
|
+
:param disk_list: List of disks to measure. If None, measure all disks.
|
|
168
|
+
:return:
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
disk_usage: dict = {}
|
|
172
|
+
for disk in psutil.disk_partitions():
|
|
173
|
+
if disk_list is None or disk.device in disk_list:
|
|
174
|
+
try:
|
|
175
|
+
disk_usage[disk.device] = psutil.disk_usage(disk.mountpoint)
|
|
176
|
+
except PermissionError as e:
|
|
177
|
+
disk_usage[disk.device] = str(e)
|
|
178
|
+
|
|
179
|
+
# Get total disk usage.
|
|
180
|
+
disk_usage['aggregated'] = psutil.disk_usage('/')
|
|
181
|
+
|
|
182
|
+
return disk_usage
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
atomicshop/__init__.py,sha256=
|
|
1
|
+
atomicshop/__init__.py,sha256=6KyCPoNOFCE-qMHSsE5D6KYM2ED2PeS6vsl4SuxrWGU,122
|
|
2
2
|
atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
|
|
3
3
|
atomicshop/_patch_import.py,sha256=ENp55sKVJ0e6-4lBvZnpz9PQCt3Otbur7F6aXDlyje4,6334
|
|
4
4
|
atomicshop/appointment_management.py,sha256=N3wVGJgrqJfsj_lqiRfaL3FxMEe57by5Stzanh189mk,7263
|
|
@@ -22,7 +22,7 @@ atomicshop/ip_addresses.py,sha256=Hvi4TumEFoTEpKWaq5WNF-YzcRzt24IxmNgv-Mgax1s,11
|
|
|
22
22
|
atomicshop/keyboard_press.py,sha256=1W5kRtOB75fulVx-uF2yarBhW0_IzdI1k73AnvXstk0,452
|
|
23
23
|
atomicshop/pbtkmultifile_argparse.py,sha256=aEk8nhvoQVu-xyfZosK3ma17CwIgOjzO1erXXdjwtS4,4574
|
|
24
24
|
atomicshop/permissions.py,sha256=GYWxm9niuJWY0pWCnASF34_HiTuvhTamn8BCgcFyVAk,2540
|
|
25
|
-
atomicshop/print_api.py,sha256=
|
|
25
|
+
atomicshop/print_api.py,sha256=DhbCQd0MWZZ5GYEk4oTu1opRFC-b31g1VWZgTGewG2Y,11568
|
|
26
26
|
atomicshop/process.py,sha256=5lqepm1psRbz66J4MB43ou6HHyac9F34QJedGukwAnk,14188
|
|
27
27
|
atomicshop/process_name_cmd.py,sha256=TNAK6kQZm5JKWzEW6QLqVHEG98ZLNDQiSS4YwDk8V8c,3830
|
|
28
28
|
atomicshop/process_poller.py,sha256=WfmwCLALfTYOq8ri0vkPeqq8ruEyA_43DaN--CU2_XY,10854
|
|
@@ -36,7 +36,8 @@ atomicshop/sound.py,sha256=KSzWRF8dkpEVXmFidIv-Eftc3kET-hQzQOxZRE7rMto,24297
|
|
|
36
36
|
atomicshop/speech_recognize.py,sha256=55-dIjgkpF93mvJnJuxSFuft5H5eRvGNlUj9BeIOZxk,5903
|
|
37
37
|
atomicshop/ssh_remote.py,sha256=Sas3nrQv8ardxR51t59xZZsYm8nvUcA7tMSqEDViRLk,17155
|
|
38
38
|
atomicshop/sys_functions.py,sha256=MTBxRve5bh58SPvhX3gMiGqHlSBuI_rdNN1NnnBBWqI,906
|
|
39
|
-
atomicshop/
|
|
39
|
+
atomicshop/system_resource_monitor.py,sha256=1LgMKCawatUiLAXkbm31vdmTu2cEE8Y1UhsESJWUslE,11289
|
|
40
|
+
atomicshop/system_resources.py,sha256=w4F1si9mJNS26UeHiFm68UwrfEb618QRNekb6Git_C8,7523
|
|
40
41
|
atomicshop/tempfiles.py,sha256=uq1ve2WlWehZ3NOTXJnpBBMt6HyCdBufqedF0HyzA6k,2517
|
|
41
42
|
atomicshop/timer.py,sha256=KxBBgVM8po6pUJDW8TgY1UXj0iiDmRmL5XDCq0VHAfU,1670
|
|
42
43
|
atomicshop/urls.py,sha256=CQl1j1kjEVDlAuYJqYD9XxPF1SUSgrmG8PjlcXNEKsQ,597
|
|
@@ -204,7 +205,7 @@ atomicshop/wrappers/playwrightw/mouse.py,sha256=-2FZbQtjgH7tdXWld6ZPGqlKFUdf5in0
|
|
|
204
205
|
atomicshop/wrappers/playwrightw/scenarios.py,sha256=Wz7aVYfG7K4fuSe_TUAc1jhFXVq5jYvZKbDtvqUiONc,5236
|
|
205
206
|
atomicshop/wrappers/playwrightw/waits.py,sha256=308fdOu6YDqQ7K7xywj7R27sSmFanPBQqpZyBC-NFmo,7015
|
|
206
207
|
atomicshop/wrappers/psutilw/cpus.py,sha256=w6LPBMINqS-T_X8vzdYkLS2Wzuve28Ydp_GafTCngrc,236
|
|
207
|
-
atomicshop/wrappers/psutilw/disks.py,sha256=
|
|
208
|
+
atomicshop/wrappers/psutilw/disks.py,sha256=7Ly6QlKI36lYw2AeUn8MlQsYMRTvDz-O9Kp-PRq_ym4,8463
|
|
208
209
|
atomicshop/wrappers/psutilw/memories.py,sha256=wpdKEkQ9Wty_r7ZJKkfli7wIHMXdQOMlmDlzmc_0FWo,161
|
|
209
210
|
atomicshop/wrappers/psutilw/psutilw.py,sha256=W9PSEZmrm_Ct_-6oKqAcbgbyF21CwcIbbHOkVqgMiow,20866
|
|
210
211
|
atomicshop/wrappers/psycopgw/psycopgw.py,sha256=XJvVf0oAUjCHkrYfKeFuGCpfn0Oxj3u4SbKMKA1508E,7118
|
|
@@ -225,8 +226,8 @@ atomicshop/wrappers/socketw/socket_server_tester.py,sha256=AhpurHJmP2kgzHaUbq5ey
|
|
|
225
226
|
atomicshop/wrappers/socketw/socket_wrapper.py,sha256=aXBwlEIJhFT0-c4i8iNlFx2It9VpCEpsv--5Oqcpxao,11624
|
|
226
227
|
atomicshop/wrappers/socketw/ssl_base.py,sha256=k4V3gwkbq10MvOH4btU4onLX2GNOsSfUAdcHmL1rpVE,2274
|
|
227
228
|
atomicshop/wrappers/socketw/statistics_csv.py,sha256=t3dtDEfN47CfYVi0CW6Kc2QHTEeZVyYhc57IYYh5nmA,826
|
|
228
|
-
atomicshop-2.
|
|
229
|
-
atomicshop-2.
|
|
230
|
-
atomicshop-2.
|
|
231
|
-
atomicshop-2.
|
|
232
|
-
atomicshop-2.
|
|
229
|
+
atomicshop-2.9.1.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
|
|
230
|
+
atomicshop-2.9.1.dist-info/METADATA,sha256=1u71oL74BRp4FmrfREViR7QCTlEwf4m1MC870Dv2RSM,10369
|
|
231
|
+
atomicshop-2.9.1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
|
|
232
|
+
atomicshop-2.9.1.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
|
|
233
|
+
atomicshop-2.9.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|