atomicshop 2.16.30__py3-none-any.whl → 2.16.32__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/basics/booleans.py +28 -40
- atomicshop/config_init.py +4 -4
- atomicshop/file_io/csvs.py +2 -3
- atomicshop/file_io/file_io.py +5 -3
- atomicshop/mitm/engines/__parent/recorder___parent.py +2 -18
- atomicshop/mitm/import_config.py +10 -9
- atomicshop/mitm/mitm_main.py +7 -0
- atomicshop/mitm/recs_files.py +4 -2
- atomicshop/mitm/shared_functions.py +0 -6
- atomicshop/mitm/statistic_analyzer.py +15 -3
- atomicshop/mitm/statistic_analyzer_helper/moving_average_helper.py +74 -46
- atomicshop/print_api.py +1 -2
- atomicshop/system_resource_monitor.py +41 -40
- atomicshop/wrappers/ctyping/win_console.py +39 -0
- atomicshop/wrappers/loggingw/loggingw.py +1 -1
- atomicshop/wrappers/loggingw/reading.py +28 -11
- atomicshop/wrappers/mongodbw/mongodbw.py +50 -31
- atomicshop/wrappers/nodejsw/install_nodejs.py +7 -4
- atomicshop/wrappers/socketw/dns_server.py +8 -5
- atomicshop/wrappers/socketw/sender.py +18 -25
- atomicshop/wrappers/socketw/socket_wrapper.py +9 -5
- {atomicshop-2.16.30.dist-info → atomicshop-2.16.32.dist-info}/METADATA +1 -1
- {atomicshop-2.16.30.dist-info → atomicshop-2.16.32.dist-info}/RECORD +27 -26
- {atomicshop-2.16.30.dist-info → atomicshop-2.16.32.dist-info}/LICENSE.txt +0 -0
- {atomicshop-2.16.30.dist-info → atomicshop-2.16.32.dist-info}/WHEEL +0 -0
- {atomicshop-2.16.30.dist-info → atomicshop-2.16.32.dist-info}/top_level.txt +0 -0
atomicshop/__init__.py
CHANGED
atomicshop/basics/booleans.py
CHANGED
|
@@ -1,47 +1,35 @@
|
|
|
1
|
-
def
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
boolean3: tuple
|
|
1
|
+
def is_only_1_true_in_list(
|
|
2
|
+
booleans_list_of_tuples: list[tuple],
|
|
3
|
+
raise_if_all_false: bool = True
|
|
5
4
|
) -> None:
|
|
6
5
|
"""
|
|
7
|
-
Check if only one boolean can be 'True' from
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
:param
|
|
11
|
-
:
|
|
12
|
-
:return:
|
|
13
|
-
|
|
14
|
-
------------------------------------------------
|
|
15
|
-
|
|
16
|
-
Example:
|
|
17
|
-
check_3_booleans_when_only_1_can_be_true(
|
|
18
|
-
(self.config['section']['default_usage'], 'default_usage'),
|
|
19
|
-
(self.config['section']['create_usage'], 'create_usage'),
|
|
20
|
-
(self.config['section']['custom_usage'], 'custom_usage'))
|
|
6
|
+
Check if only one boolean can be 'True' from a list of booleans
|
|
7
|
+
:param booleans_list_of_tuples: list of tuples, Structure:
|
|
8
|
+
[(value, string name of the setting you want to print to the user to be aware of), ...]
|
|
9
|
+
:param raise_if_all_false: bool, If True, exception will be raised if all booleans are False.
|
|
10
|
+
:return: None
|
|
21
11
|
"""
|
|
22
12
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
f"{boolean2[1]}={boolean2[0]}\n"
|
|
44
|
-
raise ValueError(message)
|
|
13
|
+
# Filter to get all the `True` conditions and their associated names
|
|
14
|
+
true_conditions = [name for value, name in booleans_list_of_tuples if value]
|
|
15
|
+
|
|
16
|
+
# Count the number of True values
|
|
17
|
+
true_count = len(true_conditions)
|
|
18
|
+
|
|
19
|
+
if true_count == 1:
|
|
20
|
+
# Only one value is True, which is acceptable
|
|
21
|
+
# print(f"Only one condition is True: {true_conditions[0]}.")
|
|
22
|
+
pass
|
|
23
|
+
elif true_count > 1:
|
|
24
|
+
# More than one value is True, raise an exception
|
|
25
|
+
raise ValueError(f"Multiple conditions are True: {', '.join(true_conditions)}.")
|
|
26
|
+
elif true_count == 0 and raise_if_all_false:
|
|
27
|
+
# None of the values are True, and the user does not want to ignore this case
|
|
28
|
+
raise ValueError("No conditions are True, and raise_if_all_false is set to True.")
|
|
29
|
+
else:
|
|
30
|
+
# If no True values and no_raise_if_all_false is True, just pass silently
|
|
31
|
+
# print("No conditions are True (but raise_if_all_false is set to False).")
|
|
32
|
+
pass
|
|
45
33
|
|
|
46
34
|
|
|
47
35
|
def convert_string_to_bool(string: str) -> bool:
|
atomicshop/config_init.py
CHANGED
|
@@ -2,7 +2,7 @@ import os
|
|
|
2
2
|
|
|
3
3
|
from .file_io import tomls
|
|
4
4
|
from . import filesystem
|
|
5
|
-
from .
|
|
5
|
+
from . import print_api
|
|
6
6
|
|
|
7
7
|
CONFIG_FILE_NAME = 'config.toml'
|
|
8
8
|
CONFIG: dict = dict()
|
|
@@ -45,7 +45,7 @@ def write_config(
|
|
|
45
45
|
'CONFIG_FILE_NAME'.
|
|
46
46
|
:param print_message: boolean, if True, the function will print the message about the created config file.
|
|
47
47
|
Also, it will wait for the user to press Enter to exit the script.
|
|
48
|
-
If False, the function will not print anything and will not exit
|
|
48
|
+
If False, the function will not print anything and will not exit.
|
|
49
49
|
:return:
|
|
50
50
|
"""
|
|
51
51
|
|
|
@@ -62,7 +62,7 @@ def write_config(
|
|
|
62
62
|
tomls.write_toml_file(config, f'{script_directory}{os.sep}{config_file_name}')
|
|
63
63
|
|
|
64
64
|
if print_message:
|
|
65
|
-
print_api(f"Created config file: {config_file_path}", color="yellow")
|
|
66
|
-
print_api(f"You need to fill it with details.", color="yellow")
|
|
65
|
+
print_api.print_api(f"Created config file: {config_file_path}", color="yellow")
|
|
66
|
+
print_api.print_api(f"You need to fill it with details.", color="yellow")
|
|
67
67
|
input("Press Enter to exit.")
|
|
68
68
|
exit()
|
atomicshop/file_io/csvs.py
CHANGED
|
@@ -2,11 +2,10 @@ import csv
|
|
|
2
2
|
import io
|
|
3
3
|
from typing import Tuple, List
|
|
4
4
|
|
|
5
|
-
from .file_io import read_file_decorator
|
|
6
5
|
from . import file_io
|
|
7
6
|
|
|
8
7
|
|
|
9
|
-
@read_file_decorator
|
|
8
|
+
@file_io.read_file_decorator
|
|
10
9
|
def read_csv_to_list_of_dicts_by_header(
|
|
11
10
|
file_path: str,
|
|
12
11
|
file_mode: str = 'r',
|
|
@@ -53,7 +52,7 @@ def read_csv_to_list_of_dicts_by_header(
|
|
|
53
52
|
return csv_list, header
|
|
54
53
|
|
|
55
54
|
|
|
56
|
-
@read_file_decorator
|
|
55
|
+
@file_io.read_file_decorator
|
|
57
56
|
def read_csv_to_list_of_lists(
|
|
58
57
|
file_path: str,
|
|
59
58
|
file_mode: str = 'r',
|
atomicshop/file_io/file_io.py
CHANGED
|
@@ -2,7 +2,7 @@ from typing import Union
|
|
|
2
2
|
import functools
|
|
3
3
|
|
|
4
4
|
from .. import print_api
|
|
5
|
-
from ..
|
|
5
|
+
from .. import inspect_wrapper
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
def get_write_file_mode_string_from_overwrite_bool(overwrite: bool) -> str:
|
|
@@ -17,7 +17,8 @@ def write_file_decorator(function_name):
|
|
|
17
17
|
def wrapper_write_file_decorator(*args, **kwargs):
|
|
18
18
|
# Put 'args' into 'kwargs' with appropriate key.
|
|
19
19
|
# args, kwargs = put_args_to_kwargs(function_name, *args, **kwargs)
|
|
20
|
-
args, kwargs = get_target_function_default_args_and_combine_with_current(
|
|
20
|
+
args, kwargs = inspect_wrapper.get_target_function_default_args_and_combine_with_current(
|
|
21
|
+
function_name, *args, **kwargs)
|
|
21
22
|
|
|
22
23
|
print_api.print_api(message=f"Writing file: {kwargs['file_path']}", **kwargs)
|
|
23
24
|
|
|
@@ -51,7 +52,8 @@ def read_file_decorator(function_name):
|
|
|
51
52
|
def wrapper_read_file_decorator(*args, **kwargs):
|
|
52
53
|
# Put 'args' into 'kwargs' with appropriate key.
|
|
53
54
|
# args, kwargs = put_args_to_kwargs(function_name, *args, **kwargs)
|
|
54
|
-
args, kwargs = get_target_function_default_args_and_combine_with_current(
|
|
55
|
+
args, kwargs = inspect_wrapper.get_target_function_default_args_and_combine_with_current(
|
|
56
|
+
function_name, *args, **kwargs)
|
|
55
57
|
|
|
56
58
|
continue_loop: bool = True
|
|
57
59
|
while continue_loop:
|
|
@@ -2,9 +2,9 @@ import os
|
|
|
2
2
|
from datetime import datetime
|
|
3
3
|
import json
|
|
4
4
|
|
|
5
|
-
from ...shared_functions import build_module_names, create_custom_logger
|
|
5
|
+
from ...shared_functions import build_module_names, create_custom_logger
|
|
6
6
|
from ... import message, recs_files
|
|
7
|
-
from .... import filesystem
|
|
7
|
+
from .... import filesystem
|
|
8
8
|
from ....file_io import file_io
|
|
9
9
|
|
|
10
10
|
|
|
@@ -46,21 +46,6 @@ class RecorderParent:
|
|
|
46
46
|
# Build the record path with file name
|
|
47
47
|
self.build_record_path_to_engine()
|
|
48
48
|
|
|
49
|
-
# Define empty 'http_path'.
|
|
50
|
-
http_path: str = str()
|
|
51
|
-
# If 'self.class_client_message.request_raw_decoded.path' will be undefined, exception will raise.
|
|
52
|
-
# This will happen if the message is not HTTP.
|
|
53
|
-
try:
|
|
54
|
-
# Parse the url to components.
|
|
55
|
-
http_path_parsed = urls.url_parser(self.class_client_message.request_raw_decoded.path)
|
|
56
|
-
# Get only directories.
|
|
57
|
-
http_path_directories_string = '-'.join(http_path_parsed['directories'])
|
|
58
|
-
# Add '_' character before 'http_path' to look better on the file name.
|
|
59
|
-
http_path = f'_{http_path_directories_string}'
|
|
60
|
-
# If 'self.class_client_message.request_raw_decoded.path' is not defined, we'll pass the exception.
|
|
61
|
-
except Exception:
|
|
62
|
-
pass
|
|
63
|
-
|
|
64
49
|
# If HTTP Path is not defined, 'http_path' will be empty, and it will not interfere with file name.
|
|
65
50
|
self.record_file_path: str = \
|
|
66
51
|
self.engine_record_path + os.sep + \
|
|
@@ -88,7 +73,6 @@ class RecorderParent:
|
|
|
88
73
|
# Convert the requests and responses to hex.
|
|
89
74
|
self.convert_messages()
|
|
90
75
|
# Get the message in dict / JSON format
|
|
91
|
-
# record_message = get_json(self.class_client_message)
|
|
92
76
|
record_message_dict: dict = dict(self.class_client_message)
|
|
93
77
|
recorded_message_json_string = json.dumps(record_message_dict)
|
|
94
78
|
|
atomicshop/mitm/import_config.py
CHANGED
|
@@ -93,19 +93,20 @@ def check_configurations() -> int:
|
|
|
93
93
|
print_api(message, color='red')
|
|
94
94
|
return 1
|
|
95
95
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
try:
|
|
100
|
-
booleans.check_3_booleans_when_only_1_can_be_true(
|
|
96
|
+
try:
|
|
97
|
+
booleans.is_only_1_true_in_list(
|
|
98
|
+
booleans_list_of_tuples=[
|
|
101
99
|
(config_static.DNSServer.set_default_dns_gateway, '[dns][set_default_dns_gateway]'),
|
|
102
100
|
(config_static.DNSServer.set_default_dns_gateway_to_localhost,
|
|
103
101
|
'[dns][set_default_dns_gateway_to_localhost]'),
|
|
104
102
|
(config_static.DNSServer.set_default_dns_gateway_to_default_interface_ipv4,
|
|
105
|
-
'[dns][set_default_dns_gateway_to_default_interface_ipv4]')
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
103
|
+
'[dns][set_default_dns_gateway_to_default_interface_ipv4]')
|
|
104
|
+
],
|
|
105
|
+
raise_if_all_false=False
|
|
106
|
+
)
|
|
107
|
+
except ValueError as e:
|
|
108
|
+
print_api(str(e), color='red')
|
|
109
|
+
return 1
|
|
109
110
|
|
|
110
111
|
if (config_static.DNSServer.set_default_dns_gateway or
|
|
111
112
|
config_static.DNSServer.set_default_dns_gateway_to_localhost or
|
atomicshop/mitm/mitm_main.py
CHANGED
|
@@ -10,6 +10,7 @@ from ..permissions import permissions
|
|
|
10
10
|
from ..python_functions import get_current_python_version_string, check_python_version_compliance
|
|
11
11
|
from ..wrappers.socketw import socket_wrapper, dns_server, base
|
|
12
12
|
from ..wrappers.loggingw import loggingw
|
|
13
|
+
from ..wrappers.ctyping import win_console
|
|
13
14
|
|
|
14
15
|
from .initialize_engines import ModuleCategory
|
|
15
16
|
from .connection_thread_worker import thread_worker_main
|
|
@@ -28,6 +29,12 @@ EXCEPTIONS_CSV_LOGGER_HEADER: str = 'time,exception'
|
|
|
28
29
|
MITM_ERROR_LOGGER: loggingw.ExceptionCsvLogger = None
|
|
29
30
|
|
|
30
31
|
|
|
32
|
+
try:
|
|
33
|
+
win_console.disable_quick_edit()
|
|
34
|
+
except win_console.NotWindowsConsoleError:
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
|
|
31
38
|
def exit_cleanup():
|
|
32
39
|
if permissions.is_admin():
|
|
33
40
|
is_dns_dynamic, current_dns_gateway = dns.get_default_dns_gateway()
|
atomicshop/mitm/recs_files.py
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import datetime
|
|
2
2
|
import os
|
|
3
3
|
import multiprocessing
|
|
4
|
+
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
from ..archiver import zips
|
|
6
7
|
from .. import filesystem
|
|
8
|
+
from .. wrappers.loggingw import consts
|
|
7
9
|
|
|
8
10
|
|
|
9
|
-
REC_FILE_DATE_TIME_FORMAT: str = "%
|
|
10
|
-
REC_FILE_DATE_FORMAT: str = REC_FILE_DATE_TIME_FORMAT.split('
|
|
11
|
+
REC_FILE_DATE_TIME_FORMAT: str = f'{consts.DEFAULT_ROTATING_SUFFIXES_FROM_WHEN["S"]}_%f'
|
|
12
|
+
REC_FILE_DATE_FORMAT: str = REC_FILE_DATE_TIME_FORMAT.split('_')[0]
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
def recs_archiver(recs_directory: str) -> list:
|
|
@@ -38,9 +38,3 @@ def create_custom_logger():
|
|
|
38
38
|
logger_name = f'{config_static.MainConfig.LOGGER_NAME}.{engine_logger_part}'
|
|
39
39
|
|
|
40
40
|
return loggingw.get_logger_with_level(logger_name)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def get_json(obj):
|
|
44
|
-
""" Convert any nested object to json / dict and values to string as is """
|
|
45
|
-
|
|
46
|
-
return json.dumps(obj, default=dicts.convert_complex_object_to_dict)
|
|
@@ -354,7 +354,8 @@ def deviation_calculator_by_moving_average(
|
|
|
354
354
|
summary: bool = False,
|
|
355
355
|
output_file_path: str = None,
|
|
356
356
|
output_file_type: Literal['json', 'csv'] = 'json',
|
|
357
|
-
convert_sizes_lists_and_ma_data_to_string: bool = False
|
|
357
|
+
convert_sizes_lists_and_ma_data_to_string: bool = False,
|
|
358
|
+
skip_total_count_less_than: int = None
|
|
358
359
|
) -> Union[list, None]:
|
|
359
360
|
"""
|
|
360
361
|
This function is the main function for the moving average calculator.
|
|
@@ -386,6 +387,12 @@ def deviation_calculator_by_moving_average(
|
|
|
386
387
|
:param output_file_type: string, the type of the output file. 'json' or 'csv'.
|
|
387
388
|
:param convert_sizes_lists_and_ma_data_to_string: bool, if True, the 'request_sizes', 'response_sizes' and 'ma_data'
|
|
388
389
|
will be converted to string. This is useful when writing to files, so the view will be more readable.
|
|
390
|
+
:param skip_total_count_less_than: integer, if specified, the deviation calculation will be skipped
|
|
391
|
+
if the total count is less than this number.
|
|
392
|
+
This means that if we have moving average window of 7 days, on the 8th day, the deviation will be calculated
|
|
393
|
+
only if the total count of the checked type
|
|
394
|
+
(request average size, response average, request count, response count)
|
|
395
|
+
is greater or equal to this number.
|
|
389
396
|
-----------------------------
|
|
390
397
|
:return: the deviation list of dicts.
|
|
391
398
|
|
|
@@ -427,7 +434,8 @@ def deviation_calculator_by_moving_average(
|
|
|
427
434
|
by_type,
|
|
428
435
|
moving_average_window_days,
|
|
429
436
|
top_bottom_deviation_percentage,
|
|
430
|
-
get_deviation_for_last_day_only
|
|
437
|
+
get_deviation_for_last_day_only,
|
|
438
|
+
skip_total_count_less_than
|
|
431
439
|
)
|
|
432
440
|
|
|
433
441
|
if deviation_list:
|
|
@@ -484,6 +492,9 @@ def deviation_calculator_by_moving_average_main():
|
|
|
484
492
|
parser.add_argument(
|
|
485
493
|
'-p', '--percentage', type=float, required=True,
|
|
486
494
|
help='Percentage of deviation from moving average. Example: 0.1 for 10%%.')
|
|
495
|
+
parser.add_argument(
|
|
496
|
+
'-slt', '--skip_total_count_less_than', type=int, required=False,
|
|
497
|
+
help='An integer to skip the deviation calculation if the total count is less than this number.')
|
|
487
498
|
|
|
488
499
|
return parser.parse_args()
|
|
489
500
|
|
|
@@ -517,7 +528,8 @@ def deviation_calculator_by_moving_average_main():
|
|
|
517
528
|
summary=summary,
|
|
518
529
|
output_file_path=args.output_file,
|
|
519
530
|
output_file_type=args.output_type,
|
|
520
|
-
convert_sizes_lists_and_ma_data_to_string=convert_sizes_lists_and_ma_data_to_string
|
|
531
|
+
convert_sizes_lists_and_ma_data_to_string=convert_sizes_lists_and_ma_data_to_string,
|
|
532
|
+
skip_total_count_less_than=args.skip_total_count_less_than
|
|
521
533
|
)
|
|
522
534
|
|
|
523
535
|
return 0
|
|
@@ -14,6 +14,7 @@ def calculate_moving_average(
|
|
|
14
14
|
moving_average_window_days,
|
|
15
15
|
top_bottom_deviation_percentage: float,
|
|
16
16
|
get_deviation_for_last_day_only: bool = False,
|
|
17
|
+
skip_total_count_less_than: int = None,
|
|
17
18
|
print_kwargs: dict = None
|
|
18
19
|
) -> list:
|
|
19
20
|
"""
|
|
@@ -37,6 +38,7 @@ def calculate_moving_average(
|
|
|
37
38
|
Files 01 to 05 will be used for moving average and the file 06 for deviation.
|
|
38
39
|
Meaning the average calculated for 2021-01-06 will be compared to the values moving average of 2021-01-01
|
|
39
40
|
to 2021-01-05.
|
|
41
|
+
:param skip_total_count_less_than: integer, if the total count is less than this number, skip the deviation.
|
|
40
42
|
:param print_kwargs: dict, the print_api arguments.
|
|
41
43
|
"""
|
|
42
44
|
|
|
@@ -55,7 +57,7 @@ def calculate_moving_average(
|
|
|
55
57
|
statistics_content: dict = {}
|
|
56
58
|
# Read each file to its day.
|
|
57
59
|
for log_atomic_path in logs_paths:
|
|
58
|
-
date_string = log_atomic_path.datetime_string
|
|
60
|
+
date_string: str = log_atomic_path.datetime_string
|
|
59
61
|
statistics_content[date_string] = {}
|
|
60
62
|
|
|
61
63
|
statistics_content[date_string]['file'] = log_atomic_path
|
|
@@ -65,11 +67,11 @@ def calculate_moving_average(
|
|
|
65
67
|
statistics_content[date_string]['content'] = log_file_content
|
|
66
68
|
statistics_content[date_string]['header'] = log_file_header
|
|
67
69
|
|
|
68
|
-
statistics_content[date_string]['
|
|
70
|
+
statistics_content[date_string]['content_no_useless'] = get_content_without_useless(log_file_content)
|
|
69
71
|
|
|
70
72
|
# Get the data dictionary from the statistics content.
|
|
71
73
|
statistics_content[date_string]['statistics_daily'] = compute_statistics_from_content(
|
|
72
|
-
statistics_content[date_string]['
|
|
74
|
+
statistics_content[date_string]['content_no_useless'], by_type)
|
|
73
75
|
|
|
74
76
|
moving_average_dict: dict = compute_moving_averages_from_average_statistics(
|
|
75
77
|
statistics_content,
|
|
@@ -85,12 +87,12 @@ def calculate_moving_average(
|
|
|
85
87
|
|
|
86
88
|
# Find deviation from the moving average to the bottom or top by specified percentage.
|
|
87
89
|
deviation_list: list = find_deviation_from_moving_average(
|
|
88
|
-
statistics_content, top_bottom_deviation_percentage)
|
|
90
|
+
statistics_content, top_bottom_deviation_percentage, skip_total_count_less_than)
|
|
89
91
|
|
|
90
92
|
return deviation_list
|
|
91
93
|
|
|
92
94
|
|
|
93
|
-
def
|
|
95
|
+
def get_content_without_useless(content: list) -> list:
|
|
94
96
|
"""
|
|
95
97
|
This function gets the 'statistics.csv' file content without errors from the 'content' list.
|
|
96
98
|
|
|
@@ -101,7 +103,7 @@ def get_content_without_errors(content: list) -> list:
|
|
|
101
103
|
traffic_statistics_without_errors: list = []
|
|
102
104
|
for line in content:
|
|
103
105
|
# Skip empty lines, headers and errors.
|
|
104
|
-
if line['host'] == 'host' or line['
|
|
106
|
+
if line['host'] == 'host' or (line['request_size_bytes'] == '' and line['response_size_bytes'] == ''):
|
|
105
107
|
continue
|
|
106
108
|
|
|
107
109
|
traffic_statistics_without_errors.append(line)
|
|
@@ -152,16 +154,13 @@ def get_data_dict_from_statistics_content(
|
|
|
152
154
|
try:
|
|
153
155
|
request_size_bytes = line['request_size_bytes']
|
|
154
156
|
response_size_bytes = line['response_size_bytes']
|
|
155
|
-
if request_size_bytes
|
|
156
|
-
|
|
157
|
-
if response_size_bytes
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
hosts_requests_responses[type_to_check]['request_sizes'].append(int(request_size_bytes))
|
|
161
|
-
hosts_requests_responses[type_to_check]['response_sizes'].append(int(response_size_bytes))
|
|
162
|
-
except ValueError:
|
|
157
|
+
if request_size_bytes != '':
|
|
158
|
+
hosts_requests_responses[type_to_check]['request_sizes'].append(int(request_size_bytes))
|
|
159
|
+
if response_size_bytes != '':
|
|
160
|
+
hosts_requests_responses[type_to_check]['response_sizes'].append(int(response_size_bytes))
|
|
161
|
+
except ValueError as e:
|
|
163
162
|
print_api(line, color='yellow')
|
|
164
|
-
raise
|
|
163
|
+
raise e
|
|
165
164
|
|
|
166
165
|
return hosts_requests_responses
|
|
167
166
|
|
|
@@ -175,13 +174,15 @@ def compute_statistics_from_data_dict(data_dict: dict):
|
|
|
175
174
|
"""
|
|
176
175
|
|
|
177
176
|
for host, host_dict in data_dict.items():
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
177
|
+
count_requests = len(host_dict['request_sizes'])
|
|
178
|
+
count_responses = len(host_dict['response_sizes'])
|
|
179
|
+
avg_request_size = statistics.mean(host_dict['request_sizes']) if count_requests > 0 else 0
|
|
180
|
+
median_request_size = statistics.median(host_dict['request_sizes']) if count_requests > 0 else 0
|
|
181
|
+
avg_response_size = statistics.mean(host_dict['response_sizes']) if count_responses > 0 else 0
|
|
182
|
+
median_response_size = statistics.median(host_dict['response_sizes']) if count_responses > 0 else 0
|
|
183
|
+
|
|
184
|
+
data_dict[host]['count_requests'] = count_requests
|
|
185
|
+
data_dict[host]['count_responses'] = count_responses
|
|
185
186
|
data_dict[host]['avg_request_size'] = avg_request_size
|
|
186
187
|
data_dict[host]['median_request_size'] = median_request_size
|
|
187
188
|
data_dict[host]['avg_response_size'] = avg_response_size
|
|
@@ -251,14 +252,16 @@ def compute_average_for_current_day_from_past_x_days(
|
|
|
251
252
|
for host, host_dict in statistics_daily.items():
|
|
252
253
|
if host not in moving_average:
|
|
253
254
|
moving_average[host] = {
|
|
254
|
-
'
|
|
255
|
+
'all_request_counts': [],
|
|
256
|
+
'all_response_counts': [],
|
|
255
257
|
'avg_request_sizes': [],
|
|
256
258
|
'avg_response_sizes': [],
|
|
257
259
|
'median_request_sizes': [],
|
|
258
260
|
'median_response_sizes': []
|
|
259
261
|
}
|
|
260
262
|
|
|
261
|
-
moving_average[host]['
|
|
263
|
+
moving_average[host]['all_request_counts'].append(int(host_dict['count_requests']))
|
|
264
|
+
moving_average[host]['all_response_counts'].append(int(host_dict['count_responses']))
|
|
262
265
|
moving_average[host]['avg_request_sizes'].append(float(host_dict['avg_request_size']))
|
|
263
266
|
moving_average[host]['avg_response_sizes'].append(float(host_dict['avg_response_size']))
|
|
264
267
|
moving_average[host]['median_request_sizes'].append(float(host_dict['median_request_size']))
|
|
@@ -267,21 +270,26 @@ def compute_average_for_current_day_from_past_x_days(
|
|
|
267
270
|
# Compute the moving average.
|
|
268
271
|
moving_average_results: dict = {}
|
|
269
272
|
for host, host_dict in moving_average.items():
|
|
270
|
-
|
|
273
|
+
ma_request_count = statistics.mean(host_dict['all_request_counts'])
|
|
274
|
+
ma_response_count = statistics.mean(host_dict['all_response_counts'])
|
|
271
275
|
ma_request_size = statistics.mean(host_dict['avg_request_sizes'])
|
|
272
276
|
ma_response_size = statistics.mean(host_dict['avg_response_sizes'])
|
|
273
|
-
|
|
277
|
+
mm_request_count = statistics.median(host_dict['all_request_counts'])
|
|
278
|
+
mm_response_count = statistics.median(host_dict['all_response_counts'])
|
|
274
279
|
mm_request_size = statistics.median(host_dict['median_request_sizes'])
|
|
275
280
|
mm_response_size = statistics.median(host_dict['median_response_sizes'])
|
|
276
281
|
|
|
277
282
|
moving_average_results[host] = {
|
|
278
|
-
'
|
|
283
|
+
'ma_request_count': ma_request_count,
|
|
284
|
+
'ma_response_count': ma_response_count,
|
|
279
285
|
'ma_request_size': ma_request_size,
|
|
280
286
|
'ma_response_size': ma_response_size,
|
|
281
|
-
'
|
|
287
|
+
'mm_request_count': mm_request_count,
|
|
288
|
+
'mm_response_count': mm_response_count,
|
|
282
289
|
'mm_request_size': mm_request_size,
|
|
283
290
|
'mm_response_size': mm_response_size,
|
|
284
|
-
'
|
|
291
|
+
'all_request_counts': host_dict['all_request_counts'],
|
|
292
|
+
'all_response_counts': host_dict['all_response_counts'],
|
|
285
293
|
'avg_request_sizes': host_dict['avg_request_sizes'],
|
|
286
294
|
'avg_response_sizes': host_dict['avg_response_sizes'],
|
|
287
295
|
'median_request_sizes': host_dict['median_request_sizes'],
|
|
@@ -293,7 +301,8 @@ def compute_average_for_current_day_from_past_x_days(
|
|
|
293
301
|
|
|
294
302
|
def find_deviation_from_moving_average(
|
|
295
303
|
statistics_content: dict,
|
|
296
|
-
top_bottom_deviation_percentage: float
|
|
304
|
+
top_bottom_deviation_percentage: float,
|
|
305
|
+
skip_total_count_less_than: int = None
|
|
297
306
|
) -> list:
|
|
298
307
|
"""
|
|
299
308
|
This function finds the deviation from the moving average to the bottom or top by specified percentage.
|
|
@@ -301,12 +310,13 @@ def find_deviation_from_moving_average(
|
|
|
301
310
|
:param statistics_content: dict, the statistics content dictionary.
|
|
302
311
|
:param top_bottom_deviation_percentage: float, the percentage of deviation from the moving average to the top or
|
|
303
312
|
bottom.
|
|
313
|
+
:param skip_total_count_less_than: integer, if the total count is less than this number, skip the deviation.
|
|
304
314
|
:return: list, the deviation list.
|
|
305
315
|
"""
|
|
306
316
|
|
|
307
317
|
def _check_deviation(
|
|
308
|
-
|
|
309
|
-
|
|
318
|
+
check: Literal['count', 'avg'],
|
|
319
|
+
traffic_direction: Literal['request', 'response'],
|
|
310
320
|
day_statistics_content_dict: dict,
|
|
311
321
|
moving_averages_dict: dict
|
|
312
322
|
):
|
|
@@ -316,6 +326,19 @@ def find_deviation_from_moving_average(
|
|
|
316
326
|
|
|
317
327
|
nonlocal message
|
|
318
328
|
|
|
329
|
+
if check == 'count':
|
|
330
|
+
check_type = f'{check}_{traffic_direction}s'
|
|
331
|
+
ma_check_type = f'ma_{traffic_direction}_{check}'
|
|
332
|
+
median_type_string = check_type
|
|
333
|
+
moving_median_type_string = f'mm_{traffic_direction}_{check}'
|
|
334
|
+
elif check == 'avg':
|
|
335
|
+
check_type = f'{check}_{traffic_direction}_size'
|
|
336
|
+
ma_check_type = f'ma_{traffic_direction}_size'
|
|
337
|
+
median_type_string = f'median_{traffic_direction}_size'
|
|
338
|
+
moving_median_type_string = f'mm_{traffic_direction}_size'
|
|
339
|
+
else:
|
|
340
|
+
raise ValueError(f'Invalid check: {check}')
|
|
341
|
+
|
|
319
342
|
host_moving_average_by_type = moving_averages_dict[host][ma_check_type]
|
|
320
343
|
check_type_moving_by_percent = (
|
|
321
344
|
host_moving_average_by_type * top_bottom_deviation_percentage)
|
|
@@ -338,29 +361,32 @@ def find_deviation_from_moving_average(
|
|
|
338
361
|
if deviation_type:
|
|
339
362
|
message = f'[{check_type}] is [{deviation_type}] the moving average.'
|
|
340
363
|
|
|
341
|
-
# Get the right moving median.
|
|
342
|
-
if check_type == 'count':
|
|
343
|
-
median_type_string: str = 'count'
|
|
344
|
-
moving_median_type_string: str = 'mm_count'
|
|
345
|
-
else:
|
|
346
|
-
median_type_string: str = check_type.replace('avg', 'median')
|
|
347
|
-
moving_median_type_string: str = check_type.replace('avg', 'mm')
|
|
348
|
-
|
|
349
364
|
# The median and the total count are None for the count, Since they are the count.
|
|
350
|
-
if
|
|
365
|
+
if 'count' in check_type:
|
|
351
366
|
total_entries_averaged = None
|
|
352
367
|
median_size = None
|
|
353
368
|
else:
|
|
354
|
-
total_entries_averaged = day_statistics_content_dict['
|
|
369
|
+
total_entries_averaged = day_statistics_content_dict[f'count_{traffic_direction}s']
|
|
355
370
|
median_size = day_statistics_content_dict[median_type_string]
|
|
356
371
|
|
|
372
|
+
value = day_statistics_content_dict[check_type]
|
|
373
|
+
|
|
374
|
+
# If the total count is less than the specified number, skip the deviation.
|
|
375
|
+
if skip_total_count_less_than:
|
|
376
|
+
if total_entries_averaged:
|
|
377
|
+
if total_entries_averaged < skip_total_count_less_than:
|
|
378
|
+
return
|
|
379
|
+
else:
|
|
380
|
+
if value < skip_total_count_less_than:
|
|
381
|
+
return
|
|
382
|
+
|
|
357
383
|
moving_median_size = moving_averages_dict[host][moving_median_type_string]
|
|
358
384
|
|
|
359
385
|
deviation_list.append({
|
|
360
386
|
'day': day,
|
|
361
387
|
'host': host,
|
|
362
388
|
'message': message,
|
|
363
|
-
'value':
|
|
389
|
+
'value': value,
|
|
364
390
|
'ma_value': host_moving_average_by_type,
|
|
365
391
|
'check_type': check_type,
|
|
366
392
|
'percentage': top_bottom_deviation_percentage,
|
|
@@ -411,10 +437,12 @@ def find_deviation_from_moving_average(
|
|
|
411
437
|
continue
|
|
412
438
|
|
|
413
439
|
_check_deviation(
|
|
414
|
-
'count', '
|
|
440
|
+
'count', 'request', host_dict, previous_day_moving_average_dict)
|
|
441
|
+
_check_deviation(
|
|
442
|
+
'count', 'response', host_dict, previous_day_moving_average_dict)
|
|
415
443
|
_check_deviation(
|
|
416
|
-
'
|
|
444
|
+
'avg', 'request', host_dict, previous_day_moving_average_dict)
|
|
417
445
|
_check_deviation(
|
|
418
|
-
'
|
|
446
|
+
'avg', 'response', host_dict, previous_day_moving_average_dict)
|
|
419
447
|
|
|
420
448
|
return deviation_list
|
atomicshop/print_api.py
CHANGED
|
@@ -2,7 +2,6 @@ import sys
|
|
|
2
2
|
import logging
|
|
3
3
|
|
|
4
4
|
from .basics import ansi_escape_codes
|
|
5
|
-
from .wrappers.loggingw import loggingw
|
|
6
5
|
from .basics import tracebacks
|
|
7
6
|
|
|
8
7
|
|
|
@@ -74,6 +73,7 @@ def print_api(
|
|
|
74
73
|
|
|
75
74
|
# Inner functions already get all the local variables of the main function.
|
|
76
75
|
def print_or_logger():
|
|
76
|
+
from .wrappers.loggingw import loggingw
|
|
77
77
|
nonlocal message
|
|
78
78
|
nonlocal color
|
|
79
79
|
nonlocal traceback_string
|
|
@@ -128,7 +128,6 @@ def print_api(
|
|
|
128
128
|
if print_end == '\n':
|
|
129
129
|
if stdcolor and color is not None:
|
|
130
130
|
# Use logger to output message.
|
|
131
|
-
# with loggingw.temporary_change_logger_stream_handler_color(logger, color=color):
|
|
132
131
|
with loggingw.temporary_change_logger_stream_handler_emit_color(logger, color):
|
|
133
132
|
getattr(logger, logger_method)(message)
|
|
134
133
|
else:
|