atomicshop 2.14.3__py3-none-any.whl → 2.14.5__py3-none-any.whl

atomicshop/__init__.py

@@ -1,4 +1,4 @@
 """Atomic Basic functions and classes to make developer life easier"""
 
 __author__ = "Den Kras"
-__version__ = '2.14.3'
+__version__ = '2.14.5'

atomicshop/datetimes.py

@@ -47,7 +47,7 @@ class MonthToNumber:
         'דצמבר': '12'}
 
 
-def get_datetime_from_complex_string_by_pattern(complex_string: str, date_pattern: str):
+def get_datetime_from_complex_string_by_pattern(complex_string: str, date_pattern: str) -> tuple[datetime, str, float]:
     """
     Function will get datetime object from a complex string by pattern.
 
@@ -65,7 +65,8 @@ def get_datetime_from_complex_string_by_pattern(complex_string: str, date_patter
     if date_str:
         # Convert the date string to a datetime object based on the given pattern
         date_obj = datetime.datetime.strptime(date_str.group(), date_pattern)
-        return date_obj
+        date_timestamp = date_obj.timestamp()
+        return date_obj, date_str.group(), date_timestamp
     else:
         raise ValueError("No valid date found in the string")
 

atomicshop/etws/providers.py

@@ -1,5 +1,21 @@
+from typing import Literal
+
 from ..wrappers.ctyping.etw_winapi import etw_functions
 
 
-def get_providers():
-    return etw_functions.get_all_providers()
+def get_providers(key_as: Literal['name', 'guid'] = 'name'):
+    return etw_functions.get_all_providers(key_as=key_as)
+
+
+def get_provider_guid_by_name(provider_name):
+    providers = get_providers(key_as='name')
+
+    try:
+        provider_guid = providers[provider_name]
+    except KeyError:
+        provider_guid = None
+
+    if not provider_guid:
+        raise ValueError(f"Provider '{provider_name}' not found")
+
+    return provider_guid

atomicshop/etws/sessions.py

@@ -1,7 +1,7 @@
 from ..wrappers.ctyping.etw_winapi import etw_functions
 
 
-def stop_and_delete(session_name) -> tuple[bool, int]:
+def stop_and_delete(session_name: str) -> tuple[bool, int]:
     """
     Stop and delete ETW session.
 

atomicshop/etws/trace.py

@@ -1,7 +1,6 @@
 import queue
 import sys
 import time
-from typing import Literal
 
 # Import FireEye Event Tracing library.
 import etw

atomicshop/file_io/csvs.py

@@ -36,7 +36,7 @@ def read_csv_to_list_of_dicts_by_header(
             All the lines of the CSV file will be considered as content.
     :param file_object: file object of the 'open()' function in the decorator. Decorator executes the 'with open()'
         statement and passes to this function. That's why the default is 'None', since we get it from the decorator.
-    :return: list.
+    :return: tuple(list of entries, header(list of cell names)).
     """
 
     # The header fields will be separated to list of "csv_reader.fieldnames".

atomicshop/mitm/initialize_engines.py

@@ -83,8 +83,14 @@ class ModuleCategory:
 
         # Initiating logger for each engine by its name
         # initiate_logger(current_module.engine_name, log_file_extension)
-        loggingw.get_logger_with_stream_handler_and_timedfilehandler(
-            logger_name=self.engine_name, directory_path=logs_path, disable_duplicate_ms=True)
+        loggingw.get_complex_logger(
+            logger_name=self.engine_name,
+            directory_path=logs_path,
+            add_stream=True,
+            add_timedfile=True,
+            formatter_streamhandler='DEFAULT',
+            formatter_filehandler='DEFAULT'
+        )
 
 
 # Assigning external class object by message domain received from client. If the domain is not in the list,

atomicshop/mitm/initialize_mitm_server.py

@@ -41,8 +41,15 @@ def initialize_mitm_server(config_static):
             config['certificates']['sni_server_certificate_from_server_socket_download_directory'])
 
     # Create a logger that will log messages to file, Initiate System logger.
-    system_logger = loggingw.get_logger_with_stream_handler_and_timedfilehandler(
-        "system", config['log']['logs_path'], disable_duplicate_ms=True)
+    logger_name = "system"
+    system_logger = loggingw.get_complex_logger(
+        logger_name=logger_name,
+        file_path=f'{config['log']['logs_path']}{os.sep}{logger_name}.txt',
+        add_stream=True,
+        add_timedfile=True,
+        formatter_streamhandler='DEFAULT',
+        formatter_filehandler='DEFAULT'
+    )
 
     # Writing first log.
     system_logger.info("======================================")
@@ -175,14 +182,24 @@ def initialize_mitm_server(config_static):
     config_static.CONFIG_EXTENDED['certificates']['domains_all_times'] = list(domains_engine_list_full)
 
     # Creating Statistics logger.
-    statistics_logger = loggingw.get_logger_with_stream_handler_and_timedfilehandler(
-        logger_name="statistics", directory_path=config['log']['logs_path'],
-        file_extension=config_static.CSV_EXTENSION, formatter_message_only=True, header=STATISTICS_HEADER
+    statistics_logger = loggingw.get_complex_logger(
+        logger_name="statistics",
+        directory_path=config['log']['logs_path'],
+        add_timedfile=True,
+        formatter_filehandler='MESSAGE',
+        file_type='csv',
+        header=STATISTICS_HEADER
     )
 
     network_logger_name = "network"
-    network_logger = loggingw.get_logger_with_stream_handler_and_timedfilehandler(
-        logger_name=network_logger_name, directory_path=config['log']['logs_path'], disable_duplicate_ms=True)
+    network_logger = loggingw.get_complex_logger(
+        logger_name=network_logger_name,
+        directory_path=config['log']['logs_path'],
+        add_stream=True,
+        add_timedfile=True,
+        formatter_streamhandler='DEFAULT',
+        formatter_filehandler='DEFAULT'
+    )
     system_logger.info(f"Loaded network logger: {network_logger}")
 
     # Initiate Listener logger, which is a child of network logger, so he uses the same settings and handlers

atomicshop/mitm/statistic_analyzer.py

@@ -1,8 +1,12 @@
+import os
 import datetime
+import statistics
+import json
+from typing import Literal
 
 from .. import filesystem, domains, datetimes, urls
 from ..basics import dicts
-from ..file_io import tomls, xlsxs
+from ..file_io import tomls, xlsxs, csvs, jsons
 from ..wrappers.loggingw import reading
 from ..print_api import print_api
 
@@ -154,10 +158,10 @@ def analyze(main_file_path: str):
     summary_path: str = filesystem.check_absolute_path___add_full(config['report_file_path'], script_directory)
 
     # Get the content from statistics files.
-    statistics_content: list = reading.get_logs(
+    statistics_content: list = reading.get_all_log_files_into_list(
         config['statistic_files_path'],
         file_name_pattern='statistics*.csv',
-        log_type='csv',
+        log_type='csv'
     )
 
     # Initialize loop.
@@ -465,3 +469,372 @@ def analyze(main_file_path: str):
         xlsxs.write_xlsx(combined_sorted_stats, file_path=summary_path)
 
     return
+
+
+# ======================================================================================================================
+
+
+def calculate_moving_average(
+        file_path: str,
+        moving_average_window_days,
+        top_bottom_deviation_percentage: float,
+        print_kwargs: dict = None
+):
+    """
+    This function calculates the moving average of the daily statistics.
+
+    :param file_path: string, the path to the 'statistics.csv' file.
+    :param moving_average_window_days: integer, the window size for the moving average.
+    :param top_bottom_deviation_percentage: float, the percentage of deviation from the moving average to the top or
+        bottom.
+    :param print_kwargs: dict, the print_api arguments.
+    """
+
+    date_pattern: str = '%Y_%m_%d'
+
+    # Get all the file paths and their midnight rotations.
+    logs_paths: list = reading.get_logs_paths(
+        log_file_path=file_path,
+        date_pattern=date_pattern
+    )
+
+    statistics_content: dict = {}
+    # Read each file to its day.
+    for log_path_dict in logs_paths:
+        date_string = log_path_dict['date_string']
+        statistics_content[date_string] = {}
+
+        statistics_content[date_string]['file'] = log_path_dict
+
+        log_file_content, log_file_header = (
+            csvs.read_csv_to_list_of_dicts_by_header(log_path_dict['file_path'], **(print_kwargs or {})))
+        statistics_content[date_string]['content'] = log_file_content
+        statistics_content[date_string]['header'] = log_file_header
+
+        statistics_content[date_string]['content_no_errors'] = get_content_without_errors(log_file_content)
+
+        # Get the data dictionary from the statistics content.
+        statistics_content[date_string]['statistics_daily'] = compute_statistics_from_content(
+            statistics_content[date_string]['content_no_errors']
+        )
+
+    moving_average_dict: dict = compute_moving_averages_from_average_statistics(
+        statistics_content,
+        moving_average_window_days
+    )
+
+    # Add the moving average to the statistics content.
+    for day, day_dict in statistics_content.items():
+        try:
+            day_dict['moving_average'] = moving_average_dict[day]
+        except KeyError:
+            day_dict['moving_average'] = {}
+
+    # Find deviation from the moving average to the bottom or top by specified percentage.
+    deviation_list: list = find_deviation_from_moving_average(
+        statistics_content, top_bottom_deviation_percentage)
+
+    return deviation_list
+
+
+def get_content_without_errors(content: list) -> list:
+    """
+    This function gets the 'statistics.csv' file content without errors from the 'content' list.
+
+    :param content: list, the content list.
+    :return: list, the content without errors.
+    """
+
+    traffic_statistics_without_errors: list = []
+    for line in content:
+        # Skip empty lines, headers and errors.
+        if line['host'] == 'host' or line['command'] == '':
+            continue
+
+        traffic_statistics_without_errors.append(line)
+
+    return traffic_statistics_without_errors
+
+
+def get_data_dict_from_statistics_content(content: list) -> dict:
+    """
+    This function gets the data dictionary from the 'statistics.csv' file content.
+
+    :param content: list, the content list.
+    :return: dict, the data dictionary.
+    """
+
+    hosts_requests_responses: dict = {}
+    for line in content:
+        # If subdomain is not in the dictionary, add it.
+        if line['host'] not in hosts_requests_responses:
+            hosts_requests_responses[line['host']] = {
+                'request_sizes': [],
+                'response_sizes': []
+            }
+
+        # Append the sizes.
+        try:
+            hosts_requests_responses[line['host']]['request_sizes'].append(int(line['request_size_bytes']))
+            hosts_requests_responses[line['host']]['response_sizes'].append(
+                int(line['response_size_bytes']))
+        except ValueError:
+            print_api(line, color='yellow')
+            raise
+
+    return hosts_requests_responses
+
+
+def compute_statistics_from_data_dict(data_dict: dict):
+    """
+    This function computes the statistics from the data dictionary.
+
+    :param data_dict: dict, the data dictionary.
+    :return: dict, the statistics dictionary.
+    """
+
+    for host, host_dict in data_dict.items():
+        count = len(host_dict['request_sizes'])
+        avg_request_size = statistics.mean(host_dict['request_sizes']) if count > 0 else 0
+        median_request_size = statistics.median(host_dict['request_sizes']) if count > 0 else 0
+        avg_response_size = statistics.mean(host_dict['response_sizes']) if count > 0 else 0
+        median_response_size = statistics.median(host_dict['response_sizes']) if count > 0 else 0
+
+        data_dict[host]['count'] = count
+        data_dict[host]['avg_request_size'] = avg_request_size
+        data_dict[host]['median_request_size'] = median_request_size
+        data_dict[host]['avg_response_size'] = avg_response_size
+        data_dict[host]['median_response_size'] = median_response_size
+
+
+def compute_statistics_from_content(content: list):
+    """
+    This function computes the statistics from the 'statistics.csv' file content.
+
+    :param content: list, the content list.
+    :return: dict, the statistics dictionary.
+    """
+
+    hosts_requests_responses: dict = get_data_dict_from_statistics_content(content)
+    compute_statistics_from_data_dict(hosts_requests_responses)
+
+    return hosts_requests_responses
+
+
+def compute_moving_averages_from_average_statistics(
+        average_statistics_dict: dict,
+        moving_average_window_days: int
+):
+    """
+    This function computes the moving averages from the average statistics dictionary.
+
+    :param average_statistics_dict: dict, the average statistics dictionary.
+    :param moving_average_window_days: integer, the window size for the moving average.
+    :return: dict, the moving averages dictionary.
+    """
+
+    moving_average: dict = {}
+    for day_index, (day, day_dict) in enumerate(average_statistics_dict.items()):
+        current_day = day_index + 1
+        if current_day < moving_average_window_days:
+            continue
+
+        # Create list of the previous 'moving_average_window_days' days.
+        previous_days_content_list = (
+            list(average_statistics_dict.values()))[current_day-moving_average_window_days:current_day]
+
+        # Compute the moving averages.
+        moving_average[day] = compute_average_for_current_day_from_past_x_days(previous_days_content_list)
+
+    return moving_average
+
+
+def compute_average_for_current_day_from_past_x_days(previous_days_content_list: list) -> dict:
+    """
+    This function computes the average for the current day from the past x days.
+
+    :param previous_days_content_list: list, the list of the previous days content.
+    :return: dict, the average dictionary.
+    """
+
+    moving_average: dict = {}
+    for entry in previous_days_content_list:
+        statistics_daily = entry['statistics_daily']
+        for host, host_dict in statistics_daily.items():
+            if host not in moving_average:
+                moving_average[host] = {
+                    'counts': [],
+                    'avg_request_sizes': [],
+                    'avg_response_sizes': [],
+                }
+
+            moving_average[host]['counts'].append(int(host_dict['count']))
+            moving_average[host]['avg_request_sizes'].append(float(host_dict['avg_request_size']))
+            moving_average[host]['avg_response_sizes'].append(float(host_dict['avg_response_size']))
+
+    # Compute the moving average.
+    moving_average_results: dict = {}
+    for host, host_dict in moving_average.items():
+        ma_count = statistics.mean(host_dict['counts'])
+        ma_request_size = statistics.mean(host_dict['avg_request_sizes'])
+        ma_response_size = statistics.mean(host_dict['avg_response_sizes'])
+
+        moving_average_results[host] = {
+            'ma_count': ma_count,
+            'ma_request_size': ma_request_size,
+            'ma_response_size': ma_response_size,
+            'counts': host_dict['counts'],
+            'avg_request_sizes': host_dict['avg_request_sizes'],
+            'avg_response_sizes': host_dict['avg_response_sizes']
+        }
+
+    return moving_average_results
+
+
+def find_deviation_from_moving_average(
+        statistics_content: dict,
+        top_bottom_deviation_percentage: float
+) -> list:
+    """
+    This function finds the deviation from the moving average to the bottom or top by specified percentage.
+
+    :param statistics_content: dict, the statistics content dictionary.
+    :param top_bottom_deviation_percentage: float, the percentage of deviation from the moving average to the top or
+        bottom.
+    :return: list, the deviation list.
+    """
+
+    def _check_deviation(
+            check_type: Literal['count', 'avg_request_size', 'avg_response_size'],
+            ma_check_type: Literal['ma_count', 'ma_request_size', 'ma_response_size'],
+            day_statistics_content_dict: dict,
+            moving_averages_dict: dict
+    ):
+        """
+        This function checks the deviation for the host.
+        """
+
+        nonlocal message
+
+        host_moving_average_by_type = moving_averages_dict[host][ma_check_type]
+        check_type_moving_by_percent = (
+                host_moving_average_by_type * top_bottom_deviation_percentage)
+        check_type_moving_above = host_moving_average_by_type + check_type_moving_by_percent
+        check_type_moving_below = host_moving_average_by_type - check_type_moving_by_percent
+
+        deviation_type = None
+        if day_statistics_content_dict[check_type] > check_type_moving_above:
+            deviation_type = 'above'
+        elif day_statistics_content_dict[check_type] < check_type_moving_below:
+            deviation_type = 'below'
+
+        if deviation_type:
+            message = f'[{check_type}] is [{deviation_type}] the moving average.'
+            deviation_list.append({
+                'day': day,
+                'host': host,
+                'message': message,
+                'value': day_statistics_content_dict[check_type],
+                'ma_value': host_moving_average_by_type,
+                'check_type': check_type,
+                'percentage': top_bottom_deviation_percentage,
+                'ma_value_checked': check_type_moving_above,
+                'deviation_type': deviation_type,
+                'data': day_statistics_content_dict,
+                'ma_data': moving_averages_dict[host]
+            })
+
+    deviation_list: list = []
+    for day_index, (day, day_dict) in enumerate(statistics_content.items()):
+        # If it's the first day, there is no previous day moving average.
+        if day_index == 0:
+            previous_day_moving_average_dict = {}
+        else:
+            previous_day_moving_average_dict = list(statistics_content.values())[day_index-1].get('moving_average', {})
+
+        # If there is no moving average for previous day continue to the next day.
+        if not previous_day_moving_average_dict:
+            continue
+
+        for host, host_dict in day_dict['statistics_daily'].items():
+            # If the host is not in the moving averages, then this is clear deviation.
+            # It means that in the current day, there were no requests for this host.
+            if host not in previous_day_moving_average_dict:
+                message = f'Host not in the moving averages: {host}'
+                deviation_list.append({
+                    'day': day,
+                    'host': host,
+                    'data': host_dict,
+                    'message': message,
+                    'type': 'clear'
+                })
+                continue
+
+            _check_deviation(
+                'count', 'ma_count', host_dict, previous_day_moving_average_dict)
+            _check_deviation(
+                'avg_request_size', 'ma_request_size', host_dict, previous_day_moving_average_dict)
+            _check_deviation(
+                'avg_response_size', 'ma_response_size', host_dict, previous_day_moving_average_dict)
+
+    return deviation_list
+
+
+def moving_average_calculator_main(
+        statistics_file_path: str,
+        output_directory: str,
+        moving_average_window_days: int,
+        top_bottom_deviation_percentage: float
+) -> int:
+    """
+    This function is the main function for the moving average calculator.
+
+    :param statistics_file_path: string, the statistics file path.
+    :param output_directory: string, the output directory.
+    :param moving_average_window_days: integer, the moving average window days.
+    :param top_bottom_deviation_percentage: float, the top bottom deviation percentage. Example: 0.1 for 10%.
+    :return: integer, the return code.
+    -----------------------------
+
+    Example:
+    import sys
+    from atomicshop.mitm import statistic_analyzer
+
+
+    def main():
+        return statistic_analyzer.moving_average_calculator_main(
+            statistics_file_path='statistics.csv',
+            output_directory='output',
+            moving_average_window_days=7,
+            top_bottom_deviation_percentage=0.1
+        )
+
+
+    if __name__ == '__main__':
+        sys.exit(main())
+    """
+
+    def convert_data_value_to_string(value_key: str, list_index: int) -> None:
+        deviation_list[list_index]['data'][value_key] = json.dumps(deviation_list[list_index]['data'][value_key])
+
+    def convert_value_to_string(value_key: str, list_index: int) -> None:
+        if value_key in deviation_list[list_index]:
+            deviation_list[list_index][value_key] = json.dumps(deviation_list[list_index][value_key])
+
+    deviation_list = calculate_moving_average(
+        statistics_file_path,
+        moving_average_window_days,
+        top_bottom_deviation_percentage
+    )
+
+    if deviation_list:
+        for deviation_list_index, deviation in enumerate(deviation_list):
+            convert_data_value_to_string('request_sizes', deviation_list_index)
+            convert_data_value_to_string('response_sizes', deviation_list_index)
+            convert_value_to_string('ma_data', deviation_list_index)
+
+        file_path = output_directory + os.sep + 'deviation.json'
+        print_api(f'Deviation Found, saving to file: {file_path}', color='blue')
+        jsons.write_json_file(deviation_list, file_path, use_default_indent=True)
+
+    return 0