atomicshop 2.14.2__py3-none-any.whl → 2.14.4__py3-none-any.whl

atomicshop/mitm/statistic_analyzer.py

@@ -1,8 +1,12 @@
+import os
 import datetime
+import statistics
+import json
+from typing import Literal
 
 from .. import filesystem, domains, datetimes, urls
 from ..basics import dicts
-from ..file_io import tomls, xlsxs
+from ..file_io import tomls, xlsxs, csvs, jsons
 from ..wrappers.loggingw import reading
 from ..print_api import print_api
 
@@ -154,10 +158,10 @@ def analyze(main_file_path: str):
     summary_path: str = filesystem.check_absolute_path___add_full(config['report_file_path'], script_directory)
 
     # Get the content from statistics files.
-    statistics_content: list = reading.get_logs(
+    statistics_content: list = reading.get_all_log_files_into_list(
         config['statistic_files_path'],
         file_name_pattern='statistics*.csv',
-        log_type='csv',
+        log_type='csv'
     )
 
     # Initialize loop.
@@ -465,3 +469,372 @@ def analyze(main_file_path: str):
         xlsxs.write_xlsx(combined_sorted_stats, file_path=summary_path)
 
     return
+
+
+# ======================================================================================================================
+
+
+def calculate_moving_average(
+        file_path: str,
+        moving_average_window_days,
+        top_bottom_deviation_percentage: float,
+        print_kwargs: dict = None
+):
+    """
+    This function calculates the moving average of the daily statistics.
+
+    :param file_path: string, the path to the 'statistics.csv' file.
+    :param moving_average_window_days: integer, the window size for the moving average.
+    :param top_bottom_deviation_percentage: float, the percentage of deviation from the moving average to the top or
+        bottom.
+    :param print_kwargs: dict, the print_api arguments.
+    """
+
+    date_pattern: str = '%Y_%m_%d'
+
+    # Get all the file paths and their midnight rotations.
+    logs_paths: list = reading.get_logs_paths(
+        log_file_path=file_path,
+        date_pattern=date_pattern
+    )
+
+    statistics_content: dict = {}
+    # Read each file to its day.
+    for log_path_dict in logs_paths:
+        date_string = log_path_dict['date_string']
+        statistics_content[date_string] = {}
+
+        statistics_content[date_string]['file'] = log_path_dict
+
+        log_file_content, log_file_header = (
+            csvs.read_csv_to_list_of_dicts_by_header(log_path_dict['file_path'], **(print_kwargs or {})))
+        statistics_content[date_string]['content'] = log_file_content
+        statistics_content[date_string]['header'] = log_file_header
+
+        statistics_content[date_string]['content_no_errors'] = get_content_without_errors(log_file_content)
+
+        # Get the data dictionary from the statistics content.
+        statistics_content[date_string]['statistics_daily'] = compute_statistics_from_content(
+            statistics_content[date_string]['content_no_errors']
+        )
+
+    moving_average_dict: dict = compute_moving_averages_from_average_statistics(
+        statistics_content,
+        moving_average_window_days
+    )
+
+    # Add the moving average to the statistics content.
+    for day, day_dict in statistics_content.items():
+        try:
+            day_dict['moving_average'] = moving_average_dict[day]
+        except KeyError:
+            day_dict['moving_average'] = {}
+
+    # Find deviation from the moving average to the bottom or top by specified percentage.
+    deviation_list: list = find_deviation_from_moving_average(
+        statistics_content, top_bottom_deviation_percentage)
+
+    return deviation_list
+
+
+def get_content_without_errors(content: list) -> list:
+    """
+    This function gets the 'statistics.csv' file content without errors from the 'content' list.
+
+    :param content: list, the content list.
+    :return: list, the content without errors.
+    """
+
+    traffic_statistics_without_errors: list = []
+    for line in content:
+        # Skip empty lines, headers and errors.
+        if line['host'] == 'host' or line['command'] == '':
+            continue
+
+        traffic_statistics_without_errors.append(line)
+
+    return traffic_statistics_without_errors
+
+
+def get_data_dict_from_statistics_content(content: list) -> dict:
+    """
+    This function gets the data dictionary from the 'statistics.csv' file content.
+
+    :param content: list, the content list.
+    :return: dict, the data dictionary.
+    """
+
+    hosts_requests_responses: dict = {}
+    for line in content:
+        # If subdomain is not in the dictionary, add it.
+        if line['host'] not in hosts_requests_responses:
+            hosts_requests_responses[line['host']] = {
+                'request_sizes': [],
+                'response_sizes': []
+            }
+
+        # Append the sizes.
+        try:
+            hosts_requests_responses[line['host']]['request_sizes'].append(int(line['request_size_bytes']))
+            hosts_requests_responses[line['host']]['response_sizes'].append(
+                int(line['response_size_bytes']))
+        except ValueError:
+            print_api(line, color='yellow')
+            raise
+
+    return hosts_requests_responses
+
+
+def compute_statistics_from_data_dict(data_dict: dict):
+    """
+    This function computes the statistics from the data dictionary.
+
+    :param data_dict: dict, the data dictionary.
+    :return: dict, the statistics dictionary.
+    """
+
+    for host, host_dict in data_dict.items():
+        count = len(host_dict['request_sizes'])
+        avg_request_size = statistics.mean(host_dict['request_sizes']) if count > 0 else 0
+        median_request_size = statistics.median(host_dict['request_sizes']) if count > 0 else 0
+        avg_response_size = statistics.mean(host_dict['response_sizes']) if count > 0 else 0
+        median_response_size = statistics.median(host_dict['response_sizes']) if count > 0 else 0
+
+        data_dict[host]['count'] = count
+        data_dict[host]['avg_request_size'] = avg_request_size
+        data_dict[host]['median_request_size'] = median_request_size
+        data_dict[host]['avg_response_size'] = avg_response_size
+        data_dict[host]['median_response_size'] = median_response_size
+
+
+def compute_statistics_from_content(content: list):
+    """
+    This function computes the statistics from the 'statistics.csv' file content.
+
+    :param content: list, the content list.
+    :return: dict, the statistics dictionary.
+    """
+
+    hosts_requests_responses: dict = get_data_dict_from_statistics_content(content)
+    compute_statistics_from_data_dict(hosts_requests_responses)
+
+    return hosts_requests_responses
+
+
+def compute_moving_averages_from_average_statistics(
+        average_statistics_dict: dict,
+        moving_average_window_days: int
+):
+    """
+    This function computes the moving averages from the average statistics dictionary.
+
+    :param average_statistics_dict: dict, the average statistics dictionary.
+    :param moving_average_window_days: integer, the window size for the moving average.
+    :return: dict, the moving averages dictionary.
+    """
+
+    moving_average: dict = {}
+    for day_index, (day, day_dict) in enumerate(average_statistics_dict.items()):
+        current_day = day_index + 1
+        if current_day < moving_average_window_days:
+            continue
+
+        # Create list of the previous 'moving_average_window_days' days.
+        previous_days_content_list = (
+            list(average_statistics_dict.values()))[current_day-moving_average_window_days:current_day]
+
+        # Compute the moving averages.
+        moving_average[day] = compute_average_for_current_day_from_past_x_days(previous_days_content_list)
+
+    return moving_average
+
+
+def compute_average_for_current_day_from_past_x_days(previous_days_content_list: list) -> dict:
+    """
+    This function computes the average for the current day from the past x days.
+
+    :param previous_days_content_list: list, the list of the previous days content.
+    :return: dict, the average dictionary.
+    """
+
+    moving_average: dict = {}
+    for entry in previous_days_content_list:
+        statistics_daily = entry['statistics_daily']
+        for host, host_dict in statistics_daily.items():
+            if host not in moving_average:
+                moving_average[host] = {
+                    'counts': [],
+                    'avg_request_sizes': [],
+                    'avg_response_sizes': [],
+                }
+
+            moving_average[host]['counts'].append(int(host_dict['count']))
+            moving_average[host]['avg_request_sizes'].append(float(host_dict['avg_request_size']))
+            moving_average[host]['avg_response_sizes'].append(float(host_dict['avg_response_size']))
+
+    # Compute the moving average.
+    moving_average_results: dict = {}
+    for host, host_dict in moving_average.items():
+        ma_count = statistics.mean(host_dict['counts'])
+        ma_request_size = statistics.mean(host_dict['avg_request_sizes'])
+        ma_response_size = statistics.mean(host_dict['avg_response_sizes'])
+
+        moving_average_results[host] = {
+            'ma_count': ma_count,
+            'ma_request_size': ma_request_size,
+            'ma_response_size': ma_response_size,
+            'counts': host_dict['counts'],
+            'avg_request_sizes': host_dict['avg_request_sizes'],
+            'avg_response_sizes': host_dict['avg_response_sizes']
+        }
+
+    return moving_average_results
+
+
+def find_deviation_from_moving_average(
+        statistics_content: dict,
+        top_bottom_deviation_percentage: float
+) -> list:
+    """
+    This function finds the deviation from the moving average to the bottom or top by specified percentage.
+
+    :param statistics_content: dict, the statistics content dictionary.
+    :param top_bottom_deviation_percentage: float, the percentage of deviation from the moving average to the top or
+        bottom.
+    :return: list, the deviation list.
+    """
+
+    def _check_deviation(
+            check_type: Literal['count', 'avg_request_size', 'avg_response_size'],
+            ma_check_type: Literal['ma_count', 'ma_request_size', 'ma_response_size'],
+            day_statistics_content_dict: dict,
+            moving_averages_dict: dict
+    ):
+        """
+        This function checks the deviation for the host.
+        """
+
+        nonlocal message
+
+        host_moving_average_by_type = moving_averages_dict[host][ma_check_type]
+        check_type_moving_by_percent = (
+                host_moving_average_by_type * top_bottom_deviation_percentage)
+        check_type_moving_above = host_moving_average_by_type + check_type_moving_by_percent
+        check_type_moving_below = host_moving_average_by_type - check_type_moving_by_percent
+
+        deviation_type = None
+        if day_statistics_content_dict[check_type] > check_type_moving_above:
+            deviation_type = 'above'
+        elif day_statistics_content_dict[check_type] < check_type_moving_below:
+            deviation_type = 'below'
+
+        if deviation_type:
+            message = f'[{check_type}] is [{deviation_type}] the moving average.'
+            deviation_list.append({
+                'day': day,
+                'host': host,
+                'message': message,
+                'value': day_statistics_content_dict[check_type],
+                'ma_value': host_moving_average_by_type,
+                'check_type': check_type,
+                'percentage': top_bottom_deviation_percentage,
+                'ma_value_checked': check_type_moving_above,
+                'deviation_type': deviation_type,
+                'data': day_statistics_content_dict,
+                'ma_data': moving_averages_dict[host]
+            })
+
+    deviation_list: list = []
+    for day_index, (day, day_dict) in enumerate(statistics_content.items()):
+        # If it's the first day, there is no previous day moving average.
+        if day_index == 0:
+            previous_day_moving_average_dict = {}
+        else:
+            previous_day_moving_average_dict = list(statistics_content.values())[day_index-1].get('moving_average', {})
+
+        # If there is no moving average for previous day continue to the next day.
+        if not previous_day_moving_average_dict:
+            continue
+
+        for host, host_dict in day_dict['statistics_daily'].items():
+            # If the host is not in the moving averages, then this is clear deviation.
+            # It means that in the current day, there were no requests for this host.
+            if host not in previous_day_moving_average_dict:
+                message = f'Host not in the moving averages: {host}'
+                deviation_list.append({
+                    'day': day,
+                    'host': host,
+                    'data': host_dict,
+                    'message': message,
+                    'type': 'clear'
+                })
+                continue
+
+            _check_deviation(
+                'count', 'ma_count', host_dict, previous_day_moving_average_dict)
+            _check_deviation(
+                'avg_request_size', 'ma_request_size', host_dict, previous_day_moving_average_dict)
+            _check_deviation(
+                'avg_response_size', 'ma_response_size', host_dict, previous_day_moving_average_dict)
+
+    return deviation_list
+
+
+def moving_average_calculator_main(
+        statistics_file_path: str,
+        output_directory: str,
+        moving_average_window_days: int,
+        top_bottom_deviation_percentage: float
+) -> int:
+    """
+    This function is the main function for the moving average calculator.
+
+    :param statistics_file_path: string, the statistics file path.
+    :param output_directory: string, the output directory.
+    :param moving_average_window_days: integer, the moving average window days.
+    :param top_bottom_deviation_percentage: float, the top bottom deviation percentage. Example: 0.1 for 10%.
+    :return: integer, the return code.
+    -----------------------------
+
+    Example:
+    import sys
+    from atomicshop.mitm import statistic_analyzer
+
+
+    def main():
+        return statistic_analyzer.moving_average_calculator_main(
+            statistics_file_path='statistics.csv',
+            output_directory='output',
+            moving_average_window_days=7,
+            top_bottom_deviation_percentage=0.1
+        )
+
+
+    if __name__ == '__main__':
+        sys.exit(main())
+    """
+
+    def convert_data_value_to_string(value_key: str, list_index: int) -> None:
+        deviation_list[list_index]['data'][value_key] = json.dumps(deviation_list[list_index]['data'][value_key])
+
+    def convert_value_to_string(value_key: str, list_index: int) -> None:
+        if value_key in deviation_list[list_index]:
+            deviation_list[list_index][value_key] = json.dumps(deviation_list[list_index][value_key])
+
+    deviation_list = calculate_moving_average(
+        statistics_file_path,
+        moving_average_window_days,
+        top_bottom_deviation_percentage
+    )
+
+    if deviation_list:
+        for deviation_list_index, deviation in enumerate(deviation_list):
+            convert_data_value_to_string('request_sizes', deviation_list_index)
+            convert_data_value_to_string('response_sizes', deviation_list_index)
+            convert_value_to_string('ma_data', deviation_list_index)
+
+        file_path = output_directory + os.sep + 'deviation.json'
+        print_api(f'Deviation Found, saving to file: {file_path}', color='blue')
+        jsons.write_json_file(deviation_list, file_path, use_default_indent=True)
+
+    return 0

atomicshop/monitor/change_monitor.py

@@ -4,6 +4,7 @@ from .checks import dns, network, file, url, process_running
 
 
 DNS__DEFAULT_SETTINGS = {
+    'skip_record_list': [],                              # List of DNS Records to skip emitting. Example: ['PTR', 'SRV']
     'learning_mode_create_unique_entries_list': True,
     'learning_hours': 24,                                   # 0 - the learning will never stop.
     'alert_about_missing_entries_after_learning': False,

atomicshop/monitor/checks/dns.py

@@ -27,7 +27,8 @@ class DnsCheck:
             trace_dns.DnsRequestResponseTrace(
                 attrs=['name', 'cmdline', 'domain', 'query_type'],
                 session_name=self.etw_session_name,
-                close_existing_session_name=True
+                close_existing_session_name=True,
+                skip_record_list=self.settings['skip_record_list'],
             )
         )
 

atomicshop/process.py

@@ -11,7 +11,7 @@ from .basics import strings
 from .wrappers import ubuntu_terminal
 
 if os.name == 'nt':
-    from . import process_poller
+    from . import get_process_list
 
 
 def is_command_exists(cmd: str) -> bool:
@@ -265,8 +265,8 @@ def match_pattern_against_running_processes_cmdlines(
     """
 
     # Get the list of all the currently running processes.
-    get_process_list = process_poller.GetProcessList(get_method='pywin32', connect_on_init=True)
-    processes = get_process_list.get_processes(as_dict=False)
+    get_process_list_instance = get_process_list.GetProcessList(get_method='pywin32', connect_on_init=True)
+    processes = get_process_list_instance.get_processes(as_dict=False)
 
     # Iterate through all the current process, while fetching executable file 'name' and the command line.
     # Name is always populated, while command line is not.

atomicshop/process_poller/pollers/psutil_pywin32wmi_dll.py

@@ -0,0 +1,95 @@
+from typing import Union, Literal
+import threading
+import time
+
+from ... import get_process_list
+
+
+class PollerPsutilPywin32Dll:
+    """
+    The class is responsible for getting the list of opened processes by using mentioned libraries.
+    """
+    def __init__(
+            self,
+            interval_seconds: Union[int, float] = 0,
+            process_get_method: Literal[
+                'poll_psutil',
+                'poll_pywin32',
+                'poll_process_dll'
+            ] = 'poll_process_dll',
+            process_queue=None
+    ):
+        """
+        :param interval_seconds: works only for pollers, float, how many seconds to wait between each cycle.
+            Default is 0, which means that the polling will be as fast as possible.
+
+            Basically, you want it to be '0' if you want to get the most recent processes.
+            Any polling by itself takes time, so if you want to get the most recent processes, you want to do it as fast
+            as possible.
+        :param process_get_method: str. Default is 'process_dll'. Available:
+            'poll_psutil': Poller, Get the list of processes by 'psutil' library. Resource intensive and slow.
+            'poll_pywin32': Poller, processes by 'pywin32' library, using WMI. Not resource intensive, but slow.
+            'poll_process_dll'. Poller, Not resource intensive and fast. Probably works only in Windows 10 x64.
+            'trace_sysmon_etw': Tracer, Get the list of processes with running SysMon by ETW - Event Tracing.
+                In this case 'interval_seconds' is irrelevant, since the ETW is real-time.
+                Steps we take:
+                    1. Check if SysMon is Running. If not, check if the executable exists in specified
+                        location and start it as a service.
+                    2. Start the "Microsoft-Windows-Sysmon" ETW session.
+                    3. Take a snapshot of current processes and their CMDs with psutil and store it in a dict.
+                    4. Each new process creation from ETW updates the dict.
+            'trace_event_log': Get the list of processes by subscribing to the Windows Event Log.
+                Log Channel: Security, Event ID: 4688.
+                We enable the necessary prerequisites in registry and subscribe to the event.
+        :param process_queue: Queue. The queue to put the processes in. If None, the processes will not be put in the
+            queue.
+        """
+
+        self.interval_seconds: Union[int, float] = interval_seconds
+        self.process_get_method = process_get_method.replace('poll_', '')
+        self.process_queue = process_queue
+
+        # noinspection PyTypeChecker
+        self.poller_instance = get_process_list.GetProcessList(get_method=self.process_get_method)
+
+        self._processes = {}
+
+    def start(self):
+        """
+        Start the poller.
+        """
+
+        thread = threading.Thread(target=self.emit_loop)
+        thread.daemon = True
+        thread.start()
+
+    def emit_loop(self):
+        """
+        Get the list of processes.
+        """
+
+        self.poller_instance.connect()
+
+        while True:
+            current_processes = self.poller_instance.get_processes(as_dict=True)
+
+            # Remove Command lines that contains only numbers, since they are useless.
+            for pid, process_info in current_processes.items():
+                if process_info['cmdline'].isnumeric():
+                    current_processes[pid]['cmdline'] = str()
+                elif process_info['cmdline'] == 'Error':
+                    current_processes[pid]['cmdline'] = str()
+
+            # This loop is essential for keeping the command lines.
+            # When the process unloads from memory, the last polling will have only pid and executable name, but not
+            # the command line. This loop will keep the command line from the previous polling if this happens.
+            for pid, process_info in current_processes.items():
+                if pid in self._processes:
+                    if self._processes[pid]['name'] == current_processes[pid]['name']:
+                        if current_processes[pid]['cmdline'] == '':
+                            current_processes[pid]['cmdline'] = self._processes[pid]['cmdline']
+            self._processes.update(current_processes)
+
+            self.process_queue.put(self._processes)
+
+            time.sleep(self.interval_seconds)