atomicshop 2.11.47__py3-none-any.whl → 3.10.5__py3-none-any.whl

atomicshop/monitor/change_monitor.py

@@ -1,6 +1,21 @@
-from .checks import dns, network, hash, process_running
-from .. import filesystem, scheduling
-from ..diff_check import DiffChecker
+from typing import Literal, Union
+
+from .checks import dns, network, file, url, process_running
+
+
+DNS__DEFAULT_SETTINGS = {
+    'skip_record_list': [],                              # List of DNS Records to skip emitting. Example: ['PTR', 'SRV']
+    'learning_mode_create_unique_entries_list': True,
+    'learning_hours': 24,                                   # 0 - the learning will never stop.
+    'alert_about_missing_entries_after_learning': False,
+    'alert_about_missing_entries_always': True,
+    'create_alert_statistics': True,
+    'statistics_rotation_hours': 'midnight'                  # 'midnight' or float of hours
+}
+
+FILE__URL__DEFAULT_SETTINGS = {
+    'store_original_object': False
+}
 
 
 class ChangeMonitor:
@@ -9,32 +24,52 @@ class ChangeMonitor:
     """
     def __init__(
             self,
-            object_type: str,
-            check_object_list: list = None,
-            input_file_directory: str = None,
+            check_object: any = None,
             input_file_name: str = None,
-            generate_input_file_name: bool = False,
+            input_statistics_file_name: str = None,
+            input_directory: str = None,
             input_file_write_only: bool = True,
-            store_original_object: bool = False,
+            object_type: Union[
+                Literal[
+                    'file',
+                    'dns',
+                    'network',
+                    'process_running',
+                    'url_urllib',
+                    'url_playwright_html',
+                    'url_playwright_pdf',
+                    'url_playwright_png',
+                    'url_playwright_jpeg'],
+                None] = None,
+            object_type_settings: dict = None,
+            etw_session_name: str = None
     ):
         """
         :param object_type: string, type of object to check. The type must be one of the following:
-            'dns': 'check_object_list' will be none, since the DNS events will be queried from the system.
-            'file': 'check_object_list' must contain strings of full path to the file.
-            'url_urllib': 'check_object_list' must contain strings of full URL to a web page. The page will be
-                downloaded using 'urllib' library in HTML.
-            'url_playwright_html': 'check_object_list' must contain strings of full URL to a web page. The page will
-                be downloaded using 'playwright' library in HTML.
-            'url_playwright_pdf': 'check_object_list' must contain strings of full URL to a web page. The page will
-                be downloaded using 'playwright' library in PDF.
-            'url_playwright_png': 'check_object_list' must contain strings of full URL to a web page. The page will
-                be downloaded using 'playwright' library in PNG.
-            'url_playwright_jpeg': 'check_object_list' must contain strings of full URL to a web page. The page will
-                be downloaded using 'playwright' library in JPEG.
-        :param check_object_list: list of objects to check if it changed. The list can contain as many objects as
-            needed and can contain only one object.
-            The list can be left empty if the object type is 'dns', 'network_sockets'.
-        :param input_file_directory: string, full directory path for storing input files for current state of objects,
+            'file': 'check_object' must contain string of full path to the file.
+            'dns': 'check_object' will be none, since the DNS events will be queried from the system.
+            'network': 'check_object' will be none, since the network events will be queried from the system.
+            'process_running': 'check_object' must contain list of strings of process names to check if they are
+                running.
+                Example: ['chrome.exe', 'firefox.exe']
+                No file is written.
+            'url_*': 'check_object' must contain string of full URL to a web page to download.
+                'url_urllib': download using 'urllib' library to HTML file.
+                'url_playwright_html': download using 'playwright' library to HTML file.
+                'url_playwright_pdf': download using 'playwright' library to PDF file.
+                'url_playwright_png': download using 'playwright' library to PNG file.
+                'url_playwright_jpeg': download using 'playwright' library to JPEG file.
+        :param object_type_settings: dict, specific settings for the object type.
+            'dns': Check the default settings example in 'DNS__DEFAULT_SETTINGS'.
+            'file': Check the default settings example in 'FILE__URL__DEFAULT_SETTINGS'.
+            'url_*': Check the default settings example in 'FILE__URL__DEFAULT_SETTINGS'.
+        :param check_object: The object to check if changed.
+            'dns': empty.
+            'network': empty.
+            'process_running': list of strings, process names to check if they are running.
+            'file': string, full path to the file.
+            'url_*': string, full URL to a web page.
+        :param input_directory: string, full directory path for storing input files for current state of objects,
             to check later if this state isn't updated. If this variable is left empty, all the content will be saved
             in memory and input file will not be used.
             If the file is not specified, the update of an object will be checked
@@ -50,91 +85,54 @@ class ChangeMonitor:
                 to the input file whether the object was updated.
             False: write to input file each time there is an update, and read each check cycle from the file and not
                 from the memory.
-        :param store_original_object: boolean, if True, the original object will be stored on the disk inside
-        'Original' folder, inside 'input_file_directory'.
+        :param etw_session_name: string, the name of the ETW session. This should help you manage your ETW sessions
+            with logman and other tools: logman query -ets
+            If not provided, a default name will be generated.
+            'dns': 'AtomicShopDnsTrace'
 
-        If 'input_file_directory' is not specified, the 'input_file_name' is not specified, and
+        If 'input_directory' is not specified, the 'input_file_name' is not specified, and
         'generate_input_file_name' is False, then the input file will not be used and the object will be stored
         in memory. This means that the object will be checked only during the time that the script is running.
         """
 
-        # =================== Exception section ============================
-        if not input_file_directory and store_original_object:
-            raise ValueError('ERROR: [input_file_directory] must be specified if [store_original_object] is True.')
+        # === Initialize Main variables ====================================
 
-        if not input_file_directory and generate_input_file_name:
-            raise ValueError('ERROR: [input_file_directory] must be specified if [generate_input_file_name] is True.')
+        self.check_object: any = check_object
+        self.input_file_name: str = input_file_name
+        self.input_statistics_file_name: str = input_statistics_file_name
+        self.input_directory: str = input_directory
+        self.input_file_write_only: bool = input_file_write_only
+        self.object_type = object_type
+        self.object_type_settings: dict = object_type_settings
+        self.etw_session_name: str = etw_session_name
 
-        if not input_file_directory and input_file_name:
-            raise ValueError('ERROR: [input_file_directory] must be specified if [input_file_name] is specified.')
+        # === Additional variables ========================================
 
-        if input_file_name and generate_input_file_name:
-            raise ValueError(
-                'ERROR: [input_file_name] and [generate_input_file_name] cannot be both specified and True.')
+        self.checks_instance = None
+        self._setup_check()
 
-        # === EOF Exception section ========================================
-        # === Initialize Main variables ====================================
+    def _setup_check(self):
+        if self.object_type == 'file':
+            if not self.object_type_settings:
+                self.object_type_settings = FILE__URL__DEFAULT_SETTINGS
 
-        if not check_object_list:
-            check_object_list = list()
+            self.checks_instance = file.FileCheck(self)
+        elif self.object_type.startswith('url_'):
+            if not self.object_type_settings:
+                self.object_type_settings = FILE__URL__DEFAULT_SETTINGS
 
-        self.object_type: str = object_type
-        self.check_object_list: list = check_object_list
-        self.input_file_directory: str = input_file_directory
-        self.input_file_name: str = input_file_name
-        self.generate_input_file_name: bool = generate_input_file_name
-        self.input_file_write_only: bool = input_file_write_only
-        self.store_original_object: bool = store_original_object
-
-        # === EOF Initialize Main variables ================================
-        # === Initialize Secondary variables ===============================
-
-        # The 'diff_check' will store the list of DiffChecker classes.
-        self.diff_check_list: list = list()
-
-        # If 'check_object_list' is a list, loop through it and create a DiffChecker object for each object.
-        if self.check_object_list:
-            for index in range(len(self.check_object_list)):
-                self.diff_check_list.append(
-                    DiffChecker(
-                        input_file_write_only=self.input_file_write_only,
-                    )
-                )
-        # Else, if 'check_object_list' is None, create a DiffChecker object only once.
+            self.checks_instance = url.UrlCheck(self)
+        elif self.object_type == 'dns':
+            if not self.object_type_settings:
+                self.object_type_settings = DNS__DEFAULT_SETTINGS
+
+            self.checks_instance = dns.DnsCheck(self)
+        elif self.object_type == 'network':
+            self.checks_instance = network.NetworkCheck(self)
+        elif self.object_type == 'process_running':
+            self.checks_instance = process_running.ProcessRunningCheck(self)
         else:
-            self.diff_check_list.append(
-                DiffChecker(
-                    input_file_write_only=self.input_file_write_only,
-                )
-            )
-
-        self.input_file_path = None
-        self.original_object_directory = None
-        self.original_object_file_path = None
-
-        # If 'store_original_object' is True, create directory for original object.
-        if self.store_original_object:
-            # Make path for original object.
-            self.original_object_directory = filesystem.add_object_to_path(
-                self.input_file_directory, 'Original')
-            # Create directory if it doesn't exist.
-            filesystem.create_directory(self.original_object_directory)
-
-        self.first_cycle = True
-
-        # Initialize objects for DNS and Network monitoring.
-        self.fetch_engine = None
-        self.thread_looper = scheduling.ThreadLooper()
-
-    def _set_input_file_path(self, check_object_index: int = 0):
-        if self.first_cycle:
-            # If 'input_file_directory' and 'input_file_name' are specified, we'll use a filename to store.
-            if self.input_file_directory and self.input_file_name:
-                self.input_file_path = filesystem.add_object_to_path(
-                    self.input_file_directory, self.input_file_name)
-
-            # Set the input file path.
-            self.diff_check_list[check_object_index].input_file_path = self.input_file_path
+            raise ValueError(f"ERROR: Unknown object type: {self.object_type}")
 
     def check_cycle(self, print_kwargs: dict = None):
         """
@@ -143,28 +141,6 @@ class ChangeMonitor:
         :return: None
         """
 
-        return_list = list()
-
-        if (
-                self.object_type == 'file' or
-                'url_' in self.object_type):
-            return_list = hash._execute_cycle(self, print_kwargs=print_kwargs)
-        if self.object_type == 'dns':
-            return_list = dns._execute_cycle(self, print_kwargs=print_kwargs)
-        elif self.object_type == 'network':
-            return_list = network._execute_cycle(self, print_kwargs=print_kwargs)
-        elif self.object_type == 'process_running':
-            return_list = process_running._execute_cycle(self, print_kwargs=print_kwargs)
-
-        # Set 'first_cycle' to False, since the first cycle is finished.
-        if self.first_cycle:
-            self.first_cycle = False
+        return_list = self.checks_instance.execute_cycle(print_kwargs=print_kwargs)
 
         return return_list
-
-    def run_loop(self, interval_seconds=0, print_kwargs: dict = None):
-        self.thread_looper.run_loop(
-            self.check_cycle, kwargs={'print_kwargs': print_kwargs}, interval_seconds=interval_seconds)
-
-    def emit_from_loop(self):
-        return self.thread_looper.emit_from_loop()

atomicshop/monitor/checks/dns.py

@@ -1,77 +1,146 @@
-from ...etw.dns_trace import DnsTrace
+from pathlib import Path
+from typing import Union
+
+from ...etws.traces import trace_dns
 from ...print_api import print_api
+from ...import diff_check
 
 
-def _execute_cycle(change_monitor_instance, print_kwargs: dict = None):
-    """
-    This function executes the cycle of the change monitor: dns.
+INPUT_FILE_DEFAULT_NAME: str = 'known_domains.json'
+INPUT_STATISTICS_FILE_DEFAULT_NAME: str = 'dns_statistics.json'
 
-    :param change_monitor_instance: Instance of the ChangeMonitor class.
 
-    :return: List of dictionaries with the results of the cycle.
+class DnsCheck:
     """
-
-    if print_kwargs is None:
-        print_kwargs = dict()
-
-    if change_monitor_instance.first_cycle:
-        original_name: str = str()
-
-        # Initialize objects for DNS monitoring.
-        change_monitor_instance.fetch_engine = DnsTrace(
-            enable_process_poller=True, attrs=['name', 'cmdline', 'domain', 'query_type'])
-
+    Class for DNS monitoring.
+    """
+    
+    def __init__(self, change_monitor_instance):
+        self.change_monitor_instance = change_monitor_instance
+        self.diff_checker_aggregation: Union[diff_check.DiffChecker, None] = None
+        self.diff_checker_statistics: Union[diff_check.DiffChecker, None] = None
+        self.settings: dict = change_monitor_instance.object_type_settings
+
+        self.etw_session_name: str = change_monitor_instance.etw_session_name
+
+        self.fetch_engine: trace_dns.DnsRequestResponseTrace = (
+            trace_dns.DnsRequestResponseTrace(
+                attrs=['name', 'cmdline', 'query', 'query_type'],
+                session_name=self.etw_session_name,
+                close_existing_session_name=True,
+                skip_record_list=self.settings['skip_record_list'],
+            )
+        )
+
+        if self.settings['alert_always'] and self.settings['alert_about_missing_entries_after_learning']:
+            raise ValueError(
+                "ERROR: [alert_always] and [alert_about_missing_entries_after_learning] "
+                "cannot be True at the same time.")
+    
+        if not change_monitor_instance.input_file_name:
+            change_monitor_instance.input_file_name = INPUT_FILE_DEFAULT_NAME
+        input_file_path = (
+            str(Path(change_monitor_instance.input_directory, change_monitor_instance.input_file_name)))
+    
+        if not change_monitor_instance.input_statistics_file_name:
+            change_monitor_instance.input_statistics_file_name = INPUT_STATISTICS_FILE_DEFAULT_NAME
+        input_statistic_file_path = (
+            str(Path(change_monitor_instance.input_directory, change_monitor_instance.input_statistics_file_name)))
+    
+        if self.settings['learning_mode_create_unique_entries_list']:
+            aggregation_display_name = \
+                f'{change_monitor_instance.input_file_name}|{change_monitor_instance.object_type}'
+            self.diff_checker_aggregation = diff_check.DiffChecker(
+                check_object=list(),                            # DNS events will be appended to this list.
+                return_first_cycle=True,
+                operation_type='new_objects',
+                input_file_write_only=change_monitor_instance.input_file_write_only,
+                check_object_display_name=aggregation_display_name,
+                input_file_path=input_file_path,
+                new_objects_hours_then_difference=self.settings['learning_hours']
+            )
+            self.diff_checker_aggregation.initiate_before_action()
+
+        if self.settings['create_alert_statistics']:
+            statistics_display_name = \
+                f'{change_monitor_instance.input_statistics_file_name}|{change_monitor_instance.object_type}'
+
+            self.diff_checker_statistics = diff_check.DiffChecker(
+                check_object=list(),  # DNS events will be appended to this list.
+                return_first_cycle=True,
+                operation_type='hit_statistics',
+                hit_statistics_enable_queue=True,
+                input_file_write_only=change_monitor_instance.input_file_write_only,
+                check_object_display_name=statistics_display_name,
+                input_file_path=input_statistic_file_path,
+                hit_statistics_input_file_rotation_cycle_hours=self.settings['statistics_rotation_hours']
+            )
+            self.diff_checker_statistics.initiate_before_action()
+    
         # Start DNS monitoring.
-        change_monitor_instance.fetch_engine.start()
-
-        # Change settings for the DiffChecker object.
-        change_monitor_instance.diff_check_list[0].return_first_cycle = True
-        change_monitor_instance.diff_check_list[0].aggregation = True
-
-        if change_monitor_instance.generate_input_file_name:
-            original_name = 'known_domains'
-            # Make path for 'input_file_name'.
-            change_monitor_instance.input_file_name = f'{original_name}.json'
-
-        change_monitor_instance.diff_check_list[0].check_object_display_name = \
-            f'{original_name}|{change_monitor_instance.object_type}'
-
-        # Set the 'check_object' to empty list, since we will append the list of DNS events.
-        change_monitor_instance.diff_check_list[0].check_object = list()
-
-        # Set the input file path.
-        change_monitor_instance._set_input_file_path()
-
-    return_list = list()
-
-    # 'emit()' method is blocking (it uses 'get' of queue instance)
-    # will return a dict with current DNS trace event.
-    event_dict = change_monitor_instance.fetch_engine.emit()
-
-    change_monitor_instance.diff_check_list[0].check_object = [event_dict]
-
-    # if event_dict not in change_monitor_instance.diff_check_list[0].check_object:
-    #     change_monitor_instance.diff_check_list[0].check_object.append(event_dict)
-    #
-    # # Sort list of dicts by process name and then by process cmdline.
-    # change_monitor_instance.diff_check_list[0].check_object = list_of_dicts.sort_by_keys(
-    #     change_monitor_instance.diff_check_list[0].check_object, ['cmdline', 'name'], case_insensitive=True)
-
-    # Check if 'known_domains' list was updated from previous cycle.
-    result, message = change_monitor_instance.diff_check_list[0].check_list_of_dicts(
-        sort_by_keys=['cmdline', 'name'], print_kwargs=print_kwargs)
-
-    if result:
-        # Get list of new connections only.
-        # new_connections_only: list = list_of_dicts.get_difference(result['old'], result['updated'])
-
-        for connection in result['updated']:
-            message = \
-                f"New domain: {connection['name']} | " \
-                f"{connection['domain']} | {connection['query_type']} | " \
-                f"{connection['cmdline']}"
-            print_api(message, color='yellow', **print_kwargs)
-
-            return_list.append(message)
-
-    return return_list
+        self.fetch_engine.start()
+
+    def execute_cycle(self, print_kwargs: dict = None) -> list:
+        """
+        This function executes the cycle of the change monitor: dns.
+        The function is blocking so while using it, the script will wait for the next DNS event.
+        No need to use 'time.sleep()'.
+    
+        :param print_kwargs: print_api kwargs.
+        :return: List of dictionaries with the results of the cycle.
+        """
+    
+        # 'emit()' method is blocking (it uses 'get' of queue instance)
+        # will return a dict with current DNS trace event.
+        event_dict = self.fetch_engine.emit()
+    
+        return_list = list()
+        if self.settings['learning_mode_create_unique_entries_list']:
+            self._aggregation_process(event_dict, return_list, print_kwargs)
+    
+        if self.settings['create_alert_statistics'] and self.settings['alert_always']:
+            self._statistics_process(event_dict, return_list, print_kwargs)
+    
+        return return_list
+
+    def _aggregation_process(self, event_dict: dict, return_list: list, print_kwargs: dict = None):
+        self.diff_checker_aggregation.check_object = [event_dict]
+
+        # Check if 'known_domains' list was updated from previous cycle.
+        result, message = self.diff_checker_aggregation.check_list_of_dicts(
+            sort_by_keys=['cmdline', 'name'], print_kwargs=print_kwargs)
+    
+        if result:
+            # Check if 'updated' key is in the result. This means that this is a regular cycle.
+            if 'updated' in result:
+                if not result['time_passed']:
+                    # Get list of new connections only.
+                    # new_connections_only: list = list_of_dicts.get_difference(result['old'], result['updated'])
+    
+                    for connection in result['updated']:
+                        message = \
+                            f"[Learning] New Domain Added: {connection['name']} | " \
+                            f"{connection['domain']} | {connection['query_type']} | " \
+                            f"{connection['cmdline']}"
+                        print_api(message, color='yellow', **(print_kwargs or {}))
+    
+                        return_list.append(message)
+    
+            # Check if learning time passed, so we can alert the new entries.
+            if 'time_passed' in result:
+                if result['time_passed']:
+                    self._statistics_process(result['updated'], return_list, print_kwargs)
+    
+    def _statistics_process(self, event_dict: dict, return_list: list, print_kwargs: dict = None):
+        self.diff_checker_statistics.check_object = [event_dict]
+    
+        # Check if 'known_domains' list was updated from previous cycle.
+        result, message = self.diff_checker_statistics.check_list_of_dicts(
+            sort_by_keys=['cmdline', 'name'], print_kwargs=print_kwargs)
+    
+        if result:
+            if 'count' in result:
+                message = f"[Alert] DNS Request: {result['entry']} | Times hit: {result['count']}"
+                print_api(message, color='yellow', **(print_kwargs or {}))
+    
+                return_list.append(message)

atomicshop/monitor/checks/file.py

@@ -0,0 +1,77 @@
+from pathlib import Path
+
+from ... import filesystem, hashing
+from ... import diff_check
+from ...print_api import print_api
+
+
+class FileCheck:
+    """
+    Class for file monitoring.
+    """
+    def __init__(self, change_monitor_instance):
+        self.diff_checker = None
+        self.change_monitor_instance = None
+        self.store_original_file_path = None
+
+        if not change_monitor_instance.input_file_name:
+            change_monitor_instance.input_file_name = Path(change_monitor_instance.check_object).name
+            change_monitor_instance.input_file_name = change_monitor_instance.input_file_name.lower()
+            change_monitor_instance.input_file_name = (
+                change_monitor_instance.input_file_name.replace(' ', '_').replace('.', '_'))
+            change_monitor_instance.input_file_name = f'{change_monitor_instance.input_file_name}.txt'
+
+        input_file_path = (
+            str(Path(change_monitor_instance.input_directory, change_monitor_instance.input_file_name)))
+
+        # If 'store_original_object' is True, create filename for the store original object.
+        if change_monitor_instance.object_type_settings['store_original_object']:
+            store_original_file_name: str = f'ORIGINAL_{Path(change_monitor_instance.check_object).name}'
+            self.store_original_file_path = str(Path(change_monitor_instance.input_directory, store_original_file_name))
+
+        self.diff_checker = diff_check.DiffChecker(
+            return_first_cycle=False,
+            operation_type='single_object',
+            input_file_path=input_file_path,
+            check_object_display_name=f'{change_monitor_instance.input_file_name}|{change_monitor_instance.object_type}'
+        )
+        self.diff_checker.initiate_before_action()
+        self.change_monitor_instance = change_monitor_instance
+
+    def execute_cycle(self, print_kwargs: dict = None):
+        """
+        This function executes the cycle of the change monitor: hash.
+
+        :param print_kwargs: print_api kwargs.
+        :return: List of dictionaries with the results of the cycle.
+        """
+
+        return_list = list()
+
+        self._get_hash()
+
+        # Check if the object was updated.
+        result, message = self.diff_checker.check_string(
+            print_kwargs=print_kwargs)
+
+        # If the object was updated, print the message in yellow color, otherwise print in green color.
+        if result:
+            print_api(message, color='yellow', **print_kwargs)
+            # create_message_file(message, self.__class__.__name__, logger=self.logger)
+
+            return_list.append(message)
+        else:
+            print_api(message, color='green', **print_kwargs)
+
+        return return_list
+
+    def _get_hash(self):
+        # Copy the file to the original object directory.
+        if self.store_original_file_path:
+            filesystem.copy_file(self.change_monitor_instance.check_object, self.store_original_file_path)
+
+        # Get hash of the file.
+        hash_string = hashing.hash_file(self.change_monitor_instance.check_object)
+
+        # Set the hash string to the 'check_object' variable.
+        self.diff_checker.check_object = hash_string