atomicshop 2.12.25__py3-none-any.whl → 2.13.0__py3-none-any.whl

atomicshop/__init__.py

@@ -1,4 +1,4 @@
 """Atomic Basic functions and classes to make developer life easier"""
 
 __author__ = "Den Kras"
-__version__ = '2.12.25'
+__version__ = '2.13.00'

atomicshop/basics/dicts.py

@@ -24,6 +24,18 @@ def get_first_key_and_value_dict(input_dict: dict) -> dict:
     return {first_key: input_dict[first_key]}
 
 
+def get_last_added_key_value(input_dict: dict) -> dict:
+    """
+    The function will return the last added key and value in a dictionary.
+
+    :param input_dict: dict, the dictionary to get the last added key and value.
+    :return: dict, the last added key and value in the dictionary.
+    """
+
+    last_key = list(input_dict.keys())[-1]
+    return {last_key: input_dict[last_key]}
+
+
 def remove_keys(input_dict: dict, key_list: list) -> None:
     """
     The function will remove a key from dictionary without raising an exception if it doesn't exist.

atomicshop/basics/package_module.py

@@ -0,0 +1,10 @@
+"""
+# Add static files to your package / module pyproject.toml:
+[tool.setuptools.package-data]
+"atomicshop.addons" = ["**"]
+
+# Read relative path of your module inside your package / module script:
+from importlib.resources import files
+PACKAGE_DLL_PATH = 'addons/process_list/compiled/Win10x64/process_list.dll'
+FULL_DLL_PATH = str(files(__package__).joinpath(PACKAGE_DLL_PATH))
+"""

atomicshop/diff_check.py

@@ -314,10 +314,10 @@ class DiffChecker:
                 try:
                     if self.save_as == 'txt':
                         self.previous_content = file_io.read_file(
-                            self.input_file_path, stderr=False, **(print_kwargs or {}))
+                            self.input_file_path, stdout=False, stderr=False, **(print_kwargs or {}))
                     elif self.save_as == 'json':
                         self.previous_content = jsons.read_json_file(
-                            self.input_file_path, stderr=False, **(print_kwargs or {}))
+                            self.input_file_path, stdout=False, stderr=False, **(print_kwargs or {}))
                 except FileNotFoundError as except_object:
                     message = f"Input File [{Path(except_object.filename).name}] doesn't exist - Will create new one."
                     print_api(message, color='yellow', **(print_kwargs or {}))
@@ -433,10 +433,11 @@ class DiffChecker:
         if self.input_file_path:
             if self.save_as == 'txt':
                 # noinspection PyTypeChecker
-                file_io.write_file(self.previous_content, self.input_file_path, **(print_kwargs or {}))
+                file_io.write_file(self.previous_content, self.input_file_path, stdout=False, **(print_kwargs or {}))
             elif self.save_as == 'json':
                 jsons.write_json_file(
-                    self.previous_content, self.input_file_path, use_default_indent=True, **(print_kwargs or {}))
+                    self.previous_content, self.input_file_path, use_default_indent=True, stdout=False,
+                    **(print_kwargs or {}))
 
         return result, message
 

atomicshop/dns.py

@@ -18,15 +18,6 @@ TYPES_DICT = {
     '255': 'ANY'
 }
 
-# Event Tracing DNS info.
-ETW_DNS_INFO = {
-    'provider_name': 'Microsoft-Windows-DNS-Client',
-    'provider_guid': '{1C95126E-7EEA-49A9-A3FE-A378B03DDB4D}',
-    # Event ID 3008 got DNS Queries and DNS Answers. Meaning, that information in ETW will arrive after DNS Response
-    # is received and not After DNS Query is sent.
-    'event_id': 3008
-}
-
 
 def resolve_dns_localhost(domain_name: str, dns_servers_list: list = None, print_kwargs: dict = None) -> str:
     """

atomicshop/etws/const.py

@@ -0,0 +1,38 @@
+ETW_DNS = {
+    'provider_name': 'Microsoft-Windows-DNS-Client',
+    'provider_guid': '{1C95126E-7EEA-49A9-A3FE-A378B03DDB4D}',
+    'event_ids': {
+        # Event ID 3008 got DNS Queries and DNS Answers. Meaning, that information in ETW will arrive after DNS Response
+        # is received and not after DNS Query Request is sent.
+        'dns_request_response': 3008
+    }
+}
+
+
+# Event Tracing Kernel-Process.
+ETW_KERNEL_PROCESS = {
+    'provider_name': 'Microsoft-Windows-Kernel-Process',
+    'provider_guid': '{22FB2CD6-0E7B-422B-A0C7-2FAD1FD0E716}',
+    'event_ids': {
+        'process_create': 1  # Emits all the processes that are started.
+    }
+}
+
+
+# You need SYSTEM privileges to execute this one. Try with psexec -s -i
+ETW_SECURITY_AUDITING = {
+    'provider_name': 'Microsoft-Windows-Security-Auditing',
+    'provider_guid': '{54849625-5478-4994-A5BA-3E3B0328C30D}',
+    'event_ids': {
+        'process_create': 4688      # Emits all the processes that are started.
+    }
+}
+
+
+ETW_SYSMON = {
+    'provider_name': 'Microsoft-Windows-Sysmon',
+    'provider_guid': '{5770385F-C22A-43E0-BF4C-06F5698FFBD9}',
+    'event_ids': {
+        'process_create': 1         # Emits all the processes that are started.
+    }
+}

atomicshop/{etw → etws/traces}/trace_dns.py

@@ -1,18 +1,31 @@
-from . import trace
-from .. import dns
-from ..wrappers.psutilw import psutilw
-from ..basics import dicts
-from ..process_poller import ProcessPollerPool
-from ..print_api import print_api
+import time
 
+from .. import trace, const
+from ...wrappers.psutilw import psutilw
+from ...basics import dicts
+from ...process_poller import ProcessPollerPool
+from ...print_api import print_api
+from ... import dns
 
-class DnsTrace:
+
+ETW_DEFAULT_SESSION_NAME: str = 'AtomicShopDnsTrace'
+
+PROVIDER_NAME: str = const.ETW_DNS['provider_name']
+PROVIDER_GUID: str = const.ETW_DNS['provider_guid']
+REQUEST_RESP_EVENT_ID: int = const.ETW_DNS['event_ids']['dns_request_response']
+
+WAIT_FOR_PROCESS_POLLER_PID_SECONDS: int = 3
+WAIT_FOR_PROCESS_POLLER_PID_COUNTS: int = WAIT_FOR_PROCESS_POLLER_PID_SECONDS * 10
+
+
+class DnsRequestResponseTrace:
     def __init__(
             self,
-            enable_process_poller: bool = False,
             attrs: list = None,
             session_name: str = None,
-            close_existing_session_name: bool = True
+            close_existing_session_name: bool = True,
+            enable_process_poller: bool = False,
+            process_poller_etw_session_name: str = None
     ):
         """
         DnsTrace class use to trace DNS events from Windows Event Tracing for EventId 3008.
@@ -28,6 +41,7 @@ class DnsTrace:
             False: if ETW session with 'session_name' exists, you will be notified and the new session will not be
                 created. Instead, the existing session will be used. If there is a buffer from the previous session,
                 you will get the events from the buffer.
+        :param process_poller_etw_session_name: The name of the ETW session for tracing process creation.
 
         -------------------------------------------------
 
@@ -35,7 +49,13 @@ class DnsTrace:
             from atomicshop.etw import dns_trace
 
 
-            dns_trace_w = dns_trace.DnsTrace(enable_process_poller=True, attrs=['pid', 'name', 'cmdline', 'domain', 'query_type'])
+            dns_trace_w = dns_trace.DnsTrace(
+                attrs=['pid', 'name', 'cmdline', 'domain', 'query_type'],
+                session_name='MyDnsTrace',
+                close_existing_session_name=True,
+                enable_process_poller=True,
+                process_poller_etw_session_name='MyProcessTrace'
+            )
             dns_trace_w.start()
             while True:
                 dns_dict = dns_trace_w.emit()
@@ -46,16 +66,21 @@ class DnsTrace:
         self.enable_process_poller = enable_process_poller
         self.attrs = attrs
 
+        if not session_name:
+            session_name = ETW_DEFAULT_SESSION_NAME
+
         self.event_trace = trace.EventTrace(
-            providers=[(dns.ETW_DNS_INFO['provider_name'], dns.ETW_DNS_INFO['provider_guid'])],
+            providers=[(PROVIDER_NAME, PROVIDER_GUID)],
             # lambda x: self.event_queue.put(x),
-            event_id_filters=[dns.ETW_DNS_INFO['event_id']],
+            event_id_filters=[REQUEST_RESP_EVENT_ID],
             session_name=session_name,
             close_existing_session_name=close_existing_session_name
         )
 
         if self.enable_process_poller:
-            self.process_poller = ProcessPollerPool(store_cycles=200, operation='process', poller_method='process_dll')
+            self.process_poller = ProcessPollerPool(
+                operation='process', poller_method='sysmon_etw',
+                sysmon_etw_session_name=process_poller_etw_session_name)
 
     def start(self):
         if self.enable_process_poller:
@@ -82,6 +107,11 @@ class DnsTrace:
         :return: Dictionary with the event data.
         """
 
+        # Get the processes first, since we need the process name and command line.
+        # If they're not ready, we will get just pids from DNS tracing.
+        if self.enable_process_poller:
+            self._get_processes_from_poller()
+
         event = self.event_trace.emit()
 
         event_dict: dict = {
@@ -92,12 +122,6 @@ class DnsTrace:
             'query_type': dns.TYPES_DICT[str(event[1]['QueryType'])]
         }
 
-        # # Get process name only from psutil, just in case.
-        # try:
-        #     process_name = psutilw.get_process_name_by_pid(event_dict['pid'])
-        # except psutil.NoSuchProcess:
-        #     process_name = str(event_dict['pid'])
-
         # Defining list if ips and other answers, which aren't IPs.
         list_of_ips = list()
         list_of_other_domains = list()
@@ -135,10 +159,19 @@ class DnsTrace:
             event_dict['status'] = 'Error'
 
         if self.enable_process_poller:
-            processes = self.process_poller.processes
-
-            if isinstance(processes, BaseException):
-                raise processes
+            processes = self.process_poller.get_processes()
+            if event_dict['pid'] not in processes:
+                counter = 0
+                while counter < WAIT_FOR_PROCESS_POLLER_PID_COUNTS:
+                    processes = self.process_poller.get_processes()
+                    if event_dict['pid'] not in processes:
+                        time.sleep(0.1)
+                        counter += 1
+                    else:
+                        break
+
+                if counter == WAIT_FOR_PROCESS_POLLER_PID_COUNTS:
+                    print_api(f"Error: Couldn't get the process name for PID: {event_dict['pid']}.", color='red')
 
             event_dict = psutilw.cross_single_connection_with_processes(event_dict, processes)
             # If it was impossible to get the process name from the process poller, get it from psutil.
@@ -150,3 +183,13 @@ class DnsTrace:
                 event_dict, self.attrs, skip_keys_not_in_list=True)
 
         return event_dict
+
+    def _get_processes_from_poller(self):
+        processes: dict = {}
+        while not processes:
+            processes = self.process_poller.get_processes()
+
+            if isinstance(processes, BaseException):
+                raise processes
+
+        return processes

atomicshop/etws/traces/trace_sysmon_process_creation.py

@@ -0,0 +1,116 @@
+from .. import trace, const
+from ...wrappers.psutilw import psutilw
+from ...wrappers import sysmonw
+from ...basics import dicts
+from ...print_api import print_api
+
+
+PROVIDER_NAME: str = const.ETW_SYSMON['provider_name']
+PROVIDER_GUID: str = const.ETW_SYSMON['provider_guid']
+PROCESS_CREATION_EVENT_ID: int = const.ETW_SYSMON['event_ids']['process_create']
+
+
+class SysmonProcessCreationTrace:
+    def __init__(
+            self,
+            attrs: list = None,
+            session_name: str = None,
+            close_existing_session_name: bool = True,
+            sysmon_directory: str = None
+    ):
+        """
+        DnsTrace class use to trace DNS events from Windows Event Tracing for EventId 3008.
+
+        :param attrs: List of attributes to return. If None, all attributes will be returned.
+        :param session_name: The name of the session to create. If not provided, a UUID will be generated.
+        :param close_existing_session_name: Boolean to close existing session names.
+            True: if ETW session with 'session_name' exists, you will be notified and the session will be closed.
+                Then the new session with this name will be created.
+            False: if ETW session with 'session_name' exists, you will be notified and the new session will not be
+                created. Instead, the existing session will be used. If there is a buffer from the previous session,
+                you will get the events from the buffer.
+        :param sysmon_directory: The directory where Sysmon is located. If not provided, "C:\\Windows\\Sysmon" will be
+            used. If 'Sysmon.exe' is not found in the directory, it will be downloaded from the internet.
+
+        -------------------------------------------------
+
+        Usage Example:
+            from atomicshop.etw import dns_trace
+
+
+            dns_trace_w = dns_trace.DnsTrace(
+                attrs=['pid', 'name', 'cmdline', 'domain', 'query_type'],
+                session_name='MyDnsTrace',
+                close_existing_session_name=True,
+                enable_process_poller=True,
+                process_poller_etw_session_name='MyProcessTrace'
+            )
+            dns_trace_w.start()
+            while True:
+                dns_dict = dns_trace_w.emit()
+                print(dns_dict)
+            dns_trace_w.stop()
+        """
+
+        self.attrs = attrs
+        self.sysmon_directory: str = sysmon_directory
+
+        self.event_trace = trace.EventTrace(
+            providers=[(PROVIDER_NAME, PROVIDER_GUID)],
+            # lambda x: self.event_queue.put(x),
+            event_id_filters=[PROCESS_CREATION_EVENT_ID],
+            session_name=session_name,
+            close_existing_session_name=close_existing_session_name
+        )
+
+    def start(self):
+        sysmonw.start_as_service(
+            installation_path=self.sysmon_directory, download_sysmon_if_not_found=True, skip_if_running=True)
+        self.event_trace.start()
+
+    def stop(self):
+        self.event_trace.stop()
+
+    def emit(self):
+        """
+        Function that will return the next event from the queue.
+        The queue is blocking, so if there is no event in the queue, the function will wait until there is one.
+
+        Usage Example:
+            while True:
+                dns_dict = dns_trace.emit()
+                print(dns_dict)
+
+        :return: Dictionary with the event data.
+        """
+
+        event = self.event_trace.emit()
+
+        event_dict: dict = {
+            'etw_id': event[0],
+            'pid': event[1]['ProcessId'],
+            'process_guid': event[1]['ProcessGuid'],
+            'image': event[1]['Image'],
+            'file_version': event[1]['FileVersion'],
+            'product': event[1]['Product'],
+            'company': event[1]['Company'],
+            'original_file_name': event[1]['OriginalFileName'],
+            'command_line': event[1]['CommandLine'],
+            'current_directory': event[1]['CurrentDirectory'],
+            'user': event[1]['User'],
+            'logon_id': event[1]['LogonId'],
+            'logon_guid': event[1]['LogonGuid'],
+            'terminal_session_id': event[1]['TerminalSessionId'],
+            'integrity_level': event[1]['IntegrityLevel'],
+            'hashes': event[1]['Hashes'],
+            'parent_process_guid': event[1]['ParentProcessGuid'],
+            'parent_process_id': event[1]['ParentProcessId'],
+            'parent_image': event[1]['ParentImage'],
+            'parent_command_line': event[1]['ParentCommandLine']
+        }
+
+        if self.attrs:
+            event_dict = dicts.reorder_keys(
+                event_dict, self.attrs, skip_keys_not_in_list=True)
+
+        return event_dict

atomicshop/monitor/change_monitor.py

@@ -41,7 +41,8 @@ class ChangeMonitor:
                     'url_playwright_jpeg'],
                 None] = None,
             object_type_settings: dict = None,
-            etw_session_name: str = None
+            etw_session_name: str = None,
+            etw_process_session_name: str = None
     ):
         """
         :param object_type: string, type of object to check. The type must be one of the following:
@@ -88,6 +89,8 @@ class ChangeMonitor:
             with logman and other tools: logman query -ets
             If not provided, a default name will be generated.
             'dns': 'AtomicShopDnsTrace'
+        :param etw_process_session_name: string, the name of the ETW session for tracing process creation.
+            This is needed to correlate the process cmd with the DNS requests PIDs.
 
         If 'input_directory' is not specified, the 'input_file_name' is not specified, and
         'generate_input_file_name' is False, then the input file will not be used and the object will be stored
@@ -104,6 +107,7 @@ class ChangeMonitor:
         self.object_type = object_type
         self.object_type_settings: dict = object_type_settings
         self.etw_session_name: str = etw_session_name
+        self.etw_process_session_name: str = etw_process_session_name
 
         # === Additional variables ========================================
 

atomicshop/monitor/checks/dns.py

@@ -1,14 +1,13 @@
 from pathlib import Path
 from typing import Union
 
-from ...etw.trace_dns import DnsTrace
+from ...etws.traces import trace_dns
 from ...print_api import print_api
 from ...import diff_check
 
 
 INPUT_FILE_DEFAULT_NAME: str = 'known_domains.json'
 INPUT_STATISTICS_FILE_DEFAULT_NAME: str = 'dns_statistics.json'
-ETW_DEFAULT_SESSION_NAME: str = 'AtomicShopDnsTrace'
 
 
 class DnsCheck:
@@ -22,15 +21,16 @@ class DnsCheck:
         self.diff_checker_statistics: Union[diff_check.DiffChecker, None] = None
         self.settings: dict = change_monitor_instance.object_type_settings
 
-        if change_monitor_instance.etw_session_name:
-            self.etw_session_name: str = change_monitor_instance.etw_session_name
-        else:
-            self.etw_session_name: str = ETW_DEFAULT_SESSION_NAME
+        self.etw_session_name: str = change_monitor_instance.etw_session_name
 
-        self.fetch_engine: DnsTrace = (
-            DnsTrace(
-                enable_process_poller=True, attrs=['name', 'cmdline', 'domain', 'query_type'],
-                session_name=self.etw_session_name, close_existing_session_name=True)
+        self.fetch_engine: trace_dns.DnsRequestResponseTrace = (
+            trace_dns.DnsRequestResponseTrace(
+                attrs=['name', 'cmdline', 'domain', 'query_type'],
+                session_name=self.etw_session_name,
+                close_existing_session_name=True,
+                enable_process_poller=True,
+                process_poller_etw_session_name=change_monitor_instance.etw_process_session_name
+            )
         )
 
         if self.settings['alert_always'] and self.settings['alert_about_missing_entries_after_learning']:
@@ -106,7 +106,7 @@ class DnsCheck:
 
     def _aggregation_process(self, event_dict: dict, return_list: list, print_kwargs: dict = None):
         self.diff_checker_aggregation.check_object = [event_dict]
-    
+
         # Check if 'known_domains' list was updated from previous cycle.
         result, message = self.diff_checker_aggregation.check_list_of_dicts(
             sort_by_keys=['cmdline', 'name'], print_kwargs=print_kwargs)

atomicshop/process.py

@@ -11,7 +11,7 @@ from .basics import strings
 from .wrappers import ubuntu_terminal
 
 if os.name == 'nt':
-    from .process_poller import GetProcessList
+    from . import process_poller
 
 
 def is_command_exists(cmd: str) -> bool:
@@ -265,7 +265,7 @@ def match_pattern_against_running_processes_cmdlines(
     """
 
     # Get the list of all the currently running processes.
-    get_process_list = GetProcessList(get_method='pywin32', connect_on_init=True)
+    get_process_list = process_poller.GetProcessList(get_method='pywin32', connect_on_init=True)
     processes = get_process_list.get_processes(as_dict=False)
 
     # Iterate through all the current process, while fetching executable file 'name' and the command line.