atomicshop 2.19.10__py3-none-any.whl → 2.19.11__py3-none-any.whl

atomicshop/__init__.py

@@ -1,4 +1,4 @@
 """Atomic Basic functions and classes to make developer life easier"""
 
 __author__ = "Den Kras"
-__version__ = '2.19.10'
+__version__ = '2.19.11'

atomicshop/a_installs/win/robocorp.py

@@ -41,6 +41,9 @@ def main():
     print_api("Initializing Robocorp Browser.")
     subprocess.check_call(["rfbrowser", "init"])
 
+    print_api("Installing Additional modules.")
+    subprocess.check_call(["pip", "install", "--upgrade", "matplotlib", "imagehash"])
+
     print_api("Installing Tesseract OCR.")
     github_wrapper = githubw.GitHubWrapper(
         user_name="tesseract-ocr",

atomicshop/etws/trace.py

@@ -83,7 +83,7 @@ class EventTrace(etw.ETW):
 
         self.self_hosted_poller: bool = False
         if self.enable_process_poller:
-            if not self.process_pool_shared_dict_proxy:
+            if self.process_pool_shared_dict_proxy is None:
                 self.self_hosted_poller = True
                 self.process_poller = simple_process_pool.SimpleProcessPool()
                 self.multiprocessing_manager: multiprocessing.managers.SyncManager = multiprocessing.Manager()
@@ -98,7 +98,7 @@ class EventTrace(etw.ETW):
         )
 
     def start(self):
-        if self.enable_process_poller:
+        if self.enable_process_poller and self.self_hosted_poller:
             self.process_poller.start()
 
         # Check if the session name already exists.
@@ -123,11 +123,10 @@ class EventTrace(etw.ETW):
     def stop(self):
         super().stop()
 
-        if self.enable_process_poller:
+        if self.self_hosted_poller:
             self.process_poller.stop()
 
-            if self.self_hosted_poller:
-                self.multiprocessing_manager.shutdown()
+            self.multiprocessing_manager.shutdown()
 
     def emit(self):
         """
@@ -146,32 +145,21 @@ class EventTrace(etw.ETW):
         :return: etw event object.
         """
 
-        # Get the processes first, since we need the process name and command line.
-        # If they're not ready, we will get just pids from DNS tracing.
-        if self.enable_process_poller:
-            self._get_processes_from_poller()
-
         event: tuple = self.event_queue.get()
 
         event_dict: dict = {
             'EventId': event[0],
-            'EventHeader': event[1],
-            'pid': event[1]['EventHeader']['ProcessId']
+            'EventHeader': event[1]
         }
 
+        if 'ProcessId' not in event[1]:
+            event_dict['pid'] = event[1]['EventHeader']['ProcessId']
+        else:
+            event_dict['pid'] = event[1]['ProcessId']
+
         if self.enable_process_poller:
             process_info: dict = self.pid_process_converter.get_process_by_pid(event_dict['pid'])
             event_dict['name'] = process_info['name']
             event_dict['cmdline'] = process_info['cmdline']
 
         return event_dict
-
-    def _get_processes_from_poller(self):
-        processes: dict = {}
-        while not processes:
-            processes = self.process_poller.get_processes()
-
-            if isinstance(processes, BaseException):
-                raise processes
-
-        return processes

atomicshop/etws/traces/trace_dns.py

@@ -47,7 +47,7 @@ class DnsRequestResponseTrace:
 
 
             dns_trace_w = dns_trace.DnsTrace(
-                attrs=['pid', 'name', 'cmdline', 'domain', 'query_type'],
+                attrs=['pid', 'name', 'cmdline', 'query', 'query_type'],
                 session_name='MyDnsTrace',
                 close_existing_session_name=True,
                 enable_process_poller=True

atomicshop/etws/traces/trace_tcp.py

@@ -0,0 +1,130 @@
+import multiprocessing.managers
+
+from .. import trace, const, providers
+from ...basics import dicts
+from ... import dns, ip_addresses
+
+
+ETW_DEFAULT_SESSION_NAME: str = 'AtomicShopTcpTrace'
+
+PROVIDER_NAME: str = "Microsoft-Windows-TCPIP"
+PROVIDER_GUID: str = '{' + providers.get_provider_guid_by_name(PROVIDER_NAME) + '}'
+REQUEST_RESP_EVENT_ID: int = 1033
+
+
+class TcpIpNewConnectionsTrace:
+    """
+    TcpIpNewConnectionsTrace class use to trace new connection events from Windows Event Tracing:
+    Provider: Microsoft-Windows-TCPIP
+    EventId: 1033
+    """
+    def __init__(
+            self,
+            attrs: list = None,
+            session_name: str = None,
+            close_existing_session_name: bool = True,
+            process_pool_shared_dict_proxy: multiprocessing.managers.DictProxy = None
+    ):
+        """
+        :param attrs: List of attributes to return. If None, all attributes will be returned.
+        :param session_name: The name of the session to create. If not provided, a UUID will be generated.
+        :param close_existing_session_name: Boolean to close existing session names.
+            True: if ETW session with 'session_name' exists, you will be notified and the session will be closed.
+                Then the new session with this name will be created.
+            False: if ETW session with 'session_name' exists, you will be notified and the new session will not be
+                created. Instead, the existing session will be used. If there is a buffer from the previous session,
+                you will get the events from the buffer.
+        :param process_pool_shared_dict_proxy: multiprocessing.managers.DictProxy, multiprocessing shared dict proxy
+            that contains current processes.
+            Check the 'atomicshop\process_poller\simple_process_pool.py' SimpleProcessPool class for more information.
+
+            For this specific class it means that you can run the process poller outside of this class and pass the
+            'process_pool_shared_dict_proxy' to this class. Then you can get the process name and command line for
+            the DNS events from the 'process_pool_shared_dict_proxy' and use it also in other classes.
+
+        -------------------------------------------------
+
+        Usage Example:
+            from atomicshop.etw import tcp_trace
+
+
+            tcp_trace_w = tcp_trace.TcpIpNewConnectionsTrace(
+                attrs=['pid', 'name', 'cmdline', 'domain', 'query_type'],
+                session_name='MyTcpTrace',
+                close_existing_session_name=True,
+                enable_process_poller=True
+            )
+            tcp_trace_w.start()
+            while True:
+                tcp_dict = tcp_trace_w.emit()
+                print(tcp_dict)
+            tcp_trace_w.stop()
+        """
+
+        self.attrs = attrs
+        self.process_pool_shared_dict_proxy: multiprocessing.managers.DictProxy = process_pool_shared_dict_proxy
+
+        if not session_name:
+            session_name = ETW_DEFAULT_SESSION_NAME
+
+        self.event_trace = trace.EventTrace(
+            providers=[(PROVIDER_NAME, PROVIDER_GUID)],
+            # lambda x: self.event_queue.put(x),
+            event_id_filters=[REQUEST_RESP_EVENT_ID],
+            session_name=session_name,
+            close_existing_session_name=close_existing_session_name,
+            enable_process_poller=True,
+            process_pool_shared_dict_proxy=self.process_pool_shared_dict_proxy
+        )
+
+    def start(self):
+        self.event_trace.start()
+
+    def stop(self):
+        self.event_trace.stop()
+
+    def emit(self):
+        """
+        Function that will return the next event from the queue.
+        The queue is blocking, so if there is no event in the queue, the function will wait until there is one.
+
+        Usage Example:
+            while True:
+                tcp_dict = tcp_trace.emit()
+                print(tcp_dict)
+
+        :return: Dictionary with the event data.
+        """
+
+        # Get the event from ETW as is.
+        event = self.event_trace.emit()
+
+        local_address_port: str = event['EventHeader']['LocalAddress']
+        remote_address_port: str = event['EventHeader']['RemoteAddress']
+
+        if 'ffff' in local_address_port:
+            pass
+
+        local_address, local_port = local_address_port.rsplit(':', 1)
+        local_address = local_address.replace('[', '').replace(']', '')
+
+        remote_address, remote_port = remote_address_port.rsplit(':', 1)
+        remote_address = remote_address.replace('[', '').replace(']', '')
+
+        event_dict: dict = {
+            'event_id': event['EventId'],
+            'local_ip': local_address,
+            'local_port': local_port,
+            'remote_ip': remote_address,
+            'remote_port': remote_port,
+            'status': event['EventHeader']['Status'],
+            'pid': event['pid'],
+            'name': event['name'],
+            'cmdline': event['cmdline']
+        }
+
+        if self.attrs:
+            event_dict = dicts.reorder_keys(
+                event_dict, self.attrs, skip_keys_not_in_list=True)
+
+        return event_dict

atomicshop/process_poller/simple_process_pool.py

@@ -23,6 +23,40 @@ class SimpleProcessPool:
     The idea is similar to the process_poller.process_pool.ProcessPool class, but this class is simpler and uses
     only the pywin32 tracing of the Windows Event Log Process Creation and Process Termination events.
     The simple process pool is used to get things simpler than the process_pool.ProcessPool class.
+
+    Example of starting the process pool in multiprocess:
+        import sys
+
+        from atomicshop.process_poller import simple_process_pool
+
+
+        def start_process_pool(process_pool_shared_dict_proxy):
+            process_poller = simple_process_pool.SimpleProcessPool(
+                process_pool_shared_dict_proxy=process_pool_shared_dict_proxy)
+            process_poller.start()
+
+            try:
+                # Keep the process alive.
+                while True:
+                    time.sleep(1)
+            except KeyboardInterrupt:
+                process_poller.stop()
+
+
+        def main():
+            # Create the shared multiprocessing dictionary of the process pool.
+            manager = multiprocessing.Manager()
+            multiprocess_dict_proxy = manager.dict()
+
+            # Start the process pool in a separate process.
+            pool_process = multiprocessing.Process(target=start_process_pool, args=(multiprocess_dict_proxy,))
+            pool_process.start()
+
+            # Pass the shared dict proxy to other functions.
+
+
+        if __name__ == '__main__':
+            sys.exit(main())
     """
 
     def __init__(
@@ -79,7 +113,7 @@ class SimpleProcessPool:
         self._processes = get_process_list.GetProcessList(
             get_method='pywin32', connect_on_init=True).get_processes(as_dict=True)
 
-        thread_get_queue = threading.Thread(target=self._start_main_thread)
+        thread_get_queue = threading.Thread(target=self._start_main_thread, args=(self.process_pool_shared_dict_proxy,))
         thread_get_queue.daemon = True
         thread_get_queue.start()
 
@@ -93,7 +127,7 @@ class SimpleProcessPool:
     def get_processes(self):
         return self._processes
 
-    def _start_main_thread(self):
+    def _start_main_thread(self, process_pool_shared_dict_proxy):
         get_instance = process_create.ProcessCreateSubscriber()
         get_instance.start()
 
@@ -109,9 +143,9 @@ class SimpleProcessPool:
             }
 
             # Update the multiprocessing shared dict proxy.
-            if self.process_pool_shared_dict_proxy:
-                self.process_pool_shared_dict_proxy.clear()
-                self.process_pool_shared_dict_proxy.update(self._processes)
+            if process_pool_shared_dict_proxy is not None:
+                process_pool_shared_dict_proxy.clear()
+                process_pool_shared_dict_proxy.update(self._processes)
 
             # print_api(f'Process [{process_id}] added to the pool.', color='blue')
 
@@ -186,6 +220,7 @@ class PidProcessConverter:
         process_dict: dict = dict()
         while counter < WAIT_FOR_PROCESS_POLLER_PID_COUNTS:
             if pid not in self.process_pool_shared_dict_proxy:
+                # print(dict(self.process_pool_shared_dict_proxy))
                 time.sleep(0.1)
                 counter += 1
             else:
@@ -193,6 +228,7 @@ class PidProcessConverter:
                 break
 
         if counter == WAIT_FOR_PROCESS_POLLER_PID_COUNTS and not process_dict:
+            print_api(f"Error: The PID [{pid}] is not in the pool, trying DLL snapshot.", color='yellow')
             # Last resort, try to get the process name by current process snapshot.
             processes = self.get_process_with_dll_instance.get_process_details(as_dict=True)
             if pid not in processes:

{atomicshop-2.19.10.dist-info → atomicshop-2.19.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: atomicshop
-Version: 2.19.10
+Version: 2.19.11
 Summary: Atomic functions and classes to make developer life easier
 Author: Denis Kras
 License: MIT License

{atomicshop-2.19.10.dist-info → atomicshop-2.19.11.dist-info}/RECORD

@@ -1,4 +1,4 @@
-atomicshop/__init__.py,sha256=G7ypN-uA8G5CeP00W-dT6ZTgWzEn9Gvxo-GY0WN0GYs,124
+atomicshop/__init__.py,sha256=mHi1iDugkcLgj5LyCc_4EL-QVNq95QCoQWv522BGF1c,124
 atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
 atomicshop/_create_pdf_demo.py,sha256=Yi-PGZuMg0RKvQmLqVeLIZYadqEZwUm-4A9JxBl_vYA,3713
 atomicshop/_patch_import.py,sha256=ENp55sKVJ0e6-4lBvZnpz9PQCt3Otbur7F6aXDlyje4,6334
@@ -57,7 +57,7 @@ atomicshop/a_installs/win/fibratus.py,sha256=TU4e9gdZ_zI73C40uueJ59pD3qmN-UFGdX5
 atomicshop/a_installs/win/mongodb.py,sha256=AqyItXu19aaoe49pppDxtEkXey6PMy0PoT2Y_RmPpPE,179
 atomicshop/a_installs/win/nodejs.py,sha256=U519Dyt4bsQPbEg_PwnZL5tsbfqDr1BbhxwoQFZsSKo,200
 atomicshop/a_installs/win/pycharm.py,sha256=j_RSd7aDOyC3yDd-_GUTMLlQTmDrqtVFG--oUfGLiZk,140
-atomicshop/a_installs/win/robocorp.py,sha256=2E28iaRlAZROoxmXwiXv8rqTjVcdBT2UJ3B8nxrtmkc,2245
+atomicshop/a_installs/win/robocorp.py,sha256=ExdMR705fW4iyNDZnH9YZMrio4FYEwNH_8nEP1GWkjM,2383
 atomicshop/a_installs/win/wsl_ubuntu_lts.py,sha256=dZbPRLNKFeMd6MotjkE6UDY9cOiIaaclIdR1kGYWI50,139
 atomicshop/a_mains/dns_gateway_setting.py,sha256=ncc2rFQCChxlNP59UshwmTonLqC6MWblrVAzbbz-13M,149
 atomicshop/a_mains/github_wrapper.py,sha256=F-PoZknVCxWPN0PTO6l7ZNiaYvo7OVFKFI_zlPt56ps,169
@@ -117,10 +117,11 @@ atomicshop/etws/_pywintrace_fix.py,sha256=nHrtnAb796eOZ6FlCqcsuRh_TSqSPp6JXLN6TB
 atomicshop/etws/const.py,sha256=v3x_IdCYeSKbCGywiZFOZln80ldpwKW5nuMDuUe51Jg,1257
 atomicshop/etws/providers.py,sha256=CXNx8pYdjtpLIpA66IwrnE64XhY4U5ExnFBMLEb8Uzk,547
 atomicshop/etws/sessions.py,sha256=b_KeiOvgOBJezJokN81TRlrvJiQNJlIWN4Z6UVjuxP0,1335
-atomicshop/etws/trace.py,sha256=LC5b_WGwk5LTVggnQ55F4Iklis7B7hVFWusOt1jEHnI,8068
+atomicshop/etws/trace.py,sha256=p-5wegZ-5SuJ3UsprCCnlor8g2LPkF9ZOynLCYaEvXQ,7647
 atomicshop/etws/traces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-atomicshop/etws/traces/trace_dns.py,sha256=0a8zoq59NyJqy6Z6lf1qkvxrHcrCSnyYEAfLfBdJIuI,6880
+atomicshop/etws/traces/trace_dns.py,sha256=SWeex1mK7rrMzCmmL3qiGaD_jzbRiiwJ_mhMZz5TwJc,6879
 atomicshop/etws/traces/trace_sysmon_process_creation.py,sha256=OM-bkK38uYMwWLZKNOTDa0Xdk3sO6sqsxoMUIiPvm5g,4656
+atomicshop/etws/traces/trace_tcp.py,sha256=pQF4A_qM-HjEhzMQUyTmUqOWdvSaqqAg5PwnkVn2fy4,5236
 atomicshop/file_io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/file_io/csvs.py,sha256=zv0kKjRT-ZWRi0WpMIUQ_FKyP9Dt0f5Bc98Qsj6ClPU,9495
 atomicshop/file_io/docxs.py,sha256=Nyt3hSpzwqUKZEP5p5efqNpjFs9XqkK40Kp7BbbPo7E,6245
@@ -168,7 +169,7 @@ atomicshop/permissions/ubuntu_permissions.py,sha256=n8z1vcIXDts4zLVue33dtJiTopjg
 atomicshop/permissions/win_permissions.py,sha256=eDQm1jfK9x_hkbLqIJjFTwfqinAWQ0iSr0kW3XrF1BE,1272
 atomicshop/process_poller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/process_poller/process_pool.py,sha256=4Qs427qd7OcBxu5PMFU5PTmyuxRy0vgj2GLsRt0IoEw,9565
-atomicshop/process_poller/simple_process_pool.py,sha256=tbeuw30MvNFbore1YooEra1ozZvFR8maKbcGNlBpBKc,8484
+atomicshop/process_poller/simple_process_pool.py,sha256=E0w66imLQQ0phHuTQfTaApOTurX0duE9vw1trjUY7rI,9894
 atomicshop/process_poller/tracer_base.py,sha256=IOiHcnmF-MccOSCErixN5mve9RifZ9cPnGVHCIRchrs,1091
 atomicshop/process_poller/pollers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/process_poller/pollers/psutil_pywin32wmi_dll.py,sha256=XRRfOIy62iOYU8IKRcyECWiL0rqQ35DeYbPsv_SHDVM,4510
@@ -327,8 +328,8 @@ atomicshop/wrappers/socketw/statistics_csv.py,sha256=fgMzDXI0cybwUEqAxprRmY3lqbh
 atomicshop/wrappers/winregw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/wrappers/winregw/winreg_installed_software.py,sha256=Qzmyktvob1qp6Tjk2DjLfAqr_yXV0sgWzdMW_9kwNjY,2345
 atomicshop/wrappers/winregw/winreg_network.py,sha256=AENV88H1qDidrcpyM9OwEZxX5svfi-Jb4N6FkS1xtqA,8851
-atomicshop-2.19.10.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
-atomicshop-2.19.10.dist-info/METADATA,sha256=Ln3I--iPj8FQjdK02tvhk-uo1MQQrkm4MLKC17rYvXc,10631
-atomicshop-2.19.10.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-atomicshop-2.19.10.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
-atomicshop-2.19.10.dist-info/RECORD,,
+atomicshop-2.19.11.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
+atomicshop-2.19.11.dist-info/METADATA,sha256=tSnPcFSBceRSRX3c335xVMPu5zruA6fmE8ifV12MW-8,10631
+atomicshop-2.19.11.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+atomicshop-2.19.11.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
+atomicshop-2.19.11.dist-info/RECORD,,