atomicshop 2.17.2__py3-none-any.whl → 2.18.0__py3-none-any.whl

atomicshop/mitm/engines/__parent/parser___parent.py

@@ -10,6 +10,6 @@ class ParserParent:
 
     def parse(self):
         # This is general parser, so we don't parse anything and 'request_body_parsed' gets empty byte string.
-        self.class_client_message.request_body_parsed = b''
+        self.class_client_message.request_custom_parsed = b''
 
-        self.logger.info(f"Parsed: {self.class_client_message.request_body_parsed[0: 100]}...")
+        self.logger.info(f"Parsed: {self.class_client_message.request_custom_parsed[0: 100]}...")

atomicshop/mitm/engines/__parent/recorder___parent.py

@@ -1,35 +1,45 @@
 import os
 from datetime import datetime
 import json
+import queue
+import threading
 
 from ...shared_functions import build_module_names, create_custom_logger
 from ... import message, recs_files
 from .... import filesystem
-from ....file_io import file_io
+from ....file_io import jsons
+from ....print_api import print_api
 
 
 # The class that is responsible for Recording Requests / Responses.
 class RecorderParent:
 
-    def __init__(self, class_client_message: message.ClientMessage, record_path: str):
-        self.class_client_message: message.ClientMessage = class_client_message
+    # noinspection PyTypeChecker
+    def __init__(self, record_path: str):
         self.record_path: str = record_path
+
         self.file_extension: str = ".json"
         self.engine_name = None
         self.module_name = None
         self.engine_record_path: str = str()
         self.record_file_path: str = str()
+        self.class_client_message: message.ClientMessage = None
 
         self.logger = create_custom_logger()
 
         # Get engine name and module name
         self.get_engine_module()
-        # Build full file path.
-        self.build_record_full_file_path()
+
+        # Build the record path with file name
+        self.build_record_path_to_engine()
 
         # Create folder.
         filesystem.create_directory(self.engine_record_path)
 
+        # Initialize a queue to hold messages
+        self.message_queue: queue.Queue = queue.Queue()
+        self.recorder_worker_thread = None
+
     # "self.__module__" is fully qualified module name: classes.engines.ENGINE-NAME.MODULE-NAME
     def get_engine_module(self):
         _, self.engine_name, self.module_name = build_module_names(self.__module__)
@@ -43,12 +53,9 @@ class RecorderParent:
         # Formatting the date and time and converting it to string object
         day_time_format: str = now.strftime(recs_files.REC_FILE_DATE_TIME_FORMAT)
 
-        # Build the record path with file name
-        self.build_record_path_to_engine()
-
         # If HTTP Path is not defined, 'http_path' will be empty, and it will not interfere with file name.
         self.record_file_path: str = (
-            f"{self.engine_record_path}{os.sep}{day_time_format}_"
+            f"{self.engine_record_path}{os.sep}{day_time_format}_th{self.class_client_message.thread_id}_"
             f"{self.class_client_message.server_name}{self.file_extension}")
 
     def convert_messages(self):
@@ -59,27 +66,79 @@ class RecorderParent:
         # We need to check that the values that we want to convert aren't empty or 'None'.
         if self.class_client_message.request_raw_bytes:
             self.class_client_message.request_raw_hex = self.class_client_message.request_raw_bytes.hex()
-        if self.class_client_message.response_list_of_raw_bytes:
-            # We checked that the list isn't empty, now we check if the first value is not empty, since if we
-            # check it in the same expression as check for list is empty, we will get an exception since
-            # the list is empty.
-            if self.class_client_message.response_list_of_raw_bytes[0]:
-                for response_raw_bytes in self.class_client_message.response_list_of_raw_bytes:
-                    self.class_client_message.response_list_of_raw_hex.append(response_raw_bytes.hex())
-
-    def record(self):
+        if self.class_client_message.response_raw_bytes:
+            self.class_client_message.response_raw_hex = self.class_client_message.response_raw_bytes.hex()
+
+    def record(self, class_client_message: message.ClientMessage):
+        self.class_client_message = class_client_message
+
+        # Build full file path if it is not already built.
+        if not self.record_file_path:
+            self.build_record_full_file_path()
+
+        # Start the worker thread if it is not already running
+        if not self.recorder_worker_thread:
+            self.recorder_worker_thread = threading.Thread(
+                target=save_message_worker,
+                args=(self.record_file_path, self.message_queue, self.logger),
+                name=f"Thread-{self.class_client_message.thread_id}_Recorder",
+                daemon=True
+            )
+            self.recorder_worker_thread.start()
+
         self.logger.info("Recording Message...")
 
         # Convert the requests and responses to hex.
         self.convert_messages()
         # Get the message in dict / JSON format
         record_message_dict: dict = dict(self.class_client_message)
-        recorded_message_json_string = json.dumps(record_message_dict)
 
-        # Since we already dumped the object to dictionary string, we'll just save the object to regular file.
-        file_io.write_file(
-            recorded_message_json_string, self.record_file_path, enable_long_file_path=True, **{'logger': self.logger})
-
-        self.logger.info(f"Recorded to file: {self.record_file_path}")
+        # Put the message in the queue to be processed by the worker thread
+        self.message_queue.put(record_message_dict)
 
         return self.record_file_path
+
+
+def save_message_worker(
+        record_file_path: str,
+        message_queue: queue.Queue,
+        logger
+):
+    """Worker function to process messages from the queue and write them to the file."""
+    while True:
+        # Get a message from the queue
+        record_message_dict = message_queue.get()
+
+        # Check for the "stop" signal
+        if record_message_dict is None:
+            break
+
+        # Read existing data from the file
+        try:
+            with open(record_file_path, 'r') as f:
+                current_json_file = json.load(f)
+        except FileNotFoundError:
+            current_json_file: list = []
+
+        # Append the new message to the existing data
+        final_json_list_of_dicts: list[dict] = []
+        if isinstance(current_json_file, list):
+            current_json_file.append(record_message_dict)
+            final_json_list_of_dicts = current_json_file
+        elif isinstance(current_json_file, dict):
+            final_json_list_of_dicts.append(current_json_file)
+            final_json_list_of_dicts.append(record_message_dict)
+        else:
+            error_message = "The current file is neither a list nor a dictionary."
+            print_api(error_message, logger_method="critical", logger=logger)
+            raise TypeError(error_message)
+
+        # Write the data back to the file
+        jsons.write_json_file(
+            final_json_list_of_dicts, record_file_path, indent=2,
+            enable_long_file_path=True, **{'logger': logger})
+
+        logger.info(f"Recorded to file: {record_file_path}")
+
+        # Indicate task completion
+        message_queue.task_done()

atomicshop/mitm/engines/__reference_general/recorder___reference_general.py

@@ -7,7 +7,7 @@ from atomicshop.mitm.message import ClientMessage
 class RecorderGeneral(RecorderParent):
     """The class that is responsible for Recording Requests / Responses"""
     # When initializing main classes through "super" you need to pass parameters to init
-    def __init__(self, class_client_message: ClientMessage, record_path):
-        super().__init__(class_client_message, record_path)
+    def __init__(self, record_path):
+        super().__init__(record_path)
 
         self.logger = create_custom_logger()

atomicshop/mitm/message.py

@@ -8,15 +8,18 @@ from ..basics import dicts
 class ClientMessage:
     """ A class that will store all the message details from the client """
     def __init__(self):
-        self.request_raw_bytes: bytearray = bytearray()
         # noinspection PyTypeChecker
-        self.request_time_received: datetime = None
-        self.request_raw_decoded: Union[http_parse.HTTPRequestParse, any] = None
-        self.request_body_parsed = None
+        self.timestamp: datetime = None
+        # noinspection PyTypeChecker
+        self.request_raw_bytes: bytes = None
+        self.request_auto_parsed: Union[http_parse.HTTPRequestParse, any] = None
+        self.request_custom_parsed: any = None
         self.request_raw_hex: hex = None
-        self.response_list_of_raw_bytes: list = list()
-        self.response_list_of_raw_decoded: list = list()
-        self.response_list_of_raw_hex: list = list()
+        # noinspection PyTypeChecker
+        self.response_raw_bytes: bytes = None
+        self.response_auto_parsed: any = None
+        self.response_custom_parsed: any = None
+        self.response_raw_hex: hex = None
         self.server_name: str = str()
         self.server_ip: str = str()
         self.client_ip: str = str()
@@ -25,26 +28,48 @@ class ClientMessage:
         self.process_name: str = str()
         self.thread_id = None
         self.info: str = str()
-        self.error: str = str()
+        self.errors: list = list()
         self.protocol: str = str()
+        self.protocol2: str = str()
         self.recorded_file_path: str = str()
+        self.action: str = str()
+
+    def reinitialize_dynamic_vars(self):
+        """
+        Reinitialize the dynamic variables of the class for the new cycle.
+        """
+        self.request_raw_bytes = None
+        self.timestamp = None
+        self.request_auto_parsed = None
+        self.request_custom_parsed = None
+        self.request_raw_hex = None
+        self.response_raw_bytes = None
+        self.response_auto_parsed = None
+        self.response_custom_parsed = None
+        self.response_raw_hex = None
+        self.action = None
+        self.info = str()
+        self.errors = list()
+        self.protocol = str()
+        self.protocol2 = str()
+        self.recorded_file_path = str()
 
     def __iter__(self):
         # __dict__ returns a dictionary containing the instance's attributes
         for key, value in self.__dict__.items():
             if key == 'request_raw_bytes':
                 value = str(value)
-            elif key == 'request_time_received':
+            elif key == 'timestamp':
                 value = value.strftime('%Y-%m-%d-%H:%M:%S.%f')
-            elif key == 'request_raw_decoded':
+            elif key == 'request_auto_parsed':
                 if isinstance(value, http_parse.HTTPRequestParse):
                     value = dicts.convert_complex_object_to_dict(value)
                 else:
                     value = str(value)
-            elif key == 'request_body_parsed':
+            elif key == 'request_custom_parsed':
+                value = dicts.convert_complex_object_to_dict(value)
+            elif key == 'response_raw_bytes':
+                value = str(value)
+            elif key == 'response_auto_parsed':
                 value = dicts.convert_complex_object_to_dict(value)
-            elif key == 'response_list_of_raw_bytes':
-                value = [str(bytes_response) for bytes_response in value]
-            elif key == 'response_list_of_raw_decoded':
-                value = [dicts.convert_complex_object_to_dict(complex_response) for complex_response in value]
             yield key, value

atomicshop/mitm/mitm_main.py

@@ -272,7 +272,7 @@ def mitm_server(config_file_path: str):
             time.sleep(1)
             return 1
 
-        dns_thread = threading.Thread(target=dns_server_instance.start)
+        dns_thread = threading.Thread(target=dns_server_instance.start, name="dns_server")
         dns_thread.daemon = True
         dns_thread.start()
 
@@ -376,7 +376,8 @@ def mitm_server(config_file_path: str):
             kwargs={
                 'reference_function_name': thread_worker_main,
                 'reference_function_args': (network_logger, statistics_writer, engines_list, reference_module,)
-            }
+            },
+            name="accepting_loop"
         )
 
         socket_thread.daemon = True

atomicshop/mitm/recs_files.py

@@ -7,7 +7,8 @@ from .. import filesystem
 from .. wrappers.loggingw import consts
 
 
-REC_FILE_DATE_TIME_FORMAT: str = f'{consts.DEFAULT_ROTATING_SUFFIXES_FROM_WHEN["S"]}_%f'
+REC_FILE_DATE_TIME_MILLISECONDS_FORMAT: str = f'{consts.DEFAULT_ROTATING_SUFFIXES_FROM_WHEN["S"]}_%f'
+REC_FILE_DATE_TIME_FORMAT: str = f'{consts.DEFAULT_ROTATING_SUFFIXES_FROM_WHEN["S"]}'
 REC_FILE_DATE_FORMAT: str = REC_FILE_DATE_TIME_FORMAT.split('_')[0]
 
 

atomicshop/system_resource_monitor.py

@@ -111,10 +111,18 @@ class SystemResourceMonitor:
         # The shared results' dictionary.
         self.results: dict = {}
 
-    def start(self, print_kwargs: dict = None):
+    def start(
+            self,
+            print_kwargs: dict = None,
+            thread_as_daemon: bool = True
+    ):
         """
         Start the monitoring process.
         :param print_kwargs:
+        :param thread_as_daemon: bool, set the thread as daemon. If you're running the monitoring process in the main
+            process, set it to True. If you're running the monitoring process in a separate process, set it to False.
+            In child processes created by multiprocessing.Process, the thread works differently. You might not
+            get the desired result.
         :return:
         """
 
@@ -127,7 +135,7 @@ class SystemResourceMonitor:
                 self.interval, self.get_cpu, self.get_memory, self.get_disk_io_bytes, self.get_disk_files_count,
                 self.get_disk_busy_time, self.get_disk_used_percent, self.calculate_maximum_changed_disk_io,
                 self.maximum_disk_io, self.queue_list, self.manager_dict))
-            self.thread.daemon = True
+            self.thread.daemon = thread_as_daemon
             self.thread.start()
         else:
             print_api.print_api("Monitoring is already running.", color='yellow', **print_kwargs)
@@ -205,6 +213,7 @@ def start_monitoring(
         calculate_maximum_changed_disk_io: bool = False,
         queue_list: list = None,
         manager_dict: multiprocessing.managers.DictProxy = None,      # multiprocessing.Manager().dict()
+        get_results_thread_as_daemon: bool = True,
         print_kwargs: dict = None
 ):
     """
@@ -235,6 +244,10 @@ def start_monitoring(
             multiprocessing.Process(
                 target=system_resource_monitor.start_monitoring, kwargs={'manager_dict': shared_dict}).start()
 
+    :param get_results_thread_as_daemon: bool, set the thread as daemon. If you're running the monitoring process in the
+        main process, set it to True. If you're running the monitoring process in a separate process, set it to False.
+        In child processes created by multiprocessing.Process, the thread works differently.
+        You might not get the desired result.
     :param print_kwargs: dict, print kwargs.
     :return:
     """
@@ -257,7 +270,7 @@ def start_monitoring(
             queue_list=queue_list,
             manager_dict=manager_dict
         )
-        SYSTEM_RESOURCES_MONITOR.start()
+        SYSTEM_RESOURCES_MONITOR.start(thread_as_daemon=get_results_thread_as_daemon)
     else:
         print_api.print_api("System resources monitoring is already running.", color='yellow', **(print_kwargs or {}))
 

atomicshop/wrappers/socketw/base.py

@@ -70,3 +70,20 @@ def get_default_ip_address() -> str:
     :return: string.
     """
     return socket.gethostbyname(socket.gethostname())
+
+
+def is_socket_closed(socket_object) -> bool:
+    """
+    Check if the socket is closed.
+    :param socket_object: socket object or ssl socket object.
+    :return: bool.
+    """
+    try:
+        # If the socket is closed, the fileno() method will raise an exception or return -1.
+
+        if socket_object.fileno() == -1:
+            return True
+        else:
+            return False
+    except socket.error:
+        return False

atomicshop/wrappers/socketw/creator.py

@@ -1,3 +1,4 @@
+import os
 import socket
 import ssl
 
@@ -33,8 +34,31 @@ def create_ssl_context_for_server():
     # return ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
 
 
-def create_ssl_context_for_client():
-    return ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+def create_ssl_context_for_client(
+        enable_sslkeylogfile_env_to_client_ssl_context: bool = False
+) -> ssl.SSLContext:
+    """
+    This function creates the SSL context for the client.
+    The SSL context is created with the "PROTOCOL_TLS_CLIENT" protocol.
+
+    :param enable_sslkeylogfile_env_to_client_ssl_context: boolean, enables the SSLKEYLOGFILE environment variable
+        to the SSL context. Default is False.
+        if True, SSLKEYLOGFILE will be added to SSL context with:
+        ssl_context.keylog_filename = os.environ.get('SSLKEYLOGFILE')
+        This is useful for debugging SSL/TLS connections with WireShark.
+        Since WireShark also uses this environment variable to read the key log file and apply to the SSL/TLS
+        connections, so you can see the decrypted traffic.
+
+    :return: ssl.SSLContext
+    """
+    ssl_context: ssl.SSLContext = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+
+    if enable_sslkeylogfile_env_to_client_ssl_context:
+        ssl_key_logfile = os.environ.get('SSLKEYLOGFILE')
+        if ssl_key_logfile:
+            ssl_context.keylog_filename = ssl_key_logfile
+
+    return ssl_context
 
 
 def set_client_ssl_context_ca_default_certs(ssl_context):
@@ -204,7 +228,8 @@ def set_listen_on_socket(socket_object, **kwargs):
 def wrap_socket_with_ssl_context_client___default_certs___ignore_verification(
         socket_object,
         server_hostname: str = None,
-        custom_pem_client_certificate_file_path: str = None
+        custom_pem_client_certificate_file_path: str = None,
+        enable_sslkeylogfile_env_to_client_ssl_context: bool = False
 ):
     """
     This function is a preset for wrapping the socket with SSL context for the client.
@@ -214,8 +239,11 @@ def wrap_socket_with_ssl_context_client___default_certs___ignore_verification(
     :param server_hostname: string, hostname of the server. Default is None.
     :param custom_pem_client_certificate_file_path: string, full file path for the client certificate PWM file.
         Default is None.
+    :param enable_sslkeylogfile_env_to_client_ssl_context: boolean, enables the SSLKEYLOGFILE environment variable
+        to the SSL context. Default is False.
     """
-    ssl_context: ssl.SSLContext = create_ssl_context_for_client()
+    ssl_context: ssl.SSLContext = create_ssl_context_for_client(
+        enable_sslkeylogfile_env_to_client_ssl_context=enable_sslkeylogfile_env_to_client_ssl_context)
     set_client_ssl_context_ca_default_certs(ssl_context)
     set_client_ssl_context_certificate_verification_ignore(ssl_context)