atomicshop 2.15.11__py3-none-any.whl → 3.10.5__py3-none-any.whl

atomicshop/mitm/initialize_engines.py

@@ -1,112 +1,189 @@
 import os
-import sys
 from pathlib import Path
 
-from .. import filesystem
+from .. import ip_addresses
 from ..file_io import tomls
 from ..basics.classes import import_first_class_name_from_file_path
-from ..wrappers.loggingw import loggingw
-from .engines.__reference_general import parser___reference_general, responder___reference_general, \
-    recorder___reference_general
+from .engines.__reference_general import parser___reference_general, requester___reference_general, \
+    responder___reference_general, recorder___reference_general
 
 
 class ModuleCategory:
     def __init__(self, script_directory: str):
-        self.domain_list: list = list()
         self.engine_name: str = str()
         self.script_directory: str = script_directory
 
+        self.domain_list: list = list()
+        self.domain_target_dict: dict = dict()
+        self.port_target_dict: dict = dict()
+
+        self.is_localhost: bool = bool()
+        self.on_port_connect: dict = dict()
+        self.mtls: dict = dict()
+
         self.parser_file_path: str = str()
+        self.requester_file_path: str = str()
         self.responder_file_path: str = str()
         self.recorder_file_path: str = str()
 
         self.parser_class_object: str = str()
+        self.requester_class_object: str = str()
         self.responder_class_object: str = str()
         self.recorder_class_object: str = str()
 
-        # The instance of the recorder class that will be initiated once in the script start
-        self.responder_instance = None
-
     def fill_engine_fields_from_general_reference(self, engines_fullpath: str):
         # Reference module variables.
         self.engine_name = '__reference_general'
         reference_folder_path: str = engines_fullpath + os.sep + self.engine_name
         # Full path to file.
         self.parser_file_path = reference_folder_path + os.sep + "parser___reference_general.py"
+        self.requester_file_path = reference_folder_path + os.sep + "requester___reference_general.py"
         self.responder_file_path = reference_folder_path + os.sep + "responder___reference_general.py"
         self.recorder_file_path = reference_folder_path + os.sep + "recorder___reference_general.py"
 
-    def fill_engine_fields_from_config(self, engine_config_file_path: str):
+    def fill_engine_fields_from_config(
+            self,
+            engine_config_file_path: str,
+            print_kwargs: dict = None
+    ) -> tuple[int, str]:
         # Read the configuration file of the engine.
-        configuration_data = tomls.read_toml_file(engine_config_file_path)
+        configuration_data = tomls.read_toml_file(engine_config_file_path, **(print_kwargs or {}))
 
         engine_directory_path: str = str(Path(engine_config_file_path).parent)
         self.engine_name = Path(engine_directory_path).name
 
         # Getting the parameters from engine config file
-        self.domain_list = configuration_data['domains']
+        self.domain_list = configuration_data['engine']['domains']
+
+        if 'on_port_connect' in configuration_data:
+            self.on_port_connect = configuration_data['on_port_connect']
+
+        if 'mtls' in configuration_data:
+            self.mtls = configuration_data['mtls']
 
         # If there's module configuration file, but no domains in it, there's no point to continue.
         # Since, each engine is based on domains.
         if not self.domain_list or self.domain_list[0] == '':
             raise ValueError(f"Engine Configuration file doesn't contain any domains: {engine_config_file_path}")
 
-        # Full path to file
-        self.parser_file_path = filesystem.get_file_paths_from_directory(
-            engine_directory_path, file_name_check_pattern=configuration_data['parser_file'])[0]
-        self.responder_file_path = filesystem.get_file_paths_from_directory(
-            engine_directory_path, file_name_check_pattern=configuration_data['responder_file'])[0]
-        self.recorder_file_path = filesystem.get_file_paths_from_directory(
-            engine_directory_path, file_name_check_pattern=configuration_data['recorder_file'])[0]
-
-    def initialize_engine(self, logs_path: str, logger=None, reference_general: bool = False, **kwargs):
-        if not reference_general:
-            self.parser_class_object = import_first_class_name_from_file_path(
-                self.script_directory, self.parser_file_path, logger=logger, stdout=False)
-            self.responder_class_object = import_first_class_name_from_file_path(
-                self.script_directory, self.responder_file_path, logger=logger, stdout=False)
-            self.recorder_class_object = import_first_class_name_from_file_path(
-                self.script_directory, self.recorder_file_path, logger=logger, stdout=False)
+        # This is needed for backwards compatibility before glass 1.8.2, atomicshop 2.20.6
+        # When the name of each file was following the pattern: parser_<EngineName>.py, responder_<EngineName>.py, recorder_<EngineName>.py
+        if os.path.isfile(f"{engine_directory_path}{os.sep}parser.py"):
+            file_name_suffix: str = ''
         else:
-            self.parser_class_object = parser___reference_general.ParserGeneral
-            self.responder_class_object = responder___reference_general.ResponderGeneral
-            self.recorder_class_object = recorder___reference_general.RecorderGeneral
+            file_name_suffix: str = f"_{self.engine_name}"
 
+        # Full path to file
+        self.parser_file_path = f"{engine_directory_path}{os.sep}parser{file_name_suffix}.py"
+        self.requester_file_path = f"{engine_directory_path}{os.sep}requester{file_name_suffix}.py"
+        self.responder_file_path = f"{engine_directory_path}{os.sep}responder{file_name_suffix}.py"
+        self.recorder_file_path = f"{engine_directory_path}{os.sep}recorder{file_name_suffix}.py"
+
+        for domain_index, domain_port_string in enumerate(self.domain_list):
+            # Splitting the domain and port
+            if ':' in domain_port_string:
+                domain, port = domain_port_string.split(':')
+            else:
+                error_string: str = f"No [domain:port] pair found in: {domain_port_string}"
+                return 1, error_string
+
+            self.domain_target_dict[domain] = {'ip': None, 'port': int(port)}
+
+        for port, value in self.on_port_connect.items():
+            self.port_target_dict[port] = {'ip': None, 'port': int(port)}
+
+            # If it is not an IP address (e.g. <IP:PORT>) it will be treated as file path.
+            if ':' not in value:
+                self.on_port_connect[port] = f'{engine_directory_path}{os.sep}{value}'
+
+        for subdomain, file_name in self.mtls.items():
+            self.mtls[subdomain] = f'{engine_directory_path}{os.sep}{file_name}'
+
+        return 0, ''
+
+    def initialize_engine(
+            self,
+            reference_general: bool = False,
+            print_kwargs: dict = None
+    ) -> tuple[int, str]:
         try:
-            # Since we're using responder to aggregate requests to build responses based on several
-            # requests, we need to initiate responder's class only once in the beginning and assign
-            # this instance to a variable that will be called later per domain.
-            self.responder_instance = self.responder_class_object()
-        except Exception as exception_object:
-            logger.error_exception(f"Exception while initializing responder: {exception_object}")
-            sys.exit()
-
-        # Initiating logger for each engine by its name
-        # initiate_logger(current_module.engine_name, log_file_extension)
-        loggingw.create_logger(
-            logger_name=self.engine_name,
-            directory_path=logs_path,
-            add_stream=True,
-            add_timedfile=True,
-            formatter_streamhandler='DEFAULT',
-            formatter_filehandler='DEFAULT'
-        )
-
-
-# Assigning external class object by message domain received from client. If the domain is not in the list,
-# the reference general module will be assigned.
+            if not reference_general:
+                self.parser_class_object = import_first_class_name_from_file_path(
+                    self.script_directory, self.parser_file_path, **(print_kwargs or {}))
+                self.requester_class_object = import_first_class_name_from_file_path(
+                    self.script_directory, self.requester_file_path, **(print_kwargs or {}))
+                self.responder_class_object = import_first_class_name_from_file_path(
+                    self.script_directory, self.responder_file_path, **(print_kwargs or {}))
+                self.recorder_class_object = import_first_class_name_from_file_path(
+                    self.script_directory, self.recorder_file_path, **(print_kwargs or {}))
+            else:
+                self.parser_class_object = parser___reference_general.ParserGeneral
+                self.requester_class_object = requester___reference_general.RequesterGeneral
+                self.responder_class_object = responder___reference_general.ResponderGeneral
+                self.recorder_class_object = recorder___reference_general.RecorderGeneral
+        except ModuleNotFoundError as e:
+            return 1, str(e)
+
+        return 0, ''
+
+
+def get_ipv4_from_engine_on_connect_port(
+        address_or_file_path: str
+) -> tuple[str, str] | None:
+    """
+    Function to get the IPv4 address from the engine on connect port.
+
+    :param address_or_file_path: string, "ip_address:port" or file path that was set in the engine on_port_connect.
+    :return: string, IPv4 address that was parsed from the 'ip_port_address'.
+
+    """
+
+    def get_ip_port_from_address(ip_port_address: str) -> tuple[str, str] | None:
+        """
+        Function to get the IP address and port from the address string.
+        If the address is a file path, it will return an empty string.
+        """
+        if ':' in ip_port_address:
+            ipv4_to_connect, port_to_connect = ip_port_address.split(':')
+            if ip_addresses.is_ip_address(ipv4_to_connect, ip_type='ipv4'):
+                return ipv4_to_connect, port_to_connect
+            else:
+                return None
+        else:
+            return None
+
+    # Try to get it as IP address.
+    ip_port_address_from_config = get_ip_port_from_address(address_or_file_path)
+
+    # If it is not an IP address, try to read it as a text file.
+    if not ip_port_address_from_config:
+        if os.path.isfile(address_or_file_path):
+            with open(address_or_file_path, 'r', encoding='utf-8') as file:
+                first_line = file.readline().strip()
+
+            ip_port_address_from_config = get_ip_port_from_address(first_line)
+        else:
+            return None
+
+    return ip_port_address_from_config
+
+
 def assign_class_by_domain(
-        engines_list: list, message_domain_name: str, reference_module, config, logger=None):
-    # Defining return variables:
-    function_parser = None
-    function_responder = None
-    function_recorder = None
+        engines_list: list,
+        message_domain_name: str,
+        reference_module
+):
+    """
+    Assigning external class object by message domain received from client. If the domain is not in the list,
+    the reference general module will be assigned.
+    """
 
     # In case SNI came empty in the request from client, then there's no point in iterating through engine domains.
+    module = None
     if message_domain_name:
-        # If the engines_usage is set to True in the config file, then we'll iterate through the list of engines
+        # If engine/s exit, the engines_list will not be empty, then we'll iterate through the list of engines
         # to find the domain in the list of domains of the engine.
-        if config['tcp']['engines_usage']:
+        if engines_list:
             # Checking if current domain is in engines' domain list to activate domain specific engine
             for function_module in engines_list:
                 # The list: matches_list = ["domain1.com", "domain2.com", "domain3.com"]
@@ -117,31 +194,34 @@ def assign_class_by_domain(
                 # On the other hand if you want to find if partial string is
                 # in the list of strings: if any(a_string in x for x in matches_list):
                 # In this case list is the same and string: a_string = domain
-                if any(x in message_domain_name for x in function_module.domain_list):
-                    # Assigning modules by current engine of the domain
-                    function_parser = function_module.parser_class_object
-                    function_recorder = function_module.recorder_class_object
-                    # Since the responder is being initiated only once, we're assigning only the instance
-                    function_responder = function_module.responder_instance
-
-                    logger.info(f"Assigned Modules for [{message_domain_name}]: "
-                                f"{function_module.parser_class_object.__name__}, "
-                                f"{function_module.responder_class_object.__name__}, "
-                                f"{function_module.recorder_class_object.__name__}")
+                if any(x in message_domain_name for x in function_module.domain_target_dict.keys()):
+                    # Assigning module by current engine of the domain
+                    module = function_module
 
                     # If the domain was found in the current list of class domains, we can stop the loop
                     break
 
+                # If the module wasn't found by the domain, check it by the port.
+                if not module:
+                    # Get the list of all the ip addresses in the on_port_connect dict.
+                    list_of_ip_addresses_per_port: list[str] = []
+                    for port, value in function_module.on_port_connect.items():
+                        ipv4_to_connect, _ = get_ipv4_from_engine_on_connect_port(value)
+                        list_of_ip_addresses_per_port.append(ipv4_to_connect)
+
+                    # Checking if the message domain name is in the list of ip addresses per port.
+                    if any(x in message_domain_name for x in list_of_ip_addresses_per_port):
+                        # Assigning module by current engine of the port
+                        module = function_module
+
+                        # If the port was found in the current list of class ports, we can stop the loop
+                        break
+
     # If none of the domains were found in the engine domains list, then we'll assign reference module.
     # It's enough to check only parser, since responder and recorder also will be empty.
     # This section is also relevant if SNI came empty in the request from the client and no domain was passed by the
     # DNS Server.
-    if not function_parser:
-        # Assigning modules by current engine of the domain
-        function_parser = reference_module.parser_class_object
-        function_recorder = reference_module.recorder_class_object
-        # Since the responder is being initiated only once, we're assigning only the instance
-        function_responder = reference_module.responder_instance
-
-    # Return all the initiated modules
-    return function_parser, function_responder, function_recorder
+    if not module:
+        module = reference_module
+
+    return module

atomicshop/mitm/message.py

@@ -1,39 +1,80 @@
+from datetime import datetime
+from typing import Union, Any
+
+from .. import http_parse
+from ..basics import dicts
+
+
 class ClientMessage:
     """ A class that will store all the message details from the client """
     def __init__(self):
-        self.request_raw_bytes: bytearray = bytearray()
-        self.request_time_received = None
-        self.request_raw_decoded = None
-        self.request_body_parsed = None
+        # noinspection PyTypeChecker
+        self.timestamp: datetime = None
+        self.engine_name: str = str()
+        # noinspection PyTypeChecker
+        self.request_raw_bytes: bytes = None
+        self.request_auto_parsed: Union[http_parse.HTTPRequestParse, any] = None
+        self.request_custom_parsed: Any = None
         self.request_raw_hex: hex = None
-        # self.response_raw_bytes: bytearray = bytearray()
-        self.response_list_of_raw_bytes: list = list()
-        self.response_list_of_raw_decoded: list = list()
-        # self.response_raw_hex: hex = None
-        self.response_list_of_raw_hex: list = list()
+        # noinspection PyTypeChecker
+        self.response_raw_bytes: bytes = None
+        self.response_auto_parsed: Any = None
+        self.response_custom_parsed: Any = None
+        self.response_raw_hex: hex = None
         self.server_name: str = str()
         self.server_ip: str = str()
+        self.client_name: str = str()
         self.client_ip: str = str()
         self.source_port: int = int()
         self.destination_port: int = int()
         self.process_name: str = str()
         self.thread_id = None
+        self.thread_process: str = str()
         self.info: str = str()
-        self.error: str = str()
+        self.errors: list = list()
+        self.protocol: str = str()
+        self.protocol2: str = str()
+        self.protocol3: str = str()
+        self.recorded_file_path: str = str()
+        self.action: str = str()
 
-    def reinitialize(self) -> None:
+    def reinitialize_dynamic_vars(self):
         """
-        'ClientMessage' is being reused, since connection is still established to the server and new requests
-        are being processed on the same socket, so we need to reinitialize variables that are being updated, like
-        lists and dictionaries. Added the rest pf the variables that are repopulated to be on the safe side.
-        :return:
+        Reinitialize the dynamic variables of the class for the new cycle.
         """
-
-        self.request_raw_bytes = bytearray()
-        self.request_time_received = None
-        self.request_raw_decoded = None
-        self.request_body_parsed = None
-        self.response_list_of_raw_bytes = list()
+        self.request_raw_bytes = None
+        self.timestamp = None
+        self.request_auto_parsed = None
+        self.request_custom_parsed = None
         self.request_raw_hex = None
-        self.response_list_of_raw_hex = list()
-        self.response_list_of_raw_decoded = list()
+        self.response_raw_bytes = None
+        self.response_auto_parsed = None
+        self.response_custom_parsed = None
+        self.response_raw_hex = None
+        self.action = None
+        self.info = str()
+        self.errors = list()
+        self.protocol = str()
+        self.protocol2 = str()
+        self.protocol3 = str()
+        self.recorded_file_path = str()
+
+    def __iter__(self):
+        # __dict__ returns a dictionary containing the instance's attributes
+        for key, value in self.__dict__.items():
+            if key == 'request_raw_bytes':
+                value = str(value)
+            elif key == 'timestamp':
+                value = value.strftime('%Y-%m-%d-%H:%M:%S.%f')
+            elif key == 'request_auto_parsed':
+                if isinstance(value, http_parse.HTTPRequestParse):
+                    value = dicts.convert_complex_object_to_dict(value)
+                else:
+                    value = str(value)
+            elif key == 'request_custom_parsed':
+                value = dicts.convert_complex_object_to_dict(value)
+            elif key == 'response_raw_bytes':
+                value = str(value)
+            elif key == 'response_auto_parsed':
+                value = dicts.convert_complex_object_to_dict(value)
+            yield key, value