atomicshop 2.20.7__py3-none-any.whl → 2.21.0__py3-none-any.whl

atomicshop/__init__.py

@@ -1,4 +1,4 @@
 """Atomic Basic functions and classes to make developer life easier"""
 
 __author__ = "Den Kras"
-__version__ = '2.20.7'
+__version__ = '2.21.0'

atomicshop/mitm/config_static.py

@@ -6,16 +6,15 @@ from . import import_config
 
 # CONFIG = None
 LIST_OF_BOOLEANS: list = [
+    ('dnstcp', 'offline'),
     ('dns', 'enable'),
-    ('dns', 'offline_mode'),
-    ('dns', 'resolve_to_tcp_server_only_engine_domains'),
-    ('dns', 'resolve_to_tcp_server_all_domains'),
-    ('dns', 'resolve_regular'),
+    ('dns', 'resolve_by_engine'),
+    ('dns', 'resolve_regular_pass_thru'),
+    ('dns', 'resolve_all_domains_to_ipv4'),
     ('dns', 'set_default_dns_gateway_to_localhost'),
     ('dns', 'set_default_dns_gateway_to_default_interface_ipv4'),
     ('tcp', 'enable'),
-    ('tcp', 'engines_usage'),
-    ('tcp', 'server_response_mode'),
+    ('tcp', 'no_engines_usage_to_listen_addresses'),
     ('logrec', 'enable_request_response_recordings_in_logs'),
     ('certificates', 'install_ca_certificate_to_root_store'),
     ('certificates', 'uninstall_unused_ca_certificates_with_mitm_ca_name'),
@@ -32,6 +31,7 @@ LIST_OF_BOOLEANS: list = [
 
 
 TOML_TO_STATIC_CATEGORIES: dict = {
+    'dnstcp': 'MainConfig',
     'dns': 'DNSServer',
     'tcp': 'TCPServer',
     'logrec': 'LogRec',
@@ -40,6 +40,10 @@ TOML_TO_STATIC_CATEGORIES: dict = {
     'process_name': 'ProcessName'
 }
 
+# noinspection PyTypeChecker
+ENGINES_LIST: list = None           # list[initialize_engines.ModuleCategory]
+REFERENCE_MODULE = None             # initialize_engines.ModuleCategory
+
 
 class MainConfig:
     LOGGER_NAME: str = 'network'
@@ -61,6 +65,7 @@ class MainConfig:
     # Default server certificate file name and path.
     default_server_certificate_filename = f'{default_server_certificate_name}.pem'
     default_server_certificate_filepath: str = None
+    offline: bool = False
 
     @classmethod
     def update(cls):
@@ -77,32 +82,34 @@ class DNSServer:
     enable: bool
     offline_mode: bool
 
-    listening_interface: str
-    listening_port: int
+    listening_address: str
     forwarding_dns_service_ipv4: str
     cache_timeout_minutes: int
 
-    resolve_to_tcp_server_only_engine_domains: bool
-    resolve_to_tcp_server_all_domains: bool
-    resolve_regular: bool
-    target_tcp_server_ipv4: str
+    resolve_by_engine: bool
+    resolve_regular_pass_thru: bool
+    resolve_all_domains_to_ipv4_enable: bool
+    target_ipv4: str
 
     set_default_dns_gateway: str
     set_default_dns_gateway_to_localhost: bool
     set_default_dns_gateway_to_default_interface_ipv4: bool
 
+    # Convertable variables.
+    resolve_all_domains_to_ipv4: dict
+
+    # Static variables.
+    forwarding_dns_service_port: int = 53
+
 
 @dataclass
 class TCPServer:
     enable: bool
+    no_engines_usage_to_listen_addresses_enable: bool
+    no_engines_listening_address_list: list[str]
 
-    engines_usage: bool
-    server_response_mode: bool
-
-    listening_address_list: list[str]
-
-    forwarding_dns_service_ipv4_list___only_for_localhost: list[str]
-
+    # Convertable variables.
+    no_engines_usage_to_listen_addresses: dict
 
 @dataclass
 class LogRec:
@@ -161,7 +168,7 @@ def load_config(config_toml_file_path: str):
     MainConfig.update()
 
     # Load the configuration file.
-    result = import_config.import_config_file(config_toml_file_path)
+    result = import_config.import_config_files(config_toml_file_path)
     return result
 
 

atomicshop/mitm/connection_thread_worker.py

@@ -205,7 +205,7 @@ def thread_worker_main(
         # If there is a custom certificate for the client for this domain, then we'll use it.
         # noinspection PyTypeChecker
         custom_client_pem_certificate_path: str = None
-        for subdomain, pem_file_path in mtls_dict.items():
+        for subdomain, pem_file_path in found_domain_module.mtls.items():
             if subdomain == client_message.server_name:
                 custom_client_pem_certificate_path = pem_file_path
                 break
@@ -217,8 +217,7 @@ def thread_worker_main(
                 service_name=client_message.server_name,
                 service_port=client_message.destination_port,
                 tls=is_tls,
-                dns_servers_list=(
-                    config_static.TCPServer.forwarding_dns_service_ipv4_list___only_for_localhost),
+                dns_servers_list=[config_static.DNSServer.forwarding_dns_service_ipv4],
                 logger=network_logger,
                 custom_pem_client_certificate_file_path=custom_client_pem_certificate_path,
                 enable_sslkeylogfile_env_to_client_ssl_context=(
@@ -415,8 +414,8 @@ def thread_worker_main(
                     finish_thread()
                     return
 
-                # If we're in response mode, execute responder.
-                if config_static.TCPServer.server_response_mode:
+                # If we're in offline mode, execute responder.
+                if config_static.MainConfig.offline:
                     responder_queue.put(copy.deepcopy(client_message))
                 else:
                     # if side == 'Client':
@@ -517,16 +516,22 @@ def thread_worker_main(
 
     # Loading parser by domain, if there is no parser for current domain - general reference parser is loaded.
     # These should be outside any loop and initialized only once entering the thread.
-    parser, responder, recorder_no_init, mtls_dict = assign_class_by_domain(
-        engines_usage=config_static.TCPServer.engines_usage,
+    found_domain_module = assign_class_by_domain(
         engines_list=engines_list,
         message_domain_name=server_name,
-        reference_module=reference_module,
-        logger=network_logger
+        reference_module=reference_module
     )
-
+    parser = found_domain_module.parser_class_object
+    responder = found_domain_module.responder_class_object
+    recorder_no_init = found_domain_module.recorder_class_object
     recorder = recorder_no_init(record_path=config_static.LogRec.recordings_path)
 
+    network_logger.info(f"Assigned Modules for [{server_name}]: "
+        f"{parser.__name__}, "
+        f"{responder.__name__}, "
+        f"{recorder_no_init.__name__}")
+
+
     # Initializing the client message object with current thread's data.
     # This is needed only to skip error alerts after 'try'.
     client_message_connection: ClientMessage = ClientMessage()
@@ -543,18 +548,18 @@ def thread_worker_main(
         client_name = socket.gethostbyaddr(client_ip)[0]
         destination_port = client_socket.getsockname()[1]
 
-        if config_static.TCPServer.server_response_mode:
-            # If in response mode, then we'll get the TCP server's input address.
+        if config_static.MainConfig.offline:
+            # If in offline mode, then we'll get the TCP server's input address.
             server_ip = client_socket.getsockname()[0]
         else:
-            # If not in response mode, we will get the ip from the socket that will connect later to the service.
+            # If not in offline mode, we will get the ip from the socket that will connect later to the service.
             server_ip = ""
 
         network_logger.info(f"Thread Created - Client [{client_ip}:{source_port}] | "
                             f"Destination service: [{server_name}:{destination_port}]")
 
-        # If we're in response mode, we'll start the responder thread.
-        if config_static.TCPServer.server_response_mode:
+        # If we're in offline mode, we'll start the responder thread.
+        if config_static.MainConfig.offline:
             responder_thread: threading.Thread = threading.Thread(
                 target=responder_thread_worker, name=f"Thread-{thread_id}-Responder", daemon=True)
             responder_thread.start()
@@ -567,11 +572,11 @@ def thread_worker_main(
         server_receive_count: int = 0
         client_message_connection = client_message_first_start()
 
-        # If we're not in response mode, then we'll create the client socket to the service.
+        # If we're not in offline mode, then we'll create the client socket to the service.
         # noinspection PyTypeChecker
         connection_error: str = None
         service_socket_instance = None
-        if not config_static.TCPServer.server_response_mode:
+        if not config_static.MainConfig.offline:
             # If "service_client" object is not defined, we'll define it.
             # If it's defined, then there's still active "ssl_socket" with connection to the service domain.
             if not service_client:
@@ -599,7 +604,7 @@ def thread_worker_main(
             client_thread.daemon = True
             client_thread.start()
 
-            if not config_static.TCPServer.server_response_mode:
+            if not config_static.MainConfig.offline:
                 service_exception_queue: queue.Queue = queue.Queue()
                 service_thread = threading.Thread(
                     target=receive_send_start, args=(service_socket_instance, client_socket, service_exception_queue),
@@ -608,7 +613,7 @@ def thread_worker_main(
                 service_thread.start()
 
             client_thread.join()
-            if config_static.TCPServer.server_response_mode:
+            if config_static.MainConfig.offline:
                 responder_thread.join()
             else:
                 service_thread.join()
@@ -616,7 +621,7 @@ def thread_worker_main(
             # If there was an exception in any of the threads, then we'll raise it here.
             if not client_exception_queue.empty():
                 raise client_exception_queue.get()
-            if not config_static.TCPServer.server_response_mode:
+            if not config_static.MainConfig.offline:
                 if not service_exception_queue.empty():
                     raise service_exception_queue.get()
 

atomicshop/mitm/engines/create_module_template.py

@@ -73,12 +73,18 @@ class CreateModuleTemplate:
 
         # Add "" to each domain.
         domains_with_quotes: list = [f'"{domain}"' for domain in self.domains]
-        config_lines_list.append(f'"domains" = [{", ".join(domains_with_quotes)}]\n')
+
+        config_lines_list.append('[engine]')
+        config_lines_list.append(f'domains = [{", ".join(domains_with_quotes)}]')
+        config_lines_list.append('dns_target = "127.0.0.1"')
+        config_lines_list.append('tcp_listening_address_list = ["0.0.0.0:443"]\n')
+        # config_lines_list.append(f'\n')
+        config_lines_list.append('[mtls]')
+        config_lines_list.append('# "subdomain.domain.com" = "file_name_in_current_dir.pem"\n')
         # config_lines_list.append(f'\n')
-        config_lines_list.append(f'[mtls]')
-        config_lines_list.append(f'# "subdomain.domain.com" = "file_name_in_current_dir.pem"\n')
-        config_lines_list.append(f'[no_sni]')
-        config_lines_list.append(f'# "domain" = "example.com"\n')
+        config_lines_list.append('[no_sni]')
+        config_lines_list.append('get_from_dns = 1         # Blocking, the accept function will wait until the domain is received from DNS.')
+        config_lines_list.append('serve_domain_on_address = {0 = [{"example.com" = "127.0.0.2:443"}]}')
 
         config_file_path = self.new_engine_directory + os.sep + CONFIG_FILE_NAME
 

atomicshop/mitm/engines/create_module_template_main_example.py

@@ -0,0 +1,13 @@
+import sys
+
+from atomicshop.mitm.engines.create_module_template import CreateModuleTemplate
+
+
+def main():
+    CreateModuleTemplate()
+    input('Press enter to exit...')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

atomicshop/mitm/import_config.py

@@ -8,7 +8,7 @@ from ..permissions import permissions
 from ..wrappers.socketw import base
 from ..basics import booleans
 
-from . import config_static
+from . import config_static, initialize_engines
 
 
 def assign_bool(dict_instance: dict, section: str, key: str):
@@ -19,16 +19,27 @@ def assign_bool(dict_instance: dict, section: str, key: str):
         dict_instance[section][key] = True
     elif dict_instance[section][key] == 0:
         dict_instance[section][key] = False
+    elif isinstance(dict_instance[section][key], dict):
+        for subkey, subvalue in dict_instance[section][key].items():
+            if subkey == '1':
+                dict_instance[section][key] = {True: subvalue}
+            elif subkey == '0':
+                dict_instance[section][key] = {False: subvalue}
+            else:
+                print_api(f"Error: {section}.{key}.{subkey} must be 0 or 1.", color='red')
+                return 1
+            break
     else:
         print_api(f"Error: {section}.{key} must be 0 or 1.", color='red')
         return 1
 
 
-def import_config_file(
+def import_config_files(
         config_file_path: str
 ):
     """
     Import the configuration file 'config.toml' and write all the values to 'config_static' dataclasses module.
+
     :param config_file_path:
     :return:
     """
@@ -50,10 +61,55 @@ def import_config_file(
 
     manipulations_after_import()
 
+    result = import_engines_configs()
+    if result != 0:
+        return result
+
     result = check_configurations()
     return result
 
 
+def import_engines_configs() -> int:
+    """
+    Import the engines configuration files and write all the values to 'config_static' dataclasses module.
+
+    :return: int, status code.
+    """
+
+    # Get full paths of all the 'engine_config.ini' files.
+    engine_config_path_list = filesystem.get_paths_from_directory(
+        directory_path=config_static.MainConfig.ENGINES_DIRECTORY_PATH,
+        get_file=True,
+        file_name_check_pattern=config_static.MainConfig.ENGINE_CONFIG_FILE_NAME)
+
+    # Iterate through all the 'engine_config.ini' file paths.
+    domains_engine_list_full: list = list()
+    engines_list: list = list()
+    for engine_config_path in engine_config_path_list:
+        # Initialize engine.
+        current_module: initialize_engines.ModuleCategory = initialize_engines.ModuleCategory(config_static.MainConfig.SCRIPT_DIRECTORY)
+        current_module.fill_engine_fields_from_config(engine_config_path.path)
+        current_module.initialize_engine()
+
+        # Extending the full engine domain list with this list.
+        domains_engine_list_full.extend(current_module.domain_list)
+        # Append the object to the engines list
+        engines_list.append(current_module)
+    # === EOF Importing engine modules =============================================================================
+    # ==== Initialize Reference Module =============================================================================
+    reference_module: initialize_engines.ModuleCategory = initialize_engines.ModuleCategory(config_static.MainConfig.SCRIPT_DIRECTORY)
+    reference_module.fill_engine_fields_from_general_reference(config_static.MainConfig.ENGINES_DIRECTORY_PATH)
+    reference_module.initialize_engine(reference_general=True)
+
+    # Assigning all the engines domains to all time domains, that will be responsible for adding new domains.
+    config_static.Certificates.domains_all_times = list(domains_engine_list_full)
+
+    config_static.ENGINES_LIST = engines_list
+    config_static.REFERENCE_MODULE = reference_module
+
+    return 0
+
+
 def check_configurations() -> int:
     """
     Check the configurations from the 'config.toml' file.
@@ -68,31 +124,103 @@ def check_configurations() -> int:
         print_api("Both DNS and TCP servers in config ini file, nothing to run. Exiting...", color='red')
         return 1
 
-    # Check [tcp_server] boolean configurations. ===================================================================
-    if not config_static.TCPServer.engines_usage and config_static.TCPServer.server_response_mode:
-        message = "You can't set [server_response_mode = True], while setting\n" \
-                    "[engines_usage = False].\n" \
-                    "No engine modules will be loaded - so nothing to respond to.\n" \
-                    "Exiting..."
-        print_api(message, color='red')
+    # Checking if listening interfaces were set.
+    if not config_static.TCPServer.no_engines_usage_to_listen_addresses_enable:
+        # If no engines were found, check if listening interfaces were set in the main config.
+        if not config_static.ENGINES_LIST:
+            message = (
+                "\n"
+                "No engines found. Create with [create_template.py].\n"
+                "Exiting...")
+            print_api(message, color="red")
+            return 1
+    else:
+        if not config_static.TCPServer.no_engines_listening_address_list:
+            message = (
+                "\n"
+                "No listening interfaces. Set [no_engines_usage_to_listen_addresses] in the main [config.toml].\n"
+                "Exiting...")
+            print_api(message, color="red")
+            return 1
+
+    if not config_static.ENGINES_LIST and config_static.DNSServer.resolve_by_engine:
+        error_message = (
+            f"No engines were found in: [{config_static.MainConfig.ENGINES_DIRECTORY_PATH}]\n"
+            f"But the DNS routing is set to use them for routing.\n"
+            f"Please check your DNS routing configuration in the [config.toml] file or create an engine with [create_template.py].")
+        print_api(error_message, color="red")
         return 1
 
+    for engine in config_static.ENGINES_LIST:
+        if engine.no_sni.get_from_dns and engine.no_sni.serve_domain_on_address_enable:
+            message = (
+                f"Both [get_from_dns] and [serve_domain_on_address] are enabled in [no_sni] section of the engine.\n"
+                f"Only one Can be True.")
+            print_api(message, color="red")
+            return 1
+        if not engine.no_sni.get_from_dns and not engine.no_sni.serve_domain_on_address_enable:
+            message = (
+                f"Both [get_from_dns] and [serve_domain_on_address] are disabled in [no_sni] section of the engine.\n"
+                f"Only one Can be True.")
+            print_api(message, color="red")
+            return 1
+
+        if engine.no_sni.serve_domain_on_address_enable:
+            # Check if the domains in no_sni are the same as in the engine. They should not be.
+            # Same goes for the address.
+            for domain, address_ip_port in engine.no_sni.serve_domain_on_address_dict.items():
+                if domain in engine.domain_list:
+                    message = (
+                        f"[*] No SNI setting: The domain [{domain}] is in the engine domains list [{engine.domain_list}].\n"
+                        f"The point of the no_sni section is to serve specific domains on separate addresses.\n")
+                    print_api(message, color="red")
+                    return 1
+
+                if address_ip_port in engine.tcp_listening_address_list:
+                    message = (
+                        f"[*] No SNI setting: The address [{address_ip_port}] is in the engine listening interfaces list [{engine.tcp_listening_address_list}].\n"
+                        f"The point of the no_sni section is to serve specific domains on separate addresses.\n")
+                    print_api(message, color="red")
+                    return 1
+
     # Check admin right if on localhost ============================================================================
-    # If the 'config.dns['target_tcp_server_ipv4']' IP address is localhost, then we need to check if the script
+    # If any of the DNS IP target addresses is localhost loopback, then we need to check if the script
     # is executed with admin rights. There are some processes that 'psutil' can't get their command line if not
     # executed with administrative privileges.
     # Also, check Admin privileges only if 'config.tcp['get_process_name']' was set to 'True' in 'config.ini' of
     # the script.
-    if (config_static.DNSServer.target_tcp_server_ipv4 in base.THIS_DEVICE_IP_LIST and
-            config_static.ProcessName.get_process_name):
-        # If we're not running with admin rights, prompt to the user and make him decide what to do.
-        # If he wants to continue running with 'psutil' exceptions or close the script and rerun with admin rights.
-        if not is_admin:
-            message: str = \
-                ("Need to run the script with administrative rights to get the process name while TCP running "
-                 "on the same computer.\nExiting...")
-            print_api(message, color='red')
-            return 1
+    if config_static.ProcessName.get_process_name:
+        # If the DNS server was set to resolve by engines, we need to check all relevant engine settings.
+        if config_static.DNSServer.resolve_by_engine:
+            for engine in config_static.ENGINES_LIST:
+                # Check if the DNS target is localhost loopback.
+                if engine.dns_target in base.THIS_DEVICE_IP_LIST or engine.dns_target.startswith('127.'):
+                    if not is_admin:
+                        message: str = \
+                            ("Need to run the script with administrative rights to get the process name while TCP "
+                             "running on the same computer.\nExiting...")
+                        print_api(message, color='red')
+                        return 1
+                if engine.no_sni.serve_domain_on_address_enable:
+                    no_sni_target_address_list: list = engine.no_sni.serve_domain_on_address_dict.values()
+                    for no_sni_target_address in no_sni_target_address_list:
+                        if no_sni_target_address in base.THIS_DEVICE_IP_LIST or \
+                                no_sni_target_address.startswith('127.'):
+                            if not is_admin:
+                                message: str = \
+                                    ("Need to run the script with administrative rights to get the process name while TCP "
+                                     "running on the same computer.\nExiting...")
+                                print_api(message, color='red')
+                                return 1
+        if config_static.DNSServer.resolve_all_domains_to_ipv4:
+            if config_static.DNSServer.target_ipv4 in base.THIS_DEVICE_IP_LIST or \
+                    config_static.DNSServer.target_ipv4.startswith('127.'):
+                if not is_admin:
+                    message: str = \
+                        ("Need to run the script with administrative rights to get the process name while TCP "
+                         "running on the same computer.\nExiting...")
+                    print_api(message, color='red')
+                    return 1
 
     try:
         booleans.is_only_1_true_in_list(
@@ -136,6 +264,14 @@ def check_configurations() -> int:
                 print_api(message, color='red')
                 return 1
 
+    if not config_static.DNSServer.resolve_by_engine and not config_static.DNSServer.resolve_regular_pass_thru and not \
+            config_static.DNSServer.resolve_all_domains_to_ipv4_enable:
+        message: str = (
+            "No DNS server resolving settings were set.\n"
+            "Please check your DNS server settings in the [config.toml] file.")
+        print_api(message, color='red')
+        return 1
+
     # This is checked directly in the SocketWrapper.
     # if (config_static.Certificates.install_ca_certificate_to_root_store and not is_admin) or \
     #         (config_static.Certificates.uninstall_unused_ca_certificates_with_mitm_ca_name and not is_admin):
@@ -144,11 +280,27 @@ def check_configurations() -> int:
     #     print_api(message, color='red')
     #     return 1
 
-
     return 0
 
 
 def manipulations_after_import():
+    for key, value in config_static.DNSServer.resolve_all_domains_to_ipv4.items():
+        config_static.DNSServer.resolve_all_domains_to_ipv4_enable = key
+        config_static.DNSServer.target_ipv4 = value
+        break
+
+    for key, value in config_static.TCPServer.no_engines_usage_to_listen_addresses.items():
+        # If the key is False, it means that the user doesn't want to use the no_engines_listening_address_list.
+        # So, we'll assign an empty list to it.
+        if not key:
+            config_static.TCPServer.no_engines_usage_to_listen_addresses_enable = False
+            config_static.TCPServer.no_engines_listening_address_list = list()
+        # If the key is True, it means that the user wants to use the no_engines_listening_address_list.
+        else:
+            config_static.TCPServer.no_engines_usage_to_listen_addresses_enable = key
+            config_static.TCPServer.no_engines_listening_address_list = value
+        break
+
     # Convert extensions to skip to a list of extension IDs.
     skip_extensions: list = list()
     if config_static.SkipExtensions.tls_web_client_authentication:
@@ -166,7 +318,7 @@ def manipulations_after_import():
         config_static.Certificates.custom_server_certificate_path, config_static.MainConfig.SCRIPT_DIRECTORY)
 
     config_static.LogRec.recordings_path = (
-            config_static.LogRec.logs_path + os.sep + config_static.LogRec.recordings_directory_name)
+        config_static.LogRec.logs_path + os.sep + config_static.LogRec.recordings_directory_name)
 
     # At this point the user that sets the config can set it to null or empty string ''. We will make sure
     # that the path is None if it's empty.