atomicshop 2.15.13__py3-none-any.whl → 2.16.0__py3-none-any.whl

atomicshop/mitm/initialize_mitm_server.py

@@ -1,53 +1,56 @@
 import os
-import sys
 import threading
 import time
 
-# Importing atomicshop package to get the version of the package.
-import atomicshop
+import atomicshop   # Importing atomicshop package to get the version of the package.
 
-from .import_config import ImportConfig
-from .initialize_engines import ModuleCategory
-from .connection_thread_worker import thread_worker_main
-from .. import filesystem, queues
+from .. import filesystem, queues, dns, on_exit, permissions
 from ..python_functions import get_current_python_version_string, check_python_version_compliance
-from ..wrappers.socketw.socket_wrapper import SocketWrapper
-from ..wrappers.socketw import dns_server
-from ..wrappers.psutilw import networks
-from ..basics import dicts_nested
+from ..wrappers.socketw import socket_wrapper, dns_server, base
 from ..wrappers.loggingw import loggingw
 from ..print_api import print_api
 
+from .initialize_engines import ModuleCategory
+from .connection_thread_worker import thread_worker_main
+from . import config_static
+
+
+def exit_cleanup():
+    if config_static.DNSServer.set_default_dns_gateway:
+        is_dns_dynamic, current_dns_gateway = dns.get_default_dns_gateway()
+        print_api(f'Current DNS Gateway: {current_dns_gateway}')
 
-STATISTICS_HEADER: str = \
-    'request_time_sent,host,path,command,status_code,request_size_bytes,response_size_bytes,file_path,process_cmd,error'
+        if current_dns_gateway == config_static.DNSServer.set_default_dns_gateway and not is_dns_dynamic:
+            if permissions.is_admin():
+                dns.set_connection_dns_gateway_dynamic(use_default_connection=True)
+                print_api("Returned default DNS gateway...", color='blue')
 
 
-def initialize_mitm_server(config_static):
+def initialize_mitm_server(config_file_path: str):
+    on_exit.register_exit_handler(exit_cleanup)
+
     # Main function should return integer with error code, 0 is successful.
     # Since listening server is infinite, this will not be reached.
     # After modules import - we check for python version.
     check_python_version_compliance(minimum_version='3.11')
 
-    # Preparing everything for the logging module.
-    # Log folder path is in the "config.ini" file, so we need to read it before setting loggers.
-    config_importer = ImportConfig(
-        file_name=config_static.CONFIG_INI_SERVER_FILE_NAME, directory_path=config_static.WORKING_DIRECTORY)
-    config_importer.open()
-    config = config_importer.config
+    # Import the configuration file.
+    result = config_static.load_config(config_file_path)
+    if result != 0:
+        return result
 
     # Create folders.
-    filesystem.create_directory(config['log']['logs_path'])
-    filesystem.create_directory(config['recorder']['recordings_path'])
-    if config['certificates']['sni_get_server_certificate_from_server_socket']:
+    filesystem.create_directory(config_static.Log.logs_path)
+    filesystem.create_directory(config_static.Recorder.recordings_path)
+    if config_static.Certificates.sni_get_server_certificate_from_server_socket:
         filesystem.create_directory(
-            config['certificates']['sni_server_certificate_from_server_socket_download_directory'])
+            config_static.Certificates.sni_server_certificate_from_server_socket_download_directory)
 
     # Create a logger that will log messages to file, Initiate System logger.
     logger_name = "system"
     system_logger = loggingw.create_logger(
         logger_name=logger_name,
-        file_path=f"{config['log']['logs_path']}{os.sep}{logger_name}.txt",
+        file_path=f"{config_static.Log.logs_path}{os.sep}{logger_name}.txt",
         add_stream=True,
         add_timedfile=True,
         formatter_streamhandler='DEFAULT',
@@ -56,44 +59,37 @@ def initialize_mitm_server(config_static):
 
     # Writing first log.
     system_logger.info("======================================")
-
-    if config_importer.admin_rights is not None:
-        if not config_importer.admin_rights:
-            system_logger.error("User continued with errors on Command Line harvesting for system processes.")
-
     system_logger.info("Server Started.")
     system_logger.info(f"Python Version: {get_current_python_version_string()}")
     system_logger.info(f"Script Version: {config_static.SCRIPT_VERSION}")
     system_logger.info(f"Atomic Workshop Version: {atomicshop.__version__}")
-    system_logger.info(f"Loaded config.ini: {config_importer.config_parser.file_path}")
-    system_logger.info(f"Log folder: {config['log']['logs_path']}")
-    system_logger.info(f"Recordings folder for Requests/Responses: {config['recorder']['recordings_path']}")
+    system_logger.info(f"Log folder: {config_static.Log.logs_path}")
+    system_logger.info(f"Recordings folder for Requests/Responses: {config_static.Recorder.recordings_path}")
     system_logger.info(f"Loaded system logger: {system_logger}")
 
-    system_logger.info(f"TCP Server Target IP: {config['dns']['target_tcp_server_ipv4']}")
+    system_logger.info(f"TCP Server Target IP: {config_static.DNSServer.target_tcp_server_ipv4}")
 
     # Some 'config.ini' settings logging ===========================================================================
-    if config['certificates']['default_server_certificate_usage']:
+    if config_static.Certificates.default_server_certificate_usage:
         system_logger.info(
             f"Default server certificate usage enabled, if no SNI available: "
-            f"{config_static.CONFIG_EXTENDED['certificates']['default_server_certificate_directory']}"
-            f"{os.sep}{config_static.CONFIG_EXTENDED['certificates']['default_server_certificate_name']}.pem")
+            f"{config_static.MainConfig.default_server_certificate_filepath}")
 
-    if config['certificates']['sni_server_certificates_cache_directory']:
+    if config_static.Certificates.sni_server_certificates_cache_directory:
         system_logger.info(
             f"SNI function certificates creation enabled. Certificates cache: "
-            f"{config['certificates']['sni_server_certificates_cache_directory']}")
+            f"{config_static.Certificates.sni_server_certificates_cache_directory}")
     else:
         system_logger.info(f"SNI function certificates creation disabled.")
 
-    if config['certificates']['custom_server_certificate_usage']:
+    if config_static.Certificates.custom_server_certificate_usage:
         system_logger.info(f"Custom server certificate usage is enabled.")
-        system_logger.info(f"Custom Certificate Path: {config['certificates']['custom_server_certificate_path']}")
+        system_logger.info(f"Custom Certificate Path: {config_static.Certificates.custom_server_certificate_path}")
 
         # If 'custom_private_key_path' field was populated.
-        if config['certificates']['custom_private_key_path']:
+        if config_static.Certificates.custom_private_key_path:
             system_logger.info(
-                f"Custom Certificate Private Key Path: {config['certificates']['custom_private_key_path']}")
+                f"Custom Certificate Private Key Path: {config_static.Certificates.custom_private_key_path}")
         else:
             system_logger.info(f"Custom Certificate Private Key Path wasn't provided in [advanced] section. "
                                f"Assuming the private key is inside the certificate file.")
@@ -103,17 +99,17 @@ def initialize_mitm_server(config_static):
 
     # Get full paths of all the 'engine_config.ini' files.
     engine_config_path_list = filesystem.get_file_paths_from_directory(
-        directory_path=config_static.ENGINES_DIRECTORY_PATH,
-        file_name_check_pattern=config_static.ENGINE_CONFIG_FILE_NAME)
+        directory_path=config_static.MainConfig.ENGINES_DIRECTORY_PATH,
+        file_name_check_pattern=config_static.MainConfig.ENGINE_CONFIG_FILE_NAME)
 
     # Iterate through all the 'engine_config.ini' file paths.
     domains_engine_list_full: list = list()
     engines_list: list = list()
     for engine_config_path in engine_config_path_list:
         # Initialize engine.
-        current_module = ModuleCategory(config_static.WORKING_DIRECTORY)
+        current_module = ModuleCategory(config_static.MainConfig.SCRIPT_DIRECTORY)
         current_module.fill_engine_fields_from_config(engine_config_path)
-        current_module.initialize_engine(logs_path=config['log']['logs_path'],
+        current_module.initialize_engine(logs_path=config_static.Log.logs_path,
                                          logger=system_logger)
 
         # Extending the full engine domain list with this list.
@@ -122,9 +118,9 @@ def initialize_mitm_server(config_static):
         engines_list.append(current_module)
     # === EOF Importing engine modules =============================================================================
     # ==== Initialize Reference Module =============================================================================
-    reference_module = ModuleCategory(config_static.WORKING_DIRECTORY)
-    reference_module.fill_engine_fields_from_general_reference(config_static.ENGINES_DIRECTORY_PATH)
-    reference_module.initialize_engine(logs_path=config['log']['logs_path'],
+    reference_module = ModuleCategory(config_static.MainConfig.SCRIPT_DIRECTORY)
+    reference_module.fill_engine_fields_from_general_reference(config_static.MainConfig.ENGINES_DIRECTORY_PATH)
+    reference_module.initialize_engine(logs_path=config_static.Log.logs_path,
                                        logger=system_logger, stdout=False, reference_general=True)
     # === EOF Initialize Reference Module ==========================================================================
     # === Engine logging ===========================================================================================
@@ -139,41 +135,41 @@ def initialize_mitm_server(config_static):
                    f"{engine.recorder_class_object.__name__}")
         print_api(message, logger=system_logger)
 
-    if config['dns']['enable_dns_server']:
+    if config_static.DNSServer.enable:
         print_api("DNS Server is enabled.", logger=system_logger)
 
         # If engines were found and dns is set to route by the engine domains.
-        if engines_list and config['dns']['route_to_tcp_server_only_engine_domains']:
+        if engines_list and config_static.DNSServer.resolve_to_tcp_server_only_engine_domains:
             print_api("Engine domains will be routed by the DNS server to Built-in TCP Server.", logger=system_logger)
         # If engines were found, but the dns isn't set to route to engines.
-        elif engines_list and not config['dns']['route_to_tcp_server_only_engine_domains']:
+        elif engines_list and not config_static.DNSServer.resolve_to_tcp_server_only_engine_domains:
             message = f"[*] Engine domains found, but the DNS routing is set not to use them for routing."
             print_api(message, color="yellow", logger=system_logger)
-        elif not engines_list and config['dns']['route_to_tcp_server_only_engine_domains']:
+        elif not engines_list and config_static.DNSServer.resolve_to_tcp_server_only_engine_domains:
             raise ValueError("No engines were found, but the DNS routing is set to use them for routing.\n"
                              "Please check your DNS configuration in the 'config.ini' file.")
 
-        if config['dns']['route_to_tcp_server_all_domains']:
+        if config_static.DNSServer.resolve_to_tcp_server_all_domains:
             print_api("All domains will be routed by the DNS server to Built-in TCP Server.", logger=system_logger)
 
-        if config['dns']['regular_resolving']:
+        if config_static.DNSServer.resolve_regular:
             print_api(
                 "Regular DNS resolving is enabled. Built-in TCP server will not be routed to",
                 logger=system_logger, color="yellow")
     else:
         print_api("DNS Server is disabled.", logger=system_logger, color="yellow")
 
-    if config['tcp']['enable_tcp_server']:
+    if config_static.TCPServer.enable:
         print_api("TCP Server is enabled.", logger=system_logger)
 
-        if engines_list and not config['tcp']['engines_usage']:
+        if engines_list and not config_static.TCPServer.engines_usage:
             message = \
                 f"Engines found, but the TCP server is set not to use them for processing. General responses only."
             print_api(message, color="yellow", logger=system_logger)
-        elif engines_list and config['tcp']['engines_usage']:
+        elif engines_list and config_static.TCPServer.engines_usage:
             message = f"Engines found, and the TCP server is set to use them for processing."
             print_api(message, logger=system_logger)
-        elif not engines_list and config['tcp']['engines_usage']:
+        elif not engines_list and config_static.TCPServer.engines_usage:
             raise ValueError("No engines were found, but the TCP server is set to use them for processing.\n"
                              "Please check your TCP configuration in the 'config.ini' file.")
     else:
@@ -182,22 +178,12 @@ def initialize_mitm_server(config_static):
     # === EOF Engine Logging =======================================================================================
 
     # Assigning all the engines domains to all time domains, that will be responsible for adding new domains.
-    config_static.CONFIG_EXTENDED['certificates']['domains_all_times'] = list(domains_engine_list_full)
-
-    # Creating Statistics logger.
-    statistics_logger = loggingw.create_logger(
-        logger_name="statistics",
-        directory_path=config['log']['logs_path'],
-        add_timedfile=True,
-        formatter_filehandler='MESSAGE',
-        file_type='csv',
-        header=STATISTICS_HEADER
-    )
+    config_static.Certificates.domains_all_times = list(domains_engine_list_full)
 
     network_logger_name = "network"
     network_logger = loggingw.create_logger(
         logger_name=network_logger_name,
-        directory_path=config['log']['logs_path'],
+        directory_path=config_static.Log.logs_path,
         add_stream=True,
         add_timedfile=True,
         formatter_streamhandler='DEFAULT',
@@ -209,60 +195,128 @@ def initialize_mitm_server(config_static):
     listener_logger = loggingw.get_logger_with_level(f'{network_logger_name}.listener')
     system_logger.info(f"Loaded listener logger: {listener_logger}")
 
+    print_api("Press [Ctrl]+[C] to stop.", color='blue')
+
     # Create request domain queue.
     domain_queue = queues.NonBlockQueue()
 
     # === Initialize DNS module ====================================================================================
-    if config['dns']['enable_dns_server']:
-        # Check if the DNS server port is in use.
-        port_in_use = networks.get_processes_using_port_list([config['dns']['listening_port']])
-        if port_in_use:
-            for port, process_info in port_in_use.items():
-                message = f"Port [{port}] is already in use by process: {process_info}"
-                print_api(message, error_type=True, logger_method='critical', logger=system_logger)
-
+    if config_static.DNSServer.enable:
+        try:
+            dns_server_instance = dns_server.DnsServer(
+                listening_interface=config_static.DNSServer.listening_interface,
+                listening_port=config_static.DNSServer.listening_port,
+                forwarding_dns_service_ipv4=config_static.DNSServer.forwarding_dns_service_ipv4,
+                tcp_target_server_ipv4=config_static.DNSServer.target_tcp_server_ipv4,
+                # Passing the engine domain list to DNS server to work with.
+                # 'list' function re-initializes the current list, or else it will be the same instance object.
+                tcp_resolve_domain_list=list(config_static.Certificates.domains_all_times),
+                log_directory_path=config_static.Log.logs_path,
+                offline_mode=config_static.DNSServer.offline_mode,
+                resolve_to_tcp_server_only_tcp_resolve_domains=(
+                    config_static.DNSServer.resolve_to_tcp_server_only_engine_domains),
+                resolve_to_tcp_server_all_domains=config_static.DNSServer.resolve_to_tcp_server_all_domains,
+                resolve_regular=config_static.DNSServer.resolve_regular,
+                cache_timeout_minutes=config_static.DNSServer.cache_timeout_minutes,
+                request_domain_queue=domain_queue
+            )
+        except (dns_server.DnsPortInUseError, dns_server.DnsConfigurationValuesError) as e:
+            print_api(e, error_type=True, color="red", logger=system_logger)
             # Wait for the message to be printed and saved to file.
             time.sleep(1)
-            sys.exit(1)
-
-        # before executing TCP sockets and after executing 'network' logger.
-        dns_server_instance = dns_server.DnsServer(config)
-        # Passing the engine domain list to DNS server to work with.
-        # 'list' function re-initializes the current list, or else it will be the same instance object.
-        dns_server_instance.domain_list = list(domains_engine_list_full)
+            return 1
 
-        dns_server_instance.request_domain_queue = domain_queue
-        # Initiate the thread.
         dns_thread = threading.Thread(target=dns_server_instance.start)
         dns_thread.daemon = True
         dns_thread.start()
 
     # === EOF Initialize DNS module ================================================================================
     # === Initialize TCP Server ====================================================================================
-    if config['tcp']['enable_tcp_server']:
-        port_in_use = networks.get_processes_using_port_list(config['tcp']['listening_port_list'])
-        if port_in_use:
-            for port, process_info in port_in_use.items():
-                print_api(f"Port [{port}] is already in use by process: {process_info}", logger=system_logger,
-                          error_type=True, logger_method='critical')
+    if config_static.TCPServer.enable:
+        try:
+            socket_wrapper_instance = socket_wrapper.SocketWrapper(
+                listening_interface=config_static.TCPServer.listening_interface,
+                listening_port_list=config_static.TCPServer.listening_port_list,
+                ca_certificate_name=config_static.MainConfig.ca_certificate_name,
+                ca_certificate_filepath=config_static.MainConfig.ca_certificate_filepath,
+                default_server_certificate_usage=config_static.Certificates.default_server_certificate_usage,
+                default_server_certificate_name=config_static.MainConfig.default_server_certificate_name,
+                default_certificate_domain_list=config_static.Certificates.domains_all_times,
+                default_server_certificate_directory=config_static.MainConfig.SCRIPT_DIRECTORY,
+                sni_use_default_callback_function=True,
+                sni_use_default_callback_function_extended=True,
+                sni_add_new_domains_to_default_server_certificate=(
+                    config_static.Certificates.sni_add_new_domains_to_default_server_certificate),
+                sni_create_server_certificate_for_each_domain=(
+                    config_static.Certificates.sni_create_server_certificate_for_each_domain),
+                sni_server_certificates_cache_directory=(
+                    config_static.Certificates.sni_server_certificates_cache_directory),
+                sni_get_server_certificate_from_server_socket=(
+                    config_static.Certificates.sni_get_server_certificate_from_server_socket),
+                sni_server_certificate_from_server_socket_download_directory=(
+                    config_static.Certificates.sni_server_certificate_from_server_socket_download_directory),
+                custom_server_certificate_usage=config_static.Certificates.custom_server_certificate_usage,
+                custom_server_certificate_path=config_static.Certificates.custom_server_certificate_path,
+                custom_private_key_path=config_static.Certificates.custom_private_key_path,
+                get_process_name=config_static.ProcessName.get_process_name,
+                ssh_user=config_static.ProcessName.ssh_user,
+                ssh_pass=config_static.ProcessName.ssh_pass,
+                ssh_script_to_execute=config_static.ProcessName.ssh_script_to_execute,
+                logger=listener_logger,
+                statistics_logs_directory=config_static.Log.logs_path,
+                forwarding_dns_service_ipv4_list___only_for_localhost=(
+                    config_static.TCPServer.forwarding_dns_service_ipv4_list___only_for_localhost),
+                skip_extension_id_list=config_static.SkipExtensions.SKIP_EXTENSION_ID_LIST
+            )
+        except socket_wrapper.SocketWrapperPortInUseError as e:
+            print_api(e, error_type=True, color="red", logger=system_logger)
             # Wait for the message to be printed and saved to file.
             time.sleep(1)
-            sys.exit(1)
-
-        socket_wrapper = SocketWrapper(
-            config=dicts_nested.merge(config, config_static.CONFIG_EXTENDED), logger=listener_logger,
-            statistics_logger=statistics_logger)
-
-        socket_wrapper.create_tcp_listening_socket_list()
-
-        socket_wrapper.requested_domain_from_dns_server = domain_queue
+            return 1
+
+        statistics_writer = socket_wrapper_instance.statistics_writer
+
+        socket_wrapper_instance.create_tcp_listening_socket_list()
+
+        socket_wrapper_instance.requested_domain_from_dns_server = domain_queue
+
+        # Before we start the loop. we can set the default gateway if specified.
+        set_dns_gateway = False
+        dns_gateway_server_list = list()
+        if config_static.DNSServer.set_default_dns_gateway:
+            dns_gateway_server_list = config_static.DNSServer.set_default_dns_gateway
+            set_dns_gateway = True
+        elif config_static.DNSServer.set_default_dns_gateway_to_localhost:
+            dns_gateway_server_list = [base.LOCALHOST_IPV4]
+            set_dns_gateway = True
+        elif config_static.DNSServer.set_default_dns_gateway_to_default_interface_ipv4:
+            dns_gateway_server_list = [base.DEFAULT_IPV4]
+            set_dns_gateway = True
+
+        if set_dns_gateway:
+            # noinspection PyTypeChecker
+            dns.set_connection_dns_gateway_static(
+                dns_servers=dns_gateway_server_list,
+                use_default_connection=True
+            )
 
         # General exception handler will catch all the exceptions that occurred in the threads and write it to the log.
+        # noinspection PyBroadException
         try:
-            socket_wrapper.loop_for_incoming_sockets(function_reference=thread_worker_main, reference_args=(
-                network_logger, statistics_logger, engines_list, reference_module, config,))
+            socket_thread = threading.Thread(
+                target=socket_wrapper_instance.loop_for_incoming_sockets,
+                kwargs={
+                    'reference_function_name': thread_worker_main,
+                    'reference_function_args': (network_logger, statistics_writer, engines_list, reference_module,)
+                }
+            )
+
+            socket_thread.daemon = True
+            socket_thread.start()
         except Exception:
             message = f"Unhandled Exception occurred in 'loop_for_incoming_sockets' function"
             print_api(message, error_type=True, color="red", logger=network_logger, traceback_string=True, oneline=True)
 
-    # === EOF Initialize TCP Server ================================================================================
+        # This is needed for Keyboard Exception.
+        while True:
+            time.sleep(1)

atomicshop/mitm/shared_functions.py

@@ -1,5 +1,4 @@
 import json
-# Needed to get the function caller module.
 import inspect
 
 from ..wrappers.loggingw import loggingw

atomicshop/mitm/statistic_analyzer.py

@@ -346,6 +346,7 @@ def analyze(main_file_path: str):
 
 def deviation_calculator_by_moving_average_main(
         statistics_file_directory: str,
+        by_type: Literal['host', 'url'],
         moving_average_window_days: int,
         top_bottom_deviation_percentage: float,
         get_deviation_for_last_day_only: bool = False,
@@ -359,6 +360,9 @@ def deviation_calculator_by_moving_average_main(
     :param statistics_file_directory: string, the directory where 'statistics.csv' file resides.
         Also, all the rotated files like: statistics_2021-01-01.csv, statistics_2021-01-02.csv, etc.
         These will be analyzed in the order of the date in the file name.
+    :param by_type: string, 'host' or 'url'. The type of the deviation calculation.
+        'host' will calculate the deviation by the host name. Example: maps.google.com, yahoo.com, etc.
+        'url' will calculate the deviation by the URL. Example: maps.google.com/maps, yahoo.com/news, etc.
     :param moving_average_window_days: integer, the moving average window days.
     :param top_bottom_deviation_percentage: float, the top bottom deviation percentage. Example: 0.1 for 10%.
     :param get_deviation_for_last_day_only: bool, if True, only the last day will be analyzed.
@@ -402,6 +406,9 @@ def deviation_calculator_by_moving_average_main(
     if output_file_type not in ['json', 'csv']:
         raise ValueError(f'output_file_type must be "json" or "csv", not [{output_file_type}]')
 
+    if by_type not in ['host', 'url']:
+        raise ValueError(f'by_type must be "host" or "url", not [{by_type}]')
+
     statistics_file_path: str = f'{statistics_file_directory}{os.sep}{STATISTICS_FILE_NAME}'
 
     def convert_data_value_to_string(value_key: str, list_index: int) -> None:
@@ -413,6 +420,7 @@ def deviation_calculator_by_moving_average_main(
 
     deviation_list = moving_average_helper.calculate_moving_average(
         statistics_file_path,
+        by_type,
         moving_average_window_days,
         top_bottom_deviation_percentage,
         get_deviation_for_last_day_only
@@ -436,7 +444,11 @@ def deviation_calculator_by_moving_average_main(
                     'value': deviation.get('value', None),
                     'ma_value': deviation.get('ma_value', None),
                     'deviation_percentage': deviation.get('deviation_percentage', None),
-                    'total_entries_averaged': total_entries_averaged
+                    'total_entries_averaged': total_entries_averaged,
+                    'median_request_size': deviation.get('median_request_size', None),
+                    'median_response_size': deviation.get('median_response_size', None),
+                    'mm_request_size': deviation.get('mm_request_size', None),
+                    'mm_response_size': deviation.get('mm_response_size', None),
                 })
 
             deviation_list = summary_deviation_list

atomicshop/mitm/statistic_analyzer_helper/moving_average_helper.py

@@ -4,10 +4,12 @@ from typing import Literal
 from ...print_api import print_api
 from ...wrappers.loggingw import reading, consts
 from ...file_io import csvs
+from ... import urls
 
 
 def calculate_moving_average(
         file_path: str,
+        by_type: Literal['host', 'url'],
         moving_average_window_days,
         top_bottom_deviation_percentage: float,
         get_deviation_for_last_day_only: bool = False,
@@ -17,6 +19,7 @@ def calculate_moving_average(
     This function calculates the moving average of the daily statistics.
 
     :param file_path: string, the path to the 'statistics.csv' file.
+    :param by_type: string, the type to calculate the moving average by. Can be 'host' or 'url'.
     :param moving_average_window_days: integer, the window size for the moving average.
     :param top_bottom_deviation_percentage: float, the percentage of deviation from the moving average to the top or
         bottom.
@@ -65,8 +68,7 @@ def calculate_moving_average(
 
         # Get the data dictionary from the statistics content.
         statistics_content[date_string]['statistics_daily'] = compute_statistics_from_content(
-            statistics_content[date_string]['content_no_errors']
-        )
+            statistics_content[date_string]['content_no_errors'], by_type)
 
     moving_average_dict: dict = compute_moving_averages_from_average_statistics(
         statistics_content,
@@ -106,19 +108,36 @@ def get_content_without_errors(content: list) -> list:
     return traffic_statistics_without_errors
 
 
-def get_data_dict_from_statistics_content(content: list) -> dict:
+def get_data_dict_from_statistics_content(
+        content: list,
+        by_type: Literal['host', 'url']
+) -> dict:
     """
     This function gets the data dictionary from the 'statistics.csv' file content.
 
     :param content: list, the content list.
+    :param by_type: string, the type to calculate the moving average by. Can be 'host' or 'url'.
     :return: dict, the data dictionary.
     """
 
     hosts_requests_responses: dict = {}
     for line in content:
+        if by_type == 'host':
+            type_to_check: str = line['host']
+        elif by_type == 'url':
+            # Combine host and path to URL.
+            type_to_check: str = line['host'] + line['path']
+            # Remove the parameters from the URL.
+            url_parsed = urls.url_parser(type_to_check)
+            type_to_check = url_parsed['path']
+            # Remove the last slash from the URL.
+            type_to_check = type_to_check.removesuffix('/')
+        else:
+            raise ValueError(f'Invalid by_type: {by_type}')
+
         # If subdomain is not in the dictionary, add it.
-        if line['host'] not in hosts_requests_responses:
-            hosts_requests_responses[line['host']] = {
+        if type_to_check not in hosts_requests_responses:
+            hosts_requests_responses[type_to_check] = {
                 'request_sizes': [],
                 'response_sizes': []
             }
@@ -132,8 +151,8 @@ def get_data_dict_from_statistics_content(content: list) -> dict:
             if response_size_bytes == '':
                 response_size_bytes = '0'
 
-            hosts_requests_responses[line['host']]['request_sizes'].append(int(request_size_bytes))
-            hosts_requests_responses[line['host']]['response_sizes'].append(int(response_size_bytes))
+            hosts_requests_responses[type_to_check]['request_sizes'].append(int(request_size_bytes))
+            hosts_requests_responses[type_to_check]['response_sizes'].append(int(response_size_bytes))
         except ValueError:
             print_api(line, color='yellow')
             raise
@@ -163,18 +182,22 @@ def compute_statistics_from_data_dict(data_dict: dict):
         data_dict[host]['median_response_size'] = median_response_size
 
 
-def compute_statistics_from_content(content: list):
+def compute_statistics_from_content(
+        content: list,
+        by_type: Literal['host', 'url']
+):
     """
     This function computes the statistics from the 'statistics.csv' file content.
 
     :param content: list, the content list.
+    :param by_type: string, the type to calculate the moving average by. Can be 'host' or 'url'.
     :return: dict, the statistics dictionary.
     """
 
-    hosts_requests_responses: dict = get_data_dict_from_statistics_content(content)
-    compute_statistics_from_data_dict(hosts_requests_responses)
+    requests_responses: dict = get_data_dict_from_statistics_content(content, by_type)
+    compute_statistics_from_data_dict(requests_responses)
 
-    return hosts_requests_responses
+    return requests_responses
 
 
 def compute_moving_averages_from_average_statistics(
@@ -200,12 +223,15 @@ def compute_moving_averages_from_average_statistics(
             list(average_statistics_dict.values()))[current_day-moving_average_window_days:current_day]
 
         # Compute the moving averages.
-        moving_average[day] = compute_average_for_current_day_from_past_x_days(last_x_window_days_content_list)
+        moving_average[day] = compute_average_for_current_day_from_past_x_days(
+            last_x_window_days_content_list)
 
     return moving_average
 
 
-def compute_average_for_current_day_from_past_x_days(previous_days_content_list: list) -> dict:
+def compute_average_for_current_day_from_past_x_days(
+        previous_days_content_list: list
+) -> dict:
     """
     This function computes the average for the current day from the past x days.
 
@@ -222,11 +248,15 @@ def compute_average_for_current_day_from_past_x_days(previous_days_content_list:
                     'counts': [],
                     'avg_request_sizes': [],
                     'avg_response_sizes': [],
+                    'median_request_sizes': [],
+                    'median_response_sizes': []
                 }
 
             moving_average[host]['counts'].append(int(host_dict['count']))
             moving_average[host]['avg_request_sizes'].append(float(host_dict['avg_request_size']))
             moving_average[host]['avg_response_sizes'].append(float(host_dict['avg_response_size']))
+            moving_average[host]['median_request_sizes'].append(float(host_dict['median_request_size']))
+            moving_average[host]['median_response_sizes'].append(float(host_dict['median_response_size']))
 
     # Compute the moving average.
     moving_average_results: dict = {}
@@ -234,14 +264,20 @@ def compute_average_for_current_day_from_past_x_days(previous_days_content_list:
         ma_count = statistics.mean(host_dict['counts'])
         ma_request_size = statistics.mean(host_dict['avg_request_sizes'])
         ma_response_size = statistics.mean(host_dict['avg_response_sizes'])
+        mm_request_size = statistics.median(host_dict['median_request_sizes'])
+        mm_response_size = statistics.median(host_dict['median_response_sizes'])
 
         moving_average_results[host] = {
             'ma_count': ma_count,
             'ma_request_size': ma_request_size,
             'ma_response_size': ma_response_size,
+            'mm_request_size': mm_request_size,
+            'mm_response_size': mm_response_size,
             'counts': host_dict['counts'],
             'avg_request_sizes': host_dict['avg_request_sizes'],
-            'avg_response_sizes': host_dict['avg_response_sizes']
+            'avg_response_sizes': host_dict['avg_response_sizes'],
+            'median_request_sizes': host_dict['median_request_sizes'],
+            'median_response_sizes': host_dict['median_response_sizes']
         }
 
     return moving_average_results
@@ -303,6 +339,10 @@ def find_deviation_from_moving_average(
                 'ma_value_checked': check_type_moving_above,
                 'deviation_percentage': deviation_percentage,
                 'deviation_type': deviation_type,
+                'median_request_size': day_statistics_content_dict['median_request_size'],
+                'median_response_size': day_statistics_content_dict['median_response_size'],
+                'mm_request_size': moving_averages_dict[host]['mm_request_size'],
+                'mm_response_size': moving_averages_dict[host]['mm_response_size'],
                 'data': day_statistics_content_dict,
                 'ma_data': moving_averages_dict[host]
             })