atomicshop 2.11.47__py3-none-any.whl → 3.10.5__py3-none-any.whl

atomicshop/mitm/statistic_analyzer.py

@@ -1,143 +1,18 @@
+import os
 import datetime
+import json
+from typing import Union, Literal
+import argparse
 
+from .statistic_analyzer_helper import analyzer_helper, moving_average_helper
 from .. import filesystem, domains, datetimes, urls
 from ..basics import dicts
-from ..file_io import tomls, xlsxs
+from ..file_io import tomls, xlsxs, jsons, csvs
 from ..wrappers.loggingw import reading
 from ..print_api import print_api
 
 
-def get_the_last_day_number(statistics_content: list, stop_after_lines: int = None) -> int:
-    """
-    This function gets the last day number from the statistics content.
-
-    :param statistics_content: list, of lines in the statistics content.
-    :param stop_after_lines: integer, if specified, the function will stop after the specified number of lines.
-    :return: integer, the last day number.
-    """
-
-    last_day_number = None
-    start_time_temp = None
-    for line_index, line in enumerate(statistics_content):
-        try:
-            request_time = datetime.datetime.strptime(line['request_time_sent'], '%Y-%m-%d %H:%M:%S.%f')
-        except ValueError:
-            continue
-
-        if not start_time_temp:
-            start_time_temp = request_time
-
-        if stop_after_lines:
-            if line_index == stop_after_lines:
-                break
-
-        last_day_number = datetimes.get_difference_between_dates_in_days(start_time_temp, request_time)
-    return last_day_number
-
-
-def create_empty_features_dict() -> dict:
-    """
-    This function creates an empty dictionary for the daily stats. This should be initiated for each 'host_type' of:
-        'domain', 'subdomain', 'url_no_parameters'.
-    :return: dict
-    """
-
-    return {
-        'total_count': {}, 'normal_count': {}, 'error_count': {},
-        'request_0_byte_count': {}, 'response_0_byte_count': {},
-        'request_sizes_list': {}, 'response_sizes_list': {},
-        'request_sizes_no_0_bytes_list': {}, 'response_sizes_no_0_bytes_list': {},
-        'average_request_size': {}, 'average_response_size': {},
-        'average_request_size_no_0_bytes': {}, 'average_response_size_no_0_bytes': {}}
-
-
-def add_to_count_to_daily_stats(
-        daily_stats: dict, current_day: int, last_day: int, host_type: str, feature: str, host_name: str) -> None:
-    """
-    This function adds 1 to the 'count' feature of the current day in the daily stats.
-
-    :param daily_stats: dict, the daily statistics dict.
-    :param current_day: integer, the current day number.
-    :param last_day: integer, the last day number.
-    :param host_type: string, the type of the host. Can be: 'domain', 'subdomain', 'url_no_parameters'.
-    :param feature: string, the feature to add the count to. Can be: 'total_count', 'normal_count', 'error_count',
-        'request_0_byte_count', 'response_0_byte_count'.
-    :param host_name: string, the name of the host.
-
-    :return: None.
-    """
-
-    # Aggregate daily domain hits.
-    if host_name not in daily_stats[host_type][feature].keys():
-        daily_stats[host_type][feature][host_name] = {}
-        # Iterate from first day to the last day.
-        for day in range(0, last_day + 1):
-            daily_stats[host_type][feature][host_name][day] = 0
-
-    # Add count to current day.
-    daily_stats[host_type][feature][host_name][current_day] += 1
-
-
-def add_to_list_to_daily_stats(
-        daily_stats: dict, current_day: int, last_day: int, host_type: str, feature: str, host_name: str,
-        size: float) -> None:
-    """
-    This function adds the 'size' to the 'feature' list of the current day in the daily stats.
-
-    :param daily_stats: dict, the daily statistics dict.
-    :param current_day: integer, the current day number.
-    :param last_day: integer, the last day number.
-    :param host_type: string, the type of the host. Can be: 'domain', 'subdomain', 'url_no_parameters'.
-    :param feature: string, the feature to add the count to. Can be: 'request_sizes_list', 'response_sizes_list',
-        'request_sizes_no_0_bytes_list', 'response_sizes_no_0_bytes_list'.
-    :param host_name: string, the name of the host.
-    :param size: float, the size in bytes to add to the list.
-
-    :return: None.
-    """
-
-    # Aggregate daily domain hits.
-    if host_name not in daily_stats[host_type][feature].keys():
-        daily_stats[host_type][feature][host_name] = {}
-        # Iterate from first day to the last day.
-        for day in range(0, last_day + 1):
-            daily_stats[host_type][feature][host_name][day] = []
-
-    # Add count to current day.
-    daily_stats[host_type][feature][host_name][current_day].append(size)
-
-
-def add_to_average_to_daily_stats(
-        daily_stats: dict, current_day: int, last_day: int, host_type: str, feature: str, host_name: str,
-        list_of_sizes: list) -> None:
-    """
-    This function adds the average size in bytes calculated from the 'list_of_sizes' to the 'feature' of the current
-        day in the daily stats.
-
-    :param daily_stats: dict, the daily statistics dict.
-    :param current_day: integer, the current day number.
-    :param last_day: integer, the last day number.
-    :param host_type: string, the type of the host. Can be: 'domain', 'subdomain', 'url_no_parameters'.
-    :param feature: string, the feature to add the count to. Can be: 'average_request_size', 'average_response_size',
-        'average_request_size_no_0_bytes', 'average_response_size_no_0_bytes'.
-    :param host_name: string, the name of the host.
-    :param list_of_sizes: list, the list of sizes to calculate the average from.
-
-    :return: None.
-    """
-
-    # Aggregate daily domain hits.
-    if host_name not in daily_stats[host_type][feature].keys():
-        daily_stats[host_type][feature][host_name] = {}
-        # Iterate from first day to the last day.
-        for day in range(0, last_day + 1):
-            daily_stats[host_type][feature][host_name][day] = 0
-
-    # If the list of size is empty, add 0 to the average, since we cannot divide by 0.
-    if len(list_of_sizes) == 0:
-        daily_stats[host_type][feature][host_name][current_day] = 0
-    else:
-        daily_stats[host_type][feature][host_name][current_day] = sum(list_of_sizes) / len(list_of_sizes)
+STATISTICS_FILE_NAME: str = 'statistics.csv'
 
 
 def analyze(main_file_path: str):
@@ -154,8 +29,10 @@ def analyze(main_file_path: str):
     summary_path: str = filesystem.check_absolute_path___add_full(config['report_file_path'], script_directory)
 
     # Get the content from statistics files.
-    statistics_content: list = reading.get_logs(
-        config['statistic_files_path'], pattern='statistics*.csv', log_type='csv',
+    log_file_path_pattern: str = f"{config['statistic_files_path']}{os.sep}statistics.csv"
+    statistics_content: list = reading.get_all_log_files_into_list(
+        log_file_path=log_file_path_pattern,
+        log_type='csv'
     )
 
     # Initialize loop.
@@ -167,9 +44,9 @@ def analyze(main_file_path: str):
         'subdomain': {'total_count': {}, 'normal_count': {}, 'error_count': {}}
     }
     daily_stats: dict = {
-        'domain': create_empty_features_dict(),
-        'subdomain': create_empty_features_dict(),
-        'url_no_parameters': create_empty_features_dict()
+        'domain': analyzer_helper.create_empty_features_dict(),
+        'subdomain': analyzer_helper.create_empty_features_dict(),
+        'url_no_parameters': analyzer_helper.create_empty_features_dict()
     }
 
     # Start the main loop.
@@ -190,7 +67,7 @@ def analyze(main_file_path: str):
 
         # Find the last day number. If 'break_after_lines' is specified, the loop will stop after the specified line.
         if not last_day_number:
-            last_day_number = get_the_last_day_number(statistics_content, break_after_lines)
+            last_day_number = analyzer_helper.get_the_last_day_number(statistics_content, break_after_lines)
 
         if break_after_lines:
             if line_index == break_after_lines:
@@ -290,87 +167,87 @@ def analyze(main_file_path: str):
         day_number = datetimes.get_difference_between_dates_in_days(start_time, request_time)
 
         # Add 1 to the total count of the current day.
-        add_to_count_to_daily_stats(
+        analyzer_helper.add_to_count_to_daily_stats(
             daily_stats, day_number, last_day_number, 'domain', 'total_count', main_domain)
-        add_to_count_to_daily_stats(
+        analyzer_helper.add_to_count_to_daily_stats(
             daily_stats, day_number, last_day_number, 'subdomain', 'total_count', subdomain)
-        add_to_count_to_daily_stats(
+        analyzer_helper.add_to_count_to_daily_stats(
             daily_stats, day_number, last_day_number, 'url_no_parameters', 'total_count', url_no_parameters)
 
         # Handle line if it has error.
         if line['error'] != '':
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'domain', 'error_count', main_domain)
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'subdomain', 'error_count', subdomain)
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'url_no_parameters', 'error_count', url_no_parameters)
         else:
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'domain', 'normal_count', main_domain)
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'subdomain', 'normal_count', subdomain)
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'url_no_parameters', 'normal_count', url_no_parameters)
 
         if request_size == 0:
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'domain', 'request_0_byte_count',
                 main_domain)
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'subdomain', 'request_0_byte_count',
                 subdomain)
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'url_no_parameters', 'request_0_byte_count',
                 url_no_parameters)
 
         if response_size == 0:
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'domain', 'response_0_byte_count',
                 main_domain)
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'subdomain', 'response_0_byte_count',
                 subdomain)
-            add_to_count_to_daily_stats(
+            analyzer_helper.add_to_count_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'url_no_parameters', 'response_0_byte_count',
                 url_no_parameters)
 
         if request_size is not None and response_size is not None:
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'domain', 'request_sizes_list', main_domain, request_size)
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'subdomain', 'request_sizes_list', subdomain, request_size)
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'url_no_parameters', 'request_sizes_list', url_no_parameters,
                 request_size)
 
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'domain', 'response_sizes_list', main_domain, response_size)
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'subdomain', 'response_sizes_list', subdomain, response_size)
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'url_no_parameters', 'response_sizes_list', url_no_parameters,
                 response_size)
 
         if request_size != 0 and request_size is not None:
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'domain', 'request_sizes_no_0_bytes_list',
                 main_domain, request_size)
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'subdomain', 'request_sizes_no_0_bytes_list',
                 subdomain, request_size)
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'url_no_parameters', 'request_sizes_no_0_bytes_list',
                 url_no_parameters, request_size)
 
         if response_size != 0 and response_size is not None:
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'domain', 'response_sizes_no_0_bytes_list',
                 main_domain, response_size)
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'subdomain', 'response_sizes_no_0_bytes_list',
                 subdomain, response_size)
-            add_to_list_to_daily_stats(
+            analyzer_helper.add_to_list_to_daily_stats(
                 daily_stats, day_number, last_day_number, 'url_no_parameters', 'response_sizes_no_0_bytes_list',
                 url_no_parameters, response_size)
 
@@ -392,7 +269,7 @@ def analyze(main_file_path: str):
 
             for host_name, days in hosts.items():
                 for day, sizes in days.items():
-                    add_to_average_to_daily_stats(
+                    analyzer_helper.add_to_average_to_daily_stats(
                         daily_stats, day, last_day_number, host_type, feature_name, host_name, sizes)
 
     # Sorting overall stats.
@@ -463,3 +340,217 @@ def analyze(main_file_path: str):
         xlsxs.write_xlsx(combined_sorted_stats, file_path=summary_path)
 
     return
+
+
+# ======================================================================================================================
+
+
+def deviation_calculator_by_moving_average(
+        statistics_file_directory: str = None,
+        statistics_content: dict = None,
+        by_type: Literal['host', 'url'] = 'url',
+        moving_average_window_days: int = 5,
+        top_bottom_deviation_percentage: float = 0.25,
+        get_deviation_for_last_day_only: bool = False,
+        get_deviation_for_date: str = None,
+        summary: bool = False,
+        output_file_path: str = None,
+        output_file_type: Literal['json', 'csv'] = 'json',
+        convert_sizes_lists_and_ma_data_to_string: bool = False,
+        skip_total_count_less_than: int = None
+) -> Union[list, None]:
+    """
+    This function is the main function for the moving average calculator.
+
+    :param statistics_file_directory: string, can be either providing the directory where 'statistics.csv' file resides
+        or None. If None, the 'statistics_content' must be provided.
+        The directory where 'statistics.csv' file resides.
+        Also, all the rotated files like: statistics_2021-01-01.csv, statistics_2021-01-02.csv, etc.
+        These will be analyzed in the order of the date in the file name.
+    :param statistics_content: dict, if specified, this will be used instead of reading the files from the directory.
+        The dict should be a result of the 'atomicshop.mitm.statistic_analyzer_helper.moving_average_helper.get_all_files_content'.
+    :param by_type: string, 'host' or 'url'. The type of the deviation calculation.
+        'host' will calculate the deviation by the host name. Example: maps.google.com, yahoo.com, etc.
+        'url' will calculate the deviation by the URL. Example: maps.google.com/maps, yahoo.com/news, etc.
+    :param moving_average_window_days: integer, the moving average window days.
+    :param top_bottom_deviation_percentage: float, the top bottom deviation percentage. Example: 0.1 for 10%.
+    :param get_deviation_for_last_day_only: bool, if True, only the last day will be analyzed.
+        Example: With 'moving_average_window_days=5', the last 6 days will be analyzed.
+        5 days for moving average and the last day for deviation.
+        File names example:
+            statistics_2021-01-01.csv
+            statistics_2021-01-02.csv
+            statistics_2021-01-03.csv
+            statistics_2021-01-04.csv
+            statistics_2021-01-05.csv
+            statistics_2021-01-06.csv
+        Files 01 to 05 will be used for moving average and the file 06 for deviation.
+        Meaning the average calculated for 2021-01-06 will be compared to the values moving average of 2021-01-01
+        to 2021-01-05.
+    :param get_deviation_for_date: string, if specified, only the specified date will be analyzed.
+        The date must be in the format of 'YYYY-MM-DD'. Example: '2021-01-06'.
+    :param summary: bool, if True, Only the summary will be generated without all the numbers that were used
+        to calculate the averages and the moving average data.
+    :param output_file_path: string, if None, no file will be written.
+    :param output_file_type: string, the type of the output file. 'json' or 'csv'.
+    :param convert_sizes_lists_and_ma_data_to_string: bool, if True, the 'request_sizes', 'response_sizes' and 'ma_data'
+        will be converted to string. This is useful when writing to files, so the view will be more readable.
+    :param skip_total_count_less_than: integer, if specified, the deviation calculation will be skipped
+        if the total count is less than this number.
+        This means that if we have moving average window of 7 days, on the 8th day, the deviation will be calculated
+        only if the total count of the checked type
+        (request average size, response average, request count, response count)
+        is greater or equal to this number.
+    -----------------------------
+    :return: the deviation list of dicts.
+
+    Example:
+    import sys
+    from atomicshop.mitm import statistic_analyzer
+
+
+    def main():
+        return statistic_analyzer.moving_average_calculator_main(
+            statistics_file_path='statistics.csv',
+            moving_average_window_days=7,
+            top_bottom_deviation_percentage=0.1,
+            output_json_file='C:\\output\\deviation_list.json'
+        )
+
+
+    if __name__ == '__main__':
+        sys.exit(main())
+    """
+
+    def convert_data_value_to_string(value_key: str, list_index: int) -> None:
+        deviation_list[list_index]['data'][value_key] = json.dumps(deviation_list[list_index]['data'][value_key])
+
+    def convert_value_to_string(value_key: str, list_index: int) -> None:
+        if value_key in deviation_list[list_index]:
+            deviation_list[list_index][value_key] = json.dumps(deviation_list[list_index][value_key])
+
+    if not statistics_file_directory and not statistics_content:
+        raise ValueError('Either [statistics_file_directory] or [statistics_content] must be provided.')
+    if statistics_file_directory and statistics_content:
+        raise ValueError('Either [statistics_file_directory] or [statistics_content] must be provided, not both.')
+
+    if output_file_type not in ['json', 'csv']:
+        raise ValueError(f'output_file_type must be "json" or "csv", not [{output_file_type}]')
+
+    if by_type not in ['host', 'url']:
+        raise ValueError(f'by_type must be "host" or "url", not [{by_type}]')
+
+    if get_deviation_for_last_day_only and get_deviation_for_date:
+        raise ValueError('Either [get_deviation_for_last_day_only] or [get_deviation_for_date] can be provided, not both.')
+
+    if statistics_file_directory:
+        statistics_file_path: str | None = f'{statistics_file_directory}{os.sep}{STATISTICS_FILE_NAME}'
+    else:
+        statistics_file_path: str | None = None
+
+    deviation_list = moving_average_helper.calculate_moving_average(
+        statistics_file_path,
+        statistics_content,
+        by_type,
+        moving_average_window_days,
+        top_bottom_deviation_percentage,
+        get_deviation_for_last_day_only,
+        get_deviation_for_date,
+        skip_total_count_less_than
+    )
+
+    if deviation_list:
+        if convert_sizes_lists_and_ma_data_to_string:
+            for deviation_list_index, deviation in enumerate(deviation_list):
+                convert_data_value_to_string('request_sizes', deviation_list_index)
+                convert_data_value_to_string('response_sizes', deviation_list_index)
+                convert_value_to_string('ma_data', deviation_list_index)
+
+        if summary:
+            for deviation in deviation_list:
+                _ = deviation.pop('check_type')
+                _ = deviation.pop('percentage')
+                _ = deviation.pop('ma_value_checked')
+                _ = deviation.pop('deviation_type')
+                _ = deviation.pop('data')
+                _ = deviation.pop('ma_data')
+
+        if output_file_path:
+            print_api(f'Deviation Found, saving to file: {output_file_path}', color='blue')
+
+            if output_file_type == 'csv':
+                csvs.write_list_to_csv(deviation_list, output_file_path)
+            elif output_file_type == 'json':
+                jsons.write_json_file(deviation_list, output_file_path, use_default_indent=True)
+
+        return deviation_list
+
+    return []
+
+
+def deviation_calculator_by_moving_average_main():
+    def parse_args():
+        description_full: str = (
+            f'Description: Calculate deviation by moving average.')
+
+        parser = argparse.ArgumentParser(
+            description=description_full)
+
+        parser.add_argument(
+            '-d', '--directory', type=str, required=True,
+            help='Provide full path to directory with statistics.csv files.')
+        parser.add_argument(
+            '-of', '--output_file', type=str, required=True, help='Provide full path to output file.')
+        parser.add_argument(
+            '-ot', '--output_type', type=str, required=True, help='Provide output type: [json] or [csv].')
+        parser.add_argument(
+            '-by', '--by_type', type=str, required=True, help='Calculate by [host] or [url].')
+        parser.add_argument(
+            '-f', '--full_details', action='store_true', required=False,
+            help='(OPTIONAL) Output full processing details instead of summary.')
+        parser.add_argument(
+            '-w', '--window', type=int, required=True, help='Moving average window in days.')
+        parser.add_argument(
+            '-p', '--percentage', type=float, required=True,
+            help='Percentage of deviation from moving average. Example: 0.1 for 10%%.')
+        parser.add_argument(
+            '-slt', '--skip_total_count_less_than', type=int, required=False,
+            help='An integer to skip the deviation calculation if the total count is less than this number.')
+
+        return parser.parse_args()
+
+    args = parse_args()
+
+    if args.output_type == 'csv' and args.full_details:
+        print_api("Full details output is not supported for csv files.", color='red')
+        return 1
+
+    if not os.path.isdir(args.directory):
+        print_api(f"Directory does not exist: {args.directory}", color='red')
+        return 1
+
+    if not filesystem.get_paths_from_directory(
+            directory_path=args.directory, get_file=True, simple_list=True, file_name_check_pattern='*statistics*.csv'):
+        print_api(f"No statistics files found in: {args.directory}", color='red')
+        return 1
+
+    if args.full_details:
+        summary = False
+        convert_sizes_lists_and_ma_data_to_string = True
+    else:
+        summary = True
+        convert_sizes_lists_and_ma_data_to_string = False
+
+    _ = deviation_calculator_by_moving_average(
+        statistics_file_directory=args.directory,
+        by_type=args.by_type,
+        moving_average_window_days=args.window,
+        top_bottom_deviation_percentage=args.percentage,
+        summary=summary,
+        output_file_path=args.output_file,
+        output_file_type=args.output_type,
+        convert_sizes_lists_and_ma_data_to_string=convert_sizes_lists_and_ma_data_to_string,
+        skip_total_count_less_than=args.skip_total_count_less_than
+    )
+
+    return 0

atomicshop/mitm/statistic_analyzer_helper/analyzer_helper.py

@@ -0,0 +1,136 @@
+import datetime
+
+from ... import datetimes
+
+
+def get_the_last_day_number(statistics_content: list, stop_after_lines: int = None) -> int:
+    """
+    This function gets the last day number from the statistics content.
+
+    :param statistics_content: list, of lines in the statistics content.
+    :param stop_after_lines: integer, if specified, the function will stop after the specified number of lines.
+    :return: integer, the last day number.
+    """
+
+    last_day_number = None
+    start_time_temp = None
+    for line_index, line in enumerate(statistics_content):
+        try:
+            request_time = datetime.datetime.strptime(line['request_time_sent'], '%Y-%m-%d %H:%M:%S.%f')
+        except ValueError:
+            continue
+
+        if not start_time_temp:
+            start_time_temp = request_time
+
+        if stop_after_lines:
+            if line_index == stop_after_lines:
+                break
+
+        last_day_number = datetimes.get_difference_between_dates_in_days(start_time_temp, request_time)
+    return last_day_number
+
+
+def create_empty_features_dict() -> dict:
+    """
+    This function creates an empty dictionary for the daily stats. This should be initiated for each 'host_type' of:
+        'domain', 'subdomain', 'url_no_parameters'.
+    :return: dict
+    """
+
+    return {
+        'total_count': {}, 'normal_count': {}, 'error_count': {},
+        'request_0_byte_count': {}, 'response_0_byte_count': {},
+        'request_sizes_list': {}, 'response_sizes_list': {},
+        'request_sizes_no_0_bytes_list': {}, 'response_sizes_no_0_bytes_list': {},
+        'average_request_size': {}, 'average_response_size': {},
+        'average_request_size_no_0_bytes': {}, 'average_response_size_no_0_bytes': {}}
+
+
+def add_to_count_to_daily_stats(
+        daily_stats: dict, current_day: int, last_day: int, host_type: str, feature: str, host_name: str) -> None:
+    """
+    This function adds 1 to the 'count' feature of the current day in the daily stats.
+
+    :param daily_stats: dict, the daily statistics dict.
+    :param current_day: integer, the current day number.
+    :param last_day: integer, the last day number.
+    :param host_type: string, the type of the host. Can be: 'domain', 'subdomain', 'url_no_parameters'.
+    :param feature: string, the feature to add the count to. Can be: 'total_count', 'normal_count', 'error_count',
+        'request_0_byte_count', 'response_0_byte_count'.
+    :param host_name: string, the name of the host.
+
+    :return: None.
+    """
+
+    # Aggregate daily domain hits.
+    if host_name not in daily_stats[host_type][feature].keys():
+        daily_stats[host_type][feature][host_name] = {}
+        # Iterate from first day to the last day.
+        for day in range(0, last_day + 1):
+            daily_stats[host_type][feature][host_name][day] = 0
+
+    # Add count to current day.
+    daily_stats[host_type][feature][host_name][current_day] += 1
+
+
+def add_to_list_to_daily_stats(
+        daily_stats: dict, current_day: int, last_day: int, host_type: str, feature: str, host_name: str,
+        size: float) -> None:
+    """
+    This function adds the 'size' to the 'feature' list of the current day in the daily stats.
+
+    :param daily_stats: dict, the daily statistics dict.
+    :param current_day: integer, the current day number.
+    :param last_day: integer, the last day number.
+    :param host_type: string, the type of the host. Can be: 'domain', 'subdomain', 'url_no_parameters'.
+    :param feature: string, the feature to add the count to. Can be: 'request_sizes_list', 'response_sizes_list',
+        'request_sizes_no_0_bytes_list', 'response_sizes_no_0_bytes_list'.
+    :param host_name: string, the name of the host.
+    :param size: float, the size in bytes to add to the list.
+
+    :return: None.
+    """
+
+    # Aggregate daily domain hits.
+    if host_name not in daily_stats[host_type][feature].keys():
+        daily_stats[host_type][feature][host_name] = {}
+        # Iterate from first day to the last day.
+        for day in range(0, last_day + 1):
+            daily_stats[host_type][feature][host_name][day] = []
+
+    # Add count to current day.
+    daily_stats[host_type][feature][host_name][current_day].append(size)
+
+
+def add_to_average_to_daily_stats(
+        daily_stats: dict, current_day: int, last_day: int, host_type: str, feature: str, host_name: str,
+        list_of_sizes: list) -> None:
+    """
+    This function adds the average size in bytes calculated from the 'list_of_sizes' to the 'feature' of the current
+        day in the daily stats.
+
+    :param daily_stats: dict, the daily statistics dict.
+    :param current_day: integer, the current day number.
+    :param last_day: integer, the last day number.
+    :param host_type: string, the type of the host. Can be: 'domain', 'subdomain', 'url_no_parameters'.
+    :param feature: string, the feature to add the count to. Can be: 'average_request_size', 'average_response_size',
+        'average_request_size_no_0_bytes', 'average_response_size_no_0_bytes'.
+    :param host_name: string, the name of the host.
+    :param list_of_sizes: list, the list of sizes to calculate the average from.
+
+    :return: None.
+    """
+
+    # Aggregate daily domain hits.
+    if host_name not in daily_stats[host_type][feature].keys():
+        daily_stats[host_type][feature][host_name] = {}
+        # Iterate from first day to the last day.
+        for day in range(0, last_day + 1):
+            daily_stats[host_type][feature][host_name][day] = 0
+
+    # If the list of size is empty, add 0 to the average, since we cannot divide by 0.
+    if len(list_of_sizes) == 0:
+        daily_stats[host_type][feature][host_name][current_day] = 0
+    else:
+        daily_stats[host_type][feature][host_name][current_day] = sum(list_of_sizes) / len(list_of_sizes)