atomicshop 2.15.11__py3-none-any.whl → 3.10.5__py3-none-any.whl

atomicshop/mitm/recs_files.py

@@ -0,0 +1,183 @@
+import datetime
+import os
+import multiprocessing
+import logging
+import zipfile
+import shutil
+
+from .. import filesystem, print_api
+from .. wrappers.loggingw import consts, loggingw
+
+
+REC_FILE_DATE_TIME_MILLISECONDS_FORMAT: str = f'{consts.DEFAULT_ROTATING_SUFFIXES_FROM_WHEN["S"]}_%f'
+REC_FILE_DATE_TIME_FORMAT: str = f'{consts.DEFAULT_ROTATING_SUFFIXES_FROM_WHEN["S"]}'
+REC_FILE_DATE_FORMAT: str = REC_FILE_DATE_TIME_FORMAT.split('_')[0]
+
+
+def archive(
+        directory_path: str,
+        include_root_directory: bool = True,
+) -> str:
+    """
+    Function archives the directory.
+    :param directory_path: string, full path to the directory.
+    :param include_root_directory: boolean, default is 'True'.
+        'True': The root directory will be included in the archive.
+        'False': The root directory will not be included in the archive.
+        True is usually the case in most archiving utilities.
+    :return: string, full path to the archived file.
+    """
+
+    # This is commonly used and supported by most ZIP utilities.
+    compression_method = zipfile.ZIP_DEFLATED
+
+    archive_path: str = directory_path + '.zip'
+    with zipfile.ZipFile(archive_path, 'w', compression_method) as zip_object:
+        for root, _, files in os.walk(directory_path):
+            for file in files:
+                file_path = os.path.join(root, file)
+
+                # If including the root directory, use the relative path from the parent directory of the root
+                if include_root_directory:
+                    arcname = os.path.relpath(file_path, os.path.dirname(directory_path))
+                else:
+                    arcname = os.path.relpath(file_path, directory_path)
+
+                zip_object.write(file_path, arcname)
+
+    return archive_path
+
+
+def recs_archiver(
+        recs_directory: str,
+        logging_queue: multiprocessing.Queue,
+        logger_name: str,
+        finalize_output_queue: multiprocessing.Queue
+) -> list | None:
+    """
+    Find recs files in a directory for each day.
+    Each day of recordings will have separate archive.
+
+    :param recs_directory: The directory where recordings are stored.
+    :param logging_queue: The queue for logging messages.
+    :param logger_name: The name of the logger to use for logging.
+        This is the base name that '.rec_packer' will be added to it.
+    :param finalize_output_queue: output queue for results/exceptions.
+    """
+
+    logger_name = f"{logger_name}.rec_packer"
+
+    rec_packer_logger_with_queue_handler: logging.Logger = loggingw.create_logger(
+        logger_name=logger_name,
+        add_queue_handler=True,
+        log_queue=logging_queue)
+
+    print_api.print_api(
+        'Starting recs archiver process.', color='blue',
+        logger=rec_packer_logger_with_queue_handler
+    )
+
+    today_date_string = datetime.datetime.now().strftime(REC_FILE_DATE_FORMAT)
+
+    # There should not be recording json files in recs root.
+    files_in_recs_root: list = filesystem.get_paths_from_directory(
+        recs_directory, get_file=True, file_name_check_pattern='*\\.json', recursive=False)
+    if files_in_recs_root:
+        raise NotImplementedError("The files in recs root directory are not implemented yet.")
+
+    # Each engine should have its own directory inside recordings. We will find all the directories inside recs folder.
+    directory_paths_in_recs: list = filesystem.get_paths_from_directory(
+        recs_directory, get_directory=True, recursive=False)
+
+    file_list_per_directory: list = list()
+    for directory_path in directory_paths_in_recs:
+        all_recs_files = filesystem.get_paths_from_directory(
+            directory_path=directory_path.path,
+            get_file=True,
+            file_name_check_pattern='*.json',
+            datetime_format=REC_FILE_DATE_FORMAT,
+            recursive=False
+        )
+        file_list_per_directory.append((directory_path, all_recs_files))
+
+
+    try:
+        archived_files: list = list()
+        for directory_path, all_recs_files in file_list_per_directory:
+            print_api.print_api(f"Archiving recs files in directory: {directory_path.path}",
+                      logger=rec_packer_logger_with_queue_handler, color='blue')
+            for recs_atomic_path in all_recs_files:
+                # We don't need to archive today's files.
+                if today_date_string == recs_atomic_path.datetime_string:
+                    continue
+
+                target_directory_path: str = f"{directory_path.path}{os.sep}{recs_atomic_path.datetime_string}"
+                filesystem.create_directory(target_directory_path)
+                filesystem.move_file(
+                    recs_atomic_path.path, target_directory_path)
+
+            # Archive directories.
+            archive_directories: list = filesystem.get_paths_from_directory(
+                directory_path.path, get_directory=True, recursive=False)
+
+            if not archive_directories:
+                print_api.print_api(
+                    f"No directories to archive in: {directory_path.path}",
+                    color='blue',
+                    logger=rec_packer_logger_with_queue_handler
+                )
+            else:
+                total_archived_files: int = 0
+                for archive_directory in archive_directories:
+                    files_to_archive: list = filesystem.get_paths_from_directory(
+                        directory_path=archive_directory.path, get_file=True, recursive=False)
+                    total_archived_files += len(files_to_archive)
+                    archived_file: str = archive(archive_directory.path, include_root_directory=True)
+                    # Remove the original directory after archiving.
+                    shutil.rmtree(archive_directory.path, ignore_errors=True)
+                    archived_files.append(archived_file)
+
+                print_api.print_api(
+                    f'Archived: 'f'Directories: {len(archive_directories)} | '
+                    f'Total Files: {total_archived_files} | In: {directory_path.path}',
+                    logger=rec_packer_logger_with_queue_handler, color='blue')
+                print_api.print_api(f'Archived files: {archived_files}', logger=rec_packer_logger_with_queue_handler)
+
+        finalize_output_queue.put(None)
+
+        print_api.print_api(
+            'Finished recs archiver process.', color='blue',
+            logger=rec_packer_logger_with_queue_handler
+        )
+
+        return archived_files
+    except Exception as e:
+        print_api.print_api(
+            f"Error while archiving recs files: {e}",
+            color='red',
+            logger=rec_packer_logger_with_queue_handler
+        )
+
+        finalize_output_queue.put(e)
+        return None
+
+
+def recs_archiver_in_process(
+        recs_directory: str,
+        logging_queue: multiprocessing.Queue,
+        logger_name: str,
+        finalize_output_queue: multiprocessing.Queue
+) -> multiprocessing.Process:
+    """
+    Archive recs files in a directory for each day in a separate process.
+
+    :param recs_directory: The directory where recordings are stored.
+    :param logging_queue: The queue for logging messages.
+    :param logger_name: The name of the logger to use for logging.
+    :param finalize_output_queue: output queue for results/exceptions.
+    """
+
+    process = multiprocessing.Process(
+        target=recs_archiver, args=(recs_directory, logging_queue, logger_name, finalize_output_queue))
+    process.start()
+    return process

atomicshop/mitm/shared_functions.py

@@ -1,9 +1,8 @@
-import json
-# Needed to get the function caller module.
 import inspect
 
 from ..wrappers.loggingw import loggingw
-from ..basics import dicts
+
+from . import config_static
 
 
 # If the string has several dot characters (".") - return the most right string after the last dot.
@@ -33,12 +32,7 @@ def create_custom_logger():
     # 'f_globals' is a dictionary of all the global variables of the calling initiated class.
     class_name = calling_frame.f_globals['__name__']
     # Get the logger name only.
-    logger_name = build_module_names(class_name)[0]
+    engine_logger_part = build_module_names(class_name)[0]
+    logger_name = f'{config_static.MainConfig.LOGGER_NAME}.{engine_logger_part}'
 
     return loggingw.get_logger_with_level(logger_name)
-
-
-def get_json(obj):
-    """ Convert any nested object to json / dict and values to string as is """
-
-    return json.dumps(obj, default=dicts.convert_complex_object_to_dict)

atomicshop/mitm/ssh_tester.py

@@ -0,0 +1,82 @@
+import socket
+
+import paramiko
+
+from .. import package_mains_processor, ssh_remote, config_init
+from ..wrappers.socketw import process_getter
+from ..print_api import print_api
+
+
+PORT_TO_CMD_FILE: str = 'process_from_port'
+TCP_PORTS_FILE: str = 'get_local_tcp_ports'
+
+
+def test_ssh_main(config: dict) -> int:
+    hosts: list = config['main']['hosts_or_ips']
+
+    for host in hosts:
+        print("-----------------------------------")
+        print_api(f"Testing cmd for host: {host}", color='blue')
+
+        if host in config['main']:
+            print("Using host-specific credentials")
+            username = config[host]['user']
+            password = config[host]['pass']
+        else:
+            print("Didn't find host-specific credential, using defaults")
+            username = config['all_hosts']['user']
+            password = config['all_hosts']['pass']
+
+        ssh_client = ssh_remote.SSHRemote(ip_address=host, username=username, password=password)
+
+        try:
+            ssh_client.connect()
+        except socket.gaierror as e:
+            if e.errno == 11001:
+                print_api(f"Couldn't resolve IP to {host}: {str(e)}\n"
+                          f"Try providing IP address instead of hostname", color='red')
+                continue
+            else:
+                raise e
+        except paramiko.ssh_exception.NoValidConnectionsError as e:
+            print_api(f"Couldn't connect to {host}: {str(e)}", color='red')
+            continue
+
+        # Read the TCP ports file to string.
+        tcp_ports_package_processor: package_mains_processor.PackageMainsProcessor = package_mains_processor.PackageMainsProcessor(
+            script_file_stem=TCP_PORTS_FILE)
+        tcp_ports_script_string: str = tcp_ports_package_processor.read_script_file_to_string()
+
+        # Execute the TCP ports script remotely via SSH to get the list of open TCP ports.
+        tcp_ports_output, tcp_ports_error = ssh_client.remote_execution_python(script_string=tcp_ports_script_string)
+        if tcp_ports_error:
+            print_api(f"Error getting TCP ports from host {host}: {tcp_ports_error}", color='red')
+            continue
+
+        tcp_ports_list: list = tcp_ports_output.strip().splitlines()
+        if not tcp_ports_list:
+            print_api(f"No TCP ports found on host {host}", color='red')
+            continue
+
+        last_port: int = int(tcp_ports_list[-1])
+
+        port_to_cmd_package_processor: package_mains_processor.PackageMainsProcessor = package_mains_processor.PackageMainsProcessor(
+            script_file_stem=PORT_TO_CMD_FILE)
+        get_command_instance = process_getter.GetCommandLine(
+            client_ip=host,
+            client_port=last_port,
+            package_processor=port_to_cmd_package_processor,
+            ssh_client=ssh_client)
+        process_name = get_command_instance.get_process_name()
+        print(f"Process for port {last_port} on host {host}: {process_name}")
+
+        print("Closing SSH connection")
+        ssh_client.close()
+
+        if not process_name:
+            print_api(f"Failed to get process name for port {last_port} on host {host}", color='red')
+            continue
+
+        print_api(f"SSH test success!", color='green')
+
+    return 0

atomicshop/mitm/statistic_analyzer.py

@@ -2,6 +2,7 @@ import os
 import datetime
 import json
 from typing import Union, Literal
+import argparse
 
 from .statistic_analyzer_helper import analyzer_helper, moving_average_helper
 from .. import filesystem, domains, datetimes, urls
@@ -28,9 +29,9 @@ def analyze(main_file_path: str):
     summary_path: str = filesystem.check_absolute_path___add_full(config['report_file_path'], script_directory)
 
     # Get the content from statistics files.
+    log_file_path_pattern: str = f"{config['statistic_files_path']}{os.sep}statistics.csv"
     statistics_content: list = reading.get_all_log_files_into_list(
-        config['statistic_files_path'],
-        file_name_pattern='statistics*.csv',
+        log_file_path=log_file_path_pattern,
         log_type='csv'
     )
 
@@ -344,21 +345,33 @@ def analyze(main_file_path: str):
 # ======================================================================================================================
 
 
-def deviation_calculator_by_moving_average_main(
-        statistics_file_directory: str,
-        moving_average_window_days: int,
-        top_bottom_deviation_percentage: float,
+def deviation_calculator_by_moving_average(
+        statistics_file_directory: str = None,
+        statistics_content: dict = None,
+        by_type: Literal['host', 'url'] = 'url',
+        moving_average_window_days: int = 5,
+        top_bottom_deviation_percentage: float = 0.25,
         get_deviation_for_last_day_only: bool = False,
+        get_deviation_for_date: str = None,
         summary: bool = False,
         output_file_path: str = None,
-        output_file_type: Literal['json', 'csv'] = 'json'
+        output_file_type: Literal['json', 'csv'] = 'json',
+        convert_sizes_lists_and_ma_data_to_string: bool = False,
+        skip_total_count_less_than: int = None
 ) -> Union[list, None]:
     """
     This function is the main function for the moving average calculator.
 
-    :param statistics_file_directory: string, the directory where 'statistics.csv' file resides.
+    :param statistics_file_directory: string, can be either providing the directory where 'statistics.csv' file resides
+        or None. If None, the 'statistics_content' must be provided.
+        The directory where 'statistics.csv' file resides.
         Also, all the rotated files like: statistics_2021-01-01.csv, statistics_2021-01-02.csv, etc.
         These will be analyzed in the order of the date in the file name.
+    :param statistics_content: dict, if specified, this will be used instead of reading the files from the directory.
+        The dict should be a result of the 'atomicshop.mitm.statistic_analyzer_helper.moving_average_helper.get_all_files_content'.
+    :param by_type: string, 'host' or 'url'. The type of the deviation calculation.
+        'host' will calculate the deviation by the host name. Example: maps.google.com, yahoo.com, etc.
+        'url' will calculate the deviation by the URL. Example: maps.google.com/maps, yahoo.com/news, etc.
     :param moving_average_window_days: integer, the moving average window days.
     :param top_bottom_deviation_percentage: float, the top bottom deviation percentage. Example: 0.1 for 10%.
     :param get_deviation_for_last_day_only: bool, if True, only the last day will be analyzed.
@@ -374,10 +387,20 @@ def deviation_calculator_by_moving_average_main(
         Files 01 to 05 will be used for moving average and the file 06 for deviation.
         Meaning the average calculated for 2021-01-06 will be compared to the values moving average of 2021-01-01
         to 2021-01-05.
+    :param get_deviation_for_date: string, if specified, only the specified date will be analyzed.
+        The date must be in the format of 'YYYY-MM-DD'. Example: '2021-01-06'.
     :param summary: bool, if True, Only the summary will be generated without all the numbers that were used
         to calculate the averages and the moving average data.
     :param output_file_path: string, if None, no file will be written.
     :param output_file_type: string, the type of the output file. 'json' or 'csv'.
+    :param convert_sizes_lists_and_ma_data_to_string: bool, if True, the 'request_sizes', 'response_sizes' and 'ma_data'
+        will be converted to string. This is useful when writing to files, so the view will be more readable.
+    :param skip_total_count_less_than: integer, if specified, the deviation calculation will be skipped
+        if the total count is less than this number.
+        This means that if we have moving average window of 7 days, on the 8th day, the deviation will be calculated
+        only if the total count of the checked type
+        (request average size, response average, request count, response count)
+        is greater or equal to this number.
     -----------------------------
     :return: the deviation list of dicts.
 
@@ -399,11 +422,6 @@ def deviation_calculator_by_moving_average_main(
         sys.exit(main())
     """
 
-    if output_file_type not in ['json', 'csv']:
-        raise ValueError(f'output_file_type must be "json" or "csv", not [{output_file_type}]')
-
-    statistics_file_path: str = f'{statistics_file_directory}{os.sep}{STATISTICS_FILE_NAME}'
-
     def convert_data_value_to_string(value_key: str, list_index: int) -> None:
         deviation_list[list_index]['data'][value_key] = json.dumps(deviation_list[list_index]['data'][value_key])
 
@@ -411,43 +429,53 @@ def deviation_calculator_by_moving_average_main(
         if value_key in deviation_list[list_index]:
             deviation_list[list_index][value_key] = json.dumps(deviation_list[list_index][value_key])
 
+    if not statistics_file_directory and not statistics_content:
+        raise ValueError('Either [statistics_file_directory] or [statistics_content] must be provided.')
+    if statistics_file_directory and statistics_content:
+        raise ValueError('Either [statistics_file_directory] or [statistics_content] must be provided, not both.')
+
+    if output_file_type not in ['json', 'csv']:
+        raise ValueError(f'output_file_type must be "json" or "csv", not [{output_file_type}]')
+
+    if by_type not in ['host', 'url']:
+        raise ValueError(f'by_type must be "host" or "url", not [{by_type}]')
+
+    if get_deviation_for_last_day_only and get_deviation_for_date:
+        raise ValueError('Either [get_deviation_for_last_day_only] or [get_deviation_for_date] can be provided, not both.')
+
+    if statistics_file_directory:
+        statistics_file_path: str | None = f'{statistics_file_directory}{os.sep}{STATISTICS_FILE_NAME}'
+    else:
+        statistics_file_path: str | None = None
+
     deviation_list = moving_average_helper.calculate_moving_average(
         statistics_file_path,
+        statistics_content,
+        by_type,
         moving_average_window_days,
         top_bottom_deviation_percentage,
-        get_deviation_for_last_day_only
+        get_deviation_for_last_day_only,
+        get_deviation_for_date,
+        skip_total_count_less_than
     )
 
     if deviation_list:
+        if convert_sizes_lists_and_ma_data_to_string:
+            for deviation_list_index, deviation in enumerate(deviation_list):
+                convert_data_value_to_string('request_sizes', deviation_list_index)
+                convert_data_value_to_string('response_sizes', deviation_list_index)
+                convert_value_to_string('ma_data', deviation_list_index)
+
         if summary:
-            summary_deviation_list: list = []
             for deviation in deviation_list:
-                value = deviation.get('value', None)
-                ma_value = deviation.get('ma_value', None)
-                if not value or not ma_value:
-                    total_entries_averaged = None
-                else:
-                    total_entries_averaged = deviation['data']['count']
-                    
-                summary_deviation_list.append({
-                    'day': deviation['day'],
-                    'host': deviation['host'],
-                    'message': deviation['message'],
-                    'value': deviation.get('value', None),
-                    'ma_value': deviation.get('ma_value', None),
-                    'deviation_percentage': deviation.get('deviation_percentage', None),
-                    'total_entries_averaged': total_entries_averaged
-                })
-
-            deviation_list = summary_deviation_list
+                _ = deviation.pop('check_type')
+                _ = deviation.pop('percentage')
+                _ = deviation.pop('ma_value_checked')
+                _ = deviation.pop('deviation_type')
+                _ = deviation.pop('data')
+                _ = deviation.pop('ma_data')
 
         if output_file_path:
-            if not summary:
-                for deviation_list_index, deviation in enumerate(deviation_list):
-                    convert_data_value_to_string('request_sizes', deviation_list_index)
-                    convert_data_value_to_string('response_sizes', deviation_list_index)
-                    convert_value_to_string('ma_data', deviation_list_index)
-
             print_api(f'Deviation Found, saving to file: {output_file_path}', color='blue')
 
             if output_file_type == 'csv':
@@ -457,4 +485,72 @@ def deviation_calculator_by_moving_average_main(
 
         return deviation_list
 
-    return None
+    return []
+
+
+def deviation_calculator_by_moving_average_main():
+    def parse_args():
+        description_full: str = (
+            f'Description: Calculate deviation by moving average.')
+
+        parser = argparse.ArgumentParser(
+            description=description_full)
+
+        parser.add_argument(
+            '-d', '--directory', type=str, required=True,
+            help='Provide full path to directory with statistics.csv files.')
+        parser.add_argument(
+            '-of', '--output_file', type=str, required=True, help='Provide full path to output file.')
+        parser.add_argument(
+            '-ot', '--output_type', type=str, required=True, help='Provide output type: [json] or [csv].')
+        parser.add_argument(
+            '-by', '--by_type', type=str, required=True, help='Calculate by [host] or [url].')
+        parser.add_argument(
+            '-f', '--full_details', action='store_true', required=False,
+            help='(OPTIONAL) Output full processing details instead of summary.')
+        parser.add_argument(
+            '-w', '--window', type=int, required=True, help='Moving average window in days.')
+        parser.add_argument(
+            '-p', '--percentage', type=float, required=True,
+            help='Percentage of deviation from moving average. Example: 0.1 for 10%%.')
+        parser.add_argument(
+            '-slt', '--skip_total_count_less_than', type=int, required=False,
+            help='An integer to skip the deviation calculation if the total count is less than this number.')
+
+        return parser.parse_args()
+
+    args = parse_args()
+
+    if args.output_type == 'csv' and args.full_details:
+        print_api("Full details output is not supported for csv files.", color='red')
+        return 1
+
+    if not os.path.isdir(args.directory):
+        print_api(f"Directory does not exist: {args.directory}", color='red')
+        return 1
+
+    if not filesystem.get_paths_from_directory(
+            directory_path=args.directory, get_file=True, simple_list=True, file_name_check_pattern='*statistics*.csv'):
+        print_api(f"No statistics files found in: {args.directory}", color='red')
+        return 1
+
+    if args.full_details:
+        summary = False
+        convert_sizes_lists_and_ma_data_to_string = True
+    else:
+        summary = True
+        convert_sizes_lists_and_ma_data_to_string = False
+
+    _ = deviation_calculator_by_moving_average(
+        statistics_file_directory=args.directory,
+        by_type=args.by_type,
+        moving_average_window_days=args.window,
+        top_bottom_deviation_percentage=args.percentage,
+        summary=summary,
+        output_file_path=args.output_file,
+        output_file_type=args.output_type,
+        convert_sizes_lists_and_ma_data_to_string=convert_sizes_lists_and_ma_data_to_string,
+        skip_total_count_less_than=args.skip_total_count_less_than
+    )
+
+    return 0