atomicshop 3.3.8__py3-none-any.whl → 3.10.0__py3-none-any.whl

atomicshop/mitm/ssh_tester.py

@@ -0,0 +1,82 @@
+import socket
+
+import paramiko
+
+from .. import package_mains_processor, ssh_remote, config_init
+from ..wrappers.socketw import process_getter
+from ..print_api import print_api
+
+
+PORT_TO_CMD_FILE: str = 'process_from_port'
+TCP_PORTS_FILE: str = 'get_local_tcp_ports'
+
+
+def test_ssh_main(config: dict) -> int:
+    hosts: list = config['main']['hosts_or_ips']
+
+    for host in hosts:
+        print("-----------------------------------")
+        print_api(f"Testing cmd for host: {host}", color='blue')
+
+        if host in config['main']:
+            print("Using host-specific credentials")
+            username = config[host]['user']
+            password = config[host]['pass']
+        else:
+            print("Didn't find host-specific credential, using defaults")
+            username = config['all_hosts']['user']
+            password = config['all_hosts']['pass']
+
+        ssh_client = ssh_remote.SSHRemote(ip_address=host, username=username, password=password)
+
+        try:
+            ssh_client.connect()
+        except socket.gaierror as e:
+            if e.errno == 11001:
+                print_api(f"Couldn't resolve IP to {host}: {str(e)}\n"
+                          f"Try providing IP address instead of hostname", color='red')
+                continue
+            else:
+                raise e
+        except paramiko.ssh_exception.NoValidConnectionsError as e:
+            print_api(f"Couldn't connect to {host}: {str(e)}", color='red')
+            continue
+
+        # Read the TCP ports file to string.
+        tcp_ports_package_processor: package_mains_processor.PackageMainsProcessor = package_mains_processor.PackageMainsProcessor(
+            script_file_stem=TCP_PORTS_FILE)
+        tcp_ports_script_string: str = tcp_ports_package_processor.read_script_file_to_string()
+
+        # Execute the TCP ports script remotely via SSH to get the list of open TCP ports.
+        tcp_ports_output, tcp_ports_error = ssh_client.remote_execution_python(script_string=tcp_ports_script_string)
+        if tcp_ports_error:
+            print_api(f"Error getting TCP ports from host {host}: {tcp_ports_error}", color='red')
+            continue
+
+        tcp_ports_list: list = tcp_ports_output.strip().splitlines()
+        if not tcp_ports_list:
+            print_api(f"No TCP ports found on host {host}", color='red')
+            continue
+
+        last_port: int = int(tcp_ports_list[-1])
+
+        port_to_cmd_package_processor: package_mains_processor.PackageMainsProcessor = package_mains_processor.PackageMainsProcessor(
+            script_file_stem=PORT_TO_CMD_FILE)
+        get_command_instance = process_getter.GetCommandLine(
+            client_ip=host,
+            client_port=last_port,
+            package_processor=port_to_cmd_package_processor,
+            ssh_client=ssh_client)
+        process_name = get_command_instance.get_process_name()
+        print(f"Process for port {last_port} on host {host}: {process_name}")
+
+        print("Closing SSH connection")
+        ssh_client.close()
+
+        if not process_name:
+            print_api(f"Failed to get process name for port {last_port} on host {host}", color='red')
+            continue
+
+        print_api(f"SSH test success!", color='green')
+
+    return 0

atomicshop/mitm/statistic_analyzer.py

@@ -346,11 +346,13 @@ def analyze(main_file_path: str):
 
 
 def deviation_calculator_by_moving_average(
-        statistics_file_directory: str,
-        by_type: Literal['host', 'url'],
-        moving_average_window_days: int,
-        top_bottom_deviation_percentage: float,
+        statistics_file_directory: str = None,
+        statistics_content: dict = None,
+        by_type: Literal['host', 'url'] = 'url',
+        moving_average_window_days: int = 5,
+        top_bottom_deviation_percentage: float = 0.25,
         get_deviation_for_last_day_only: bool = False,
+        get_deviation_for_date: str = None,
         summary: bool = False,
         output_file_path: str = None,
         output_file_type: Literal['json', 'csv'] = 'json',
@@ -360,9 +362,13 @@ def deviation_calculator_by_moving_average(
     """
     This function is the main function for the moving average calculator.
 
-    :param statistics_file_directory: string, the directory where 'statistics.csv' file resides.
+    :param statistics_file_directory: string, can be either providing the directory where 'statistics.csv' file resides
+        or None. If None, the 'statistics_content' must be provided.
+        The directory where 'statistics.csv' file resides.
         Also, all the rotated files like: statistics_2021-01-01.csv, statistics_2021-01-02.csv, etc.
         These will be analyzed in the order of the date in the file name.
+    :param statistics_content: dict, if specified, this will be used instead of reading the files from the directory.
+        The dict should be a result of the 'atomicshop.mitm.statistic_analyzer_helper.moving_average_helper.get_all_files_content'.
     :param by_type: string, 'host' or 'url'. The type of the deviation calculation.
         'host' will calculate the deviation by the host name. Example: maps.google.com, yahoo.com, etc.
         'url' will calculate the deviation by the URL. Example: maps.google.com/maps, yahoo.com/news, etc.
@@ -381,6 +387,8 @@ def deviation_calculator_by_moving_average(
         Files 01 to 05 will be used for moving average and the file 06 for deviation.
         Meaning the average calculated for 2021-01-06 will be compared to the values moving average of 2021-01-01
         to 2021-01-05.
+    :param get_deviation_for_date: string, if specified, only the specified date will be analyzed.
+        The date must be in the format of 'YYYY-MM-DD'. Example: '2021-01-06'.
     :param summary: bool, if True, Only the summary will be generated without all the numbers that were used
         to calculate the averages and the moving average data.
     :param output_file_path: string, if None, no file will be written.
@@ -414,27 +422,40 @@ def deviation_calculator_by_moving_average(
         sys.exit(main())
     """
 
+    def convert_data_value_to_string(value_key: str, list_index: int) -> None:
+        deviation_list[list_index]['data'][value_key] = json.dumps(deviation_list[list_index]['data'][value_key])
+
+    def convert_value_to_string(value_key: str, list_index: int) -> None:
+        if value_key in deviation_list[list_index]:
+            deviation_list[list_index][value_key] = json.dumps(deviation_list[list_index][value_key])
+
+    if not statistics_file_directory and not statistics_content:
+        raise ValueError('Either [statistics_file_directory] or [statistics_content] must be provided.')
+    if statistics_file_directory and statistics_content:
+        raise ValueError('Either [statistics_file_directory] or [statistics_content] must be provided, not both.')
+
     if output_file_type not in ['json', 'csv']:
         raise ValueError(f'output_file_type must be "json" or "csv", not [{output_file_type}]')
 
     if by_type not in ['host', 'url']:
         raise ValueError(f'by_type must be "host" or "url", not [{by_type}]')
 
-    statistics_file_path: str = f'{statistics_file_directory}{os.sep}{STATISTICS_FILE_NAME}'
+    if get_deviation_for_last_day_only and get_deviation_for_date:
+        raise ValueError('Either [get_deviation_for_last_day_only] or [get_deviation_for_date] can be provided, not both.')
 
-    def convert_data_value_to_string(value_key: str, list_index: int) -> None:
-        deviation_list[list_index]['data'][value_key] = json.dumps(deviation_list[list_index]['data'][value_key])
-
-    def convert_value_to_string(value_key: str, list_index: int) -> None:
-        if value_key in deviation_list[list_index]:
-            deviation_list[list_index][value_key] = json.dumps(deviation_list[list_index][value_key])
+    if statistics_file_directory:
+        statistics_file_path: str | None = f'{statistics_file_directory}{os.sep}{STATISTICS_FILE_NAME}'
+    else:
+        statistics_file_path: str | None = None
 
     deviation_list = moving_average_helper.calculate_moving_average(
         statistics_file_path,
+        statistics_content,
         by_type,
         moving_average_window_days,
         top_bottom_deviation_percentage,
         get_deviation_for_last_day_only,
+        get_deviation_for_date,
         skip_total_count_less_than
     )
 

atomicshop/mitm/statistic_analyzer_helper/moving_average_helper.py

@@ -1,6 +1,7 @@
 import statistics
 from pathlib import Path
 from typing import Literal
+import datetime
 
 from ...print_api import print_api
 from ...wrappers.loggingw import reading, consts
@@ -9,11 +10,13 @@ from ... import urls, filesystem
 
 
 def calculate_moving_average(
-        file_path: str,
-        by_type: Literal['host', 'url'],
-        moving_average_window_days,
-        top_bottom_deviation_percentage: float,
+        file_path: str = None,
+        statistics_content: dict = None,
+        by_type: Literal['host', 'url'] = 'url',
+        moving_average_window_days: int = 5,
+        top_bottom_deviation_percentage: float = 0.25,
         get_deviation_for_last_day_only: bool = False,
+        get_deviation_for_date: str = None,
         skip_total_count_less_than: int = None,
         print_kwargs: dict = None
 ) -> list:
@@ -21,14 +24,75 @@ def calculate_moving_average(
     This function calculates the moving average of the daily statistics.
 
     :param file_path: string, the path to the 'statistics.csv' file.
+    :param statistics_content: dict, the statistics content dictionary. If provided, 'file_path' will be ignored.
+        The dictionary should be in the format returned by 'get_all_files_content' function.
     :param by_type: string, the type to calculate the moving average by. Can be 'host' or 'url'.
     :param moving_average_window_days: integer, the window size for the moving average.
     :param top_bottom_deviation_percentage: float, the percentage of deviation from the moving average to the top or
         bottom.
+    :param get_deviation_for_last_day_only: bool, check the 'get_all_files_content' function.
+    :param get_deviation_for_date: str, check the 'get_all_files_content' function.
+    :param skip_total_count_less_than: integer, if the total count is less than this number, skip the deviation.
+    :param print_kwargs: dict, the print_api arguments.
+    """
+
+    if not file_path and not statistics_content:
+        raise ValueError('Either file_path or statistics_content must be provided.')
+    if file_path and statistics_content:
+        raise ValueError('Only one of file_path or statistics_content must be provided.')
+
+    if get_deviation_for_last_day_only and get_deviation_for_date:
+        raise ValueError('Only one of get_deviation_for_last_day_only or get_deviation_for_date can be set.')
+
+    if not statistics_content:
+        statistics_content: dict = get_all_files_content(
+            file_path=file_path, moving_average_window_days=moving_average_window_days,
+            get_deviation_for_last_day_only=get_deviation_for_last_day_only,
+            get_deviation_for_date=get_deviation_for_date,
+            print_kwargs=print_kwargs)
+
+    for date_string, day_dict in statistics_content.items():
+        day_dict['content_no_useless'] = get_content_without_useless(day_dict['content'])
+
+        # Get the data dictionary from the statistics content.
+        day_dict['statistics_daily'] = compute_statistics_from_content(
+            day_dict['content_no_useless'], by_type)
+
+    moving_average_dict: dict = compute_moving_averages_from_average_statistics(
+        statistics_content,
+        moving_average_window_days
+    )
+
+    # Add the moving average to the statistics content.
+    for day, day_dict in statistics_content.items():
+        try:
+            day_dict['moving_average'] = moving_average_dict[day]
+        except KeyError:
+            day_dict['moving_average'] = {}
+
+    # Find deviation from the moving average to the bottom or top by specified percentage.
+    deviation_list: list = find_deviation_from_moving_average(
+        statistics_content, top_bottom_deviation_percentage, skip_total_count_less_than)
+
+    return deviation_list
+
+
+def get_all_files_content(
+        file_path: str,
+        moving_average_window_days: int,
+        get_deviation_for_last_day_only: bool = False,
+        get_deviation_for_date: str = None,
+        print_kwargs: dict = None
+) -> dict:
+    """
+    Get the dictionary that will contain all the details of the file, like date, header and content, to prepare for the MA analysis.
+
+    :param file_path: string, the path to the 'statistics.csv' file.
+    :param moving_average_window_days: integer, the window size for the moving average.
     :param get_deviation_for_last_day_only: bool, if True, only the last day will be analyzed.
         Example: With 'moving_average_window_days=5', the last 6 days will be analyzed.
         5 days for moving average and the last day for deviation.
-        File names example:
+        File names example the last day is 2021-01-06:
             statistics_2021-01-01.csv
             statistics_2021-01-02.csv
             statistics_2021-01-03.csv
@@ -38,12 +102,24 @@ def calculate_moving_average(
         Files 01 to 05 will be used for moving average and the file 06 for deviation.
         Meaning the average calculated for 2021-01-06 will be compared to the values moving average of 2021-01-01
         to 2021-01-05.
-    :param skip_total_count_less_than: integer, if the total count is less than this number, skip the deviation.
+    :param get_deviation_for_date: str, if set, the last day is considered the date that you set here.
+        The format should be the same as in the file names, e.g. 'YYYY-MM-DD'.
     :param print_kwargs: dict, the print_api arguments.
+    :return:
     """
 
+    if get_deviation_for_last_day_only and get_deviation_for_date:
+        raise ValueError('Only one of get_deviation_for_last_day_only or get_deviation_for_date can be set.')
+
     date_format: str = consts.DEFAULT_ROTATING_SUFFIXES_FROM_WHEN['midnight']
 
+    def is_valid_date(date_str: str) -> bool:
+        try:
+            datetime.datetime.strptime(date_str, date_format)
+            return True
+        except ValueError:
+            return False
+
     # Get all the file paths and their midnight rotations.
     logs_paths: list[filesystem.AtomicPath] = reading.get_logs_paths(
         log_file_path=file_path,
@@ -54,6 +130,24 @@ def calculate_moving_average(
         days_back_to_analyze: int = moving_average_window_days + 1
         logs_paths = logs_paths[-days_back_to_analyze:]
 
+    if get_deviation_for_date:
+        # Check if the date format is correct.
+        if not is_valid_date(get_deviation_for_date):
+            raise ValueError(f'Date [{get_deviation_for_date}] is not in the correct format: {date_format}')
+
+        # Find the index of the date in the logs_paths list.
+        date_index: int | None = None
+        for index, log_atomic_path in enumerate(logs_paths):
+            if log_atomic_path.datetime_string == get_deviation_for_date:
+                date_index = index
+                break
+
+        if date_index is None:
+            raise ValueError(f'Date {get_deviation_for_date} not found in the log files.')
+
+        start_index: int = max(0, date_index - moving_average_window_days)
+        logs_paths = logs_paths[start_index:date_index + 1]
+
     statistics_content: dict = {}
     # Read each file to its day.
     for log_atomic_path in logs_paths:
@@ -67,29 +161,7 @@ def calculate_moving_average(
         statistics_content[date_string]['content'] = log_file_content
         statistics_content[date_string]['header'] = log_file_header
 
-        statistics_content[date_string]['content_no_useless'] = get_content_without_useless(log_file_content)
-
-        # Get the data dictionary from the statistics content.
-        statistics_content[date_string]['statistics_daily'] = compute_statistics_from_content(
-            statistics_content[date_string]['content_no_useless'], by_type)
-
-    moving_average_dict: dict = compute_moving_averages_from_average_statistics(
-        statistics_content,
-        moving_average_window_days
-    )
-
-    # Add the moving average to the statistics content.
-    for day, day_dict in statistics_content.items():
-        try:
-            day_dict['moving_average'] = moving_average_dict[day]
-        except KeyError:
-            day_dict['moving_average'] = {}
-
-    # Find deviation from the moving average to the bottom or top by specified percentage.
-    deviation_list: list = find_deviation_from_moving_average(
-        statistics_content, top_bottom_deviation_percentage, skip_total_count_less_than)
-
-    return deviation_list
+    return statistics_content
 
 
 def get_content_without_useless(content: list) -> list:
@@ -227,7 +299,7 @@ def compute_moving_averages_from_average_statistics(
 
         # Create list of the last 'moving_average_window_days' days, including the current day.
         last_x_window_days_content_list = (
-            list(average_statistics_dict.values()))[current_day-moving_average_window_days:current_day]
+            list(average_statistics_dict.values()))[current_day - moving_average_window_days:current_day]
 
         # Compute the moving averages.
         moving_average[day] = compute_average_for_current_day_from_past_x_days(
@@ -410,7 +482,8 @@ def find_deviation_from_moving_average(
         if day_index == 0:
             previous_day_moving_average_dict = {}
         else:
-            previous_day_moving_average_dict = list(statistics_content.values())[day_index-1].get('moving_average', {})
+            previous_day_moving_average_dict = list(statistics_content.values())[day_index - 1].get('moving_average',
+                                                                                                    {})
 
         # If there is no moving average for previous day continue to the next day.
         if not previous_day_moving_average_dict: