atomicshop 3.2.5__py3-none-any.whl → 3.2.6__py3-none-any.whl

atomicshop/__init__.py

@@ -1,4 +1,4 @@
 """Atomic Basic functions and classes to make developer life easier"""
 
 __author__ = "Den Kras"
-__version__ = '3.2.5'
+__version__ = '3.2.6'

atomicshop/mitm/mitm_main.py

@@ -3,6 +3,7 @@ import multiprocessing
 import time
 import datetime
 import os
+import logging
 
 import atomicshop   # Importing atomicshop package to get the version of the package.
 
@@ -71,6 +72,9 @@ MITM_ERROR_LOGGER: loggingw.ExceptionCsvLogger = None
 # Create logger's queue.
 NETWORK_LOGGER_QUEUE: multiprocessing.Queue = multiprocessing.Queue()
 
+# Create finalization queue for the rec archiving process.
+FINALIZE_RECS_ARCHIVE_QUEUE: multiprocessing.Queue = multiprocessing.Queue()
+
 
 try:
     win_console.disable_quick_edit()
@@ -115,7 +119,7 @@ def exit_cleanup():
 
     # The process will not be executed if there was an exception in the beginning.
     if RECS_PROCESS_INSTANCE is not None:
-        print_api.print_api(RECS_PROCESS_INSTANCE.is_alive())
+        print_api.print_api(f'Recs archive process alive: {RECS_PROCESS_INSTANCE.is_alive()}')
         RECS_PROCESS_INSTANCE.terminate()
         RECS_PROCESS_INSTANCE.join()
 
@@ -324,12 +328,6 @@ def mitm_server(config_file_path: str, script_version: str):
     # Create folders.
     filesystem.create_directory(config_static.LogRec.logs_path)
 
-    if config_static.LogRec.enable_request_response_recordings_in_logs:
-        filesystem.create_directory(config_static.LogRec.recordings_path)
-        # Compress recordings of the previous days if there are any.
-        global RECS_PROCESS_INSTANCE
-        RECS_PROCESS_INSTANCE = recs_files.recs_archiver_in_process(config_static.LogRec.recordings_path)
-
     if config_static.Certificates.sni_get_server_certificate_from_server_socket:
         filesystem.create_directory(
             config_static.Certificates.sni_server_certificate_from_server_socket_download_directory)
@@ -347,14 +345,35 @@ def mitm_server(config_file_path: str, script_version: str):
         formatter_filehandler='DEFAULT',
         backupCount=config_static.LogRec.store_logs_for_x_days)
 
-    network_logger_with_queue_handler = loggingw.create_logger(
+    network_logger_with_queue_handler: logging.Logger = loggingw.create_logger(
         logger_name=network_logger_name,
         add_queue_handler=True,
         log_queue=NETWORK_LOGGER_QUEUE)
 
     # Initiate Listener logger, which is a child of network logger, so he uses the same settings and handlers
-    listener_logger = loggingw.get_logger_with_level(f'{network_logger_name}.listener')
-    system_logger = loggingw.get_logger_with_level(f'{network_logger_name}.system')
+    listener_logger: logging.Logger = loggingw.get_logger_with_level(f'{network_logger_name}.listener')
+    system_logger: logging.Logger = loggingw.get_logger_with_level(f'{network_logger_name}.system')
+
+    if config_static.LogRec.enable_request_response_recordings_in_logs:
+        filesystem.create_directory(config_static.LogRec.recordings_path)
+        # Compress recordings of the previous days if there are any.
+        global RECS_PROCESS_INSTANCE
+        RECS_PROCESS_INSTANCE = recs_files.recs_archiver_in_process(
+            config_static.LogRec.recordings_path,
+            logging_queue=NETWORK_LOGGER_QUEUE,
+            logger_name=network_logger_name,
+            finalize_output_queue=FINALIZE_RECS_ARCHIVE_QUEUE
+        )
+
+        archiver_result = FINALIZE_RECS_ARCHIVE_QUEUE.get()
+        if isinstance(archiver_result, Exception):
+            print_api.print_api(
+                f"Error while archiving recordings: {archiver_result}",
+                error_type=True, color="red", logger=system_logger, logger_method='critical')
+            # Wait for the message to be printed and saved to file.
+            time.sleep(1)
+            network_logger_queue_listener.stop()
+            return 1
 
     # Logging Startup information.
     startup_output(system_logger, script_version)
@@ -528,7 +547,9 @@ def mitm_server(config_file_path: str, script_version: str):
 
         socket_wrapper_instance.start_listening_sockets(
             reference_function_name=thread_worker_main,
-            reference_function_args=(network_logger_with_queue_handler, statistics_writer, config_static.ENGINES_LIST, config_static.REFERENCE_MODULE)
+            reference_function_args=(
+                network_logger_with_queue_handler, statistics_writer, config_static.ENGINES_LIST,
+                config_static.REFERENCE_MODULE)
         )
 
         # socket_thread = threading.Thread(
@@ -544,8 +565,7 @@ def mitm_server(config_file_path: str, script_version: str):
         # socket_thread.start()
 
         # Compress recordings each day in a separate process.
-        recs_archiver_thread = threading.Thread(target=_loop_at_midnight_recs_archive)
-        recs_archiver_thread.daemon = True
+        recs_archiver_thread = threading.Thread(target=_loop_at_midnight_recs_archive, args=(network_logger_name,), daemon=True)
         recs_archiver_thread.start()
 
     if config_static.DNSServer.enable or config_static.TCPServer.enable:
@@ -554,7 +574,7 @@ def mitm_server(config_file_path: str, script_version: str):
             time.sleep(1)
 
 
-def _loop_at_midnight_recs_archive():
+def _loop_at_midnight_recs_archive(network_logger_name):
     previous_date = datetime.datetime.now().strftime('%d')
     while True:
         # Get current time.
@@ -563,7 +583,17 @@ def _loop_at_midnight_recs_archive():
         if current_date != previous_date:
             if config_static.LogRec.enable_request_response_recordings_in_logs:
                 global RECS_PROCESS_INSTANCE
-                RECS_PROCESS_INSTANCE = recs_files.recs_archiver_in_process(config_static.LogRec.recordings_path)
+                RECS_PROCESS_INSTANCE = recs_files.recs_archiver_in_process(
+                    config_static.LogRec.recordings_path,
+                    logging_queue=NETWORK_LOGGER_QUEUE,
+                    logger_name=network_logger_name,
+                    finalize_output_queue=FINALIZE_RECS_ARCHIVE_QUEUE
+                )
+
+                archiver_result = FINALIZE_RECS_ARCHIVE_QUEUE.get()
+                if isinstance(archiver_result, Exception):
+                    MITM_ERROR_LOGGER.write(archiver_result)
+
             # Update the previous date.
             previous_date = current_date
         # Sleep for 1 minute.

atomicshop/mitm/recs_files.py

@@ -1,10 +1,11 @@
 import datetime
 import os
 import multiprocessing
+import logging
 
 from ..archiver import zips
-from .. import filesystem
-from .. wrappers.loggingw import consts
+from .. import filesystem, print_api
+from .. wrappers.loggingw import consts, loggingw
 
 
 REC_FILE_DATE_TIME_MILLISECONDS_FORMAT: str = f'{consts.DEFAULT_ROTATING_SUFFIXES_FROM_WHEN["S"]}_%f'
@@ -12,14 +13,35 @@ REC_FILE_DATE_TIME_FORMAT: str = f'{consts.DEFAULT_ROTATING_SUFFIXES_FROM_WHEN["
 REC_FILE_DATE_FORMAT: str = REC_FILE_DATE_TIME_FORMAT.split('_')[0]
 
 
-def recs_archiver(recs_directory: str) -> list:
+def recs_archiver(
+        recs_directory: str,
+        logging_queue: multiprocessing.Queue,
+        logger_name: str,
+        finalize_output_queue: multiprocessing.Queue
+) -> list | None:
     """
     Find recs files in a directory for each day.
     Each day of recordings will have separate archive.
 
     :param recs_directory: The directory where recordings are stored.
+    :param logging_queue: The queue for logging messages.
+    :param logger_name: The name of the logger to use for logging.
+        This is the base name that '.rec_packer' will be added to it.
+    :param finalize_output_queue: output queue for results/exceptions.
     """
 
+    logger_name = f"{logger_name}.rec_packer"
+
+    rec_packer_logger_with_queue_handler: logging.Logger = loggingw.create_logger(
+        logger_name=logger_name,
+        add_queue_handler=True,
+        log_queue=logging_queue)
+
+    print_api.print_api(
+        'Starting recs archiver process.', color='blue',
+        logger=rec_packer_logger_with_queue_handler
+    )
+
     today_date_string = datetime.datetime.now().strftime(REC_FILE_DATE_FORMAT)
 
     # There should not be recording json files in recs root.
@@ -43,34 +65,58 @@ def recs_archiver(recs_directory: str) -> list:
         )
         file_list_per_directory.append((directory_path, all_recs_files))
 
-    archived_files: list = list()
-    for directory_path, all_recs_files in file_list_per_directory:
-        for recs_atomic_path in all_recs_files:
-            # We don't need to archive today's files.
-            if today_date_string == recs_atomic_path.datetime_string:
-                continue
-
-            target_directory_path: str = f"{directory_path.path}{os.sep}{recs_atomic_path.datetime_string}"
-            filesystem.create_directory(target_directory_path)
-            filesystem.move_file(
-                recs_atomic_path.path, target_directory_path)
 
-        # Archive directories.
-        archive_directories: list = filesystem.get_paths_from_directory(
-            directory_path.path, get_directory=True, recursive=False)
-        for archive_directory in archive_directories:
-            archived_file: str = zips.archive_directory(
-                archive_directory.path, remove_original=True, include_root_directory=True)
-            archived_files.append(archived_file)
+    try:
+        archived_files: list = list()
+        for directory_path, all_recs_files in file_list_per_directory:
+            for recs_atomic_path in all_recs_files:
+                # We don't need to archive today's files.
+                if today_date_string == recs_atomic_path.datetime_string:
+                    continue
+
+                target_directory_path: str = f"{directory_path.path}{os.sep}{recs_atomic_path.datetime_string}"
+                filesystem.create_directory(target_directory_path)
+                filesystem.move_file(
+                    recs_atomic_path.path, target_directory_path)
+
+            # Archive directories.
+            archive_directories: list = filesystem.get_paths_from_directory(
+                directory_path.path, get_directory=True, recursive=False)
+            for archive_directory in archive_directories:
+                archived_file: str = zips.archive_directory(
+                    archive_directory.path, remove_original=True, include_root_directory=True)
+                archived_files.append(archived_file)
+
+        finalize_output_queue.put(None)
+
+        return archived_files
+    except Exception as e:
+        print_api.print_api(
+            f"Error while archiving recs files: {e}",
+            color='red',
+            logger=rec_packer_logger_with_queue_handler
+        )
 
-    return archived_files
+        finalize_output_queue.put(e)
+        return None
 
 
-def recs_archiver_in_process(recs_directory: str) -> multiprocessing.Process:
+def recs_archiver_in_process(
+        recs_directory: str,
+        logging_queue: multiprocessing.Queue,
+        logger_name: str,
+        finalize_output_queue: multiprocessing.Queue
+) -> multiprocessing.Process:
     """
     Archive recs files in a directory for each day in a separate process.
+
+    :param recs_directory: The directory where recordings are stored.
+    :param logging_queue: The queue for logging messages.
+    :param logger_name: The name of the logger to use for logging.
+    :param finalize_output_queue: output queue for results/exceptions.
     """
 
-    process = multiprocessing.Process(target=recs_archiver, args=(recs_directory,))
+    process = multiprocessing.Process(
+        target=recs_archiver, args=(recs_directory, logging_queue, logger_name, finalize_output_queue))
     process.start()
     return process

atomicshop/wrappers/socketw/dns_server.py

@@ -948,6 +948,7 @@ def start_dns_server_multiprocessing_worker(
             logger_name=logger_name
         )
     except (DnsPortInUseError, DnsConfigurationValuesError) as e:
+        _ = e
         # Wait for the message to be printed and saved to file.
         time.sleep(1)
         return 1

{atomicshop-3.2.5.dist-info → atomicshop-3.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: atomicshop
-Version: 3.2.5
+Version: 3.2.6
 Summary: Atomic functions and classes to make developer life easier
 Author: Denis Kras
 License: MIT License

{atomicshop-3.2.5.dist-info → atomicshop-3.2.6.dist-info}/RECORD

@@ -1,4 +1,4 @@
-atomicshop/__init__.py,sha256=Fe2MPgseNiPA-8oAyxLSss-5daH3Di8iuZ8RhRCAKPk,122
+atomicshop/__init__.py,sha256=E9W72TNe3wsH81NNRHbb0v1IMM-1rJY-s7J7IozQvlU,122
 atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
 atomicshop/_create_pdf_demo.py,sha256=Yi-PGZuMg0RKvQmLqVeLIZYadqEZwUm-4A9JxBl_vYA,3713
 atomicshop/_patch_import.py,sha256=ENp55sKVJ0e6-4lBvZnpz9PQCt3Otbur7F6aXDlyje4,6334
@@ -140,8 +140,8 @@ atomicshop/mitm/connection_thread_worker.py,sha256=zo9YVliRtanmt3BDnxlePOuZLYk8H
 atomicshop/mitm/import_config.py,sha256=_RJPxoVEEsJQwQY-K_8qPw2PbzsDcV23_sEJEWSKqXY,17831
 atomicshop/mitm/initialize_engines.py,sha256=EYGW6gEYVETmmh9uNdsFT89YAdzeducFTIkdF-as33s,10534
 atomicshop/mitm/message.py,sha256=CDhhm4BTuZE7oNZCjvIZ4BuPOW4MuIzQLOg91hJaxDI,3065
-atomicshop/mitm/mitm_main.py,sha256=at9uxnXF_4b273HRXt9UGkERkHantRTOAvRtlYYI68E,30774
-atomicshop/mitm/recs_files.py,sha256=ZAAD0twun-FtmbSniXe3XQhIlawvANNB_HxwbHj7kwI,3151
+atomicshop/mitm/mitm_main.py,sha256=StOAvsswTx325kVIXjHiH97QP2Syxq5o0mnoeiPsy_4,32149
+atomicshop/mitm/recs_files.py,sha256=cxmM52CNg53mpQhVTc_kdppXCuPQWcSHEsEqifz8HFQ,4856
 atomicshop/mitm/shared_functions.py,sha256=0lzeyINd44sVEfFbahJxQmz6KAMWbYrW5ou3UYfItvw,1777
 atomicshop/mitm/statistic_analyzer.py,sha256=5_sAYGX2Xunzo_pS2W5WijNCwr_BlGJbbOO462y_wN4,27533
 atomicshop/mitm/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -323,7 +323,7 @@ atomicshop/wrappers/socketw/accepter.py,sha256=4I9ORugRDvwaqSzm_gWSjZnRwQGY8hDTl
 atomicshop/wrappers/socketw/base.py,sha256=EcosGkD8VzgBY3GeIHDSG29ThQfXwg3-GQPmBTAqTdw,3048
 atomicshop/wrappers/socketw/certificator.py,sha256=mtWPJ_ew3OSwt0-1W4jaoco1VIY4NRCrMv3mDUxb_Cc,12418
 atomicshop/wrappers/socketw/creator.py,sha256=LGI4gcgJ47thx6f96rjwjPz3CsTAIv6VxWFY4EyUF2E,13667
-atomicshop/wrappers/socketw/dns_server.py,sha256=aWrO16-c9Zt5SnVnmTUEv_j7XENYVBz1udIW9qMeC18,54011
+atomicshop/wrappers/socketw/dns_server.py,sha256=6dtZT3foDy4tKGQKC7JL1eN4kWfUYCepVm8nUA4lpoo,54026
 atomicshop/wrappers/socketw/exception_wrapper.py,sha256=qW_1CKyPgGlsIt7_jusKkMV4A4hih4bX324u0PLnoO8,7382
 atomicshop/wrappers/socketw/get_process.py,sha256=aJC-_qFUv3NgWCSUzDI72E4z8_-VTZE9NVZ0CwUoNlM,5698
 atomicshop/wrappers/socketw/receiver.py,sha256=9B3MvcDqr4C3x2fsnjG5SQognd1wRqsBgikxZa0wXG8,8243
@@ -337,8 +337,8 @@ atomicshop/wrappers/socketw/statistics_csv.py,sha256=WcNyaqEZ82S5-f3kzqi1nllNT2N
 atomicshop/wrappers/winregw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/wrappers/winregw/winreg_installed_software.py,sha256=Qzmyktvob1qp6Tjk2DjLfAqr_yXV0sgWzdMW_9kwNjY,2345
 atomicshop/wrappers/winregw/winreg_network.py,sha256=ih0BVNwByLvf9F_Lac4EdmDYYJA3PzMvmG0PieDZrsE,9905
-atomicshop-3.2.5.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
-atomicshop-3.2.5.dist-info/METADATA,sha256=vQliSRQcXSJH2H5esW4IpzObnUX8s3xRyTc27tJXgpM,10670
-atomicshop-3.2.5.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-atomicshop-3.2.5.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
-atomicshop-3.2.5.dist-info/RECORD,,
+atomicshop-3.2.6.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
+atomicshop-3.2.6.dist-info/METADATA,sha256=u4Jh_IqIDPxXiAdCQiTWxBgvgJERae4r_wc513d6GFU,10670
+atomicshop-3.2.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+atomicshop-3.2.6.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
+atomicshop-3.2.6.dist-info/RECORD,,