atomicshop 2.19.19__py3-none-any.whl → 2.20.1__py3-none-any.whl

atomicshop/wrappers/loggingw/loggingw.py

@@ -1,9 +1,13 @@
 import logging
 import os
+from logging import Logger
+from logging.handlers import QueueListener
 from typing import Literal, Union
 import datetime
 import contextlib
 import threading
+import queue
+import multiprocessing
 
 from . import loggers, handlers, filters
 from ...file_io import csvs
@@ -17,11 +21,17 @@ class LoggingwLoggerAlreadyExistsError(Exception):
 
 # noinspection PyPep8Naming
 def create_logger(
-        logger_name: str,
-        file_path: str = None,
-        directory_path: str = None,
+        logger_name: str = None,
+        get_queue_listener: bool = False,
+
         add_stream: bool = False,
         add_timedfile: bool = False,
+        add_timedfile_with_internal_queue: bool = False,
+        add_queue_handler: bool = False,
+
+        log_queue: Union[queue.Queue, multiprocessing.Queue] = None,
+        file_path: str = None,
+        directory_path: str = None,
         file_type: Literal[
             'txt',
             'csv',
@@ -47,17 +57,30 @@ def create_logger(
         delay: bool = False,
         encoding=None,
         header: str = None
-) -> logging.Logger:
+) -> None | QueueListener | Logger:
     """
     Function to get a logger and add StreamHandler and TimedRotatingFileHandler to it.
 
     :param logger_name: Name of the logger.
+    :param get_queue_listener: bool, If set to True, QueueListener will be started with all the handlers
+        like 'add_timedfile' and 'add_stream', using the 'log_queue'.
+
+    Only one of the following parameters can be set at a time: 'logger_name', 'get_queue_listener'.
+
     :param file_path: full path to the log file. If you don't want to use the file, set it to None.
         You can set the directory_path only and then the 'logger_name' will be used as the file name with the
         'file_type' as the file extension.
     :param directory_path: full path to the directory where the log file will be saved.
     :param add_stream: bool, If set to True, StreamHandler will be added to the logger.
-    :param add_timedfile: bool, If set to True, TimedRotatingFileHandler will be added to the logger.
+    :param add_timedfile: bool, If set to True, TimedRotatingFileHandler will be added to the logger directly.
+    :param add_timedfile_with_internal_queue: bool, If set to True, TimedRotatingFileHandler will be added
+        to the logger, but not directly.
+        Internal queue.Queue will be created, then used by the QueueListener, which will get the
+        TimerRotatingFileHandler as the handler.
+        Then the QueueHandler using the same internal queue will be added to the logger.
+        This is done to improve the multithreading compatibility.
+    :param add_queue_handler: bool, If set to True, QueueHandler will be added to the logger, using the 'log_queue'.
+    :param log_queue: queue.Queue or multiprocessing.Queue, Queue to use for the QueueHandler.
     :param file_type: string, file type of the log file. Default is 'txt'.
         'txt': Text file.
         'csv': CSV file.
@@ -133,7 +156,7 @@ def create_logger(
         error_logger = loggingw.create_logger(
             logger_name=f'{self.__class__.__name__}_CSV',
             file_path=output_log_file,
-            add_timedfile=True,
+            add_timedfile_with_internal_queue=True,
             file_type='csv',
             formatter_filehandler='MESSAGE',
             header=header
@@ -159,7 +182,7 @@ def create_logger(
             logger_name=f'{self.__class__.__name__}',
             file_path=output_log_file,
             add_stream=True,
-            add_timedfile=True,
+            add_timedfile_with_internal_queue=True,
             file_type='txt',
             formatter_streamhandler='DEFAULT',
             formatter_filehandler='DEFAULT'
@@ -170,16 +193,93 @@ def create_logger(
 
     if __name__ == "__main__":
         main()
+
+    ------------------------------
+
+    Example to use StreamHandler to output to console and TimedRotatingFileHandler to write to file in multiprocessing,
+    while QueueListener is in the main process writes to the file and outputs to the console and the QueueHandler
+    in two child subprocesses sends the logs to the main process through the multiprocessing.Queue:
+
+    import sys
+    import multiprocessing
+    from atomicshop.wrappers.loggingw import loggingw
+
+
+    def worker1(log_queue: multiprocessing.Queue):
+        error_logger = loggingw.create_logger(
+            logger_name='network',
+            add_queue_handler=True,
+            log_queue=log_queue
+        )
+
+        error_logger.info("Worker1 log message for 'network' logger.")
+
+
+    def worker2(log_queue: multiprocessing.Queue):
+        error_logger = loggingw.create_logger(
+            logger_name='network',
+            add_queue_handler=True,
+            log_queue=log_queue
+        )
+
+        error_logger.info("Worker2 log message for 'network' logger.")
+
+
+    def main():
+        log_queue = multiprocessing.Queue()
+
+        queue_listener = loggingw.create_logger(
+            get_queue_listener=True,
+            add_stream=True,
+            add_timedfile=True,
+            log_queue=log_queue,
+            file_type='txt',
+            formatter_streamhandler='DEFAULT',
+            formatter_filehandler='DEFAULT'
+        )
+
+        process1 = multiprocessing.Process(target=worker1, args=(log_queue,))
+        process2 = multiprocessing.Process(target=worker2, args=(log_queue,))
+
+        process1.start()
+        process2.start()
+
+        process1.join()
+        process2.join()
+
+        queue_listener.stop()
+
+        return 0
+
+
+    if __name__ == "__main__":
+        sys.exit(main())
     """
 
+    if logger_name and get_queue_listener:
+        raise ValueError("You can't set both 'logger_name' and 'get_queue_listener'.")
+    if not logger_name and not get_queue_listener:
+        raise ValueError("You need to provide 'logger_name' or 'get_queue_listener'.")
+
     # Check if the logger exists before creating it.
-    if loggers.is_logger_exists(logger_name):
-        raise LoggingwLoggerAlreadyExistsError(f"Logger '{logger_name}' already exists.")
+    if logger_name:
+        if loggers.is_logger_exists(logger_name):
+            raise LoggingwLoggerAlreadyExistsError(f"Logger '{logger_name}' already exists.")
+
+    if not logger_name and not file_path:
+        raise ValueError("You need to provide 'file_path' if 'logger_name' is not set.")
+
+    if get_queue_listener and not log_queue:
+        raise ValueError("You need to provide 'log_queue' if 'get_queue_listener' is set to True.")
 
-    if not directory_path and not file_path:
-        raise ValueError("You need to provide 'directory_path' or 'file_path'.")
-    if directory_path and file_path:
-        raise ValueError("You can't provide both 'directory_path' and 'file_path'.")
+    if add_queue_handler and not log_queue:
+        raise ValueError("You need to provide 'log_queue' if 'add_queue_handler' is set to True.")
+
+    if add_timedfile or add_timedfile_with_internal_queue:
+        if not directory_path and not file_path:
+            raise ValueError("You need to provide 'directory_path' or 'file_path'.")
+        if directory_path and file_path:
+            raise ValueError("You can't provide both 'directory_path' and 'file_path'.")
 
     if directory_path:
         if directory_path.endswith(os.sep):
@@ -187,24 +287,83 @@ def create_logger(
 
         file_path = f"{directory_path}{os.sep}{logger_name}.{file_type}"
 
-    logger = get_logger_with_level(logger_name, logging_level)
+    # --- Add the handlers to a tuple ---
 
+    handlers_tuple: tuple = ()
     if add_stream:
-        handlers.add_stream_handler(
-            logger=logger, logging_level=logging_level, formatter=formatter_streamhandler,
+        stream_handler = handlers.get_stream_handler_extended(
+            logging_level=logging_level,
+            formatter=formatter_streamhandler,
             formatter_use_nanoseconds=formatter_streamhandler_use_nanoseconds)
 
+        handlers_tuple += (stream_handler,)
+
     if add_timedfile:
-        handlers.add_timedfilehandler_with_queuehandler(
-            logger=logger, file_path=file_path, logging_level=logging_level, formatter=formatter_filehandler,
-            formatter_use_nanoseconds=formatter_filehandler_use_nanoseconds, file_type=file_type,
+        timed_file_handler = handlers.get_timed_rotating_file_handler_extended(
+            file_path=file_path,
+            logging_level=logging_level,
+            formatter=formatter_filehandler,
+            formatter_use_nanoseconds=formatter_filehandler_use_nanoseconds,
+            file_type=file_type,
             rotate_at_rollover_time=filehandler_rotate_at_rollover_time,
             rotation_date_format=filehandler_rotation_date_format,
             rotation_callback_namer_function=filehandler_rotation_callback_namer_function,
             rotation_use_default_callback_namer_function=filehandler_rotation_use_default_namer_function,
-            when=when, interval=interval, delay=delay, backupCount=backupCount, encoding=encoding, header=header)
+            when=when,
+            interval=interval,
+            delay=delay,
+            backupCount=backupCount,
+            encoding=encoding,
+            header=header
+        )
 
-    return logger
+        handlers_tuple += (timed_file_handler,)
+
+    if add_timedfile_with_internal_queue:
+        timed_file_handler_with_queue = handlers.get_timed_rotating_file_handler_extended(
+            file_path=file_path,
+            logging_level=logging_level,
+            formatter=formatter_filehandler,
+            formatter_use_nanoseconds=formatter_filehandler_use_nanoseconds,
+            file_type=file_type,
+            rotate_at_rollover_time=filehandler_rotate_at_rollover_time,
+            rotation_date_format=filehandler_rotation_date_format,
+            rotation_callback_namer_function=filehandler_rotation_callback_namer_function,
+            rotation_use_default_callback_namer_function=filehandler_rotation_use_default_namer_function,
+            use_internal_queue_listener=True,
+            when=when,
+            interval=interval,
+            delay=delay,
+            backupCount=backupCount,
+            encoding=encoding,
+            header=header
+        )
+
+        handlers_tuple += (timed_file_handler_with_queue,)
+
+    if add_queue_handler:
+        queue_handler = handlers.get_queue_handler_extended(log_queue)
+        handlers_tuple += (queue_handler,)
+
+    # --- Create the logger ---
+
+    if logger_name:
+        logger = get_logger_with_level(logger_name, logging_level)
+
+        # Add the handlers to the logger.
+        for handler in handlers_tuple:
+            loggers.add_handler(logger, handler)
+
+        # Disable propagation from the 'root' logger, so we will not see the messages twice.
+        loggers.set_propagation(logger)
+
+        return logger
+
+    # --- create the QueueListener ---
+
+    if get_queue_listener:
+        queue_listener: logging.handlers.QueueListener = handlers.start_queue_listener_for_handlers(handlers_tuple, log_queue)
+        return queue_listener
 
 
 def get_logger_with_level(
@@ -276,7 +435,7 @@ def is_logger_exists(logger_name: str) -> bool:
     return loggers.is_logger_exists(logger_name)
 
 
-def find_the_parent_logger_with_stream_handler(logger: logging.Logger) -> logging.Logger:
+def find_the_parent_logger_with_stream_handler(logger: logging.Logger) -> logging.Logger | None:
     """
     Function to find the parent logger with StreamHandler.
     Example:
@@ -286,7 +445,7 @@ def find_the_parent_logger_with_stream_handler(logger: logging.Logger) -> loggin
         StreamHandler from the 'parent' logger.
 
     :param logger: Logger to find the parent logger with StreamHandler.
-    :return: Parent logger with StreamHandler.
+    :return: Parent logger with StreamHandler or None if the logger doesn't have StreamHandler.
     """
 
     # Start with current logger to see if it has a stream handler.
@@ -302,6 +461,10 @@ def find_the_parent_logger_with_stream_handler(logger: logging.Logger) -> loggin
             # If the current logger doesn't have the stream handler, let's move to the parent.
             current_logger = current_logger.parent
 
+            # If none of the parent loggers have the stream handler, break the loop.
+            if current_logger is None:
+                break
+
     return current_logger
 
 

atomicshop/wrappers/playwrightw/scenarios.py

@@ -12,6 +12,7 @@ from bs4 import BeautifulSoup
 
 from . import engine, base, combos
 from ...basics import threads, multiprocesses
+from ...web import USER_AGENTS
 
 
 def get_text_from_html_tag(url: str, tag_name: str, attribute: str, value: str) -> str:
@@ -218,10 +219,12 @@ def _fetch_content(
     with sync_playwright() as p:
         browser = p.chromium.launch(headless=headless)  # Set headless=True if you don't want to see the browser
 
+        user_agent: str = USER_AGENTS['Windows_Chrome_Latest']
+
         if text_fetch_method == "playwright_copypaste":
-            context = browser.new_context(permissions=["clipboard-read", "clipboard-write"])
+            context = browser.new_context(permissions=["clipboard-read", "clipboard-write"], user_agent=user_agent)
         else:
-            context = browser.new_context()
+            context = browser.new_context(user_agent=user_agent)
 
         page = context.new_page()
 

atomicshop/wrappers/socketw/base.py

@@ -87,3 +87,16 @@ def is_socket_closed(socket_object) -> bool:
             return False
     except socket.error:
         return False
+
+
+def get_host_name_from_ip_address(ip_address: str) -> str:
+    """
+    Get the host name from the IP address.
+    :param ip_address: string, IP address.
+    :return: string, host name.
+    """
+
+    host_name, alias_list, ipaddr_list = socket.gethostbyaddr(ip_address)
+    _ = alias_list, ipaddr_list
+
+    return host_name

atomicshop/wrappers/socketw/dns_server.py

@@ -6,6 +6,7 @@ import socket
 import logging
 from pathlib import Path
 from typing import Literal
+import multiprocessing
 
 from ...print_api import print_api
 from ..loggingw import loggingw
@@ -45,7 +46,7 @@ class DnsStatisticsCSVWriter:
         self.csv_logger = loggingw.create_logger(
             logger_name=LOGGER_NAME,
             directory_path=statistics_directory_path,
-            add_timedfile=True,
+            add_timedfile_with_internal_queue=True,
             formatter_filehandler='MESSAGE',
             file_type='csv',
             header=DNS_STATISTICS_HEADER
@@ -156,12 +157,14 @@ class DnsServer:
             offline_mode: bool = False,
             tcp_target_server_ipv4: str = '127.0.0.1',
             tcp_resolve_domain_list: list = None,
-            request_domain_queue: queues.NonBlockQueue = None,
+            request_domain_queue: multiprocessing.Queue = None,
             buffer_size_receive: int = 8192,
             response_ttl: int = 60,
             dns_service_retries: int = 5,
             cache_timeout_minutes: int = 60,
-            logger: logging.Logger = None
+            logger: logging.Logger = None,
+            logging_queue: multiprocessing.Queue = None,
+            logger_name: str = None
     ):
         """
         Initialize the DNS Server object with all the necessary settings.
@@ -186,14 +189,20 @@ class DnsServer:
             the domains to.
         :param tcp_resolve_domain_list: list: List of domains that will be resolved to the TCP Server.
             This means that all the requests will be resolved to the specified offline IPv4 address.
-        :param request_domain_queue: queues.NonBlockQueue: Queue to pass all the requested domains that hit the DNS
+        :param request_domain_queue: multiprocessing Queue to pass all the requested domains that hit the DNS
         :param buffer_size_receive: int: Buffer size of the connection while receiving messages.
         :param response_ttl: int, Time to live of the DNS Response that will be returned. Default is 60 seconds.
         :param dns_service_retries: int, How many times the request will be sent to forwarded DNS Service on errors:
             (socket connect / request send / response receive).
         :param cache_timeout_minutes: int: Timeout in minutes to clear the DNS Cache.
             server. Each domain will be pass in the queue as a string.
+
         :param logger: logging.Logger: Logger object to use for logging. If not provided, a new logger will be created.
+        :param logging_queue: multiprocessing.Queue: Queue to pass the logs to the QueueListener.
+            You will use this in case you run the DNS Server in a separate process.
+            Of course, you need to have a QueueListener to listen to this queue.
+
+        You can pass only one of the following: 'logger', 'logging_queue'.
         """
 
         self.listening_interface: str = listening_interface
@@ -207,11 +216,16 @@ class DnsServer:
         self.resolve_to_tcp_server_only_tcp_resolve_domains: bool = resolve_to_tcp_server_only_tcp_resolve_domains
         self.resolve_to_tcp_server_all_domains: bool = resolve_to_tcp_server_all_domains
         self.resolve_regular: bool = resolve_regular
-        self.request_domain_queue: queues.NonBlockQueue = request_domain_queue
+        self.request_domain_queue: multiprocessing.Queue = request_domain_queue
         self.buffer_size_receive: int = buffer_size_receive
         self.response_ttl: int = response_ttl
         self.dns_service_retries: int = dns_service_retries
         self.cache_timeout_minutes: int = cache_timeout_minutes
+        self.logging_queue: multiprocessing.Queue = logging_queue
+        self.logging_name: str = logger_name
+
+        if logger and logging_queue:
+            raise ValueError("You can pass only one of the following: 'logger', 'logging_queue'.")
 
         if not tcp_resolve_domain_list:
             self.tcp_resolve_domain_list = list()
@@ -242,20 +256,31 @@ class DnsServer:
         # network log of TCP Server module.
         self.dns_statistics_csv_writer = DnsStatisticsCSVWriter(statistics_directory_path=log_directory_path)
 
+        if not logger_name and not logger and not logging_queue:
+            self.logger_name = Path(__file__).stem
+        elif logger_name and (logger or logging_queue):
+            self.logger_name = f'{logger_name}.{Path(__file__).stem}'
+
         # Check if the logger was provided, if not, create a new logger.
-        if not logger:
-            self.logger = loggingw.create_logger(
+        if not logger and not logging_queue:
+            self.logger: logging.Logger = loggingw.create_logger(
                 logger_name=Path(__file__).stem,
                 directory_path=self.log_directory_path,
                 add_stream=True,
-                add_timedfile=True,
+                add_timedfile_with_internal_queue=True,
                 formatter_streamhandler='DEFAULT',
                 formatter_filehandler='DEFAULT',
                 backupCount=backupCount_log_files_x_days
             )
-        else:
+        elif logger:
             # Create child logger for the provided logger with the module's name.
-            self.logger: logging.Logger = loggingw.get_logger_with_level(f'{logger.name}.{Path(__file__).stem}')
+            self.logger: logging.Logger = loggingw.get_logger_with_level(self.logger_name)
+        elif logging_queue:
+            self.logger: logging.Logger = loggingw.create_logger(
+                logger_name=self.logger_name,
+                add_queue_handler=True,
+                log_queue=self.logging_queue
+            )
 
         self.test_config()
 
@@ -271,6 +296,9 @@ class DnsServer:
                 raise_if_all_false=True
             )
         except ValueError as e:
+            print_api(f'DnsConfigurationValuesError: {str(e)}', error_type=True, color="red", logger=self.logger)
+            # Wait for the message to be printed and saved to file.
+            time.sleep(1)
             raise DnsConfigurationValuesError(e)
 
         ips_ports: list[str] = [f'{self.listening_interface}:{self.listening_port}']
@@ -279,7 +307,12 @@ class DnsServer:
             error_messages: list = list()
             for port, process_info in port_in_use.items():
                 error_messages.append(f"Port [{port}] is already in use by process: {process_info}")
-            raise DnsPortInUseError("\n".join(error_messages))
+
+            message = "\n".join(error_messages)
+            print_api(f'DnsPortInUseError: {str(e)}', error_type=True, color="red", logger=self.logger)
+            # Wait for the message to be printed and saved to file.
+            time.sleep(1)
+            raise DnsPortInUseError(message)
 
     def thread_worker_empty_dns_cache(self, function_sleep_time: int):
         """
@@ -457,7 +490,7 @@ class DnsServer:
                         if forward_to_tcp_server:
                             # If the request is forwarded to TCP server, then we'll put the domain in the domain queue.
                             # self.request_domain_queue.put(question_domain)
-                            self.request_domain_queue.queue = question_domain
+                            self.request_domain_queue.put(question_domain)
 
                             # Make DNS response that will refer TCP traffic to our server
                             dns_built_response = DNSRecord(
@@ -846,3 +879,58 @@ class DnsServer:
                     self.logger.info("==========")
                     pass
                     continue
+
+
+# noinspection PyPep8Naming
+def start_dns_server_multiprocessing_worker(
+        listening_interface: str,
+        listening_port: int,
+        log_directory_path: str,
+        backupCount_log_files_x_days: int,
+        forwarding_dns_service_ipv4: str,
+        tcp_target_server_ipv4: str,
+        # Passing the engine domain list to DNS server to work with.
+        # 'list' function re-initializes the current list, or else it will be the same instance object.
+        tcp_resolve_domain_list: list,
+        offline_mode: bool,
+        resolve_to_tcp_server_only_tcp_resolve_domains: bool,
+        resolve_to_tcp_server_all_domains: bool,
+        resolve_regular: bool,
+        cache_timeout_minutes: int,
+        request_domain_queue: multiprocessing.Queue,
+        logging_queue: multiprocessing.Queue,
+        logger_name: str
+):
+    # Setting the current thread name to the current process name.
+    current_process_name = multiprocessing.current_process().name
+    threading.current_thread().name = current_process_name
+
+
+
+    try:
+        dns_server_instance = DnsServer(
+            listening_interface=listening_interface,
+            listening_port=listening_port,
+            log_directory_path=log_directory_path,
+            backupCount_log_files_x_days=backupCount_log_files_x_days,
+            forwarding_dns_service_ipv4=forwarding_dns_service_ipv4,
+            tcp_target_server_ipv4=tcp_target_server_ipv4,
+            # Passing the engine domain list to DNS server to work with.
+            # 'list' function re-initializes the current list, or else it will be the same instance object.
+            tcp_resolve_domain_list=tcp_resolve_domain_list,
+            offline_mode=offline_mode,
+            resolve_to_tcp_server_only_tcp_resolve_domains=resolve_to_tcp_server_only_tcp_resolve_domains,
+            resolve_to_tcp_server_all_domains=resolve_to_tcp_server_all_domains,
+            resolve_regular=resolve_regular,
+            cache_timeout_minutes=cache_timeout_minutes,
+            request_domain_queue=request_domain_queue,
+            logging_queue=logging_queue,
+            logger_name=logger_name
+        )
+    except (DnsPortInUseError, DnsConfigurationValuesError) as e:
+        print_api(e, error_type=True, color="red", logger=dns_server_instance.logger)
+        # Wait for the message to be printed and saved to file.
+        time.sleep(1)
+        return 1
+
+    dns_server_instance.start()