atomicshop 2.15.11__py3-none-any.whl → 3.10.5__py3-none-any.whl

atomicshop/mitm/engines/__parent/parser___parent.py

@@ -1,28 +1,15 @@
-# v1.0.0 - 26.03.2023 14:00
 from ...shared_functions import create_custom_logger
 from ...message import ClientMessage
 
 
-# Class that parses the message received from client.
 class ParserParent:
-    # Initializing the logger in the "class variable" section will leave the instance of the logger initiated
-    # and the rest of the instances of the class will use the same logger.
-    # It is not in the "__init__" section, so it's not going to be initiated again.
-    # The name of the logger using "__name__" variable, which is the full name of the module package.
-    # Example: classes.parsers.parser_1_reference_general
-
-    # The code outside the functions will be executed during import of the module. When initializing a class
-    # in the script these lines will not be called again, only the "init" function.
-    logger = create_custom_logger()
-
+    """Class that parses the message received from client."""
     def __init__(self, class_client_message: ClientMessage):
         self.class_client_message: ClientMessage = class_client_message
+        self.logger = create_custom_logger()
 
     def parse(self):
         # This is general parser, so we don't parse anything and 'request_body_parsed' gets empty byte string.
-        self.class_client_message.request_body_parsed = b''
+        self.class_client_message.request_custom_parsed = b''
 
-        try:
-            self.logger.info(f"Parsed: {self.class_client_message.request_body_parsed[0: 100]}...")
-        except Exception:
-            pass
+        self.logger.info(f"Parsed: {self.class_client_message.request_custom_parsed[0: 100]}...")

atomicshop/mitm/engines/__parent/recorder___parent.py

@@ -1,35 +1,45 @@
 import os
 from datetime import datetime
+import queue
+import threading
+from pathlib import Path
 
-from ...shared_functions import build_module_names, create_custom_logger, get_json
-from ...message import ClientMessage
-from ....urls import url_parser
+from ...shared_functions import build_module_names, create_custom_logger
+from ... import message, recs_files
 from .... import filesystem
+from ....file_io import jsons
+from ....print_api import print_api
 
 
 # The class that is responsible for Recording Requests / Responses.
 class RecorderParent:
-    # The code outside the functions will be executed during import of the module. When initializing a class
-    # in the script these lines will not be called again, only the "init" function.
-    logger = create_custom_logger()
 
-    def __init__(self, class_client_message: ClientMessage, record_path: str):
-        self.class_client_message: ClientMessage = class_client_message
+    # noinspection PyTypeChecker
+    def __init__(self, record_path: str):
         self.record_path: str = record_path
+
         self.file_extension: str = ".json"
         self.engine_name = None
         self.module_name = None
         self.engine_record_path: str = str()
         self.record_file_path: str = str()
+        self.class_client_message: message.ClientMessage = None
+
+        self.logger = create_custom_logger()
 
         # Get engine name and module name
         self.get_engine_module()
-        # Build full file path.
-        self.build_record_full_file_path()
+
+        # Build the record path with file name
+        self.build_record_path_to_engine()
 
         # Create folder.
         filesystem.create_directory(self.engine_record_path)
 
+        # Initialize a queue to hold messages
+        self.message_queue: queue.Queue = queue.Queue()
+        self.recorder_worker_thread = None
+
     # "self.__module__" is fully qualified module name: classes.engines.ENGINE-NAME.MODULE-NAME
     def get_engine_module(self):
         _, self.engine_name, self.module_name = build_module_names(self.__module__)
@@ -38,33 +48,10 @@ class RecorderParent:
         self.engine_record_path = self.record_path + os.sep + self.engine_name
 
     def build_record_full_file_path(self):
-        # current date and time in object
-        now = datetime.now()
-        # Formatting the date and time and converting it to string object
-        day_time_format: str = now.strftime("%Y_%m_%d-%H_%M_%S_%f")
-
-        # Build the record path with file name
-        self.build_record_path_to_engine()
-
-        # Define empty 'http_path'.
-        http_path: str = str()
-        # If 'self.class_client_message.request_raw_decoded.path' will be undefined, exception will raise.
-        # This will happen if the message is not HTTP.
-        try:
-            # Parse the url to components.
-            http_path_parsed = url_parser(self.class_client_message.request_raw_decoded.path)
-            # Get only directories.
-            http_path_directories_string = '-'.join(http_path_parsed['directories'])
-            # Add '_' character before 'http_path' to look better on the file name.
-            http_path = f'_{http_path_directories_string}'
-        # If 'self.class_client_message.request_raw_decoded.path' is not defined, we'll pass the exception.
-        except Exception:
-            pass
-
         # If HTTP Path is not defined, 'http_path' will be empty, and it will not interfere with file name.
-        self.record_file_path: str = \
-            self.engine_record_path + os.sep + \
-            day_time_format + "_" + self.class_client_message.server_name + http_path + self.file_extension
+        self.record_file_path: str = (
+            f"{self.engine_record_path}{os.sep}th{self.class_client_message.thread_id}_"
+            f"{self.class_client_message.server_name}{self.file_extension}")
 
     def convert_messages(self):
         """
@@ -74,26 +61,96 @@ class RecorderParent:
         # We need to check that the values that we want to convert aren't empty or 'None'.
         if self.class_client_message.request_raw_bytes:
             self.class_client_message.request_raw_hex = self.class_client_message.request_raw_bytes.hex()
-        if self.class_client_message.response_list_of_raw_bytes:
-            # We checked that the list isn't empty, now we check if the first value is not empty, since if we
-            # check it in the same expression as check for list is empty, we will get an exception since
-            # the list is empty.
-            if self.class_client_message.response_list_of_raw_bytes[0]:
-                for response_raw_bytes in self.class_client_message.response_list_of_raw_bytes:
-                    self.class_client_message.response_list_of_raw_hex.append(response_raw_bytes.hex())
+        if self.class_client_message.response_raw_bytes:
+            self.class_client_message.response_raw_hex = self.class_client_message.response_raw_bytes.hex()
+
+    def record(self, class_client_message: message.ClientMessage):
+        self.class_client_message = class_client_message
 
-    def record(self):
-        self.logger.info("Recording Message...")
+        # Build full file path if it is not already built.
+        if not self.record_file_path:
+            self.build_record_full_file_path()
+
+        # Start the worker thread if it is not already running
+        if not self.recorder_worker_thread:
+            self.recorder_worker_thread = threading.Thread(
+                target=save_message_worker,
+                args=(self.record_file_path, self.message_queue, self.logger),
+                name=f"{self.class_client_message.thread_process} | Thread-{self.class_client_message.thread_id}-Recorder",
+                daemon=True
+            )
+            self.recorder_worker_thread.start()
+
+        self.logger.info("Putting Message to Recorder Thread Queue...")
 
         # Convert the requests and responses to hex.
         self.convert_messages()
         # Get the message in dict / JSON format
-        record_message = get_json(self.class_client_message)
+        record_message_dict: dict = dict(self.class_client_message)
 
-        # Since we already dumped the object to dictionary string, we'll just save the object to regular file.
-        with open(self.record_file_path, 'w') as output_file:
-            output_file.write(record_message)
-
-        self.logger.info(f"Recorded to file: {self.record_file_path}")
+        # Put the message in the queue to be processed by the worker thread
+        self.message_queue.put(record_message_dict)
 
         return self.record_file_path
+
+
+def save_message_worker(
+        record_file_path_no_date: str,
+        message_queue: queue.Queue,
+        logger
+):
+    """Worker function to process messages from the queue and write them to the file."""
+    original_file_path_object: Path = Path(record_file_path_no_date)
+    original_file_stem: str = original_file_path_object.stem
+    original_file_extension: str = original_file_path_object.suffix
+    original_file_directory: str = str(original_file_path_object.parent)
+
+    original_datetime_string: str = get_datetime_string()
+    previous_date_string: str = get_date_string()
+
+    record_file_path: str = f'{original_file_directory}{os.sep}{original_datetime_string}_{original_file_stem}{original_file_extension}'
+
+    while True:
+        # Get a message from the queue
+        record_message_dict = message_queue.get()
+
+        # Check for the "stop" signal
+        if record_message_dict is None:
+            break
+
+        current_date_string: str = get_date_string()
+
+        # If current datetime string is different from the original datetime string, we will create a new file path.
+        if current_date_string != previous_date_string:
+            previous_date_string = current_date_string
+            current_datetime_string: str = get_datetime_string()
+            record_file_path = f'{original_file_directory}{os.sep}{current_datetime_string}_{original_file_stem}_partof_{original_datetime_string}{original_file_extension}'
+
+        try:
+            jsons.append_to_json(
+                record_message_dict, record_file_path, indent=2,
+                enable_long_file_path=True, print_kwargs={'logger': logger}
+            )
+        except TypeError as e:
+            print_api(str(e), logger_method="critical", logger=logger)
+            raise e
+
+        logger.info(f"Recorded to file: {record_file_path}")
+
+        # Indicate task completion
+        message_queue.task_done()
+
+
+def get_datetime_string():
+    # current date and time in object
+    now = datetime.now()
+    # Formatting the date and time and converting it to string object
+    day_time_format: str = now.strftime(recs_files.REC_FILE_DATE_TIME_FORMAT)
+    return day_time_format
+
+def get_date_string():
+    # current date and time in object
+    now = datetime.now()
+    # Formatting the date and time and converting it to string object
+    date_format: str = now.strftime(recs_files.REC_FILE_DATE_FORMAT)
+    return date_format

atomicshop/mitm/engines/__parent/requester___parent.py

@@ -0,0 +1,116 @@
+# Using to convert status code to status phrase / string.
+from http import HTTPStatus
+# Parsing PATH template to variables.
+from pathlib import PurePosixPath
+from urllib.parse import unquote
+# Needed to extract parameters after question mark in URL / Path.
+from urllib.parse import urlparse
+from urllib.parse import parse_qs
+
+from ...message import ClientMessage
+from .... import http_parse
+from ....print_api import print_api
+
+from atomicshop.mitm.shared_functions import create_custom_logger
+
+
+class RequesterParent:
+    """The class that is responsible for generating request to client based on the received message."""
+    def __init__(self):
+        self.logger = create_custom_logger()
+
+    def build_byte_request(
+            self,
+            http_method: str,
+            endpoint: str,
+            http_version: str,
+            headers: dict,
+            body: bytes
+    ) -> bytes:
+        # noinspection GrazieInspection
+        """
+                Create genuine request from input parameters.
+                ---------------
+                The request is built from:
+                <http_method> <endpoint> <http_version>\r\n
+                Headers1: Value\r\n
+                Headers2: Value\r\n
+                \r\n                        # This is meant to end the headers' section
+                Body                        # Request doesn't end with '\r\n\r\n'
+                ---------------
+                Example for POST request:
+                POST /api/v1/resource HTTP/1.1\r\n
+                Cache-Control: max-age=86400\r\n
+                Content-Type: application/json; charset=utf-8\r\n
+                \r\n
+                {"id":1,"name":"something"}
+                ---------------
+                You can create response as:
+
+                ...POST endpoint/api/1 HTTP/1.1
+                header1: value
+                header2: value
+
+                {data: value}...
+
+                Change 3 dots ("...") to 3 double quotes before "POST" and after "value}".
+                This way there will be "\n" added automatically after each line.
+                While, the HTTP Client does the parsing of the text and not raw data, most probably it will be parsed well,
+                but genuine requests from HTTP sources come with "\r\n" at the end of the line, so better use these for
+                better compatibility.
+                ---------------
+
+                :param http_method: HTTP Method of Request, e.g. 'GET', 'POST', etc.
+                :param endpoint: Endpoint of Request, e.g. '/api/v1/resource'.
+                :param http_version: HTTP Version of Response in HTTP Status line.
+                :param headers: HTTP Headers of Response.
+                :param body: HTTP body data of Response, bytes.
+                :return: bytes of the response.
+                """
+
+        try:
+            # CHeck if the HTTP method is valid.
+            if http_method not in http_parse.get_request_methods():
+                raise ValueError(f"Invalid HTTP Method: {http_method}")
+
+            # Building the full method endpoint string line and the "\r\n" in the end.
+            method_full: str = f"{http_method} {endpoint} {http_version}\r\n"
+
+            # Defining headers string.
+            headers_string: str = str()
+            # Adding all the headers to the full response
+            for keys, values in headers.items():
+                headers_string = headers_string + str(keys) + ": " + str(values) + "\r\n"
+
+            # Building full string request.
+            # 1. Adding full method line.
+            # 2. Adding headers string.
+            # 3. Adding a line that end headers (with "\r\n").
+            # 4. Adding body as byte string.
+            request_full_no_body: str = method_full + headers_string + "\r\n"
+
+            # Converting the HTTP Request string to bytes and adding 'body' bytes.
+            request_raw_bytes = request_full_no_body.encode() + body
+        except ValueError as exception_object:
+            message = \
+                f'Create Byte request function error, of the of values provided is not standard: {exception_object}'
+            print_api(message, error_type=True, logger=self.logger, logger_method='error', color='red')
+
+            request_raw_bytes = b''
+
+        # Parsing the request we created.
+        request_parse_test = http_parse.HTTPRequestParse(request_raw_bytes)
+        # If there were errors during parsing, it means that something is wrong with response created.
+        if request_parse_test.error_message:
+            self.logger.error(request_parse_test.error_message)
+            request_raw_bytes = b''
+        else:
+            self.logger.info("Created Valid Byte Request.")
+
+        return request_raw_bytes
+
+    def create_request(self, class_client_message: ClientMessage, **kwargs) -> bytes:
+        """ This function should be overridden in the child class. """
+
+        request_bytes: bytes = None
+        return request_bytes

atomicshop/mitm/engines/__parent/responder___parent.py

@@ -1,4 +1,3 @@
-# v1.0.0 - 26.03.2023 14:20
 # Using to convert status code to status phrase / string.
 from http import HTTPStatus
 # Parsing PATH template to variables.
@@ -8,20 +7,31 @@ from urllib.parse import unquote
 from urllib.parse import urlparse
 from urllib.parse import parse_qs
 
-from ...shared_functions import create_custom_logger
 from ...message import ClientMessage
 from ....http_parse import HTTPResponseParse
 from ....print_api import print_api
 
+from atomicshop.mitm.shared_functions import create_custom_logger
 
-# The class that is responsible for generating response to client based on the received message.
-class ResponderParent:
-    # The code outside the functions will be executed during import of the module. When initializing a class
-    # in the script these lines will not be called again, only the "init" function.
-    logger = create_custom_logger()
 
+class ResponderParent:
+    """The class that is responsible for generating response to client based on the received message."""
     def __init__(self):
-        return
+        self.logger = create_custom_logger()
+        # engine: initialize_engines.ModuleCategory
+        self.engine = None
+
+    def add_args(
+            self,
+            # engine: initialize_engines.ModuleCategory
+            engine = None
+    ):
+        """
+        Add more arguments to the class.
+        This is needed to be backwards compatible and not to change the child class apis.
+        """
+
+        self.engine = engine
 
     @staticmethod
     def get_path_parts(path: str):
@@ -107,51 +117,56 @@ class ResponderParent:
 
         return parameter_value
 
-    def build_byte_response_and_fill_lists(
-            self, http_version: str, status_code: int, headers: dict, body: bytes, client_message: ClientMessage):
-        """
-        Create genuine response from input parameters.
-        ---------------
-        The response is built from:
-        HTTP-Version HTTP-Status HTTP-Status-String\r\n
-        Headers1: Value\r\n
-        Headers2: Value\r\n
-        \r\n                        # This is meant to end the headers' section
-        Body\r\n\r\n                # In most cases Body is ended with '\r\n\r\n'
-        ---------------
-        Example for 200 response:
-        HTTP/1.1 200 OK\r\n
-        Cache-Control: max-age=86400\r\n
-        Content-Type: application/json; charset=utf-8\r\n
-        \r\n
-        {"id":1,"name":"something"}
-        ---------------
-        The final response will look like oneline string:
-        HTTP/1.1 200 OK\r\nCache-Control: max-age=86400\r\n
-        Content-Type: application/json; charset=utf-8\r\n\r\n{"id":1,"name":"something"}
-        ---------------
-        You can create response as:
-
-        ...HTTP/1.1 200 OK
-        header1: value
-        header2: value
-
-        {data: value}...
-
-        Change 3 dots ("...") to 3 double quotes before "HTTP" and after "value}".
-        This way there will be "\n" added automatically after each line.
-        While, the HTTP Client does the parsing of the text and not raw data, most probably it will be parsed well,
-        but genuine responses from HTTP sources come with "\r\n" at the end of the line, so better use these for
-        better compatibility.
-        ---------------
-
-        :param http_version: HTTP Version of Response in HTTP Status line.
-        :param status_code: HTTP Status Code of Response in HTTP Status line.
-        :param headers: HTTP Headers of Response.
-        :param body: HTTP body data of Response, bytes.
-        :param client_message: client message class.
-        :return:
+    def build_byte_response(
+            self,
+            http_version: str,
+            status_code: int,
+            headers: dict,
+            body: bytes
+    ) -> bytes:
+        # noinspection GrazieInspection
         """
+                Create genuine response from input parameters.
+                ---------------
+                The response is built from:
+                HTTP-Version HTTP-Status HTTP-Status-String\r\n
+                Headers1: Value\r\n
+                Headers2: Value\r\n
+                \r\n                        # This is meant to end the headers' section
+                Body\r\n\r\n                # In most cases Body is ended with '\r\n\r\n'
+                ---------------
+                Example for 200 response:
+                HTTP/1.1 200 OK\r\n
+                Cache-Control: max-age=86400\r\n
+                Content-Type: application/json; charset=utf-8\r\n
+                \r\n
+                {"id":1,"name":"something"}
+                ---------------
+                The final response will look like oneline string:
+                HTTP/1.1 200 OK\r\nCache-Control: max-age=86400\r\n
+                Content-Type: application/json; charset=utf-8\r\n\r\n{"id":1,"name":"something"}
+                ---------------
+                You can create response as:
+
+                ...HTTP/1.1 200 OK
+                header1: value
+                header2: value
+
+                {data: value}...
+
+                Change 3 dots ("...") to 3 double quotes before "HTTP" and after "value}".
+                This way there will be "\n" added automatically after each line.
+                While, the HTTP Client does the parsing of the text and not raw data, most probably it will be parsed well,
+                but genuine responses from HTTP sources come with "\r\n" at the end of the line, so better use these for
+                better compatibility.
+                ---------------
+
+                :param http_version: HTTP Version of Response in HTTP Status line.
+                :param status_code: HTTP Status Code of Response in HTTP Status line.
+                :param headers: HTTP Headers of Response.
+                :param body: HTTP body data of Response, bytes.
+                :return: bytes of the response.
+                """
 
         try:
             # Building full status string line and the "\r\n" to the end of the status line
@@ -178,7 +193,6 @@ class ResponderParent:
             print_api(message, error_type=True, logger=self.logger, logger_method='error', color='red')
 
             response_raw_bytes = b''
-            pass
 
         # Parsing the response we created.
         response_parse_test = HTTPResponseParse(response_raw_bytes)
@@ -186,73 +200,20 @@ class ResponderParent:
         if response_parse_test.error:
             self.logger.error(response_parse_test.error)
             response_raw_bytes = b''
-            response_decoded = None
         else:
             self.logger.info("Created Valid Byte Response.")
-            response_decoded = response_parse_test.response_raw_decoded
 
-        # Add 'response_raw_bytes' and 'response_decoded' to appropriate response lists in 'class_message'.
-        self.add_response_elements_to_lists(client_message, response_raw_bytes, response_decoded)
+        return response_raw_bytes
 
     @staticmethod
-    def add_response_elements_to_lists(class_client_message: ClientMessage, byte_response, decoded_response):
-        """
-        Function just adds the byte response to the 'response_list_of_raw_bytes'.
-        :param class_client_message:
-        :param byte_response:
-        :param decoded_response:
-        :return:
-        """
+    def create_connect_response(class_client_message: ClientMessage):
+        """ This function should be overridden in the child class. """
 
-        class_client_message.response_list_of_raw_bytes.append(byte_response)
-        class_client_message.response_list_of_raw_decoded.append(decoded_response)
+        _ = class_client_message
+        response_bytes_list: list[bytes] = list()
+        return response_bytes_list
 
     def create_response(self, class_client_message: ClientMessage):
-        # noinspection GrazieInspection
-        """
-        Function to create Response based on ClientMessage and its Request.
-
-        :param class_client_message: contains request and other parameters to help creating response.
-        :return: "class_client_message.response_list_of_raw_bytes" is populated with list of responses in bytes.
-        -----------------------------------
-        # Remember that 'response_list_of_raw_bytes' is a list object that will contain byte response/s.
-
-        # Example of creating 'response_list_of_raw_bytes' using 'build_byte_response' function:
-        class_client_message.response_list_of_raw_bytes.append(
-            self.build_byte_response(
-                http_version=class_client_message.request_raw_decoded.request_version,
-                status_code=200,
-                headers=response_headers,
-                body=b''
-            )
-        )
-
-        # Or you can use the 'add_byte_response_to_response_list' function that will do it for you:
-        byte_response = self.build_byte_response(
-            http_version=class_client_message.request_raw_decoded.request_version,
-            status_code=200,
-            headers=response_headers,
-            body=b''
-        )
-        self.add_byte_response_to_response_list(class_client_message, byte_response)
-        -----------------------------------
-        # Example of extracting variables from URL PATH based on custom PATH TEMPLATE:
-        # (more examples in 'self.extract_variables_from_path_template' function description)
-        template_path: str = "/hithere/<variable1>/else/<variable2>/tested/"
-        path_variables: dict = extract_variables_from_path_template(
-            path=class_client_message.request_raw_decoded.path,
-            template_path=template_path
-        )
-        -----------------------------------
-        # Example of extracting value from URL PATH parameters after question mark:
-        parameter_value = extract_value_from_path_parameter(
-            path=class_client_message.request_raw_decoded.path,
-            parameter='test_id'
-        )
-        """
+        """ This function should be overridden in the child class. """
 
-        byte_response = None
-        decoded_response = None
-        # class_client_message.response_list_of_raw_bytes.append(byte_response)
-        self.add_response_elements_to_lists(class_client_message, byte_response, decoded_response)
-        self.logger.info(f"Response: {class_client_message.response_list_of_raw_bytes}")
+        return None

atomicshop/mitm/engines/__reference_general/parser___reference_general.py

@@ -3,21 +3,24 @@ from atomicshop.mitm.engines.__parent.parser___parent import ParserParent
 from atomicshop.mitm.shared_functions import create_custom_logger
 from atomicshop.mitm.message import ClientMessage
 
-# # This is 'example' '.proto' file that contains message 'ExampleRequest'.
-# from .example_pb2 import ExampleRequest
-# # Import from 'protobuf' library of function 'MessageToDict' that
-# converts protobuf message object type to python dict.
-# from google.protobuf.json_format import MessageToDict
+
+"""
+# This is 'example' '.proto' file that contains message 'ExampleRequest'.
+from .example_pb2 import ExampleRequest
+# Import from 'protobuf' library of function 'MessageToDict' that
+converts protobuf message object type to python dict.
+from google.protobuf.json_format import MessageToDict
+"""
 
 
 # Class that parses the message received from client.
 class ParserGeneral(ParserParent):
-    logger = create_custom_logger()
-
     # When initializing main classes through "super" you need to pass parameters to init
     def __init__(self, class_client_message: ClientMessage):
         super().__init__(class_client_message)
 
+        self.logger = create_custom_logger()
+
     # ==================================================================================================================
     # Uncomment this section in order to begin building custom responder.
     # def parse_example_request(self):

atomicshop/mitm/engines/__reference_general/recorder___reference_general.py

@@ -4,10 +4,10 @@ from atomicshop.mitm.shared_functions import create_custom_logger
 from atomicshop.mitm.message import ClientMessage
 
 
-# The class that is responsible for Recording Requests / Responses
 class RecorderGeneral(RecorderParent):
-    logger = create_custom_logger()
-
+    """The class that is responsible for Recording Requests / Responses"""
     # When initializing main classes through "super" you need to pass parameters to init
-    def __init__(self, class_client_message: ClientMessage, record_path):
-        super().__init__(class_client_message, record_path)
+    def __init__(self, record_path):
+        super().__init__(record_path)
+
+        self.logger = create_custom_logger()