atomicshop 2.11.47__py3-none-any.whl → 3.10.5__py3-none-any.whl

atomicshop/etws/traces/trace_sysmon_process_creation.py

@@ -0,0 +1,126 @@
+from .. import trace, const
+from ...wrappers import sysmonw
+from ...basics import dicts
+
+
+DEFAULT_SESSION_NAME: str = 'AtomicShopSysmonProcessCreationTrace'
+
+PROVIDER_NAME: str = const.ETW_SYSMON['provider_name']
+PROVIDER_GUID: str = const.ETW_SYSMON['provider_guid']
+PROCESS_CREATION_EVENT_ID: int = const.ETW_SYSMON['event_ids']['process_create']
+
+
+class SysmonProcessCreationTrace:
+    def __init__(
+            self,
+            attrs: list = None,
+            session_name: str = None,
+            close_existing_session_name: bool = True,
+            sysmon_directory: str = None
+    ):
+        """
+        DnsTrace class use to trace DNS events from Windows Event Tracing for EventId 3008.
+
+        :param attrs: List of attributes to return. If None, all attributes will be returned.
+        :param session_name: The name of the session to create. If not provided, a UUID will be generated.
+        :param close_existing_session_name: Boolean to close existing session names.
+            True: if ETW session with 'session_name' exists, you will be notified and the session will be closed.
+                Then the new session with this name will be created.
+            False: if ETW session with 'session_name' exists, you will be notified and the new session will not be
+                created. Instead, the existing session will be used. If there is a buffer from the previous session,
+                you will get the events from the buffer.
+        :param sysmon_directory: The directory where Sysmon is located. If not provided, "C:\\Windows\\Sysmon" will be
+            used. If 'Sysmon.exe' is not found in the directory, it will be downloaded from the internet.
+
+        -------------------------------------------------
+
+        Usage Example:
+            from atomicshop.etw import dns_trace
+
+
+            dns_trace_w = dns_trace.DnsTrace(
+                attrs=['pid', 'name', 'cmdline', 'domain', 'query_type'],
+                session_name='MyDnsTrace',
+                close_existing_session_name=True,
+                enable_process_poller=True,
+                process_poller_etw_session_name='MyProcessTrace'
+            )
+            dns_trace_w.start()
+            while True:
+                dns_dict = dns_trace_w.emit()
+                print(dns_dict)
+            dns_trace_w.stop()
+        """
+
+        self.attrs = attrs
+        self.sysmon_directory: str = sysmon_directory
+
+        if not session_name:
+            session_name = DEFAULT_SESSION_NAME
+
+        self.event_trace = trace.EventTrace(
+            providers=[(PROVIDER_NAME, PROVIDER_GUID)],
+            # lambda x: self.event_queue.put(x),
+            event_id_filters=[PROCESS_CREATION_EVENT_ID],
+            session_name=session_name,
+            close_existing_session_name=close_existing_session_name
+        )
+
+    def start(self):
+        sysmonw.start_as_service(
+            installation_path=self.sysmon_directory, download_sysmon_if_not_found=True, skip_if_running=True)
+        self.event_trace.start()
+
+    def stop(self):
+        self.event_trace.stop()
+
+    def emit(self):
+        """
+        Function that will return the next event from the queue.
+        The queue is blocking, so if there is no event in the queue, the function will wait until there is one.
+
+        Usage Example:
+            while True:
+                dns_dict = dns_trace.emit()
+                print(dns_dict)
+
+        :return: Dictionary with the event data.
+
+        -----------------------------------------------
+
+        Structure of the returned dictionary:
+        {
+            'event_id': int,
+            'ProcessId': int,
+            'ProcessGuid': str,
+            'Image': str,
+            'FileVersion': str,
+            'Product': str,
+            'Company': str,
+            'OriginalFileName': str,
+            'CommandLine': str,
+            'CurrentDirectory': str,
+            'User': str,
+            'LogonId': str,
+            'LogonGuid': str,
+            'TerminalSessionId': int,
+            'IntegrityLevel': str,
+            'Hashes': dict,
+            'ParentProcessGuid': str,
+            'ParentProcessId': int,
+            'ParentImage': str,
+            'ParentCommandLine': str
+        }
+
+        """
+
+        event = self.event_trace.emit()
+
+        event_dict = {'EventId': event['EventId']}
+        event_dict.update(event['EventHeader'])
+
+        if self.attrs:
+            event_dict = dicts.reorder_keys(
+                event_dict, self.attrs, skip_keys_not_in_list=True)
+
+        return event_dict

atomicshop/etws/traces/trace_tcp.py

@@ -0,0 +1,130 @@
+import multiprocessing.managers
+
+from .. import trace, providers
+from ...basics import dicts
+
+
+ETW_DEFAULT_SESSION_NAME: str = 'AtomicShopTcpTrace'
+
+PROVIDER_NAME: str = "Microsoft-Windows-TCPIP"
+PROVIDER_GUID: str = '{' + providers.get_provider_guid_by_name(PROVIDER_NAME) + '}'
+REQUEST_RESP_EVENT_ID: int = 1033
+
+
+class TcpIpNewConnectionsTrace:
+    """
+    TcpIpNewConnectionsTrace class use to trace new connection events from Windows Event Tracing:
+    Provider: Microsoft-Windows-TCPIP
+    EventId: 1033
+    """
+    def __init__(
+            self,
+            attrs: list = None,
+            session_name: str = None,
+            close_existing_session_name: bool = True,
+            process_pool_shared_dict_proxy: multiprocessing.managers.DictProxy = None
+    ):
+        """
+        :param attrs: List of attributes to return. If None, all attributes will be returned.
+        :param session_name: The name of the session to create. If not provided, a UUID will be generated.
+        :param close_existing_session_name: Boolean to close existing session names.
+            True: if ETW session with 'session_name' exists, you will be notified and the session will be closed.
+                Then the new session with this name will be created.
+            False: if ETW session with 'session_name' exists, you will be notified and the new session will not be
+                created. Instead, the existing session will be used. If there is a buffer from the previous session,
+                you will get the events from the buffer.
+        :param process_pool_shared_dict_proxy: multiprocessing.managers.DictProxy, multiprocessing shared dict proxy
+            that contains current processes.
+            Check the 'atomicshop\process_poller\simple_process_pool.py' SimpleProcessPool class for more information.
+
+            For this specific class it means that you can run the process poller outside of this class and pass the
+            'process_pool_shared_dict_proxy' to this class. Then you can get the process name and command line for
+            the DNS events from the 'process_pool_shared_dict_proxy' and use it also in other classes.
+
+        -------------------------------------------------
+
+        Usage Example:
+            from atomicshop.etw import tcp_trace
+
+
+            tcp_trace_w = tcp_trace.TcpIpNewConnectionsTrace(
+                attrs=['pid', 'name', 'cmdline', 'domain', 'query_type'],
+                session_name='MyTcpTrace',
+                close_existing_session_name=True,
+                enable_process_poller=True
+            )
+            tcp_trace_w.start()
+            while True:
+                tcp_dict = tcp_trace_w.emit()
+                print(tcp_dict)
+            tcp_trace_w.stop()
+        """
+
+        self.attrs = attrs
+        self.process_pool_shared_dict_proxy: multiprocessing.managers.DictProxy = process_pool_shared_dict_proxy
+
+        if not session_name:
+            session_name = ETW_DEFAULT_SESSION_NAME
+
+        self.event_trace = trace.EventTrace(
+            providers=[(PROVIDER_NAME, PROVIDER_GUID)],
+            # lambda x: self.event_queue.put(x),
+            event_id_filters=[REQUEST_RESP_EVENT_ID],
+            session_name=session_name,
+            close_existing_session_name=close_existing_session_name,
+            enable_process_poller=True,
+            process_pool_shared_dict_proxy=self.process_pool_shared_dict_proxy
+        )
+
+    def start(self):
+        self.event_trace.start()
+
+    def stop(self):
+        self.event_trace.stop()
+
+    def emit(self):
+        """
+        Function that will return the next event from the queue.
+        The queue is blocking, so if there is no event in the queue, the function will wait until there is one.
+
+        Usage Example:
+            while True:
+                tcp_dict = tcp_trace.emit()
+                print(tcp_dict)
+
+        :return: Dictionary with the event data.
+        """
+
+        # Get the event from ETW as is.
+        event = self.event_trace.emit()
+
+        local_address_port: str = event['EventHeader']['LocalAddress']
+        remote_address_port: str = event['EventHeader']['RemoteAddress']
+
+        if 'ffff' in local_address_port:
+            pass
+
+        local_address, local_port = local_address_port.rsplit(':', 1)
+        local_address = local_address.replace('[', '').replace(']', '')
+
+        remote_address, remote_port = remote_address_port.rsplit(':', 1)
+        remote_address = remote_address.replace('[', '').replace(']', '')
+
+        event_dict: dict = {
+            'timestamp': event['timestamp'],
+            'event_id': event['EventId'],
+            'local_ip': local_address,
+            'local_port': local_port,
+            'remote_ip': remote_address,
+            'remote_port': remote_port,
+            'status': event['EventHeader']['Status'],
+            'pid': event['pid'],
+            'name': event['name'],
+            'cmdline': event['cmdline']
+        }
+
+        if self.attrs:
+            event_dict = dicts.reorder_keys(
+                event_dict, self.attrs, skip_keys_not_in_list=True)
+
+        return event_dict

atomicshop/file_io/csvs.py

@@ -1,29 +1,42 @@
 import csv
+import io
 from typing import Tuple, List
 
-from .file_io import read_file_decorator
 from . import file_io
 
 
-@read_file_decorator
-def read_csv_to_list(file_path: str,
-                     file_mode: str = 'r',
-                     encoding=None,
-                     header: list = None,
-                     file_object=None,
-                     **kwargs) -> Tuple[List, List | None]:
+@file_io.read_file_decorator
+def read_csv_to_list_of_dicts_by_header(
+        file_path: str,
+        file_mode: str = 'r',
+        encoding=None,
+        header: list = None,
+        file_object=None,
+        **kwargs
+) -> Tuple[List, List | None]:
     """
     Function to read csv file and output its contents as list of dictionaries for each row.
+    Each key of the dictionary is a header field.
+
+    Example:
+        CSV file:
+        name,age,city
+        John,25,New York
+
+        Output:
+        [{'name': 'John', 'age': '25', 'city': 'New York'}]
 
     :param file_path: String with full file path to json file.
     :param file_mode: string, file reading mode. Examples: 'r', 'rb'. Default is 'r'.
     :param encoding: string, encoding of the file. Default is 'None'.
     :param header: list, list of strings that will be the header of the CSV file. Default is 'None'.
-        If you want to use the header from the CSV file, use 'None'. In this case, the first row of the CSV file will
-        be the header.
+        None: the header from the CSV file will be used. The first row of the CSV file will be the header.
+            Meaning, that the first line will be skipped and the second line will be the first row of the content.
+        List: the list will be used as header.
+            All the lines of the CSV file will be considered as content.
     :param file_object: file object of the 'open()' function in the decorator. Decorator executes the 'with open()'
         statement and passes to this function. That's why the default is 'None', since we get it from the decorator.
-    :return: list.
+    :return: tuple(list of entries, header(list of cell names)).
     """
 
     # The header fields will be separated to list of "csv_reader.fieldnames".
@@ -39,25 +52,92 @@ def read_csv_to_list(file_path: str,
     return csv_list, header
 
 
-def write_list_to_csv(csv_list: list, csv_filepath: str) -> None:
+@file_io.read_file_decorator
+def read_csv_to_list_of_lists(
+        file_path: str,
+        file_mode: str = 'r',
+        encoding=None,
+        exclude_header_from_content: bool = False,
+        file_object=None,
+        **kwargs
+) -> Tuple[List, List | None]:
+    """
+    Function to read csv file and output its contents as list of lists for each row.
+
+    Example:
+        CSV file:
+        name,age,city
+        John,25,New York
+
+        Output:
+        [['name', 'age', 'city'], ['John', '25', 'New York']]
+
+    :param file_path: String with full file path to json file.
+    :param file_mode: string, file reading mode. Examples: 'r', 'rb'. Default is 'r'.
+    :param encoding: string, encoding of the file. Default is 'None'.
+    :param exclude_header_from_content: Boolean, if True, the header will be excluded from the content.
+    :param file_object: file object of the 'open()' function in the decorator. Decorator executes the 'with open()'
+        statement and passes to this function. That's why the default is 'None', since we get it from the decorator.
+    :param kwargs: Keyword arguments for 'read_file' function.
+    :return: list.
     """
-    Function to write list object that each iteration of it contains dict object with same keys and different values.
 
-    :param csv_list: List object that each iteration contains dictionary with same keys and different values.
-    :param csv_filepath: Full file path to CSV file.
+    # Read CSV file to list of lists.
+    csv_reader = csv.reader(file_object)
+
+    csv_list = list(csv_reader)
+
+    # Get the header if there is only something in the content.
+    if csv_list:
+        header = csv_list[0]
+    else:
+        header = []
+
+    if exclude_header_from_content and csv_list:
+        csv_list.pop(0)
+
+    return csv_list, header
+
+
+def write_list_to_csv(
+        content_list: list,
+        file_path: str,
+        mode: str = 'w',
+        encoding: str = None
+) -> None:
+    """
+    This function got dual purpose:
+    1. Write list object that each iteration of it contains list object with same length.
+    2. Write list object that each iteration of it contains dict object with same keys and different values.
+    The dictionary inside the function will be identified by the first iteration of the list.
+    Other objects (inside the provided list) than dictionary will be identified as regular objects.
+
+    :param content_list: List object that each iteration contains dictionary with same keys and different values.
+    :param file_path: Full file path to CSV file.
+    :param mode: String, file writing mode. Default is 'w'.
+    :param encoding: String, encoding of the file. Default is 'None'.
+        Example: 'utf-8', 'utf-16', 'cp1252'.
     :return: None.
     """
 
-    with open(csv_filepath, mode='w') as csv_file:
-        # Create header from keys of the first dictionary in list.
-        header = csv_list[0].keys()
-        # Create CSV writer.
-        writer = csv.DictWriter(csv_file, fieldnames=header, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
+    with open(file_path, mode=mode, newline='', encoding=encoding) as csv_file:
+        if len(content_list) > 0 and isinstance(content_list[0], dict):
+            # Treat the list as list of dictionaries.
+            header = content_list[0].keys()
 
-        # Write header.
-        writer.writeheader()
-        # Write list of dits as rows.
-        writer.writerows(csv_list)
+            # Create CSV writer.
+            writer = csv.DictWriter(csv_file, fieldnames=header, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
+
+            # Write header.
+            writer.writeheader()
+            # Write list of dits as rows.
+            writer.writerows(content_list)
+        # Else, treat the list as list of lists.
+        else:
+            # Create CSV writer.
+            writer = csv.writer(csv_file)
+            # Write list of lists as rows.
+            writer.writerows(content_list)
 
 
 def get_header(file_path: str, print_kwargs: dict = None) -> list:
@@ -78,3 +158,121 @@ def get_header(file_path: str, print_kwargs: dict = None) -> list:
     # Split the header to list of keys.
     header = header.split(',')
     return header
+
+
+def _escape_csv_value_ref(text):
+    """
+    FOR REFERENCE ONLY, better use csv module to do it natively.
+    Function to escape text for CSV file.
+    This function escapes commas (,) and double quotes (") for csv cell (between commas).
+
+    Example:
+        test1 = 'test1'
+        test2 = 'test,2'
+        test3 = 'test3,"3",3'
+
+        csv_line = f'{escape_csv_value(test1)},{escape_csv_value(test2)},{escape_csv_value(test3)}'
+
+        Output: 'test1,"test,2","test3,""3"",3"'
+    """
+
+    if '"' in text:
+        text = text.replace('"', '""')  # Escape double quotes
+    if ',' in text or '"' in text:
+        text = f'"{text}"'  # Enclose in double quotes if there are commas or double quotes
+    return text
+
+
+def escape_csv_value(value):
+    """
+    Function to escape text for CSV file.
+    This function escapes commas (,) and double quotes (") for csv cell (between commas).
+
+    Example:
+        test1 = 'test1'
+        test2 = 'test,2'
+        test3 = 'test3,"3",3'
+
+        csv_line = f'{escape_csv_value(test1)},{escape_csv_value(test2)},{escape_csv_value(test3)}'
+
+        Output: 'test1,"test,2","test3,""3"",3"'
+    """
+    output = io.StringIO()
+    writer = csv.writer(output, quoting=csv.QUOTE_MINIMAL)
+    writer.writerow([value])
+    return output.getvalue().strip()
+
+
+def escape_csv_line_to_string(csv_line: list) -> str:
+    """
+    Function to escape list of strings for CSV file.
+    This function escapes commas (,) and double quotes (") for csv cell (between commas).
+
+    Example:
+        test1 = 'test1'
+        test2 = 'test,2'
+        test3 = 'test3,"3",3'
+
+        csv_line = escape_csv_line_to_string([test1, test2, test3])
+
+        Output:
+        csv_line == 'test1,"test,2","test3,""3"",3"'
+    """
+
+    # Prepare the data as a list of lists
+    data = [csv_line]
+
+    # Use StringIO to create an in-memory file-like object
+    output = io.StringIO()
+    writer = csv.writer(output, quoting=csv.QUOTE_MINIMAL)
+
+    # Write the data to the CSV writer
+    writer.writerows(data)
+
+    # Get the CSV string from the StringIO object, Strip to remove any trailing newlines.
+    csv_line = output.getvalue().strip()
+
+    return csv_line
+
+
+def escape_csv_line_to_list(csv_line: list) -> list:
+    """
+    Function to escape list of strings for CSV file.
+    This function escapes commas (,) and double quotes (") for csv cell (between commas).
+
+    Example:
+        test1 = 'test1'
+        test2 = 'test,2'
+        test3 = 'test3,"3",3'
+
+        csv_entries_list = escape_csv_line_to_list([test1, test2, test3])
+
+        Output:
+        csv_entries_list == ['test1', '"test,2"', '"test3,""3"",3"']
+    """
+
+    result_csv_entries: list = []
+    for entry in csv_line:
+        result_csv_entries.append(escape_csv_value(entry))
+
+    return result_csv_entries
+
+
+def get_number_of_cells_in_string_line(line: str) -> int:
+    """
+    Function to get number of cells in CSV line.
+
+    :param line: String, line of CSV file.
+    :return: int, number of cells in the line.
+    """
+
+    # Create CSV reader from 'input_file'. By default, the first row will be the header if 'fieldnames' is None.
+    csv_reader = csv.reader([line])
+
+    # Get the first row of the CSV file.
+    csv_list = list(csv_reader)
+
+    # Get the number of cells in the first row.
+    number_of_cells = len(csv_list[0])
+
+    return number_of_cells

atomicshop/file_io/docxs.py

@@ -12,7 +12,7 @@ def get_hyperlinks(docx_path):
     :return: list of strings, hyperlinks.
     """
 
-    hyperlinks: list = list()
+    hyperlinks: list[dict] = list()
 
     try:
         doc = Document(docx_path)
@@ -26,7 +26,19 @@ def get_hyperlinks(docx_path):
         if not paragraph.hyperlinks:
             continue
         for hyperlink in paragraph.hyperlinks:
-            hyperlinks.append(hyperlink.address)
+            # Hyperlinks are stored in docx (document.xml) without the fragment part.
+            # Fragment is the anchor of the link, for example: 'https://www.example.com#anchor'.
+            # So the hyperlink.address is stored as 'https://www.example.com'.
+            # And the fragment is stored in the hyperlink.fragment as 'anchor'.
+            # For the full hyperlink, we need to concatenate the address and the fragment.
+            # If there is no anchor in the link the fragment will be empty string ('').
+            # Basically, we don't need to add the fragment to the hyperlink if it's empty, we can just use the url.
+            # if hyperlink.fragment:
+            #     hyperlinks.append(hyperlink.address + "#" + hyperlink.fragment)
+            hyperlinks.append({
+                'url': hyperlink.url,
+                'text': hyperlink.text
+            })
 
     return hyperlinks
 
@@ -62,28 +74,31 @@ def search_for_hyperlink_in_files(directory_path: str, hyperlink: str, relative_
             input('press Enter')
         """
 
-    if not filesystem.check_directory_existence(directory_path):
+    if not filesystem.is_directory_exists(directory_path):
         raise NotADirectoryError(f"Directory doesn't exist: {directory_path}")
 
     # Get all the docx files in the specified directory.
-    files = filesystem.get_file_paths_from_directory(
-        directory_path, file_name_check_pattern="*\.docx",
+    files = filesystem.get_paths_from_directory(
+        directory_path, get_file=True, file_name_check_pattern="*.docx",
         add_relative_directory=True, relative_file_name_as_directory=True)
 
-    found_in_files: list = list()
+    found_in_files: list[dict] = list()
     for file_path in files:
-        hyperlinks = get_hyperlinks(file_path['file_path'])
-        if hyperlink in hyperlinks:
-            found_in_files.append(file_path)
+        doc_hyperlinks = get_hyperlinks(file_path.path)
+        for doc_link in doc_hyperlinks:
+            if hyperlink in doc_link['url']:
+                if relative_paths:
+                    path: str = file_path.relative_dir
+                else:
+                    path: str = file_path.path
 
-    if relative_paths:
-        result_list = list()
-        for found_file in found_in_files:
-            result_list.append(found_file['relative_dir'])
-    else:
-        result_list = found_in_files
+                found_in_files.append({
+                    'path':path,
+                    'link':doc_link['url'],
+                    'text':doc_link['text']
+                })
 
-    return result_list
+    return found_in_files
 
 
 def search_for_hyperlink_in_files_interface_main(script_directory: str = None):
@@ -126,10 +141,12 @@ def search_for_hyperlink_in_files_interface_main(script_directory: str = None):
     found_in_files = search_for_hyperlink_in_files(
         config['directory_path'], config['hyperlink'], relative_paths=config['relative_paths'])
 
-    print_api(f"Found in [{len(found_in_files)}] files:", color="blue")
+    print_api(f"Found [{len(found_in_files)}] links:", color="blue")
 
     for index, found_file in enumerate(found_in_files):
         print_api(f"[{index+1}]", print_end="", color="green")
-        print_api(f" {found_file}")
+        print_api(f" {found_file['path']}")
+        print_api(f" {found_file['link']}", color="cyan")
+        print_api(f" {found_file['text']}", color="orange")
 
     input('[*] Press [Enter] to exit...')