maco 1.2.15__py3-none-any.whl → 1.2.16__py3-none-any.whl

maco/model/model.py

@@ -391,6 +391,18 @@ class ExtractorModel(ForbidModel):
 
     proxy: List[Proxy] = []
 
+    class ICMP(ForbidModel):
+        """Usage of ICMP."""
+
+        type: Optional[int] = None
+        code: Optional[int] = None
+        header: Optional[str] = None  # Some malware uses non-standard header fields
+        hostname: Optional[str] = None
+
+        usage: Optional[ConnUsageEnum] = None
+
+    icmp: List[ICMP] = []
+
     #
     # inter process communication (IPC)
     #

maco/utils.py

@@ -1,8 +1,6 @@
 """Common utilities shared between the MACO collector and configextractor-py."""
 
 import importlib
-import importlib.machinery
-import importlib.util
 import inspect
 import json
 import logging
@@ -13,8 +11,9 @@ import shutil
 import subprocess
 import sys
 import tempfile
+from importlib.machinery import SourceFileLoader
 
-from multiprocess import Queue
+from multiprocess import Process, Queue
 
 from maco import yara
 
@@ -27,9 +26,8 @@ from base64 import b64decode
 from copy import deepcopy
 from glob import glob
 from logging import Logger
-from pkgutil import walk_packages
 from types import ModuleType
-from typing import Callable, Dict, List, Set, Tuple, Union
+from typing import Callable, Dict, List, Tuple, Union
 
 from uv import find_uv_bin
 
@@ -343,13 +341,43 @@ def find_and_insert_venv(path: str, venvs: List[str]) -> Tuple[str, str]:
     return venv, site_package
 
 
+def register_extractor_module(
+    extractor_source_file: str,
+    module_name: str,
+    venvs: List[str],
+    extractor_module_callback: Callable[[ModuleType, str], None],
+    logger: Logger,
+):
+    """Register the extractor module in isolation.
+
+    Args:
+        extractor_source_file (str): Path to source file of extractor
+        module_name (str): The name of the module relative to the package directory
+        venvs (List[str]): List of virtual environments
+        extractor_module_callback (Callable[[ModuleType, str], None]): Callback used to register extractors
+        logger (Logger): Logger to use
+
+    """
+    try:
+        logger.info(f"Inspecting '{extractor_source_file}' for extractors..")
+        venv, site_packages = find_and_insert_venv(extractor_source_file, venvs)
+        loader = SourceFileLoader(
+            module_name,
+            extractor_source_file,
+        )
+        extractor_module_callback(loader.load_module(), venv)
+    finally:
+        # Cleanup virtual environment that was loaded into PATH
+        if venv and site_packages in sys.path:
+            sys.path.remove(site_packages)
+
+
 def register_extractors(
     current_directory: str,
     venvs: List[str],
     extractor_files: List[str],
     extractor_module_callback: Callable[[ModuleType, str], None],
     logger: Logger,
-    default_loaded_modules: Set[str] = set(sys.modules.keys()),
 ):
     """Register extractors with in the current directory.
 
@@ -359,7 +387,6 @@ def register_extractors(
         extractor_files (List[str]): List of extractor files found
         extractor_module_callback (Callable[[ModuleType, str], None]): Callback used to register extractors
         logger (Logger): Logger to use
-        default_loaded_modules (Set[str]): Set of default loaded modules
     """
     package_name = os.path.basename(current_directory)
     parent_directory = os.path.dirname(current_directory)
@@ -375,74 +402,25 @@ def register_extractors(
         sys.path.insert(1, current_directory)
         sys.path.insert(1, parent_directory)
 
-        # Insert any virtual environment necessary to load directory as package
-        package_venv, package_site_packages = find_and_insert_venv(current_directory, venvs)
-        package = importlib.import_module(package_name)
+        # Load the potential extractors directly from the source file
+        registration_processes = []
+        for extractor_source_file in extractor_files:
+            module_name = extractor_source_file.replace(f"{parent_directory}/", "").replace("/", ".")[:-3]
+            p = Process(
+                target=register_extractor_module,
+                args=(extractor_source_file, module_name, venvs, extractor_module_callback, logger),
+            )
+            p.start()
+            registration_processes.append(p)
 
-        # Walk through our new package and find the extractors that YARA identified
-        for module_path, module_name, ispkg in walk_packages(package.__path__, package.__name__ + "."):
-            if ispkg:
-                # Skip packages
-                continue
+        for p in registration_processes:
+            p.join()
 
-            module_path = os.path.realpath(os.path.join(module_path.path, module_name.rsplit(".", 1)[1]) + ".py")
-            if module_path in extractor_files:
-                # Cross this extractor off the list of extractors to find
-                logger.debug(f"Inspecting '{module_name}' for extractors..")
-                extractor_files.remove(module_path)
-                try:
-                    # This is an extractor we've been looking for, load the module and invoke callback
-                    venv, site_packages = find_and_insert_venv(module_path, venvs)
-                    module = importlib.import_module(module_name)
-                    module.__file__ = os.path.realpath(module.__file__)
-
-                    # Patch the original directory information into the module
-                    original_package_name = os.path.basename(current_directory)
-                    module.__name__ = module.__name__.replace(package_name, original_package_name)
-                    module.__package__ = module.__package__.replace(package_name, original_package_name)
-                    extractor_module_callback(module, venv)
-                finally:
-                    # Cleanup virtual environment that was loaded into PATH
-                    if venv and site_packages in sys.path:
-                        sys.path.remove(site_packages)
-
-            if not extractor_files:
-                return
     finally:
         # Cleanup changes made to PATH
         sys.path.remove(parent_directory)
         sys.path.remove(current_directory)
 
-        if package_venv and package_site_packages in sys.path:
-            sys.path.remove(package_site_packages)
-
-        # Remove any modules that were loaded to deconflict with later modules loads
-        [sys.modules.pop(k) for k in set(sys.modules.keys()) - default_loaded_modules]
-
-    # If there still exists extractor files we haven't found yet, try searching in the available subdirectories
-    if extractor_files:
-        for dir in os.listdir(current_directory):
-            path = os.path.join(current_directory, dir)
-            if dir == "__pycache__":
-                # Ignore the cache created
-                continue
-            elif dir.endswith(".egg-info"):
-                # Ignore these directories
-                continue
-            elif dir.startswith("."):
-                # Ignore hidden directories
-                continue
-
-            if os.path.isdir(path):
-                # Check subdirectory to find the rest of the detected extractors
-                register_extractors(
-                    path, venvs, extractor_files, extractor_module_callback, logger, default_loaded_modules
-                )
-
-            if not extractor_files:
-                # We were able to find all the extractor files
-                break
-
 
 def proxy_logging(queue: Queue, callback: Callable[[ModuleType, str], None], *args, **kwargs):
     """Ensures logging is set up correctly for a child process and then executes the callback."""

{maco-1.2.15.dist-info → maco-1.2.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: maco
-Version: 1.2.15
+Version: 1.2.16
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License

{maco-1.2.15.dist-info → maco-1.2.16.dist-info}/RECORD

@@ -14,21 +14,21 @@ maco/cli.py,sha256=nrSukAJAthbstZT3-lQNPz4zOOMcBhvfYQqLh_B5Jdk,9457
 maco/collector.py,sha256=R3zw-fUJBlwmcSqvkQ-PnoJdHfRm2V0JAOl7N8MTAbY,8240
 maco/exceptions.py,sha256=XBHUrs1kr1ZayPI9B_W-WejKgVmC8sWL_o4RL0b4DQE,745
 maco/extractor.py,sha256=s36aGcsXSc-9iCik6iihVt5G1a1DZUA7TquvWYQNwdE,2912
-maco/utils.py,sha256=rXKrrKTNi7DEC5SZUnUQcxnRRmJXRp0y4DuVaDkBYvY,24977
+maco/utils.py,sha256=wzecqu1W_BbhDHSs07sweOKNeFrmC4CDYmAN_qyYJS4,23362
 maco/yara.py,sha256=gkHHxwZNxzZV7nHZM3HNUmhHXB7VW82voCHK5mHpt2Q,3970
 maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
-maco/model/model.py,sha256=whdeqwphReHpgQ5f6kODB7pQI3UEylTTiVqNq_FHNBg,24156
-maco-1.2.15.dist-info/licenses/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
+maco/model/model.py,sha256=DBHTmZXMzjpVq0s2mzZv3VCzPhwPnv7sH6u_QZCTcA4,24484
+maco-1.2.16.dist-info/licenses/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
 model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 model_setup/maco/base_test.py,sha256=DrVE7vOazeLQpOQeIDwBYK1WtlmdJrRe50JOqP5t4Y0,3198
 model_setup/maco/cli.py,sha256=nrSukAJAthbstZT3-lQNPz4zOOMcBhvfYQqLh_B5Jdk,9457
 model_setup/maco/collector.py,sha256=R3zw-fUJBlwmcSqvkQ-PnoJdHfRm2V0JAOl7N8MTAbY,8240
 model_setup/maco/exceptions.py,sha256=XBHUrs1kr1ZayPI9B_W-WejKgVmC8sWL_o4RL0b4DQE,745
 model_setup/maco/extractor.py,sha256=s36aGcsXSc-9iCik6iihVt5G1a1DZUA7TquvWYQNwdE,2912
-model_setup/maco/utils.py,sha256=rXKrrKTNi7DEC5SZUnUQcxnRRmJXRp0y4DuVaDkBYvY,24977
+model_setup/maco/utils.py,sha256=wzecqu1W_BbhDHSs07sweOKNeFrmC4CDYmAN_qyYJS4,23362
 model_setup/maco/yara.py,sha256=gkHHxwZNxzZV7nHZM3HNUmhHXB7VW82voCHK5mHpt2Q,3970
 model_setup/maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
-model_setup/maco/model/model.py,sha256=whdeqwphReHpgQ5f6kODB7pQI3UEylTTiVqNq_FHNBg,24156
+model_setup/maco/model/model.py,sha256=DBHTmZXMzjpVq0s2mzZv3VCzPhwPnv7sH6u_QZCTcA4,24484
 pipelines/publish.yaml,sha256=xt3WNU-5kIICJgKIiiE94M3dWjS3uEiun-n4OmIssK8,1471
 pipelines/test.yaml,sha256=btJVI-R39UBeYosGu7TOpU6V9ogFW3FT3ROtWygQGQ0,1472
 tests/data/example.txt.cart,sha256=j4ZdDnFNVq7lb-Qi4pY4evOXKQPKG-GSg-n-uEqPhV0,289
@@ -42,8 +42,8 @@ tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 tests/extractors/bob/bob.py,sha256=4fpqy_O6NDinJImghyW5OwYgnaB05aY4kgoIS_C3c_U,253
 tests/extractors/import_rewriting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/extractors/import_rewriting/importer.py,sha256=wqF1AG2zXXuj9EMt9qlDorab-UD0GYuFggtrCuz4sf0,289735
-maco-1.2.15.dist-info/METADATA,sha256=BBjPNqDyPQPPwFHL0G0LqOrM2zYURFENydH2K63J6aU,15232
-maco-1.2.15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-maco-1.2.15.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
-maco-1.2.15.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
-maco-1.2.15.dist-info/RECORD,,
+maco-1.2.16.dist-info/METADATA,sha256=v5ZV9RoifsSn6im1tO0wpUJeKPGW3Wf-_C_BjF7qNmM,15232
+maco-1.2.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+maco-1.2.16.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
+maco-1.2.16.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
+maco-1.2.16.dist-info/RECORD,,

model_setup/maco/model/model.py

@@ -391,6 +391,18 @@ class ExtractorModel(ForbidModel):
 
     proxy: List[Proxy] = []
 
+    class ICMP(ForbidModel):
+        """Usage of ICMP."""
+
+        type: Optional[int] = None
+        code: Optional[int] = None
+        header: Optional[str] = None  # Some malware uses non-standard header fields
+        hostname: Optional[str] = None
+
+        usage: Optional[ConnUsageEnum] = None
+
+    icmp: List[ICMP] = []
+
     #
     # inter process communication (IPC)
     #

model_setup/maco/utils.py

@@ -1,8 +1,6 @@
 """Common utilities shared between the MACO collector and configextractor-py."""
 
 import importlib
-import importlib.machinery
-import importlib.util
 import inspect
 import json
 import logging
@@ -13,8 +11,9 @@ import shutil
 import subprocess
 import sys
 import tempfile
+from importlib.machinery import SourceFileLoader
 
-from multiprocess import Queue
+from multiprocess import Process, Queue
 
 from maco import yara
 
@@ -27,9 +26,8 @@ from base64 import b64decode
 from copy import deepcopy
 from glob import glob
 from logging import Logger
-from pkgutil import walk_packages
 from types import ModuleType
-from typing import Callable, Dict, List, Set, Tuple, Union
+from typing import Callable, Dict, List, Tuple, Union
 
 from uv import find_uv_bin
 
@@ -343,13 +341,43 @@ def find_and_insert_venv(path: str, venvs: List[str]) -> Tuple[str, str]:
     return venv, site_package
 
 
+def register_extractor_module(
+    extractor_source_file: str,
+    module_name: str,
+    venvs: List[str],
+    extractor_module_callback: Callable[[ModuleType, str], None],
+    logger: Logger,
+):
+    """Register the extractor module in isolation.
+
+    Args:
+        extractor_source_file (str): Path to source file of extractor
+        module_name (str): The name of the module relative to the package directory
+        venvs (List[str]): List of virtual environments
+        extractor_module_callback (Callable[[ModuleType, str], None]): Callback used to register extractors
+        logger (Logger): Logger to use
+
+    """
+    try:
+        logger.info(f"Inspecting '{extractor_source_file}' for extractors..")
+        venv, site_packages = find_and_insert_venv(extractor_source_file, venvs)
+        loader = SourceFileLoader(
+            module_name,
+            extractor_source_file,
+        )
+        extractor_module_callback(loader.load_module(), venv)
+    finally:
+        # Cleanup virtual environment that was loaded into PATH
+        if venv and site_packages in sys.path:
+            sys.path.remove(site_packages)
+
+
 def register_extractors(
     current_directory: str,
     venvs: List[str],
     extractor_files: List[str],
     extractor_module_callback: Callable[[ModuleType, str], None],
     logger: Logger,
-    default_loaded_modules: Set[str] = set(sys.modules.keys()),
 ):
     """Register extractors with in the current directory.
 
@@ -359,7 +387,6 @@ def register_extractors(
         extractor_files (List[str]): List of extractor files found
         extractor_module_callback (Callable[[ModuleType, str], None]): Callback used to register extractors
         logger (Logger): Logger to use
-        default_loaded_modules (Set[str]): Set of default loaded modules
     """
     package_name = os.path.basename(current_directory)
     parent_directory = os.path.dirname(current_directory)
@@ -375,74 +402,25 @@ def register_extractors(
         sys.path.insert(1, current_directory)
         sys.path.insert(1, parent_directory)
 
-        # Insert any virtual environment necessary to load directory as package
-        package_venv, package_site_packages = find_and_insert_venv(current_directory, venvs)
-        package = importlib.import_module(package_name)
+        # Load the potential extractors directly from the source file
+        registration_processes = []
+        for extractor_source_file in extractor_files:
+            module_name = extractor_source_file.replace(f"{parent_directory}/", "").replace("/", ".")[:-3]
+            p = Process(
+                target=register_extractor_module,
+                args=(extractor_source_file, module_name, venvs, extractor_module_callback, logger),
+            )
+            p.start()
+            registration_processes.append(p)
 
-        # Walk through our new package and find the extractors that YARA identified
-        for module_path, module_name, ispkg in walk_packages(package.__path__, package.__name__ + "."):
-            if ispkg:
-                # Skip packages
-                continue
+        for p in registration_processes:
+            p.join()
 
-            module_path = os.path.realpath(os.path.join(module_path.path, module_name.rsplit(".", 1)[1]) + ".py")
-            if module_path in extractor_files:
-                # Cross this extractor off the list of extractors to find
-                logger.debug(f"Inspecting '{module_name}' for extractors..")
-                extractor_files.remove(module_path)
-                try:
-                    # This is an extractor we've been looking for, load the module and invoke callback
-                    venv, site_packages = find_and_insert_venv(module_path, venvs)
-                    module = importlib.import_module(module_name)
-                    module.__file__ = os.path.realpath(module.__file__)
-
-                    # Patch the original directory information into the module
-                    original_package_name = os.path.basename(current_directory)
-                    module.__name__ = module.__name__.replace(package_name, original_package_name)
-                    module.__package__ = module.__package__.replace(package_name, original_package_name)
-                    extractor_module_callback(module, venv)
-                finally:
-                    # Cleanup virtual environment that was loaded into PATH
-                    if venv and site_packages in sys.path:
-                        sys.path.remove(site_packages)
-
-            if not extractor_files:
-                return
     finally:
         # Cleanup changes made to PATH
         sys.path.remove(parent_directory)
         sys.path.remove(current_directory)
 
-        if package_venv and package_site_packages in sys.path:
-            sys.path.remove(package_site_packages)
-
-        # Remove any modules that were loaded to deconflict with later modules loads
-        [sys.modules.pop(k) for k in set(sys.modules.keys()) - default_loaded_modules]
-
-    # If there still exists extractor files we haven't found yet, try searching in the available subdirectories
-    if extractor_files:
-        for dir in os.listdir(current_directory):
-            path = os.path.join(current_directory, dir)
-            if dir == "__pycache__":
-                # Ignore the cache created
-                continue
-            elif dir.endswith(".egg-info"):
-                # Ignore these directories
-                continue
-            elif dir.startswith("."):
-                # Ignore hidden directories
-                continue
-
-            if os.path.isdir(path):
-                # Check subdirectory to find the rest of the detected extractors
-                register_extractors(
-                    path, venvs, extractor_files, extractor_module_callback, logger, default_loaded_modules
-                )
-
-            if not extractor_files:
-                # We were able to find all the extractor files
-                break
-
 
 def proxy_logging(queue: Queue, callback: Callable[[ModuleType, str], None], *args, **kwargs):
     """Ensures logging is set up correctly for a child process and then executes the callback."""