maco 1.2.4__py3-none-any.whl → 1.2.6__py3-none-any.whl

maco/utils.py

@@ -4,14 +4,14 @@ import importlib.machinery
 import importlib.util
 import inspect
 import json
+import logging
 import logging.handlers
+import multiprocessing
 import os
 import re
 import shutil
 import subprocess
 import sys
-import multiprocessing
-import logging
 import tempfile
 
 from maco import yara
@@ -27,8 +27,11 @@ from glob import glob
 from logging import Logger
 from pkgutil import walk_packages
 from types import ModuleType
-from typing import Callable, Dict, List, Set, Tuple
+from typing import Callable, Dict, List, Set, Tuple, Union
+
+from uv import find_uv_bin
 
+from maco import model
 from maco.extractor import Extractor
 
 logger = logging.getLogger("maco.lib.utils")
@@ -38,22 +41,10 @@ VENV_DIRECTORY_NAME = ".venv"
 RELATIVE_FROM_RE = re.compile(r"from (\.+)")
 RELATIVE_FROM_IMPORT_RE = re.compile(r"from (\.+) import")
 
-try:
-    # Attempt to use the uv package manager (Recommended)
-    from uv import find_uv_bin
+UV_BIN = find_uv_bin()
 
-    UV_BIN = find_uv_bin()
-
-    PIP_CMD = f"{UV_BIN} pip"
-    VENV_CREATE_CMD = f"{UV_BIN} venv"
-    PACKAGE_MANAGER = "uv"
-except ImportError:
-    # Otherwise default to pip
-    from sys import executable
-
-    PIP_CMD = "pip"
-    VENV_CREATE_CMD = f"{executable} -m venv"
-    PACKAGE_MANAGER = "pip"
+PIP_CMD = f"{UV_BIN} pip"
+VENV_CREATE_CMD = f"{UV_BIN} venv"
 
 
 class Base64Decoder(json.JSONDecoder):
@@ -189,11 +180,9 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
                         for pattern in [RELATIVE_FROM_IMPORT_RE, RELATIVE_FROM_RE]:
                             for match in pattern.findall(data):
                                 depth = match.count(".")
-                                data = data.replace(
-                                    f"from {match}",
-                                    f"from {'.'.join(split[depth - 1 : split.index(package) + 1][::-1])}{'.' if pattern == RELATIVE_FROM_RE else ''}",
-                                    1,
-                                )
+                                abspath = ".".join(split[depth - 1 : split.index(package) + 1][::-1])
+                                abspath += "." if pattern == RELATIVE_FROM_RE else ""
+                                data = data.replace(f"from {match}", f"from {abspath}", 1)
                         f.write(data)
 
                 if scanner.match(path):
@@ -210,9 +199,8 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
     return extractor_dirs, extractor_files
 
 
-def create_virtual_environments(directories: List[str], python_version: str, logger: Logger):
+def _install_required_packages(create_venv: bool, directories: List[str], python_version: str, logger: Logger):
     venvs = []
-    logger.info("Creating virtual environment(s)..")
     env = deepcopy(os.environ)
     stop_directory = os.path.dirname(sorted(directories)[0])
     # Track directories that we've already visited
@@ -222,17 +210,23 @@ def create_virtual_environments(directories: List[str], python_version: str, log
         while dir != stop_directory and dir not in visited_dirs:
             req_files = list({"requirements.txt", "pyproject.toml"}.intersection(set(os.listdir(dir))))
             if req_files:
-                venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
-                env.update({"VIRTUAL_ENV": venv_path})
-                # Create a virtual environment for the directory
-                if not os.path.exists(venv_path):
-                    cmd = VENV_CREATE_CMD
-                    if PACKAGE_MANAGER == "uv":
-                        cmd += f" --python {python_version}"
-                    subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
+                # create a virtual environment, otherwise directly install into current env
+                if create_venv:
+                    venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
+                    logger.info(f"Updating virtual environment {venv_path}")
+                    env.update({"VIRTUAL_ENV": venv_path})
+                    # Create a virtual environment for the directory
+                    if not os.path.exists(venv_path):
+                        cmd = f"{VENV_CREATE_CMD} --python {python_version}"
+                        subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
 
                 # Install/Update the packages in the environment
-                install_command = PIP_CMD.split(" ") + ["install", "-U"]
+                install_command = PIP_CMD.split(" ") + ["install"]
+                # When running locally, only install packages to required spec.
+                # This prevents issues during maco development and building extractors against local libraries.
+                if create_venv:
+                    # when running in custom virtual environment, always upgrade packages.
+                    install_command.append("-U")
 
                 # Update the pip install command depending on where the dependencies are coming from
                 if "requirements.txt" in req_files:
@@ -253,7 +247,10 @@ def create_virtual_environments(directories: List[str], python_version: str, log
 
                     install_command.extend(pyproject_command)
 
+                # always require maco to be installed
+                install_command.append("maco")
                 logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
+                # this uses VIRTUAL_ENV to control usage of a virtual environment
                 p = subprocess.run(
                     install_command,
                     cwd=dir,
@@ -264,10 +261,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
                     if b"is being installed using the legacy" in p.stderr:
                         # Ignore these types of errors
                         continue
-                    logger.error(f"Error installing into venv:\n{p.stderr.decode()}")
+                    logger.error(f"Error installing into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
                 else:
-                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
-                    venvs.append(venv_path)
+                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
+                    if create_venv:
+                        venvs.append(venv_path)
 
                 # Cleanup any build directories that are the product of package installation
                 expected_build_path = os.path.join(dir, "build")
@@ -311,10 +309,12 @@ def register_extractors(
 ):
     package_name = os.path.basename(current_directory)
     parent_directory = os.path.dirname(current_directory)
-    if package_name in sys.modules:
+    if venvs and package_name in sys.modules:
         # this may happen as part of testing if some part of the extractor code was directly imported
-        logger.warning(f"Looks like {package_name} is already loaded. "
-                       "If your maco extractor overlaps an existing package name this could cause problems.")
+        logger.warning(
+            f"Looks like {package_name} is already loaded. "
+            "If your maco extractor overlaps an existing package name this could cause problems."
+        )
 
     try:
         # Modify the PATH so we can recognize this new package on import
@@ -389,6 +389,7 @@ def register_extractors(
                 # We were able to find all the extractor files
                 break
 
+
 def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType, str], None], *args, **kwargs):
     """Ensures logging is set up correctly for a child process and then executes the callback."""
     logger = logging.getLogger()
@@ -397,37 +398,33 @@ def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType,
     logger.addHandler(qh)
     callback(*args, **kwargs, logger=logger)
 
+
 def import_extractors(
     extractor_module_callback: Callable[[ModuleType, str], bool],
     *,
     root_directory: str,
     scanner: yara.Rules,
-    create_venv: bool = False,
-    python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
+    create_venv: bool,
     logger: Logger,
+    python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
 ):
     extractor_dirs, extractor_files = scan_for_extractors(root_directory, scanner, logger)
 
     logger.info(f"Extractor files found based on scanner ({len(extractor_files)}).")
     logger.debug(extractor_files)
 
-    venvs = []
-    if create_venv:
-        venvs = create_virtual_environments(extractor_dirs, python_version, logger)
-    else:
-        # Look for pre-existing virtual environments, if any
-        logger.info("Checking for pre-existing virtual environment(s)..")
-        venvs = [
-            os.path.join(root, VENV_DIRECTORY_NAME)
-            for root, dirs, _ in os.walk(root_directory)
-            if VENV_DIRECTORY_NAME in dirs
-        ]
+    # Install packages into the current environment or dynamically created virtual environments
+    venvs = _install_required_packages(create_venv, extractor_dirs, python_version, logger)
 
     # With the environment prepared, we can now hunt for the extractors and register them
     logger.info("Registering extractors..")
     register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
 
 
+# holds cached extractors when not running in venv mode
+_loaded_extractors: Dict[str, Extractor] = {}
+
+
 def run_extractor(
     sample_path,
     module_name,
@@ -436,55 +433,69 @@ def run_extractor(
     venv,
     venv_script=VENV_SCRIPT,
     json_decoder=Base64Decoder,
-) -> Dict[str, dict]:
-    # Write temporary script in the same directory as extractor to resolve relative imports
-    python_exe = sys.executable
-    if venv:
-        # If there is a linked virtual environment, execute within that environment
+) -> Union[Dict[str, dict], model.ExtractorModel]:
+    """Runs the maco extractor against sample either in current process or child process."""
+    if not venv:
+        key = f"{module_name}_{extractor_class}"
+        if key not in _loaded_extractors:
+            # dynamic import of extractor
+            mod = importlib.import_module(module_name)
+            extractor_cls = mod.__getattribute__(extractor_class)
+            extractor = extractor_cls()
+        else:
+            # retrieve cached extractor
+            extractor = _loaded_extractors[key]
+        if extractor.yara_compiled:
+            matches = extractor.yara_compiled.match(sample_path)
+        loaded = extractor.run(open(sample_path, "rb"), matches=matches)
+    else:
+        # execute extractor in child process with separate virtual environment
+        # Write temporary script in the same directory as extractor to resolve relative imports
         python_exe = os.path.join(venv, "bin", "python")
-    dirname = os.path.dirname(module_path)
-    with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
-        with tempfile.NamedTemporaryFile() as output:
-            parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
-            root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
-
-            script.write(
-                venv_script.format(
-                    parent_package_path=parent_package_path,
-                    module_name=module_name,
-                    module_class=extractor_class,
-                    sample_path=sample_path,
-                    output_path=output.name,
+        dirname = os.path.dirname(module_path)
+        with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
+            with tempfile.NamedTemporaryFile() as output:
+                parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
+                root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
+
+                script.write(
+                    venv_script.format(
+                        parent_package_path=parent_package_path,
+                        module_name=module_name,
+                        module_class=extractor_class,
+                        sample_path=sample_path,
+                        output_path=output.name,
+                    )
                 )
-            )
-            script.flush()
-            cwd = root_directory
-            custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
-
-            if custom_module.startswith("src."):
-                # src layout found, which means the actual module content is within 'src' directory
-                custom_module = custom_module[4:]
-                cwd = os.path.join(cwd, "src")
-
-            proc = subprocess.run(
-                [python_exe, "-m", custom_module],
-                cwd=cwd,
-                capture_output=True,
-            )
-            stderr = proc.stderr.decode()
-            try:
-                # Load results and return them
-                output.seek(0)
-                loaded =  json.load(output, cls=json_decoder)
-            except Exception as e:
-                # If there was an error raised during runtime, then propagate
-                delim = f'File "{module_path}"'
-                exception = stderr
-                if delim in exception:
-                    exception = f"{delim}{exception.split(delim, 1)[1]}"
-                # print extractor logging at error level
-                logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
-                raise Exception(exception) from e
-            # ensure that extractor logging is available
-            logger.info(f"maco extractor stderr:\n{stderr}")
-            return loaded
+                script.flush()
+                cwd = root_directory
+                custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
+
+                if custom_module.startswith("src."):
+                    # src layout found, which means the actual module content is within 'src' directory
+                    custom_module = custom_module[4:]
+                    cwd = os.path.join(cwd, "src")
+
+                # run the maco extractor in full venv process isolation (slow)
+                proc = subprocess.run(
+                    [python_exe, "-m", custom_module],
+                    cwd=cwd,
+                    capture_output=True,
+                )
+                stderr = proc.stderr.decode()
+                try:
+                    # Load results and return them
+                    output.seek(0)
+                    loaded = json.load(output, cls=json_decoder)
+                except Exception as e:
+                    # If there was an error raised during runtime, then propagate
+                    delim = f'File "{module_path}"'
+                    exception = stderr
+                    if delim in exception:
+                        exception = f"{delim}{exception.split(delim, 1)[1]}"
+                    # print extractor logging at error level
+                    logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
+                    raise Exception(exception) from e
+                # ensure that extractor logging is available
+                logger.info(f"maco extractor stderr:\n{stderr}")
+    return loaded

maco/yara.py

@@ -3,7 +3,6 @@ from collections import namedtuple
 from itertools import cycle
 from typing import Dict
 
-import yara
 import yara_x
 
 RULE_ID_RE = re.compile("(\w+)? ?rule (\w+)")

{maco-1.2.4.dist-info → maco-1.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.4
+Version: 1.2.6
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License
@@ -38,7 +38,7 @@ Requires-Dist: yara-x==0.11.0
 
 # Maco - Malware config extractor framework
 
-## Maco is a framework for ***ma***lware ***co***nfig extractors.
+## Maco is a framework for <ins>ma</ins>lware <ins>co</ins>nfig extractors.
 
 It aims to solve two problems:
 
@@ -272,3 +272,15 @@ run Complex extractor from rules ['ComplexAlt']
 The demo extractors are designed to trigger when run over the '`demo_extractors`' folder.
 
 e.g. `maco demo_extractors demo_extractors`
+
+# Contributions
+
+Please use ruff to format and lint PRs. This may be the cause of PR test failures.
+
+Ruff will attempt to fix most issues, but some may require manual resolution.
+
+```
+pip install ruff
+ruff format
+ruff check --fix
+```

maco-1.2.6.dist-info/RECORD

@@ -0,0 +1,44 @@
+demo_extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+demo_extractors/elfy.py,sha256=Jo_GKExCeFOKGENJnNM_9ONfJO7LQFucCNz0ryTAo9U,765
+demo_extractors/limit_other.py,sha256=BWjeyOxB75kw4eRla5zvSzdcXtELOS8R6hc71rLPh1s,1295
+demo_extractors/nothing.py,sha256=MNPlb0IsBjrlU5e438JlJ4DIKoBpBRAaYY3JhD3yHqk,601
+demo_extractors/requirements.txt,sha256=E0tD6xBZldq6sQGTHng6k88lBeASOhmLJcdcjpcqBNE,6
+demo_extractors/shared.py,sha256=2P1cyuRbHDvM9IRt3UZnwdyhxx7OWqNC83xLyV8Y190,305
+demo_extractors/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+demo_extractors/complex/complex.py,sha256=tXrzj_zWIXbTOwj7Lezapk-qkrM-lfwcyjd5m-BYzdg,2322
+demo_extractors/complex/complex_utils.py,sha256=aec8kJsYUrMPo-waihkVLt-0QpiOPkw7dDqfT9MNuHk,123
+maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+maco/base_test.py,sha256=cjGLEy2c69wl9sjn74QFz7X-VxWOfdin4W8MvYsXc4Q,2718
+maco/cli.py,sha256=NTzV8eu9V0qQNttRo592j-Rdzac7q1NAMraqJF2h_6k,8171
+maco/collector.py,sha256=LraWYlCA72FCmQP0dHWc-ekd7R1SxR6h6rMD95_6mMs,7077
+maco/extractor.py,sha256=uGSGiCQ4jd8jFmfw2T99BGcY5iQJzXHcG_RoTIxClTE,2802
+maco/utils.py,sha256=vo8zGSoFP0k2APUlQXmLssdrVrknjS2YcpvbRM78J68,20480
+maco/yara.py,sha256=8RVaGyeUWY5f8_wfQ25lDX1bcXsb_VoSja85ZC2SqGw,2913
+maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
+maco/model/model.py,sha256=4uY88WphbP3iu-L2WjuYwtgZCS_wNul_hr0bAVuTpvc,23740
+model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+model_setup/maco/base_test.py,sha256=cjGLEy2c69wl9sjn74QFz7X-VxWOfdin4W8MvYsXc4Q,2718
+model_setup/maco/cli.py,sha256=NTzV8eu9V0qQNttRo592j-Rdzac7q1NAMraqJF2h_6k,8171
+model_setup/maco/collector.py,sha256=LraWYlCA72FCmQP0dHWc-ekd7R1SxR6h6rMD95_6mMs,7077
+model_setup/maco/extractor.py,sha256=uGSGiCQ4jd8jFmfw2T99BGcY5iQJzXHcG_RoTIxClTE,2802
+model_setup/maco/utils.py,sha256=vo8zGSoFP0k2APUlQXmLssdrVrknjS2YcpvbRM78J68,20480
+model_setup/maco/yara.py,sha256=8RVaGyeUWY5f8_wfQ25lDX1bcXsb_VoSja85ZC2SqGw,2913
+model_setup/maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
+model_setup/maco/model/model.py,sha256=4uY88WphbP3iu-L2WjuYwtgZCS_wNul_hr0bAVuTpvc,23740
+pipelines/publish.yaml,sha256=xt3WNU-5kIICJgKIiiE94M3dWjS3uEiun-n4OmIssK8,1471
+pipelines/test.yaml,sha256=btJVI-R39UBeYosGu7TOpU6V9ogFW3FT3ROtWygQGQ0,1472
+tests/data/example.txt.cart,sha256=j4ZdDnFNVq7lb-Qi4pY4evOXKQPKG-GSg-n-uEqPhV0,289
+tests/data/trigger_complex.txt,sha256=uqnLSrnyDGCmXwuPmZ2s8vdhH0hJs8DxvyaW_tuYY24,64
+tests/data/trigger_complex.txt.cart,sha256=Z7qF1Zi640O45Znkl9ooP2RhSLAEqY0NRf51d-q7utU,345
+tests/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/extractors/basic.py,sha256=r5eLCL6Ynr14nCBgtbLvUbm0NdrXizyc9c-4xBCNShU,828
+tests/extractors/basic_longer.py,sha256=1ClU2QD-Y0TOl_loNFvEqIEpTR5TSVJ6zg9ZmC-ESJo,860
+tests/extractors/test_basic.py,sha256=FLKekfSGM69HaiF7Vu_7D7KDXHZko-9hZkMO8_DoyYA,697
+tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/extractors/bob/bob.py,sha256=G5aOoz58J0ZQK2_lA7HRxAzeLzBxssWxBTZcv1pSbi8,176
+maco-1.2.6.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
+maco-1.2.6.dist-info/METADATA,sha256=-ZxVnBbAHn-iGDizJJa8knrTf4DUArPzANp-SVqDzZ4,15855
+maco-1.2.6.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+maco-1.2.6.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
+maco-1.2.6.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
+maco-1.2.6.dist-info/RECORD,,

model_setup/maco/base_test.py

@@ -32,14 +32,19 @@ class BaseTest(unittest.TestCase):
     # I recommend something like os.path.join(__file__, "../../extractors")
     # if your extractors are in a folder 'extractors' next to a folder of tests
     path: str = None
+    create_venv: bool = False
 
-    def setUp(self) -> None:
-        if not self.name or not self.path:
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not cls.name or not cls.path:
             raise Exception("name and path must be set")
-        self.c = collector.Collector(self.path, include=[self.name])
+        cls.c = collector.Collector(cls.path, include=[cls.name], create_venv=cls.create_venv)
+        return super().setUpClass()
+
+    def test_default_metadata(self):
+        """Require extractor to be loadable and valid."""
         self.assertIn(self.name, self.c.extractors)
         self.assertEqual(len(self.c.extractors), 1)
-        return super().setUp()
 
     def extract(self, stream):
         """Return results for running extractor over stream, including yara check."""
@@ -49,18 +54,20 @@ class BaseTest(unittest.TestCase):
         resp = self.c.extract(stream, self.name)
         return resp
 
-    def _get_location(self) -> str:
+    @classmethod
+    def _get_location(cls) -> str:
         """Return path to child class that implements this class."""
         # import child module
-        module = type(self).__module__
+        module = cls.__module__
         i = importlib.import_module(module)
         # get location to child module
         return i.__file__
 
-    def load_cart(self, filepath: str) -> io.BytesIO:
+    @classmethod
+    def load_cart(cls, filepath: str) -> io.BytesIO:
         """Load and unneuter a test file (likely malware) into memory for processing."""
         # it is nice if we can load files relative to whatever is implementing base_test
-        dirpath = os.path.split(self._get_location())[0]
+        dirpath = os.path.split(cls._get_location())[0]
         # either filepath is absolute, or should be loaded relative to child of base_test
         filepath = os.path.join(dirpath, filepath)
         if not os.path.isfile(filepath):

model_setup/maco/cli.py

@@ -1,4 +1,5 @@
 """CLI example of how extractors can be executed."""
+
 import argparse
 import base64
 import binascii
@@ -150,6 +151,7 @@ def process_filesystem(
         logger.info(f"{num_analysed} analysed, {num_hits} hits, {num_extracted} extracted")
     return num_analysed, num_hits, num_extracted
 
+
 def main():
     parser = argparse.ArgumentParser(description="Run extractors over samples.")
     parser.add_argument("extractors", type=str, help="path to extractors")
@@ -165,7 +167,8 @@ def main():
     parser.add_argument(
         "--base64",
         action="store_true",
-        help="Include base64 encoded binary data in output (can be large, consider printing to file rather than console)",
+        help="Include base64 encoded binary data in output "
+        "(can be large, consider printing to file rather than console)",
     )
     parser.add_argument("--logfile", type=str, help="file to log output")
     parser.add_argument("--include", type=str, help="comma separated extractors to run")
@@ -179,7 +182,9 @@ def main():
     parser.add_argument(
         "--create_venv",
         action="store_true",
-        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory)",
+        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). "
+        "This runs much slower than the alternative but may be necessary "
+        "when there are many extractors with conflicting dependencies.",
     )
     args = parser.parse_args()
     inc = args.include.split(",") if args.include else []
@@ -225,7 +230,7 @@ def main():
         pretty=args.pretty,
         force=args.force,
         include_base64=args.base64,
-        create_venv=args.create_venv
+        create_venv=args.create_venv,
     )
 
 

model_setup/maco/collector.py

@@ -4,6 +4,7 @@ import inspect
 import logging
 import logging.handlers
 import os
+import sys
 from multiprocessing import Manager, Process, Queue
 from tempfile import NamedTemporaryFile
 from types import ModuleType
@@ -48,6 +49,15 @@ class Collector:
         create_venv: bool = False,
     ):
         """Discover and load extractors from file system."""
+        # maco requires the extractor to be imported directly, so ensure they are available on the path
+        full_path_extractors = os.path.abspath(path_extractors)
+        full_path_above_extractors = os.path.dirname(full_path_extractors)
+        # Modify the PATH so we can recognize this new package on import
+        if full_path_extractors not in sys.path:
+            sys.path.insert(1, full_path_extractors)
+        if full_path_above_extractors not in sys.path:
+            sys.path.insert(1, full_path_above_extractors)
+
         path_extractors = os.path.realpath(path_extractors)
         self.path: str = path_extractors
         self.extractors: Dict[str, Dict[str, str]] = {}
@@ -89,7 +99,7 @@ class Collector:
 
             # multiprocess logging is awkward - set up a queue to ensure we can log
             logging_queue = Queue()
-            queue_handler = logging.handlers.QueueListener(logging_queue,*logging.getLogger().handlers)
+            queue_handler = logging.handlers.QueueListener(logging_queue, *logging.getLogger().handlers)
             queue_handler.start()
 
             # Find the extractors within the given directory

model_setup/maco/extractor.py

@@ -51,14 +51,14 @@ class Extractor:
         # check yara rules conform to expected structure
         # we throw away these compiled rules as we need all rules in system compiled together
         try:
-            rules = yara.compile(source=self.yara_rule)
+            self.yara_compiled = yara.compile(source=self.yara_rule)
         except yara.SyntaxError as e:
             raise InvalidExtractor(f"{self.name} - invalid yara rule") from e
         # need to track which plugin owns the rules
-        self.yara_rule_names = [x.identifier for x in rules]
-        if not len(list(rules)):
+        self.yara_rule_names = [x.identifier for x in self.yara_compiled]
+        if not len(list(self.yara_compiled)):
             raise InvalidExtractor(f"{name} must define at least one yara rule")
-        for x in rules:
+        for x in self.yara_compiled:
             if x.is_global:
                 raise InvalidExtractor(f"{x.identifier} yara rule must not be global")
 

model_setup/maco/model/__init__.py

@@ -1 +1 @@
-from maco.model.model import *
+from maco.model.model import *  # noqa: F403