maco 1.2.4__py3-none-any.whl → 1.2.5__py3-none-any.whl

demo_extractors/complex/complex.py

@@ -1,9 +1,8 @@
 from io import BytesIO
 from typing import List, Optional
 
-from maco import extractor, model, yara
-
 from demo_extractors.complex import complex_utils
+from maco import extractor, model, yara
 
 
 class Complex(extractor.Extractor):

demo_extractors/limit_other.py

@@ -1,9 +1,8 @@
 from io import BytesIO
 from typing import Dict, List, Optional
 
-from maco import extractor, model, yara
-
 from demo_extractors import shared
+from maco import extractor, model, yara
 
 
 class LimitOther(extractor.Extractor):
@@ -24,6 +23,10 @@ class LimitOther(extractor.Extractor):
         """
 
     def run(self, stream: BytesIO, matches: List[yara.Match]) -> Optional[model.ExtractorModel]:
+        # import httpx at runtime so we can test that requirements.txt is installed dynamically without breaking
+        # the tests that do direct importing
+        import httpx
+
         # use a custom model that inherits from ExtractorModel
         # this model defines what can go in the 'other' dict
         tmp = shared.MyCustomModel(family="specify_other")

demo_extractors/requirements.txt

@@ -0,0 +1 @@
+httpx

demo_extractors/shared.py

@@ -1,4 +1,5 @@
 from typing import Optional
+
 import pydantic
 
 from maco import model

maco/base_test.py

@@ -32,14 +32,19 @@ class BaseTest(unittest.TestCase):
     # I recommend something like os.path.join(__file__, "../../extractors")
     # if your extractors are in a folder 'extractors' next to a folder of tests
     path: str = None
+    create_venv: bool=False
 
-    def setUp(self) -> None:
-        if not self.name or not self.path:
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not cls.name or not cls.path:
             raise Exception("name and path must be set")
-        self.c = collector.Collector(self.path, include=[self.name])
+        cls.c = collector.Collector(cls.path, include=[cls.name], create_venv=cls.create_venv)
+        return super().setUpClass()
+
+    def test_default_metadata(self):
+        """Require extractor to be loadable and valid."""
         self.assertIn(self.name, self.c.extractors)
         self.assertEqual(len(self.c.extractors), 1)
-        return super().setUp()
 
     def extract(self, stream):
         """Return results for running extractor over stream, including yara check."""
@@ -49,18 +54,20 @@ class BaseTest(unittest.TestCase):
         resp = self.c.extract(stream, self.name)
         return resp
 
-    def _get_location(self) -> str:
+    @classmethod
+    def _get_location(cls) -> str:
         """Return path to child class that implements this class."""
         # import child module
-        module = type(self).__module__
+        module = cls.__module__
         i = importlib.import_module(module)
         # get location to child module
         return i.__file__
 
-    def load_cart(self, filepath: str) -> io.BytesIO:
+    @classmethod
+    def load_cart(cls, filepath: str) -> io.BytesIO:
         """Load and unneuter a test file (likely malware) into memory for processing."""
         # it is nice if we can load files relative to whatever is implementing base_test
-        dirpath = os.path.split(self._get_location())[0]
+        dirpath = os.path.split(cls._get_location())[0]
         # either filepath is absolute, or should be loaded relative to child of base_test
         filepath = os.path.join(dirpath, filepath)
         if not os.path.isfile(filepath):

maco/cli.py

@@ -179,7 +179,7 @@ def main():
     parser.add_argument(
         "--create_venv",
         action="store_true",
-        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory)",
+        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). This runs much slower than the alternative but may be necessary when there are many extractors with conflicting dependencies.",
     )
     args = parser.parse_args()
     inc = args.include.split(",") if args.include else []

maco/extractor.py

@@ -51,14 +51,14 @@ class Extractor:
         # check yara rules conform to expected structure
         # we throw away these compiled rules as we need all rules in system compiled together
         try:
-            rules = yara.compile(source=self.yara_rule)
+            self.yara_compiled = yara.compile(source=self.yara_rule)
         except yara.SyntaxError as e:
             raise InvalidExtractor(f"{self.name} - invalid yara rule") from e
         # need to track which plugin owns the rules
-        self.yara_rule_names = [x.identifier for x in rules]
-        if not len(list(rules)):
+        self.yara_rule_names = [x.identifier for x in self.yara_compiled]
+        if not len(list(self.yara_compiled)):
             raise InvalidExtractor(f"{name} must define at least one yara rule")
-        for x in rules:
+        for x in self.yara_compiled:
             if x.is_global:
                 raise InvalidExtractor(f"{x.identifier} yara rule must not be global")
 

maco/utils.py

@@ -4,14 +4,14 @@ import importlib.machinery
 import importlib.util
 import inspect
 import json
+import logging
 import logging.handlers
+import multiprocessing
 import os
 import re
 import shutil
 import subprocess
 import sys
-import multiprocessing
-import logging
 import tempfile
 
 from maco import yara
@@ -27,8 +27,11 @@ from glob import glob
 from logging import Logger
 from pkgutil import walk_packages
 from types import ModuleType
-from typing import Callable, Dict, List, Set, Tuple
+from typing import Callable, Dict, List, Set, Tuple, Union
+
+from uv import find_uv_bin
 
+from maco import model
 from maco.extractor import Extractor
 
 logger = logging.getLogger("maco.lib.utils")
@@ -38,23 +41,10 @@ VENV_DIRECTORY_NAME = ".venv"
 RELATIVE_FROM_RE = re.compile(r"from (\.+)")
 RELATIVE_FROM_IMPORT_RE = re.compile(r"from (\.+) import")
 
-try:
-    # Attempt to use the uv package manager (Recommended)
-    from uv import find_uv_bin
-
-    UV_BIN = find_uv_bin()
-
-    PIP_CMD = f"{UV_BIN} pip"
-    VENV_CREATE_CMD = f"{UV_BIN} venv"
-    PACKAGE_MANAGER = "uv"
-except ImportError:
-    # Otherwise default to pip
-    from sys import executable
-
-    PIP_CMD = "pip"
-    VENV_CREATE_CMD = f"{executable} -m venv"
-    PACKAGE_MANAGER = "pip"
+UV_BIN = find_uv_bin()
 
+PIP_CMD = f"{UV_BIN} pip"
+VENV_CREATE_CMD = f"{UV_BIN} venv"
 
 class Base64Decoder(json.JSONDecoder):
     def __init__(self, *args, **kwargs):
@@ -210,9 +200,8 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
     return extractor_dirs, extractor_files
 
 
-def create_virtual_environments(directories: List[str], python_version: str, logger: Logger):
+def _install_required_packages(create_venv: bool, directories: List[str], python_version: str, logger: Logger):
     venvs = []
-    logger.info("Creating virtual environment(s)..")
     env = deepcopy(os.environ)
     stop_directory = os.path.dirname(sorted(directories)[0])
     # Track directories that we've already visited
@@ -222,14 +211,15 @@ def create_virtual_environments(directories: List[str], python_version: str, log
         while dir != stop_directory and dir not in visited_dirs:
             req_files = list({"requirements.txt", "pyproject.toml"}.intersection(set(os.listdir(dir))))
             if req_files:
-                venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
-                env.update({"VIRTUAL_ENV": venv_path})
-                # Create a virtual environment for the directory
-                if not os.path.exists(venv_path):
-                    cmd = VENV_CREATE_CMD
-                    if PACKAGE_MANAGER == "uv":
-                        cmd += f" --python {python_version}"
-                    subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
+                # create a virtual environment, otherwise directly install into current env
+                if create_venv:
+                    venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
+                    logger.info(f"Updating virtual environment {venv_path}")
+                    env.update({"VIRTUAL_ENV": venv_path})
+                    # Create a virtual environment for the directory
+                    if not os.path.exists(venv_path):
+                        cmd = f"{VENV_CREATE_CMD} --python {python_version}"
+                        subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
 
                 # Install/Update the packages in the environment
                 install_command = PIP_CMD.split(" ") + ["install", "-U"]
@@ -253,7 +243,10 @@ def create_virtual_environments(directories: List[str], python_version: str, log
 
                     install_command.extend(pyproject_command)
 
+                # always require maco to be installed
+                install_command.append("maco")
                 logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
+                # this uses VIRTUAL_ENV to control usage of a virtual environment
                 p = subprocess.run(
                     install_command,
                     cwd=dir,
@@ -264,10 +257,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
                     if b"is being installed using the legacy" in p.stderr:
                         # Ignore these types of errors
                         continue
-                    logger.error(f"Error installing into venv:\n{p.stderr.decode()}")
+                    logger.error(f"Error installing into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
                 else:
-                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
-                    venvs.append(venv_path)
+                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
+                    if create_venv:
+                        venvs.append(venv_path)
 
                 # Cleanup any build directories that are the product of package installation
                 expected_build_path = os.path.join(dir, "build")
@@ -311,7 +305,7 @@ def register_extractors(
 ):
     package_name = os.path.basename(current_directory)
     parent_directory = os.path.dirname(current_directory)
-    if package_name in sys.modules:
+    if venvs and package_name in sys.modules:
         # this may happen as part of testing if some part of the extractor code was directly imported
         logger.warning(f"Looks like {package_name} is already loaded. "
                        "If your maco extractor overlaps an existing package name this could cause problems.")
@@ -402,32 +396,26 @@ def import_extractors(
     *,
     root_directory: str,
     scanner: yara.Rules,
-    create_venv: bool = False,
-    python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
+    create_venv: bool,
     logger: Logger,
+    python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
 ):
     extractor_dirs, extractor_files = scan_for_extractors(root_directory, scanner, logger)
 
     logger.info(f"Extractor files found based on scanner ({len(extractor_files)}).")
     logger.debug(extractor_files)
 
-    venvs = []
-    if create_venv:
-        venvs = create_virtual_environments(extractor_dirs, python_version, logger)
-    else:
-        # Look for pre-existing virtual environments, if any
-        logger.info("Checking for pre-existing virtual environment(s)..")
-        venvs = [
-            os.path.join(root, VENV_DIRECTORY_NAME)
-            for root, dirs, _ in os.walk(root_directory)
-            if VENV_DIRECTORY_NAME in dirs
-        ]
+    # Install packages into the current environment or dynamically created virtual environments
+    venvs = _install_required_packages(create_venv, extractor_dirs, python_version, logger)
 
     # With the environment prepared, we can now hunt for the extractors and register them
     logger.info("Registering extractors..")
     register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
 
 
+# holds cached extractors when not running in venv mode
+_loaded_extractors: Dict[str, Extractor] = {}
+
 def run_extractor(
     sample_path,
     module_name,
@@ -436,55 +424,69 @@ def run_extractor(
     venv,
     venv_script=VENV_SCRIPT,
     json_decoder=Base64Decoder,
-) -> Dict[str, dict]:
-    # Write temporary script in the same directory as extractor to resolve relative imports
-    python_exe = sys.executable
-    if venv:
-        # If there is a linked virtual environment, execute within that environment
+) -> Union[Dict[str, dict], model.ExtractorModel]:
+    """Runs the maco extractor against sample either in current process or child process."""
+    if not venv:
+        key = f"{module_name}_{extractor_class}"
+        if key not in _loaded_extractors:
+            # dynamic import of extractor
+            mod = importlib.import_module(module_name)
+            extractor_cls = mod.__getattribute__(extractor_class)
+            extractor = extractor_cls()
+        else:
+            # retrieve cached extractor
+            extractor = _loaded_extractors[key]
+        if extractor.yara_compiled:
+            matches = extractor.yara_compiled.match(sample_path)
+        loaded = extractor.run(open(sample_path, 'rb'), matches=matches)
+    else:
+        # execute extractor in child process with separate virtual environment
+        # Write temporary script in the same directory as extractor to resolve relative imports
         python_exe = os.path.join(venv, "bin", "python")
-    dirname = os.path.dirname(module_path)
-    with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
-        with tempfile.NamedTemporaryFile() as output:
-            parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
-            root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
-
-            script.write(
-                venv_script.format(
-                    parent_package_path=parent_package_path,
-                    module_name=module_name,
-                    module_class=extractor_class,
-                    sample_path=sample_path,
-                    output_path=output.name,
+        dirname = os.path.dirname(module_path)
+        with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
+            with tempfile.NamedTemporaryFile() as output:
+                parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
+                root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
+
+                script.write(
+                    venv_script.format(
+                        parent_package_path=parent_package_path,
+                        module_name=module_name,
+                        module_class=extractor_class,
+                        sample_path=sample_path,
+                        output_path=output.name,
+                    )
                 )
-            )
-            script.flush()
-            cwd = root_directory
-            custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
-
-            if custom_module.startswith("src."):
-                # src layout found, which means the actual module content is within 'src' directory
-                custom_module = custom_module[4:]
-                cwd = os.path.join(cwd, "src")
-
-            proc = subprocess.run(
-                [python_exe, "-m", custom_module],
-                cwd=cwd,
-                capture_output=True,
-            )
-            stderr = proc.stderr.decode()
-            try:
-                # Load results and return them
-                output.seek(0)
-                loaded =  json.load(output, cls=json_decoder)
-            except Exception as e:
-                # If there was an error raised during runtime, then propagate
-                delim = f'File "{module_path}"'
-                exception = stderr
-                if delim in exception:
-                    exception = f"{delim}{exception.split(delim, 1)[1]}"
-                # print extractor logging at error level
-                logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
-                raise Exception(exception) from e
-            # ensure that extractor logging is available
-            logger.info(f"maco extractor stderr:\n{stderr}")
-            return loaded
+                script.flush()
+                cwd = root_directory
+                custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
+
+                if custom_module.startswith("src."):
+                    # src layout found, which means the actual module content is within 'src' directory
+                    custom_module = custom_module[4:]
+                    cwd = os.path.join(cwd, "src")
+
+                # run the maco extractor in full venv process isolation (slow)
+                proc = subprocess.run(
+                    [python_exe, "-m", custom_module],
+                    cwd=cwd,
+                    capture_output=True,
+                )
+                stderr = proc.stderr.decode()
+                try:
+                    # Load results and return them
+                    output.seek(0)
+                    loaded =  json.load(output, cls=json_decoder)
+                except Exception as e:
+                    # If there was an error raised during runtime, then propagate
+                    delim = f'File "{module_path}"'
+                    exception = stderr
+                    if delim in exception:
+                        exception = f"{delim}{exception.split(delim, 1)[1]}"
+                    # print extractor logging at error level
+                    logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
+                    raise Exception(exception) from e
+                # ensure that extractor logging is available
+                logger.info(f"maco extractor stderr:\n{stderr}")
+    return loaded

{maco-1.2.4.dist-info → maco-1.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.4
+Version: 1.2.5
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License

{maco-1.2.4.dist-info → maco-1.2.5.dist-info}/RECORD

@@ -1,26 +1,27 @@
 demo_extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 demo_extractors/elfy.py,sha256=AAFr5i1aivPwO4nycyXJEud57EpVNA-5k_2GicWesbY,771
-demo_extractors/limit_other.py,sha256=RAFx_0K_WnhUURA5uwXGmjrWODrAuLZFBxqZcWaxf64,944
+demo_extractors/limit_other.py,sha256=8Z7X0cXUyZuK3MhDtObMWmdruRj5hgFdDi_VVGXqRx4,1123
 demo_extractors/nothing.py,sha256=3aeQJTY-dakmVXmyfmrRM8YCQVT7q3bq880DFH1Ol_Y,607
-demo_extractors/shared.py,sha256=Wlvy77SCAR97gxi8uUhGYyjxGmDb-pOSvN8b1rXrVWs,304
+demo_extractors/requirements.txt,sha256=E0tD6xBZldq6sQGTHng6k88lBeASOhmLJcdcjpcqBNE,6
+demo_extractors/shared.py,sha256=2P1cyuRbHDvM9IRt3UZnwdyhxx7OWqNC83xLyV8Y190,305
 demo_extractors/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-demo_extractors/complex/complex.py,sha256=JFKqBGKwkuDSz4zZUJuqhCLUQv6dlCMhBqNj33grBsE,2323
+demo_extractors/complex/complex.py,sha256=tXrzj_zWIXbTOwj7Lezapk-qkrM-lfwcyjd5m-BYzdg,2322
 demo_extractors/complex/complex_utils.py,sha256=aec8kJsYUrMPo-waihkVLt-0QpiOPkw7dDqfT9MNuHk,123
 maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-maco/base_test.py,sha256=pqet9ofMwFRTj3JHgPdh9WHwdyp8kxNMi1vJNUzkSNA,2518
-maco/cli.py,sha256=1I3U54yPddTxqWclCtZ5Ma5hW6RoVTZMzLSFOjjfM1g,8008
+maco/base_test.py,sha256=EPxCun9Tv91V-lFpaenn14tPyW17TPvXVH4AjE3t6js,2716
+maco/cli.py,sha256=fIeUXOgOxcecmAkl6OAdnjBKqk1gBPv1ryWe50pT60g,8135
 maco/collector.py,sha256=Vlo7KcJC7TKZFTElv8i_f_hvWEnlWCRzOP1xOc9x7vk,6532
-maco/extractor.py,sha256=4ZQd8OfvEQYUIkUS3LzZ5tcioembuLhT9_uRVNKSsyM,2750
-maco/utils.py,sha256=vQeJKw4whWTXp1mTd2oEhfvL4nvvgAzWjBnCp2XxWLI,19275
+maco/extractor.py,sha256=uGSGiCQ4jd8jFmfw2T99BGcY5iQJzXHcG_RoTIxClTE,2802
+maco/utils.py,sha256=K41c-H7naaoiEYf0WNfP054IxwvHPujsbmmzgTizuLU,20159
 maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
 maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
 maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
 model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-model_setup/maco/base_test.py,sha256=pqet9ofMwFRTj3JHgPdh9WHwdyp8kxNMi1vJNUzkSNA,2518
-model_setup/maco/cli.py,sha256=1I3U54yPddTxqWclCtZ5Ma5hW6RoVTZMzLSFOjjfM1g,8008
+model_setup/maco/base_test.py,sha256=EPxCun9Tv91V-lFpaenn14tPyW17TPvXVH4AjE3t6js,2716
+model_setup/maco/cli.py,sha256=fIeUXOgOxcecmAkl6OAdnjBKqk1gBPv1ryWe50pT60g,8135
 model_setup/maco/collector.py,sha256=Vlo7KcJC7TKZFTElv8i_f_hvWEnlWCRzOP1xOc9x7vk,6532
-model_setup/maco/extractor.py,sha256=4ZQd8OfvEQYUIkUS3LzZ5tcioembuLhT9_uRVNKSsyM,2750
-model_setup/maco/utils.py,sha256=vQeJKw4whWTXp1mTd2oEhfvL4nvvgAzWjBnCp2XxWLI,19275
+model_setup/maco/extractor.py,sha256=uGSGiCQ4jd8jFmfw2T99BGcY5iQJzXHcG_RoTIxClTE,2802
+model_setup/maco/utils.py,sha256=K41c-H7naaoiEYf0WNfP054IxwvHPujsbmmzgTizuLU,20159
 model_setup/maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
 model_setup/maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
 model_setup/maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
@@ -35,9 +36,9 @@ tests/extractors/basic_longer.py,sha256=1ClU2QD-Y0TOl_loNFvEqIEpTR5TSVJ6zg9ZmC-E
 tests/extractors/test_basic.py,sha256=FLKekfSGM69HaiF7Vu_7D7KDXHZko-9hZkMO8_DoyYA,697
 tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/extractors/bob/bob.py,sha256=Gy5p8KssJX87cwa9vVv8UBODF_ulbUteZXh15frW2hs,247
-maco-1.2.4.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
-maco-1.2.4.dist-info/METADATA,sha256=EoAEnCfbaXe8eUFAMHj3-C9lQIDR9Ss31NcY4CbEXjI,15610
-maco-1.2.4.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-maco-1.2.4.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
-maco-1.2.4.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
-maco-1.2.4.dist-info/RECORD,,
+maco-1.2.5.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
+maco-1.2.5.dist-info/METADATA,sha256=cJ7x_shBhDgKVjkq_e2d94aj3qiUzi0lt7f3lPO334U,15610
+maco-1.2.5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+maco-1.2.5.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
+maco-1.2.5.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
+maco-1.2.5.dist-info/RECORD,,

model_setup/maco/base_test.py

@@ -32,14 +32,19 @@ class BaseTest(unittest.TestCase):
     # I recommend something like os.path.join(__file__, "../../extractors")
     # if your extractors are in a folder 'extractors' next to a folder of tests
     path: str = None
+    create_venv: bool=False
 
-    def setUp(self) -> None:
-        if not self.name or not self.path:
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not cls.name or not cls.path:
             raise Exception("name and path must be set")
-        self.c = collector.Collector(self.path, include=[self.name])
+        cls.c = collector.Collector(cls.path, include=[cls.name], create_venv=cls.create_venv)
+        return super().setUpClass()
+
+    def test_default_metadata(self):
+        """Require extractor to be loadable and valid."""
         self.assertIn(self.name, self.c.extractors)
         self.assertEqual(len(self.c.extractors), 1)
-        return super().setUp()
 
     def extract(self, stream):
         """Return results for running extractor over stream, including yara check."""
@@ -49,18 +54,20 @@ class BaseTest(unittest.TestCase):
         resp = self.c.extract(stream, self.name)
         return resp
 
-    def _get_location(self) -> str:
+    @classmethod
+    def _get_location(cls) -> str:
         """Return path to child class that implements this class."""
         # import child module
-        module = type(self).__module__
+        module = cls.__module__
         i = importlib.import_module(module)
         # get location to child module
         return i.__file__
 
-    def load_cart(self, filepath: str) -> io.BytesIO:
+    @classmethod
+    def load_cart(cls, filepath: str) -> io.BytesIO:
         """Load and unneuter a test file (likely malware) into memory for processing."""
         # it is nice if we can load files relative to whatever is implementing base_test
-        dirpath = os.path.split(self._get_location())[0]
+        dirpath = os.path.split(cls._get_location())[0]
         # either filepath is absolute, or should be loaded relative to child of base_test
         filepath = os.path.join(dirpath, filepath)
         if not os.path.isfile(filepath):

model_setup/maco/cli.py

@@ -179,7 +179,7 @@ def main():
     parser.add_argument(
         "--create_venv",
         action="store_true",
-        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory)",
+        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). This runs much slower than the alternative but may be necessary when there are many extractors with conflicting dependencies.",
     )
     args = parser.parse_args()
     inc = args.include.split(",") if args.include else []

model_setup/maco/extractor.py

@@ -51,14 +51,14 @@ class Extractor:
         # check yara rules conform to expected structure
         # we throw away these compiled rules as we need all rules in system compiled together
         try:
-            rules = yara.compile(source=self.yara_rule)
+            self.yara_compiled = yara.compile(source=self.yara_rule)
         except yara.SyntaxError as e:
             raise InvalidExtractor(f"{self.name} - invalid yara rule") from e
         # need to track which plugin owns the rules
-        self.yara_rule_names = [x.identifier for x in rules]
-        if not len(list(rules)):
+        self.yara_rule_names = [x.identifier for x in self.yara_compiled]
+        if not len(list(self.yara_compiled)):
             raise InvalidExtractor(f"{name} must define at least one yara rule")
-        for x in rules:
+        for x in self.yara_compiled:
             if x.is_global:
                 raise InvalidExtractor(f"{x.identifier} yara rule must not be global")
 

model_setup/maco/utils.py

@@ -4,14 +4,14 @@ import importlib.machinery
 import importlib.util
 import inspect
 import json
+import logging
 import logging.handlers
+import multiprocessing
 import os
 import re
 import shutil
 import subprocess
 import sys
-import multiprocessing
-import logging
 import tempfile
 
 from maco import yara
@@ -27,8 +27,11 @@ from glob import glob
 from logging import Logger
 from pkgutil import walk_packages
 from types import ModuleType
-from typing import Callable, Dict, List, Set, Tuple
+from typing import Callable, Dict, List, Set, Tuple, Union
+
+from uv import find_uv_bin
 
+from maco import model
 from maco.extractor import Extractor
 
 logger = logging.getLogger("maco.lib.utils")
@@ -38,23 +41,10 @@ VENV_DIRECTORY_NAME = ".venv"
 RELATIVE_FROM_RE = re.compile(r"from (\.+)")
 RELATIVE_FROM_IMPORT_RE = re.compile(r"from (\.+) import")
 
-try:
-    # Attempt to use the uv package manager (Recommended)
-    from uv import find_uv_bin
-
-    UV_BIN = find_uv_bin()
-
-    PIP_CMD = f"{UV_BIN} pip"
-    VENV_CREATE_CMD = f"{UV_BIN} venv"
-    PACKAGE_MANAGER = "uv"
-except ImportError:
-    # Otherwise default to pip
-    from sys import executable
-
-    PIP_CMD = "pip"
-    VENV_CREATE_CMD = f"{executable} -m venv"
-    PACKAGE_MANAGER = "pip"
+UV_BIN = find_uv_bin()
 
+PIP_CMD = f"{UV_BIN} pip"
+VENV_CREATE_CMD = f"{UV_BIN} venv"
 
 class Base64Decoder(json.JSONDecoder):
     def __init__(self, *args, **kwargs):
@@ -210,9 +200,8 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
     return extractor_dirs, extractor_files
 
 
-def create_virtual_environments(directories: List[str], python_version: str, logger: Logger):
+def _install_required_packages(create_venv: bool, directories: List[str], python_version: str, logger: Logger):
     venvs = []
-    logger.info("Creating virtual environment(s)..")
     env = deepcopy(os.environ)
     stop_directory = os.path.dirname(sorted(directories)[0])
     # Track directories that we've already visited
@@ -222,14 +211,15 @@ def create_virtual_environments(directories: List[str], python_version: str, log
         while dir != stop_directory and dir not in visited_dirs:
             req_files = list({"requirements.txt", "pyproject.toml"}.intersection(set(os.listdir(dir))))
             if req_files:
-                venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
-                env.update({"VIRTUAL_ENV": venv_path})
-                # Create a virtual environment for the directory
-                if not os.path.exists(venv_path):
-                    cmd = VENV_CREATE_CMD
-                    if PACKAGE_MANAGER == "uv":
-                        cmd += f" --python {python_version}"
-                    subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
+                # create a virtual environment, otherwise directly install into current env
+                if create_venv:
+                    venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
+                    logger.info(f"Updating virtual environment {venv_path}")
+                    env.update({"VIRTUAL_ENV": venv_path})
+                    # Create a virtual environment for the directory
+                    if not os.path.exists(venv_path):
+                        cmd = f"{VENV_CREATE_CMD} --python {python_version}"
+                        subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
 
                 # Install/Update the packages in the environment
                 install_command = PIP_CMD.split(" ") + ["install", "-U"]
@@ -253,7 +243,10 @@ def create_virtual_environments(directories: List[str], python_version: str, log
 
                     install_command.extend(pyproject_command)
 
+                # always require maco to be installed
+                install_command.append("maco")
                 logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
+                # this uses VIRTUAL_ENV to control usage of a virtual environment
                 p = subprocess.run(
                     install_command,
                     cwd=dir,
@@ -264,10 +257,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
                     if b"is being installed using the legacy" in p.stderr:
                         # Ignore these types of errors
                         continue
-                    logger.error(f"Error installing into venv:\n{p.stderr.decode()}")
+                    logger.error(f"Error installing into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
                 else:
-                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
-                    venvs.append(venv_path)
+                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
+                    if create_venv:
+                        venvs.append(venv_path)
 
                 # Cleanup any build directories that are the product of package installation
                 expected_build_path = os.path.join(dir, "build")
@@ -311,7 +305,7 @@ def register_extractors(
 ):
     package_name = os.path.basename(current_directory)
     parent_directory = os.path.dirname(current_directory)
-    if package_name in sys.modules:
+    if venvs and package_name in sys.modules:
         # this may happen as part of testing if some part of the extractor code was directly imported
         logger.warning(f"Looks like {package_name} is already loaded. "
                        "If your maco extractor overlaps an existing package name this could cause problems.")
@@ -402,32 +396,26 @@ def import_extractors(
     *,
     root_directory: str,
     scanner: yara.Rules,
-    create_venv: bool = False,
-    python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
+    create_venv: bool,
     logger: Logger,
+    python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
 ):
     extractor_dirs, extractor_files = scan_for_extractors(root_directory, scanner, logger)
 
     logger.info(f"Extractor files found based on scanner ({len(extractor_files)}).")
     logger.debug(extractor_files)
 
-    venvs = []
-    if create_venv:
-        venvs = create_virtual_environments(extractor_dirs, python_version, logger)
-    else:
-        # Look for pre-existing virtual environments, if any
-        logger.info("Checking for pre-existing virtual environment(s)..")
-        venvs = [
-            os.path.join(root, VENV_DIRECTORY_NAME)
-            for root, dirs, _ in os.walk(root_directory)
-            if VENV_DIRECTORY_NAME in dirs
-        ]
+    # Install packages into the current environment or dynamically created virtual environments
+    venvs = _install_required_packages(create_venv, extractor_dirs, python_version, logger)
 
     # With the environment prepared, we can now hunt for the extractors and register them
     logger.info("Registering extractors..")
     register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
 
 
+# holds cached extractors when not running in venv mode
+_loaded_extractors: Dict[str, Extractor] = {}
+
 def run_extractor(
     sample_path,
     module_name,
@@ -436,55 +424,69 @@ def run_extractor(
     venv,
     venv_script=VENV_SCRIPT,
     json_decoder=Base64Decoder,
-) -> Dict[str, dict]:
-    # Write temporary script in the same directory as extractor to resolve relative imports
-    python_exe = sys.executable
-    if venv:
-        # If there is a linked virtual environment, execute within that environment
+) -> Union[Dict[str, dict], model.ExtractorModel]:
+    """Runs the maco extractor against sample either in current process or child process."""
+    if not venv:
+        key = f"{module_name}_{extractor_class}"
+        if key not in _loaded_extractors:
+            # dynamic import of extractor
+            mod = importlib.import_module(module_name)
+            extractor_cls = mod.__getattribute__(extractor_class)
+            extractor = extractor_cls()
+        else:
+            # retrieve cached extractor
+            extractor = _loaded_extractors[key]
+        if extractor.yara_compiled:
+            matches = extractor.yara_compiled.match(sample_path)
+        loaded = extractor.run(open(sample_path, 'rb'), matches=matches)
+    else:
+        # execute extractor in child process with separate virtual environment
+        # Write temporary script in the same directory as extractor to resolve relative imports
         python_exe = os.path.join(venv, "bin", "python")
-    dirname = os.path.dirname(module_path)
-    with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
-        with tempfile.NamedTemporaryFile() as output:
-            parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
-            root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
-
-            script.write(
-                venv_script.format(
-                    parent_package_path=parent_package_path,
-                    module_name=module_name,
-                    module_class=extractor_class,
-                    sample_path=sample_path,
-                    output_path=output.name,
+        dirname = os.path.dirname(module_path)
+        with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
+            with tempfile.NamedTemporaryFile() as output:
+                parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
+                root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
+
+                script.write(
+                    venv_script.format(
+                        parent_package_path=parent_package_path,
+                        module_name=module_name,
+                        module_class=extractor_class,
+                        sample_path=sample_path,
+                        output_path=output.name,
+                    )
                 )
-            )
-            script.flush()
-            cwd = root_directory
-            custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
-
-            if custom_module.startswith("src."):
-                # src layout found, which means the actual module content is within 'src' directory
-                custom_module = custom_module[4:]
-                cwd = os.path.join(cwd, "src")
-
-            proc = subprocess.run(
-                [python_exe, "-m", custom_module],
-                cwd=cwd,
-                capture_output=True,
-            )
-            stderr = proc.stderr.decode()
-            try:
-                # Load results and return them
-                output.seek(0)
-                loaded =  json.load(output, cls=json_decoder)
-            except Exception as e:
-                # If there was an error raised during runtime, then propagate
-                delim = f'File "{module_path}"'
-                exception = stderr
-                if delim in exception:
-                    exception = f"{delim}{exception.split(delim, 1)[1]}"
-                # print extractor logging at error level
-                logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
-                raise Exception(exception) from e
-            # ensure that extractor logging is available
-            logger.info(f"maco extractor stderr:\n{stderr}")
-            return loaded
+                script.flush()
+                cwd = root_directory
+                custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
+
+                if custom_module.startswith("src."):
+                    # src layout found, which means the actual module content is within 'src' directory
+                    custom_module = custom_module[4:]
+                    cwd = os.path.join(cwd, "src")
+
+                # run the maco extractor in full venv process isolation (slow)
+                proc = subprocess.run(
+                    [python_exe, "-m", custom_module],
+                    cwd=cwd,
+                    capture_output=True,
+                )
+                stderr = proc.stderr.decode()
+                try:
+                    # Load results and return them
+                    output.seek(0)
+                    loaded =  json.load(output, cls=json_decoder)
+                except Exception as e:
+                    # If there was an error raised during runtime, then propagate
+                    delim = f'File "{module_path}"'
+                    exception = stderr
+                    if delim in exception:
+                        exception = f"{delim}{exception.split(delim, 1)[1]}"
+                    # print extractor logging at error level
+                    logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
+                    raise Exception(exception) from e
+                # ensure that extractor logging is available
+                logger.info(f"maco extractor stderr:\n{stderr}")
+    return loaded