maco 1.2.4__tar.gz → 1.2.5__tar.gz

{maco-1.2.4/maco.egg-info → maco-1.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.4
+Version: 1.2.5
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License

{maco-1.2.4 → maco-1.2.5}/demo_extractors/complex/complex.py

@@ -1,9 +1,8 @@
 from io import BytesIO
 from typing import List, Optional
 
-from maco import extractor, model, yara
-
 from demo_extractors.complex import complex_utils
+from maco import extractor, model, yara
 
 
 class Complex(extractor.Extractor):

{maco-1.2.4 → maco-1.2.5}/demo_extractors/limit_other.py

@@ -1,9 +1,8 @@
 from io import BytesIO
 from typing import Dict, List, Optional
 
-from maco import extractor, model, yara
-
 from demo_extractors import shared
+from maco import extractor, model, yara
 
 
 class LimitOther(extractor.Extractor):
@@ -24,6 +23,10 @@ class LimitOther(extractor.Extractor):
         """
 
     def run(self, stream: BytesIO, matches: List[yara.Match]) -> Optional[model.ExtractorModel]:
+        # import httpx at runtime so we can test that requirements.txt is installed dynamically without breaking
+        # the tests that do direct importing
+        import httpx
+
         # use a custom model that inherits from ExtractorModel
         # this model defines what can go in the 'other' dict
         tmp = shared.MyCustomModel(family="specify_other")

maco-1.2.5/demo_extractors/requirements.txt

@@ -0,0 +1 @@
+httpx

{maco-1.2.4 → maco-1.2.5}/demo_extractors/shared.py

@@ -1,4 +1,5 @@
 from typing import Optional
+
 import pydantic
 
 from maco import model

{maco-1.2.4/model_setup → maco-1.2.5}/maco/base_test.py

@@ -32,14 +32,19 @@ class BaseTest(unittest.TestCase):
     # I recommend something like os.path.join(__file__, "../../extractors")
     # if your extractors are in a folder 'extractors' next to a folder of tests
     path: str = None
+    create_venv: bool=False
 
-    def setUp(self) -> None:
-        if not self.name or not self.path:
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not cls.name or not cls.path:
             raise Exception("name and path must be set")
-        self.c = collector.Collector(self.path, include=[self.name])
+        cls.c = collector.Collector(cls.path, include=[cls.name], create_venv=cls.create_venv)
+        return super().setUpClass()
+
+    def test_default_metadata(self):
+        """Require extractor to be loadable and valid."""
         self.assertIn(self.name, self.c.extractors)
         self.assertEqual(len(self.c.extractors), 1)
-        return super().setUp()
 
     def extract(self, stream):
         """Return results for running extractor over stream, including yara check."""
@@ -49,18 +54,20 @@ class BaseTest(unittest.TestCase):
         resp = self.c.extract(stream, self.name)
         return resp
 
-    def _get_location(self) -> str:
+    @classmethod
+    def _get_location(cls) -> str:
         """Return path to child class that implements this class."""
         # import child module
-        module = type(self).__module__
+        module = cls.__module__
         i = importlib.import_module(module)
         # get location to child module
         return i.__file__
 
-    def load_cart(self, filepath: str) -> io.BytesIO:
+    @classmethod
+    def load_cart(cls, filepath: str) -> io.BytesIO:
         """Load and unneuter a test file (likely malware) into memory for processing."""
         # it is nice if we can load files relative to whatever is implementing base_test
-        dirpath = os.path.split(self._get_location())[0]
+        dirpath = os.path.split(cls._get_location())[0]
         # either filepath is absolute, or should be loaded relative to child of base_test
         filepath = os.path.join(dirpath, filepath)
         if not os.path.isfile(filepath):

{maco-1.2.4/model_setup → maco-1.2.5}/maco/cli.py

@@ -179,7 +179,7 @@ def main():
     parser.add_argument(
         "--create_venv",
         action="store_true",
-        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory)",
+        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). This runs much slower than the alternative but may be necessary when there are many extractors with conflicting dependencies.",
     )
     args = parser.parse_args()
     inc = args.include.split(",") if args.include else []

{maco-1.2.4/model_setup → maco-1.2.5}/maco/extractor.py

@@ -51,14 +51,14 @@ class Extractor:
         # check yara rules conform to expected structure
         # we throw away these compiled rules as we need all rules in system compiled together
         try:
-            rules = yara.compile(source=self.yara_rule)
+            self.yara_compiled = yara.compile(source=self.yara_rule)
         except yara.SyntaxError as e:
             raise InvalidExtractor(f"{self.name} - invalid yara rule") from e
         # need to track which plugin owns the rules
-        self.yara_rule_names = [x.identifier for x in rules]
-        if not len(list(rules)):
+        self.yara_rule_names = [x.identifier for x in self.yara_compiled]
+        if not len(list(self.yara_compiled)):
             raise InvalidExtractor(f"{name} must define at least one yara rule")
-        for x in rules:
+        for x in self.yara_compiled:
             if x.is_global:
                 raise InvalidExtractor(f"{x.identifier} yara rule must not be global")
 

{maco-1.2.4 → maco-1.2.5}/maco/utils.py

@@ -4,14 +4,14 @@ import importlib.machinery
 import importlib.util
 import inspect
 import json
+import logging
 import logging.handlers
+import multiprocessing
 import os
 import re
 import shutil
 import subprocess
 import sys
-import multiprocessing
-import logging
 import tempfile
 
 from maco import yara
@@ -27,8 +27,11 @@ from glob import glob
 from logging import Logger
 from pkgutil import walk_packages
 from types import ModuleType
-from typing import Callable, Dict, List, Set, Tuple
+from typing import Callable, Dict, List, Set, Tuple, Union
+
+from uv import find_uv_bin
 
+from maco import model
 from maco.extractor import Extractor
 
 logger = logging.getLogger("maco.lib.utils")
@@ -38,23 +41,10 @@ VENV_DIRECTORY_NAME = ".venv"
 RELATIVE_FROM_RE = re.compile(r"from (\.+)")
 RELATIVE_FROM_IMPORT_RE = re.compile(r"from (\.+) import")
 
-try:
-    # Attempt to use the uv package manager (Recommended)
-    from uv import find_uv_bin
-
-    UV_BIN = find_uv_bin()
-
-    PIP_CMD = f"{UV_BIN} pip"
-    VENV_CREATE_CMD = f"{UV_BIN} venv"
-    PACKAGE_MANAGER = "uv"
-except ImportError:
-    # Otherwise default to pip
-    from sys import executable
-
-    PIP_CMD = "pip"
-    VENV_CREATE_CMD = f"{executable} -m venv"
-    PACKAGE_MANAGER = "pip"
+UV_BIN = find_uv_bin()
 
+PIP_CMD = f"{UV_BIN} pip"
+VENV_CREATE_CMD = f"{UV_BIN} venv"
 
 class Base64Decoder(json.JSONDecoder):
     def __init__(self, *args, **kwargs):
@@ -210,9 +200,8 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
     return extractor_dirs, extractor_files
 
 
-def create_virtual_environments(directories: List[str], python_version: str, logger: Logger):
+def _install_required_packages(create_venv: bool, directories: List[str], python_version: str, logger: Logger):
     venvs = []
-    logger.info("Creating virtual environment(s)..")
     env = deepcopy(os.environ)
     stop_directory = os.path.dirname(sorted(directories)[0])
     # Track directories that we've already visited
@@ -222,14 +211,15 @@ def create_virtual_environments(directories: List[str], python_version: str, log
         while dir != stop_directory and dir not in visited_dirs:
             req_files = list({"requirements.txt", "pyproject.toml"}.intersection(set(os.listdir(dir))))
             if req_files:
-                venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
-                env.update({"VIRTUAL_ENV": venv_path})
-                # Create a virtual environment for the directory
-                if not os.path.exists(venv_path):
-                    cmd = VENV_CREATE_CMD
-                    if PACKAGE_MANAGER == "uv":
-                        cmd += f" --python {python_version}"
-                    subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
+                # create a virtual environment, otherwise directly install into current env
+                if create_venv:
+                    venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
+                    logger.info(f"Updating virtual environment {venv_path}")
+                    env.update({"VIRTUAL_ENV": venv_path})
+                    # Create a virtual environment for the directory
+                    if not os.path.exists(venv_path):
+                        cmd = f"{VENV_CREATE_CMD} --python {python_version}"
+                        subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
 
                 # Install/Update the packages in the environment
                 install_command = PIP_CMD.split(" ") + ["install", "-U"]
@@ -253,7 +243,10 @@ def create_virtual_environments(directories: List[str], python_version: str, log
 
                     install_command.extend(pyproject_command)
 
+                # always require maco to be installed
+                install_command.append("maco")
                 logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
+                # this uses VIRTUAL_ENV to control usage of a virtual environment
                 p = subprocess.run(
                     install_command,
                     cwd=dir,
@@ -264,10 +257,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
                     if b"is being installed using the legacy" in p.stderr:
                         # Ignore these types of errors
                         continue
-                    logger.error(f"Error installing into venv:\n{p.stderr.decode()}")
+                    logger.error(f"Error installing into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
                 else:
-                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
-                    venvs.append(venv_path)
+                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
+                    if create_venv:
+                        venvs.append(venv_path)
 
                 # Cleanup any build directories that are the product of package installation
                 expected_build_path = os.path.join(dir, "build")
@@ -311,7 +305,7 @@ def register_extractors(
 ):
     package_name = os.path.basename(current_directory)
     parent_directory = os.path.dirname(current_directory)
-    if package_name in sys.modules:
+    if venvs and package_name in sys.modules:
         # this may happen as part of testing if some part of the extractor code was directly imported
         logger.warning(f"Looks like {package_name} is already loaded. "
                        "If your maco extractor overlaps an existing package name this could cause problems.")
@@ -402,32 +396,26 @@ def import_extractors(
     *,
     root_directory: str,
     scanner: yara.Rules,
-    create_venv: bool = False,
-    python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
+    create_venv: bool,
     logger: Logger,
+    python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
 ):
     extractor_dirs, extractor_files = scan_for_extractors(root_directory, scanner, logger)
 
     logger.info(f"Extractor files found based on scanner ({len(extractor_files)}).")
     logger.debug(extractor_files)
 
-    venvs = []
-    if create_venv:
-        venvs = create_virtual_environments(extractor_dirs, python_version, logger)
-    else:
-        # Look for pre-existing virtual environments, if any
-        logger.info("Checking for pre-existing virtual environment(s)..")
-        venvs = [
-            os.path.join(root, VENV_DIRECTORY_NAME)
-            for root, dirs, _ in os.walk(root_directory)
-            if VENV_DIRECTORY_NAME in dirs
-        ]
+    # Install packages into the current environment or dynamically created virtual environments
+    venvs = _install_required_packages(create_venv, extractor_dirs, python_version, logger)
 
     # With the environment prepared, we can now hunt for the extractors and register them
     logger.info("Registering extractors..")
     register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
 
 
+# holds cached extractors when not running in venv mode
+_loaded_extractors: Dict[str, Extractor] = {}
+
 def run_extractor(
     sample_path,
     module_name,
@@ -436,55 +424,69 @@ def run_extractor(
     venv,
     venv_script=VENV_SCRIPT,
     json_decoder=Base64Decoder,
-) -> Dict[str, dict]:
-    # Write temporary script in the same directory as extractor to resolve relative imports
-    python_exe = sys.executable
-    if venv:
-        # If there is a linked virtual environment, execute within that environment
+) -> Union[Dict[str, dict], model.ExtractorModel]:
+    """Runs the maco extractor against sample either in current process or child process."""
+    if not venv:
+        key = f"{module_name}_{extractor_class}"
+        if key not in _loaded_extractors:
+            # dynamic import of extractor
+            mod = importlib.import_module(module_name)
+            extractor_cls = mod.__getattribute__(extractor_class)
+            extractor = extractor_cls()
+        else:
+            # retrieve cached extractor
+            extractor = _loaded_extractors[key]
+        if extractor.yara_compiled:
+            matches = extractor.yara_compiled.match(sample_path)
+        loaded = extractor.run(open(sample_path, 'rb'), matches=matches)
+    else:
+        # execute extractor in child process with separate virtual environment
+        # Write temporary script in the same directory as extractor to resolve relative imports
         python_exe = os.path.join(venv, "bin", "python")
-    dirname = os.path.dirname(module_path)
-    with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
-        with tempfile.NamedTemporaryFile() as output:
-            parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
-            root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
-
-            script.write(
-                venv_script.format(
-                    parent_package_path=parent_package_path,
-                    module_name=module_name,
-                    module_class=extractor_class,
-                    sample_path=sample_path,
-                    output_path=output.name,
+        dirname = os.path.dirname(module_path)
+        with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
+            with tempfile.NamedTemporaryFile() as output:
+                parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
+                root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
+
+                script.write(
+                    venv_script.format(
+                        parent_package_path=parent_package_path,
+                        module_name=module_name,
+                        module_class=extractor_class,
+                        sample_path=sample_path,
+                        output_path=output.name,
+                    )
                 )
-            )
-            script.flush()
-            cwd = root_directory
-            custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
-
-            if custom_module.startswith("src."):
-                # src layout found, which means the actual module content is within 'src' directory
-                custom_module = custom_module[4:]
-                cwd = os.path.join(cwd, "src")
-
-            proc = subprocess.run(
-                [python_exe, "-m", custom_module],
-                cwd=cwd,
-                capture_output=True,
-            )
-            stderr = proc.stderr.decode()
-            try:
-                # Load results and return them
-                output.seek(0)
-                loaded =  json.load(output, cls=json_decoder)
-            except Exception as e:
-                # If there was an error raised during runtime, then propagate
-                delim = f'File "{module_path}"'
-                exception = stderr
-                if delim in exception:
-                    exception = f"{delim}{exception.split(delim, 1)[1]}"
-                # print extractor logging at error level
-                logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
-                raise Exception(exception) from e
-            # ensure that extractor logging is available
-            logger.info(f"maco extractor stderr:\n{stderr}")
-            return loaded
+                script.flush()
+                cwd = root_directory
+                custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
+
+                if custom_module.startswith("src."):
+                    # src layout found, which means the actual module content is within 'src' directory
+                    custom_module = custom_module[4:]
+                    cwd = os.path.join(cwd, "src")
+
+                # run the maco extractor in full venv process isolation (slow)
+                proc = subprocess.run(
+                    [python_exe, "-m", custom_module],
+                    cwd=cwd,
+                    capture_output=True,
+                )
+                stderr = proc.stderr.decode()
+                try:
+                    # Load results and return them
+                    output.seek(0)
+                    loaded =  json.load(output, cls=json_decoder)
+                except Exception as e:
+                    # If there was an error raised during runtime, then propagate
+                    delim = f'File "{module_path}"'
+                    exception = stderr
+                    if delim in exception:
+                        exception = f"{delim}{exception.split(delim, 1)[1]}"
+                    # print extractor logging at error level
+                    logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
+                    raise Exception(exception) from e
+                # ensure that extractor logging is available
+                logger.info(f"maco extractor stderr:\n{stderr}")
+    return loaded

{maco-1.2.4 → maco-1.2.5/maco.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.4
+Version: 1.2.5
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License

{maco-1.2.4 → maco-1.2.5}/maco.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ demo_extractors/__init__.py
 demo_extractors/elfy.py
 demo_extractors/limit_other.py
 demo_extractors/nothing.py
+demo_extractors/requirements.txt
 demo_extractors/shared.py
 demo_extractors/complex/__init__.py
 demo_extractors/complex/complex.py
@@ -44,6 +45,7 @@ model_setup/maco/model/__init__.py
 model_setup/maco/model/model.py
 pipelines/publish.yaml
 pipelines/test.yaml
+tests/benchmark.py
 tests/pytest.ini
 tests/requirements.txt
 tests/test_base_test.py

{maco-1.2.4 → maco-1.2.5/model_setup}/maco/base_test.py

@@ -32,14 +32,19 @@ class BaseTest(unittest.TestCase):
     # I recommend something like os.path.join(__file__, "../../extractors")
     # if your extractors are in a folder 'extractors' next to a folder of tests
     path: str = None
+    create_venv: bool=False
 
-    def setUp(self) -> None:
-        if not self.name or not self.path:
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not cls.name or not cls.path:
             raise Exception("name and path must be set")
-        self.c = collector.Collector(self.path, include=[self.name])
+        cls.c = collector.Collector(cls.path, include=[cls.name], create_venv=cls.create_venv)
+        return super().setUpClass()
+
+    def test_default_metadata(self):
+        """Require extractor to be loadable and valid."""
         self.assertIn(self.name, self.c.extractors)
         self.assertEqual(len(self.c.extractors), 1)
-        return super().setUp()
 
     def extract(self, stream):
         """Return results for running extractor over stream, including yara check."""
@@ -49,18 +54,20 @@ class BaseTest(unittest.TestCase):
         resp = self.c.extract(stream, self.name)
         return resp
 
-    def _get_location(self) -> str:
+    @classmethod
+    def _get_location(cls) -> str:
         """Return path to child class that implements this class."""
         # import child module
-        module = type(self).__module__
+        module = cls.__module__
         i = importlib.import_module(module)
         # get location to child module
         return i.__file__
 
-    def load_cart(self, filepath: str) -> io.BytesIO:
+    @classmethod
+    def load_cart(cls, filepath: str) -> io.BytesIO:
         """Load and unneuter a test file (likely malware) into memory for processing."""
         # it is nice if we can load files relative to whatever is implementing base_test
-        dirpath = os.path.split(self._get_location())[0]
+        dirpath = os.path.split(cls._get_location())[0]
         # either filepath is absolute, or should be loaded relative to child of base_test
         filepath = os.path.join(dirpath, filepath)
         if not os.path.isfile(filepath):

{maco-1.2.4 → maco-1.2.5/model_setup}/maco/cli.py

@@ -179,7 +179,7 @@ def main():
     parser.add_argument(
         "--create_venv",
         action="store_true",
-        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory)",
+        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). This runs much slower than the alternative but may be necessary when there are many extractors with conflicting dependencies.",
     )
     args = parser.parse_args()
     inc = args.include.split(",") if args.include else []

{maco-1.2.4 → maco-1.2.5/model_setup}/maco/extractor.py

@@ -51,14 +51,14 @@ class Extractor:
         # check yara rules conform to expected structure
         # we throw away these compiled rules as we need all rules in system compiled together
         try:
-            rules = yara.compile(source=self.yara_rule)
+            self.yara_compiled = yara.compile(source=self.yara_rule)
         except yara.SyntaxError as e:
             raise InvalidExtractor(f"{self.name} - invalid yara rule") from e
         # need to track which plugin owns the rules
-        self.yara_rule_names = [x.identifier for x in rules]
-        if not len(list(rules)):
+        self.yara_rule_names = [x.identifier for x in self.yara_compiled]
+        if not len(list(self.yara_compiled)):
             raise InvalidExtractor(f"{name} must define at least one yara rule")
-        for x in rules:
+        for x in self.yara_compiled:
             if x.is_global:
                 raise InvalidExtractor(f"{x.identifier} yara rule must not be global")
 

{maco-1.2.4 → maco-1.2.5}/model_setup/maco/utils.py

@@ -4,14 +4,14 @@ import importlib.machinery
 import importlib.util
 import inspect
 import json
+import logging
 import logging.handlers
+import multiprocessing
 import os
 import re
 import shutil
 import subprocess
 import sys
-import multiprocessing
-import logging
 import tempfile
 
 from maco import yara
@@ -27,8 +27,11 @@ from glob import glob
 from logging import Logger
 from pkgutil import walk_packages
 from types import ModuleType
-from typing import Callable, Dict, List, Set, Tuple
+from typing import Callable, Dict, List, Set, Tuple, Union
+
+from uv import find_uv_bin
 
+from maco import model
 from maco.extractor import Extractor
 
 logger = logging.getLogger("maco.lib.utils")
@@ -38,23 +41,10 @@ VENV_DIRECTORY_NAME = ".venv"
 RELATIVE_FROM_RE = re.compile(r"from (\.+)")
 RELATIVE_FROM_IMPORT_RE = re.compile(r"from (\.+) import")
 
-try:
-    # Attempt to use the uv package manager (Recommended)
-    from uv import find_uv_bin
-
-    UV_BIN = find_uv_bin()
-
-    PIP_CMD = f"{UV_BIN} pip"
-    VENV_CREATE_CMD = f"{UV_BIN} venv"
-    PACKAGE_MANAGER = "uv"
-except ImportError:
-    # Otherwise default to pip
-    from sys import executable
-
-    PIP_CMD = "pip"
-    VENV_CREATE_CMD = f"{executable} -m venv"
-    PACKAGE_MANAGER = "pip"
+UV_BIN = find_uv_bin()
 
+PIP_CMD = f"{UV_BIN} pip"
+VENV_CREATE_CMD = f"{UV_BIN} venv"
 
 class Base64Decoder(json.JSONDecoder):
     def __init__(self, *args, **kwargs):
@@ -210,9 +200,8 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
     return extractor_dirs, extractor_files
 
 
-def create_virtual_environments(directories: List[str], python_version: str, logger: Logger):
+def _install_required_packages(create_venv: bool, directories: List[str], python_version: str, logger: Logger):
     venvs = []
-    logger.info("Creating virtual environment(s)..")
     env = deepcopy(os.environ)
     stop_directory = os.path.dirname(sorted(directories)[0])
     # Track directories that we've already visited
@@ -222,14 +211,15 @@ def create_virtual_environments(directories: List[str], python_version: str, log
         while dir != stop_directory and dir not in visited_dirs:
             req_files = list({"requirements.txt", "pyproject.toml"}.intersection(set(os.listdir(dir))))
             if req_files:
-                venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
-                env.update({"VIRTUAL_ENV": venv_path})
-                # Create a virtual environment for the directory
-                if not os.path.exists(venv_path):
-                    cmd = VENV_CREATE_CMD
-                    if PACKAGE_MANAGER == "uv":
-                        cmd += f" --python {python_version}"
-                    subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
+                # create a virtual environment, otherwise directly install into current env
+                if create_venv:
+                    venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
+                    logger.info(f"Updating virtual environment {venv_path}")
+                    env.update({"VIRTUAL_ENV": venv_path})
+                    # Create a virtual environment for the directory
+                    if not os.path.exists(venv_path):
+                        cmd = f"{VENV_CREATE_CMD} --python {python_version}"
+                        subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
 
                 # Install/Update the packages in the environment
                 install_command = PIP_CMD.split(" ") + ["install", "-U"]
@@ -253,7 +243,10 @@ def create_virtual_environments(directories: List[str], python_version: str, log
 
                     install_command.extend(pyproject_command)
 
+                # always require maco to be installed
+                install_command.append("maco")
                 logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
+                # this uses VIRTUAL_ENV to control usage of a virtual environment
                 p = subprocess.run(
                     install_command,
                     cwd=dir,
@@ -264,10 +257,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
                     if b"is being installed using the legacy" in p.stderr:
                         # Ignore these types of errors
                         continue
-                    logger.error(f"Error installing into venv:\n{p.stderr.decode()}")
+                    logger.error(f"Error installing into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
                 else:
-                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
-                    venvs.append(venv_path)
+                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
+                    if create_venv:
+                        venvs.append(venv_path)
 
                 # Cleanup any build directories that are the product of package installation
                 expected_build_path = os.path.join(dir, "build")
@@ -311,7 +305,7 @@ def register_extractors(
 ):
     package_name = os.path.basename(current_directory)
     parent_directory = os.path.dirname(current_directory)
-    if package_name in sys.modules:
+    if venvs and package_name in sys.modules:
         # this may happen as part of testing if some part of the extractor code was directly imported
         logger.warning(f"Looks like {package_name} is already loaded. "
                        "If your maco extractor overlaps an existing package name this could cause problems.")
@@ -402,32 +396,26 @@ def import_extractors(
     *,
     root_directory: str,
     scanner: yara.Rules,
-    create_venv: bool = False,
-    python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
+    create_venv: bool,
     logger: Logger,
+    python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
 ):
     extractor_dirs, extractor_files = scan_for_extractors(root_directory, scanner, logger)
 
     logger.info(f"Extractor files found based on scanner ({len(extractor_files)}).")
     logger.debug(extractor_files)
 
-    venvs = []
-    if create_venv:
-        venvs = create_virtual_environments(extractor_dirs, python_version, logger)
-    else:
-        # Look for pre-existing virtual environments, if any
-        logger.info("Checking for pre-existing virtual environment(s)..")
-        venvs = [
-            os.path.join(root, VENV_DIRECTORY_NAME)
-            for root, dirs, _ in os.walk(root_directory)
-            if VENV_DIRECTORY_NAME in dirs
-        ]
+    # Install packages into the current environment or dynamically created virtual environments
+    venvs = _install_required_packages(create_venv, extractor_dirs, python_version, logger)
 
     # With the environment prepared, we can now hunt for the extractors and register them
     logger.info("Registering extractors..")
     register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
 
 
+# holds cached extractors when not running in venv mode
+_loaded_extractors: Dict[str, Extractor] = {}
+
 def run_extractor(
     sample_path,
     module_name,
@@ -436,55 +424,69 @@ def run_extractor(
     venv,
     venv_script=VENV_SCRIPT,
     json_decoder=Base64Decoder,
-) -> Dict[str, dict]:
-    # Write temporary script in the same directory as extractor to resolve relative imports
-    python_exe = sys.executable
-    if venv:
-        # If there is a linked virtual environment, execute within that environment
+) -> Union[Dict[str, dict], model.ExtractorModel]:
+    """Runs the maco extractor against sample either in current process or child process."""
+    if not venv:
+        key = f"{module_name}_{extractor_class}"
+        if key not in _loaded_extractors:
+            # dynamic import of extractor
+            mod = importlib.import_module(module_name)
+            extractor_cls = mod.__getattribute__(extractor_class)
+            extractor = extractor_cls()
+        else:
+            # retrieve cached extractor
+            extractor = _loaded_extractors[key]
+        if extractor.yara_compiled:
+            matches = extractor.yara_compiled.match(sample_path)
+        loaded = extractor.run(open(sample_path, 'rb'), matches=matches)
+    else:
+        # execute extractor in child process with separate virtual environment
+        # Write temporary script in the same directory as extractor to resolve relative imports
         python_exe = os.path.join(venv, "bin", "python")
-    dirname = os.path.dirname(module_path)
-    with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
-        with tempfile.NamedTemporaryFile() as output:
-            parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
-            root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
-
-            script.write(
-                venv_script.format(
-                    parent_package_path=parent_package_path,
-                    module_name=module_name,
-                    module_class=extractor_class,
-                    sample_path=sample_path,
-                    output_path=output.name,
+        dirname = os.path.dirname(module_path)
+        with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
+            with tempfile.NamedTemporaryFile() as output:
+                parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
+                root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
+
+                script.write(
+                    venv_script.format(
+                        parent_package_path=parent_package_path,
+                        module_name=module_name,
+                        module_class=extractor_class,
+                        sample_path=sample_path,
+                        output_path=output.name,
+                    )
                 )
-            )
-            script.flush()
-            cwd = root_directory
-            custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
-
-            if custom_module.startswith("src."):
-                # src layout found, which means the actual module content is within 'src' directory
-                custom_module = custom_module[4:]
-                cwd = os.path.join(cwd, "src")
-
-            proc = subprocess.run(
-                [python_exe, "-m", custom_module],
-                cwd=cwd,
-                capture_output=True,
-            )
-            stderr = proc.stderr.decode()
-            try:
-                # Load results and return them
-                output.seek(0)
-                loaded =  json.load(output, cls=json_decoder)
-            except Exception as e:
-                # If there was an error raised during runtime, then propagate
-                delim = f'File "{module_path}"'
-                exception = stderr
-                if delim in exception:
-                    exception = f"{delim}{exception.split(delim, 1)[1]}"
-                # print extractor logging at error level
-                logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
-                raise Exception(exception) from e
-            # ensure that extractor logging is available
-            logger.info(f"maco extractor stderr:\n{stderr}")
-            return loaded
+                script.flush()
+                cwd = root_directory
+                custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
+
+                if custom_module.startswith("src."):
+                    # src layout found, which means the actual module content is within 'src' directory
+                    custom_module = custom_module[4:]
+                    cwd = os.path.join(cwd, "src")
+
+                # run the maco extractor in full venv process isolation (slow)
+                proc = subprocess.run(
+                    [python_exe, "-m", custom_module],
+                    cwd=cwd,
+                    capture_output=True,
+                )
+                stderr = proc.stderr.decode()
+                try:
+                    # Load results and return them
+                    output.seek(0)
+                    loaded =  json.load(output, cls=json_decoder)
+                except Exception as e:
+                    # If there was an error raised during runtime, then propagate
+                    delim = f'File "{module_path}"'
+                    exception = stderr
+                    if delim in exception:
+                        exception = f"{delim}{exception.split(delim, 1)[1]}"
+                    # print extractor logging at error level
+                    logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
+                    raise Exception(exception) from e
+                # ensure that extractor logging is available
+                logger.info(f"maco extractor stderr:\n{stderr}")
+    return loaded

maco-1.2.5/tests/benchmark.py

@@ -0,0 +1,107 @@
+import os
+import timeit
+
+from demo_extractors.complex import complex
+from maco import base_test
+
+# instance of extractor for synthetic comparison to maco
+instance = complex.Complex()
+
+
+class LocalBaseTest(base_test.BaseTest):
+    name = "Complex"
+    path = os.path.join(__file__, "../../demo_extractors")
+    create_venv = False
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        super().setUpClass()
+        cls.input_file = cls.load_cart("data/trigger_complex.txt.cart")
+        cls.input_file.seek(0)
+
+
+class TestComplexSynthetic(LocalBaseTest):
+    """Test extractors work bypassing maco."""
+
+    def test_extract(self):
+        self.input_file.seek(0)
+        raw = self.input_file.read()
+        self.input_file.seek(0)
+        # run yara rules against sample
+        matches = instance.yara_compiled.match(data=raw)
+        self.assertEqual(len(matches), 2)
+        result = instance.run(self.input_file, [])
+        self.assertEqual(result.family, "complex")
+
+
+class TestComplexNoVenv(LocalBaseTest):
+    """Test extractors work without full venv isolation."""
+
+    def test_extract(self):
+        self.input_file.seek(0)
+        ret = self.extract(self.input_file)
+        self.assertEqual(ret["family"], "complex")
+        self.assertEqual(ret["version"], "5")
+
+
+class TestComplexVenv(LocalBaseTest):
+    """Test extractors work when run with virtual environments."""
+
+    create_venv = True
+
+    def test_extract(self):
+        self.input_file.seek(0)
+        ret = self.extract(self.input_file)
+        self.assertEqual(ret["family"], "complex")
+        self.assertEqual(ret["version"], "5")
+
+
+def make_synthetic():
+    TestComplexSynthetic.setUpClass()
+    tc = TestComplexSynthetic()
+    tc.setUp()
+    return tc
+
+
+def make_no_venv():
+    TestComplexNoVenv.setUpClass()
+    tc = TestComplexNoVenv()
+    tc.setUp()
+    return tc
+
+
+def make_venv():
+    TestComplexVenv.setUpClass()
+    tc = TestComplexVenv()
+    tc.setUp()
+    return tc
+
+
+if __name__ == "__main__":
+    trials = 1000
+    print(f"num trials: {trials}")
+    print("results are number of seconds to execute total number of trials")
+    print("synthetic comparison (directly import and execute extractor)")
+    print(
+        timeit.timeit(
+            "tc.test_extract()",
+            setup="from __main__ import make_synthetic; tc=make_synthetic()",
+            number=trials,
+        )
+    )
+    print("maco no venv isolation")
+    print(
+        timeit.timeit(
+            "tc.test_extract()",
+            setup="from __main__ import make_no_venv; tc=make_no_venv()",
+            number=trials,
+        )
+    )
+    print("maco venv isolation")
+    print(
+        timeit.timeit(
+            "tc.test_extract()",
+            setup="from __main__ import make_venv; tc=make_venv()",
+            number=trials,
+        )
+    )

{maco-1.2.4 → maco-1.2.5}/tests/test_base_test.py

@@ -19,9 +19,12 @@ class TestLimitOther(base_test.BaseTest):
         self.assertEqual(ret["family"], "specify_other")
         self.assertEqual(ret["campaign_id"], ["12345"])
 
+
 class TestComplex(base_test.BaseTest):
+    """Test that complex extractor can be used in base environment."""
     name = "Complex"
     path = os.path.join(__file__, "../../demo_extractors")
+    create_venv = False
 
     def test_extract(self):
         """Tests that we can run an extractor through maco."""
@@ -43,3 +46,15 @@ class TestComplex(base_test.BaseTest):
         data = io.BytesIO(b"my malwarez")
         result = instance.run(data, [])
         self.assertEqual(result.family, "complex")
+
+class TestComplexVenv(base_test.BaseTest):
+    """Test that complex extractor can be used in full venv isolation."""
+    name = "Complex"
+    path = os.path.join(__file__, "../../demo_extractors")
+    create_venv = True
+
+    def test_extract(self):
+        """Tests that we can run an extractor through maco."""
+        ret = self.extract(self.load_cart("data/trigger_complex.txt.cart"))
+        self.assertEqual(ret["family"], "complex")
+        self.assertEqual(ret["version"], "5")

{maco-1.2.4 → maco-1.2.5}/tests/test_detection.py

@@ -1,7 +1,8 @@
 import os
-import pytest
 import sys
 
+import pytest
+
 from maco.collector import Collector
 
 INIT_MODULES = list(sys.modules.keys())
@@ -105,9 +106,9 @@ def test_public_projects(repository_url: str, extractors: list, python_minor: in
     # Ensure that any changes we make doesn't break usage of public projects
     # which can affect downstream systems using like library (ie. Assemblyline)
     import sys
+    from tempfile import TemporaryDirectory
 
     from git import Repo
-    from tempfile import TemporaryDirectory
 
     if sys.version_info >= (3, python_minor):
         with TemporaryDirectory() as working_dir:
@@ -123,8 +124,9 @@ def test_public_projects(repository_url: str, extractors: list, python_minor: in
 
 
 def test_module_confusion():
-    from tempfile import TemporaryDirectory
     import shutil
+    from tempfile import TemporaryDirectory
+
     import git
 
     # Directories that have the same name as the Python module, shouldn't cause confusion on loading the right module

{maco-1.2.4 → maco-1.2.5}/tox.ini

@@ -7,4 +7,4 @@ deps =
     -r requirements.txt
     -r tests/requirements.txt
 # run the tests
-commands = python -m pytest -p no:cacheprovider --durations=10 -ra -q -k "not git and not extractors" -vv -W ignore::DeprecationWarning
+commands = python -m pytest tests/ -p no:cacheprovider --durations=10 -ra -q -k "not git and not extractors" -vv -W ignore::DeprecationWarning

{maco-1.2.4 → maco-1.2.5}/tests/test_cli.py

@@ -1,6 +1,6 @@
 import os
-
 import unittest
+
 from maco import cli