maco 1.2.3__py3-none-any.whl → 1.2.5__py3-none-any.whl

demo_extractors/complex/complex.py

@@ -1,10 +1,9 @@
 from io import BytesIO
 from typing import List, Optional
 
+from demo_extractors.complex import complex_utils
 from maco import extractor, model, yara
 
-from complex import complex_utils
-
 
 class Complex(extractor.Extractor):
     """This script has multiple yara rules and coverage of the data model."""

demo_extractors/limit_other.py

@@ -1,10 +1,9 @@
 from io import BytesIO
 from typing import Dict, List, Optional
 
+from demo_extractors import shared
 from maco import extractor, model, yara
 
-from . import shared
-
 
 class LimitOther(extractor.Extractor):
     """An example of how the 'other' dictionary can be limited in a custom way."""
@@ -24,6 +23,10 @@ class LimitOther(extractor.Extractor):
         """
 
     def run(self, stream: BytesIO, matches: List[yara.Match]) -> Optional[model.ExtractorModel]:
+        # import httpx at runtime so we can test that requirements.txt is installed dynamically without breaking
+        # the tests that do direct importing
+        import httpx
+
         # use a custom model that inherits from ExtractorModel
         # this model defines what can go in the 'other' dict
         tmp = shared.MyCustomModel(family="specify_other")

demo_extractors/requirements.txt

@@ -0,0 +1 @@
+httpx

demo_extractors/shared.py

@@ -1,4 +1,5 @@
 from typing import Optional
+
 import pydantic
 
 from maco import model

maco/base_test.py

@@ -32,14 +32,19 @@ class BaseTest(unittest.TestCase):
     # I recommend something like os.path.join(__file__, "../../extractors")
     # if your extractors are in a folder 'extractors' next to a folder of tests
     path: str = None
+    create_venv: bool=False
 
-    def setUp(self) -> None:
-        if not self.name or not self.path:
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not cls.name or not cls.path:
             raise Exception("name and path must be set")
-        self.c = collector.Collector(self.path, include=[self.name])
+        cls.c = collector.Collector(cls.path, include=[cls.name], create_venv=cls.create_venv)
+        return super().setUpClass()
+
+    def test_default_metadata(self):
+        """Require extractor to be loadable and valid."""
         self.assertIn(self.name, self.c.extractors)
         self.assertEqual(len(self.c.extractors), 1)
-        return super().setUp()
 
     def extract(self, stream):
         """Return results for running extractor over stream, including yara check."""
@@ -49,18 +54,20 @@ class BaseTest(unittest.TestCase):
         resp = self.c.extract(stream, self.name)
         return resp
 
-    def _get_location(self) -> str:
+    @classmethod
+    def _get_location(cls) -> str:
         """Return path to child class that implements this class."""
         # import child module
-        module = type(self).__module__
+        module = cls.__module__
         i = importlib.import_module(module)
         # get location to child module
         return i.__file__
 
-    def load_cart(self, filepath: str) -> io.BytesIO:
+    @classmethod
+    def load_cart(cls, filepath: str) -> io.BytesIO:
         """Load and unneuter a test file (likely malware) into memory for processing."""
         # it is nice if we can load files relative to whatever is implementing base_test
-        dirpath = os.path.split(self._get_location())[0]
+        dirpath = os.path.split(cls._get_location())[0]
         # either filepath is absolute, or should be loaded relative to child of base_test
         filepath = os.path.join(dirpath, filepath)
         if not os.path.isfile(filepath):

maco/cli.py

@@ -179,7 +179,7 @@ def main():
     parser.add_argument(
         "--create_venv",
         action="store_true",
-        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory)",
+        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). This runs much slower than the alternative but may be necessary when there are many extractors with conflicting dependencies.",
     )
     args = parser.parse_args()
     inc = args.include.split(",") if args.include else []

maco/collector.py

@@ -2,8 +2,9 @@
 
 import inspect
 import logging
+import logging.handlers
 import os
-from multiprocessing import Manager, Process
+from multiprocessing import Manager, Process, Queue
 from tempfile import NamedTemporaryFile
 from types import ModuleType
 from typing import Any, BinaryIO, Dict, List, Union
@@ -86,21 +87,33 @@ class Collector:
                     )
                     namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
 
+            # multiprocess logging is awkward - set up a queue to ensure we can log
+            logging_queue = Queue()
+            queue_handler = logging.handlers.QueueListener(logging_queue,*logging.getLogger().handlers)
+            queue_handler.start()
+
             # Find the extractors within the given directory
             # Execute within a child process to ensure main process interpreter is kept clean
             p = Process(
-                target=utils.import_extractors,
+                target=utils.proxy_logging,
                 args=(
-                    path_extractors,
-                    yara.compile(source=utils.MACO_YARA_RULE),
+                    logging_queue,
+                    utils.import_extractors,
                     extractor_module_callback,
-                    logger,
-                    create_venv and os.path.isdir(path_extractors),
+                ),
+                kwargs=dict(
+                    root_directory=path_extractors,
+                    scanner=yara.compile(source=utils.MACO_YARA_RULE),
+                    create_venv=create_venv and os.path.isdir(path_extractors),
                 ),
             )
             p.start()
             p.join()
 
+            # stop multiprocess logging
+            queue_handler.stop()
+            logging_queue.close()
+
             self.extractors = dict(extractors)
             if not self.extractors:
                 raise ExtractorLoadError("no extractors were loaded")

maco/extractor.py

@@ -51,14 +51,14 @@ class Extractor:
         # check yara rules conform to expected structure
         # we throw away these compiled rules as we need all rules in system compiled together
         try:
-            rules = yara.compile(source=self.yara_rule)
+            self.yara_compiled = yara.compile(source=self.yara_rule)
         except yara.SyntaxError as e:
             raise InvalidExtractor(f"{self.name} - invalid yara rule") from e
         # need to track which plugin owns the rules
-        self.yara_rule_names = [x.identifier for x in rules]
-        if not len(list(rules)):
+        self.yara_rule_names = [x.identifier for x in self.yara_compiled]
+        if not len(list(self.yara_compiled)):
             raise InvalidExtractor(f"{name} must define at least one yara rule")
-        for x in rules:
+        for x in self.yara_compiled:
             if x.is_global:
                 raise InvalidExtractor(f"{x.identifier} yara rule must not be global")
 

maco/utils.py

@@ -4,6 +4,9 @@ import importlib.machinery
 import importlib.util
 import inspect
 import json
+import logging
+import logging.handlers
+import multiprocessing
 import os
 import re
 import shutil
@@ -24,32 +27,24 @@ from glob import glob
 from logging import Logger
 from pkgutil import walk_packages
 from types import ModuleType
-from typing import Callable, Dict, List, Set, Tuple
+from typing import Callable, Dict, List, Set, Tuple, Union
 
-from maco.extractor import Extractor
-
-VENV_DIRECTORY_NAME = ".venv"
+from uv import find_uv_bin
 
-RELATIVE_FROM_RE = re.compile("from (\.+)")
-RELATIVE_FROM_IMPORT_RE = re.compile("from (\.+) import")
+from maco import model
+from maco.extractor import Extractor
 
-try:
-    # Attempt to use the uv package manager (Recommended)
-    from uv import find_uv_bin
+logger = logging.getLogger("maco.lib.utils")
 
-    UV_BIN = find_uv_bin()
+VENV_DIRECTORY_NAME = ".venv"
 
-    PIP_CMD = f"{UV_BIN} pip"
-    VENV_CREATE_CMD = f"{UV_BIN} venv"
-    PACKAGE_MANAGER = "uv"
-except ImportError:
-    # Otherwise default to pip
-    from sys import executable
+RELATIVE_FROM_RE = re.compile(r"from (\.+)")
+RELATIVE_FROM_IMPORT_RE = re.compile(r"from (\.+) import")
 
-    PIP_CMD = "pip"
-    VENV_CREATE_CMD = f"{executable} -m venv"
-    PACKAGE_MANAGER = "pip"
+UV_BIN = find_uv_bin()
 
+PIP_CMD = f"{UV_BIN} pip"
+VENV_CREATE_CMD = f"{UV_BIN} venv"
 
 class Base64Decoder(json.JSONDecoder):
     def __init__(self, *args, **kwargs):
@@ -69,6 +64,7 @@ import importlib
 import json
 import os
 import sys
+import logging
 
 try:
     from maco import yara
@@ -76,6 +72,19 @@ except:
     import yara
 
 from base64 import b64encode
+
+# ensure we have a logger to stderr
+import logging
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+sh = logging.StreamHandler()
+logger.addHandler(sh)
+sh.setLevel(logging.DEBUG)
+formatter = logging.Formatter(
+    fmt="%(asctime)s, [%(levelname)s] %(module)s.%(funcName)s: %(message)s", datefmt="%Y-%m-%d (%H:%M:%S)"
+)
+sh.setFormatter(formatter)
+
 parent_package_path = "{parent_package_path}"
 sys.path.insert(1, parent_package_path)
 mod = importlib.import_module("{module_name}")
@@ -101,7 +110,7 @@ with open("{output_path}", 'w') as fp:
             json.dump(result.dict(exclude_defaults=True, exclude_none=True), fp, cls=Base64Encoder)
 """
 
-MACO_YARA_RULE = """
+MACO_YARA_RULE = r"""
 rule MACO {
     meta:
         desc = "Used to match on Python files that contain MACO extractors"
@@ -191,9 +200,8 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
     return extractor_dirs, extractor_files
 
 
-def create_virtual_environments(directories: List[str], python_version: str, logger: Logger):
+def _install_required_packages(create_venv: bool, directories: List[str], python_version: str, logger: Logger):
     venvs = []
-    logger.info("Creating virtual environment(s)..")
     env = deepcopy(os.environ)
     stop_directory = os.path.dirname(sorted(directories)[0])
     # Track directories that we've already visited
@@ -203,14 +211,15 @@ def create_virtual_environments(directories: List[str], python_version: str, log
         while dir != stop_directory and dir not in visited_dirs:
             req_files = list({"requirements.txt", "pyproject.toml"}.intersection(set(os.listdir(dir))))
             if req_files:
-                venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
-                env.update({"VIRTUAL_ENV": venv_path})
-                # Create a virtual environment for the directory
-                if not os.path.exists(venv_path):
-                    cmd = VENV_CREATE_CMD
-                    if PACKAGE_MANAGER == "uv":
-                        cmd += f" --python {python_version}"
-                    subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
+                # create a virtual environment, otherwise directly install into current env
+                if create_venv:
+                    venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
+                    logger.info(f"Updating virtual environment {venv_path}")
+                    env.update({"VIRTUAL_ENV": venv_path})
+                    # Create a virtual environment for the directory
+                    if not os.path.exists(venv_path):
+                        cmd = f"{VENV_CREATE_CMD} --python {python_version}"
+                        subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
 
                 # Install/Update the packages in the environment
                 install_command = PIP_CMD.split(" ") + ["install", "-U"]
@@ -234,7 +243,10 @@ def create_virtual_environments(directories: List[str], python_version: str, log
 
                     install_command.extend(pyproject_command)
 
+                # always require maco to be installed
+                install_command.append("maco")
                 logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
+                # this uses VIRTUAL_ENV to control usage of a virtual environment
                 p = subprocess.run(
                     install_command,
                     cwd=dir,
@@ -245,10 +257,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
                     if b"is being installed using the legacy" in p.stderr:
                         # Ignore these types of errors
                         continue
-                    logger.error(f"Error installing into venv:\n{p.stderr.decode()}")
+                    logger.error(f"Error installing into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
                 else:
-                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
-                    venvs.append(venv_path)
+                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
+                    if create_venv:
+                        venvs.append(venv_path)
 
                 # Cleanup any build directories that are the product of package installation
                 expected_build_path = os.path.join(dir, "build")
@@ -292,15 +305,10 @@ def register_extractors(
 ):
     package_name = os.path.basename(current_directory)
     parent_directory = os.path.dirname(current_directory)
-    symlink = None
-    while package_name in sys.modules:
-        # Package name conflicts with an existing loaded module, let's deconflict that
-        package_name = f"_{package_name}"
-
-        # We'll need to create a link back to the original
-        if package_name not in sys.modules:
-            symlink = os.path.join(parent_directory, package_name)
-            os.symlink(current_directory, symlink)
+    if venvs and package_name in sys.modules:
+        # this may happen as part of testing if some part of the extractor code was directly imported
+        logger.warning(f"Looks like {package_name} is already loaded. "
+                       "If your maco extractor overlaps an existing package name this could cause problems.")
 
     try:
         # Modify the PATH so we can recognize this new package on import
@@ -351,10 +359,6 @@ def register_extractors(
         # Remove any modules that were loaded to deconflict with later modules loads
         [sys.modules.pop(k) for k in set(sys.modules.keys()) - default_loaded_modules]
 
-        # Cleanup any symlinks
-        if symlink:
-            os.remove(symlink)
-
     # If there still exists extractor files we haven't found yet, try searching in the available subdirectories
     if extractor_files:
         for dir in os.listdir(current_directory):
@@ -379,13 +383,21 @@ def register_extractors(
                 # We were able to find all the extractor files
                 break
 
+def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType, str], None], *args, **kwargs):
+    """Ensures logging is set up correctly for a child process and then executes the callback."""
+    logger = logging.getLogger()
+    qh = logging.handlers.QueueHandler(queue)
+    qh.setLevel(logging.DEBUG)
+    logger.addHandler(qh)
+    callback(*args, **kwargs, logger=logger)
 
 def import_extractors(
+    extractor_module_callback: Callable[[ModuleType, str], bool],
+    *,
     root_directory: str,
     scanner: yara.Rules,
-    extractor_module_callback: Callable[[ModuleType, str], bool],
+    create_venv: bool,
     logger: Logger,
-    create_venv: bool = False,
     python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
 ):
     extractor_dirs, extractor_files = scan_for_extractors(root_directory, scanner, logger)
@@ -393,23 +405,17 @@ def import_extractors(
     logger.info(f"Extractor files found based on scanner ({len(extractor_files)}).")
     logger.debug(extractor_files)
 
-    venvs = []
-    if create_venv:
-        venvs = create_virtual_environments(extractor_dirs, python_version, logger)
-    else:
-        # Look for pre-existing virtual environments, if any
-        logger.info("Checking for pre-existing virtual environment(s)..")
-        venvs = [
-            os.path.join(root, VENV_DIRECTORY_NAME)
-            for root, dirs, _ in os.walk(root_directory)
-            if VENV_DIRECTORY_NAME in dirs
-        ]
+    # Install packages into the current environment or dynamically created virtual environments
+    venvs = _install_required_packages(create_venv, extractor_dirs, python_version, logger)
 
     # With the environment prepared, we can now hunt for the extractors and register them
     logger.info("Registering extractors..")
     register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
 
 
+# holds cached extractors when not running in venv mode
+_loaded_extractors: Dict[str, Extractor] = {}
+
 def run_extractor(
     sample_path,
     module_name,
@@ -418,49 +424,69 @@ def run_extractor(
     venv,
     venv_script=VENV_SCRIPT,
     json_decoder=Base64Decoder,
-) -> Dict[str, dict]:
-    # Write temporary script in the same directory as extractor to resolve relative imports
-    python_exe = sys.executable
-    if venv:
-        # If there is a linked virtual environment, execute within that environment
+) -> Union[Dict[str, dict], model.ExtractorModel]:
+    """Runs the maco extractor against sample either in current process or child process."""
+    if not venv:
+        key = f"{module_name}_{extractor_class}"
+        if key not in _loaded_extractors:
+            # dynamic import of extractor
+            mod = importlib.import_module(module_name)
+            extractor_cls = mod.__getattribute__(extractor_class)
+            extractor = extractor_cls()
+        else:
+            # retrieve cached extractor
+            extractor = _loaded_extractors[key]
+        if extractor.yara_compiled:
+            matches = extractor.yara_compiled.match(sample_path)
+        loaded = extractor.run(open(sample_path, 'rb'), matches=matches)
+    else:
+        # execute extractor in child process with separate virtual environment
+        # Write temporary script in the same directory as extractor to resolve relative imports
         python_exe = os.path.join(venv, "bin", "python")
-    dirname = os.path.dirname(module_path)
-    with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
-        with tempfile.NamedTemporaryFile() as output:
-            parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
-            root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
-
-            script.write(
-                venv_script.format(
-                    parent_package_path=parent_package_path,
-                    module_name=module_name,
-                    module_class=extractor_class,
-                    sample_path=sample_path,
-                    output_path=output.name,
+        dirname = os.path.dirname(module_path)
+        with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
+            with tempfile.NamedTemporaryFile() as output:
+                parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
+                root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
+
+                script.write(
+                    venv_script.format(
+                        parent_package_path=parent_package_path,
+                        module_name=module_name,
+                        module_class=extractor_class,
+                        sample_path=sample_path,
+                        output_path=output.name,
+                    )
                 )
-            )
-            script.flush()
-            cwd = root_directory
-            custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
-
-            if custom_module.startswith("src."):
-                # src layout found, which means the actual module content is within 'src' directory
-                custom_module = custom_module[4:]
-                cwd = os.path.join(cwd, "src")
-
-            proc = subprocess.run(
-                [python_exe, "-m", custom_module],
-                cwd=cwd,
-                capture_output=True,
-            )
-            try:
-                # Load results and return them
-                output.seek(0)
-                return json.load(output, cls=json_decoder)
-            except Exception:
-                # If there was an error raised during runtime, then propagate
-                delim = f'File "{module_path}"'
-                exception = proc.stderr.decode()
-                if delim in exception:
-                    exception = f"{delim}{exception.split(delim, 1)[1]}"
-                raise Exception(exception)
+                script.flush()
+                cwd = root_directory
+                custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
+
+                if custom_module.startswith("src."):
+                    # src layout found, which means the actual module content is within 'src' directory
+                    custom_module = custom_module[4:]
+                    cwd = os.path.join(cwd, "src")
+
+                # run the maco extractor in full venv process isolation (slow)
+                proc = subprocess.run(
+                    [python_exe, "-m", custom_module],
+                    cwd=cwd,
+                    capture_output=True,
+                )
+                stderr = proc.stderr.decode()
+                try:
+                    # Load results and return them
+                    output.seek(0)
+                    loaded =  json.load(output, cls=json_decoder)
+                except Exception as e:
+                    # If there was an error raised during runtime, then propagate
+                    delim = f'File "{module_path}"'
+                    exception = stderr
+                    if delim in exception:
+                        exception = f"{delim}{exception.split(delim, 1)[1]}"
+                    # print extractor logging at error level
+                    logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
+                    raise Exception(exception) from e
+                # ensure that extractor logging is available
+                logger.info(f"maco extractor stderr:\n{stderr}")
+    return loaded

{maco-1.2.3.dist-info → maco-1.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.3
+Version: 1.2.5
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License

{maco-1.2.3.dist-info → maco-1.2.5.dist-info}/RECORD

@@ -1,25 +1,27 @@
+demo_extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 demo_extractors/elfy.py,sha256=AAFr5i1aivPwO4nycyXJEud57EpVNA-5k_2GicWesbY,771
-demo_extractors/limit_other.py,sha256=oscnNFkwD9Dm8Lae66GsRaAwUoTWUmkYpJu_wIsJ6sg,930
+demo_extractors/limit_other.py,sha256=8Z7X0cXUyZuK3MhDtObMWmdruRj5hgFdDi_VVGXqRx4,1123
 demo_extractors/nothing.py,sha256=3aeQJTY-dakmVXmyfmrRM8YCQVT7q3bq880DFH1Ol_Y,607
-demo_extractors/shared.py,sha256=Wlvy77SCAR97gxi8uUhGYyjxGmDb-pOSvN8b1rXrVWs,304
+demo_extractors/requirements.txt,sha256=E0tD6xBZldq6sQGTHng6k88lBeASOhmLJcdcjpcqBNE,6
+demo_extractors/shared.py,sha256=2P1cyuRbHDvM9IRt3UZnwdyhxx7OWqNC83xLyV8Y190,305
 demo_extractors/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-demo_extractors/complex/complex.py,sha256=TkNmR9UUYo1f2JVpJhGasLG-5wHZI05JYxMIXj16GKM,2307
+demo_extractors/complex/complex.py,sha256=tXrzj_zWIXbTOwj7Lezapk-qkrM-lfwcyjd5m-BYzdg,2322
 demo_extractors/complex/complex_utils.py,sha256=aec8kJsYUrMPo-waihkVLt-0QpiOPkw7dDqfT9MNuHk,123
 maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-maco/base_test.py,sha256=pqet9ofMwFRTj3JHgPdh9WHwdyp8kxNMi1vJNUzkSNA,2518
-maco/cli.py,sha256=1I3U54yPddTxqWclCtZ5Ma5hW6RoVTZMzLSFOjjfM1g,8008
-maco/collector.py,sha256=sxP374lK16lzdi0lhm5e4GT3a5lzeyE1VCicE4m_E3k,6003
-maco/extractor.py,sha256=4ZQd8OfvEQYUIkUS3LzZ5tcioembuLhT9_uRVNKSsyM,2750
-maco/utils.py,sha256=nyRTTsiwIE3L9XjFME6R_vR_EvLluwNL0kLb0r96jcg,18238
+maco/base_test.py,sha256=EPxCun9Tv91V-lFpaenn14tPyW17TPvXVH4AjE3t6js,2716
+maco/cli.py,sha256=fIeUXOgOxcecmAkl6OAdnjBKqk1gBPv1ryWe50pT60g,8135
+maco/collector.py,sha256=Vlo7KcJC7TKZFTElv8i_f_hvWEnlWCRzOP1xOc9x7vk,6532
+maco/extractor.py,sha256=uGSGiCQ4jd8jFmfw2T99BGcY5iQJzXHcG_RoTIxClTE,2802
+maco/utils.py,sha256=K41c-H7naaoiEYf0WNfP054IxwvHPujsbmmzgTizuLU,20159
 maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
 maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
 maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
 model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-model_setup/maco/base_test.py,sha256=pqet9ofMwFRTj3JHgPdh9WHwdyp8kxNMi1vJNUzkSNA,2518
-model_setup/maco/cli.py,sha256=1I3U54yPddTxqWclCtZ5Ma5hW6RoVTZMzLSFOjjfM1g,8008
-model_setup/maco/collector.py,sha256=sxP374lK16lzdi0lhm5e4GT3a5lzeyE1VCicE4m_E3k,6003
-model_setup/maco/extractor.py,sha256=4ZQd8OfvEQYUIkUS3LzZ5tcioembuLhT9_uRVNKSsyM,2750
-model_setup/maco/utils.py,sha256=nyRTTsiwIE3L9XjFME6R_vR_EvLluwNL0kLb0r96jcg,18238
+model_setup/maco/base_test.py,sha256=EPxCun9Tv91V-lFpaenn14tPyW17TPvXVH4AjE3t6js,2716
+model_setup/maco/cli.py,sha256=fIeUXOgOxcecmAkl6OAdnjBKqk1gBPv1ryWe50pT60g,8135
+model_setup/maco/collector.py,sha256=Vlo7KcJC7TKZFTElv8i_f_hvWEnlWCRzOP1xOc9x7vk,6532
+model_setup/maco/extractor.py,sha256=uGSGiCQ4jd8jFmfw2T99BGcY5iQJzXHcG_RoTIxClTE,2802
+model_setup/maco/utils.py,sha256=K41c-H7naaoiEYf0WNfP054IxwvHPujsbmmzgTizuLU,20159
 model_setup/maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
 model_setup/maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
 model_setup/maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
@@ -27,15 +29,16 @@ pipelines/publish.yaml,sha256=xt3WNU-5kIICJgKIiiE94M3dWjS3uEiun-n4OmIssK8,1471
 pipelines/test.yaml,sha256=3KOoo-8SqP_bTAscsz5V3xxnuL91J-62mTjnQD1Btag,1019
 tests/data/example.txt.cart,sha256=j4ZdDnFNVq7lb-Qi4pY4evOXKQPKG-GSg-n-uEqPhV0,289
 tests/data/trigger_complex.txt,sha256=uqnLSrnyDGCmXwuPmZ2s8vdhH0hJs8DxvyaW_tuYY24,64
+tests/data/trigger_complex.txt.cart,sha256=Z7qF1Zi640O45Znkl9ooP2RhSLAEqY0NRf51d-q7utU,345
 tests/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/extractors/basic.py,sha256=r5eLCL6Ynr14nCBgtbLvUbm0NdrXizyc9c-4xBCNShU,828
 tests/extractors/basic_longer.py,sha256=1ClU2QD-Y0TOl_loNFvEqIEpTR5TSVJ6zg9ZmC-ESJo,860
 tests/extractors/test_basic.py,sha256=FLKekfSGM69HaiF7Vu_7D7KDXHZko-9hZkMO8_DoyYA,697
 tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/extractors/bob/bob.py,sha256=Gy5p8KssJX87cwa9vVv8UBODF_ulbUteZXh15frW2hs,247
-maco-1.2.3.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
-maco-1.2.3.dist-info/METADATA,sha256=81_65WF-TCXVfF3NeY5m3zCr831F7PVsTswjvZ34Ok4,15610
-maco-1.2.3.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-maco-1.2.3.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
-maco-1.2.3.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
-maco-1.2.3.dist-info/RECORD,,
+maco-1.2.5.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
+maco-1.2.5.dist-info/METADATA,sha256=cJ7x_shBhDgKVjkq_e2d94aj3qiUzi0lt7f3lPO334U,15610
+maco-1.2.5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+maco-1.2.5.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
+maco-1.2.5.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
+maco-1.2.5.dist-info/RECORD,,

model_setup/maco/base_test.py

@@ -32,14 +32,19 @@ class BaseTest(unittest.TestCase):
     # I recommend something like os.path.join(__file__, "../../extractors")
     # if your extractors are in a folder 'extractors' next to a folder of tests
     path: str = None
+    create_venv: bool=False
 
-    def setUp(self) -> None:
-        if not self.name or not self.path:
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not cls.name or not cls.path:
             raise Exception("name and path must be set")
-        self.c = collector.Collector(self.path, include=[self.name])
+        cls.c = collector.Collector(cls.path, include=[cls.name], create_venv=cls.create_venv)
+        return super().setUpClass()
+
+    def test_default_metadata(self):
+        """Require extractor to be loadable and valid."""
         self.assertIn(self.name, self.c.extractors)
         self.assertEqual(len(self.c.extractors), 1)
-        return super().setUp()
 
     def extract(self, stream):
         """Return results for running extractor over stream, including yara check."""
@@ -49,18 +54,20 @@ class BaseTest(unittest.TestCase):
         resp = self.c.extract(stream, self.name)
         return resp
 
-    def _get_location(self) -> str:
+    @classmethod
+    def _get_location(cls) -> str:
         """Return path to child class that implements this class."""
         # import child module
-        module = type(self).__module__
+        module = cls.__module__
         i = importlib.import_module(module)
         # get location to child module
         return i.__file__
 
-    def load_cart(self, filepath: str) -> io.BytesIO:
+    @classmethod
+    def load_cart(cls, filepath: str) -> io.BytesIO:
         """Load and unneuter a test file (likely malware) into memory for processing."""
         # it is nice if we can load files relative to whatever is implementing base_test
-        dirpath = os.path.split(self._get_location())[0]
+        dirpath = os.path.split(cls._get_location())[0]
         # either filepath is absolute, or should be loaded relative to child of base_test
         filepath = os.path.join(dirpath, filepath)
         if not os.path.isfile(filepath):

model_setup/maco/cli.py

@@ -179,7 +179,7 @@ def main():
     parser.add_argument(
         "--create_venv",
         action="store_true",
-        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory)",
+        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). This runs much slower than the alternative but may be necessary when there are many extractors with conflicting dependencies.",
     )
     args = parser.parse_args()
     inc = args.include.split(",") if args.include else []

model_setup/maco/collector.py

@@ -2,8 +2,9 @@
 
 import inspect
 import logging
+import logging.handlers
 import os
-from multiprocessing import Manager, Process
+from multiprocessing import Manager, Process, Queue
 from tempfile import NamedTemporaryFile
 from types import ModuleType
 from typing import Any, BinaryIO, Dict, List, Union
@@ -86,21 +87,33 @@ class Collector:
                     )
                     namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
 
+            # multiprocess logging is awkward - set up a queue to ensure we can log
+            logging_queue = Queue()
+            queue_handler = logging.handlers.QueueListener(logging_queue,*logging.getLogger().handlers)
+            queue_handler.start()
+
             # Find the extractors within the given directory
             # Execute within a child process to ensure main process interpreter is kept clean
             p = Process(
-                target=utils.import_extractors,
+                target=utils.proxy_logging,
                 args=(
-                    path_extractors,
-                    yara.compile(source=utils.MACO_YARA_RULE),
+                    logging_queue,
+                    utils.import_extractors,
                     extractor_module_callback,
-                    logger,
-                    create_venv and os.path.isdir(path_extractors),
+                ),
+                kwargs=dict(
+                    root_directory=path_extractors,
+                    scanner=yara.compile(source=utils.MACO_YARA_RULE),
+                    create_venv=create_venv and os.path.isdir(path_extractors),
                 ),
             )
             p.start()
             p.join()
 
+            # stop multiprocess logging
+            queue_handler.stop()
+            logging_queue.close()
+
             self.extractors = dict(extractors)
             if not self.extractors:
                 raise ExtractorLoadError("no extractors were loaded")

model_setup/maco/extractor.py

@@ -51,14 +51,14 @@ class Extractor:
         # check yara rules conform to expected structure
         # we throw away these compiled rules as we need all rules in system compiled together
         try:
-            rules = yara.compile(source=self.yara_rule)
+            self.yara_compiled = yara.compile(source=self.yara_rule)
         except yara.SyntaxError as e:
             raise InvalidExtractor(f"{self.name} - invalid yara rule") from e
         # need to track which plugin owns the rules
-        self.yara_rule_names = [x.identifier for x in rules]
-        if not len(list(rules)):
+        self.yara_rule_names = [x.identifier for x in self.yara_compiled]
+        if not len(list(self.yara_compiled)):
             raise InvalidExtractor(f"{name} must define at least one yara rule")
-        for x in rules:
+        for x in self.yara_compiled:
             if x.is_global:
                 raise InvalidExtractor(f"{x.identifier} yara rule must not be global")
 

model_setup/maco/utils.py

@@ -4,6 +4,9 @@ import importlib.machinery
 import importlib.util
 import inspect
 import json
+import logging
+import logging.handlers
+import multiprocessing
 import os
 import re
 import shutil
@@ -24,32 +27,24 @@ from glob import glob
 from logging import Logger
 from pkgutil import walk_packages
 from types import ModuleType
-from typing import Callable, Dict, List, Set, Tuple
+from typing import Callable, Dict, List, Set, Tuple, Union
 
-from maco.extractor import Extractor
-
-VENV_DIRECTORY_NAME = ".venv"
+from uv import find_uv_bin
 
-RELATIVE_FROM_RE = re.compile("from (\.+)")
-RELATIVE_FROM_IMPORT_RE = re.compile("from (\.+) import")
+from maco import model
+from maco.extractor import Extractor
 
-try:
-    # Attempt to use the uv package manager (Recommended)
-    from uv import find_uv_bin
+logger = logging.getLogger("maco.lib.utils")
 
-    UV_BIN = find_uv_bin()
+VENV_DIRECTORY_NAME = ".venv"
 
-    PIP_CMD = f"{UV_BIN} pip"
-    VENV_CREATE_CMD = f"{UV_BIN} venv"
-    PACKAGE_MANAGER = "uv"
-except ImportError:
-    # Otherwise default to pip
-    from sys import executable
+RELATIVE_FROM_RE = re.compile(r"from (\.+)")
+RELATIVE_FROM_IMPORT_RE = re.compile(r"from (\.+) import")
 
-    PIP_CMD = "pip"
-    VENV_CREATE_CMD = f"{executable} -m venv"
-    PACKAGE_MANAGER = "pip"
+UV_BIN = find_uv_bin()
 
+PIP_CMD = f"{UV_BIN} pip"
+VENV_CREATE_CMD = f"{UV_BIN} venv"
 
 class Base64Decoder(json.JSONDecoder):
     def __init__(self, *args, **kwargs):
@@ -69,6 +64,7 @@ import importlib
 import json
 import os
 import sys
+import logging
 
 try:
     from maco import yara
@@ -76,6 +72,19 @@ except:
     import yara
 
 from base64 import b64encode
+
+# ensure we have a logger to stderr
+import logging
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+sh = logging.StreamHandler()
+logger.addHandler(sh)
+sh.setLevel(logging.DEBUG)
+formatter = logging.Formatter(
+    fmt="%(asctime)s, [%(levelname)s] %(module)s.%(funcName)s: %(message)s", datefmt="%Y-%m-%d (%H:%M:%S)"
+)
+sh.setFormatter(formatter)
+
 parent_package_path = "{parent_package_path}"
 sys.path.insert(1, parent_package_path)
 mod = importlib.import_module("{module_name}")
@@ -101,7 +110,7 @@ with open("{output_path}", 'w') as fp:
             json.dump(result.dict(exclude_defaults=True, exclude_none=True), fp, cls=Base64Encoder)
 """
 
-MACO_YARA_RULE = """
+MACO_YARA_RULE = r"""
 rule MACO {
     meta:
         desc = "Used to match on Python files that contain MACO extractors"
@@ -191,9 +200,8 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
     return extractor_dirs, extractor_files
 
 
-def create_virtual_environments(directories: List[str], python_version: str, logger: Logger):
+def _install_required_packages(create_venv: bool, directories: List[str], python_version: str, logger: Logger):
     venvs = []
-    logger.info("Creating virtual environment(s)..")
     env = deepcopy(os.environ)
     stop_directory = os.path.dirname(sorted(directories)[0])
     # Track directories that we've already visited
@@ -203,14 +211,15 @@ def create_virtual_environments(directories: List[str], python_version: str, log
         while dir != stop_directory and dir not in visited_dirs:
             req_files = list({"requirements.txt", "pyproject.toml"}.intersection(set(os.listdir(dir))))
             if req_files:
-                venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
-                env.update({"VIRTUAL_ENV": venv_path})
-                # Create a virtual environment for the directory
-                if not os.path.exists(venv_path):
-                    cmd = VENV_CREATE_CMD
-                    if PACKAGE_MANAGER == "uv":
-                        cmd += f" --python {python_version}"
-                    subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
+                # create a virtual environment, otherwise directly install into current env
+                if create_venv:
+                    venv_path = os.path.join(dir, VENV_DIRECTORY_NAME)
+                    logger.info(f"Updating virtual environment {venv_path}")
+                    env.update({"VIRTUAL_ENV": venv_path})
+                    # Create a virtual environment for the directory
+                    if not os.path.exists(venv_path):
+                        cmd = f"{VENV_CREATE_CMD} --python {python_version}"
+                        subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
 
                 # Install/Update the packages in the environment
                 install_command = PIP_CMD.split(" ") + ["install", "-U"]
@@ -234,7 +243,10 @@ def create_virtual_environments(directories: List[str], python_version: str, log
 
                     install_command.extend(pyproject_command)
 
+                # always require maco to be installed
+                install_command.append("maco")
                 logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
+                # this uses VIRTUAL_ENV to control usage of a virtual environment
                 p = subprocess.run(
                     install_command,
                     cwd=dir,
@@ -245,10 +257,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
                     if b"is being installed using the legacy" in p.stderr:
                         # Ignore these types of errors
                         continue
-                    logger.error(f"Error installing into venv:\n{p.stderr.decode()}")
+                    logger.error(f"Error installing into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
                 else:
-                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
-                    venvs.append(venv_path)
+                    logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}\n{p.stderr.decode()}")
+                    if create_venv:
+                        venvs.append(venv_path)
 
                 # Cleanup any build directories that are the product of package installation
                 expected_build_path = os.path.join(dir, "build")
@@ -292,15 +305,10 @@ def register_extractors(
 ):
     package_name = os.path.basename(current_directory)
     parent_directory = os.path.dirname(current_directory)
-    symlink = None
-    while package_name in sys.modules:
-        # Package name conflicts with an existing loaded module, let's deconflict that
-        package_name = f"_{package_name}"
-
-        # We'll need to create a link back to the original
-        if package_name not in sys.modules:
-            symlink = os.path.join(parent_directory, package_name)
-            os.symlink(current_directory, symlink)
+    if venvs and package_name in sys.modules:
+        # this may happen as part of testing if some part of the extractor code was directly imported
+        logger.warning(f"Looks like {package_name} is already loaded. "
+                       "If your maco extractor overlaps an existing package name this could cause problems.")
 
     try:
         # Modify the PATH so we can recognize this new package on import
@@ -351,10 +359,6 @@ def register_extractors(
         # Remove any modules that were loaded to deconflict with later modules loads
         [sys.modules.pop(k) for k in set(sys.modules.keys()) - default_loaded_modules]
 
-        # Cleanup any symlinks
-        if symlink:
-            os.remove(symlink)
-
     # If there still exists extractor files we haven't found yet, try searching in the available subdirectories
     if extractor_files:
         for dir in os.listdir(current_directory):
@@ -379,13 +383,21 @@ def register_extractors(
                 # We were able to find all the extractor files
                 break
 
+def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType, str], None], *args, **kwargs):
+    """Ensures logging is set up correctly for a child process and then executes the callback."""
+    logger = logging.getLogger()
+    qh = logging.handlers.QueueHandler(queue)
+    qh.setLevel(logging.DEBUG)
+    logger.addHandler(qh)
+    callback(*args, **kwargs, logger=logger)
 
 def import_extractors(
+    extractor_module_callback: Callable[[ModuleType, str], bool],
+    *,
     root_directory: str,
     scanner: yara.Rules,
-    extractor_module_callback: Callable[[ModuleType, str], bool],
+    create_venv: bool,
     logger: Logger,
-    create_venv: bool = False,
     python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
 ):
     extractor_dirs, extractor_files = scan_for_extractors(root_directory, scanner, logger)
@@ -393,23 +405,17 @@ def import_extractors(
     logger.info(f"Extractor files found based on scanner ({len(extractor_files)}).")
     logger.debug(extractor_files)
 
-    venvs = []
-    if create_venv:
-        venvs = create_virtual_environments(extractor_dirs, python_version, logger)
-    else:
-        # Look for pre-existing virtual environments, if any
-        logger.info("Checking for pre-existing virtual environment(s)..")
-        venvs = [
-            os.path.join(root, VENV_DIRECTORY_NAME)
-            for root, dirs, _ in os.walk(root_directory)
-            if VENV_DIRECTORY_NAME in dirs
-        ]
+    # Install packages into the current environment or dynamically created virtual environments
+    venvs = _install_required_packages(create_venv, extractor_dirs, python_version, logger)
 
     # With the environment prepared, we can now hunt for the extractors and register them
     logger.info("Registering extractors..")
     register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
 
 
+# holds cached extractors when not running in venv mode
+_loaded_extractors: Dict[str, Extractor] = {}
+
 def run_extractor(
     sample_path,
     module_name,
@@ -418,49 +424,69 @@ def run_extractor(
     venv,
     venv_script=VENV_SCRIPT,
     json_decoder=Base64Decoder,
-) -> Dict[str, dict]:
-    # Write temporary script in the same directory as extractor to resolve relative imports
-    python_exe = sys.executable
-    if venv:
-        # If there is a linked virtual environment, execute within that environment
+) -> Union[Dict[str, dict], model.ExtractorModel]:
+    """Runs the maco extractor against sample either in current process or child process."""
+    if not venv:
+        key = f"{module_name}_{extractor_class}"
+        if key not in _loaded_extractors:
+            # dynamic import of extractor
+            mod = importlib.import_module(module_name)
+            extractor_cls = mod.__getattribute__(extractor_class)
+            extractor = extractor_cls()
+        else:
+            # retrieve cached extractor
+            extractor = _loaded_extractors[key]
+        if extractor.yara_compiled:
+            matches = extractor.yara_compiled.match(sample_path)
+        loaded = extractor.run(open(sample_path, 'rb'), matches=matches)
+    else:
+        # execute extractor in child process with separate virtual environment
+        # Write temporary script in the same directory as extractor to resolve relative imports
         python_exe = os.path.join(venv, "bin", "python")
-    dirname = os.path.dirname(module_path)
-    with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
-        with tempfile.NamedTemporaryFile() as output:
-            parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
-            root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
-
-            script.write(
-                venv_script.format(
-                    parent_package_path=parent_package_path,
-                    module_name=module_name,
-                    module_class=extractor_class,
-                    sample_path=sample_path,
-                    output_path=output.name,
+        dirname = os.path.dirname(module_path)
+        with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
+            with tempfile.NamedTemporaryFile() as output:
+                parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
+                root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
+
+                script.write(
+                    venv_script.format(
+                        parent_package_path=parent_package_path,
+                        module_name=module_name,
+                        module_class=extractor_class,
+                        sample_path=sample_path,
+                        output_path=output.name,
+                    )
                 )
-            )
-            script.flush()
-            cwd = root_directory
-            custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
-
-            if custom_module.startswith("src."):
-                # src layout found, which means the actual module content is within 'src' directory
-                custom_module = custom_module[4:]
-                cwd = os.path.join(cwd, "src")
-
-            proc = subprocess.run(
-                [python_exe, "-m", custom_module],
-                cwd=cwd,
-                capture_output=True,
-            )
-            try:
-                # Load results and return them
-                output.seek(0)
-                return json.load(output, cls=json_decoder)
-            except Exception:
-                # If there was an error raised during runtime, then propagate
-                delim = f'File "{module_path}"'
-                exception = proc.stderr.decode()
-                if delim in exception:
-                    exception = f"{delim}{exception.split(delim, 1)[1]}"
-                raise Exception(exception)
+                script.flush()
+                cwd = root_directory
+                custom_module = script.name[:-3].replace(root_directory, "").replace("/", ".")
+
+                if custom_module.startswith("src."):
+                    # src layout found, which means the actual module content is within 'src' directory
+                    custom_module = custom_module[4:]
+                    cwd = os.path.join(cwd, "src")
+
+                # run the maco extractor in full venv process isolation (slow)
+                proc = subprocess.run(
+                    [python_exe, "-m", custom_module],
+                    cwd=cwd,
+                    capture_output=True,
+                )
+                stderr = proc.stderr.decode()
+                try:
+                    # Load results and return them
+                    output.seek(0)
+                    loaded =  json.load(output, cls=json_decoder)
+                except Exception as e:
+                    # If there was an error raised during runtime, then propagate
+                    delim = f'File "{module_path}"'
+                    exception = stderr
+                    if delim in exception:
+                        exception = f"{delim}{exception.split(delim, 1)[1]}"
+                    # print extractor logging at error level
+                    logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
+                    raise Exception(exception) from e
+                # ensure that extractor logging is available
+                logger.info(f"maco extractor stderr:\n{stderr}")
+    return loaded