PyPI - maco - Versions diffs - 1.2.0__py3-none-any.whl → 1.2.1__py3-none-any.whl - Mend

maco 1.2.0py3-none-any.whl → 1.2.1py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (16) hide show

demo_extractors/complex/complex.py +3 -3
maco/collector.py +66 -60
maco/utils.py +21 -8
maco/yara.py +4 -4
{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/METADATA +4 -5
{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/RECORD +16 -15
{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/WHEEL +1 -1
model_setup/maco/collector.py +66 -60
model_setup/maco/utils.py +21 -8
model_setup/maco/yara.py +4 -4
pipelines/publish.yaml +7 -7
pipelines/test.yaml +14 -10
tests/data/trigger_complex.txt +6 -0
{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/LICENSE.md +0 -0
{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/entry_points.txt +0 -0
{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/top_level.txt +0 -0

demo_extractors/complex/complex.py CHANGED Viewed

@@ -1,9 +1,9 @@
 from io import BytesIO
-from typing import Dict, List, Optional
+from typing import List, Optional
 from maco import extractor, model, yara
-from . import complex_utils
+from complex import complex_utils
 class Complex(extractor.Extractor):
@@ -50,7 +50,7 @@ class Complex(extractor.Extractor):
         other = complex_utils.getdata()["result"]
         self.logger.debug("got data from lib")
         # example - accessing yara strings
-        strings = {y[2].decode("utf8") for x in matches for y in x.strings}
+        strings = sorted({z.plaintext().decode("utf8") for x in matches for y in x.strings for z in y.instances})
         self.logger.debug(f"{strings=}")
         # construct model of results
         tmp = model.ExtractorModel(family=self.family)

maco/collector.py CHANGED Viewed

@@ -3,15 +3,14 @@
 import inspect
 import logging
 import os
+from multiprocessing import Manager, Process
 from tempfile import NamedTemporaryFile
-from typing import Any, BinaryIO, Dict, List
 from types import ModuleType
+from typing import Any, BinaryIO, Dict, List, Union
-from maco import yara
 from pydantic import BaseModel
-from maco import extractor, model, utils
+from maco import extractor, model, utils, yara
 class ExtractorLoadError(Exception):
@@ -21,12 +20,14 @@ class ExtractorLoadError(Exception):
 logger = logging.getLogger("maco.lib.helpers")
-def _verify_response(resp: BaseModel) -> Dict:
+def _verify_response(resp: Union[BaseModel, dict]) -> Dict:
     """Enforce types and verify properties, and remove defaults."""
+    if not resp:
+        return None
     # check the response is valid for its own model
     # this is useful if a restriction on the 'other' dictionary is needed
     resp_model = type(resp)
-    if resp_model != model.ExtractorModel:
+    if resp_model != model.ExtractorModel and hasattr(resp_model, "model_validate"):
         resp = resp_model.model_validate(resp)
     # check the response is valid according to the ExtractorModel
     resp = model.ExtractorModel.model_validate(resp)
@@ -43,44 +44,59 @@ class Collector:
     ):
         """Discover and load extractors from file system."""
         path_extractors = os.path.realpath(path_extractors)
-        self.path = path_extractors
-        self.extractors = {}
-        namespaced_rules = {}
-        def extractor_module_callback(module: ModuleType, venv: str):
-            members = inspect.getmembers(module, predicate=utils.maco_extractor_validation)
-            for member in members:
-                name, member = member
-                if exclude and name in exclude:
-                    # Module is part of the exclusion list, skip
-                    logger.debug(f"exclude excluded '{name}'")
-                    return
-                if include and name not in include:
-                    # Module wasn't part of the inclusion list, skip
-                    logger.debug(f"include excluded '{name}'")
-                    return
-                # initialise and register
-                logger.debug(f"register '{name}'")
-                self.extractors[name] = dict(module=member, venv=venv, module_path=module.__file__)
-                namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
-        # Find the extractors within the given directory
-        utils.import_extractors(
-            path_extractors,
-            yara.compile(source=utils.MACO_YARA_RULE),
-            extractor_module_callback,
-            logger,
-            create_venv and os.path.isdir(path_extractors),
-        )
-        if not self.extractors:
-            raise ExtractorLoadError("no extractors were loaded")
-        logger.debug(f"found extractors {list(self.extractors.keys())}\n")
-        # compile yara rules gathered from extractors
-        self.rules = yara.compile(sources=namespaced_rules)
+        self.path: str = path_extractors
+        self.extractors: Dict[str, Dict[str, str]] = {}
+        with Manager() as manager:
+            extractors = manager.dict()
+            namespaced_rules = manager.dict()
+            def extractor_module_callback(module: ModuleType, venv: str):
+                members = inspect.getmembers(module, predicate=utils.maco_extractor_validation)
+                for member in members:
+                    name, member = member
+                    if exclude and name in exclude:
+                        # Module is part of the exclusion list, skip
+                        logger.debug(f"exclude excluded '{name}'")
+                        return
+                    if include and name not in include:
+                        # Module wasn't part of the inclusion list, skip
+                        logger.debug(f"include excluded '{name}'")
+                        return
+                    # initialise and register
+                    logger.debug(f"register '{name}'")
+                    extractors[name] = dict(
+                        venv=venv,
+                        module_path=module.__file__,
+                        module_name=member.__module__,
+                        extractor_class=member.__name__,
+                    )
+                    namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
+            # Find the extractors within the given directory
+            # Execute within a child process to ensure main process interpreter is kept clean
+            p = Process(
+                target=utils.import_extractors,
+                args=(
+                    path_extractors,
+                    yara.compile(source=utils.MACO_YARA_RULE),
+                    extractor_module_callback,
+                    logger,
+                    create_venv and os.path.isdir(path_extractors),
+                ),
+            )
+            p.start()
+            p.join()
+            self.extractors = dict(extractors)
+            if not self.extractors:
+                raise ExtractorLoadError("no extractors were loaded")
+            logger.debug(f"found extractors {list(self.extractors.keys())}\n")
+            # compile yara rules gathered from extractors
+            self.rules = yara.compile(sources=dict(namespaced_rules))
     def match(self, stream: BinaryIO) -> Dict[str, List[yara.Match]]:
         """Return extractors that should run based on yara rules."""
@@ -105,17 +121,13 @@ class Collector:
     ) -> Dict[str, Any]:
         """Run extractor with stream and verify output matches the model."""
         extractor = self.extractors[extractor_name]
-        resp = None
         try:
-            if extractor["venv"]:
-                # Run extractor within a virtual environment
-                with NamedTemporaryFile() as sample_path:
-                    sample_path.write(stream.read())
-                    sample_path.flush()
-                    return utils.run_in_venv(sample_path.name, **extractor)
-            else:
-                # Run extractor within on host environment
-                resp = extractor["module"]().run(stream, matches)
+            # Run extractor on a copy of the sample
+            with NamedTemporaryFile() as sample_path:
+                sample_path.write(stream.read())
+                sample_path.flush()
+                # enforce types and verify properties, and remove defaults
+                return _verify_response(utils.run_extractor(sample_path.name, **extractor))
         except Exception:
             # caller can deal with the exception
             raise
@@ -123,9 +135,3 @@ class Collector:
             # make sure to reset where we are in the file
             # otherwise follow on extractors are going to read 0 bytes
             stream.seek(0)
-        # enforce types and verify properties, and remove defaults
-        if resp is not None:
-            resp = _verify_response(resp)
-        return resp

maco/utils.py CHANGED Viewed

@@ -6,9 +6,11 @@ import inspect
 import json
 import os
 import re
+import shutil
 import subprocess
 import sys
 import tempfile
 from maco import yara
 if sys.version_info >= (3, 11):
@@ -21,8 +23,8 @@ from copy import deepcopy
 from glob import glob
 from logging import Logger
 from pkgutil import walk_packages
-from typing import Callable, Dict, Tuple, List, Set
 from types import ModuleType
+from typing import Callable, Dict, List, Set, Tuple
 from maco.extractor import Extractor
@@ -67,7 +69,11 @@ import importlib
 import json
 import os
 import sys
-import yara
+try:
+    from maco import yara
+except:
+    import yara
 from base64 import b64encode
 parent_package_path = "{parent_package_path}"
@@ -244,6 +250,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
                     logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
                     venvs.append(venv_path)
+                # Cleanup any build directories that are the product of package installation
+                expected_build_path = os.path.join(dir, "build")
+                if os.path.exists(expected_build_path):
+                    shutil.rmtree(expected_build_path)
             # Add directories to our visited list and check the parent of this directory on the next loop
             visited_dirs.append(dir)
             dir = os.path.dirname(dir)
@@ -399,21 +410,23 @@ def import_extractors(
     register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
-def run_in_venv(
+def run_extractor(
     sample_path,
-    module,
+    module_name,
+    extractor_class,
     module_path,
     venv,
     venv_script=VENV_SCRIPT,
     json_decoder=Base64Decoder,
 ) -> Dict[str, dict]:
     # Write temporary script in the same directory as extractor to resolve relative imports
-    python_exe = os.path.join(venv, "bin", "python")
+    python_exe = sys.executable
+    if venv:
+        # If there is a linked virtual environment, execute within that environment
+        python_exe = os.path.join(venv, "bin", "python")
     dirname = os.path.dirname(module_path)
     with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
         with tempfile.NamedTemporaryFile() as output:
-            module_name = module.__module__
-            module_class = module.__name__
             parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
             root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
@@ -421,7 +434,7 @@ def run_in_venv(
                 venv_script.format(
                     parent_package_path=parent_package_path,
                     module_name=module_name,
-                    module_class=module_class,
+                    module_class=extractor_class,
                     sample_path=sample_path,
                     output_path=output.name,
                 )

maco/yara.py CHANGED Viewed

@@ -1,11 +1,11 @@
 import re
-import yara
-import yara_x
 from collections import namedtuple
 from itertools import cycle
 from typing import Dict
+import yara
+import yara_x
 RULE_ID_RE = re.compile("(\w+)? ?rule (\w+)")
@@ -42,7 +42,7 @@ class Match:
     def __init__(self, rule: yara_x.Rule, file_content: bytes):
         self.rule = rule.identifier
         self.namespace = rule.namespace
-        self.tags = rule.tags if hasattr(rule, "tags") else []
+        self.tags = list(rule.tags) or []
         self.meta = dict()
         # Ensure metadata doesn't get overwritten
         for k, v in rule.metadata:

{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/METADATA RENAMED Viewed

@@ -1,8 +1,7 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.0
+Version: 1.2.1
 Author: sl-govau
-Author-email:
 Maintainer: cccs-rs
 License: MIT License
@@ -31,11 +30,11 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: cart
-Requires-Dist: pydantic >=2.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: tomli>=1.1.0; python_version < "3.11"
 Requires-Dist: uv
 Requires-Dist: yara-python
-Requires-Dist: yara-x ==0.10.0
-Requires-Dist: tomli >=1.1.0 ; python_version < "3.11"
+Requires-Dist: yara-x==0.11.0
 # Maco - Malware config extractor framework

{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/RECORD RENAMED Viewed

@@ -3,38 +3,39 @@ demo_extractors/limit_other.py,sha256=oscnNFkwD9Dm8Lae66GsRaAwUoTWUmkYpJu_wIsJ6s
 demo_extractors/nothing.py,sha256=3aeQJTY-dakmVXmyfmrRM8YCQVT7q3bq880DFH1Ol_Y,607
 demo_extractors/shared.py,sha256=Wlvy77SCAR97gxi8uUhGYyjxGmDb-pOSvN8b1rXrVWs,304
 demo_extractors/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-demo_extractors/complex/complex.py,sha256=U4yVLFTBeW9ORyL7Q4Hu2vzJPSCFgR9u--lIjMl2tZU,2269
+demo_extractors/complex/complex.py,sha256=TkNmR9UUYo1f2JVpJhGasLG-5wHZI05JYxMIXj16GKM,2307
 demo_extractors/complex/complex_utils.py,sha256=aec8kJsYUrMPo-waihkVLt-0QpiOPkw7dDqfT9MNuHk,123
 maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 maco/base_test.py,sha256=7uZTprPoFZRnPaO_hLkMGz5rVW9F9TozAUWtNeXouig,2555
 maco/cli.py,sha256=56xM2qqLNlvQgWBF3Uc1fDbG8_46PSx66Wqj2mMxl1E,7713
-maco/collector.py,sha256=Ja8WaoVVr00zMvGFwIJ5TtVV3f5LxMukbGcP13QQnPc,4811
+maco/collector.py,sha256=uBk_RrnJAoiQkBxnNA-B7zIPj9fDm5gkFb4D_VyMWi8,5342
 maco/extractor.py,sha256=4ZQd8OfvEQYUIkUS3LzZ5tcioembuLhT9_uRVNKSsyM,2750
-maco/utils.py,sha256=JyAeuZpz_1Zm-hTUl54fVuI6FAoGOtcQel5zYQlKdXU,17826
-maco/yara.py,sha256=9YnN8Q0XAOJSnrQLuvBYZmwq7ROzDMICjlQfq0rU8rg,2946
+maco/utils.py,sha256=lkZ4yb-LjkzjVpyY6INNWKyimYg5k3Y2N_Hr9CdrFbw,18232
+maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
 maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
 maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
 model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 model_setup/maco/base_test.py,sha256=7uZTprPoFZRnPaO_hLkMGz5rVW9F9TozAUWtNeXouig,2555
 model_setup/maco/cli.py,sha256=56xM2qqLNlvQgWBF3Uc1fDbG8_46PSx66Wqj2mMxl1E,7713
-model_setup/maco/collector.py,sha256=Ja8WaoVVr00zMvGFwIJ5TtVV3f5LxMukbGcP13QQnPc,4811
+model_setup/maco/collector.py,sha256=uBk_RrnJAoiQkBxnNA-B7zIPj9fDm5gkFb4D_VyMWi8,5342
 model_setup/maco/extractor.py,sha256=4ZQd8OfvEQYUIkUS3LzZ5tcioembuLhT9_uRVNKSsyM,2750
-model_setup/maco/utils.py,sha256=JyAeuZpz_1Zm-hTUl54fVuI6FAoGOtcQel5zYQlKdXU,17826
-model_setup/maco/yara.py,sha256=9YnN8Q0XAOJSnrQLuvBYZmwq7ROzDMICjlQfq0rU8rg,2946
+model_setup/maco/utils.py,sha256=lkZ4yb-LjkzjVpyY6INNWKyimYg5k3Y2N_Hr9CdrFbw,18232
+model_setup/maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
 model_setup/maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
 model_setup/maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
-pipelines/publish.yaml,sha256=_1vDLcsOwDM3mIZCEvhB7UJOvyHvyiOSjg3BmmrYsWM,1483
-pipelines/test.yaml,sha256=WNeL3ZSvzYDbTas_4vqzLRwMingo63cUGgU5y-iMsN4,1288
+pipelines/publish.yaml,sha256=xt3WNU-5kIICJgKIiiE94M3dWjS3uEiun-n4OmIssK8,1471
+pipelines/test.yaml,sha256=3KOoo-8SqP_bTAscsz5V3xxnuL91J-62mTjnQD1Btag,1019
 tests/data/example.txt.cart,sha256=j4ZdDnFNVq7lb-Qi4pY4evOXKQPKG-GSg-n-uEqPhV0,289
+tests/data/trigger_complex.txt,sha256=uqnLSrnyDGCmXwuPmZ2s8vdhH0hJs8DxvyaW_tuYY24,64
 tests/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/extractors/basic.py,sha256=r5eLCL6Ynr14nCBgtbLvUbm0NdrXizyc9c-4xBCNShU,828
 tests/extractors/basic_longer.py,sha256=1ClU2QD-Y0TOl_loNFvEqIEpTR5TSVJ6zg9ZmC-ESJo,860
 tests/extractors/test_basic.py,sha256=FLKekfSGM69HaiF7Vu_7D7KDXHZko-9hZkMO8_DoyYA,697
 tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/extractors/bob/bob.py,sha256=Gy5p8KssJX87cwa9vVv8UBODF_ulbUteZXh15frW2hs,247
-maco-1.2.0.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
-maco-1.2.0.dist-info/METADATA,sha256=pqS4bzX_eaOjO3XuJKWYrRz49UnF32Xq4sYHKU1hRBc,15629
-maco-1.2.0.dist-info/WHEEL,sha256=a7TGlA-5DaHMRrarXjVbQagU3Man_dCnGIWMJr5kRWo,91
-maco-1.2.0.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
-maco-1.2.0.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
-maco-1.2.0.dist-info/RECORD,,
+maco-1.2.1.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
+maco-1.2.1.dist-info/METADATA,sha256=iaJIFA_TcNHwAxT6ysbTLSaRmLib7lF2BC3IBXOoQg4,15610
+maco-1.2.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+maco-1.2.1.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
+maco-1.2.1.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
+maco-1.2.1.dist-info/RECORD,,

{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/WHEEL RENAMED Viewed

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.4.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any

model_setup/maco/collector.py CHANGED Viewed

@@ -3,15 +3,14 @@
 import inspect
 import logging
 import os
+from multiprocessing import Manager, Process
 from tempfile import NamedTemporaryFile
-from typing import Any, BinaryIO, Dict, List
 from types import ModuleType
+from typing import Any, BinaryIO, Dict, List, Union
-from maco import yara
 from pydantic import BaseModel
-from maco import extractor, model, utils
+from maco import extractor, model, utils, yara
 class ExtractorLoadError(Exception):
@@ -21,12 +20,14 @@ class ExtractorLoadError(Exception):
 logger = logging.getLogger("maco.lib.helpers")
-def _verify_response(resp: BaseModel) -> Dict:
+def _verify_response(resp: Union[BaseModel, dict]) -> Dict:
     """Enforce types and verify properties, and remove defaults."""
+    if not resp:
+        return None
     # check the response is valid for its own model
     # this is useful if a restriction on the 'other' dictionary is needed
     resp_model = type(resp)
-    if resp_model != model.ExtractorModel:
+    if resp_model != model.ExtractorModel and hasattr(resp_model, "model_validate"):
         resp = resp_model.model_validate(resp)
     # check the response is valid according to the ExtractorModel
     resp = model.ExtractorModel.model_validate(resp)
@@ -43,44 +44,59 @@ class Collector:
     ):
         """Discover and load extractors from file system."""
         path_extractors = os.path.realpath(path_extractors)
-        self.path = path_extractors
-        self.extractors = {}
-        namespaced_rules = {}
-        def extractor_module_callback(module: ModuleType, venv: str):
-            members = inspect.getmembers(module, predicate=utils.maco_extractor_validation)
-            for member in members:
-                name, member = member
-                if exclude and name in exclude:
-                    # Module is part of the exclusion list, skip
-                    logger.debug(f"exclude excluded '{name}'")
-                    return
-                if include and name not in include:
-                    # Module wasn't part of the inclusion list, skip
-                    logger.debug(f"include excluded '{name}'")
-                    return
-                # initialise and register
-                logger.debug(f"register '{name}'")
-                self.extractors[name] = dict(module=member, venv=venv, module_path=module.__file__)
-                namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
-        # Find the extractors within the given directory
-        utils.import_extractors(
-            path_extractors,
-            yara.compile(source=utils.MACO_YARA_RULE),
-            extractor_module_callback,
-            logger,
-            create_venv and os.path.isdir(path_extractors),
-        )
-        if not self.extractors:
-            raise ExtractorLoadError("no extractors were loaded")
-        logger.debug(f"found extractors {list(self.extractors.keys())}\n")
-        # compile yara rules gathered from extractors
-        self.rules = yara.compile(sources=namespaced_rules)
+        self.path: str = path_extractors
+        self.extractors: Dict[str, Dict[str, str]] = {}
+        with Manager() as manager:
+            extractors = manager.dict()
+            namespaced_rules = manager.dict()
+            def extractor_module_callback(module: ModuleType, venv: str):
+                members = inspect.getmembers(module, predicate=utils.maco_extractor_validation)
+                for member in members:
+                    name, member = member
+                    if exclude and name in exclude:
+                        # Module is part of the exclusion list, skip
+                        logger.debug(f"exclude excluded '{name}'")
+                        return
+                    if include and name not in include:
+                        # Module wasn't part of the inclusion list, skip
+                        logger.debug(f"include excluded '{name}'")
+                        return
+                    # initialise and register
+                    logger.debug(f"register '{name}'")
+                    extractors[name] = dict(
+                        venv=venv,
+                        module_path=module.__file__,
+                        module_name=member.__module__,
+                        extractor_class=member.__name__,
+                    )
+                    namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
+            # Find the extractors within the given directory
+            # Execute within a child process to ensure main process interpreter is kept clean
+            p = Process(
+                target=utils.import_extractors,
+                args=(
+                    path_extractors,
+                    yara.compile(source=utils.MACO_YARA_RULE),
+                    extractor_module_callback,
+                    logger,
+                    create_venv and os.path.isdir(path_extractors),
+                ),
+            )
+            p.start()
+            p.join()
+            self.extractors = dict(extractors)
+            if not self.extractors:
+                raise ExtractorLoadError("no extractors were loaded")
+            logger.debug(f"found extractors {list(self.extractors.keys())}\n")
+            # compile yara rules gathered from extractors
+            self.rules = yara.compile(sources=dict(namespaced_rules))
     def match(self, stream: BinaryIO) -> Dict[str, List[yara.Match]]:
         """Return extractors that should run based on yara rules."""
@@ -105,17 +121,13 @@ class Collector:
     ) -> Dict[str, Any]:
         """Run extractor with stream and verify output matches the model."""
         extractor = self.extractors[extractor_name]
-        resp = None
         try:
-            if extractor["venv"]:
-                # Run extractor within a virtual environment
-                with NamedTemporaryFile() as sample_path:
-                    sample_path.write(stream.read())
-                    sample_path.flush()
-                    return utils.run_in_venv(sample_path.name, **extractor)
-            else:
-                # Run extractor within on host environment
-                resp = extractor["module"]().run(stream, matches)
+            # Run extractor on a copy of the sample
+            with NamedTemporaryFile() as sample_path:
+                sample_path.write(stream.read())
+                sample_path.flush()
+                # enforce types and verify properties, and remove defaults
+                return _verify_response(utils.run_extractor(sample_path.name, **extractor))
         except Exception:
             # caller can deal with the exception
             raise
@@ -123,9 +135,3 @@ class Collector:
             # make sure to reset where we are in the file
             # otherwise follow on extractors are going to read 0 bytes
             stream.seek(0)
-        # enforce types and verify properties, and remove defaults
-        if resp is not None:
-            resp = _verify_response(resp)
-        return resp

model_setup/maco/utils.py CHANGED Viewed

@@ -6,9 +6,11 @@ import inspect
 import json
 import os
 import re
+import shutil
 import subprocess
 import sys
 import tempfile
 from maco import yara
 if sys.version_info >= (3, 11):
@@ -21,8 +23,8 @@ from copy import deepcopy
 from glob import glob
 from logging import Logger
 from pkgutil import walk_packages
-from typing import Callable, Dict, Tuple, List, Set
 from types import ModuleType
+from typing import Callable, Dict, List, Set, Tuple
 from maco.extractor import Extractor
@@ -67,7 +69,11 @@ import importlib
 import json
 import os
 import sys
-import yara
+try:
+    from maco import yara
+except:
+    import yara
 from base64 import b64encode
 parent_package_path = "{parent_package_path}"
@@ -244,6 +250,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
                     logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
                     venvs.append(venv_path)
+                # Cleanup any build directories that are the product of package installation
+                expected_build_path = os.path.join(dir, "build")
+                if os.path.exists(expected_build_path):
+                    shutil.rmtree(expected_build_path)
             # Add directories to our visited list and check the parent of this directory on the next loop
             visited_dirs.append(dir)
             dir = os.path.dirname(dir)
@@ -399,21 +410,23 @@ def import_extractors(
     register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
-def run_in_venv(
+def run_extractor(
     sample_path,
-    module,
+    module_name,
+    extractor_class,
     module_path,
     venv,
     venv_script=VENV_SCRIPT,
     json_decoder=Base64Decoder,
 ) -> Dict[str, dict]:
     # Write temporary script in the same directory as extractor to resolve relative imports
-    python_exe = os.path.join(venv, "bin", "python")
+    python_exe = sys.executable
+    if venv:
+        # If there is a linked virtual environment, execute within that environment
+        python_exe = os.path.join(venv, "bin", "python")
     dirname = os.path.dirname(module_path)
     with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
         with tempfile.NamedTemporaryFile() as output:
-            module_name = module.__module__
-            module_class = module.__name__
             parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
             root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
@@ -421,7 +434,7 @@ def run_in_venv(
                 venv_script.format(
                     parent_package_path=parent_package_path,
                     module_name=module_name,
-                    module_class=module_class,
+                    module_class=extractor_class,
                     sample_path=sample_path,
                     output_path=output.name,
                 )

model_setup/maco/yara.py CHANGED Viewed

@@ -1,11 +1,11 @@
 import re
-import yara
-import yara_x
 from collections import namedtuple
 from itertools import cycle
 from typing import Dict
+import yara
+import yara_x
 RULE_ID_RE = re.compile("(\w+)? ?rule (\w+)")
@@ -42,7 +42,7 @@ class Match:
     def __init__(self, rule: yara_x.Rule, file_content: bytes):
         self.rule = rule.identifier
         self.namespace = rule.namespace
-        self.tags = rule.tags if hasattr(rule, "tags") else []
+        self.tags = list(rule.tags) or []
         self.meta = dict()
         # Ensure metadata doesn't get overwritten
         for k, v in rule.metadata:

pipelines/publish.yaml CHANGED Viewed

@@ -9,7 +9,7 @@ trigger:
 pr: none
 pool:
-  vmImage: "ubuntu-20.04"
+  vmImage: "ubuntu-22.04"
 jobs:
 - job: test
@@ -20,12 +20,12 @@ jobs:
         python.version: '3.8'
       Python39:
         python.version: '3.9'
-      # Python310:
-      #   python.version: '3.10'
-      # Python311:
-      #   python.version: '3.11'
-      # Python312:
-      #   python.version: '3.12'
+      Python310:
+        python.version: '3.10'
+      Python311:
+        python.version: '3.11'
+      Python312:
+        python.version: '3.12'
   steps:
   - task: UsePythonVersion@0
     displayName: 'Use Python $(python.version)'

pipelines/test.yaml CHANGED Viewed

@@ -4,7 +4,7 @@ trigger: ["*"]
 pr: ["*"]
 pool:
-  vmImage: "ubuntu-20.04"
+  vmImage: "ubuntu-22.04"
 jobs:
   - job: run_test
@@ -27,15 +27,19 @@ jobs:
         displayName: Set python version
         inputs:
           versionSpec: "$(python.version)"
       - script: |
-          [ ! -d "$(pwd)/tests" ] && echo "No tests found" && exit
-          [ -f $(pwd)/requirements.txt ] && sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir -r $(pwd)/requirements.txt
-          [ -f $(pwd)/tests/requirements.txt ] && sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir -r $(pwd)/tests/requirements.txt
-          sudo rm -rf /tmp/* /var/lib/apt/lists/* ~/.cache/pip
+          runtests=true
+          if [ ! -d "$(pwd)/tests" ]; then
+            echo "No tests found"
+            runtest=false
+          else
+            python -m pip install -U tox
+          fi
+          echo "##vso[task.setvariable variable=runtests;]$runtests"
+        displayName: Install tox
-        displayName: Setup environment
       - script: |
-          [ ! -d "$(pwd)/tests" ] && echo "No tests found" && exit
-          export REPO_NAME=${BUILD_REPOSITORY_NAME##*/}
-          python -m pytest -p no:cacheprovider --durations=10 -rsx -vv -W ignore::DeprecationWarning
-        displayName: Test
+          python -m tox -e py
+        displayName: "Run tests"
+        condition: and(succeeded(), eq(variables.runtests, true))

tests/data/trigger_complex.txt ADDED Viewed

@@ -0,0 +1,6 @@
+file to trigger demo extractors
+self_trigger
+Complex
+Paradise

{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/LICENSE.md RENAMED Viewed

File without changes

{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/entry_points.txt RENAMED Viewed

File without changes

{maco-1.2.0.dist-info → maco-1.2.1.dist-info}/top_level.txt RENAMED Viewed

File without changes

maco 1.2.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

maco 1.2.0py3-none-any.whl → 1.2.1py3-none-any.whl