maco 1.2.0__tar.gz → 1.2.1__tar.gz

{maco-1.2.0/maco.egg-info → maco-1.2.1}/PKG-INFO

@@ -1,8 +1,7 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.0
+Version: 1.2.1
 Author: sl-govau
-Author-email: 
 Maintainer: cccs-rs
 License: MIT License
         
@@ -35,7 +34,7 @@ Requires-Dist: pydantic>=2.0.0
 Requires-Dist: tomli>=1.1.0; python_version < "3.11"
 Requires-Dist: uv
 Requires-Dist: yara-python
-Requires-Dist: yara-x==0.10.0
+Requires-Dist: yara-x==0.11.0
 
 # Maco - Malware config extractor framework
 

{maco-1.2.0 → maco-1.2.1}/demo_extractors/complex/complex.py

@@ -1,9 +1,9 @@
 from io import BytesIO
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 from maco import extractor, model, yara
 
-from . import complex_utils
+from complex import complex_utils
 
 
 class Complex(extractor.Extractor):
@@ -50,7 +50,7 @@ class Complex(extractor.Extractor):
         other = complex_utils.getdata()["result"]
         self.logger.debug("got data from lib")
         # example - accessing yara strings
-        strings = {y[2].decode("utf8") for x in matches for y in x.strings}
+        strings = sorted({z.plaintext().decode("utf8") for x in matches for y in x.strings for z in y.instances})
         self.logger.debug(f"{strings=}")
         # construct model of results
         tmp = model.ExtractorModel(family=self.family)

maco-1.2.1/maco/collector.py

@@ -0,0 +1,137 @@
+"""Convenience functions for discovering your extractors."""
+
+import inspect
+import logging
+import os
+from multiprocessing import Manager, Process
+from tempfile import NamedTemporaryFile
+from types import ModuleType
+from typing import Any, BinaryIO, Dict, List, Union
+
+from pydantic import BaseModel
+
+from maco import extractor, model, utils, yara
+
+
+class ExtractorLoadError(Exception):
+    pass
+
+
+logger = logging.getLogger("maco.lib.helpers")
+
+
+def _verify_response(resp: Union[BaseModel, dict]) -> Dict:
+    """Enforce types and verify properties, and remove defaults."""
+    if not resp:
+        return None
+    # check the response is valid for its own model
+    # this is useful if a restriction on the 'other' dictionary is needed
+    resp_model = type(resp)
+    if resp_model != model.ExtractorModel and hasattr(resp_model, "model_validate"):
+        resp = resp_model.model_validate(resp)
+    # check the response is valid according to the ExtractorModel
+    resp = model.ExtractorModel.model_validate(resp)
+    # coerce sets to correct types
+    # otherwise we end up with sets where we expect lists
+    resp = model.ExtractorModel(**resp.model_dump())
+    # dump model to dict
+    return resp.model_dump(exclude_defaults=True)
+
+
+class Collector:
+    def __init__(
+        self, path_extractors: str, include: List[str] = None, exclude: List[str] = None, create_venv: bool = False
+    ):
+        """Discover and load extractors from file system."""
+        path_extractors = os.path.realpath(path_extractors)
+        self.path: str = path_extractors
+        self.extractors: Dict[str, Dict[str, str]] = {}
+
+        with Manager() as manager:
+            extractors = manager.dict()
+            namespaced_rules = manager.dict()
+
+            def extractor_module_callback(module: ModuleType, venv: str):
+                members = inspect.getmembers(module, predicate=utils.maco_extractor_validation)
+                for member in members:
+                    name, member = member
+                    if exclude and name in exclude:
+                        # Module is part of the exclusion list, skip
+                        logger.debug(f"exclude excluded '{name}'")
+                        return
+
+                    if include and name not in include:
+                        # Module wasn't part of the inclusion list, skip
+                        logger.debug(f"include excluded '{name}'")
+                        return
+
+                    # initialise and register
+                    logger.debug(f"register '{name}'")
+                    extractors[name] = dict(
+                        venv=venv,
+                        module_path=module.__file__,
+                        module_name=member.__module__,
+                        extractor_class=member.__name__,
+                    )
+                    namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
+
+            # Find the extractors within the given directory
+            # Execute within a child process to ensure main process interpreter is kept clean
+            p = Process(
+                target=utils.import_extractors,
+                args=(
+                    path_extractors,
+                    yara.compile(source=utils.MACO_YARA_RULE),
+                    extractor_module_callback,
+                    logger,
+                    create_venv and os.path.isdir(path_extractors),
+                ),
+            )
+            p.start()
+            p.join()
+
+            self.extractors = dict(extractors)
+            if not self.extractors:
+                raise ExtractorLoadError("no extractors were loaded")
+            logger.debug(f"found extractors {list(self.extractors.keys())}\n")
+
+            # compile yara rules gathered from extractors
+            self.rules = yara.compile(sources=dict(namespaced_rules))
+
+    def match(self, stream: BinaryIO) -> Dict[str, List[yara.Match]]:
+        """Return extractors that should run based on yara rules."""
+        # execute yara rules on file to find extractors we should run
+        # yara can't run on a stream so we give it a bytestring
+        matches = self.rules.match(data=stream.read())
+        stream.seek(0)
+        if not matches:
+            return
+        # get all rules that hit for each extractor
+        runs = {}
+        for match in matches:
+            runs.setdefault(match.namespace, []).append(match)
+
+        return runs
+
+    def extract(
+        self,
+        stream: BinaryIO,
+        matches: List[yara.Match],
+        extractor_name: str,
+    ) -> Dict[str, Any]:
+        """Run extractor with stream and verify output matches the model."""
+        extractor = self.extractors[extractor_name]
+        try:
+            # Run extractor on a copy of the sample
+            with NamedTemporaryFile() as sample_path:
+                sample_path.write(stream.read())
+                sample_path.flush()
+                # enforce types and verify properties, and remove defaults
+                return _verify_response(utils.run_extractor(sample_path.name, **extractor))
+        except Exception:
+            # caller can deal with the exception
+            raise
+        finally:
+            # make sure to reset where we are in the file
+            # otherwise follow on extractors are going to read 0 bytes
+            stream.seek(0)

{maco-1.2.0 → maco-1.2.1}/maco/utils.py

@@ -6,9 +6,11 @@ import inspect
 import json
 import os
 import re
+import shutil
 import subprocess
 import sys
 import tempfile
+
 from maco import yara
 
 if sys.version_info >= (3, 11):
@@ -21,8 +23,8 @@ from copy import deepcopy
 from glob import glob
 from logging import Logger
 from pkgutil import walk_packages
-from typing import Callable, Dict, Tuple, List, Set
 from types import ModuleType
+from typing import Callable, Dict, List, Set, Tuple
 
 from maco.extractor import Extractor
 
@@ -67,7 +69,11 @@ import importlib
 import json
 import os
 import sys
-import yara
+
+try:
+    from maco import yara
+except:
+    import yara
 
 from base64 import b64encode
 parent_package_path = "{parent_package_path}"
@@ -244,6 +250,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
                     logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
                     venvs.append(venv_path)
 
+                # Cleanup any build directories that are the product of package installation
+                expected_build_path = os.path.join(dir, "build")
+                if os.path.exists(expected_build_path):
+                    shutil.rmtree(expected_build_path)
+
             # Add directories to our visited list and check the parent of this directory on the next loop
             visited_dirs.append(dir)
             dir = os.path.dirname(dir)
@@ -399,21 +410,23 @@ def import_extractors(
     register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
 
 
-def run_in_venv(
+def run_extractor(
     sample_path,
-    module,
+    module_name,
+    extractor_class,
     module_path,
     venv,
     venv_script=VENV_SCRIPT,
     json_decoder=Base64Decoder,
 ) -> Dict[str, dict]:
     # Write temporary script in the same directory as extractor to resolve relative imports
-    python_exe = os.path.join(venv, "bin", "python")
+    python_exe = sys.executable
+    if venv:
+        # If there is a linked virtual environment, execute within that environment
+        python_exe = os.path.join(venv, "bin", "python")
     dirname = os.path.dirname(module_path)
     with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
         with tempfile.NamedTemporaryFile() as output:
-            module_name = module.__module__
-            module_class = module.__name__
             parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
             root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
 
@@ -421,7 +434,7 @@ def run_in_venv(
                 venv_script.format(
                     parent_package_path=parent_package_path,
                     module_name=module_name,
-                    module_class=module_class,
+                    module_class=extractor_class,
                     sample_path=sample_path,
                     output_path=output.name,
                 )

{maco-1.2.0/model_setup → maco-1.2.1}/maco/yara.py

@@ -1,11 +1,11 @@
 import re
-import yara
-import yara_x
-
 from collections import namedtuple
 from itertools import cycle
 from typing import Dict
 
+import yara
+import yara_x
+
 RULE_ID_RE = re.compile("(\w+)? ?rule (\w+)")
 
 
@@ -42,7 +42,7 @@ class Match:
     def __init__(self, rule: yara_x.Rule, file_content: bytes):
         self.rule = rule.identifier
         self.namespace = rule.namespace
-        self.tags = rule.tags if hasattr(rule, "tags") else []
+        self.tags = list(rule.tags) or []
         self.meta = dict()
         # Ensure metadata doesn't get overwritten
         for k, v in rule.metadata:

{maco-1.2.0 → maco-1.2.1/maco.egg-info}/PKG-INFO

@@ -1,8 +1,7 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.0
+Version: 1.2.1
 Author: sl-govau
-Author-email: 
 Maintainer: cccs-rs
 License: MIT License
         
@@ -35,7 +34,7 @@ Requires-Dist: pydantic>=2.0.0
 Requires-Dist: tomli>=1.1.0; python_version < "3.11"
 Requires-Dist: uv
 Requires-Dist: yara-python
-Requires-Dist: yara-x==0.10.0
+Requires-Dist: yara-x==0.11.0
 
 # Maco - Malware config extractor framework
 

{maco-1.2.0 → maco-1.2.1}/maco.egg-info/SOURCES.txt

@@ -3,7 +3,6 @@ LICENSE.md
 README.md
 pyproject.toml
 requirements.txt
-setup.py
 tox.ini
 .vscode/settings.json
 demo_extractors/elfy.py
@@ -47,11 +46,13 @@ pipelines/test.yaml
 tests/pytest.ini
 tests/requirements.txt
 tests/test_base_test.py
+tests/test_demo_extractors.py
 tests/test_detection.py
 tests/test_extractor.py
 tests/test_helpers.py
 tests/test_model.py
 tests/data/example.txt.cart
+tests/data/trigger_complex.txt
 tests/extractors/__init__.py
 tests/extractors/basic.py
 tests/extractors/basic_longer.py

{maco-1.2.0 → maco-1.2.1}/maco.egg-info/requires.txt

@@ -2,7 +2,7 @@ cart
 pydantic>=2.0.0
 uv
 yara-python
-yara-x==0.10.0
+yara-x==0.11.0
 
 [:python_version < "3.11"]
 tomli>=1.1.0

maco-1.2.1/model_setup/maco/collector.py

@@ -0,0 +1,137 @@
+"""Convenience functions for discovering your extractors."""
+
+import inspect
+import logging
+import os
+from multiprocessing import Manager, Process
+from tempfile import NamedTemporaryFile
+from types import ModuleType
+from typing import Any, BinaryIO, Dict, List, Union
+
+from pydantic import BaseModel
+
+from maco import extractor, model, utils, yara
+
+
+class ExtractorLoadError(Exception):
+    pass
+
+
+logger = logging.getLogger("maco.lib.helpers")
+
+
+def _verify_response(resp: Union[BaseModel, dict]) -> Dict:
+    """Enforce types and verify properties, and remove defaults."""
+    if not resp:
+        return None
+    # check the response is valid for its own model
+    # this is useful if a restriction on the 'other' dictionary is needed
+    resp_model = type(resp)
+    if resp_model != model.ExtractorModel and hasattr(resp_model, "model_validate"):
+        resp = resp_model.model_validate(resp)
+    # check the response is valid according to the ExtractorModel
+    resp = model.ExtractorModel.model_validate(resp)
+    # coerce sets to correct types
+    # otherwise we end up with sets where we expect lists
+    resp = model.ExtractorModel(**resp.model_dump())
+    # dump model to dict
+    return resp.model_dump(exclude_defaults=True)
+
+
+class Collector:
+    def __init__(
+        self, path_extractors: str, include: List[str] = None, exclude: List[str] = None, create_venv: bool = False
+    ):
+        """Discover and load extractors from file system."""
+        path_extractors = os.path.realpath(path_extractors)
+        self.path: str = path_extractors
+        self.extractors: Dict[str, Dict[str, str]] = {}
+
+        with Manager() as manager:
+            extractors = manager.dict()
+            namespaced_rules = manager.dict()
+
+            def extractor_module_callback(module: ModuleType, venv: str):
+                members = inspect.getmembers(module, predicate=utils.maco_extractor_validation)
+                for member in members:
+                    name, member = member
+                    if exclude and name in exclude:
+                        # Module is part of the exclusion list, skip
+                        logger.debug(f"exclude excluded '{name}'")
+                        return
+
+                    if include and name not in include:
+                        # Module wasn't part of the inclusion list, skip
+                        logger.debug(f"include excluded '{name}'")
+                        return
+
+                    # initialise and register
+                    logger.debug(f"register '{name}'")
+                    extractors[name] = dict(
+                        venv=venv,
+                        module_path=module.__file__,
+                        module_name=member.__module__,
+                        extractor_class=member.__name__,
+                    )
+                    namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
+
+            # Find the extractors within the given directory
+            # Execute within a child process to ensure main process interpreter is kept clean
+            p = Process(
+                target=utils.import_extractors,
+                args=(
+                    path_extractors,
+                    yara.compile(source=utils.MACO_YARA_RULE),
+                    extractor_module_callback,
+                    logger,
+                    create_venv and os.path.isdir(path_extractors),
+                ),
+            )
+            p.start()
+            p.join()
+
+            self.extractors = dict(extractors)
+            if not self.extractors:
+                raise ExtractorLoadError("no extractors were loaded")
+            logger.debug(f"found extractors {list(self.extractors.keys())}\n")
+
+            # compile yara rules gathered from extractors
+            self.rules = yara.compile(sources=dict(namespaced_rules))
+
+    def match(self, stream: BinaryIO) -> Dict[str, List[yara.Match]]:
+        """Return extractors that should run based on yara rules."""
+        # execute yara rules on file to find extractors we should run
+        # yara can't run on a stream so we give it a bytestring
+        matches = self.rules.match(data=stream.read())
+        stream.seek(0)
+        if not matches:
+            return
+        # get all rules that hit for each extractor
+        runs = {}
+        for match in matches:
+            runs.setdefault(match.namespace, []).append(match)
+
+        return runs
+
+    def extract(
+        self,
+        stream: BinaryIO,
+        matches: List[yara.Match],
+        extractor_name: str,
+    ) -> Dict[str, Any]:
+        """Run extractor with stream and verify output matches the model."""
+        extractor = self.extractors[extractor_name]
+        try:
+            # Run extractor on a copy of the sample
+            with NamedTemporaryFile() as sample_path:
+                sample_path.write(stream.read())
+                sample_path.flush()
+                # enforce types and verify properties, and remove defaults
+                return _verify_response(utils.run_extractor(sample_path.name, **extractor))
+        except Exception:
+            # caller can deal with the exception
+            raise
+        finally:
+            # make sure to reset where we are in the file
+            # otherwise follow on extractors are going to read 0 bytes
+            stream.seek(0)

{maco-1.2.0 → maco-1.2.1}/model_setup/maco/utils.py

@@ -6,9 +6,11 @@ import inspect
 import json
 import os
 import re
+import shutil
 import subprocess
 import sys
 import tempfile
+
 from maco import yara
 
 if sys.version_info >= (3, 11):
@@ -21,8 +23,8 @@ from copy import deepcopy
 from glob import glob
 from logging import Logger
 from pkgutil import walk_packages
-from typing import Callable, Dict, Tuple, List, Set
 from types import ModuleType
+from typing import Callable, Dict, List, Set, Tuple
 
 from maco.extractor import Extractor
 
@@ -67,7 +69,11 @@ import importlib
 import json
 import os
 import sys
-import yara
+
+try:
+    from maco import yara
+except:
+    import yara
 
 from base64 import b64encode
 parent_package_path = "{parent_package_path}"
@@ -244,6 +250,11 @@ def create_virtual_environments(directories: List[str], python_version: str, log
                     logger.debug(f"Installed dependencies into venv:\n{p.stdout.decode()}")
                     venvs.append(venv_path)
 
+                # Cleanup any build directories that are the product of package installation
+                expected_build_path = os.path.join(dir, "build")
+                if os.path.exists(expected_build_path):
+                    shutil.rmtree(expected_build_path)
+
             # Add directories to our visited list and check the parent of this directory on the next loop
             visited_dirs.append(dir)
             dir = os.path.dirname(dir)
@@ -399,21 +410,23 @@ def import_extractors(
     register_extractors(root_directory, venvs, extractor_files, extractor_module_callback, logger)
 
 
-def run_in_venv(
+def run_extractor(
     sample_path,
-    module,
+    module_name,
+    extractor_class,
     module_path,
     venv,
     venv_script=VENV_SCRIPT,
     json_decoder=Base64Decoder,
 ) -> Dict[str, dict]:
     # Write temporary script in the same directory as extractor to resolve relative imports
-    python_exe = os.path.join(venv, "bin", "python")
+    python_exe = sys.executable
+    if venv:
+        # If there is a linked virtual environment, execute within that environment
+        python_exe = os.path.join(venv, "bin", "python")
     dirname = os.path.dirname(module_path)
     with tempfile.NamedTemporaryFile("w", dir=dirname, suffix=".py") as script:
         with tempfile.NamedTemporaryFile() as output:
-            module_name = module.__module__
-            module_class = module.__name__
             parent_package_path = dirname.rsplit(module_name.split(".", 1)[0], 1)[0]
             root_directory = module_path[:-3].rsplit(module_name.split(".", 1)[1].replace(".", "/"))[0]
 
@@ -421,7 +434,7 @@ def run_in_venv(
                 venv_script.format(
                     parent_package_path=parent_package_path,
                     module_name=module_name,
-                    module_class=module_class,
+                    module_class=extractor_class,
                     sample_path=sample_path,
                     output_path=output.name,
                 )

{maco-1.2.0 → maco-1.2.1/model_setup}/maco/yara.py

@@ -1,11 +1,11 @@
 import re
-import yara
-import yara_x
-
 from collections import namedtuple
 from itertools import cycle
 from typing import Dict
 
+import yara
+import yara_x
+
 RULE_ID_RE = re.compile("(\w+)? ?rule (\w+)")
 
 
@@ -42,7 +42,7 @@ class Match:
     def __init__(self, rule: yara_x.Rule, file_content: bytes):
         self.rule = rule.identifier
         self.namespace = rule.namespace
-        self.tags = rule.tags if hasattr(rule, "tags") else []
+        self.tags = list(rule.tags) or []
         self.meta = dict()
         # Ensure metadata doesn't get overwritten
         for k, v in rule.metadata:

{maco-1.2.0 → maco-1.2.1}/pipelines/publish.yaml

@@ -9,7 +9,7 @@ trigger:
 pr: none
 
 pool:
-  vmImage: "ubuntu-20.04"
+  vmImage: "ubuntu-22.04"
 
 jobs:
 - job: test
@@ -20,12 +20,12 @@ jobs:
         python.version: '3.8'
       Python39:
         python.version: '3.9'
-      # Python310:
-      #   python.version: '3.10'
-      # Python311:
-      #   python.version: '3.11'
-      # Python312:
-      #   python.version: '3.12'
+      Python310:
+        python.version: '3.10'
+      Python311:
+        python.version: '3.11'
+      Python312:
+        python.version: '3.12'
   steps:
   - task: UsePythonVersion@0
     displayName: 'Use Python $(python.version)'

maco-1.2.1/pipelines/test.yaml

@@ -0,0 +1,45 @@
+name: tests
+
+trigger: ["*"]
+pr: ["*"]
+
+pool:
+  vmImage: "ubuntu-22.04"
+
+jobs:
+  - job: run_test
+    strategy:
+      matrix:
+        Python3_8:
+          python.version: "3.8"
+        Python3_9:
+          python.version: "3.9"
+        Python3_10:
+          python.version: "3.10"
+        Python3_11:
+          python.version: "3.11"
+        Python3_12:
+          python.version: "3.12"
+    timeoutInMinutes: 10
+
+    steps:
+      - task: UsePythonVersion@0
+        displayName: Set python version
+        inputs:
+          versionSpec: "$(python.version)"
+
+      - script: |
+          runtests=true
+          if [ ! -d "$(pwd)/tests" ]; then
+            echo "No tests found"
+            runtest=false
+          else
+            python -m pip install -U tox
+          fi
+          echo "##vso[task.setvariable variable=runtests;]$runtests"
+        displayName: Install tox
+
+      - script: |
+          python -m tox -e py
+        displayName: "Run tests"
+        condition: and(succeeded(), eq(variables.runtests, true))

{maco-1.2.0 → maco-1.2.1}/pyproject.toml

@@ -4,19 +4,10 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "maco"
-dynamic = ["version"]
-dependencies = [
-    "cart",
-    "pydantic>=2.0.0",
-    'tomli >= 1.1.0 ; python_version < "3.11"',
-    "uv",
-    "yara-python",
-    "yara-x==0.10.0",
-]
+dynamic = ["version", "readme", "dependencies"]
 requires-python = ">=3.8"
 authors = [{ name = "sl-govau" }]
 maintainers = [{ name = "cccs-rs" }]
-readme = "README.md"
 license = { file = "LICENSE.md" }
 
 classifiers = [
@@ -43,6 +34,10 @@ Issues = "https://github.com/CybercentreCanada/Maco/issues"
 
 [tool.setuptools_scm]
 
+[tool.setuptools.dynamic]
+readme = { file = ["README.md"], content-type = "text/markdown" }
+dependencies = { file = ["requirements.txt"] }
+
 [tool.setuptools.packages.find]
 where = ["."]
 exclude = ["test", "tests", "extractors", "model_setup"]

{maco-1.2.0 → maco-1.2.1}/requirements.txt

@@ -3,4 +3,4 @@ pydantic>=2.0.0
 tomli >= 1.1.0 ; python_version < "3.11"
 uv
 yara-python
-yara-x==0.10.0
+yara-x==0.11.0

maco-1.2.1/tests/data/trigger_complex.txt

@@ -0,0 +1,6 @@
+file to trigger demo extractors
+
+self_trigger
+
+Complex
+Paradise

maco-1.2.1/tests/test_demo_extractors.py

@@ -0,0 +1,60 @@
+import os
+import unittest
+
+from maco import cli
+from maco.collector import Collector
+
+
+class TestDemoExtractors(unittest.TestCase):
+    def test_complex(self):
+        path_file = os.path.normpath(
+            os.path.join(__file__, "../data/trigger_complex.txt")
+        )
+        collector = Collector(os.path.join(__file__, "../../demo_extractors"))
+        self.assertEqual(
+            set(collector.extractors.keys()),
+            {
+                "Elfy",
+                "Nothing",
+                "Complex",
+                "LimitOther",
+            },
+        )
+
+        with open(path_file, "rb") as stream:
+            ret = cli.process_file(
+                collector,
+                path_file,
+                stream,
+                pretty=True,
+                force=False,
+                include_base64=False,
+            )
+        self.assertEqual(
+            ret,
+            {
+                "Complex": {
+                    "family": "complex",
+                    "version": "5",
+                    "decoded_strings": sorted(["Paradise", "Complex"]),
+                    "binaries": [
+                        {
+                            "datatype": "payload",
+                            "encryption": {"algorithm": "something"},
+                            "sha256": "1307990e6ba5ca145eb35e99182a9bec46531bc54ddf656a602c780fa0240dee",
+                            "size": 9,
+                            "hex_sample": "736F6D652064617461",
+                        }
+                    ],
+                    "http": [
+                        {
+                            "protocol": "https",
+                            "hostname": "blarg5.com",
+                            "path": "/malz/64",
+                            "usage": "c2",
+                        }
+                    ],
+                    "encryption": [{"algorithm": "sha256"}],
+                }
+            },
+        )

maco-1.2.1/tox.ini

@@ -0,0 +1,10 @@
+[tox]
+envlist = py38,py39,py310,py311,py312
+[testenv]
+# install testing framework
+deps =
+    pytest
+    -r requirements.txt
+    -r tests/requirements.txt
+# run the tests
+commands = python -m pytest -p no:cacheprovider --durations=10 -ra -q -k "not git and not extractors" -vv -W ignore::DeprecationWarning

maco-1.2.0/maco/collector.py

@@ -1,131 +0,0 @@
-"""Convenience functions for discovering your extractors."""
-
-import inspect
-import logging
-import os
-
-from tempfile import NamedTemporaryFile
-from typing import Any, BinaryIO, Dict, List
-from types import ModuleType
-
-from maco import yara
-from pydantic import BaseModel
-
-from maco import extractor, model, utils
-
-
-class ExtractorLoadError(Exception):
-    pass
-
-
-logger = logging.getLogger("maco.lib.helpers")
-
-
-def _verify_response(resp: BaseModel) -> Dict:
-    """Enforce types and verify properties, and remove defaults."""
-    # check the response is valid for its own model
-    # this is useful if a restriction on the 'other' dictionary is needed
-    resp_model = type(resp)
-    if resp_model != model.ExtractorModel:
-        resp = resp_model.model_validate(resp)
-    # check the response is valid according to the ExtractorModel
-    resp = model.ExtractorModel.model_validate(resp)
-    # coerce sets to correct types
-    # otherwise we end up with sets where we expect lists
-    resp = model.ExtractorModel(**resp.model_dump())
-    # dump model to dict
-    return resp.model_dump(exclude_defaults=True)
-
-
-class Collector:
-    def __init__(
-        self, path_extractors: str, include: List[str] = None, exclude: List[str] = None, create_venv: bool = False
-    ):
-        """Discover and load extractors from file system."""
-        path_extractors = os.path.realpath(path_extractors)
-        self.path = path_extractors
-        self.extractors = {}
-        namespaced_rules = {}
-
-        def extractor_module_callback(module: ModuleType, venv: str):
-            members = inspect.getmembers(module, predicate=utils.maco_extractor_validation)
-            for member in members:
-                name, member = member
-                if exclude and name in exclude:
-                    # Module is part of the exclusion list, skip
-                    logger.debug(f"exclude excluded '{name}'")
-                    return
-
-                if include and name not in include:
-                    # Module wasn't part of the inclusion list, skip
-                    logger.debug(f"include excluded '{name}'")
-                    return
-
-                # initialise and register
-                logger.debug(f"register '{name}'")
-                self.extractors[name] = dict(module=member, venv=venv, module_path=module.__file__)
-                namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
-
-        # Find the extractors within the given directory
-        utils.import_extractors(
-            path_extractors,
-            yara.compile(source=utils.MACO_YARA_RULE),
-            extractor_module_callback,
-            logger,
-            create_venv and os.path.isdir(path_extractors),
-        )
-
-        if not self.extractors:
-            raise ExtractorLoadError("no extractors were loaded")
-        logger.debug(f"found extractors {list(self.extractors.keys())}\n")
-
-        # compile yara rules gathered from extractors
-        self.rules = yara.compile(sources=namespaced_rules)
-
-    def match(self, stream: BinaryIO) -> Dict[str, List[yara.Match]]:
-        """Return extractors that should run based on yara rules."""
-        # execute yara rules on file to find extractors we should run
-        # yara can't run on a stream so we give it a bytestring
-        matches = self.rules.match(data=stream.read())
-        stream.seek(0)
-        if not matches:
-            return
-        # get all rules that hit for each extractor
-        runs = {}
-        for match in matches:
-            runs.setdefault(match.namespace, []).append(match)
-
-        return runs
-
-    def extract(
-        self,
-        stream: BinaryIO,
-        matches: List[yara.Match],
-        extractor_name: str,
-    ) -> Dict[str, Any]:
-        """Run extractor with stream and verify output matches the model."""
-        extractor = self.extractors[extractor_name]
-        resp = None
-        try:
-            if extractor["venv"]:
-                # Run extractor within a virtual environment
-                with NamedTemporaryFile() as sample_path:
-                    sample_path.write(stream.read())
-                    sample_path.flush()
-                    return utils.run_in_venv(sample_path.name, **extractor)
-            else:
-                # Run extractor within on host environment
-                resp = extractor["module"]().run(stream, matches)
-        except Exception:
-            # caller can deal with the exception
-            raise
-        finally:
-            # make sure to reset where we are in the file
-            # otherwise follow on extractors are going to read 0 bytes
-            stream.seek(0)
-
-        # enforce types and verify properties, and remove defaults
-        if resp is not None:
-            resp = _verify_response(resp)
-
-        return resp

maco-1.2.0/model_setup/maco/collector.py

@@ -1,131 +0,0 @@
-"""Convenience functions for discovering your extractors."""
-
-import inspect
-import logging
-import os
-
-from tempfile import NamedTemporaryFile
-from typing import Any, BinaryIO, Dict, List
-from types import ModuleType
-
-from maco import yara
-from pydantic import BaseModel
-
-from maco import extractor, model, utils
-
-
-class ExtractorLoadError(Exception):
-    pass
-
-
-logger = logging.getLogger("maco.lib.helpers")
-
-
-def _verify_response(resp: BaseModel) -> Dict:
-    """Enforce types and verify properties, and remove defaults."""
-    # check the response is valid for its own model
-    # this is useful if a restriction on the 'other' dictionary is needed
-    resp_model = type(resp)
-    if resp_model != model.ExtractorModel:
-        resp = resp_model.model_validate(resp)
-    # check the response is valid according to the ExtractorModel
-    resp = model.ExtractorModel.model_validate(resp)
-    # coerce sets to correct types
-    # otherwise we end up with sets where we expect lists
-    resp = model.ExtractorModel(**resp.model_dump())
-    # dump model to dict
-    return resp.model_dump(exclude_defaults=True)
-
-
-class Collector:
-    def __init__(
-        self, path_extractors: str, include: List[str] = None, exclude: List[str] = None, create_venv: bool = False
-    ):
-        """Discover and load extractors from file system."""
-        path_extractors = os.path.realpath(path_extractors)
-        self.path = path_extractors
-        self.extractors = {}
-        namespaced_rules = {}
-
-        def extractor_module_callback(module: ModuleType, venv: str):
-            members = inspect.getmembers(module, predicate=utils.maco_extractor_validation)
-            for member in members:
-                name, member = member
-                if exclude and name in exclude:
-                    # Module is part of the exclusion list, skip
-                    logger.debug(f"exclude excluded '{name}'")
-                    return
-
-                if include and name not in include:
-                    # Module wasn't part of the inclusion list, skip
-                    logger.debug(f"include excluded '{name}'")
-                    return
-
-                # initialise and register
-                logger.debug(f"register '{name}'")
-                self.extractors[name] = dict(module=member, venv=venv, module_path=module.__file__)
-                namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
-
-        # Find the extractors within the given directory
-        utils.import_extractors(
-            path_extractors,
-            yara.compile(source=utils.MACO_YARA_RULE),
-            extractor_module_callback,
-            logger,
-            create_venv and os.path.isdir(path_extractors),
-        )
-
-        if not self.extractors:
-            raise ExtractorLoadError("no extractors were loaded")
-        logger.debug(f"found extractors {list(self.extractors.keys())}\n")
-
-        # compile yara rules gathered from extractors
-        self.rules = yara.compile(sources=namespaced_rules)
-
-    def match(self, stream: BinaryIO) -> Dict[str, List[yara.Match]]:
-        """Return extractors that should run based on yara rules."""
-        # execute yara rules on file to find extractors we should run
-        # yara can't run on a stream so we give it a bytestring
-        matches = self.rules.match(data=stream.read())
-        stream.seek(0)
-        if not matches:
-            return
-        # get all rules that hit for each extractor
-        runs = {}
-        for match in matches:
-            runs.setdefault(match.namespace, []).append(match)
-
-        return runs
-
-    def extract(
-        self,
-        stream: BinaryIO,
-        matches: List[yara.Match],
-        extractor_name: str,
-    ) -> Dict[str, Any]:
-        """Run extractor with stream and verify output matches the model."""
-        extractor = self.extractors[extractor_name]
-        resp = None
-        try:
-            if extractor["venv"]:
-                # Run extractor within a virtual environment
-                with NamedTemporaryFile() as sample_path:
-                    sample_path.write(stream.read())
-                    sample_path.flush()
-                    return utils.run_in_venv(sample_path.name, **extractor)
-            else:
-                # Run extractor within on host environment
-                resp = extractor["module"]().run(stream, matches)
-        except Exception:
-            # caller can deal with the exception
-            raise
-        finally:
-            # make sure to reset where we are in the file
-            # otherwise follow on extractors are going to read 0 bytes
-            stream.seek(0)
-
-        # enforce types and verify properties, and remove defaults
-        if resp is not None:
-            resp = _verify_response(resp)
-
-        return resp

maco-1.2.0/pipelines/test.yaml

@@ -1,41 +0,0 @@
-name: tests
-
-trigger: ["*"]
-pr: ["*"]
-
-pool:
-  vmImage: "ubuntu-20.04"
-
-jobs:
-  - job: run_test
-    strategy:
-      matrix:
-        Python3_8:
-          python.version: "3.8"
-        Python3_9:
-          python.version: "3.9"
-        Python3_10:
-          python.version: "3.10"
-        Python3_11:
-          python.version: "3.11"
-        Python3_12:
-          python.version: "3.12"
-    timeoutInMinutes: 10
-
-    steps:
-      - task: UsePythonVersion@0
-        displayName: Set python version
-        inputs:
-          versionSpec: "$(python.version)"
-      - script: |
-          [ ! -d "$(pwd)/tests" ] && echo "No tests found" && exit
-          [ -f $(pwd)/requirements.txt ] && sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir -r $(pwd)/requirements.txt
-          [ -f $(pwd)/tests/requirements.txt ] && sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir -r $(pwd)/tests/requirements.txt
-          sudo rm -rf /tmp/* /var/lib/apt/lists/* ~/.cache/pip
-
-        displayName: Setup environment
-      - script: |
-          [ ! -d "$(pwd)/tests" ] && echo "No tests found" && exit
-          export REPO_NAME=${BUILD_REPOSITORY_NAME##*/}
-          python -m pytest -p no:cacheprovider --durations=10 -rsx -vv -W ignore::DeprecationWarning
-        displayName: Test

maco-1.2.0/setup.py

@@ -1,25 +0,0 @@
-#!/usr/bin/env python3
-"""Setup script."""
-
-from setuptools import find_packages, setup
-
-setup(
-    python_requires=">=3.8",
-    use_scm_version=True,
-    setup_requires=["setuptools_scm"],
-    packages=find_packages(".", exclude=["test", "tests", "extractors"]),
-    include_package_data=True,
-    install_requires=[
-        r.strip() for r in open("requirements.txt", "r") if not r.startswith("#")
-    ],
-    name="maco",
-    description="",
-    author="",
-    author_email="",
-    classifiers=[],
-    entry_points={
-        "console_scripts": [
-            "maco = maco.cli:main",
-        ],
-    },
-)

maco-1.2.0/tox.ini

@@ -1,10 +0,0 @@
-[tox]
-envlist = py38,py39
-[testenv]
-# install testing framework
-deps =
-    pytest
-    -r requirements.txt
-    -r tests/requirements.txt
-# run the tests
-commands = pytest ./tests