maco 1.2.1__tar.gz → 1.2.3__tar.gz

{maco-1.2.1/maco.egg-info → maco-1.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.1
+Version: 1.2.3
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License

{maco-1.2.1/model_setup → maco-1.2.3}/maco/base_test.py

@@ -46,8 +46,7 @@ class BaseTest(unittest.TestCase):
         runs = self.c.match(stream)
         if not runs:
             raise NoHitException("no yara rule hit")
-        hits = runs[self.name]
-        resp = self.c.extract(stream, hits, self.name)
+        resp = self.c.extract(stream, self.name)
         return resp
 
     def _get_location(self) -> str:

{maco-1.2.1/model_setup → maco-1.2.3}/maco/cli.py

@@ -8,7 +8,7 @@ import json
 import logging
 import os
 import sys
-from typing import BinaryIO, List
+from typing import BinaryIO, List, Tuple
 
 import cart
 
@@ -57,7 +57,7 @@ def process_file(
         # run and store results for extractor
         logger.info(f"run {extractor_name} extractor from rules {[x.rule for x in hits]}")
         try:
-            resp = collected.extract(stream, hits, extractor_name)
+            resp = collected.extract(stream, extractor_name)
         except Exception as e:
             logger.exception(f"extractor error with {path_file} ({e})")
             resp = None
@@ -91,18 +91,22 @@ def process_filesystem(
     force: bool,
     include_base64: bool,
     create_venv: bool = False,
-):
+) -> Tuple[int, int, int]:
+    """Process filesystem with extractors and print results of extraction.
+
+    Returns total number of analysed files, yara hits and successful maco extractions.
+    """
     if force:
         logger.warning("force execute will cause errors if an extractor requires a yara rule hit during execution")
     collected = collector.Collector(path_extractors, include=include, exclude=exclude, create_venv=create_venv)
 
     logger.info(f"extractors loaded: {[x for x in collected.extractors.keys()]}\n")
     for _, extractor in collected.extractors.items():
-        extractor = extractor["module"]
+        extractor_meta = extractor["metadata"]
         logger.info(
-            f"{extractor.family} by {extractor.author}"
-            f" {extractor.last_modified} {extractor.sharing}"
-            f"\n{extractor.__doc__}\n"
+            f"{extractor_meta['family']} by {extractor_meta['author']}"
+            f" {extractor_meta['last_modified']} {extractor_meta['sharing']}"
+            f"\n{extractor_meta['description']}\n"
         )
 
     num_analysed = 0
@@ -144,7 +148,7 @@ def process_filesystem(
     finally:
         logger.info("")
         logger.info(f"{num_analysed} analysed, {num_hits} hits, {num_extracted} extracted")
-
+    return num_analysed, num_hits, num_extracted
 
 def main():
     parser = argparse.ArgumentParser(description="Run extractors over samples.")

{maco-1.2.1/model_setup → maco-1.2.3}/maco/collector.py

@@ -40,7 +40,11 @@ def _verify_response(resp: Union[BaseModel, dict]) -> Dict:
 
 class Collector:
     def __init__(
-        self, path_extractors: str, include: List[str] = None, exclude: List[str] = None, create_venv: bool = False
+        self,
+        path_extractors: str,
+        include: List[str] = None,
+        exclude: List[str] = None,
+        create_venv: bool = False,
     ):
         """Discover and load extractors from file system."""
         path_extractors = os.path.realpath(path_extractors)
@@ -72,6 +76,13 @@ class Collector:
                         module_path=module.__file__,
                         module_name=member.__module__,
                         extractor_class=member.__name__,
+                        metadata={
+                            "family": member.family,
+                            "author": member.author,
+                            "last_modified": member.last_modified,
+                            "sharing": member.sharing,
+                            "description": member.__doc__,
+                        },
                     )
                     namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
 
@@ -116,7 +127,6 @@ class Collector:
     def extract(
         self,
         stream: BinaryIO,
-        matches: List[yara.Match],
         extractor_name: str,
     ) -> Dict[str, Any]:
         """Run extractor with stream and verify output matches the model."""
@@ -127,7 +137,15 @@ class Collector:
                 sample_path.write(stream.read())
                 sample_path.flush()
                 # enforce types and verify properties, and remove defaults
-                return _verify_response(utils.run_extractor(sample_path.name, **extractor))
+                return _verify_response(
+                    utils.run_extractor(
+                        sample_path.name,
+                        module_name=extractor["module_name"],
+                        extractor_class=extractor["extractor_class"],
+                        module_path=extractor["module_path"],
+                        venv=extractor["venv"],
+                    )
+                )
         except Exception:
             # caller can deal with the exception
             raise

{maco-1.2.1 → maco-1.2.3}/maco/utils.py

@@ -265,7 +265,7 @@ def find_and_insert_venv(path: str, venvs: List[str]):
     venv = None
     for venv in sorted(venvs, reverse=True):
         venv_parent = os.path.dirname(venv)
-        if path.startswith(venv_parent):
+        if path.startswith(f"{venv_parent}/"):
             # Found the virtual environment that's the closest to extractor
             break
 

{maco-1.2.1 → maco-1.2.3/maco.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.1
+Version: 1.2.3
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License

{maco-1.2.1 → maco-1.2.3}/maco.egg-info/SOURCES.txt

@@ -46,6 +46,7 @@ pipelines/test.yaml
 tests/pytest.ini
 tests/requirements.txt
 tests/test_base_test.py
+tests/test_cli.py
 tests/test_demo_extractors.py
 tests/test_detection.py
 tests/test_extractor.py

{maco-1.2.1 → maco-1.2.3/model_setup}/maco/base_test.py

@@ -46,8 +46,7 @@ class BaseTest(unittest.TestCase):
         runs = self.c.match(stream)
         if not runs:
             raise NoHitException("no yara rule hit")
-        hits = runs[self.name]
-        resp = self.c.extract(stream, hits, self.name)
+        resp = self.c.extract(stream, self.name)
         return resp
 
     def _get_location(self) -> str:

{maco-1.2.1 → maco-1.2.3/model_setup}/maco/cli.py

@@ -8,7 +8,7 @@ import json
 import logging
 import os
 import sys
-from typing import BinaryIO, List
+from typing import BinaryIO, List, Tuple
 
 import cart
 
@@ -57,7 +57,7 @@ def process_file(
         # run and store results for extractor
         logger.info(f"run {extractor_name} extractor from rules {[x.rule for x in hits]}")
         try:
-            resp = collected.extract(stream, hits, extractor_name)
+            resp = collected.extract(stream, extractor_name)
         except Exception as e:
             logger.exception(f"extractor error with {path_file} ({e})")
             resp = None
@@ -91,18 +91,22 @@ def process_filesystem(
     force: bool,
     include_base64: bool,
     create_venv: bool = False,
-):
+) -> Tuple[int, int, int]:
+    """Process filesystem with extractors and print results of extraction.
+
+    Returns total number of analysed files, yara hits and successful maco extractions.
+    """
     if force:
         logger.warning("force execute will cause errors if an extractor requires a yara rule hit during execution")
     collected = collector.Collector(path_extractors, include=include, exclude=exclude, create_venv=create_venv)
 
     logger.info(f"extractors loaded: {[x for x in collected.extractors.keys()]}\n")
     for _, extractor in collected.extractors.items():
-        extractor = extractor["module"]
+        extractor_meta = extractor["metadata"]
         logger.info(
-            f"{extractor.family} by {extractor.author}"
-            f" {extractor.last_modified} {extractor.sharing}"
-            f"\n{extractor.__doc__}\n"
+            f"{extractor_meta['family']} by {extractor_meta['author']}"
+            f" {extractor_meta['last_modified']} {extractor_meta['sharing']}"
+            f"\n{extractor_meta['description']}\n"
         )
 
     num_analysed = 0
@@ -144,7 +148,7 @@ def process_filesystem(
     finally:
         logger.info("")
         logger.info(f"{num_analysed} analysed, {num_hits} hits, {num_extracted} extracted")
-
+    return num_analysed, num_hits, num_extracted
 
 def main():
     parser = argparse.ArgumentParser(description="Run extractors over samples.")

{maco-1.2.1 → maco-1.2.3/model_setup}/maco/collector.py

@@ -40,7 +40,11 @@ def _verify_response(resp: Union[BaseModel, dict]) -> Dict:
 
 class Collector:
     def __init__(
-        self, path_extractors: str, include: List[str] = None, exclude: List[str] = None, create_venv: bool = False
+        self,
+        path_extractors: str,
+        include: List[str] = None,
+        exclude: List[str] = None,
+        create_venv: bool = False,
     ):
         """Discover and load extractors from file system."""
         path_extractors = os.path.realpath(path_extractors)
@@ -72,6 +76,13 @@ class Collector:
                         module_path=module.__file__,
                         module_name=member.__module__,
                         extractor_class=member.__name__,
+                        metadata={
+                            "family": member.family,
+                            "author": member.author,
+                            "last_modified": member.last_modified,
+                            "sharing": member.sharing,
+                            "description": member.__doc__,
+                        },
                     )
                     namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
 
@@ -116,7 +127,6 @@ class Collector:
     def extract(
         self,
         stream: BinaryIO,
-        matches: List[yara.Match],
         extractor_name: str,
     ) -> Dict[str, Any]:
         """Run extractor with stream and verify output matches the model."""
@@ -127,7 +137,15 @@ class Collector:
                 sample_path.write(stream.read())
                 sample_path.flush()
                 # enforce types and verify properties, and remove defaults
-                return _verify_response(utils.run_extractor(sample_path.name, **extractor))
+                return _verify_response(
+                    utils.run_extractor(
+                        sample_path.name,
+                        module_name=extractor["module_name"],
+                        extractor_class=extractor["extractor_class"],
+                        module_path=extractor["module_path"],
+                        venv=extractor["venv"],
+                    )
+                )
         except Exception:
             # caller can deal with the exception
             raise

{maco-1.2.1 → maco-1.2.3}/model_setup/maco/utils.py

@@ -265,7 +265,7 @@ def find_and_insert_venv(path: str, venvs: List[str]):
     venv = None
     for venv in sorted(venvs, reverse=True):
         venv_parent = os.path.dirname(venv)
-        if path.startswith(venv_parent):
+        if path.startswith(f"{venv_parent}/"):
             # Found the virtual environment that's the closest to extractor
             break
 

maco-1.2.3/tests/test_cli.py

@@ -0,0 +1,20 @@
+import os
+
+import unittest
+from maco import cli
+
+
+class TestCLI(unittest.TestCase):
+    def test_process_filesystem(self):
+        maco_path = os.path.abspath(os.path.join(__file__, "../../demo_extractors"))
+        test_path = os.path.abspath(os.path.join(__file__, "../data"))
+        results = cli.process_filesystem(
+            maco_path,
+            test_path,
+            include=[],
+            exclude=[],
+            pretty=True,
+            force=False,
+            include_base64=False,
+        )
+        self.assertEqual(results, (2, 2, 2))

{maco-1.2.1 → maco-1.2.3}/tests/test_helpers.py

@@ -43,11 +43,11 @@ class TestHelpersAnalyseStream(unittest.TestCase):
 
     def test_analyse_stream(self):
         data = b""
-        resp = self.m.extract(io.BytesIO(data), [], "Complex")
+        resp = self.m.extract(io.BytesIO(data), "Complex")
         self.assertEqual(resp, None)
 
         data = b"data"
-        resp = self.m.extract(io.BytesIO(data), [], "Complex")
+        resp = self.m.extract(io.BytesIO(data), "Complex")
         self.assertEqual(
             resp,
             {