maco 1.2.4__py3-none-any.whl → 1.2.6__py3-none-any.whl

demo_extractors/complex/complex.py

@@ -1,9 +1,8 @@
 from io import BytesIO
 from typing import List, Optional
 
-from maco import extractor, model, yara
-
 from demo_extractors.complex import complex_utils
+from maco import extractor, model, yara
 
 
 class Complex(extractor.Extractor):

demo_extractors/elfy.py

@@ -1,5 +1,5 @@
 from io import BytesIO
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 from maco import extractor, model, yara
 

demo_extractors/limit_other.py

@@ -1,9 +1,8 @@
 from io import BytesIO
-from typing import Dict, List, Optional
-
-from maco import extractor, model, yara
+from typing import List, Optional
 
 from demo_extractors import shared
+from maco import extractor, model, yara
 
 
 class LimitOther(extractor.Extractor):
@@ -24,6 +23,14 @@ class LimitOther(extractor.Extractor):
         """
 
     def run(self, stream: BytesIO, matches: List[yara.Match]) -> Optional[model.ExtractorModel]:
+        # import httpx at runtime so we can test that requirements.txt is installed dynamically without breaking
+        # the tests that do direct importing
+        import httpx
+
+        # use httpx so it doesn't get deleted by auto linter
+        if not httpx.__name__:
+            raise Exception("wow I really want to use this library in a useful way")
+
         # use a custom model that inherits from ExtractorModel
         # this model defines what can go in the 'other' dict
         tmp = shared.MyCustomModel(family="specify_other")

demo_extractors/nothing.py

@@ -1,5 +1,5 @@
 from io import BytesIO
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 from maco import extractor, model, yara
 

demo_extractors/requirements.txt

@@ -0,0 +1 @@
+httpx

demo_extractors/shared.py

@@ -1,4 +1,5 @@
 from typing import Optional
+
 import pydantic
 
 from maco import model

maco/base_test.py

@@ -32,14 +32,19 @@ class BaseTest(unittest.TestCase):
     # I recommend something like os.path.join(__file__, "../../extractors")
     # if your extractors are in a folder 'extractors' next to a folder of tests
     path: str = None
+    create_venv: bool = False
 
-    def setUp(self) -> None:
-        if not self.name or not self.path:
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not cls.name or not cls.path:
             raise Exception("name and path must be set")
-        self.c = collector.Collector(self.path, include=[self.name])
+        cls.c = collector.Collector(cls.path, include=[cls.name], create_venv=cls.create_venv)
+        return super().setUpClass()
+
+    def test_default_metadata(self):
+        """Require extractor to be loadable and valid."""
         self.assertIn(self.name, self.c.extractors)
         self.assertEqual(len(self.c.extractors), 1)
-        return super().setUp()
 
     def extract(self, stream):
         """Return results for running extractor over stream, including yara check."""
@@ -49,18 +54,20 @@ class BaseTest(unittest.TestCase):
         resp = self.c.extract(stream, self.name)
         return resp
 
-    def _get_location(self) -> str:
+    @classmethod
+    def _get_location(cls) -> str:
         """Return path to child class that implements this class."""
         # import child module
-        module = type(self).__module__
+        module = cls.__module__
         i = importlib.import_module(module)
         # get location to child module
         return i.__file__
 
-    def load_cart(self, filepath: str) -> io.BytesIO:
+    @classmethod
+    def load_cart(cls, filepath: str) -> io.BytesIO:
         """Load and unneuter a test file (likely malware) into memory for processing."""
         # it is nice if we can load files relative to whatever is implementing base_test
-        dirpath = os.path.split(self._get_location())[0]
+        dirpath = os.path.split(cls._get_location())[0]
         # either filepath is absolute, or should be loaded relative to child of base_test
         filepath = os.path.join(dirpath, filepath)
         if not os.path.isfile(filepath):

maco/cli.py

@@ -1,4 +1,5 @@
 """CLI example of how extractors can be executed."""
+
 import argparse
 import base64
 import binascii
@@ -150,6 +151,7 @@ def process_filesystem(
         logger.info(f"{num_analysed} analysed, {num_hits} hits, {num_extracted} extracted")
     return num_analysed, num_hits, num_extracted
 
+
 def main():
     parser = argparse.ArgumentParser(description="Run extractors over samples.")
     parser.add_argument("extractors", type=str, help="path to extractors")
@@ -165,7 +167,8 @@ def main():
     parser.add_argument(
         "--base64",
         action="store_true",
-        help="Include base64 encoded binary data in output (can be large, consider printing to file rather than console)",
+        help="Include base64 encoded binary data in output "
+        "(can be large, consider printing to file rather than console)",
     )
     parser.add_argument("--logfile", type=str, help="file to log output")
     parser.add_argument("--include", type=str, help="comma separated extractors to run")
@@ -179,7 +182,9 @@ def main():
     parser.add_argument(
         "--create_venv",
         action="store_true",
-        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory)",
+        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). "
+        "This runs much slower than the alternative but may be necessary "
+        "when there are many extractors with conflicting dependencies.",
     )
     args = parser.parse_args()
     inc = args.include.split(",") if args.include else []
@@ -225,7 +230,7 @@ def main():
         pretty=args.pretty,
         force=args.force,
         include_base64=args.base64,
-        create_venv=args.create_venv
+        create_venv=args.create_venv,
     )
 
 

maco/collector.py

@@ -4,6 +4,7 @@ import inspect
 import logging
 import logging.handlers
 import os
+import sys
 from multiprocessing import Manager, Process, Queue
 from tempfile import NamedTemporaryFile
 from types import ModuleType
@@ -48,6 +49,15 @@ class Collector:
         create_venv: bool = False,
     ):
         """Discover and load extractors from file system."""
+        # maco requires the extractor to be imported directly, so ensure they are available on the path
+        full_path_extractors = os.path.abspath(path_extractors)
+        full_path_above_extractors = os.path.dirname(full_path_extractors)
+        # Modify the PATH so we can recognize this new package on import
+        if full_path_extractors not in sys.path:
+            sys.path.insert(1, full_path_extractors)
+        if full_path_above_extractors not in sys.path:
+            sys.path.insert(1, full_path_above_extractors)
+
         path_extractors = os.path.realpath(path_extractors)
         self.path: str = path_extractors
         self.extractors: Dict[str, Dict[str, str]] = {}
@@ -89,7 +99,7 @@ class Collector:
 
             # multiprocess logging is awkward - set up a queue to ensure we can log
             logging_queue = Queue()
-            queue_handler = logging.handlers.QueueListener(logging_queue,*logging.getLogger().handlers)
+            queue_handler = logging.handlers.QueueListener(logging_queue, *logging.getLogger().handlers)
             queue_handler.start()
 
             # Find the extractors within the given directory

maco/extractor.py

@@ -51,14 +51,14 @@ class Extractor:
         # check yara rules conform to expected structure
         # we throw away these compiled rules as we need all rules in system compiled together
         try:
-            rules = yara.compile(source=self.yara_rule)
+            self.yara_compiled = yara.compile(source=self.yara_rule)
         except yara.SyntaxError as e:
             raise InvalidExtractor(f"{self.name} - invalid yara rule") from e
         # need to track which plugin owns the rules
-        self.yara_rule_names = [x.identifier for x in rules]
-        if not len(list(rules)):
+        self.yara_rule_names = [x.identifier for x in self.yara_compiled]
+        if not len(list(self.yara_compiled)):
             raise InvalidExtractor(f"{name} must define at least one yara rule")
-        for x in rules:
+        for x in self.yara_compiled:
             if x.is_global:
                 raise InvalidExtractor(f"{x.identifier} yara rule must not be global")
 

maco/model/__init__.py

@@ -1 +1 @@
-from maco.model.model import *
+from maco.model.model import *  # noqa: F403

maco/model/model.py

@@ -59,31 +59,48 @@ class CategoryEnum(str, Enum):
     # Malware related to an Advanced Persistent Threat (APT) group.
     apt = "apt"
 
-    # A backdoor Trojan gives malicious users remote control over the infected computer. They enable the author to do anything they wish on the infected computer including sending, receiving, launching and deleting files, displaying data and rebooting the computer. Backdoor Trojans are often used to unite a group of victim computers to form a botnet or zombie network that can be used for criminal purposes.
+    # A backdoor Trojan gives malicious users remote control over the infected computer.
+    # They enable the author to do anything they wish on the infected computer including
+    # sending, receiving, launching and deleting files, displaying data and rebooting the computer.
+    # Backdoor Trojans are often used to unite a group of victim computers to form a botnet or
+    # zombie network that can be used for criminal purposes.
     backdoor = "backdoor"
 
-    # Trojan Banker programs are designed to steal your account data for online banking systems, e-payment systems and credit or debit cards.
+    # Trojan Banker programs are designed to steal your account data for online banking systems,
+    # e-payment systems and credit or debit cards.
     banker = "banker"
 
-    # A malware variant that modifies the boot sectors of a hard drive, including the Master Boot Record (MBR) and Volume Boot Record (VBR).
+    # A malware variant that modifies the boot sectors of a hard drive, including the Master Boot Record (MBR)
+    # and Volume Boot Record (VBR).
     bootkit = "bootkit"
 
-    # A malicious bot is self-propagating malware designed to infect a host and connect back to a central server or servers that act as a command and control (C&C) center for an entire network of compromised devices, or botnet.
+    # A malicious bot is self-propagating malware designed to infect a host and connect back to a central server
+    # or servers that act as a command and control (C&C) center for an entire network of compromised devices,
+    # or botnet.
     bot = "bot"
 
-    # A browser hijacker is defined as a form of unwanted software that modifies a web browser's settings without the user's permission. The result is the placement of unwanted advertising into the browser, and possibly the replacement of an existing home page or search page with the hijacker page.
+    # A browser hijacker is defined as a form of unwanted software that modifies a web browser's settings without
+    # the user's permission. The result is the placement of unwanted advertising into the browser,
+    # and possibly the replacement of an existing home page or search page with the hijacker page.
     browser_hijacker = "browser_hijacker"
 
-    # Trojan bruteforcer are trying to brute force website in order to achieve something else (EX: Finding  WordPress websites with default credentials).
+    # Trojan bruteforcer are trying to brute force website in order to achieve something else
+    # (EX: Finding  WordPress websites with default credentials).
     bruteforcer = "bruteforcer"
 
-    # A type of trojan that can use your PC to 'click' on websites or applications. They are usually used to make money for a malicious hacker by clicking on online advertisements and making it look like the website gets more traffic than it does. They can also be used to skew online polls, install programs on your PC, or make unwanted software appear more popular than it is.
+    # A type of trojan that can use your PC to 'click' on websites or applications.
+    # They are usually used to make money for a malicious hacker by clicking on online advertisements
+    # and making it look like the website gets more traffic than it does.
+    # They can also be used to skew online polls, install programs on your PC, or make unwanted software
+    # appear more popular than it is.
     clickfraud = "clickfraud"
 
     # Cryptocurrency mining malware.
     cryptominer = "cryptominer"
 
-    # These programs conduct DoS (Denial of Service) attacks against a targeted web address. By sending multiple requests from your computer and several other infected computers, the attack can overwhelm the target address leading to a denial of service.
+    # These programs conduct DoS (Denial of Service) attacks against a targeted web address.
+    # By sending multiple requests from your computer and several other infected computers,
+    # the attack can overwhelm the target address leading to a denial of service.
     ddos = "ddos"
 
     # Trojan Downloaders can download and install new versions of malicious programs in the target system.
@@ -92,49 +109,66 @@ class CategoryEnum(str, Enum):
     # These programs are used by hackers in order to install malware or to prevent the detection of malicious programs.
     dropper = "dropper"
 
-    # Exploit kits are programs that contain data or code that takes advantage of a vulnerability within an application that is running in the target system.
+    # Exploit kits are programs that contain data or code that takes advantage of a vulnerability
+    # within an application that is running in the target system.
     exploitkit = "exploitkit"
 
-    # Trojan FakeAV programs simulate the activity of antivirus software. They are designed to extort money in return for the detection and removal of threat, even though the threats that they report are actually non-existent.
+    # Trojan FakeAV programs simulate the activity of antivirus software.
+    # They are designed to extort money in return for the detection and removal of threat, even though the
+    # threats that they report are actually non-existent.
     fakeav = "fakeav"
 
     # A type of tool that can be used to allow and maintain unauthorized access to your PC.
     hacktool = "hacktool"
 
-    # A program that collects your personal information, such as your browsing history, and uses it without adequate consent.
+    # A program that collects your personal information, such as your browsing history,
+    # and uses it without adequate consent.
     infostealer = "infostealer"
 
-    # A keylogger monitors and logs every keystroke it can identify. Once installed, the virus either keeps track of all the keys and stores the information locally, after which the hacker needs physical access to the computer to retrieve the information, or the logs are sent over the internet back to the hacker.
+    # A keylogger monitors and logs every keystroke it can identify.
+    # Once installed, the virus either keeps track of all the keys and stores the information locally,
+    # after which the hacker needs physical access to the computer to retrieve the information,
+    # or the logs are sent over the internet back to the hacker.
     keylogger = "keylogger"
 
     # A program that loads another application / memory space.
     loader = "loader"
 
-    # A type of malware that hides its code and purpose to make it more difficult for security software to detect or remove it.
+    # A type of malware that hides its code and purpose to make it more difficult for
+    # security software to detect or remove it.
     obfuscator = "obfuscator"
 
-    # Point-of-sale malware is usually a type of malware that is used by cybercriminals to target point of sale (POS) and payment terminals with the intent to obtain credit card and debit card information.
+    # Point-of-sale malware is usually a type of malware that is used by cybercriminals to target point of sale (POS)
+    # and payment terminals with the intent to obtain credit card and debit card information.
     pos = "pos"
 
-    # This type of trojan allows unauthorized parties to use the infected computer as a proxy server to access the Internet anonymously.
+    # This type of trojan allows unauthorized parties to use the infected computer as a proxy server
+    # to access the Internet anonymously.
     proxy = "proxy"
 
     # A program that can be used by a remote hacker to gain access and control of an infected machine.
     rat = "rat"
 
-    # This type of malware can modify data in the target computer so the operating system will stop running correctly or the data is no longer accessible. The criminal will only restore the computer state or data after a ransom is paid to them (mostly using cryptocurrency).
+    # This type of malware can modify data in the target computer so the operating system
+    # will stop running correctly or the data is no longer accessible.
+    # The criminal will only restore the computer state or data after a ransom is paid to them
+    # (mostly using cryptocurrency).
     ransomware = "ransomware"
 
     # A reverse proxy is a server that receives requests from the internet and forwards them to a small set of servers.
     reverse_proxy = "reverse_proxy"
 
-    # Rootkits are designed to conceal certain objects or activities in the system. Often their main purpose is to prevent malicious programs being detected in order to extend the period in which programs can run on an infected computer.
+    # Rootkits are designed to conceal certain objects or activities in the system.
+    # Often their main purpose is to prevent malicious programs being detected
+    # in order to extend the period in which programs can run on an infected computer.
     rootkit = "rootkit"
 
-    # This type of malware scan the internet / network(s) / system(s) / service(s) to collect information. That information could be used later to perpetuate an cyber attack.
+    # This type of malware scan the internet / network(s) / system(s) / service(s) to collect information.
+    # That information could be used later to perpetuate an cyber attack.
     scanner = "scanner"
 
-    # Scareware is a form of malware which uses social engineering to cause shock, anxiety, or the perception of a threat in order to manipulate users into buying unwanted software.
+    # Scareware is a form of malware which uses social engineering to cause shock, anxiety,
+    # or the perception of a threat in order to manipulate users into buying unwanted software.
     scareware = "scareware"
 
     # Malware that is sending spam.