maco 1.2.5__py3-none-any.whl → 1.2.6__py3-none-any.whl

demo_extractors/elfy.py

@@ -1,5 +1,5 @@
 from io import BytesIO
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 from maco import extractor, model, yara
 

demo_extractors/limit_other.py

@@ -1,5 +1,5 @@
 from io import BytesIO
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 from demo_extractors import shared
 from maco import extractor, model, yara
@@ -27,6 +27,10 @@ class LimitOther(extractor.Extractor):
         # the tests that do direct importing
         import httpx
 
+        # use httpx so it doesn't get deleted by auto linter
+        if not httpx.__name__:
+            raise Exception("wow I really want to use this library in a useful way")
+
         # use a custom model that inherits from ExtractorModel
         # this model defines what can go in the 'other' dict
         tmp = shared.MyCustomModel(family="specify_other")

demo_extractors/nothing.py

@@ -1,5 +1,5 @@
 from io import BytesIO
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 from maco import extractor, model, yara
 

maco/base_test.py

@@ -32,7 +32,7 @@ class BaseTest(unittest.TestCase):
     # I recommend something like os.path.join(__file__, "../../extractors")
     # if your extractors are in a folder 'extractors' next to a folder of tests
     path: str = None
-    create_venv: bool=False
+    create_venv: bool = False
 
     @classmethod
     def setUpClass(cls) -> None:

maco/cli.py

@@ -1,4 +1,5 @@
 """CLI example of how extractors can be executed."""
+
 import argparse
 import base64
 import binascii
@@ -150,6 +151,7 @@ def process_filesystem(
         logger.info(f"{num_analysed} analysed, {num_hits} hits, {num_extracted} extracted")
     return num_analysed, num_hits, num_extracted
 
+
 def main():
     parser = argparse.ArgumentParser(description="Run extractors over samples.")
     parser.add_argument("extractors", type=str, help="path to extractors")
@@ -165,7 +167,8 @@ def main():
     parser.add_argument(
         "--base64",
         action="store_true",
-        help="Include base64 encoded binary data in output (can be large, consider printing to file rather than console)",
+        help="Include base64 encoded binary data in output "
+        "(can be large, consider printing to file rather than console)",
     )
     parser.add_argument("--logfile", type=str, help="file to log output")
     parser.add_argument("--include", type=str, help="comma separated extractors to run")
@@ -179,7 +182,9 @@ def main():
     parser.add_argument(
         "--create_venv",
         action="store_true",
-        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). This runs much slower than the alternative but may be necessary when there are many extractors with conflicting dependencies.",
+        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). "
+        "This runs much slower than the alternative but may be necessary "
+        "when there are many extractors with conflicting dependencies.",
     )
     args = parser.parse_args()
     inc = args.include.split(",") if args.include else []
@@ -225,7 +230,7 @@ def main():
         pretty=args.pretty,
         force=args.force,
         include_base64=args.base64,
-        create_venv=args.create_venv
+        create_venv=args.create_venv,
     )
 
 

maco/collector.py

@@ -4,6 +4,7 @@ import inspect
 import logging
 import logging.handlers
 import os
+import sys
 from multiprocessing import Manager, Process, Queue
 from tempfile import NamedTemporaryFile
 from types import ModuleType
@@ -48,6 +49,15 @@ class Collector:
         create_venv: bool = False,
     ):
         """Discover and load extractors from file system."""
+        # maco requires the extractor to be imported directly, so ensure they are available on the path
+        full_path_extractors = os.path.abspath(path_extractors)
+        full_path_above_extractors = os.path.dirname(full_path_extractors)
+        # Modify the PATH so we can recognize this new package on import
+        if full_path_extractors not in sys.path:
+            sys.path.insert(1, full_path_extractors)
+        if full_path_above_extractors not in sys.path:
+            sys.path.insert(1, full_path_above_extractors)
+
         path_extractors = os.path.realpath(path_extractors)
         self.path: str = path_extractors
         self.extractors: Dict[str, Dict[str, str]] = {}
@@ -89,7 +99,7 @@ class Collector:
 
             # multiprocess logging is awkward - set up a queue to ensure we can log
             logging_queue = Queue()
-            queue_handler = logging.handlers.QueueListener(logging_queue,*logging.getLogger().handlers)
+            queue_handler = logging.handlers.QueueListener(logging_queue, *logging.getLogger().handlers)
             queue_handler.start()
 
             # Find the extractors within the given directory

maco/model/__init__.py

@@ -1 +1 @@
-from maco.model.model import *
+from maco.model.model import *  # noqa: F403

maco/model/model.py

@@ -59,31 +59,48 @@ class CategoryEnum(str, Enum):
     # Malware related to an Advanced Persistent Threat (APT) group.
     apt = "apt"
 
-    # A backdoor Trojan gives malicious users remote control over the infected computer. They enable the author to do anything they wish on the infected computer including sending, receiving, launching and deleting files, displaying data and rebooting the computer. Backdoor Trojans are often used to unite a group of victim computers to form a botnet or zombie network that can be used for criminal purposes.
+    # A backdoor Trojan gives malicious users remote control over the infected computer.
+    # They enable the author to do anything they wish on the infected computer including
+    # sending, receiving, launching and deleting files, displaying data and rebooting the computer.
+    # Backdoor Trojans are often used to unite a group of victim computers to form a botnet or
+    # zombie network that can be used for criminal purposes.
     backdoor = "backdoor"
 
-    # Trojan Banker programs are designed to steal your account data for online banking systems, e-payment systems and credit or debit cards.
+    # Trojan Banker programs are designed to steal your account data for online banking systems,
+    # e-payment systems and credit or debit cards.
     banker = "banker"
 
-    # A malware variant that modifies the boot sectors of a hard drive, including the Master Boot Record (MBR) and Volume Boot Record (VBR).
+    # A malware variant that modifies the boot sectors of a hard drive, including the Master Boot Record (MBR)
+    # and Volume Boot Record (VBR).
     bootkit = "bootkit"
 
-    # A malicious bot is self-propagating malware designed to infect a host and connect back to a central server or servers that act as a command and control (C&C) center for an entire network of compromised devices, or botnet.
+    # A malicious bot is self-propagating malware designed to infect a host and connect back to a central server
+    # or servers that act as a command and control (C&C) center for an entire network of compromised devices,
+    # or botnet.
     bot = "bot"
 
-    # A browser hijacker is defined as a form of unwanted software that modifies a web browser's settings without the user's permission. The result is the placement of unwanted advertising into the browser, and possibly the replacement of an existing home page or search page with the hijacker page.
+    # A browser hijacker is defined as a form of unwanted software that modifies a web browser's settings without
+    # the user's permission. The result is the placement of unwanted advertising into the browser,
+    # and possibly the replacement of an existing home page or search page with the hijacker page.
     browser_hijacker = "browser_hijacker"
 
-    # Trojan bruteforcer are trying to brute force website in order to achieve something else (EX: Finding  WordPress websites with default credentials).
+    # Trojan bruteforcer are trying to brute force website in order to achieve something else
+    # (EX: Finding  WordPress websites with default credentials).
     bruteforcer = "bruteforcer"
 
-    # A type of trojan that can use your PC to 'click' on websites or applications. They are usually used to make money for a malicious hacker by clicking on online advertisements and making it look like the website gets more traffic than it does. They can also be used to skew online polls, install programs on your PC, or make unwanted software appear more popular than it is.
+    # A type of trojan that can use your PC to 'click' on websites or applications.
+    # They are usually used to make money for a malicious hacker by clicking on online advertisements
+    # and making it look like the website gets more traffic than it does.
+    # They can also be used to skew online polls, install programs on your PC, or make unwanted software
+    # appear more popular than it is.
     clickfraud = "clickfraud"
 
     # Cryptocurrency mining malware.
     cryptominer = "cryptominer"
 
-    # These programs conduct DoS (Denial of Service) attacks against a targeted web address. By sending multiple requests from your computer and several other infected computers, the attack can overwhelm the target address leading to a denial of service.
+    # These programs conduct DoS (Denial of Service) attacks against a targeted web address.
+    # By sending multiple requests from your computer and several other infected computers,
+    # the attack can overwhelm the target address leading to a denial of service.
     ddos = "ddos"
 
     # Trojan Downloaders can download and install new versions of malicious programs in the target system.
@@ -92,49 +109,66 @@ class CategoryEnum(str, Enum):
     # These programs are used by hackers in order to install malware or to prevent the detection of malicious programs.
     dropper = "dropper"
 
-    # Exploit kits are programs that contain data or code that takes advantage of a vulnerability within an application that is running in the target system.
+    # Exploit kits are programs that contain data or code that takes advantage of a vulnerability
+    # within an application that is running in the target system.
     exploitkit = "exploitkit"
 
-    # Trojan FakeAV programs simulate the activity of antivirus software. They are designed to extort money in return for the detection and removal of threat, even though the threats that they report are actually non-existent.
+    # Trojan FakeAV programs simulate the activity of antivirus software.
+    # They are designed to extort money in return for the detection and removal of threat, even though the
+    # threats that they report are actually non-existent.
     fakeav = "fakeav"
 
     # A type of tool that can be used to allow and maintain unauthorized access to your PC.
     hacktool = "hacktool"
 
-    # A program that collects your personal information, such as your browsing history, and uses it without adequate consent.
+    # A program that collects your personal information, such as your browsing history,
+    # and uses it without adequate consent.
     infostealer = "infostealer"
 
-    # A keylogger monitors and logs every keystroke it can identify. Once installed, the virus either keeps track of all the keys and stores the information locally, after which the hacker needs physical access to the computer to retrieve the information, or the logs are sent over the internet back to the hacker.
+    # A keylogger monitors and logs every keystroke it can identify.
+    # Once installed, the virus either keeps track of all the keys and stores the information locally,
+    # after which the hacker needs physical access to the computer to retrieve the information,
+    # or the logs are sent over the internet back to the hacker.
     keylogger = "keylogger"
 
     # A program that loads another application / memory space.
     loader = "loader"
 
-    # A type of malware that hides its code and purpose to make it more difficult for security software to detect or remove it.
+    # A type of malware that hides its code and purpose to make it more difficult for
+    # security software to detect or remove it.
     obfuscator = "obfuscator"
 
-    # Point-of-sale malware is usually a type of malware that is used by cybercriminals to target point of sale (POS) and payment terminals with the intent to obtain credit card and debit card information.
+    # Point-of-sale malware is usually a type of malware that is used by cybercriminals to target point of sale (POS)
+    # and payment terminals with the intent to obtain credit card and debit card information.
     pos = "pos"
 
-    # This type of trojan allows unauthorized parties to use the infected computer as a proxy server to access the Internet anonymously.
+    # This type of trojan allows unauthorized parties to use the infected computer as a proxy server
+    # to access the Internet anonymously.
     proxy = "proxy"
 
     # A program that can be used by a remote hacker to gain access and control of an infected machine.
     rat = "rat"
 
-    # This type of malware can modify data in the target computer so the operating system will stop running correctly or the data is no longer accessible. The criminal will only restore the computer state or data after a ransom is paid to them (mostly using cryptocurrency).
+    # This type of malware can modify data in the target computer so the operating system
+    # will stop running correctly or the data is no longer accessible.
+    # The criminal will only restore the computer state or data after a ransom is paid to them
+    # (mostly using cryptocurrency).
     ransomware = "ransomware"
 
     # A reverse proxy is a server that receives requests from the internet and forwards them to a small set of servers.
     reverse_proxy = "reverse_proxy"
 
-    # Rootkits are designed to conceal certain objects or activities in the system. Often their main purpose is to prevent malicious programs being detected in order to extend the period in which programs can run on an infected computer.
+    # Rootkits are designed to conceal certain objects or activities in the system.
+    # Often their main purpose is to prevent malicious programs being detected
+    # in order to extend the period in which programs can run on an infected computer.
     rootkit = "rootkit"
 
-    # This type of malware scan the internet / network(s) / system(s) / service(s) to collect information. That information could be used later to perpetuate an cyber attack.
+    # This type of malware scan the internet / network(s) / system(s) / service(s) to collect information.
+    # That information could be used later to perpetuate an cyber attack.
     scanner = "scanner"
 
-    # Scareware is a form of malware which uses social engineering to cause shock, anxiety, or the perception of a threat in order to manipulate users into buying unwanted software.
+    # Scareware is a form of malware which uses social engineering to cause shock, anxiety,
+    # or the perception of a threat in order to manipulate users into buying unwanted software.
     scareware = "scareware"
 
     # Malware that is sending spam.

maco/utils.py

@@ -46,6 +46,7 @@ UV_BIN = find_uv_bin()
 PIP_CMD = f"{UV_BIN} pip"
 VENV_CREATE_CMD = f"{UV_BIN} venv"
 
+
 class Base64Decoder(json.JSONDecoder):
     def __init__(self, *args, **kwargs):
         json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)
@@ -179,11 +180,9 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
                         for pattern in [RELATIVE_FROM_IMPORT_RE, RELATIVE_FROM_RE]:
                             for match in pattern.findall(data):
                                 depth = match.count(".")
-                                data = data.replace(
-                                    f"from {match}",
-                                    f"from {'.'.join(split[depth - 1 : split.index(package) + 1][::-1])}{'.' if pattern == RELATIVE_FROM_RE else ''}",
-                                    1,
-                                )
+                                abspath = ".".join(split[depth - 1 : split.index(package) + 1][::-1])
+                                abspath += "." if pattern == RELATIVE_FROM_RE else ""
+                                data = data.replace(f"from {match}", f"from {abspath}", 1)
                         f.write(data)
 
                 if scanner.match(path):
@@ -222,7 +221,12 @@ def _install_required_packages(create_venv: bool, directories: List[str], python
                         subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
 
                 # Install/Update the packages in the environment
-                install_command = PIP_CMD.split(" ") + ["install", "-U"]
+                install_command = PIP_CMD.split(" ") + ["install"]
+                # When running locally, only install packages to required spec.
+                # This prevents issues during maco development and building extractors against local libraries.
+                if create_venv:
+                    # when running in custom virtual environment, always upgrade packages.
+                    install_command.append("-U")
 
                 # Update the pip install command depending on where the dependencies are coming from
                 if "requirements.txt" in req_files:
@@ -307,8 +311,10 @@ def register_extractors(
     parent_directory = os.path.dirname(current_directory)
     if venvs and package_name in sys.modules:
         # this may happen as part of testing if some part of the extractor code was directly imported
-        logger.warning(f"Looks like {package_name} is already loaded. "
-                       "If your maco extractor overlaps an existing package name this could cause problems.")
+        logger.warning(
+            f"Looks like {package_name} is already loaded. "
+            "If your maco extractor overlaps an existing package name this could cause problems."
+        )
 
     try:
         # Modify the PATH so we can recognize this new package on import
@@ -383,6 +389,7 @@ def register_extractors(
                 # We were able to find all the extractor files
                 break
 
+
 def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType, str], None], *args, **kwargs):
     """Ensures logging is set up correctly for a child process and then executes the callback."""
     logger = logging.getLogger()
@@ -391,6 +398,7 @@ def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType,
     logger.addHandler(qh)
     callback(*args, **kwargs, logger=logger)
 
+
 def import_extractors(
     extractor_module_callback: Callable[[ModuleType, str], bool],
     *,
@@ -416,6 +424,7 @@ def import_extractors(
 # holds cached extractors when not running in venv mode
 _loaded_extractors: Dict[str, Extractor] = {}
 
+
 def run_extractor(
     sample_path,
     module_name,
@@ -438,7 +447,7 @@ def run_extractor(
             extractor = _loaded_extractors[key]
         if extractor.yara_compiled:
             matches = extractor.yara_compiled.match(sample_path)
-        loaded = extractor.run(open(sample_path, 'rb'), matches=matches)
+        loaded = extractor.run(open(sample_path, "rb"), matches=matches)
     else:
         # execute extractor in child process with separate virtual environment
         # Write temporary script in the same directory as extractor to resolve relative imports
@@ -477,7 +486,7 @@ def run_extractor(
                 try:
                     # Load results and return them
                     output.seek(0)
-                    loaded =  json.load(output, cls=json_decoder)
+                    loaded = json.load(output, cls=json_decoder)
                 except Exception as e:
                     # If there was an error raised during runtime, then propagate
                     delim = f'File "{module_path}"'

maco/yara.py

@@ -3,7 +3,6 @@ from collections import namedtuple
 from itertools import cycle
 from typing import Dict
 
-import yara
 import yara_x
 
 RULE_ID_RE = re.compile("(\w+)? ?rule (\w+)")

{maco-1.2.5.dist-info → maco-1.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.5
+Version: 1.2.6
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License
@@ -38,7 +38,7 @@ Requires-Dist: yara-x==0.11.0
 
 # Maco - Malware config extractor framework
 
-## Maco is a framework for ***ma***lware ***co***nfig extractors.
+## Maco is a framework for <ins>ma</ins>lware <ins>co</ins>nfig extractors.
 
 It aims to solve two problems:
 
@@ -272,3 +272,15 @@ run Complex extractor from rules ['ComplexAlt']
 The demo extractors are designed to trigger when run over the '`demo_extractors`' folder.
 
 e.g. `maco demo_extractors demo_extractors`
+
+# Contributions
+
+Please use ruff to format and lint PRs. This may be the cause of PR test failures.
+
+Ruff will attempt to fix most issues, but some may require manual resolution.
+
+```
+pip install ruff
+ruff format
+ruff check --fix
+```

maco-1.2.6.dist-info/RECORD

@@ -0,0 +1,44 @@
+demo_extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+demo_extractors/elfy.py,sha256=Jo_GKExCeFOKGENJnNM_9ONfJO7LQFucCNz0ryTAo9U,765
+demo_extractors/limit_other.py,sha256=BWjeyOxB75kw4eRla5zvSzdcXtELOS8R6hc71rLPh1s,1295
+demo_extractors/nothing.py,sha256=MNPlb0IsBjrlU5e438JlJ4DIKoBpBRAaYY3JhD3yHqk,601
+demo_extractors/requirements.txt,sha256=E0tD6xBZldq6sQGTHng6k88lBeASOhmLJcdcjpcqBNE,6
+demo_extractors/shared.py,sha256=2P1cyuRbHDvM9IRt3UZnwdyhxx7OWqNC83xLyV8Y190,305
+demo_extractors/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+demo_extractors/complex/complex.py,sha256=tXrzj_zWIXbTOwj7Lezapk-qkrM-lfwcyjd5m-BYzdg,2322
+demo_extractors/complex/complex_utils.py,sha256=aec8kJsYUrMPo-waihkVLt-0QpiOPkw7dDqfT9MNuHk,123
+maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+maco/base_test.py,sha256=cjGLEy2c69wl9sjn74QFz7X-VxWOfdin4W8MvYsXc4Q,2718
+maco/cli.py,sha256=NTzV8eu9V0qQNttRo592j-Rdzac7q1NAMraqJF2h_6k,8171
+maco/collector.py,sha256=LraWYlCA72FCmQP0dHWc-ekd7R1SxR6h6rMD95_6mMs,7077
+maco/extractor.py,sha256=uGSGiCQ4jd8jFmfw2T99BGcY5iQJzXHcG_RoTIxClTE,2802
+maco/utils.py,sha256=vo8zGSoFP0k2APUlQXmLssdrVrknjS2YcpvbRM78J68,20480
+maco/yara.py,sha256=8RVaGyeUWY5f8_wfQ25lDX1bcXsb_VoSja85ZC2SqGw,2913
+maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
+maco/model/model.py,sha256=4uY88WphbP3iu-L2WjuYwtgZCS_wNul_hr0bAVuTpvc,23740
+model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+model_setup/maco/base_test.py,sha256=cjGLEy2c69wl9sjn74QFz7X-VxWOfdin4W8MvYsXc4Q,2718
+model_setup/maco/cli.py,sha256=NTzV8eu9V0qQNttRo592j-Rdzac7q1NAMraqJF2h_6k,8171
+model_setup/maco/collector.py,sha256=LraWYlCA72FCmQP0dHWc-ekd7R1SxR6h6rMD95_6mMs,7077
+model_setup/maco/extractor.py,sha256=uGSGiCQ4jd8jFmfw2T99BGcY5iQJzXHcG_RoTIxClTE,2802
+model_setup/maco/utils.py,sha256=vo8zGSoFP0k2APUlQXmLssdrVrknjS2YcpvbRM78J68,20480
+model_setup/maco/yara.py,sha256=8RVaGyeUWY5f8_wfQ25lDX1bcXsb_VoSja85ZC2SqGw,2913
+model_setup/maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
+model_setup/maco/model/model.py,sha256=4uY88WphbP3iu-L2WjuYwtgZCS_wNul_hr0bAVuTpvc,23740
+pipelines/publish.yaml,sha256=xt3WNU-5kIICJgKIiiE94M3dWjS3uEiun-n4OmIssK8,1471
+pipelines/test.yaml,sha256=btJVI-R39UBeYosGu7TOpU6V9ogFW3FT3ROtWygQGQ0,1472
+tests/data/example.txt.cart,sha256=j4ZdDnFNVq7lb-Qi4pY4evOXKQPKG-GSg-n-uEqPhV0,289
+tests/data/trigger_complex.txt,sha256=uqnLSrnyDGCmXwuPmZ2s8vdhH0hJs8DxvyaW_tuYY24,64
+tests/data/trigger_complex.txt.cart,sha256=Z7qF1Zi640O45Znkl9ooP2RhSLAEqY0NRf51d-q7utU,345
+tests/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/extractors/basic.py,sha256=r5eLCL6Ynr14nCBgtbLvUbm0NdrXizyc9c-4xBCNShU,828
+tests/extractors/basic_longer.py,sha256=1ClU2QD-Y0TOl_loNFvEqIEpTR5TSVJ6zg9ZmC-ESJo,860
+tests/extractors/test_basic.py,sha256=FLKekfSGM69HaiF7Vu_7D7KDXHZko-9hZkMO8_DoyYA,697
+tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/extractors/bob/bob.py,sha256=G5aOoz58J0ZQK2_lA7HRxAzeLzBxssWxBTZcv1pSbi8,176
+maco-1.2.6.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
+maco-1.2.6.dist-info/METADATA,sha256=-ZxVnBbAHn-iGDizJJa8knrTf4DUArPzANp-SVqDzZ4,15855
+maco-1.2.6.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+maco-1.2.6.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
+maco-1.2.6.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
+maco-1.2.6.dist-info/RECORD,,

model_setup/maco/base_test.py

@@ -32,7 +32,7 @@ class BaseTest(unittest.TestCase):
     # I recommend something like os.path.join(__file__, "../../extractors")
     # if your extractors are in a folder 'extractors' next to a folder of tests
     path: str = None
-    create_venv: bool=False
+    create_venv: bool = False
 
     @classmethod
     def setUpClass(cls) -> None:

model_setup/maco/cli.py

@@ -1,4 +1,5 @@
 """CLI example of how extractors can be executed."""
+
 import argparse
 import base64
 import binascii
@@ -150,6 +151,7 @@ def process_filesystem(
         logger.info(f"{num_analysed} analysed, {num_hits} hits, {num_extracted} extracted")
     return num_analysed, num_hits, num_extracted
 
+
 def main():
     parser = argparse.ArgumentParser(description="Run extractors over samples.")
     parser.add_argument("extractors", type=str, help="path to extractors")
@@ -165,7 +167,8 @@ def main():
     parser.add_argument(
         "--base64",
         action="store_true",
-        help="Include base64 encoded binary data in output (can be large, consider printing to file rather than console)",
+        help="Include base64 encoded binary data in output "
+        "(can be large, consider printing to file rather than console)",
     )
     parser.add_argument("--logfile", type=str, help="file to log output")
     parser.add_argument("--include", type=str, help="comma separated extractors to run")
@@ -179,7 +182,9 @@ def main():
     parser.add_argument(
         "--create_venv",
         action="store_true",
-        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). This runs much slower than the alternative but may be necessary when there are many extractors with conflicting dependencies.",
+        help="Creates venvs for every requirements.txt found (only applies when extractor path is a directory). "
+        "This runs much slower than the alternative but may be necessary "
+        "when there are many extractors with conflicting dependencies.",
     )
     args = parser.parse_args()
     inc = args.include.split(",") if args.include else []
@@ -225,7 +230,7 @@ def main():
         pretty=args.pretty,
         force=args.force,
         include_base64=args.base64,
-        create_venv=args.create_venv
+        create_venv=args.create_venv,
     )
 
 

model_setup/maco/collector.py

@@ -4,6 +4,7 @@ import inspect
 import logging
 import logging.handlers
 import os
+import sys
 from multiprocessing import Manager, Process, Queue
 from tempfile import NamedTemporaryFile
 from types import ModuleType
@@ -48,6 +49,15 @@ class Collector:
         create_venv: bool = False,
     ):
         """Discover and load extractors from file system."""
+        # maco requires the extractor to be imported directly, so ensure they are available on the path
+        full_path_extractors = os.path.abspath(path_extractors)
+        full_path_above_extractors = os.path.dirname(full_path_extractors)
+        # Modify the PATH so we can recognize this new package on import
+        if full_path_extractors not in sys.path:
+            sys.path.insert(1, full_path_extractors)
+        if full_path_above_extractors not in sys.path:
+            sys.path.insert(1, full_path_above_extractors)
+
         path_extractors = os.path.realpath(path_extractors)
         self.path: str = path_extractors
         self.extractors: Dict[str, Dict[str, str]] = {}
@@ -89,7 +99,7 @@ class Collector:
 
             # multiprocess logging is awkward - set up a queue to ensure we can log
             logging_queue = Queue()
-            queue_handler = logging.handlers.QueueListener(logging_queue,*logging.getLogger().handlers)
+            queue_handler = logging.handlers.QueueListener(logging_queue, *logging.getLogger().handlers)
             queue_handler.start()
 
             # Find the extractors within the given directory

model_setup/maco/model/__init__.py

@@ -1 +1 @@
-from maco.model.model import *
+from maco.model.model import *  # noqa: F403

model_setup/maco/model/model.py

@@ -59,31 +59,48 @@ class CategoryEnum(str, Enum):
     # Malware related to an Advanced Persistent Threat (APT) group.
     apt = "apt"
 
-    # A backdoor Trojan gives malicious users remote control over the infected computer. They enable the author to do anything they wish on the infected computer including sending, receiving, launching and deleting files, displaying data and rebooting the computer. Backdoor Trojans are often used to unite a group of victim computers to form a botnet or zombie network that can be used for criminal purposes.
+    # A backdoor Trojan gives malicious users remote control over the infected computer.
+    # They enable the author to do anything they wish on the infected computer including
+    # sending, receiving, launching and deleting files, displaying data and rebooting the computer.
+    # Backdoor Trojans are often used to unite a group of victim computers to form a botnet or
+    # zombie network that can be used for criminal purposes.
     backdoor = "backdoor"
 
-    # Trojan Banker programs are designed to steal your account data for online banking systems, e-payment systems and credit or debit cards.
+    # Trojan Banker programs are designed to steal your account data for online banking systems,
+    # e-payment systems and credit or debit cards.
     banker = "banker"
 
-    # A malware variant that modifies the boot sectors of a hard drive, including the Master Boot Record (MBR) and Volume Boot Record (VBR).
+    # A malware variant that modifies the boot sectors of a hard drive, including the Master Boot Record (MBR)
+    # and Volume Boot Record (VBR).
     bootkit = "bootkit"
 
-    # A malicious bot is self-propagating malware designed to infect a host and connect back to a central server or servers that act as a command and control (C&C) center for an entire network of compromised devices, or botnet.
+    # A malicious bot is self-propagating malware designed to infect a host and connect back to a central server
+    # or servers that act as a command and control (C&C) center for an entire network of compromised devices,
+    # or botnet.
     bot = "bot"
 
-    # A browser hijacker is defined as a form of unwanted software that modifies a web browser's settings without the user's permission. The result is the placement of unwanted advertising into the browser, and possibly the replacement of an existing home page or search page with the hijacker page.
+    # A browser hijacker is defined as a form of unwanted software that modifies a web browser's settings without
+    # the user's permission. The result is the placement of unwanted advertising into the browser,
+    # and possibly the replacement of an existing home page or search page with the hijacker page.
     browser_hijacker = "browser_hijacker"
 
-    # Trojan bruteforcer are trying to brute force website in order to achieve something else (EX: Finding  WordPress websites with default credentials).
+    # Trojan bruteforcer are trying to brute force website in order to achieve something else
+    # (EX: Finding  WordPress websites with default credentials).
     bruteforcer = "bruteforcer"
 
-    # A type of trojan that can use your PC to 'click' on websites or applications. They are usually used to make money for a malicious hacker by clicking on online advertisements and making it look like the website gets more traffic than it does. They can also be used to skew online polls, install programs on your PC, or make unwanted software appear more popular than it is.
+    # A type of trojan that can use your PC to 'click' on websites or applications.
+    # They are usually used to make money for a malicious hacker by clicking on online advertisements
+    # and making it look like the website gets more traffic than it does.
+    # They can also be used to skew online polls, install programs on your PC, or make unwanted software
+    # appear more popular than it is.
     clickfraud = "clickfraud"
 
     # Cryptocurrency mining malware.
     cryptominer = "cryptominer"
 
-    # These programs conduct DoS (Denial of Service) attacks against a targeted web address. By sending multiple requests from your computer and several other infected computers, the attack can overwhelm the target address leading to a denial of service.
+    # These programs conduct DoS (Denial of Service) attacks against a targeted web address.
+    # By sending multiple requests from your computer and several other infected computers,
+    # the attack can overwhelm the target address leading to a denial of service.
     ddos = "ddos"
 
     # Trojan Downloaders can download and install new versions of malicious programs in the target system.
@@ -92,49 +109,66 @@ class CategoryEnum(str, Enum):
     # These programs are used by hackers in order to install malware or to prevent the detection of malicious programs.
     dropper = "dropper"
 
-    # Exploit kits are programs that contain data or code that takes advantage of a vulnerability within an application that is running in the target system.
+    # Exploit kits are programs that contain data or code that takes advantage of a vulnerability
+    # within an application that is running in the target system.
     exploitkit = "exploitkit"
 
-    # Trojan FakeAV programs simulate the activity of antivirus software. They are designed to extort money in return for the detection and removal of threat, even though the threats that they report are actually non-existent.
+    # Trojan FakeAV programs simulate the activity of antivirus software.
+    # They are designed to extort money in return for the detection and removal of threat, even though the
+    # threats that they report are actually non-existent.
     fakeav = "fakeav"
 
     # A type of tool that can be used to allow and maintain unauthorized access to your PC.
     hacktool = "hacktool"
 
-    # A program that collects your personal information, such as your browsing history, and uses it without adequate consent.
+    # A program that collects your personal information, such as your browsing history,
+    # and uses it without adequate consent.
     infostealer = "infostealer"
 
-    # A keylogger monitors and logs every keystroke it can identify. Once installed, the virus either keeps track of all the keys and stores the information locally, after which the hacker needs physical access to the computer to retrieve the information, or the logs are sent over the internet back to the hacker.
+    # A keylogger monitors and logs every keystroke it can identify.
+    # Once installed, the virus either keeps track of all the keys and stores the information locally,
+    # after which the hacker needs physical access to the computer to retrieve the information,
+    # or the logs are sent over the internet back to the hacker.
     keylogger = "keylogger"
 
     # A program that loads another application / memory space.
     loader = "loader"
 
-    # A type of malware that hides its code and purpose to make it more difficult for security software to detect or remove it.
+    # A type of malware that hides its code and purpose to make it more difficult for
+    # security software to detect or remove it.
     obfuscator = "obfuscator"
 
-    # Point-of-sale malware is usually a type of malware that is used by cybercriminals to target point of sale (POS) and payment terminals with the intent to obtain credit card and debit card information.
+    # Point-of-sale malware is usually a type of malware that is used by cybercriminals to target point of sale (POS)
+    # and payment terminals with the intent to obtain credit card and debit card information.
     pos = "pos"
 
-    # This type of trojan allows unauthorized parties to use the infected computer as a proxy server to access the Internet anonymously.
+    # This type of trojan allows unauthorized parties to use the infected computer as a proxy server
+    # to access the Internet anonymously.
     proxy = "proxy"
 
     # A program that can be used by a remote hacker to gain access and control of an infected machine.
     rat = "rat"
 
-    # This type of malware can modify data in the target computer so the operating system will stop running correctly or the data is no longer accessible. The criminal will only restore the computer state or data after a ransom is paid to them (mostly using cryptocurrency).
+    # This type of malware can modify data in the target computer so the operating system
+    # will stop running correctly or the data is no longer accessible.
+    # The criminal will only restore the computer state or data after a ransom is paid to them
+    # (mostly using cryptocurrency).
     ransomware = "ransomware"
 
     # A reverse proxy is a server that receives requests from the internet and forwards them to a small set of servers.
     reverse_proxy = "reverse_proxy"
 
-    # Rootkits are designed to conceal certain objects or activities in the system. Often their main purpose is to prevent malicious programs being detected in order to extend the period in which programs can run on an infected computer.
+    # Rootkits are designed to conceal certain objects or activities in the system.
+    # Often their main purpose is to prevent malicious programs being detected
+    # in order to extend the period in which programs can run on an infected computer.
     rootkit = "rootkit"
 
-    # This type of malware scan the internet / network(s) / system(s) / service(s) to collect information. That information could be used later to perpetuate an cyber attack.
+    # This type of malware scan the internet / network(s) / system(s) / service(s) to collect information.
+    # That information could be used later to perpetuate an cyber attack.
     scanner = "scanner"
 
-    # Scareware is a form of malware which uses social engineering to cause shock, anxiety, or the perception of a threat in order to manipulate users into buying unwanted software.
+    # Scareware is a form of malware which uses social engineering to cause shock, anxiety,
+    # or the perception of a threat in order to manipulate users into buying unwanted software.
     scareware = "scareware"
 
     # Malware that is sending spam.

model_setup/maco/utils.py

@@ -46,6 +46,7 @@ UV_BIN = find_uv_bin()
 PIP_CMD = f"{UV_BIN} pip"
 VENV_CREATE_CMD = f"{UV_BIN} venv"
 
+
 class Base64Decoder(json.JSONDecoder):
     def __init__(self, *args, **kwargs):
         json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)
@@ -179,11 +180,9 @@ def scan_for_extractors(root_directory: str, scanner: yara.Rules, logger: Logger
                         for pattern in [RELATIVE_FROM_IMPORT_RE, RELATIVE_FROM_RE]:
                             for match in pattern.findall(data):
                                 depth = match.count(".")
-                                data = data.replace(
-                                    f"from {match}",
-                                    f"from {'.'.join(split[depth - 1 : split.index(package) + 1][::-1])}{'.' if pattern == RELATIVE_FROM_RE else ''}",
-                                    1,
-                                )
+                                abspath = ".".join(split[depth - 1 : split.index(package) + 1][::-1])
+                                abspath += "." if pattern == RELATIVE_FROM_RE else ""
+                                data = data.replace(f"from {match}", f"from {abspath}", 1)
                         f.write(data)
 
                 if scanner.match(path):
@@ -222,7 +221,12 @@ def _install_required_packages(create_venv: bool, directories: List[str], python
                         subprocess.run(cmd.split(" ") + [venv_path], capture_output=True, env=env)
 
                 # Install/Update the packages in the environment
-                install_command = PIP_CMD.split(" ") + ["install", "-U"]
+                install_command = PIP_CMD.split(" ") + ["install"]
+                # When running locally, only install packages to required spec.
+                # This prevents issues during maco development and building extractors against local libraries.
+                if create_venv:
+                    # when running in custom virtual environment, always upgrade packages.
+                    install_command.append("-U")
 
                 # Update the pip install command depending on where the dependencies are coming from
                 if "requirements.txt" in req_files:
@@ -307,8 +311,10 @@ def register_extractors(
     parent_directory = os.path.dirname(current_directory)
     if venvs and package_name in sys.modules:
         # this may happen as part of testing if some part of the extractor code was directly imported
-        logger.warning(f"Looks like {package_name} is already loaded. "
-                       "If your maco extractor overlaps an existing package name this could cause problems.")
+        logger.warning(
+            f"Looks like {package_name} is already loaded. "
+            "If your maco extractor overlaps an existing package name this could cause problems."
+        )
 
     try:
         # Modify the PATH so we can recognize this new package on import
@@ -383,6 +389,7 @@ def register_extractors(
                 # We were able to find all the extractor files
                 break
 
+
 def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType, str], None], *args, **kwargs):
     """Ensures logging is set up correctly for a child process and then executes the callback."""
     logger = logging.getLogger()
@@ -391,6 +398,7 @@ def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType,
     logger.addHandler(qh)
     callback(*args, **kwargs, logger=logger)
 
+
 def import_extractors(
     extractor_module_callback: Callable[[ModuleType, str], bool],
     *,
@@ -416,6 +424,7 @@ def import_extractors(
 # holds cached extractors when not running in venv mode
 _loaded_extractors: Dict[str, Extractor] = {}
 
+
 def run_extractor(
     sample_path,
     module_name,
@@ -438,7 +447,7 @@ def run_extractor(
             extractor = _loaded_extractors[key]
         if extractor.yara_compiled:
             matches = extractor.yara_compiled.match(sample_path)
-        loaded = extractor.run(open(sample_path, 'rb'), matches=matches)
+        loaded = extractor.run(open(sample_path, "rb"), matches=matches)
     else:
         # execute extractor in child process with separate virtual environment
         # Write temporary script in the same directory as extractor to resolve relative imports
@@ -477,7 +486,7 @@ def run_extractor(
                 try:
                     # Load results and return them
                     output.seek(0)
-                    loaded =  json.load(output, cls=json_decoder)
+                    loaded = json.load(output, cls=json_decoder)
                 except Exception as e:
                     # If there was an error raised during runtime, then propagate
                     delim = f'File "{module_path}"'

model_setup/maco/yara.py

@@ -3,7 +3,6 @@ from collections import namedtuple
 from itertools import cycle
 from typing import Dict
 
-import yara
 import yara_x
 
 RULE_ID_RE = re.compile("(\w+)? ?rule (\w+)")

pipelines/test.yaml

@@ -7,6 +7,27 @@ pool:
   vmImage: "ubuntu-22.04"
 
 jobs:
+  - job: style_test
+    strategy:
+      matrix:
+        Python3_12:
+          python.version: "3.12"
+    timeoutInMinutes: 10
+
+    steps:
+      - task: UsePythonVersion@0
+        displayName: Set python version
+        inputs:
+          versionSpec: "$(python.version)"
+
+      - script: |
+          python -m pip install -U tox
+        displayName: Install tox
+
+      - script: |
+          python -m tox -e style
+        displayName: "Run style tests"
+
   - job: run_test
     strategy:
       matrix:

tests/extractors/bob/bob.py

@@ -1,7 +1,4 @@
-from io import BytesIO
-from typing import List, Optional
-
-from maco import extractor, model, yara
+from maco import extractor
 
 
 class Bob(extractor.Extractor):

maco-1.2.5.dist-info/RECORD

@@ -1,44 +0,0 @@
-demo_extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-demo_extractors/elfy.py,sha256=AAFr5i1aivPwO4nycyXJEud57EpVNA-5k_2GicWesbY,771
-demo_extractors/limit_other.py,sha256=8Z7X0cXUyZuK3MhDtObMWmdruRj5hgFdDi_VVGXqRx4,1123
-demo_extractors/nothing.py,sha256=3aeQJTY-dakmVXmyfmrRM8YCQVT7q3bq880DFH1Ol_Y,607
-demo_extractors/requirements.txt,sha256=E0tD6xBZldq6sQGTHng6k88lBeASOhmLJcdcjpcqBNE,6
-demo_extractors/shared.py,sha256=2P1cyuRbHDvM9IRt3UZnwdyhxx7OWqNC83xLyV8Y190,305
-demo_extractors/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-demo_extractors/complex/complex.py,sha256=tXrzj_zWIXbTOwj7Lezapk-qkrM-lfwcyjd5m-BYzdg,2322
-demo_extractors/complex/complex_utils.py,sha256=aec8kJsYUrMPo-waihkVLt-0QpiOPkw7dDqfT9MNuHk,123
-maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-maco/base_test.py,sha256=EPxCun9Tv91V-lFpaenn14tPyW17TPvXVH4AjE3t6js,2716
-maco/cli.py,sha256=fIeUXOgOxcecmAkl6OAdnjBKqk1gBPv1ryWe50pT60g,8135
-maco/collector.py,sha256=Vlo7KcJC7TKZFTElv8i_f_hvWEnlWCRzOP1xOc9x7vk,6532
-maco/extractor.py,sha256=uGSGiCQ4jd8jFmfw2T99BGcY5iQJzXHcG_RoTIxClTE,2802
-maco/utils.py,sha256=K41c-H7naaoiEYf0WNfP054IxwvHPujsbmmzgTizuLU,20159
-maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
-maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
-maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
-model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-model_setup/maco/base_test.py,sha256=EPxCun9Tv91V-lFpaenn14tPyW17TPvXVH4AjE3t6js,2716
-model_setup/maco/cli.py,sha256=fIeUXOgOxcecmAkl6OAdnjBKqk1gBPv1ryWe50pT60g,8135
-model_setup/maco/collector.py,sha256=Vlo7KcJC7TKZFTElv8i_f_hvWEnlWCRzOP1xOc9x7vk,6532
-model_setup/maco/extractor.py,sha256=uGSGiCQ4jd8jFmfw2T99BGcY5iQJzXHcG_RoTIxClTE,2802
-model_setup/maco/utils.py,sha256=K41c-H7naaoiEYf0WNfP054IxwvHPujsbmmzgTizuLU,20159
-model_setup/maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
-model_setup/maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
-model_setup/maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
-pipelines/publish.yaml,sha256=xt3WNU-5kIICJgKIiiE94M3dWjS3uEiun-n4OmIssK8,1471
-pipelines/test.yaml,sha256=3KOoo-8SqP_bTAscsz5V3xxnuL91J-62mTjnQD1Btag,1019
-tests/data/example.txt.cart,sha256=j4ZdDnFNVq7lb-Qi4pY4evOXKQPKG-GSg-n-uEqPhV0,289
-tests/data/trigger_complex.txt,sha256=uqnLSrnyDGCmXwuPmZ2s8vdhH0hJs8DxvyaW_tuYY24,64
-tests/data/trigger_complex.txt.cart,sha256=Z7qF1Zi640O45Znkl9ooP2RhSLAEqY0NRf51d-q7utU,345
-tests/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/extractors/basic.py,sha256=r5eLCL6Ynr14nCBgtbLvUbm0NdrXizyc9c-4xBCNShU,828
-tests/extractors/basic_longer.py,sha256=1ClU2QD-Y0TOl_loNFvEqIEpTR5TSVJ6zg9ZmC-ESJo,860
-tests/extractors/test_basic.py,sha256=FLKekfSGM69HaiF7Vu_7D7KDXHZko-9hZkMO8_DoyYA,697
-tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/extractors/bob/bob.py,sha256=Gy5p8KssJX87cwa9vVv8UBODF_ulbUteZXh15frW2hs,247
-maco-1.2.5.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
-maco-1.2.5.dist-info/METADATA,sha256=cJ7x_shBhDgKVjkq_e2d94aj3qiUzi0lt7f3lPO334U,15610
-maco-1.2.5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-maco-1.2.5.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
-maco-1.2.5.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
-maco-1.2.5.dist-info/RECORD,,