maco 1.2.17__py3-none-any.whl → 1.2.19__py3-none-any.whl

extractor_setup/maco/yara.py

@@ -0,0 +1,129 @@
+"""yara-python facade that uses yara-x."""
+
+import re
+from collections import namedtuple
+from itertools import cycle
+from typing import Dict, List, Union
+
+import yara_x
+
+from maco.exceptions import SyntaxError
+
+RULE_ID_RE = re.compile("(\w+)? ?rule (\w+)")
+
+
+# Create interfaces that resembles yara-python (but is running yara-x under the hood)
+class StringMatchInstance:
+    """Instance of a string match."""
+
+    def __init__(self, match: yara_x.Match, file_content: bytes):
+        """Initializes StringMatchInstance."""
+        self.matched_data = file_content[match.offset : match.offset + match.length]
+        self.matched_length = match.length
+        self.offset = match.offset
+        self.xor_key = match.xor_key
+
+    def plaintext(self) -> bytes:
+        """Plaintext of the matched data.
+
+        Returns:
+            (bytes): Plaintext of the matched cipher text
+        """
+        if not self.xor_key:
+            # No need to XOR the matched data
+            return self.matched_data
+        else:
+            return bytes(c ^ k for c, k in zip(self.matched_data, cycle(self.xor_key)))
+
+
+class StringMatch:
+    """String match."""
+
+    def __init__(self, pattern: yara_x.Pattern, file_content: bytes):
+        """Initializes StringMatch."""
+        self.identifier = pattern.identifier
+        self.instances = [StringMatchInstance(match, file_content) for match in pattern.matches]
+        self._is_xor = any([match.xor_key for match in pattern.matches])
+
+    def is_xor(self):
+        """Checks if string match is xor'd.
+
+        Returns:
+            (bool): True if match is xor'd
+        """
+        return self._is_xor
+
+
+class Match:
+    """Match."""
+
+    def __init__(self, rule: yara_x.Rule, file_content: bytes):
+        """Initializes Match."""
+        self.rule = rule.identifier
+        self.namespace = rule.namespace
+        self.tags = list(rule.tags) or []
+        self.meta = dict()
+        # Ensure metadata doesn't get overwritten
+        for k, v in rule.metadata:
+            self.meta.setdefault(k, []).append(v)
+        self.strings = [StringMatch(pattern, file_content) for pattern in rule.patterns]
+
+
+class Rules:
+    """Rules."""
+
+    def __init__(self, source: str = None, sources: Dict[str, str] = None):
+        """Initializes Rules.
+
+        Raises:
+            SyntaxError: Raised when there's a syntax error in the YARA rule.
+        """
+        Rule = namedtuple("Rule", "identifier namespace is_global")
+        if source:
+            sources = {"default": source}
+
+        try:
+            self._rules = []
+            compiler = yara_x.Compiler(relaxed_re_syntax=True)
+            for namespace, source in sources.items():
+                compiler.new_namespace(namespace)
+                for rule_type, id in RULE_ID_RE.findall(source):
+                    is_global = True if rule_type == "global" else False
+                    self._rules.append(Rule(namespace=namespace, identifier=id, is_global=is_global))
+                compiler.add_source(source)
+            self.scanner = yara_x.Scanner(compiler.build())
+        except yara_x.CompileError as e:
+            raise SyntaxError(e)
+
+    def __iter__(self):
+        """Iterate over rules.
+
+        Yields:
+            YARA rules
+        """
+        for rule in self._rules:
+            yield rule
+
+    def match(self, filepath: str = None, data: Union[bytes, bytearray] = None) -> List[Match]:
+        """Performs a scan to check for YARA rules matches based on the file, either given by path or buffer.
+
+        Returns:
+            (List[Match]): A list of YARA matches.
+        """
+        if filepath:
+            with open(filepath, "rb") as fp:
+                data = fp.read()
+
+        if isinstance(data, bytearray):
+            data = bytes(data)
+
+        return [Match(m, data) for m in self.scanner.scan(data).matching_rules]
+
+
+def compile(source: str = None, sources: Dict[str, str] = None) -> Rules:
+    """Compiles YARA rules from source or from sources.
+
+    Returns:
+        (Rules): a Rules object
+    """
+    return Rules(source, sources)

maco/collector.py

@@ -122,6 +122,7 @@ class Collector:
                             "author": member.author,
                             "last_modified": member.last_modified,
                             "sharing": member.sharing,
+                            "result_sharing": member.result_sharing,
                             "description": member.__doc__,
                         },
                     )

maco/extractor.py

@@ -25,7 +25,8 @@ class Extractor:
     family: Union[str, List[str]] = None  # family or families of malware that is detected by the extractor
     author: str = None  # author of the extractor (name@organisation)
     last_modified: str = None  # last modified date (YYYY-MM-DD)
-    sharing: str = "TLP:WHITE"  # who can this be shared with?
+    sharing: str = "TLP:CLEAR"  # who can this be shared with?
+    result_sharing: str = sharing  # who can the results be shared with? (defaults to sharing)
     yara_rule: str = None  # yara rule that we filter inputs with
     reference: str = None  # link to malware report or other reference information
     logger: logging.Logger = None  # logger for use when debugging

maco/utils.py

@@ -280,8 +280,8 @@ def _install_required_packages(create_venv: bool, directories: List[str], python
 
                     install_command.extend(pyproject_command)
 
-                # always require maco to be installed
-                install_command.append("maco")
+                # Always require maco-extractor to be installed
+                install_command.append("maco-extractor")
                 logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
                 # this uses VIRTUAL_ENV to control usage of a virtual environment
                 p = subprocess.run(

{maco-1.2.17.dist-info → maco-1.2.19.dist-info}/METADATA

@@ -1,6 +1,7 @@
 Metadata-Version: 2.4
 Name: maco
-Version: 1.2.17
+Version: 1.2.19
+Summary: Maco is a framework for creating and using malware configuration extractors.
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License

{maco-1.2.17.dist-info → maco-1.2.19.dist-info}/RECORD

@@ -8,28 +8,38 @@ demo_extractors/terminator.py,sha256=nxoZYRteYDQS7wp-aAsCaxCSJ9FSE54jPrW3fJpRVho
 demo_extractors/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 demo_extractors/complex/complex.py,sha256=GYKmPOD8-fyVHxwjZb-3t1IghKVMuLtdUvCs5C5yPe0,2625
 demo_extractors/complex/complex_utils.py,sha256=5kdMl-niSH9d-d3ChuItpmlPT4U-S9g-iyBZlR4tfmQ,296
+extractor_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extractor_setup/maco/base_test.py,sha256=DrVE7vOazeLQpOQeIDwBYK1WtlmdJrRe50JOqP5t4Y0,3198
+extractor_setup/maco/cli.py,sha256=nrSukAJAthbstZT3-lQNPz4zOOMcBhvfYQqLh_B5Jdk,9457
+extractor_setup/maco/collector.py,sha256=I0Nidf4-xcvoe6X0bbCsAXjr66iPf00JDO6ocKkaZLc,8309
+extractor_setup/maco/exceptions.py,sha256=XBHUrs1kr1ZayPI9B_W-WejKgVmC8sWL_o4RL0b4DQE,745
+extractor_setup/maco/extractor.py,sha256=nqIfUcrc_l57FicKZc6HLtN223-_zkYWL1AYMy1WAmA,3007
+extractor_setup/maco/utils.py,sha256=yNm5CiHc9033ONX_gFwg9Lng5IYFujLDtw6FfiJkAao,23425
+extractor_setup/maco/yara.py,sha256=y141t8NqDDXHY23uE1d6BDPeNmSuUuohR6Yr_LKa7GI,4067
+extractor_setup/maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
+extractor_setup/maco/model/model.py,sha256=DBHTmZXMzjpVq0s2mzZv3VCzPhwPnv7sH6u_QZCTcA4,24484
 maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 maco/base_test.py,sha256=DrVE7vOazeLQpOQeIDwBYK1WtlmdJrRe50JOqP5t4Y0,3198
 maco/cli.py,sha256=nrSukAJAthbstZT3-lQNPz4zOOMcBhvfYQqLh_B5Jdk,9457
-maco/collector.py,sha256=R3zw-fUJBlwmcSqvkQ-PnoJdHfRm2V0JAOl7N8MTAbY,8240
+maco/collector.py,sha256=I0Nidf4-xcvoe6X0bbCsAXjr66iPf00JDO6ocKkaZLc,8309
 maco/exceptions.py,sha256=XBHUrs1kr1ZayPI9B_W-WejKgVmC8sWL_o4RL0b4DQE,745
-maco/extractor.py,sha256=s36aGcsXSc-9iCik6iihVt5G1a1DZUA7TquvWYQNwdE,2912
-maco/utils.py,sha256=7Xf-kWCDm1DdpBCGOvecEb_hqKoRgNJi0M_OYsJeSrM,23405
+maco/extractor.py,sha256=nqIfUcrc_l57FicKZc6HLtN223-_zkYWL1AYMy1WAmA,3007
+maco/utils.py,sha256=yNm5CiHc9033ONX_gFwg9Lng5IYFujLDtw6FfiJkAao,23425
 maco/yara.py,sha256=y141t8NqDDXHY23uE1d6BDPeNmSuUuohR6Yr_LKa7GI,4067
 maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
 maco/model/model.py,sha256=DBHTmZXMzjpVq0s2mzZv3VCzPhwPnv7sH6u_QZCTcA4,24484
-maco-1.2.17.dist-info/licenses/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
+maco-1.2.19.dist-info/licenses/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
 model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 model_setup/maco/base_test.py,sha256=DrVE7vOazeLQpOQeIDwBYK1WtlmdJrRe50JOqP5t4Y0,3198
 model_setup/maco/cli.py,sha256=nrSukAJAthbstZT3-lQNPz4zOOMcBhvfYQqLh_B5Jdk,9457
-model_setup/maco/collector.py,sha256=R3zw-fUJBlwmcSqvkQ-PnoJdHfRm2V0JAOl7N8MTAbY,8240
+model_setup/maco/collector.py,sha256=I0Nidf4-xcvoe6X0bbCsAXjr66iPf00JDO6ocKkaZLc,8309
 model_setup/maco/exceptions.py,sha256=XBHUrs1kr1ZayPI9B_W-WejKgVmC8sWL_o4RL0b4DQE,745
-model_setup/maco/extractor.py,sha256=s36aGcsXSc-9iCik6iihVt5G1a1DZUA7TquvWYQNwdE,2912
-model_setup/maco/utils.py,sha256=7Xf-kWCDm1DdpBCGOvecEb_hqKoRgNJi0M_OYsJeSrM,23405
+model_setup/maco/extractor.py,sha256=nqIfUcrc_l57FicKZc6HLtN223-_zkYWL1AYMy1WAmA,3007
+model_setup/maco/utils.py,sha256=yNm5CiHc9033ONX_gFwg9Lng5IYFujLDtw6FfiJkAao,23425
 model_setup/maco/yara.py,sha256=y141t8NqDDXHY23uE1d6BDPeNmSuUuohR6Yr_LKa7GI,4067
 model_setup/maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
 model_setup/maco/model/model.py,sha256=DBHTmZXMzjpVq0s2mzZv3VCzPhwPnv7sH6u_QZCTcA4,24484
-pipelines/publish.yaml,sha256=xt3WNU-5kIICJgKIiiE94M3dWjS3uEiun-n4OmIssK8,1471
+pipelines/publish.yaml,sha256=BfsbDsg2ijtXF8lhRUjzkenw3zi2mL7ESNv3KuC1cVE,1626
 pipelines/test.yaml,sha256=btJVI-R39UBeYosGu7TOpU6V9ogFW3FT3ROtWygQGQ0,1472
 tests/data/example.txt.cart,sha256=j4ZdDnFNVq7lb-Qi4pY4evOXKQPKG-GSg-n-uEqPhV0,289
 tests/data/trigger_complex.txt,sha256=uqnLSrnyDGCmXwuPmZ2s8vdhH0hJs8DxvyaW_tuYY24,64
@@ -42,8 +52,8 @@ tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 tests/extractors/bob/bob.py,sha256=4fpqy_O6NDinJImghyW5OwYgnaB05aY4kgoIS_C3c_U,253
 tests/extractors/import_rewriting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/extractors/import_rewriting/importer.py,sha256=wqF1AG2zXXuj9EMt9qlDorab-UD0GYuFggtrCuz4sf0,289735
-maco-1.2.17.dist-info/METADATA,sha256=7NtcX0tJ94BxrswwvqTQO9Ou-UVURL5-j0nBOSGoWQY,15224
-maco-1.2.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-maco-1.2.17.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
-maco-1.2.17.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
-maco-1.2.17.dist-info/RECORD,,
+maco-1.2.19.dist-info/METADATA,sha256=bcX2cXg2jbw6epAkczHoP9WOlcRTKcFtutfrVAc3mIg,15310
+maco-1.2.19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+maco-1.2.19.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
+maco-1.2.19.dist-info/top_level.txt,sha256=xiVS11ZoyN8ChHJQGpOzTH4ZyQ3YJe1qT3Yt4gcKGUk,65
+maco-1.2.19.dist-info/RECORD,,

{maco-1.2.17.dist-info → maco-1.2.19.dist-info}/top_level.txt

@@ -1,4 +1,5 @@
 demo_extractors
+extractor_setup
 maco
 model_setup
 pipelines

model_setup/maco/collector.py

@@ -122,6 +122,7 @@ class Collector:
                             "author": member.author,
                             "last_modified": member.last_modified,
                             "sharing": member.sharing,
+                            "result_sharing": member.result_sharing,
                             "description": member.__doc__,
                         },
                     )

model_setup/maco/extractor.py

@@ -25,7 +25,8 @@ class Extractor:
     family: Union[str, List[str]] = None  # family or families of malware that is detected by the extractor
     author: str = None  # author of the extractor (name@organisation)
     last_modified: str = None  # last modified date (YYYY-MM-DD)
-    sharing: str = "TLP:WHITE"  # who can this be shared with?
+    sharing: str = "TLP:CLEAR"  # who can this be shared with?
+    result_sharing: str = sharing  # who can the results be shared with? (defaults to sharing)
     yara_rule: str = None  # yara rule that we filter inputs with
     reference: str = None  # link to malware report or other reference information
     logger: logging.Logger = None  # logger for use when debugging

model_setup/maco/utils.py

@@ -280,8 +280,8 @@ def _install_required_packages(create_venv: bool, directories: List[str], python
 
                     install_command.extend(pyproject_command)
 
-                # always require maco to be installed
-                install_command.append("maco")
+                # Always require maco-extractor to be installed
+                install_command.append("maco-extractor")
                 logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
                 # this uses VIRTUAL_ENV to control usage of a virtual environment
                 p = subprocess.run(

pipelines/publish.yaml

@@ -64,6 +64,13 @@ jobs:
       ls ../dist
     displayName: Build (Model Only)
 
+  - script: |
+      set -x
+      cd extractor_setup
+      python -m build --outdir ../dist
+      ls ../dist
+    displayName: Build (Extractor Essentials)
+
   - script: |
       set -xv  # Echo commands before they are run
       sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine