maco 1.2.16__py3-none-any.whl → 1.2.18__py3-none-any.whl

extractor_setup/maco/yara.py

@@ -0,0 +1,129 @@
+"""yara-python facade that uses yara-x."""
+
+import re
+from collections import namedtuple
+from itertools import cycle
+from typing import Dict, List, Union
+
+import yara_x
+
+from maco.exceptions import SyntaxError
+
+RULE_ID_RE = re.compile("(\w+)? ?rule (\w+)")
+
+
+# Create interfaces that resembles yara-python (but is running yara-x under the hood)
+class StringMatchInstance:
+    """Instance of a string match."""
+
+    def __init__(self, match: yara_x.Match, file_content: bytes):
+        """Initializes StringMatchInstance."""
+        self.matched_data = file_content[match.offset : match.offset + match.length]
+        self.matched_length = match.length
+        self.offset = match.offset
+        self.xor_key = match.xor_key
+
+    def plaintext(self) -> bytes:
+        """Plaintext of the matched data.
+
+        Returns:
+            (bytes): Plaintext of the matched cipher text
+        """
+        if not self.xor_key:
+            # No need to XOR the matched data
+            return self.matched_data
+        else:
+            return bytes(c ^ k for c, k in zip(self.matched_data, cycle(self.xor_key)))
+
+
+class StringMatch:
+    """String match."""
+
+    def __init__(self, pattern: yara_x.Pattern, file_content: bytes):
+        """Initializes StringMatch."""
+        self.identifier = pattern.identifier
+        self.instances = [StringMatchInstance(match, file_content) for match in pattern.matches]
+        self._is_xor = any([match.xor_key for match in pattern.matches])
+
+    def is_xor(self):
+        """Checks if string match is xor'd.
+
+        Returns:
+            (bool): True if match is xor'd
+        """
+        return self._is_xor
+
+
+class Match:
+    """Match."""
+
+    def __init__(self, rule: yara_x.Rule, file_content: bytes):
+        """Initializes Match."""
+        self.rule = rule.identifier
+        self.namespace = rule.namespace
+        self.tags = list(rule.tags) or []
+        self.meta = dict()
+        # Ensure metadata doesn't get overwritten
+        for k, v in rule.metadata:
+            self.meta.setdefault(k, []).append(v)
+        self.strings = [StringMatch(pattern, file_content) for pattern in rule.patterns]
+
+
+class Rules:
+    """Rules."""
+
+    def __init__(self, source: str = None, sources: Dict[str, str] = None):
+        """Initializes Rules.
+
+        Raises:
+            SyntaxError: Raised when there's a syntax error in the YARA rule.
+        """
+        Rule = namedtuple("Rule", "identifier namespace is_global")
+        if source:
+            sources = {"default": source}
+
+        try:
+            self._rules = []
+            compiler = yara_x.Compiler(relaxed_re_syntax=True)
+            for namespace, source in sources.items():
+                compiler.new_namespace(namespace)
+                for rule_type, id in RULE_ID_RE.findall(source):
+                    is_global = True if rule_type == "global" else False
+                    self._rules.append(Rule(namespace=namespace, identifier=id, is_global=is_global))
+                compiler.add_source(source)
+            self.scanner = yara_x.Scanner(compiler.build())
+        except yara_x.CompileError as e:
+            raise SyntaxError(e)
+
+    def __iter__(self):
+        """Iterate over rules.
+
+        Yields:
+            YARA rules
+        """
+        for rule in self._rules:
+            yield rule
+
+    def match(self, filepath: str = None, data: Union[bytes, bytearray] = None) -> List[Match]:
+        """Performs a scan to check for YARA rules matches based on the file, either given by path or buffer.
+
+        Returns:
+            (List[Match]): A list of YARA matches.
+        """
+        if filepath:
+            with open(filepath, "rb") as fp:
+                data = fp.read()
+
+        if isinstance(data, bytearray):
+            data = bytes(data)
+
+        return [Match(m, data) for m in self.scanner.scan(data).matching_rules]
+
+
+def compile(source: str = None, sources: Dict[str, str] = None) -> Rules:
+    """Compiles YARA rules from source or from sources.
+
+    Returns:
+        (Rules): a Rules object
+    """
+    return Rules(source, sources)

maco/utils.py

@@ -259,15 +259,15 @@ def _install_required_packages(create_venv: bool, directories: List[str], python
                 # This prevents issues during maco development and building extractors against local libraries.
                 if create_venv:
                     # when running in custom virtual environment, always upgrade packages.
-                    install_command.append("-U")
+                    install_command.extend(["--upgrade", "--no-cache"])
 
                 # Update the pip install command depending on where the dependencies are coming from
                 if "requirements.txt" in req_files:
                     # Perform a pip install using the requirements flag
-                    install_command.extend(["-r", "requirements.txt"])
+                    install_command.extend(["--requirements", "requirements.txt"])
                 elif "pyproject.toml" in req_files:
                     # Assume we're dealing with a project directory
-                    pyproject_command = ["-e", "."]
+                    pyproject_command = ["--editable", "."]
 
                     # Check to see if there are optional dependencies required
                     with open(os.path.join(dir, "pyproject.toml"), "rb") as f:
@@ -280,8 +280,8 @@ def _install_required_packages(create_venv: bool, directories: List[str], python
 
                     install_command.extend(pyproject_command)
 
-                # always require maco to be installed
-                install_command.append("maco")
+                # Always require maco-extractor to be installed
+                install_command.append("maco-extractor")
                 logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
                 # this uses VIRTUAL_ENV to control usage of a virtual environment
                 p = subprocess.run(

maco/yara.py

@@ -3,7 +3,7 @@
 import re
 from collections import namedtuple
 from itertools import cycle
-from typing import Dict, List
+from typing import Dict, List, Union
 
 import yara_x
 
@@ -104,7 +104,7 @@ class Rules:
         for rule in self._rules:
             yield rule
 
-    def match(self, filepath: str = None, data: bytes = None) -> List[Match]:
+    def match(self, filepath: str = None, data: Union[bytes, bytearray] = None) -> List[Match]:
         """Performs a scan to check for YARA rules matches based on the file, either given by path or buffer.
 
         Returns:
@@ -114,6 +114,9 @@ class Rules:
             with open(filepath, "rb") as fp:
                 data = fp.read()
 
+        if isinstance(data, bytearray):
+            data = bytes(data)
+
         return [Match(m, data) for m in self.scanner.scan(data).matching_rules]
 
 

{maco-1.2.16.dist-info → maco-1.2.18.dist-info}/METADATA

@@ -1,6 +1,7 @@
 Metadata-Version: 2.4
 Name: maco
-Version: 1.2.16
+Version: 1.2.18
+Summary: Maco is a framework for creating and using malware configuration extractors.
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License
@@ -33,7 +34,7 @@ Requires-Dist: cart
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: tomli>=1.1.0; python_version < "3.11"
 Requires-Dist: uv
-Requires-Dist: yara-x==0.11.0
+Requires-Dist: yara-x
 Requires-Dist: multiprocess>=0.70.17
 Dynamic: license-file
 

{maco-1.2.16.dist-info → maco-1.2.18.dist-info}/RECORD

@@ -8,28 +8,38 @@ demo_extractors/terminator.py,sha256=nxoZYRteYDQS7wp-aAsCaxCSJ9FSE54jPrW3fJpRVho
 demo_extractors/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 demo_extractors/complex/complex.py,sha256=GYKmPOD8-fyVHxwjZb-3t1IghKVMuLtdUvCs5C5yPe0,2625
 demo_extractors/complex/complex_utils.py,sha256=5kdMl-niSH9d-d3ChuItpmlPT4U-S9g-iyBZlR4tfmQ,296
+extractor_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+extractor_setup/maco/base_test.py,sha256=DrVE7vOazeLQpOQeIDwBYK1WtlmdJrRe50JOqP5t4Y0,3198
+extractor_setup/maco/cli.py,sha256=nrSukAJAthbstZT3-lQNPz4zOOMcBhvfYQqLh_B5Jdk,9457
+extractor_setup/maco/collector.py,sha256=R3zw-fUJBlwmcSqvkQ-PnoJdHfRm2V0JAOl7N8MTAbY,8240
+extractor_setup/maco/exceptions.py,sha256=XBHUrs1kr1ZayPI9B_W-WejKgVmC8sWL_o4RL0b4DQE,745
+extractor_setup/maco/extractor.py,sha256=s36aGcsXSc-9iCik6iihVt5G1a1DZUA7TquvWYQNwdE,2912
+extractor_setup/maco/utils.py,sha256=yNm5CiHc9033ONX_gFwg9Lng5IYFujLDtw6FfiJkAao,23425
+extractor_setup/maco/yara.py,sha256=y141t8NqDDXHY23uE1d6BDPeNmSuUuohR6Yr_LKa7GI,4067
+extractor_setup/maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
+extractor_setup/maco/model/model.py,sha256=DBHTmZXMzjpVq0s2mzZv3VCzPhwPnv7sH6u_QZCTcA4,24484
 maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 maco/base_test.py,sha256=DrVE7vOazeLQpOQeIDwBYK1WtlmdJrRe50JOqP5t4Y0,3198
 maco/cli.py,sha256=nrSukAJAthbstZT3-lQNPz4zOOMcBhvfYQqLh_B5Jdk,9457
 maco/collector.py,sha256=R3zw-fUJBlwmcSqvkQ-PnoJdHfRm2V0JAOl7N8MTAbY,8240
 maco/exceptions.py,sha256=XBHUrs1kr1ZayPI9B_W-WejKgVmC8sWL_o4RL0b4DQE,745
 maco/extractor.py,sha256=s36aGcsXSc-9iCik6iihVt5G1a1DZUA7TquvWYQNwdE,2912
-maco/utils.py,sha256=wzecqu1W_BbhDHSs07sweOKNeFrmC4CDYmAN_qyYJS4,23362
-maco/yara.py,sha256=gkHHxwZNxzZV7nHZM3HNUmhHXB7VW82voCHK5mHpt2Q,3970
+maco/utils.py,sha256=yNm5CiHc9033ONX_gFwg9Lng5IYFujLDtw6FfiJkAao,23425
+maco/yara.py,sha256=y141t8NqDDXHY23uE1d6BDPeNmSuUuohR6Yr_LKa7GI,4067
 maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
 maco/model/model.py,sha256=DBHTmZXMzjpVq0s2mzZv3VCzPhwPnv7sH6u_QZCTcA4,24484
-maco-1.2.16.dist-info/licenses/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
+maco-1.2.18.dist-info/licenses/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
 model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 model_setup/maco/base_test.py,sha256=DrVE7vOazeLQpOQeIDwBYK1WtlmdJrRe50JOqP5t4Y0,3198
 model_setup/maco/cli.py,sha256=nrSukAJAthbstZT3-lQNPz4zOOMcBhvfYQqLh_B5Jdk,9457
 model_setup/maco/collector.py,sha256=R3zw-fUJBlwmcSqvkQ-PnoJdHfRm2V0JAOl7N8MTAbY,8240
 model_setup/maco/exceptions.py,sha256=XBHUrs1kr1ZayPI9B_W-WejKgVmC8sWL_o4RL0b4DQE,745
 model_setup/maco/extractor.py,sha256=s36aGcsXSc-9iCik6iihVt5G1a1DZUA7TquvWYQNwdE,2912
-model_setup/maco/utils.py,sha256=wzecqu1W_BbhDHSs07sweOKNeFrmC4CDYmAN_qyYJS4,23362
-model_setup/maco/yara.py,sha256=gkHHxwZNxzZV7nHZM3HNUmhHXB7VW82voCHK5mHpt2Q,3970
+model_setup/maco/utils.py,sha256=yNm5CiHc9033ONX_gFwg9Lng5IYFujLDtw6FfiJkAao,23425
+model_setup/maco/yara.py,sha256=y141t8NqDDXHY23uE1d6BDPeNmSuUuohR6Yr_LKa7GI,4067
 model_setup/maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
 model_setup/maco/model/model.py,sha256=DBHTmZXMzjpVq0s2mzZv3VCzPhwPnv7sH6u_QZCTcA4,24484
-pipelines/publish.yaml,sha256=xt3WNU-5kIICJgKIiiE94M3dWjS3uEiun-n4OmIssK8,1471
+pipelines/publish.yaml,sha256=yjc3eqrI-LHLSfZ0DPtxwdfPDT0NI6LUA_zy61UxN_8,1654
 pipelines/test.yaml,sha256=btJVI-R39UBeYosGu7TOpU6V9ogFW3FT3ROtWygQGQ0,1472
 tests/data/example.txt.cart,sha256=j4ZdDnFNVq7lb-Qi4pY4evOXKQPKG-GSg-n-uEqPhV0,289
 tests/data/trigger_complex.txt,sha256=uqnLSrnyDGCmXwuPmZ2s8vdhH0hJs8DxvyaW_tuYY24,64
@@ -42,8 +52,8 @@ tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 tests/extractors/bob/bob.py,sha256=4fpqy_O6NDinJImghyW5OwYgnaB05aY4kgoIS_C3c_U,253
 tests/extractors/import_rewriting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/extractors/import_rewriting/importer.py,sha256=wqF1AG2zXXuj9EMt9qlDorab-UD0GYuFggtrCuz4sf0,289735
-maco-1.2.16.dist-info/METADATA,sha256=v5ZV9RoifsSn6im1tO0wpUJeKPGW3Wf-_C_BjF7qNmM,15232
-maco-1.2.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-maco-1.2.16.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
-maco-1.2.16.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
-maco-1.2.16.dist-info/RECORD,,
+maco-1.2.18.dist-info/METADATA,sha256=citbYasnfKhc-PAxK7tLQt_Dc2LZRbhKn26ChD0PC3g,15310
+maco-1.2.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+maco-1.2.18.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
+maco-1.2.18.dist-info/top_level.txt,sha256=xiVS11ZoyN8ChHJQGpOzTH4ZyQ3YJe1qT3Yt4gcKGUk,65
+maco-1.2.18.dist-info/RECORD,,

{maco-1.2.16.dist-info → maco-1.2.18.dist-info}/top_level.txt

@@ -1,4 +1,5 @@
 demo_extractors
+extractor_setup
 maco
 model_setup
 pipelines

model_setup/maco/utils.py

@@ -259,15 +259,15 @@ def _install_required_packages(create_venv: bool, directories: List[str], python
                 # This prevents issues during maco development and building extractors against local libraries.
                 if create_venv:
                     # when running in custom virtual environment, always upgrade packages.
-                    install_command.append("-U")
+                    install_command.extend(["--upgrade", "--no-cache"])
 
                 # Update the pip install command depending on where the dependencies are coming from
                 if "requirements.txt" in req_files:
                     # Perform a pip install using the requirements flag
-                    install_command.extend(["-r", "requirements.txt"])
+                    install_command.extend(["--requirements", "requirements.txt"])
                 elif "pyproject.toml" in req_files:
                     # Assume we're dealing with a project directory
-                    pyproject_command = ["-e", "."]
+                    pyproject_command = ["--editable", "."]
 
                     # Check to see if there are optional dependencies required
                     with open(os.path.join(dir, "pyproject.toml"), "rb") as f:
@@ -280,8 +280,8 @@ def _install_required_packages(create_venv: bool, directories: List[str], python
 
                     install_command.extend(pyproject_command)
 
-                # always require maco to be installed
-                install_command.append("maco")
+                # Always require maco-extractor to be installed
+                install_command.append("maco-extractor")
                 logger.debug(f"Install command: {' '.join(install_command)} [{dir}]")
                 # this uses VIRTUAL_ENV to control usage of a virtual environment
                 p = subprocess.run(

model_setup/maco/yara.py

@@ -3,7 +3,7 @@
 import re
 from collections import namedtuple
 from itertools import cycle
-from typing import Dict, List
+from typing import Dict, List, Union
 
 import yara_x
 
@@ -104,7 +104,7 @@ class Rules:
         for rule in self._rules:
             yield rule
 
-    def match(self, filepath: str = None, data: bytes = None) -> List[Match]:
+    def match(self, filepath: str = None, data: Union[bytes, bytearray] = None) -> List[Match]:
         """Performs a scan to check for YARA rules matches based on the file, either given by path or buffer.
 
         Returns:
@@ -114,6 +114,9 @@ class Rules:
             with open(filepath, "rb") as fp:
                 data = fp.read()
 
+        if isinstance(data, bytearray):
+            data = bytes(data)
+
         return [Match(m, data) for m in self.scanner.scan(data).matching_rules]
 
 

pipelines/publish.yaml

@@ -12,34 +12,33 @@ pool:
   vmImage: "ubuntu-22.04"
 
 jobs:
-- job: test
-  displayName: Test
-  strategy:
-    matrix:
-      Python38:
-        python.version: '3.8'
-      Python39:
-        python.version: '3.9'
-      Python310:
-        python.version: '3.10'
-      Python311:
-        python.version: '3.11'
-      Python312:
-        python.version: '3.12'
-  steps:
-  - task: UsePythonVersion@0
-    displayName: 'Use Python $(python.version)'
-    inputs:
-      versionSpec: '$(python.version)'
+# - job: test
+#   displayName: Test
+#   strategy:
+#     matrix:
+#       Python38:
+#         python.version: '3.8'
+#       Python39:
+#         python.version: '3.9'
+#       Python310:
+#         python.version: '3.10'
+#       Python311:
+#         python.version: '3.11'
+#       Python312:
+#         python.version: '3.12'
+#   steps:
+#   - task: UsePythonVersion@0
+#     displayName: 'Use Python $(python.version)'
+#     inputs:
+#       versionSpec: '$(python.version)'
 
-  - script: |
-      set -x
+#   - script: |
+#       set -x
 
-      python -m pip install -U tox
-      python -m tox -e py
+#       python -m pip install -U tox
+#       python -m tox -e py
 
 - job: build_and_deploy
-  dependsOn: test
   displayName: Build and Deploy
   variables:
   - group: deployment-information
@@ -64,6 +63,13 @@ jobs:
       ls ../dist
     displayName: Build (Model Only)
 
+  - script: |
+      set -x
+      cd extractor_setup
+      python -m build --outdir ../dist
+      ls ../dist
+    displayName: Build (Extractor Essentials)
+
   - script: |
       set -xv  # Echo commands before they are run
       sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine