maco 1.2.3__py3-none-any.whl → 1.2.4__py3-none-any.whl

demo_extractors/complex/complex.py

@@ -3,7 +3,7 @@ from typing import List, Optional
 
 from maco import extractor, model, yara
 
-from complex import complex_utils
+from demo_extractors.complex import complex_utils
 
 
 class Complex(extractor.Extractor):

demo_extractors/limit_other.py

@@ -3,7 +3,7 @@ from typing import Dict, List, Optional
 
 from maco import extractor, model, yara
 
-from . import shared
+from demo_extractors import shared
 
 
 class LimitOther(extractor.Extractor):

maco/collector.py

@@ -2,8 +2,9 @@
 
 import inspect
 import logging
+import logging.handlers
 import os
-from multiprocessing import Manager, Process
+from multiprocessing import Manager, Process, Queue
 from tempfile import NamedTemporaryFile
 from types import ModuleType
 from typing import Any, BinaryIO, Dict, List, Union
@@ -86,21 +87,33 @@ class Collector:
                     )
                     namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
 
+            # multiprocess logging is awkward - set up a queue to ensure we can log
+            logging_queue = Queue()
+            queue_handler = logging.handlers.QueueListener(logging_queue,*logging.getLogger().handlers)
+            queue_handler.start()
+
             # Find the extractors within the given directory
             # Execute within a child process to ensure main process interpreter is kept clean
             p = Process(
-                target=utils.import_extractors,
+                target=utils.proxy_logging,
                 args=(
-                    path_extractors,
-                    yara.compile(source=utils.MACO_YARA_RULE),
+                    logging_queue,
+                    utils.import_extractors,
                     extractor_module_callback,
-                    logger,
-                    create_venv and os.path.isdir(path_extractors),
+                ),
+                kwargs=dict(
+                    root_directory=path_extractors,
+                    scanner=yara.compile(source=utils.MACO_YARA_RULE),
+                    create_venv=create_venv and os.path.isdir(path_extractors),
                 ),
             )
             p.start()
             p.join()
 
+            # stop multiprocess logging
+            queue_handler.stop()
+            logging_queue.close()
+
             self.extractors = dict(extractors)
             if not self.extractors:
                 raise ExtractorLoadError("no extractors were loaded")

maco/utils.py

@@ -4,11 +4,14 @@ import importlib.machinery
 import importlib.util
 import inspect
 import json
+import logging.handlers
 import os
 import re
 import shutil
 import subprocess
 import sys
+import multiprocessing
+import logging
 import tempfile
 
 from maco import yara
@@ -28,10 +31,12 @@ from typing import Callable, Dict, List, Set, Tuple
 
 from maco.extractor import Extractor
 
+logger = logging.getLogger("maco.lib.utils")
+
 VENV_DIRECTORY_NAME = ".venv"
 
-RELATIVE_FROM_RE = re.compile("from (\.+)")
-RELATIVE_FROM_IMPORT_RE = re.compile("from (\.+) import")
+RELATIVE_FROM_RE = re.compile(r"from (\.+)")
+RELATIVE_FROM_IMPORT_RE = re.compile(r"from (\.+) import")
 
 try:
     # Attempt to use the uv package manager (Recommended)
@@ -69,6 +74,7 @@ import importlib
 import json
 import os
 import sys
+import logging
 
 try:
     from maco import yara
@@ -76,6 +82,19 @@ except:
     import yara
 
 from base64 import b64encode
+
+# ensure we have a logger to stderr
+import logging
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+sh = logging.StreamHandler()
+logger.addHandler(sh)
+sh.setLevel(logging.DEBUG)
+formatter = logging.Formatter(
+    fmt="%(asctime)s, [%(levelname)s] %(module)s.%(funcName)s: %(message)s", datefmt="%Y-%m-%d (%H:%M:%S)"
+)
+sh.setFormatter(formatter)
+
 parent_package_path = "{parent_package_path}"
 sys.path.insert(1, parent_package_path)
 mod = importlib.import_module("{module_name}")
@@ -101,7 +120,7 @@ with open("{output_path}", 'w') as fp:
             json.dump(result.dict(exclude_defaults=True, exclude_none=True), fp, cls=Base64Encoder)
 """
 
-MACO_YARA_RULE = """
+MACO_YARA_RULE = r"""
 rule MACO {
     meta:
         desc = "Used to match on Python files that contain MACO extractors"
@@ -292,15 +311,10 @@ def register_extractors(
 ):
     package_name = os.path.basename(current_directory)
     parent_directory = os.path.dirname(current_directory)
-    symlink = None
-    while package_name in sys.modules:
-        # Package name conflicts with an existing loaded module, let's deconflict that
-        package_name = f"_{package_name}"
-
-        # We'll need to create a link back to the original
-        if package_name not in sys.modules:
-            symlink = os.path.join(parent_directory, package_name)
-            os.symlink(current_directory, symlink)
+    if package_name in sys.modules:
+        # this may happen as part of testing if some part of the extractor code was directly imported
+        logger.warning(f"Looks like {package_name} is already loaded. "
+                       "If your maco extractor overlaps an existing package name this could cause problems.")
 
     try:
         # Modify the PATH so we can recognize this new package on import
@@ -351,10 +365,6 @@ def register_extractors(
         # Remove any modules that were loaded to deconflict with later modules loads
         [sys.modules.pop(k) for k in set(sys.modules.keys()) - default_loaded_modules]
 
-        # Cleanup any symlinks
-        if symlink:
-            os.remove(symlink)
-
     # If there still exists extractor files we haven't found yet, try searching in the available subdirectories
     if extractor_files:
         for dir in os.listdir(current_directory):
@@ -379,14 +389,22 @@ def register_extractors(
                 # We were able to find all the extractor files
                 break
 
+def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType, str], None], *args, **kwargs):
+    """Ensures logging is set up correctly for a child process and then executes the callback."""
+    logger = logging.getLogger()
+    qh = logging.handlers.QueueHandler(queue)
+    qh.setLevel(logging.DEBUG)
+    logger.addHandler(qh)
+    callback(*args, **kwargs, logger=logger)
 
 def import_extractors(
+    extractor_module_callback: Callable[[ModuleType, str], bool],
+    *,
     root_directory: str,
     scanner: yara.Rules,
-    extractor_module_callback: Callable[[ModuleType, str], bool],
-    logger: Logger,
     create_venv: bool = False,
     python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
+    logger: Logger,
 ):
     extractor_dirs, extractor_files = scan_for_extractors(root_directory, scanner, logger)
 
@@ -453,14 +471,20 @@ def run_extractor(
                 cwd=cwd,
                 capture_output=True,
             )
+            stderr = proc.stderr.decode()
             try:
                 # Load results and return them
                 output.seek(0)
-                return json.load(output, cls=json_decoder)
-            except Exception:
+                loaded =  json.load(output, cls=json_decoder)
+            except Exception as e:
                 # If there was an error raised during runtime, then propagate
                 delim = f'File "{module_path}"'
-                exception = proc.stderr.decode()
+                exception = stderr
                 if delim in exception:
                     exception = f"{delim}{exception.split(delim, 1)[1]}"
-                raise Exception(exception)
+                # print extractor logging at error level
+                logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
+                raise Exception(exception) from e
+            # ensure that extractor logging is available
+            logger.info(f"maco extractor stderr:\n{stderr}")
+            return loaded

{maco-1.2.3.dist-info → maco-1.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: maco
-Version: 1.2.3
+Version: 1.2.4
 Author: sl-govau
 Maintainer: cccs-rs
 License: MIT License

{maco-1.2.3.dist-info → maco-1.2.4.dist-info}/RECORD

@@ -1,25 +1,26 @@
+demo_extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 demo_extractors/elfy.py,sha256=AAFr5i1aivPwO4nycyXJEud57EpVNA-5k_2GicWesbY,771
-demo_extractors/limit_other.py,sha256=oscnNFkwD9Dm8Lae66GsRaAwUoTWUmkYpJu_wIsJ6sg,930
+demo_extractors/limit_other.py,sha256=RAFx_0K_WnhUURA5uwXGmjrWODrAuLZFBxqZcWaxf64,944
 demo_extractors/nothing.py,sha256=3aeQJTY-dakmVXmyfmrRM8YCQVT7q3bq880DFH1Ol_Y,607
 demo_extractors/shared.py,sha256=Wlvy77SCAR97gxi8uUhGYyjxGmDb-pOSvN8b1rXrVWs,304
 demo_extractors/complex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-demo_extractors/complex/complex.py,sha256=TkNmR9UUYo1f2JVpJhGasLG-5wHZI05JYxMIXj16GKM,2307
+demo_extractors/complex/complex.py,sha256=JFKqBGKwkuDSz4zZUJuqhCLUQv6dlCMhBqNj33grBsE,2323
 demo_extractors/complex/complex_utils.py,sha256=aec8kJsYUrMPo-waihkVLt-0QpiOPkw7dDqfT9MNuHk,123
 maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 maco/base_test.py,sha256=pqet9ofMwFRTj3JHgPdh9WHwdyp8kxNMi1vJNUzkSNA,2518
 maco/cli.py,sha256=1I3U54yPddTxqWclCtZ5Ma5hW6RoVTZMzLSFOjjfM1g,8008
-maco/collector.py,sha256=sxP374lK16lzdi0lhm5e4GT3a5lzeyE1VCicE4m_E3k,6003
+maco/collector.py,sha256=Vlo7KcJC7TKZFTElv8i_f_hvWEnlWCRzOP1xOc9x7vk,6532
 maco/extractor.py,sha256=4ZQd8OfvEQYUIkUS3LzZ5tcioembuLhT9_uRVNKSsyM,2750
-maco/utils.py,sha256=nyRTTsiwIE3L9XjFME6R_vR_EvLluwNL0kLb0r96jcg,18238
+maco/utils.py,sha256=vQeJKw4whWTXp1mTd2oEhfvL4nvvgAzWjBnCp2XxWLI,19275
 maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
 maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
 maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
 model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 model_setup/maco/base_test.py,sha256=pqet9ofMwFRTj3JHgPdh9WHwdyp8kxNMi1vJNUzkSNA,2518
 model_setup/maco/cli.py,sha256=1I3U54yPddTxqWclCtZ5Ma5hW6RoVTZMzLSFOjjfM1g,8008
-model_setup/maco/collector.py,sha256=sxP374lK16lzdi0lhm5e4GT3a5lzeyE1VCicE4m_E3k,6003
+model_setup/maco/collector.py,sha256=Vlo7KcJC7TKZFTElv8i_f_hvWEnlWCRzOP1xOc9x7vk,6532
 model_setup/maco/extractor.py,sha256=4ZQd8OfvEQYUIkUS3LzZ5tcioembuLhT9_uRVNKSsyM,2750
-model_setup/maco/utils.py,sha256=nyRTTsiwIE3L9XjFME6R_vR_EvLluwNL0kLb0r96jcg,18238
+model_setup/maco/utils.py,sha256=vQeJKw4whWTXp1mTd2oEhfvL4nvvgAzWjBnCp2XxWLI,19275
 model_setup/maco/yara.py,sha256=vPzCqauVp52ivcTdt8zwrYqDdkLutGlesma9DhKPzHw,2925
 model_setup/maco/model/__init__.py,sha256=SJrwdn12wklUFm2KoIgWjX_KgvJxCM7Ca9ntXOneuzc,31
 model_setup/maco/model/model.py,sha256=ngen4ViyLdRo_z_TqZBjw2DN0NrRLpuxOy15-6QmtNw,23536
@@ -27,15 +28,16 @@ pipelines/publish.yaml,sha256=xt3WNU-5kIICJgKIiiE94M3dWjS3uEiun-n4OmIssK8,1471
 pipelines/test.yaml,sha256=3KOoo-8SqP_bTAscsz5V3xxnuL91J-62mTjnQD1Btag,1019
 tests/data/example.txt.cart,sha256=j4ZdDnFNVq7lb-Qi4pY4evOXKQPKG-GSg-n-uEqPhV0,289
 tests/data/trigger_complex.txt,sha256=uqnLSrnyDGCmXwuPmZ2s8vdhH0hJs8DxvyaW_tuYY24,64
+tests/data/trigger_complex.txt.cart,sha256=Z7qF1Zi640O45Znkl9ooP2RhSLAEqY0NRf51d-q7utU,345
 tests/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/extractors/basic.py,sha256=r5eLCL6Ynr14nCBgtbLvUbm0NdrXizyc9c-4xBCNShU,828
 tests/extractors/basic_longer.py,sha256=1ClU2QD-Y0TOl_loNFvEqIEpTR5TSVJ6zg9ZmC-ESJo,860
 tests/extractors/test_basic.py,sha256=FLKekfSGM69HaiF7Vu_7D7KDXHZko-9hZkMO8_DoyYA,697
 tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/extractors/bob/bob.py,sha256=Gy5p8KssJX87cwa9vVv8UBODF_ulbUteZXh15frW2hs,247
-maco-1.2.3.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
-maco-1.2.3.dist-info/METADATA,sha256=81_65WF-TCXVfF3NeY5m3zCr831F7PVsTswjvZ34Ok4,15610
-maco-1.2.3.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-maco-1.2.3.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
-maco-1.2.3.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
-maco-1.2.3.dist-info/RECORD,,
+maco-1.2.4.dist-info/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
+maco-1.2.4.dist-info/METADATA,sha256=EoAEnCfbaXe8eUFAMHj3-C9lQIDR9Ss31NcY4CbEXjI,15610
+maco-1.2.4.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+maco-1.2.4.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
+maco-1.2.4.dist-info/top_level.txt,sha256=iMRwuzmrHA3zSwiSeMIl6FWhzRpn_st-I4fAv-kw5_o,49
+maco-1.2.4.dist-info/RECORD,,

model_setup/maco/collector.py

@@ -2,8 +2,9 @@
 
 import inspect
 import logging
+import logging.handlers
 import os
-from multiprocessing import Manager, Process
+from multiprocessing import Manager, Process, Queue
 from tempfile import NamedTemporaryFile
 from types import ModuleType
 from typing import Any, BinaryIO, Dict, List, Union
@@ -86,21 +87,33 @@ class Collector:
                     )
                     namespaced_rules[name] = member.yara_rule or extractor.DEFAULT_YARA_RULE.format(name=name)
 
+            # multiprocess logging is awkward - set up a queue to ensure we can log
+            logging_queue = Queue()
+            queue_handler = logging.handlers.QueueListener(logging_queue,*logging.getLogger().handlers)
+            queue_handler.start()
+
             # Find the extractors within the given directory
             # Execute within a child process to ensure main process interpreter is kept clean
             p = Process(
-                target=utils.import_extractors,
+                target=utils.proxy_logging,
                 args=(
-                    path_extractors,
-                    yara.compile(source=utils.MACO_YARA_RULE),
+                    logging_queue,
+                    utils.import_extractors,
                     extractor_module_callback,
-                    logger,
-                    create_venv and os.path.isdir(path_extractors),
+                ),
+                kwargs=dict(
+                    root_directory=path_extractors,
+                    scanner=yara.compile(source=utils.MACO_YARA_RULE),
+                    create_venv=create_venv and os.path.isdir(path_extractors),
                 ),
             )
             p.start()
             p.join()
 
+            # stop multiprocess logging
+            queue_handler.stop()
+            logging_queue.close()
+
             self.extractors = dict(extractors)
             if not self.extractors:
                 raise ExtractorLoadError("no extractors were loaded")

model_setup/maco/utils.py

@@ -4,11 +4,14 @@ import importlib.machinery
 import importlib.util
 import inspect
 import json
+import logging.handlers
 import os
 import re
 import shutil
 import subprocess
 import sys
+import multiprocessing
+import logging
 import tempfile
 
 from maco import yara
@@ -28,10 +31,12 @@ from typing import Callable, Dict, List, Set, Tuple
 
 from maco.extractor import Extractor
 
+logger = logging.getLogger("maco.lib.utils")
+
 VENV_DIRECTORY_NAME = ".venv"
 
-RELATIVE_FROM_RE = re.compile("from (\.+)")
-RELATIVE_FROM_IMPORT_RE = re.compile("from (\.+) import")
+RELATIVE_FROM_RE = re.compile(r"from (\.+)")
+RELATIVE_FROM_IMPORT_RE = re.compile(r"from (\.+) import")
 
 try:
     # Attempt to use the uv package manager (Recommended)
@@ -69,6 +74,7 @@ import importlib
 import json
 import os
 import sys
+import logging
 
 try:
     from maco import yara
@@ -76,6 +82,19 @@ except:
     import yara
 
 from base64 import b64encode
+
+# ensure we have a logger to stderr
+import logging
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+sh = logging.StreamHandler()
+logger.addHandler(sh)
+sh.setLevel(logging.DEBUG)
+formatter = logging.Formatter(
+    fmt="%(asctime)s, [%(levelname)s] %(module)s.%(funcName)s: %(message)s", datefmt="%Y-%m-%d (%H:%M:%S)"
+)
+sh.setFormatter(formatter)
+
 parent_package_path = "{parent_package_path}"
 sys.path.insert(1, parent_package_path)
 mod = importlib.import_module("{module_name}")
@@ -101,7 +120,7 @@ with open("{output_path}", 'w') as fp:
             json.dump(result.dict(exclude_defaults=True, exclude_none=True), fp, cls=Base64Encoder)
 """
 
-MACO_YARA_RULE = """
+MACO_YARA_RULE = r"""
 rule MACO {
     meta:
         desc = "Used to match on Python files that contain MACO extractors"
@@ -292,15 +311,10 @@ def register_extractors(
 ):
     package_name = os.path.basename(current_directory)
     parent_directory = os.path.dirname(current_directory)
-    symlink = None
-    while package_name in sys.modules:
-        # Package name conflicts with an existing loaded module, let's deconflict that
-        package_name = f"_{package_name}"
-
-        # We'll need to create a link back to the original
-        if package_name not in sys.modules:
-            symlink = os.path.join(parent_directory, package_name)
-            os.symlink(current_directory, symlink)
+    if package_name in sys.modules:
+        # this may happen as part of testing if some part of the extractor code was directly imported
+        logger.warning(f"Looks like {package_name} is already loaded. "
+                       "If your maco extractor overlaps an existing package name this could cause problems.")
 
     try:
         # Modify the PATH so we can recognize this new package on import
@@ -351,10 +365,6 @@ def register_extractors(
         # Remove any modules that were loaded to deconflict with later modules loads
         [sys.modules.pop(k) for k in set(sys.modules.keys()) - default_loaded_modules]
 
-        # Cleanup any symlinks
-        if symlink:
-            os.remove(symlink)
-
     # If there still exists extractor files we haven't found yet, try searching in the available subdirectories
     if extractor_files:
         for dir in os.listdir(current_directory):
@@ -379,14 +389,22 @@ def register_extractors(
                 # We were able to find all the extractor files
                 break
 
+def proxy_logging(queue: multiprocessing.Queue, callback: Callable[[ModuleType, str], None], *args, **kwargs):
+    """Ensures logging is set up correctly for a child process and then executes the callback."""
+    logger = logging.getLogger()
+    qh = logging.handlers.QueueHandler(queue)
+    qh.setLevel(logging.DEBUG)
+    logger.addHandler(qh)
+    callback(*args, **kwargs, logger=logger)
 
 def import_extractors(
+    extractor_module_callback: Callable[[ModuleType, str], bool],
+    *,
     root_directory: str,
     scanner: yara.Rules,
-    extractor_module_callback: Callable[[ModuleType, str], bool],
-    logger: Logger,
     create_venv: bool = False,
     python_version: str = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
+    logger: Logger,
 ):
     extractor_dirs, extractor_files = scan_for_extractors(root_directory, scanner, logger)
 
@@ -453,14 +471,20 @@ def run_extractor(
                 cwd=cwd,
                 capture_output=True,
             )
+            stderr = proc.stderr.decode()
             try:
                 # Load results and return them
                 output.seek(0)
-                return json.load(output, cls=json_decoder)
-            except Exception:
+                loaded =  json.load(output, cls=json_decoder)
+            except Exception as e:
                 # If there was an error raised during runtime, then propagate
                 delim = f'File "{module_path}"'
-                exception = proc.stderr.decode()
+                exception = stderr
                 if delim in exception:
                     exception = f"{delim}{exception.split(delim, 1)[1]}"
-                raise Exception(exception)
+                # print extractor logging at error level
+                logger.error(f"maco extractor raised exception, stderr:\n{stderr}")
+                raise Exception(exception) from e
+            # ensure that extractor logging is available
+            logger.info(f"maco extractor stderr:\n{stderr}")
+            return loaded