maco 1.2.18__py3-none-any.whl → 1.2.20__py3-none-any.whl

extractor_setup/maco/cli.py

@@ -27,6 +27,7 @@ def process_file(
     pretty: bool,
     force: bool,
     include_base64: bool,
+    extracted_dir: str = "",
 ):
     """Process a filestream with the extractors and rules.
 
@@ -37,6 +38,7 @@ def process_file(
         pretty (bool): Pretty print the JSON output
         force (bool): Run all extractors regardless of YARA rule match
         include_base64 (bool): include base64'd data in output
+        extracted_dir (str): directory to write CaRTed binary data to
 
     Returns:
         (dict): The output from the extractors analyzing the sample
@@ -87,6 +89,34 @@ def process_file(
                 if include_base64:
                     # this can be large
                     row["base64"] = base64.b64encode(row["data"]).decode("utf8")
+
+                # write binary data to disk if enabled
+                if extracted_dir:
+                    # only allow writes to already existing directories with permissions
+                    if os.path.isdir(extracted_dir) and os.access(extracted_dir, os.W_OK):
+                        filepath = os.path.abspath(os.path.join(extracted_dir, f"{row['sha256']}.cart"))
+                        # don't overwrite existing files
+                        if os.path.exists(filepath):
+                            logger.debug(f"{filepath} already exists.")
+                        else:
+                            # CaRT data before writing to disk
+                            in_stream = io.BytesIO(row["data"])
+                            output_stream = io.BytesIO()
+                            try:
+                                cart.pack_stream(in_stream, output_stream)
+                            except Exception:
+                                logger.error(f"Error trying to CaRT binary output ({row['sha256']}) from {path_file}.")
+                            else:
+                                output_stream.seek(0)
+                                try:
+                                    with open(filepath, "wb") as f:
+                                        f.write(output_stream.getbuffer())
+                                    logger.debug(f"Wrote binary output to {filepath}.")
+                                except (FileNotFoundError, PermissionError, OSError):
+                                    logger.error(f"Error trying to write binary output to {filepath}")
+                    else:
+                        logger.error(f"Cannot write files to {extracted_dir}")
+
                 # do not print raw bytes to console
                 row.pop("data")
         ret[extractor_name] = resp
@@ -107,6 +137,7 @@ def process_filesystem(
     include_base64: bool,
     create_venv: bool = False,
     skip_install: bool = False,
+    extracted_dir: str = "",
 ) -> Tuple[int, int, int]:
     """Process filesystem with extractors and print results of extraction.
 
@@ -159,6 +190,7 @@ def process_filesystem(
                             pretty=pretty,
                             force=force,
                             include_base64=include_base64,
+                            extracted_dir=extracted_dir,
                         )
                         if resp:
                             num_hits += 1
@@ -194,6 +226,7 @@ def main():
         help="Include base64 encoded binary data in output "
         "(can be large, consider printing to file rather than console)",
     )
+    parser.add_argument("--binarydir", type=str, help="directory to write extracted binary data to")
     parser.add_argument("--logfile", type=str, help="file to log output")
     parser.add_argument("--include", type=str, help="comma separated extractors to run")
     parser.add_argument("--exclude", type=str, help="comma separated extractors to not run")
@@ -268,6 +301,7 @@ def main():
         include_base64=args.base64,
         create_venv=args.create_venv,
         skip_install=not args.force_install,
+        extracted_dir=args.binarydir,
     )
 
 

extractor_setup/maco/collector.py

@@ -122,6 +122,7 @@ class Collector:
                             "author": member.author,
                             "last_modified": member.last_modified,
                             "sharing": member.sharing,
+                            "result_sharing": member.result_sharing,
                             "description": member.__doc__,
                         },
                     )

extractor_setup/maco/extractor.py

@@ -25,7 +25,8 @@ class Extractor:
     family: Union[str, List[str]] = None  # family or families of malware that is detected by the extractor
     author: str = None  # author of the extractor (name@organisation)
     last_modified: str = None  # last modified date (YYYY-MM-DD)
-    sharing: str = "TLP:WHITE"  # who can this be shared with?
+    sharing: str = "TLP:CLEAR"  # who can this be shared with?
+    result_sharing: str = sharing  # who can the results be shared with? (defaults to sharing)
     yara_rule: str = None  # yara rule that we filter inputs with
     reference: str = None  # link to malware report or other reference information
     logger: logging.Logger = None  # logger for use when debugging

maco/cli.py

@@ -27,6 +27,7 @@ def process_file(
     pretty: bool,
     force: bool,
     include_base64: bool,
+    extracted_dir: str = "",
 ):
     """Process a filestream with the extractors and rules.
 
@@ -37,6 +38,7 @@ def process_file(
         pretty (bool): Pretty print the JSON output
         force (bool): Run all extractors regardless of YARA rule match
         include_base64 (bool): include base64'd data in output
+        extracted_dir (str): directory to write CaRTed binary data to
 
     Returns:
         (dict): The output from the extractors analyzing the sample
@@ -87,6 +89,34 @@ def process_file(
                 if include_base64:
                     # this can be large
                     row["base64"] = base64.b64encode(row["data"]).decode("utf8")
+
+                # write binary data to disk if enabled
+                if extracted_dir:
+                    # only allow writes to already existing directories with permissions
+                    if os.path.isdir(extracted_dir) and os.access(extracted_dir, os.W_OK):
+                        filepath = os.path.abspath(os.path.join(extracted_dir, f"{row['sha256']}.cart"))
+                        # don't overwrite existing files
+                        if os.path.exists(filepath):
+                            logger.debug(f"{filepath} already exists.")
+                        else:
+                            # CaRT data before writing to disk
+                            in_stream = io.BytesIO(row["data"])
+                            output_stream = io.BytesIO()
+                            try:
+                                cart.pack_stream(in_stream, output_stream)
+                            except Exception:
+                                logger.error(f"Error trying to CaRT binary output ({row['sha256']}) from {path_file}.")
+                            else:
+                                output_stream.seek(0)
+                                try:
+                                    with open(filepath, "wb") as f:
+                                        f.write(output_stream.getbuffer())
+                                    logger.debug(f"Wrote binary output to {filepath}.")
+                                except (FileNotFoundError, PermissionError, OSError):
+                                    logger.error(f"Error trying to write binary output to {filepath}")
+                    else:
+                        logger.error(f"Cannot write files to {extracted_dir}")
+
                 # do not print raw bytes to console
                 row.pop("data")
         ret[extractor_name] = resp
@@ -107,6 +137,7 @@ def process_filesystem(
     include_base64: bool,
     create_venv: bool = False,
     skip_install: bool = False,
+    extracted_dir: str = "",
 ) -> Tuple[int, int, int]:
     """Process filesystem with extractors and print results of extraction.
 
@@ -159,6 +190,7 @@ def process_filesystem(
                             pretty=pretty,
                             force=force,
                             include_base64=include_base64,
+                            extracted_dir=extracted_dir,
                         )
                         if resp:
                             num_hits += 1
@@ -194,6 +226,7 @@ def main():
         help="Include base64 encoded binary data in output "
         "(can be large, consider printing to file rather than console)",
     )
+    parser.add_argument("--binarydir", type=str, help="directory to write extracted binary data to")
     parser.add_argument("--logfile", type=str, help="file to log output")
     parser.add_argument("--include", type=str, help="comma separated extractors to run")
     parser.add_argument("--exclude", type=str, help="comma separated extractors to not run")
@@ -268,6 +301,7 @@ def main():
         include_base64=args.base64,
         create_venv=args.create_venv,
         skip_install=not args.force_install,
+        extracted_dir=args.binarydir,
     )
 
 

maco/collector.py

@@ -122,6 +122,7 @@ class Collector:
                             "author": member.author,
                             "last_modified": member.last_modified,
                             "sharing": member.sharing,
+                            "result_sharing": member.result_sharing,
                             "description": member.__doc__,
                         },
                     )

maco/extractor.py

@@ -25,7 +25,8 @@ class Extractor:
     family: Union[str, List[str]] = None  # family or families of malware that is detected by the extractor
     author: str = None  # author of the extractor (name@organisation)
     last_modified: str = None  # last modified date (YYYY-MM-DD)
-    sharing: str = "TLP:WHITE"  # who can this be shared with?
+    sharing: str = "TLP:CLEAR"  # who can this be shared with?
+    result_sharing: str = sharing  # who can the results be shared with? (defaults to sharing)
     yara_rule: str = None  # yara rule that we filter inputs with
     reference: str = None  # link to malware report or other reference information
     logger: logging.Logger = None  # logger for use when debugging

{maco-1.2.18.dist-info → maco-1.2.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: maco
-Version: 1.2.18
+Version: 1.2.20
 Summary: Maco is a framework for creating and using malware configuration extractors.
 Author: sl-govau
 Maintainer: cccs-rs

{maco-1.2.18.dist-info → maco-1.2.20.dist-info}/RECORD

@@ -10,36 +10,36 @@ demo_extractors/complex/complex.py,sha256=GYKmPOD8-fyVHxwjZb-3t1IghKVMuLtdUvCs5C
 demo_extractors/complex/complex_utils.py,sha256=5kdMl-niSH9d-d3ChuItpmlPT4U-S9g-iyBZlR4tfmQ,296
 extractor_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extractor_setup/maco/base_test.py,sha256=DrVE7vOazeLQpOQeIDwBYK1WtlmdJrRe50JOqP5t4Y0,3198
-extractor_setup/maco/cli.py,sha256=nrSukAJAthbstZT3-lQNPz4zOOMcBhvfYQqLh_B5Jdk,9457
-extractor_setup/maco/collector.py,sha256=R3zw-fUJBlwmcSqvkQ-PnoJdHfRm2V0JAOl7N8MTAbY,8240
+extractor_setup/maco/cli.py,sha256=jFtYWuKNlqcnSl0W4vgITctpgfd3J9kgN37IwoW3pDY,11477
+extractor_setup/maco/collector.py,sha256=I0Nidf4-xcvoe6X0bbCsAXjr66iPf00JDO6ocKkaZLc,8309
 extractor_setup/maco/exceptions.py,sha256=XBHUrs1kr1ZayPI9B_W-WejKgVmC8sWL_o4RL0b4DQE,745
-extractor_setup/maco/extractor.py,sha256=s36aGcsXSc-9iCik6iihVt5G1a1DZUA7TquvWYQNwdE,2912
+extractor_setup/maco/extractor.py,sha256=nqIfUcrc_l57FicKZc6HLtN223-_zkYWL1AYMy1WAmA,3007
 extractor_setup/maco/utils.py,sha256=yNm5CiHc9033ONX_gFwg9Lng5IYFujLDtw6FfiJkAao,23425
 extractor_setup/maco/yara.py,sha256=y141t8NqDDXHY23uE1d6BDPeNmSuUuohR6Yr_LKa7GI,4067
 extractor_setup/maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
 extractor_setup/maco/model/model.py,sha256=DBHTmZXMzjpVq0s2mzZv3VCzPhwPnv7sH6u_QZCTcA4,24484
 maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 maco/base_test.py,sha256=DrVE7vOazeLQpOQeIDwBYK1WtlmdJrRe50JOqP5t4Y0,3198
-maco/cli.py,sha256=nrSukAJAthbstZT3-lQNPz4zOOMcBhvfYQqLh_B5Jdk,9457
-maco/collector.py,sha256=R3zw-fUJBlwmcSqvkQ-PnoJdHfRm2V0JAOl7N8MTAbY,8240
+maco/cli.py,sha256=jFtYWuKNlqcnSl0W4vgITctpgfd3J9kgN37IwoW3pDY,11477
+maco/collector.py,sha256=I0Nidf4-xcvoe6X0bbCsAXjr66iPf00JDO6ocKkaZLc,8309
 maco/exceptions.py,sha256=XBHUrs1kr1ZayPI9B_W-WejKgVmC8sWL_o4RL0b4DQE,745
-maco/extractor.py,sha256=s36aGcsXSc-9iCik6iihVt5G1a1DZUA7TquvWYQNwdE,2912
+maco/extractor.py,sha256=nqIfUcrc_l57FicKZc6HLtN223-_zkYWL1AYMy1WAmA,3007
 maco/utils.py,sha256=yNm5CiHc9033ONX_gFwg9Lng5IYFujLDtw6FfiJkAao,23425
 maco/yara.py,sha256=y141t8NqDDXHY23uE1d6BDPeNmSuUuohR6Yr_LKa7GI,4067
 maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
 maco/model/model.py,sha256=DBHTmZXMzjpVq0s2mzZv3VCzPhwPnv7sH6u_QZCTcA4,24484
-maco-1.2.18.dist-info/licenses/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
+maco-1.2.20.dist-info/licenses/LICENSE.md,sha256=gMSjshPhXvV_F1qxmeNkKdBqGWkd__fEJf4glS504bM,1478
 model_setup/maco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 model_setup/maco/base_test.py,sha256=DrVE7vOazeLQpOQeIDwBYK1WtlmdJrRe50JOqP5t4Y0,3198
-model_setup/maco/cli.py,sha256=nrSukAJAthbstZT3-lQNPz4zOOMcBhvfYQqLh_B5Jdk,9457
-model_setup/maco/collector.py,sha256=R3zw-fUJBlwmcSqvkQ-PnoJdHfRm2V0JAOl7N8MTAbY,8240
+model_setup/maco/cli.py,sha256=jFtYWuKNlqcnSl0W4vgITctpgfd3J9kgN37IwoW3pDY,11477
+model_setup/maco/collector.py,sha256=I0Nidf4-xcvoe6X0bbCsAXjr66iPf00JDO6ocKkaZLc,8309
 model_setup/maco/exceptions.py,sha256=XBHUrs1kr1ZayPI9B_W-WejKgVmC8sWL_o4RL0b4DQE,745
-model_setup/maco/extractor.py,sha256=s36aGcsXSc-9iCik6iihVt5G1a1DZUA7TquvWYQNwdE,2912
+model_setup/maco/extractor.py,sha256=nqIfUcrc_l57FicKZc6HLtN223-_zkYWL1AYMy1WAmA,3007
 model_setup/maco/utils.py,sha256=yNm5CiHc9033ONX_gFwg9Lng5IYFujLDtw6FfiJkAao,23425
 model_setup/maco/yara.py,sha256=y141t8NqDDXHY23uE1d6BDPeNmSuUuohR6Yr_LKa7GI,4067
 model_setup/maco/model/__init__.py,sha256=ULdyHx8R5D2ICHZo3VoCk1YTlewTok36TYIpwx__pNY,45
 model_setup/maco/model/model.py,sha256=DBHTmZXMzjpVq0s2mzZv3VCzPhwPnv7sH6u_QZCTcA4,24484
-pipelines/publish.yaml,sha256=yjc3eqrI-LHLSfZ0DPtxwdfPDT0NI6LUA_zy61UxN_8,1654
+pipelines/publish.yaml,sha256=BfsbDsg2ijtXF8lhRUjzkenw3zi2mL7ESNv3KuC1cVE,1626
 pipelines/test.yaml,sha256=btJVI-R39UBeYosGu7TOpU6V9ogFW3FT3ROtWygQGQ0,1472
 tests/data/example.txt.cart,sha256=j4ZdDnFNVq7lb-Qi4pY4evOXKQPKG-GSg-n-uEqPhV0,289
 tests/data/trigger_complex.txt,sha256=uqnLSrnyDGCmXwuPmZ2s8vdhH0hJs8DxvyaW_tuYY24,64
@@ -52,8 +52,8 @@ tests/extractors/bob/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 tests/extractors/bob/bob.py,sha256=4fpqy_O6NDinJImghyW5OwYgnaB05aY4kgoIS_C3c_U,253
 tests/extractors/import_rewriting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/extractors/import_rewriting/importer.py,sha256=wqF1AG2zXXuj9EMt9qlDorab-UD0GYuFggtrCuz4sf0,289735
-maco-1.2.18.dist-info/METADATA,sha256=citbYasnfKhc-PAxK7tLQt_Dc2LZRbhKn26ChD0PC3g,15310
-maco-1.2.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-maco-1.2.18.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
-maco-1.2.18.dist-info/top_level.txt,sha256=xiVS11ZoyN8ChHJQGpOzTH4ZyQ3YJe1qT3Yt4gcKGUk,65
-maco-1.2.18.dist-info/RECORD,,
+maco-1.2.20.dist-info/METADATA,sha256=8ZvRVvxy741jFWHAP1g-B_57zuamXaiglsEglweURUI,15310
+maco-1.2.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+maco-1.2.20.dist-info/entry_points.txt,sha256=TpcwG1gedIg8Y7a9ZOv8aQpuwEUftCefDrAjzeP-o6U,39
+maco-1.2.20.dist-info/top_level.txt,sha256=xiVS11ZoyN8ChHJQGpOzTH4ZyQ3YJe1qT3Yt4gcKGUk,65
+maco-1.2.20.dist-info/RECORD,,

model_setup/maco/cli.py

@@ -27,6 +27,7 @@ def process_file(
     pretty: bool,
     force: bool,
     include_base64: bool,
+    extracted_dir: str = "",
 ):
     """Process a filestream with the extractors and rules.
 
@@ -37,6 +38,7 @@ def process_file(
         pretty (bool): Pretty print the JSON output
         force (bool): Run all extractors regardless of YARA rule match
         include_base64 (bool): include base64'd data in output
+        extracted_dir (str): directory to write CaRTed binary data to
 
     Returns:
         (dict): The output from the extractors analyzing the sample
@@ -87,6 +89,34 @@ def process_file(
                 if include_base64:
                     # this can be large
                     row["base64"] = base64.b64encode(row["data"]).decode("utf8")
+
+                # write binary data to disk if enabled
+                if extracted_dir:
+                    # only allow writes to already existing directories with permissions
+                    if os.path.isdir(extracted_dir) and os.access(extracted_dir, os.W_OK):
+                        filepath = os.path.abspath(os.path.join(extracted_dir, f"{row['sha256']}.cart"))
+                        # don't overwrite existing files
+                        if os.path.exists(filepath):
+                            logger.debug(f"{filepath} already exists.")
+                        else:
+                            # CaRT data before writing to disk
+                            in_stream = io.BytesIO(row["data"])
+                            output_stream = io.BytesIO()
+                            try:
+                                cart.pack_stream(in_stream, output_stream)
+                            except Exception:
+                                logger.error(f"Error trying to CaRT binary output ({row['sha256']}) from {path_file}.")
+                            else:
+                                output_stream.seek(0)
+                                try:
+                                    with open(filepath, "wb") as f:
+                                        f.write(output_stream.getbuffer())
+                                    logger.debug(f"Wrote binary output to {filepath}.")
+                                except (FileNotFoundError, PermissionError, OSError):
+                                    logger.error(f"Error trying to write binary output to {filepath}")
+                    else:
+                        logger.error(f"Cannot write files to {extracted_dir}")
+
                 # do not print raw bytes to console
                 row.pop("data")
         ret[extractor_name] = resp
@@ -107,6 +137,7 @@ def process_filesystem(
     include_base64: bool,
     create_venv: bool = False,
     skip_install: bool = False,
+    extracted_dir: str = "",
 ) -> Tuple[int, int, int]:
     """Process filesystem with extractors and print results of extraction.
 
@@ -159,6 +190,7 @@ def process_filesystem(
                             pretty=pretty,
                             force=force,
                             include_base64=include_base64,
+                            extracted_dir=extracted_dir,
                         )
                         if resp:
                             num_hits += 1
@@ -194,6 +226,7 @@ def main():
         help="Include base64 encoded binary data in output "
         "(can be large, consider printing to file rather than console)",
     )
+    parser.add_argument("--binarydir", type=str, help="directory to write extracted binary data to")
     parser.add_argument("--logfile", type=str, help="file to log output")
     parser.add_argument("--include", type=str, help="comma separated extractors to run")
     parser.add_argument("--exclude", type=str, help="comma separated extractors to not run")
@@ -268,6 +301,7 @@ def main():
         include_base64=args.base64,
         create_venv=args.create_venv,
         skip_install=not args.force_install,
+        extracted_dir=args.binarydir,
     )
 
 

model_setup/maco/collector.py

@@ -122,6 +122,7 @@ class Collector:
                             "author": member.author,
                             "last_modified": member.last_modified,
                             "sharing": member.sharing,
+                            "result_sharing": member.result_sharing,
                             "description": member.__doc__,
                         },
                     )

model_setup/maco/extractor.py

@@ -25,7 +25,8 @@ class Extractor:
     family: Union[str, List[str]] = None  # family or families of malware that is detected by the extractor
     author: str = None  # author of the extractor (name@organisation)
     last_modified: str = None  # last modified date (YYYY-MM-DD)
-    sharing: str = "TLP:WHITE"  # who can this be shared with?
+    sharing: str = "TLP:CLEAR"  # who can this be shared with?
+    result_sharing: str = sharing  # who can the results be shared with? (defaults to sharing)
     yara_rule: str = None  # yara rule that we filter inputs with
     reference: str = None  # link to malware report or other reference information
     logger: logging.Logger = None  # logger for use when debugging

pipelines/publish.yaml

@@ -12,33 +12,34 @@ pool:
   vmImage: "ubuntu-22.04"
 
 jobs:
-# - job: test
-#   displayName: Test
-#   strategy:
-#     matrix:
-#       Python38:
-#         python.version: '3.8'
-#       Python39:
-#         python.version: '3.9'
-#       Python310:
-#         python.version: '3.10'
-#       Python311:
-#         python.version: '3.11'
-#       Python312:
-#         python.version: '3.12'
-#   steps:
-#   - task: UsePythonVersion@0
-#     displayName: 'Use Python $(python.version)'
-#     inputs:
-#       versionSpec: '$(python.version)'
+- job: test
+  displayName: Test
+  strategy:
+    matrix:
+      Python38:
+        python.version: '3.8'
+      Python39:
+        python.version: '3.9'
+      Python310:
+        python.version: '3.10'
+      Python311:
+        python.version: '3.11'
+      Python312:
+        python.version: '3.12'
+  steps:
+  - task: UsePythonVersion@0
+    displayName: 'Use Python $(python.version)'
+    inputs:
+      versionSpec: '$(python.version)'
 
-#   - script: |
-#       set -x
+  - script: |
+      set -x
 
-#       python -m pip install -U tox
-#       python -m tox -e py
+      python -m pip install -U tox
+      python -m tox -e py
 
 - job: build_and_deploy
+  dependsOn: test
   displayName: Build and Deploy
   variables:
   - group: deployment-information