dissect.target 3.19.dev21__tar.gz → 3.19.dev22__tar.gz

{dissect_target-3.19.dev21/dissect.target.egg-info → dissect_target-3.19.dev22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dissect.target
-Version: 3.19.dev21
+Version: 3.19.dev22
 Summary: This module ties all other Dissect modules together, it provides a programming API and command line tools which allow easy access to various data sources inside disk images or file collections (a.k.a. targets)
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

dissect_target-3.19.dev22/dissect/target/plugins/filesystem/yara.py

@@ -0,0 +1,186 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+from io import BytesIO
+from pathlib import Path
+from typing import Iterator
+
+from dissect.target.helpers import hashutil
+
+try:
+    import yara
+
+    HAS_YARA = True
+
+except ImportError:
+    HAS_YARA = False
+
+from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
+from dissect.target.helpers.record import TargetRecordDescriptor
+from dissect.target.plugin import Plugin, arg, export
+
+log = logging.getLogger(__name__)
+
+YaraMatchRecord = TargetRecordDescriptor(
+    "filesystem/yara/match",
+    [
+        ("path", "path"),
+        ("digest", "digest"),
+        ("string", "rule"),
+        ("string[]", "tags"),
+        ("string", "namespace"),
+    ],
+)
+
+DEFAULT_MAX_SCAN_SIZE = 10 * 1024 * 1024
+
+
+class YaraPlugin(Plugin):
+    """Plugin to scan files against a local YARA rules file."""
+
+    def check_compatible(self) -> None:
+        if not HAS_YARA:
+            raise UnsupportedPluginError("Please install 'yara-python' to use the yara plugin.")
+
+    @arg("-r", "--rules", required=True, nargs="*", help="path(s) to YARA rule file(s) or folder(s)")
+    @arg("-p", "--path", default="/", help="path on target(s) to recursively scan")
+    @arg("-m", "--max-size", default=DEFAULT_MAX_SCAN_SIZE, help="maximum file size in bytes to scan")
+    @arg("-c", "--check", default=False, action="store_true", help="check if every YARA rule is valid")
+    @export(record=YaraMatchRecord)
+    def yara(
+        self,
+        rules: list[str | Path],
+        path: str = "/",
+        max_size: int = DEFAULT_MAX_SCAN_SIZE,
+        check: bool = False,
+    ) -> Iterator[YaraMatchRecord]:
+        """Scan files inside the target up to a given maximum size with YARA rule file(s).
+
+        Args:
+            rules: ``list`` of strings or ``Path`` objects pointing to rule files to use.
+            path: ``string`` of absolute target path to scan.
+            max_size: Files larger than this size will not be scanned.
+            check: Check if provided rules are valid, only compiles valid rules.
+
+        Returns:
+            Iterator yields ``YaraMatchRecord``.
+        """
+
+        compiled_rules = process_rules(rules, check)
+
+        if not rules:
+            self.target.log.error("No working rules found in '%s'", ",".join(rules))
+            return
+
+        if hasattr(compiled_rules, "warnings") and (num_warns := len(compiled_rules.warnings)) > 0:
+            self.target.log.warning("YARA generated %s warnings while compiling rules", num_warns)
+            for warning in compiled_rules.warnings:
+                self.target.log.debug(warning)
+
+        self.target.log.warning("Will not scan files larger than %s MB", max_size // 1024 // 1024)
+
+        for _, _, files in self.target.fs.walk_ext(path):
+            for file in files:
+                try:
+                    if file_size := file.stat().st_size > max_size:
+                        self.target.log.debug(
+                            "Skipping file '%s' as it is larger than %s bytes (size is %s)", file, file_size, max_size
+                        )
+                        continue
+
+                    buf = file.open().read()
+                    for match in compiled_rules.match(data=buf):
+                        yield YaraMatchRecord(
+                            path=self.target.fs.path(file.path),
+                            digest=hashutil.common(BytesIO(buf)),
+                            rule=match.rule,
+                            tags=match.tags,
+                            namespace=match.namespace,
+                            _target=self.target,
+                        )
+
+                except FileNotFoundError:
+                    continue
+                except RuntimeWarning as e:
+                    self.target.log.warning("Runtime warning while scanning file '%s': %s", file, e)
+                except Exception as e:
+                    self.target.log.error("Exception scanning file '%s'", file)
+                    self.target.log.debug("", exc_info=e)
+
+
+def process_rules(paths: list[str | Path], check: bool = False) -> yara.Rules | None:
+    """Generate compiled YARA rules from the given path(s).
+
+    Provide path to one (compiled) YARA file or directory containing YARA files.
+
+    Args:
+        paths: Path to file(s) or folder(s) containing YARA files.
+        check: Attempt to compile every rule file before appending to rules.
+
+    Returns:
+        Compiled YARA rules or None.
+    """
+    files = set()
+    compiled_rules = None
+
+    for rules_path in paths:
+        if isinstance(rules_path, str):
+            rules_path = Path(rules_path)
+
+        if not rules_path.exists():
+            log.warning("File %s does not exist!", rules_path)
+            continue
+
+        if rules_path.is_dir():
+            for file in rules_path.rglob("*"):
+                if not file.is_file():
+                    continue
+                files.add(file)
+        else:
+            files.add(rules_path)
+
+    for file in set(files):
+        with file.open("rb") as fh:
+            magic = fh.read(4)
+
+        if magic == b"YARA":
+            if len(files) > 1:
+                log.error("Providing multiple compiled YARA files is not supported. Did not add %s", file)
+                continue
+            else:
+                log.info("Adding single compiled YARA file %s", file)
+                compiled_rules = compile_yara(file, is_compiled=True)
+                break
+
+        elif check and not is_valid_yara({"check_namespace": file}):
+            log.warning("File %s contains invalid rule(s)!", file)
+            files.remove(file)
+            continue
+
+    if files and not compiled_rules:
+        try:
+            compiled_rules = compile_yara({hashlib.md5(file.as_posix().encode()).hexdigest(): file for file in files})
+        except yara.Error as e:
+            log.error("Failed to compile YARA file(s): %s", e)
+
+    return compiled_rules
+
+
+def compile_yara(files: dict[str, Path] | Path, is_compiled: bool = False) -> yara.Rules | None:
+    """Compile or load the given YARA file(s) to rules."""
+    if is_compiled and isinstance(files, Path):
+        return yara.load(files.as_posix())
+    else:
+        return yara.compile(filepaths={ns: Path(path).as_posix() for ns, path in files.items()})
+
+
+def is_valid_yara(files: dict[str, Path] | Path, is_compiled: bool = False) -> bool:
+    """Determine if the given YARA file(s) compile without errors or warnings."""
+    try:
+        compile_yara(files, is_compiled)
+        return True
+
+    except (yara.SyntaxError, yara.WarningError, yara.Error) as e:
+        log.debug("Rule file(s) '%s' invalid: %s", files, e)
+        return False

dissect_target-3.19.dev22/dissect/target/tools/yara.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import argparse
+import logging
+
+from dissect.target import Target
+from dissect.target.exceptions import TargetError
+from dissect.target.plugins.filesystem.yara import HAS_YARA, YaraPlugin
+from dissect.target.tools.query import record_output
+from dissect.target.tools.utils import (
+    catch_sigpipe,
+    configure_generic_arguments,
+    process_generic_arguments,
+)
+
+log = logging.getLogger(__name__)
+
+
+@catch_sigpipe
+def main():
+    help_formatter = argparse.ArgumentDefaultsHelpFormatter
+    parser = argparse.ArgumentParser(
+        description="target-yara",
+        fromfile_prefix_chars="@",
+        formatter_class=help_formatter,
+    )
+
+    parser.add_argument("targets", metavar="TARGETS", nargs="*", help="Targets to load")
+    parser.add_argument("-s", "--strings", default=False, action="store_true", help="print output as string")
+
+    for args, kwargs in getattr(YaraPlugin.yara, "__args__", []):
+        parser.add_argument(*args, **kwargs)
+
+    configure_generic_arguments(parser)
+
+    args = parser.parse_args()
+    process_generic_arguments(args)
+
+    if not HAS_YARA:
+        log.error("yara-python is not installed: pip install yara-python")
+        parser.exit(1)
+
+    if not args.targets:
+        log.error("No targets provided")
+        parser.exit(1)
+
+    try:
+        for target in Target.open_all(args.targets):
+            target.log.info("Scanning target")
+            rs = record_output(args.strings, False)
+            for record in target.yara(args.rules, args.path, args.max_size, args.check):
+                rs.write(record)
+
+    except TargetError as e:
+        log.error(e)
+        log.debug("", exc_info=e)
+        parser.exit(1)
+
+
+if __name__ == "__main__":
+    main()

{dissect_target-3.19.dev21 → dissect_target-3.19.dev22/dissect.target.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dissect.target
-Version: 3.19.dev21
+Version: 3.19.dev22
 Summary: This module ties all other Dissect modules together, it provides a programming API and command line tools which allow easy access to various data sources inside disk images or file collections (a.k.a. targets)
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{dissect_target-3.19.dev21 → dissect_target-3.19.dev22}/dissect.target.egg-info/SOURCES.txt

@@ -345,6 +345,7 @@ dissect/target/tools/query.py
 dissect/target/tools/reg.py
 dissect/target/tools/shell.py
 dissect/target/tools/utils.py
+dissect/target/tools/yara.py
 dissect/target/tools/dump/__init__.py
 dissect/target/tools/dump/run.py
 dissect/target/tools/dump/state.py
@@ -586,6 +587,7 @@ tests/tools/test_query.py
 tests/tools/test_reg.py
 tests/tools/test_shell.py
 tests/tools/test_utils.py
+tests/tools/test_yara.py
 tests/volumes/__init__.py
 tests/volumes/test_bde.py
 tests/volumes/test_md.py

{dissect_target-3.19.dev21 → dissect_target-3.19.dev22}/dissect.target.egg-info/entry_points.txt

@@ -8,3 +8,4 @@ target-mount = dissect.target.tools.mount:main
 target-query = dissect.target.tools.query:main
 target-reg = dissect.target.tools.reg:main
 target-shell = dissect.target.tools.shell:main
+target-yara = dissect.target.tools.yara:main

{dissect_target-3.19.dev21 → dissect_target-3.19.dev22}/pyproject.toml

@@ -133,6 +133,7 @@ target-mount = "dissect.target.tools.mount:main"
 target-query = "dissect.target.tools.query:main"
 target-reg = "dissect.target.tools.reg:main"
 target-shell = "dissect.target.tools.shell:main"
+target-yara = "dissect.target.tools.yara:main"
 
 [tool.black]
 line-length = 120

dissect_target-3.19.dev22/tests/plugins/filesystem/test_yara.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import tempfile
+from io import BytesIO
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+from dissect.target import Target
+from dissect.target.filesystem import VirtualFilesystem
+from dissect.target.plugins.filesystem.yara import HAS_YARA, YaraPlugin, is_valid_yara
+from tests._utils import absolute_path
+
+if HAS_YARA:
+    import yara
+
+rule_file = absolute_path("_data/plugins/filesystem/yara/rule.yar")
+another_rule_file = absolute_path("_data/plugins/filesystem/yara/another.yar")
+invalid_rule = absolute_path("_data/plugins/filesystem/yara/invalid.yar")
+rule_dir = Path(rule_file).parent
+
+
+@pytest.fixture
+def target_yara(target_default: Target) -> Iterator[Target]:
+    vfs = VirtualFilesystem()
+    vfs.map_file_fh("test_file", BytesIO(b"test string"))
+    vfs.map_file_fh("/test/dir/to/test_file", BytesIO(b"test string"))
+    vfs.map_file_fh("should_not_hit", BytesIO(b"this is another file."))
+    target_default.fs.mount("/", vfs)
+    target_default.add_plugin(YaraPlugin)
+    yield target_default
+
+
+@pytest.mark.skipif(not HAS_YARA, reason="requires python-yara")
+def test_yara_plugin(target_yara: Target) -> None:
+    results = list(target_yara.yara(rules=[Path(rule_file)]))
+
+    assert len(results) == 2
+    assert results[0].path == "/test_file"
+    assert results[1].path == "/test/dir/to/test_file"
+    assert results[0].rule == "test_rule_name"
+
+
+@pytest.mark.skipif(not HAS_YARA, reason="requires python-yara")
+@pytest.mark.parametrize(
+    "rules,expected_hits,should_be_valid",
+    [
+        (["/does/not/exist"], 0, False),
+        ([rule_file, rule_file], 2, True),
+        ([rule_file, another_rule_file], 4, True),
+        ([rule_dir], 4, False),  # contains invalid.yar
+        ([invalid_rule], 0, False),
+    ],
+)
+def test_yara_plugin_invalid_rules(
+    target_yara: Target, rules: list[str | Path], expected_hits: int, should_be_valid: bool
+) -> None:
+    assert is_valid_yara(files={str(file): file for file in rules}) == should_be_valid
+
+    results = list(target_yara.yara(rules=rules, check=True))
+    assert len(results) == expected_hits
+
+
+@pytest.mark.skipif(not HAS_YARA, reason="requires python-yara")
+def test_yara_plugin_invalid_rule_warn(target_yara: Target, caplog: pytest.CaptureFixture) -> None:
+    results = list(target_yara.yara(rules=[invalid_rule, another_rule_file], check=True))
+    assert "invalid.yar contains invalid rule(s)!" in caplog.text
+    assert len(results) == 2
+
+
+@pytest.mark.skipif(not HAS_YARA, reason="requires python-yara")
+def test_yara_plugin_compiled_rule(target_yara: Target, tmp_path: str) -> None:
+    with tempfile.NamedTemporaryFile(mode="w", dir=tmp_path, delete=False) as tf:
+        rules = yara.compile(rule_file)
+        rules.save(tf.name)
+        tf.close()
+
+        results = list(target_yara.yara(rules=[tf.name]))
+
+        assert len(results) == 2
+
+        assert results[0].path == "/test_file"
+        assert results[0].rule == "test_rule_name"
+        assert results[0].tags == ["tag1", "tag2", "tag3"]
+        assert results[0].namespace == "default"
+        assert results[0].digest.md5 == "6f8db599de986fab7a21625b7916589c"
+        assert results[0].digest.sha1 == "661295c9cbf9d6b2f6428414504a8deed3020641"
+        assert results[0].digest.sha256 == "d5579c46dfcc7f18207013e65b44e4cb4e2c2298f4ac457ba8f82743f31e930b"

dissect_target-3.19.dev22/tests/tools/test_yara.py

@@ -0,0 +1,43 @@
+from io import BytesIO
+from unittest.mock import patch
+
+import pytest
+
+from dissect.target import Target
+from dissect.target.filesystem import VirtualFilesystem
+from dissect.target.tools.yara import HAS_YARA
+from dissect.target.tools.yara import main as target_yara
+from tests._utils import absolute_path
+
+
+@pytest.mark.skipif(not HAS_YARA, reason="requires python-yara")
+def test_yara(target_default: Target, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture) -> None:
+    vfs = VirtualFilesystem()
+    vfs.map_file_fh("test_file", BytesIO(b"hello there this is a test string!"))
+    vfs.map_file_fh("/test/dir/to/test_file", BytesIO(b"this is another test string for YARA testing."))
+    vfs.map_file_fh("should_not_hit", BytesIO(b"this is another file."))
+    target_default.fs.mount("/", vfs)
+
+    with patch("dissect.target.Target.open_all", return_value=[target_default]), monkeypatch.context() as m:
+        m.setattr(
+            "sys.argv",
+            [
+                "target-yara",
+                "example.img",
+                "--rules",
+                absolute_path("_data/plugins/filesystem/yara/rule.yar"),
+                "--path",
+                "/",
+                "--check",
+                "-s",
+            ],
+        )
+        target_yara()
+
+        out, _ = capsys.readouterr()
+
+        hit1 = "<filesystem/yara/match hostname=None domain=None path='/test_file' digest=(md5=d690ba32b59d28614aebefe9b03c74d4, sha1=4b1ced217aabe37138e96fb93bf40026639b9d3b, sha256=7a644118588ff0dcf2fadbe198ae1f1629c29374bac491ba41d5cf957edf0dfc) rule='test_rule_name' tags=['tag1', 'tag2', 'tag3']"  # noqa E501
+        hit2 = "<filesystem/yara/match hostname=None domain=None path='/test/dir/to/test_file' digest=(md5=bd7490dd2978ce983e2e1613ac8444c0, sha1=849a062cf09280f5c7dce4c7f87c69a1d9262e08, sha256=9bf7629a67c7ce8019910f1c1251fe44b61b3fff55a59a5e148af3c207dc102f) rule='test_rule_name' tags=['tag1', 'tag2', 'tag3']"  # noqa E501
+
+        assert hit1 in out
+        assert hit2 in out

dissect_target-3.19.dev21/dissect/target/plugins/filesystem/yara.py

@@ -1,63 +0,0 @@
-from pathlib import Path
-
-try:
-    import yara
-except ImportError:
-    raise ImportError("Please install 'yara-python' to use 'target-query -f yara'.")
-
-from dissect.target.exceptions import FileNotFoundError
-from dissect.target.helpers.record import TargetRecordDescriptor
-from dissect.target.plugin import Plugin, arg, export
-
-YaraMatchRecord = TargetRecordDescriptor(
-    "filesystem/yara/match",
-    [
-        ("path", "path"),
-        ("digest", "digest"),
-        ("string", "rule"),
-        ("string[]", "tags"),
-    ],
-)
-
-
-class YaraPlugin(Plugin):
-    """Plugin to scan files against a local YARA rules file."""
-
-    DEFAULT_MAX_SIZE = 10 * 1024 * 1024
-
-    def check_compatible(self) -> None:
-        pass
-
-    @arg("--rule-files", "-r", type=Path, nargs="+", required=True, help="path to YARA rule file")
-    @arg("--scan-path", default="/", help="path to recursively scan")
-    @arg("--max-size", "-m", default=DEFAULT_MAX_SIZE, help="maximum file size in bytes to scan")
-    @export(record=YaraMatchRecord)
-    def yara(self, rule_files, scan_path="/", max_size=DEFAULT_MAX_SIZE):
-        """Scan files up to a given maximum size with a local YARA rule file.
-
-        Example:
-            target-query <TARGET> -f yara --rule-file /path/to/yara_sigs.rule
-        """
-
-        rule_data = "\n".join([rule_file.read_text() for rule_file in rule_files])
-
-        rules = yara.compile(source=rule_data)
-        for _, _, files in self.target.fs.walk_ext(scan_path):
-            for file_entry in files:
-                path = self.target.fs.path(file_entry.path)
-                try:
-                    if path.stat().st_size > max_size:
-                        continue
-
-                    for match in rules.match(data=path.read_bytes()):
-                        yield YaraMatchRecord(
-                            path=path,
-                            digest=path.get().hash(),
-                            rule=match.rule,
-                            tags=match.tags,
-                            _target=self.target,
-                        )
-                except FileNotFoundError:
-                    continue
-                except Exception:
-                    self.target.log.exception("Error scanning file: %s", path)

dissect_target-3.19.dev21/tests/plugins/filesystem/test_yara.py

@@ -1,38 +0,0 @@
-import tempfile
-from io import BytesIO
-from pathlib import Path
-
-import pytest
-
-from dissect.target.filesystem import VirtualFilesystem
-
-yara = pytest.importorskip("dissect.target.plugins.filesystem.yara", reason="yara-python module unavailable")
-
-
-def test_yara_plugin(tmp_path, target_default):
-    test_rule = """
-    rule test_rule_name {
-        strings:
-            $ = "test string"
-
-        condition:
-            any of them
-    }
-    """
-    vfs = VirtualFilesystem()
-    vfs.map_file_fh("test_file", BytesIO(b"test string"))
-    vfs.map_file_fh("/test/dir/to/test_file", BytesIO(b"test string"))
-
-    target_default.fs.mount("/", vfs)
-
-    with tempfile.NamedTemporaryFile(mode="w+t", dir=tmp_path, delete=False) as tmp_file:
-        tmp_file.write(test_rule)
-        tmp_file.close()
-
-        target_default.add_plugin(yara.YaraPlugin)
-        results = list(target_default.yara(rule_files=[Path(tmp_file.name)]))
-
-    assert len(results) == 2
-    assert results[0].path == "/test_file"
-    assert results[1].path == "/test/dir/to/test_file"
-    assert results[0].rule == "test_rule_name"