dissect.target 3.19.dev21__py3-none-any.whl → 3.19.dev23__py3-none-any.whl

dissect/target/helpers/network_managers.py

@@ -7,12 +7,14 @@ from configparser import ConfigParser, MissingSectionHeaderError
 from io import StringIO
 from itertools import chain
 from re import compile, sub
-from typing import Any, Callable, Iterable, Match, Optional
+from typing import Any, Callable, Iterable, Iterator, Match, Optional
 
 from defusedxml import ElementTree
 
 from dissect.target.exceptions import PluginError
 from dissect.target.helpers.fsutil import TargetPath
+from dissect.target.plugins.os.unix.log.journal import JournalRecord
+from dissect.target.plugins.os.unix.log.messages import MessagesRecord
 from dissect.target.target import Target
 
 log = logging.getLogger(__name__)
@@ -509,14 +511,15 @@ class LinuxNetworkManager:
         return values
 
 
-def parse_unix_dhcp_log_messages(target) -> list[str]:
+def parse_unix_dhcp_log_messages(target: Target, iter_all: bool = False) -> set[str]:
     """Parse local syslog, journal and cloud init-log files for DHCP lease IPs.
 
     Args:
         target: Target to discover and obtain network information from.
+        iter_all: Parse limited amount of journal messages (first 10000) or all of them.
 
     Returns:
-        List of DHCP ip addresses.
+        A set of found DHCP IP addresses.
     """
     ips = set()
     messages = set()
@@ -530,9 +533,19 @@ def parse_unix_dhcp_log_messages(target) -> list[str]:
     if not messages:
         target.log.warning(f"Could not search for DHCP leases using {log_func}: No log entries found.")
 
-    for record in messages:
+    def records_enumerate(iterable: Iterable) -> Iterator[tuple[int, JournalRecord | MessagesRecord]]:
+        count = 0
+        for rec in iterable:
+            if rec._desc.name == "linux/log/journal":
+                count += 1
+            yield count, rec
+
+    for count, record in records_enumerate(messages):
         line = record.message
 
+        if not line:
+            continue
+
         # Ubuntu cloud-init
         if "Received dhcp lease on" in line:
             interface, ip, netmask = re.search(r"Received dhcp lease on (\w{0,}) for (\S+)\/(\S+)", line).groups()
@@ -576,9 +589,11 @@ def parse_unix_dhcp_log_messages(target) -> list[str]:
             ips.add(ip)
             continue
 
-        # Journals and syslogs can be large and slow to iterate,
-        # so we stop if we have some results and have reached the journal plugin.
-        if len(ips) >= 2 and record._desc.name == "linux/log/journal":
+        # The journal parser is relatively slow, so we stop when we have read 10000 journal entries,
+        # or if we have found at least one ip address. When `iter_all` is `True` we continue searching.
+        if not iter_all and (ips or count > 10_000):
+            if not ips:
+                target.log.warning("No DHCP IP addresses found in first 10000 journal entries.")
             break
 
     return ips

dissect/target/plugins/filesystem/yara.py

@@ -1,14 +1,27 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+from io import BytesIO
 from pathlib import Path
+from typing import Iterator
+
+from dissect.target.helpers import hashutil
 
 try:
     import yara
+
+    HAS_YARA = True
+
 except ImportError:
-    raise ImportError("Please install 'yara-python' to use 'target-query -f yara'.")
+    HAS_YARA = False
 
-from dissect.target.exceptions import FileNotFoundError
+from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
 from dissect.target.helpers.record import TargetRecordDescriptor
 from dissect.target.plugin import Plugin, arg, export
 
+log = logging.getLogger(__name__)
+
 YaraMatchRecord = TargetRecordDescriptor(
     "filesystem/yara/match",
     [
@@ -16,48 +29,158 @@ YaraMatchRecord = TargetRecordDescriptor(
         ("digest", "digest"),
         ("string", "rule"),
         ("string[]", "tags"),
+        ("string", "namespace"),
     ],
 )
 
+DEFAULT_MAX_SCAN_SIZE = 10 * 1024 * 1024
+
 
 class YaraPlugin(Plugin):
     """Plugin to scan files against a local YARA rules file."""
 
-    DEFAULT_MAX_SIZE = 10 * 1024 * 1024
-
     def check_compatible(self) -> None:
-        pass
+        if not HAS_YARA:
+            raise UnsupportedPluginError("Please install 'yara-python' to use the yara plugin.")
 
-    @arg("--rule-files", "-r", type=Path, nargs="+", required=True, help="path to YARA rule file")
-    @arg("--scan-path", default="/", help="path to recursively scan")
-    @arg("--max-size", "-m", default=DEFAULT_MAX_SIZE, help="maximum file size in bytes to scan")
+    @arg("-r", "--rules", required=True, nargs="*", help="path(s) to YARA rule file(s) or folder(s)")
+    @arg("-p", "--path", default="/", help="path on target(s) to recursively scan")
+    @arg("-m", "--max-size", default=DEFAULT_MAX_SCAN_SIZE, help="maximum file size in bytes to scan")
+    @arg("-c", "--check", default=False, action="store_true", help="check if every YARA rule is valid")
     @export(record=YaraMatchRecord)
-    def yara(self, rule_files, scan_path="/", max_size=DEFAULT_MAX_SIZE):
-        """Scan files up to a given maximum size with a local YARA rule file.
+    def yara(
+        self,
+        rules: list[str | Path],
+        path: str = "/",
+        max_size: int = DEFAULT_MAX_SCAN_SIZE,
+        check: bool = False,
+    ) -> Iterator[YaraMatchRecord]:
+        """Scan files inside the target up to a given maximum size with YARA rule file(s).
+
+        Args:
+            rules: ``list`` of strings or ``Path`` objects pointing to rule files to use.
+            path: ``string`` of absolute target path to scan.
+            max_size: Files larger than this size will not be scanned.
+            check: Check if provided rules are valid, only compiles valid rules.
 
-        Example:
-            target-query <TARGET> -f yara --rule-file /path/to/yara_sigs.rule
+        Returns:
+            Iterator yields ``YaraMatchRecord``.
         """
 
-        rule_data = "\n".join([rule_file.read_text() for rule_file in rule_files])
+        compiled_rules = process_rules(rules, check)
 
-        rules = yara.compile(source=rule_data)
-        for _, _, files in self.target.fs.walk_ext(scan_path):
-            for file_entry in files:
-                path = self.target.fs.path(file_entry.path)
+        if not rules:
+            self.target.log.error("No working rules found in '%s'", ",".join(rules))
+            return
+
+        if hasattr(compiled_rules, "warnings") and (num_warns := len(compiled_rules.warnings)) > 0:
+            self.target.log.warning("YARA generated %s warnings while compiling rules", num_warns)
+            for warning in compiled_rules.warnings:
+                self.target.log.debug(warning)
+
+        self.target.log.warning("Will not scan files larger than %s MB", max_size // 1024 // 1024)
+
+        for _, _, files in self.target.fs.walk_ext(path):
+            for file in files:
                 try:
-                    if path.stat().st_size > max_size:
+                    if file_size := file.stat().st_size > max_size:
+                        self.target.log.debug(
+                            "Skipping file '%s' as it is larger than %s bytes (size is %s)", file, file_size, max_size
+                        )
                         continue
 
-                    for match in rules.match(data=path.read_bytes()):
+                    buf = file.open().read()
+                    for match in compiled_rules.match(data=buf):
                         yield YaraMatchRecord(
-                            path=path,
-                            digest=path.get().hash(),
+                            path=self.target.fs.path(file.path),
+                            digest=hashutil.common(BytesIO(buf)),
                             rule=match.rule,
                             tags=match.tags,
+                            namespace=match.namespace,
                             _target=self.target,
                         )
+
                 except FileNotFoundError:
                     continue
-                except Exception:
-                    self.target.log.exception("Error scanning file: %s", path)
+                except RuntimeWarning as e:
+                    self.target.log.warning("Runtime warning while scanning file '%s': %s", file, e)
+                except Exception as e:
+                    self.target.log.error("Exception scanning file '%s'", file)
+                    self.target.log.debug("", exc_info=e)
+
+
+def process_rules(paths: list[str | Path], check: bool = False) -> yara.Rules | None:
+    """Generate compiled YARA rules from the given path(s).
+
+    Provide path to one (compiled) YARA file or directory containing YARA files.
+
+    Args:
+        paths: Path to file(s) or folder(s) containing YARA files.
+        check: Attempt to compile every rule file before appending to rules.
+
+    Returns:
+        Compiled YARA rules or None.
+    """
+    files = set()
+    compiled_rules = None
+
+    for rules_path in paths:
+        if isinstance(rules_path, str):
+            rules_path = Path(rules_path)
+
+        if not rules_path.exists():
+            log.warning("File %s does not exist!", rules_path)
+            continue
+
+        if rules_path.is_dir():
+            for file in rules_path.rglob("*"):
+                if not file.is_file():
+                    continue
+                files.add(file)
+        else:
+            files.add(rules_path)
+
+    for file in set(files):
+        with file.open("rb") as fh:
+            magic = fh.read(4)
+
+        if magic == b"YARA":
+            if len(files) > 1:
+                log.error("Providing multiple compiled YARA files is not supported. Did not add %s", file)
+                continue
+            else:
+                log.info("Adding single compiled YARA file %s", file)
+                compiled_rules = compile_yara(file, is_compiled=True)
+                break
+
+        elif check and not is_valid_yara({"check_namespace": file}):
+            log.warning("File %s contains invalid rule(s)!", file)
+            files.remove(file)
+            continue
+
+    if files and not compiled_rules:
+        try:
+            compiled_rules = compile_yara({hashlib.md5(file.as_posix().encode()).hexdigest(): file for file in files})
+        except yara.Error as e:
+            log.error("Failed to compile YARA file(s): %s", e)
+
+    return compiled_rules
+
+
+def compile_yara(files: dict[str, Path] | Path, is_compiled: bool = False) -> yara.Rules | None:
+    """Compile or load the given YARA file(s) to rules."""
+    if is_compiled and isinstance(files, Path):
+        return yara.load(files.as_posix())
+    else:
+        return yara.compile(filepaths={ns: Path(path).as_posix() for ns, path in files.items()})
+
+
+def is_valid_yara(files: dict[str, Path] | Path, is_compiled: bool = False) -> bool:
+    """Determine if the given YARA file(s) compile without errors or warnings."""
+    try:
+        compile_yara(files, is_compiled)
+        return True
+
+    except (yara.SyntaxError, yara.WarningError, yara.Error) as e:
+        log.debug("Rule file(s) '%s' invalid: %s", files, e)
+        return False

dissect/target/plugins/os/unix/linux/_os.py

@@ -41,7 +41,7 @@ class LinuxPlugin(UnixPlugin, LinuxNetworkManager):
             for ip in ip_set:
                 ips.append(ip)
 
-        for ip in parse_unix_dhcp_log_messages(self.target):
+        for ip in parse_unix_dhcp_log_messages(self.target, iter_all=False):
             if ip not in ips:
                 ips.append(ip)
 

dissect/target/tools/yara.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import argparse
+import logging
+
+from dissect.target import Target
+from dissect.target.exceptions import TargetError
+from dissect.target.plugins.filesystem.yara import HAS_YARA, YaraPlugin
+from dissect.target.tools.query import record_output
+from dissect.target.tools.utils import (
+    catch_sigpipe,
+    configure_generic_arguments,
+    process_generic_arguments,
+)
+
+log = logging.getLogger(__name__)
+
+
+@catch_sigpipe
+def main():
+    help_formatter = argparse.ArgumentDefaultsHelpFormatter
+    parser = argparse.ArgumentParser(
+        description="target-yara",
+        fromfile_prefix_chars="@",
+        formatter_class=help_formatter,
+    )
+
+    parser.add_argument("targets", metavar="TARGETS", nargs="*", help="Targets to load")
+    parser.add_argument("-s", "--strings", default=False, action="store_true", help="print output as string")
+
+    for args, kwargs in getattr(YaraPlugin.yara, "__args__", []):
+        parser.add_argument(*args, **kwargs)
+
+    configure_generic_arguments(parser)
+
+    args = parser.parse_args()
+    process_generic_arguments(args)
+
+    if not HAS_YARA:
+        log.error("yara-python is not installed: pip install yara-python")
+        parser.exit(1)
+
+    if not args.targets:
+        log.error("No targets provided")
+        parser.exit(1)
+
+    try:
+        for target in Target.open_all(args.targets):
+            target.log.info("Scanning target")
+            rs = record_output(args.strings, False)
+            for record in target.yara(args.rules, args.path, args.max_size, args.check):
+                rs.write(record)
+
+    except TargetError as e:
+        log.error(e)
+        log.debug("", exc_info=e)
+        parser.exit(1)
+
+
+if __name__ == "__main__":
+    main()

{dissect.target-3.19.dev21.dist-info → dissect.target-3.19.dev23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dissect.target
-Version: 3.19.dev21
+Version: 3.19.dev23
 Summary: This module ties all other Dissect modules together, it provides a programming API and command line tools which allow easy access to various data sources inside disk images or file collections (a.k.a. targets)
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{dissect.target-3.19.dev21.dist-info → dissect.target-3.19.dev23.dist-info}/RECORD

@@ -58,7 +58,7 @@ dissect/target/helpers/loaderutil.py,sha256=kiyMWra_gVxfNSGwLlgxLcuuqAYuCMDc5NiC
 dissect/target/helpers/localeutil.py,sha256=Y4Fh4jDSGfm5356xSLMriUCN8SZP_FAHg_iodkAxNq4,1504
 dissect/target/helpers/mount.py,sha256=JxhUYyEbDnHfzPpfuWy4nV9OwCJPoDSGdHHNiyvd_l0,3949
 dissect/target/helpers/mui.py,sha256=i-7XoHbu4WO2fYapK9yGAMW04rFlgRispknc1KQIS5Q,22258
-dissect/target/helpers/network_managers.py,sha256=uRh_P8ICbKke2N7eFJ6AS2-I5DmIRiaQUlxR7oqxPaU,24975
+dissect/target/helpers/network_managers.py,sha256=ByBSe2K3c8hgQC6dokcf-hHdmPcD8PmrOj0xs1C3yhs,25743
 dissect/target/helpers/polypath.py,sha256=h8p7m_OCNiQljGwoZh5Aflr9H2ot6CZr6WKq1OSw58o,2175
 dissect/target/helpers/protobuf.py,sha256=b4DsnqrRLrefcDjx7rQno-_LBcwtJXxuKf5RdOegzfE,1537
 dissect/target/helpers/record.py,sha256=lWl7k2Mp9Axllm0tXzPGJx2zj2zONsyY_p5g424T0Lc,4826
@@ -164,7 +164,7 @@ dissect/target/plugins/filesystem/acquire_hash.py,sha256=OVxI19-Bl1tdqCiFMscFMLm
 dissect/target/plugins/filesystem/icat.py,sha256=bOMi04IlljnKwxTWTZJKtK7RxKnabFu3WcXyUwzkE-4,4090
 dissect/target/plugins/filesystem/resolver.py,sha256=HfyASUFV4F9uD-yFXilFpPTORAsRDvdmTvuYHgOaOWg,4776
 dissect/target/plugins/filesystem/walkfs.py,sha256=e8HEZcV5Wiua26FGWL3xgiQ_PIhcNvGI5KCdsAx2Nmo,2298
-dissect/target/plugins/filesystem/yara.py,sha256=q_pbrQArNaWP4ILRzK7VQhukIw16LhUvntoviHmZ38Q,2241
+dissect/target/plugins/filesystem/yara.py,sha256=JdWqbqDBhKrht3fTroqX7NpBU9khEQUWyMcDgLv2l2g,6686
 dissect/target/plugins/filesystem/ntfs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dissect/target/plugins/filesystem/ntfs/mft.py,sha256=AD3w2FIjDAf8x2KEbBhz2NeOA_lxIAmw353w6J3ObYU,9565
 dissect/target/plugins/filesystem/ntfs/mft_timeline.py,sha256=vvNFAZbr7s3X2OTYf4ES_L6-XsouTXcTymfxnHfZ1Rw,6791
@@ -212,7 +212,7 @@ dissect/target/plugins/os/unix/esxi/_os.py,sha256=JOJ6j57vFCojgBNkju-7MG2nQqwl4Q
 dissect/target/plugins/os/unix/etc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dissect/target/plugins/os/unix/etc/etc.py,sha256=WNCtO7NWOKRDBiV-XjXqgPuGRDE_Zyw6XWz3kTm__QE,2493
 dissect/target/plugins/os/unix/linux/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dissect/target/plugins/os/unix/linux/_os.py,sha256=YJYwuq_iAinOrPqTE49Q4DLYMWBeRCly1uTbDvPhp6Q,2796
+dissect/target/plugins/os/unix/linux/_os.py,sha256=n6VkfGYIdZUxcK2C1aPDUY_ZZQEIl0GkrpvIKeguv5o,2812
 dissect/target/plugins/os/unix/linux/cmdline.py,sha256=AyMfndt3UsmJtoOyZYC8nWq2GZg9oPvn8SiI3M4NxnE,1622
 dissect/target/plugins/os/unix/linux/environ.py,sha256=UOQD7Xmu754u2oAh3L5g5snuz-gv4jbWbVy46qszYjo,1881
 dissect/target/plugins/os/unix/linux/iptables.py,sha256=qTzY5PHHXA33WnPYb5NESgoSwI7ECZ8YPoEe_Fmln-8,6045
@@ -333,6 +333,7 @@ dissect/target/tools/query.py,sha256=ONHu2FVomLccikb84qBrlhNmEfRoHYFQMcahk_y2c9A
 dissect/target/tools/reg.py,sha256=FDsiBBDxjWVUBTRj8xn82vZe-J_d9piM-TKS3PHZCcM,3193
 dissect/target/tools/shell.py,sha256=_widEuIRqZhYzcFR52NYI8O2aPFm6tG5Uiv-AIrC32U,45155
 dissect/target/tools/utils.py,sha256=sQizexY3ui5vmWw4KOBLg5ecK3TPFjD-uxDqRn56ZTY,11304
+dissect/target/tools/yara.py,sha256=xIom_n78oBiDg6VEBMVk8qmvhYMOPzY5yv9Vl1rDbB4,1754
 dissect/target/tools/dump/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dissect/target/tools/dump/run.py,sha256=aD84peRS4zHqC78fH7Vd4ni3m1ZmVP70LyMwBRvoDGY,9463
 dissect/target/tools/dump/state.py,sha256=YYgCff0kZZ-tx27lJlc9LQ7AfoGnLK5Gyi796OnktA8,9205
@@ -345,10 +346,10 @@ dissect/target/volumes/luks.py,sha256=OmCMsw6rCUXG1_plnLVLTpsvE1n_6WtoRUGQbpmu1z
 dissect/target/volumes/lvm.py,sha256=wwQVR9I3G9YzmY6UxFsH2Y4MXGBcKL9aayWGCDTiWMU,2269
 dissect/target/volumes/md.py,sha256=j1K1iKmspl0C_OJFc7-Q1BMWN2OCC5EVANIgVlJ_fIE,1673
 dissect/target/volumes/vmfs.py,sha256=-LoUbn9WNwTtLi_4K34uV_-wDw2W5hgaqxZNj4UmqAQ,1730
-dissect.target-3.19.dev21.dist-info/COPYRIGHT,sha256=m-9ih2RVhMiXHI2bf_oNSSgHgkeIvaYRVfKTwFbnJPA,301
-dissect.target-3.19.dev21.dist-info/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
-dissect.target-3.19.dev21.dist-info/METADATA,sha256=EnMvW_6eQ-p-R4fxSi-ZjcQ_MvkH6xnkcW6F0YE7oMo,12719
-dissect.target-3.19.dev21.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-dissect.target-3.19.dev21.dist-info/entry_points.txt,sha256=tvFPa-Ap-gakjaPwRc6Fl6mxHzxEZ_arAVU-IUYeo_s,447
-dissect.target-3.19.dev21.dist-info/top_level.txt,sha256=Mn-CQzEYsAbkxrUI0TnplHuXnGVKzxpDw_po_sXpvv4,8
-dissect.target-3.19.dev21.dist-info/RECORD,,
+dissect.target-3.19.dev23.dist-info/COPYRIGHT,sha256=m-9ih2RVhMiXHI2bf_oNSSgHgkeIvaYRVfKTwFbnJPA,301
+dissect.target-3.19.dev23.dist-info/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+dissect.target-3.19.dev23.dist-info/METADATA,sha256=c2q_-Jvv4xRoYAE0jWyTGNV-oOKAD73yjU9Rk5-mavY,12719
+dissect.target-3.19.dev23.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+dissect.target-3.19.dev23.dist-info/entry_points.txt,sha256=BWuxAb_6AvUAQpIQOQU0IMTlaF6TDht2AIZK8bHd-zE,492
+dissect.target-3.19.dev23.dist-info/top_level.txt,sha256=Mn-CQzEYsAbkxrUI0TnplHuXnGVKzxpDw_po_sXpvv4,8
+dissect.target-3.19.dev23.dist-info/RECORD,,

{dissect.target-3.19.dev21.dist-info → dissect.target-3.19.dev23.dist-info}/entry_points.txt

@@ -8,3 +8,4 @@ target-mount = dissect.target.tools.mount:main
 target-query = dissect.target.tools.query:main
 target-reg = dissect.target.tools.reg:main
 target-shell = dissect.target.tools.shell:main
+target-yara = dissect.target.tools.yara:main