dissect.target 3.17.dev26__py3-none-any.whl → 3.17.dev28__py3-none-any.whl

dissect/target/helpers/network_managers.py

@@ -5,6 +5,7 @@ import re
 from collections import defaultdict
 from configparser import ConfigParser, MissingSectionHeaderError
 from io import StringIO
+from itertools import chain
 from re import compile, sub
 from typing import Any, Callable, Iterable, Match, Optional
 
@@ -299,7 +300,8 @@ class Parser:
             return
 
         if section:
-            config = config.get(section, {})
+            # account for values of sections which are None
+            config = config.get(section, {}) or {}
 
         for key, value in config.items():
             if key == option:
@@ -508,7 +510,7 @@ class LinuxNetworkManager:
 
 
 def parse_unix_dhcp_log_messages(target) -> list[str]:
-    """Parse local syslog and cloud init log files for DHCP lease IPs.
+    """Parse local syslog, journal and cloud init-log files for DHCP lease IPs.
 
     Args:
         target: Target to discover and obtain network information from.
@@ -516,53 +518,68 @@ def parse_unix_dhcp_log_messages(target) -> list[str]:
     Returns:
         List of DHCP ip addresses.
     """
-    ips = []
-
-    # Search through parsed syslogs for DHCP leases.
-    try:
-        messages = target.messages()
-        for record in messages:
-            line = record.message
-
-            # Ubuntu DHCP
-            if ("DHCPv4" in line or "DHCPv6" in line) and " address " in line and " via " in line:
-                ip = line.split(" address ")[1].split(" via ")[0].strip().split("/")[0]
-                if ip not in ips:
-                    ips.append(ip)
-
-            # Ubuntu DHCP NetworkManager
-            elif "option ip_address" in line and ("dhcp4" in line or "dhcp6" in line) and "=> '" in line:
-                ip = line.split("=> '")[1].replace("'", "").strip()
-                if ip not in ips:
-                    ips.append(ip)
-
-            # Debian and CentOS dhclient
-            elif record.daemon == "dhclient" and "bound to" in line:
-                ip = line.split("bound to")[1].split(" ")[1].strip()
-                if ip not in ips:
-                    ips.append(ip)
-
-            # CentOS DHCP and general NetworkManager
-            elif " address " in line and ("dhcp4" in line or "dhcp6" in line):
-                ip = line.split(" address ")[1].strip()
-                if ip not in ips:
-                    ips.append(ip)
-
-    except PluginError:
-        target.log.debug("Can not search for DHCP leases in syslog files as they does not exist.")
-
-    # A unix system might be provisioned using Ubuntu's cloud-init (https://cloud-init.io/).
-    if (path := target.fs.path("/var/log/cloud-init.log")).exists():
-        for line in path.open("rt"):
-            # We are interested in the following log line:
-            # YYYY-MM-DD HH:MM:SS,000 - dhcp.py[DEBUG]: Received dhcp lease on IFACE for IP/MASK
-            if "Received dhcp lease on" in line:
-                interface, ip, netmask = re.search(r"Received dhcp lease on (\w{0,}) for (\S+)\/(\S+)", line).groups()
-                if ip not in ips:
-                    ips.append(ip)
-
-    if not path and not messages:
-        target.log.warning("Can not search for DHCP leases in syslog or cloud-init.log files as they does not exist.")
+    ips = set()
+    messages = set()
+
+    for log_func in ["messages", "journal"]:
+        try:
+            messages = chain(messages, getattr(target, log_func)())
+        except PluginError:
+            target.log.debug(f"Could not search for DHCP leases in {log_func} files.")
+
+    if not messages:
+        target.log.warning(f"Could not search for DHCP leases using {log_func}: No log entries found.")
+
+    for record in messages:
+        line = record.message
+
+        # Ubuntu cloud-init
+        if "Received dhcp lease on" in line:
+            interface, ip, netmask = re.search(r"Received dhcp lease on (\w{0,}) for (\S+)\/(\S+)", line).groups()
+            ips.add(ip)
+            continue
+
+        # Ubuntu DHCP
+        if ("DHCPv4" in line or "DHCPv6" in line) and " address " in line and " via " in line:
+            ip = line.split(" address ")[1].split(" via ")[0].strip().split("/")[0]
+            ips.add(ip)
+            continue
+
+        # Ubuntu DHCP NetworkManager
+        if "option ip_address" in line and ("dhcp4" in line or "dhcp6" in line) and "=> '" in line:
+            ip = line.split("=> '")[1].replace("'", "").strip()
+            ips.add(ip)
+            continue
+
+        # Debian and CentOS dhclient
+        if hasattr(record, "daemon") and record.daemon == "dhclient" and "bound to" in line:
+            ip = line.split("bound to")[1].split(" ")[1].strip()
+            ips.add(ip)
+            continue
+
+        # CentOS DHCP and general NetworkManager
+        if " address " in line and ("dhcp4" in line or "dhcp6" in line):
+            ip = line.split(" address ")[1].strip()
+            ips.add(ip)
+            continue
+
+        # Ubuntu/Debian DHCP networkd (Journal)
+        if (
+            hasattr(record, "code_func")
+            and record.code_func == "dhcp_lease_acquired"
+            and " address " in line
+            and " via " in line
+        ):
+            interface, ip, netmask, gateway = re.search(
+                r"^(\S+): DHCPv[4|6] address (\S+)\/(\S+) via (\S+)", line
+            ).groups()
+            ips.add(ip)
+            continue
+
+        # Journals and syslogs can be large and slow to iterate,
+        # so we stop if we have some results and have reached the journal plugin.
+        if len(ips) >= 2 and record._desc.name == "linux/log/journal":
+            break
 
     return ips
 

dissect/target/loaders/mqtt.py

@@ -65,6 +65,9 @@ class MQTTStream(AlignedStream):
 class MQTTConnection:
     broker = None
     host = None
+    prev = -1
+    factor = 1
+    prefetch_factor_inc = 10
 
     def __init__(self, broker: Broker, host: str):
         self.broker = broker
@@ -95,20 +98,32 @@ class MQTTConnection:
 
     def read(self, disk_id: int, offset: int, length: int, optimization_strategy: int) -> bytes:
         message = None
-        self.broker.seek(self.host, disk_id, offset, length, optimization_strategy)
 
+        message = self.broker.read(self.host, disk_id, offset, length)
+        if message:
+            return message.data
+
+        if self.prev == offset - (length * self.factor):
+            if self.factor < 500:
+                self.factor += self.prefetch_factor_inc
+        else:
+            self.factor = 1
+
+        self.prev = offset
+        flength = length * self.factor
+        self.broker.factor = self.factor
+        self.broker.seek(self.host, disk_id, offset, flength, optimization_strategy)
         attempts = 0
         while True:
-            message = self.broker.read(self.host, disk_id, offset, length)
-            # don't waste time with sleep if we have a response
-            if message:
+            if message := self.broker.read(self.host, disk_id, offset, length):
+                # don't waste time with sleep if we have a response
                 break
 
             attempts += 1
-            time.sleep(0.01)
-            if attempts > 100:
+            time.sleep(0.1)
+            if attempts > 300:
                 # message might have not reached agent, resend...
-                self.broker.seek(self.host, disk_id, offset, length, optimization_strategy)
+                self.broker.seek(self.host, disk_id, offset, flength, optimization_strategy)
                 attempts = 0
 
         return message.data
@@ -127,6 +142,7 @@ class Broker:
     diskinfo = {}
     index = {}
     topo = {}
+    factor = 1
 
     def __init__(self, broker: Broker, port: str, key: str, crt: str, ca: str, case: str, **kwargs):
         self.broker_host = broker
@@ -137,10 +153,13 @@ class Broker:
         self.case = case
         self.command = kwargs.get("command", None)
 
+    def clear_cache(self) -> None:
+        self.index = {}
+
     @suppress
     def read(self, host: str, disk_id: int, seek_address: int, read_length: int) -> SeekMessage:
         key = f"{host}-{disk_id}-{seek_address}-{read_length}"
-        return self.index.pop(key)
+        return self.index.get(key)
 
     @suppress
     def disk(self, host: str) -> DiskMessage:
@@ -165,14 +184,15 @@ class Broker:
         disk_id = tokens[3]
         seek_address = int(tokens[4], 16)
         read_length = int(tokens[5], 16)
-        msg = SeekMessage(data=payload)
 
-        key = f"{hostname}-{disk_id}-{seek_address}-{read_length}"
+        for i in range(self.factor):
+            sublength = int(read_length / self.factor)
+            start = i * sublength
+            key = f"{hostname}-{disk_id}-{seek_address+start}-{sublength}"
+            if key in self.index:
+                continue
 
-        if key in self.index:
-            return
-
-        self.index[key] = msg
+            self.index[key] = SeekMessage(data=payload[start : start + sublength])
 
     def _on_id(self, hostname: str, payload: bytes) -> None:
         key = hostname
@@ -204,9 +224,14 @@ class Broker:
         elif response == "ID":
             self._on_id(hostname, msg.payload)
 
-    def seek(self, host: str, disk_id: int, offset: int, length: int, optimization_strategy: int) -> None:
+    def seek(self, host: str, disk_id: int, offset: int, flength: int, optimization_strategy: int) -> None:
+        length = int(flength / self.factor)
+        key = f"{host}-{disk_id}-{offset}-{length}"
+        if key in self.index:
+            return
+
         self.mqtt_client.publish(
-            f"{self.case}/{host}/SEEK/{disk_id}/{hex(offset)}/{hex(length)}", pack("<I", optimization_strategy)
+            f"{self.case}/{host}/SEEK/{disk_id}/{hex(offset)}/{hex(flength)}", pack("<I", optimization_strategy)
         )
 
     def info(self, host: str) -> None:

dissect/target/plugins/os/unix/log/messages.py

@@ -1,7 +1,8 @@
 import re
-from itertools import chain
+from pathlib import Path
 from typing import Iterator
 
+from dissect.target import Target
 from dissect.target.exceptions import UnsupportedPluginError
 from dissect.target.helpers.record import TargetRecordDescriptor
 from dissect.target.helpers.utils import year_rollover_helper
@@ -23,17 +24,28 @@ RE_TS = re.compile(r"(\w+\s{1,2}\d+\s\d{2}:\d{2}:\d{2})")
 RE_DAEMON = re.compile(r"^[^:]+:\d+:\d+[^\[\]:]+\s([^\[:]+)[\[|:]{1}")
 RE_PID = re.compile(r"\w\[(\d+)\]")
 RE_MSG = re.compile(r"[^:]+:\d+:\d+[^:]+:\s(.*)$")
+RE_CLOUD_INIT_LINE = re.compile(r"(?P<ts>.*) - (?P<daemon>.*)\[(?P<log_level>\w+)\]\: (?P<message>.*)$")
 
 
 class MessagesPlugin(Plugin):
+    def __init__(self, target: Target):
+        super().__init__(target)
+        self.log_files = set(self._find_log_files())
+
+    def _find_log_files(self) -> Iterator[Path]:
+        log_dirs = ["/var/log/", "/var/log/installer/"]
+        file_globs = ["syslog*", "messages*", "cloud-init.log*"]
+        for log_dir in log_dirs:
+            for glob in file_globs:
+                yield from self.target.fs.path(log_dir).glob(glob)
+
     def check_compatible(self) -> None:
-        var_log = self.target.fs.path("/var/log")
-        if not any(var_log.glob("syslog*")) and not any(var_log.glob("messages*")):
-            raise UnsupportedPluginError("No message files found")
+        if not self.log_files:
+            raise UnsupportedPluginError("No log files found")
 
     @export(record=MessagesRecord)
     def syslog(self) -> Iterator[MessagesRecord]:
-        """Return contents of /var/log/messages* and /var/log/syslog*.
+        """Return contents of /var/log/messages*, /var/log/syslog* and cloud-init logs.
 
         See ``messages`` for more information.
         """
@@ -41,7 +53,7 @@ class MessagesPlugin(Plugin):
 
     @export(record=MessagesRecord)
     def messages(self) -> Iterator[MessagesRecord]:
-        """Return contents of /var/log/messages* and /var/log/syslog*.
+        """Return contents of /var/log/messages*, /var/log/syslog* and cloud-init logs.
 
         Note: due to year rollover detection, the contents of the files are returned in reverse.
 
@@ -52,12 +64,16 @@ class MessagesPlugin(Plugin):
         References:
             - https://geek-university.com/linux/var-log-messages-file/
             - https://www.geeksforgeeks.org/file-timestamps-mtime-ctime-and-atime-in-linux/
+            - https://cloudinit.readthedocs.io/en/latest/development/logging.html#logging-command-output
         """
 
         tzinfo = self.target.datetime.tzinfo
 
-        var_log = self.target.fs.path("/var/log")
-        for log_file in chain(var_log.glob("syslog*"), var_log.glob("messages*")):
+        for log_file in self.log_files:
+            if "cloud-init" in log_file.name:
+                yield from self._parse_cloud_init_log(log_file)
+                continue
+
             for ts, line in year_rollover_helper(log_file, RE_TS, DEFAULT_TS_LOG_FORMAT, tzinfo):
                 daemon = dict(enumerate(RE_DAEMON.findall(line))).get(0)
                 pid = dict(enumerate(RE_PID.findall(line))).get(0)
@@ -71,3 +87,32 @@ class MessagesPlugin(Plugin):
                     source=log_file,
                     _target=self.target,
                 )
+
+    def _parse_cloud_init_log(self, log_file: Path) -> Iterator[MessagesRecord]:
+        """Parse a cloud-init.log file.
+
+        Lines are structured in the following format:
+        ``YYYY-MM-DD HH:MM:SS,000 - dhcp.py[DEBUG]: Received dhcp lease on IFACE for IP/MASK``
+
+        NOTE: ``cloud-init-output.log`` files are not supported as they do not contain structured logs.
+
+        Args:
+            ``log_file``: path to cloud-init.log file.
+
+        Returns: ``MessagesRecord``
+        """
+        for line in log_file.open("rt").readlines():
+            if line := line.strip():
+                if match := RE_CLOUD_INIT_LINE.match(line):
+                    match = match.groupdict()
+                    yield MessagesRecord(
+                        ts=match["ts"].split(",")[0],
+                        daemon=match["daemon"],
+                        pid=None,
+                        message=match["message"],
+                        source=log_file,
+                        _target=self.target,
+                    )
+                else:
+                    self.target.log.warning("Could not match cloud-init log line")
+                    self.target.log.debug("No match for line '%s'", line)

dissect/target/tools/query.py

@@ -173,8 +173,7 @@ def main():
         collected_plugins = {}
 
         if targets:
-            for target in targets:
-                plugin_target = Target.open(target)
+            for plugin_target in Target.open_all(targets, args.children):
                 if isinstance(plugin_target._loader, ProxyLoader):
                     parser.error("can't list compatible plugins for remote targets.")
                 funcs, _ = find_plugin_functions(plugin_target, args.list, compatibility=True, show_hidden=True)

{dissect.target-3.17.dev26.dist-info → dissect.target-3.17.dev28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dissect.target
-Version: 3.17.dev26
+Version: 3.17.dev28
 Summary: This module ties all other Dissect modules together, it provides a programming API and command line tools which allow easy access to various data sources inside disk images or file collections (a.k.a. targets)
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{dissect.target-3.17.dev26.dist-info → dissect.target-3.17.dev28.dist-info}/RECORD

@@ -57,7 +57,7 @@ dissect/target/helpers/loaderutil.py,sha256=kiyMWra_gVxfNSGwLlgxLcuuqAYuCMDc5NiC
 dissect/target/helpers/localeutil.py,sha256=Y4Fh4jDSGfm5356xSLMriUCN8SZP_FAHg_iodkAxNq4,1504
 dissect/target/helpers/mount.py,sha256=JxhUYyEbDnHfzPpfuWy4nV9OwCJPoDSGdHHNiyvd_l0,3949
 dissect/target/helpers/mui.py,sha256=i-7XoHbu4WO2fYapK9yGAMW04rFlgRispknc1KQIS5Q,22258
-dissect/target/helpers/network_managers.py,sha256=OgrYhbBqM6K5OfUnCdTLG0RBrR-DcpR1CPezbNddK7k,24667
+dissect/target/helpers/network_managers.py,sha256=uRh_P8ICbKke2N7eFJ6AS2-I5DmIRiaQUlxR7oqxPaU,24975
 dissect/target/helpers/polypath.py,sha256=h8p7m_OCNiQljGwoZh5Aflr9H2ot6CZr6WKq1OSw58o,2175
 dissect/target/helpers/protobuf.py,sha256=NwKrZD4q9v7J8GnZX9gbzMUMV5pR78eAV17jgWOz_EY,1730
 dissect/target/helpers/record.py,sha256=lWl7k2Mp9Axllm0tXzPGJx2zj2zONsyY_p5g424T0Lc,4826
@@ -85,7 +85,7 @@ dissect/target/loaders/itunes.py,sha256=69aMTQiiGYpmD_EYSmf9mO1re8C3jAZIEStmwlMx
 dissect/target/loaders/kape.py,sha256=t5TfrGLqPeIpUUpXzIl6aHsqXMEGDqJ5YwDCs07DiBA,1237
 dissect/target/loaders/local.py,sha256=Ul-LCd_fY7SyWOVR6nH-NqbkuNpxoZVmffwrkvQElU8,16453
 dissect/target/loaders/log.py,sha256=cCkDIRS4aPlX3U-n_jUKaI2FPSV3BDpfqKceaU7rBbo,1507
-dissect/target/loaders/mqtt.py,sha256=b0VrQ75_tmc4POkcfnUwKJoj1qmcjm1OKsVBQ9MjgqI,9552
+dissect/target/loaders/mqtt.py,sha256=D8AmdOz2atD92z8bhjVFi3tC1H7pYmP4UrOCtMgfwMY,10396
 dissect/target/loaders/multiraw.py,sha256=4a3ZST0NwjnfPDxHkcEfAcX2ddUlT_C-rcrMHNg1wp4,1046
 dissect/target/loaders/ova.py,sha256=6h4O-7i87J394C6KgLsPkdXRAKNwtPubzLNS3vBGs7U,744
 dissect/target/loaders/ovf.py,sha256=ELMq6J2y6cPKbp7pjWAqMMnFYefWxXNqzIiAQdvGGXQ,1061
@@ -247,7 +247,7 @@ dissect/target/plugins/os/unix/log/audit.py,sha256=OjorWTmCFvCI5RJq6m6WNW0Lhb-po
 dissect/target/plugins/os/unix/log/auth.py,sha256=l7gCuRdvv9gL0U1N0yrR9hVsMnr4t_k4t-n-f6PrOxg,2388
 dissect/target/plugins/os/unix/log/journal.py,sha256=eiNNVLmKWFj4dTQX8PNRNgKpVwzQWEHEsKyYfGUAPXQ,17376
 dissect/target/plugins/os/unix/log/lastlog.py,sha256=eL_dbB1sPoy0tyavIjT457ZLVfXcCr17GiwDrMEEh8A,2458
-dissect/target/plugins/os/unix/log/messages.py,sha256=W3CeI0tchdRql9SKLFDxk9AKwUvqIrnpCujcERvDt90,2846
+dissect/target/plugins/os/unix/log/messages.py,sha256=CXA-SkMPLaCgnTQg9nzII-7tO8Il_ENQmuYvDxo33rI,4698
 dissect/target/plugins/os/unix/log/utmp.py,sha256=21tvzG977LqzRShV6uAoU-83WDcLUrI_Tv__2ZVi9rw,7756
 dissect/target/plugins/os/windows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dissect/target/plugins/os/windows/_os.py,sha256=EA9B9Rgb1C9NMvlX3gXhTRFXYaI6zrrKRg0OYq4v1ts,12589
@@ -320,7 +320,7 @@ dissect/target/tools/fs.py,sha256=cizCrW8rqdpT1irA8g6mslkaXX7CynWVQ7fvRUrcxNU,37
 dissect/target/tools/info.py,sha256=3smHr8I71yj3kCjsQ5nXkOHI9T_N8UwvkVa1CNOxB-s,5461
 dissect/target/tools/logging.py,sha256=5ZnumtMWLyslxfrUGZ4ntRyf3obOOhmn8SBjKfdLcEg,4174
 dissect/target/tools/mount.py,sha256=L_0tSmiBdW4aSaF0vXjB0bAkTC0kmT2N1hrbW6s5Jow,3254
-dissect/target/tools/query.py,sha256=6zz9SXS6YnHj7eguORS8Je7N4iM0i1PZDIQ-gyJ1nPY,15593
+dissect/target/tools/query.py,sha256=ONHu2FVomLccikb84qBrlhNmEfRoHYFQMcahk_y2c9A,15580
 dissect/target/tools/reg.py,sha256=FDsiBBDxjWVUBTRj8xn82vZe-J_d9piM-TKS3PHZCcM,3193
 dissect/target/tools/shell.py,sha256=4v6Z06YJDjKv6e6SRvWNjQ2n_KHo_CjL4P0w1_gY_ro,44827
 dissect/target/tools/utils.py,sha256=sQizexY3ui5vmWw4KOBLg5ecK3TPFjD-uxDqRn56ZTY,11304
@@ -336,10 +336,10 @@ dissect/target/volumes/luks.py,sha256=OmCMsw6rCUXG1_plnLVLTpsvE1n_6WtoRUGQbpmu1z
 dissect/target/volumes/lvm.py,sha256=wwQVR9I3G9YzmY6UxFsH2Y4MXGBcKL9aayWGCDTiWMU,2269
 dissect/target/volumes/md.py,sha256=j1K1iKmspl0C_OJFc7-Q1BMWN2OCC5EVANIgVlJ_fIE,1673
 dissect/target/volumes/vmfs.py,sha256=-LoUbn9WNwTtLi_4K34uV_-wDw2W5hgaqxZNj4UmqAQ,1730
-dissect.target-3.17.dev26.dist-info/COPYRIGHT,sha256=m-9ih2RVhMiXHI2bf_oNSSgHgkeIvaYRVfKTwFbnJPA,301
-dissect.target-3.17.dev26.dist-info/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
-dissect.target-3.17.dev26.dist-info/METADATA,sha256=fyuJSNpaOXUDx5rJFQYpsaxFKwa7VqFttG1XIvZxXco,11300
-dissect.target-3.17.dev26.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-dissect.target-3.17.dev26.dist-info/entry_points.txt,sha256=tvFPa-Ap-gakjaPwRc6Fl6mxHzxEZ_arAVU-IUYeo_s,447
-dissect.target-3.17.dev26.dist-info/top_level.txt,sha256=Mn-CQzEYsAbkxrUI0TnplHuXnGVKzxpDw_po_sXpvv4,8
-dissect.target-3.17.dev26.dist-info/RECORD,,
+dissect.target-3.17.dev28.dist-info/COPYRIGHT,sha256=m-9ih2RVhMiXHI2bf_oNSSgHgkeIvaYRVfKTwFbnJPA,301
+dissect.target-3.17.dev28.dist-info/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+dissect.target-3.17.dev28.dist-info/METADATA,sha256=BUIYxmU-67ACgU91l-B3EP-B6QpBkjk-OBY7EOMDfBU,11300
+dissect.target-3.17.dev28.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+dissect.target-3.17.dev28.dist-info/entry_points.txt,sha256=tvFPa-Ap-gakjaPwRc6Fl6mxHzxEZ_arAVU-IUYeo_s,447
+dissect.target-3.17.dev28.dist-info/top_level.txt,sha256=Mn-CQzEYsAbkxrUI0TnplHuXnGVKzxpDw_po_sXpvv4,8
+dissect.target-3.17.dev28.dist-info/RECORD,,