dissect.target 3.17.dev26__tar.gz → 3.17.dev28__tar.gz

{dissect.target-3.17.dev26/dissect.target.egg-info → dissect_target-3.17.dev28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dissect.target
-Version: 3.17.dev26
+Version: 3.17.dev28
 Summary: This module ties all other Dissect modules together, it provides a programming API and command line tools which allow easy access to various data sources inside disk images or file collections (a.k.a. targets)
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{dissect.target-3.17.dev26 → dissect_target-3.17.dev28}/dissect/target/helpers/network_managers.py

@@ -5,6 +5,7 @@ import re
 from collections import defaultdict
 from configparser import ConfigParser, MissingSectionHeaderError
 from io import StringIO
+from itertools import chain
 from re import compile, sub
 from typing import Any, Callable, Iterable, Match, Optional
 
@@ -299,7 +300,8 @@ class Parser:
             return
 
         if section:
-            config = config.get(section, {})
+            # account for values of sections which are None
+            config = config.get(section, {}) or {}
 
         for key, value in config.items():
             if key == option:
@@ -508,7 +510,7 @@ class LinuxNetworkManager:
 
 
 def parse_unix_dhcp_log_messages(target) -> list[str]:
-    """Parse local syslog and cloud init log files for DHCP lease IPs.
+    """Parse local syslog, journal and cloud init-log files for DHCP lease IPs.
 
     Args:
         target: Target to discover and obtain network information from.
@@ -516,53 +518,68 @@ def parse_unix_dhcp_log_messages(target) -> list[str]:
     Returns:
         List of DHCP ip addresses.
     """
-    ips = []
-
-    # Search through parsed syslogs for DHCP leases.
-    try:
-        messages = target.messages()
-        for record in messages:
-            line = record.message
-
-            # Ubuntu DHCP
-            if ("DHCPv4" in line or "DHCPv6" in line) and " address " in line and " via " in line:
-                ip = line.split(" address ")[1].split(" via ")[0].strip().split("/")[0]
-                if ip not in ips:
-                    ips.append(ip)
-
-            # Ubuntu DHCP NetworkManager
-            elif "option ip_address" in line and ("dhcp4" in line or "dhcp6" in line) and "=> '" in line:
-                ip = line.split("=> '")[1].replace("'", "").strip()
-                if ip not in ips:
-                    ips.append(ip)
-
-            # Debian and CentOS dhclient
-            elif record.daemon == "dhclient" and "bound to" in line:
-                ip = line.split("bound to")[1].split(" ")[1].strip()
-                if ip not in ips:
-                    ips.append(ip)
-
-            # CentOS DHCP and general NetworkManager
-            elif " address " in line and ("dhcp4" in line or "dhcp6" in line):
-                ip = line.split(" address ")[1].strip()
-                if ip not in ips:
-                    ips.append(ip)
-
-    except PluginError:
-        target.log.debug("Can not search for DHCP leases in syslog files as they does not exist.")
-
-    # A unix system might be provisioned using Ubuntu's cloud-init (https://cloud-init.io/).
-    if (path := target.fs.path("/var/log/cloud-init.log")).exists():
-        for line in path.open("rt"):
-            # We are interested in the following log line:
-            # YYYY-MM-DD HH:MM:SS,000 - dhcp.py[DEBUG]: Received dhcp lease on IFACE for IP/MASK
-            if "Received dhcp lease on" in line:
-                interface, ip, netmask = re.search(r"Received dhcp lease on (\w{0,}) for (\S+)\/(\S+)", line).groups()
-                if ip not in ips:
-                    ips.append(ip)
-
-    if not path and not messages:
-        target.log.warning("Can not search for DHCP leases in syslog or cloud-init.log files as they does not exist.")
+    ips = set()
+    messages = set()
+
+    for log_func in ["messages", "journal"]:
+        try:
+            messages = chain(messages, getattr(target, log_func)())
+        except PluginError:
+            target.log.debug(f"Could not search for DHCP leases in {log_func} files.")
+
+    if not messages:
+        target.log.warning(f"Could not search for DHCP leases using {log_func}: No log entries found.")
+
+    for record in messages:
+        line = record.message
+
+        # Ubuntu cloud-init
+        if "Received dhcp lease on" in line:
+            interface, ip, netmask = re.search(r"Received dhcp lease on (\w{0,}) for (\S+)\/(\S+)", line).groups()
+            ips.add(ip)
+            continue
+
+        # Ubuntu DHCP
+        if ("DHCPv4" in line or "DHCPv6" in line) and " address " in line and " via " in line:
+            ip = line.split(" address ")[1].split(" via ")[0].strip().split("/")[0]
+            ips.add(ip)
+            continue
+
+        # Ubuntu DHCP NetworkManager
+        if "option ip_address" in line and ("dhcp4" in line or "dhcp6" in line) and "=> '" in line:
+            ip = line.split("=> '")[1].replace("'", "").strip()
+            ips.add(ip)
+            continue
+
+        # Debian and CentOS dhclient
+        if hasattr(record, "daemon") and record.daemon == "dhclient" and "bound to" in line:
+            ip = line.split("bound to")[1].split(" ")[1].strip()
+            ips.add(ip)
+            continue
+
+        # CentOS DHCP and general NetworkManager
+        if " address " in line and ("dhcp4" in line or "dhcp6" in line):
+            ip = line.split(" address ")[1].strip()
+            ips.add(ip)
+            continue
+
+        # Ubuntu/Debian DHCP networkd (Journal)
+        if (
+            hasattr(record, "code_func")
+            and record.code_func == "dhcp_lease_acquired"
+            and " address " in line
+            and " via " in line
+        ):
+            interface, ip, netmask, gateway = re.search(
+                r"^(\S+): DHCPv[4|6] address (\S+)\/(\S+) via (\S+)", line
+            ).groups()
+            ips.add(ip)
+            continue
+
+        # Journals and syslogs can be large and slow to iterate,
+        # so we stop if we have some results and have reached the journal plugin.
+        if len(ips) >= 2 and record._desc.name == "linux/log/journal":
+            break
 
     return ips
 

{dissect.target-3.17.dev26 → dissect_target-3.17.dev28}/dissect/target/loaders/mqtt.py

@@ -65,6 +65,9 @@ class MQTTStream(AlignedStream):
 class MQTTConnection:
     broker = None
     host = None
+    prev = -1
+    factor = 1
+    prefetch_factor_inc = 10
 
     def __init__(self, broker: Broker, host: str):
         self.broker = broker
@@ -95,20 +98,32 @@ class MQTTConnection:
 
     def read(self, disk_id: int, offset: int, length: int, optimization_strategy: int) -> bytes:
         message = None
-        self.broker.seek(self.host, disk_id, offset, length, optimization_strategy)
 
+        message = self.broker.read(self.host, disk_id, offset, length)
+        if message:
+            return message.data
+
+        if self.prev == offset - (length * self.factor):
+            if self.factor < 500:
+                self.factor += self.prefetch_factor_inc
+        else:
+            self.factor = 1
+
+        self.prev = offset
+        flength = length * self.factor
+        self.broker.factor = self.factor
+        self.broker.seek(self.host, disk_id, offset, flength, optimization_strategy)
         attempts = 0
         while True:
-            message = self.broker.read(self.host, disk_id, offset, length)
-            # don't waste time with sleep if we have a response
-            if message:
+            if message := self.broker.read(self.host, disk_id, offset, length):
+                # don't waste time with sleep if we have a response
                 break
 
             attempts += 1
-            time.sleep(0.01)
-            if attempts > 100:
+            time.sleep(0.1)
+            if attempts > 300:
                 # message might have not reached agent, resend...
-                self.broker.seek(self.host, disk_id, offset, length, optimization_strategy)
+                self.broker.seek(self.host, disk_id, offset, flength, optimization_strategy)
                 attempts = 0
 
         return message.data
@@ -127,6 +142,7 @@ class Broker:
     diskinfo = {}
     index = {}
     topo = {}
+    factor = 1
 
     def __init__(self, broker: Broker, port: str, key: str, crt: str, ca: str, case: str, **kwargs):
         self.broker_host = broker
@@ -137,10 +153,13 @@ class Broker:
         self.case = case
         self.command = kwargs.get("command", None)
 
+    def clear_cache(self) -> None:
+        self.index = {}
+
     @suppress
     def read(self, host: str, disk_id: int, seek_address: int, read_length: int) -> SeekMessage:
         key = f"{host}-{disk_id}-{seek_address}-{read_length}"
-        return self.index.pop(key)
+        return self.index.get(key)
 
     @suppress
     def disk(self, host: str) -> DiskMessage:
@@ -165,14 +184,15 @@ class Broker:
         disk_id = tokens[3]
         seek_address = int(tokens[4], 16)
         read_length = int(tokens[5], 16)
-        msg = SeekMessage(data=payload)
 
-        key = f"{hostname}-{disk_id}-{seek_address}-{read_length}"
+        for i in range(self.factor):
+            sublength = int(read_length / self.factor)
+            start = i * sublength
+            key = f"{hostname}-{disk_id}-{seek_address+start}-{sublength}"
+            if key in self.index:
+                continue
 
-        if key in self.index:
-            return
-
-        self.index[key] = msg
+            self.index[key] = SeekMessage(data=payload[start : start + sublength])
 
     def _on_id(self, hostname: str, payload: bytes) -> None:
         key = hostname
@@ -204,9 +224,14 @@ class Broker:
         elif response == "ID":
             self._on_id(hostname, msg.payload)
 
-    def seek(self, host: str, disk_id: int, offset: int, length: int, optimization_strategy: int) -> None:
+    def seek(self, host: str, disk_id: int, offset: int, flength: int, optimization_strategy: int) -> None:
+        length = int(flength / self.factor)
+        key = f"{host}-{disk_id}-{offset}-{length}"
+        if key in self.index:
+            return
+
         self.mqtt_client.publish(
-            f"{self.case}/{host}/SEEK/{disk_id}/{hex(offset)}/{hex(length)}", pack("<I", optimization_strategy)
+            f"{self.case}/{host}/SEEK/{disk_id}/{hex(offset)}/{hex(flength)}", pack("<I", optimization_strategy)
         )
 
     def info(self, host: str) -> None:

dissect_target-3.17.dev28/dissect/target/plugins/os/unix/log/messages.py

@@ -0,0 +1,118 @@
+import re
+from pathlib import Path
+from typing import Iterator
+
+from dissect.target import Target
+from dissect.target.exceptions import UnsupportedPluginError
+from dissect.target.helpers.record import TargetRecordDescriptor
+from dissect.target.helpers.utils import year_rollover_helper
+from dissect.target.plugin import Plugin, export
+
+MessagesRecord = TargetRecordDescriptor(
+    "linux/log/messages",
+    [
+        ("datetime", "ts"),
+        ("string", "daemon"),
+        ("varint", "pid"),
+        ("string", "message"),
+        ("path", "source"),
+    ],
+)
+
+DEFAULT_TS_LOG_FORMAT = "%b %d %H:%M:%S"
+RE_TS = re.compile(r"(\w+\s{1,2}\d+\s\d{2}:\d{2}:\d{2})")
+RE_DAEMON = re.compile(r"^[^:]+:\d+:\d+[^\[\]:]+\s([^\[:]+)[\[|:]{1}")
+RE_PID = re.compile(r"\w\[(\d+)\]")
+RE_MSG = re.compile(r"[^:]+:\d+:\d+[^:]+:\s(.*)$")
+RE_CLOUD_INIT_LINE = re.compile(r"(?P<ts>.*) - (?P<daemon>.*)\[(?P<log_level>\w+)\]\: (?P<message>.*)$")
+
+
+class MessagesPlugin(Plugin):
+    def __init__(self, target: Target):
+        super().__init__(target)
+        self.log_files = set(self._find_log_files())
+
+    def _find_log_files(self) -> Iterator[Path]:
+        log_dirs = ["/var/log/", "/var/log/installer/"]
+        file_globs = ["syslog*", "messages*", "cloud-init.log*"]
+        for log_dir in log_dirs:
+            for glob in file_globs:
+                yield from self.target.fs.path(log_dir).glob(glob)
+
+    def check_compatible(self) -> None:
+        if not self.log_files:
+            raise UnsupportedPluginError("No log files found")
+
+    @export(record=MessagesRecord)
+    def syslog(self) -> Iterator[MessagesRecord]:
+        """Return contents of /var/log/messages*, /var/log/syslog* and cloud-init logs.
+
+        See ``messages`` for more information.
+        """
+        return self.messages()
+
+    @export(record=MessagesRecord)
+    def messages(self) -> Iterator[MessagesRecord]:
+        """Return contents of /var/log/messages*, /var/log/syslog* and cloud-init logs.
+
+        Note: due to year rollover detection, the contents of the files are returned in reverse.
+
+        The messages log file holds information about a variety of events such as the system error messages, system
+        startups and shutdowns, change in the network configuration, etc. Aims to store valuable, non-debug and
+        non-critical messages. This log should be considered the "general system activity" log.
+
+        References:
+            - https://geek-university.com/linux/var-log-messages-file/
+            - https://www.geeksforgeeks.org/file-timestamps-mtime-ctime-and-atime-in-linux/
+            - https://cloudinit.readthedocs.io/en/latest/development/logging.html#logging-command-output
+        """
+
+        tzinfo = self.target.datetime.tzinfo
+
+        for log_file in self.log_files:
+            if "cloud-init" in log_file.name:
+                yield from self._parse_cloud_init_log(log_file)
+                continue
+
+            for ts, line in year_rollover_helper(log_file, RE_TS, DEFAULT_TS_LOG_FORMAT, tzinfo):
+                daemon = dict(enumerate(RE_DAEMON.findall(line))).get(0)
+                pid = dict(enumerate(RE_PID.findall(line))).get(0)
+                message = dict(enumerate(RE_MSG.findall(line))).get(0, line)
+
+                yield MessagesRecord(
+                    ts=ts,
+                    daemon=daemon,
+                    pid=pid,
+                    message=message,
+                    source=log_file,
+                    _target=self.target,
+                )
+
+    def _parse_cloud_init_log(self, log_file: Path) -> Iterator[MessagesRecord]:
+        """Parse a cloud-init.log file.
+
+        Lines are structured in the following format:
+        ``YYYY-MM-DD HH:MM:SS,000 - dhcp.py[DEBUG]: Received dhcp lease on IFACE for IP/MASK``
+
+        NOTE: ``cloud-init-output.log`` files are not supported as they do not contain structured logs.
+
+        Args:
+            ``log_file``: path to cloud-init.log file.
+
+        Returns: ``MessagesRecord``
+        """
+        for line in log_file.open("rt").readlines():
+            if line := line.strip():
+                if match := RE_CLOUD_INIT_LINE.match(line):
+                    match = match.groupdict()
+                    yield MessagesRecord(
+                        ts=match["ts"].split(",")[0],
+                        daemon=match["daemon"],
+                        pid=None,
+                        message=match["message"],
+                        source=log_file,
+                        _target=self.target,
+                    )
+                else:
+                    self.target.log.warning("Could not match cloud-init log line")
+                    self.target.log.debug("No match for line '%s'", line)

{dissect.target-3.17.dev26 → dissect_target-3.17.dev28}/dissect/target/tools/query.py

@@ -173,8 +173,7 @@ def main():
         collected_plugins = {}
 
         if targets:
-            for target in targets:
-                plugin_target = Target.open(target)
+            for plugin_target in Target.open_all(targets, args.children):
                 if isinstance(plugin_target._loader, ProxyLoader):
                     parser.error("can't list compatible plugins for remote targets.")
                 funcs, _ = find_plugin_functions(plugin_target, args.list, compatibility=True, show_hidden=True)

{dissect.target-3.17.dev26 → dissect_target-3.17.dev28/dissect.target.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dissect.target
-Version: 3.17.dev26
+Version: 3.17.dev28
 Summary: This module ties all other Dissect modules together, it provides a programming API and command line tools which allow easy access to various data sources inside disk images or file collections (a.k.a. targets)
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{dissect.target-3.17.dev26 → dissect_target-3.17.dev28}/tests/loaders/test_mqtt.py

@@ -1,5 +1,8 @@
+from __future__ import annotations
+
 import sys
 import time
+from dataclasses import dataclass
 from struct import pack
 from typing import Iterator
 from unittest.mock import MagicMock, patch
@@ -44,6 +47,24 @@ class MQTTMock(MagicMock):
         self.on_message(self, None, response)
 
 
+@dataclass
+class MockSeekMessage:
+    data: bytes = b""
+
+
+class MockBroker(MagicMock):
+    _seek = False
+
+    def seek(self, *args) -> None:
+        self._seek = True
+
+    def read(self, *args) -> MockSeekMessage | None:
+        if self._seek:
+            self._seek = False
+            return MockSeekMessage(data=b"010101")
+        return None
+
+
 @pytest.fixture
 def mock_paho(monkeypatch: pytest.MonkeyPatch) -> Iterator[MagicMock]:
     with monkeypatch.context() as m:
@@ -62,6 +83,11 @@ def mock_client(mock_paho: MagicMock) -> Iterator[MagicMock]:
     yield mock_client
 
 
+@pytest.fixture
+def mock_broker() -> Iterator[MockBroker]:
+    yield MockBroker()
+
+
 @pytest.mark.parametrize(
     "alias, hosts, disks, disk, seek, read, expected",
     [
@@ -102,3 +128,21 @@ def test_remote_loader_stream(
         target.disks[disk].seek(seek)
         data = target.disks[disk].read(read)
         assert data == expected
+
+
+def test_mqtt_loader_prefetch(mock_broker: MockBroker) -> None:
+    from dissect.target.loaders.mqtt import MQTTConnection
+
+    connection = MQTTConnection(mock_broker, "")
+    connection.prefetch_factor_inc = 10
+    assert connection.factor == 1
+    assert connection.prev == -1
+    connection.read(1, 0, 100, 0)
+    assert connection.factor == 1
+    assert connection.prev == 0
+    connection.read(1, 100, 100, 0)
+    assert connection.factor == connection.prefetch_factor_inc + 1
+    assert connection.prev == 100
+    connection.read(1, 1200, 100, 0)
+    assert connection.factor == (connection.prefetch_factor_inc * 2) + 1
+    assert connection.prev == 1200

{dissect.target-3.17.dev26 → dissect_target-3.17.dev28}/tests/plugins/os/unix/log/test_messages.py

@@ -8,16 +8,17 @@ from zoneinfo import ZoneInfo
 from flow.record.fieldtypes import datetime as dt
 
 from dissect.target import Target
+from dissect.target.filesystem import VirtualFilesystem
 from dissect.target.filesystems.tar import TarFilesystem
 from dissect.target.plugins.general import default
 from dissect.target.plugins.os.unix.log.messages import MessagesPlugin, MessagesRecord
 from tests._utils import absolute_path
 
 
-def test_unix_log_messages_plugin(target_unix_users, fs_unix):
+def test_unix_log_messages_plugin(target_unix_users: Target, fs_unix: VirtualFilesystem) -> None:
     fs_unix.map_file_fh("/etc/timezone", BytesIO(b"Europe/Amsterdam"))
 
-    data_file = absolute_path("_data/plugins/os/unix/log//messages/messages")
+    data_file = absolute_path("_data/plugins/os/unix/log/messages/messages")
     fs_unix.map_file("var/log/messages", data_file)
 
     entry = fs_unix.get("var/log/messages")
@@ -46,7 +47,7 @@ def test_unix_log_messages_plugin(target_unix_users, fs_unix):
     assert isinstance(syslogs[0], type(MessagesRecord()))
 
 
-def test_unix_log_messages_compressed_timezone_year_rollover():
+def test_unix_log_messages_compressed_timezone_year_rollover() -> None:
     target = Target()
     bio = BytesIO()
 
@@ -86,7 +87,7 @@ def test_unix_log_messages_compressed_timezone_year_rollover():
     assert results[1].ts == dt(2021, 1, 1, 13, 37, 0, tzinfo=ZoneInfo("America/Chicago"))
 
 
-def test_unix_log_messages_malformed_log_year_rollover(target_unix_users, fs_unix):
+def test_unix_log_messages_malformed_log_year_rollover(target_unix_users: Target, fs_unix: VirtualFilesystem) -> None:
     fs_unix.map_file_fh("/etc/timezone", BytesIO(b"Europe/Amsterdam"))
 
     messages = BytesIO(
@@ -105,3 +106,21 @@ def test_unix_log_messages_malformed_log_year_rollover(target_unix_users, fs_uni
 
         results = list(target_unix_users.messages())
         assert len(results) == 2
+
+
+def test_unix_messages_cloud_init(target_unix_users: Target, fs_unix: VirtualFilesystem) -> None:
+    messages = """
+    2005-08-09 11:55:21,000 - foo.py[DEBUG]: This is a cloud-init message!
+    2005-08-09 11:55:21,001 - util.py[DEBUG]: Cloud-init v. 1.2.3-4ubuntu5 running 'init-local' at Tue, 9 Aug 2005 11:55:21 +0000. Up 13.37 seconds.
+    """  # noqa: E501
+    fs_unix.map_file_fh("/var/log/installer/cloud-init.log", BytesIO(textwrap.dedent(messages).encode()))
+
+    target_unix_users.add_plugin(MessagesPlugin)
+
+    results = list(target_unix_users.messages())
+    assert len(results) == 2
+    assert results[0].ts == dt(2005, 8, 9, 11, 55, 21)
+    assert results[0].daemon == "foo.py"
+    assert results[0].pid is None
+    assert results[0].message == "This is a cloud-init message!"
+    assert results[0].source == "/var/log/installer/cloud-init.log"