dissect.target 3.16.dev44__py3-none-any.whl → 3.17__py3-none-any.whl

dissect/target/loaders/mqtt.py

@@ -0,0 +1,309 @@
+from __future__ import annotations
+
+import logging
+import ssl
+import time
+import urllib
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from struct import pack, unpack_from
+from typing import Any, Callable, Iterator, Optional, Union
+
+import paho.mqtt.client as mqtt
+from dissect.util.stream import AlignedStream
+
+from dissect.target.containers.raw import RawContainer
+from dissect.target.exceptions import LoaderError
+from dissect.target.loader import Loader
+from dissect.target.plugin import arg
+from dissect.target.target import Target
+
+log = logging.getLogger(__name__)
+
+DISK_INDEX_OFFSET = 9
+
+
+def suppress(func: Callable) -> Callable:
+    def suppressed(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception:
+            return
+
+    return suppressed
+
+
+@dataclass
+class InfoMessage:
+    disks: list[DiskMessage]
+
+
+@dataclass
+class DiskMessage:
+    index: int = 0
+    sector_size: int = 0
+    total_size: int = 0
+
+
+@dataclass
+class SeekMessage:
+    data: bytes = b""
+
+
+class MQTTStream(AlignedStream):
+    def __init__(self, stream: MQTTConnection, disk_id: int, size: Optional[int] = None):
+        self.stream = stream
+        self.disk_id = disk_id
+        super().__init__(size)
+
+    def _read(self, offset: int, length: int, optimization_strategy: int = 0) -> bytes:
+        data = self.stream.read(self.disk_id, offset, length, optimization_strategy)
+        return data
+
+
+class MQTTConnection:
+    broker = None
+    host = None
+    prev = -1
+    factor = 1
+    prefetch_factor_inc = 10
+
+    def __init__(self, broker: Broker, host: str):
+        self.broker = broker
+        self.host = str(host)
+        self.info = lru_cache(128)(self.info)
+        self.read = lru_cache(128)(self.read)
+
+    def topo(self, peers: int) -> list[str]:
+        self.broker.topology(self.host)
+
+        while len(self.broker.peers(self.host)) < peers:
+            self.broker.topology(self.host)
+            time.sleep(1)
+        return self.broker.peers(self.host)
+
+    def info(self) -> list[MQTTStream]:
+        disks = []
+        self.broker.info(self.host)
+
+        message = None
+        while message is None:
+            message = self.broker.disk(self.host)
+
+        for idx, disk in enumerate(message.disks):
+            disks.append(MQTTStream(self, idx, disk.total_size))
+
+        return disks
+
+    def read(self, disk_id: int, offset: int, length: int, optimization_strategy: int) -> bytes:
+        message = None
+
+        message = self.broker.read(self.host, disk_id, offset, length)
+        if message:
+            return message.data
+
+        if self.prev == offset - (length * self.factor):
+            if self.factor < 500:
+                self.factor += self.prefetch_factor_inc
+        else:
+            self.factor = 1
+
+        self.prev = offset
+        flength = length * self.factor
+        self.broker.factor = self.factor
+        self.broker.seek(self.host, disk_id, offset, flength, optimization_strategy)
+        attempts = 0
+        while True:
+            if message := self.broker.read(self.host, disk_id, offset, length):
+                # don't waste time with sleep if we have a response
+                break
+
+            attempts += 1
+            time.sleep(0.1)
+            if attempts > 300:
+                # message might have not reached agent, resend...
+                self.broker.seek(self.host, disk_id, offset, flength, optimization_strategy)
+                attempts = 0
+
+        return message.data
+
+
+class Broker:
+    broker_host = None
+    broker_port = None
+    private_key_file = None
+    certificate_file = None
+    cacert_file = None
+    mqtt_client = None
+    connected = False
+    case = None
+
+    diskinfo = {}
+    index = {}
+    topo = {}
+    factor = 1
+
+    def __init__(self, broker: Broker, port: str, key: str, crt: str, ca: str, case: str, **kwargs):
+        self.broker_host = broker
+        self.broker_port = int(port)
+        self.private_key_file = key
+        self.certificate_file = crt
+        self.cacert_file = ca
+        self.case = case
+        self.command = kwargs.get("command", None)
+
+    def clear_cache(self) -> None:
+        self.index = {}
+
+    @suppress
+    def read(self, host: str, disk_id: int, seek_address: int, read_length: int) -> SeekMessage:
+        key = f"{host}-{disk_id}-{seek_address}-{read_length}"
+        return self.index.get(key)
+
+    @suppress
+    def disk(self, host: str) -> DiskMessage:
+        return self.diskinfo[host]
+
+    def peers(self, host: str) -> list[str]:
+        return self.topo[host]
+
+    def _on_disk(self, hostname: str, payload: bytes) -> None:
+        (num_of_disks,) = unpack_from("<B", payload, offset=0)
+        disks = []
+        for disk_index in range(num_of_disks):
+            (
+                sector_size,
+                total_size,
+            ) = unpack_from("<IQ", payload, offset=1 + (disk_index * DISK_INDEX_OFFSET))
+            disks.append(DiskMessage(index=disk_index, sector_size=sector_size, total_size=total_size))
+
+        self.diskinfo[hostname] = InfoMessage(disks=disks)
+
+    def _on_read(self, hostname: str, tokens: list[str], payload: bytes) -> None:
+        disk_id = tokens[3]
+        seek_address = int(tokens[4], 16)
+        read_length = int(tokens[5], 16)
+
+        for i in range(self.factor):
+            sublength = int(read_length / self.factor)
+            start = i * sublength
+            key = f"{hostname}-{disk_id}-{seek_address+start}-{sublength}"
+            if key in self.index:
+                continue
+
+            self.index[key] = SeekMessage(data=payload[start : start + sublength])
+
+    def _on_id(self, hostname: str, payload: bytes) -> None:
+        key = hostname
+        host = payload.decode("utf-8")
+        if host not in self.topo[key]:
+            self.topo[key].append(payload.decode("utf-8"))
+            self.mqtt_client.subscribe(f"{self.case}/{host}/DISKS")
+            self.mqtt_client.subscribe(f"{self.case}/{host}/READ/#")
+            if self.command is not None:
+                self.mqtt_client.publish(f"{self.case}/{host}/COMM", self.command.encode("utf-8"))
+            time.sleep(1)
+
+    def _on_log(self, client: mqtt.Client, userdata: Any, log_level: int, message: str) -> None:
+        log.debug(message)
+
+    def _on_connect(self, client: mqtt.Client, userdata: Any, flags: dict, rc: int) -> None:
+        self.connected = True
+
+    def _on_message(self, client: mqtt.Client, userdata: Any, msg: mqtt.client.MQTTMessage) -> None:
+        tokens = msg.topic.split("/")
+        casename, hostname, response, *_ = tokens
+        if casename != self.case:
+            return
+
+        if response == "DISKS":
+            self._on_disk(hostname, msg.payload)
+        elif response == "READ":
+            self._on_read(hostname, tokens, msg.payload)
+        elif response == "ID":
+            self._on_id(hostname, msg.payload)
+
+    def seek(self, host: str, disk_id: int, offset: int, flength: int, optimization_strategy: int) -> None:
+        length = int(flength / self.factor)
+        key = f"{host}-{disk_id}-{offset}-{length}"
+        if key in self.index:
+            return
+
+        self.mqtt_client.publish(
+            f"{self.case}/{host}/SEEK/{disk_id}/{hex(offset)}/{hex(flength)}", pack("<I", optimization_strategy)
+        )
+
+    def info(self, host: str) -> None:
+        self.mqtt_client.publish(f"{self.case}/{host}/INFO")
+
+    def topology(self, host: str) -> None:
+        self.topo[host] = []
+        self.mqtt_client.subscribe(f"{self.case}/{host}/ID")
+        time.sleep(1)  # need some time to avoid race condition, i.e. MQTT might react too fast
+        self.mqtt_client.publish(f"{self.case}/{host}/TOPO")
+
+    def connect(self) -> None:
+        self.mqtt_client = mqtt.Client(
+            client_id="", clean_session=True, userdata=None, protocol=mqtt.MQTTv311, transport="tcp"
+        )
+        self.mqtt_client.tls_set(
+            ca_certs=self.cacert_file,
+            certfile=self.certificate_file,
+            keyfile=self.private_key_file,
+            cert_reqs=ssl.CERT_REQUIRED,
+            tls_version=ssl.PROTOCOL_TLS,
+            ciphers=None,
+        )
+        self.mqtt_client.tls_insecure_set(True)  # merely having the correct cert is ok
+        self.mqtt_client.on_connect = self._on_connect
+        self.mqtt_client.on_message = self._on_message
+        if log.getEffectiveLevel() == logging.DEBUG:
+            self.mqtt_client.on_log = self._on_log
+        self.mqtt_client.connect(self.broker_host, port=self.broker_port, keepalive=60)
+        self.mqtt_client.loop_start()
+
+
+@arg("--mqtt-peers", type=int, dest="peers", help="minimum number of peers to await for first alias")
+@arg("--mqtt-case", dest="case", help="case name (broker will determine if you are allowed to access this data)")
+@arg("--mqtt-port", type=int, dest="port", help="broker connection port")
+@arg("--mqtt-broker", dest="broker", help="broker ip-address")
+@arg("--mqtt-key", dest="key", help="private key file")
+@arg("--mqtt-crt", dest="crt", help="client certificate file")
+@arg("--mqtt-ca", dest="ca", help="certificate authority file")
+@arg("--mqtt-command", dest="command", help="direct command to client(s)")
+class MQTTLoader(Loader):
+    """Load remote targets through a broker."""
+
+    connection = None
+    broker = None
+    peers = []
+
+    def __init__(self, path: Union[Path, str], **kwargs):
+        super().__init__(path)
+        cls = MQTTLoader
+        self.broker = cls.broker
+        self.connection = MQTTConnection(self.broker, path)
+
+    @staticmethod
+    def detect(path: Path) -> bool:
+        return False
+
+    def find_all(path: Path, **kwargs) -> Iterator[str]:
+        cls = MQTTLoader
+        num_peers = 1
+        if cls.broker is None:
+            if (uri := kwargs.get("parsed_path")) is None:
+                raise LoaderError("No URI connection details have been passed.")
+            options = dict(urllib.parse.parse_qsl(uri.query, keep_blank_values=True))
+            cls.broker = Broker(**options)
+            cls.broker.connect()
+            num_peers = int(options.get("peers", 1))
+
+        cls.connection = MQTTConnection(cls.broker, path)
+        cls.peers = cls.connection.topo(num_peers)
+        yield from cls.peers
+
+    def map(self, target: Target) -> None:
+        for disk in self.connection.info():
+            target.disks.add(RawContainer(disk))

dissect/target/loaders/overlay.py

@@ -0,0 +1,31 @@
+from dissect.target.filesystems.overlay import Overlay2Filesystem
+from dissect.target.helpers.fsutil import TargetPath
+from dissect.target.loader import Loader
+from dissect.target.target import Target
+
+
+class Overlay2Loader(Loader):
+    """Load overlay2 filesystems"""
+
+    def __init__(self, path: TargetPath, **kwargs):
+        super().__init__(path.resolve(), **kwargs)
+
+    @staticmethod
+    def detect(path: TargetPath) -> bool:
+        # path should be a folder
+        if not path.is_dir():
+            return False
+
+        # with the following three files
+        for required_file in ["init-id", "parent", "mount-id"]:
+            if not path.joinpath(required_file).exists():
+                return False
+
+        # and should have the following parent folders
+        if "image/overlay2/layerdb/mounts/" not in path.as_posix():
+            return False
+
+        return True
+
+    def map(self, target: Target) -> None:
+        target.filesystems.add(Overlay2Filesystem(self.path))

dissect/target/loaders/target.py

@@ -1,27 +1,30 @@
+from pathlib import Path
+from typing import TYPE_CHECKING
+
 try:
-    import yaml
+    from ruamel.yaml import YAML
 except ImportError:
-    raise ImportError("Missing PyYAML dependency")
+    raise ImportError("Missing ruamel.yaml dependency")
 
 from dissect.target import container
 from dissect.target.loader import Loader
 
+if TYPE_CHECKING:
+    from dissect.target.target import Target
+
 
 class TargetLoader(Loader):
     """Load target files."""
 
-    def __init__(self, path, **kwargs):
+    def __init__(self, path: Path, **kwargs):
         super().__init__(path)
         self.base_dir = path.parent
-        self.definition = yaml.safe_load(path.open("rb"))
+        self.definition = YAML(typ="safe").load(path.open("rb"))
 
     @staticmethod
-    def detect(path):
+    def detect(path: Path) -> bool:
         return path.suffix.lower() == ".target"
 
-    def map(self, target):
+    def map(self, target: Target) -> None:
         for disk in self.definition["disks"]:
             target.disks.add(container.open(disk))
-
-    def open(self, path):
-        return self.base_dir.joinpath(path).open("rb")

dissect/target/loaders/vb.py

@@ -14,8 +14,8 @@ class VBLoader(Loader):
         return (mft_exists or c_drive_exists) and config_exists
 
     def map(self, target):
-        ntfs_overlay = target.fs.add_layer()
-        remap_overlay = target.fs.add_layer()
+        remap_overlay = target.fs.append_layer()
+        ntfs_overlay = target.fs.append_layer()
         dfs = DirectoryFilesystem(self.path, case_sensitive=False)
         target.filesystems.add(dfs)
 

dissect/target/loaders/velociraptor.py

@@ -61,6 +61,10 @@ def extract_drive_letter(name: str) -> Optional[str]:
     if len(name) == 14 and name.startswith("%5C%5C.%5C") and name.endswith("%3A"):
         return name[10].lower()
 
+    # X: in URL encoding
+    if len(name) == 4 and name.endswith("%3A"):
+        return name[0].lower()
+
 
 class VelociraptorLoader(DirLoader):
     """Load Rapid7 Velociraptor forensic image files.
@@ -71,10 +75,7 @@ class VelociraptorLoader(DirLoader):
         {"Generic.Collectors.File":{"Root":"/","collectionSpec":"Glob\\netc/**\\nvar/log/**"}}
 
     Generic.Collectors.File (Windows) and Windows.KapeFiles.Targets (Windows) uses the accessors mft, ntfs, lazy_ntfs,
-    ntfs_vss and auto. The loader only supports a collection where a single accessor is used, which can be forced by
-    using the following configuration::
-
-        {"Windows.KapeFiles.Targets":{"VSSAnalysisAge":"1000","_SANS_Triage":"Y"}}
+    ntfs_vss and auto. The loader supports a collection where multiple accessors were used.
 
     References:
         - https://www.rapid7.com/products/velociraptor/

dissect/target/plugin.py

@@ -82,7 +82,7 @@ def export(*args, **kwargs) -> Callable:
         - default: Single return value
         - record: Yields records. Implicit when record argument is given.
         - yield: Yields printable values.
-        - none: No return value.
+        - none: No return value. Plugin is responsible for output formatting and should return ``None``.
 
     The ``export`` decorator adds some additional private attributes to an exported method or property:
 

dissect/target/plugins/apps/browser/brave.py

@@ -8,6 +8,7 @@ from dissect.target.plugins.apps.browser.browser import (
     GENERIC_DOWNLOAD_RECORD_FIELDS,
     GENERIC_EXTENSION_RECORD_FIELDS,
     GENERIC_HISTORY_RECORD_FIELDS,
+    GENERIC_PASSWORD_RECORD_FIELDS,
     BrowserPlugin,
 )
 from dissect.target.plugins.apps.browser.chromium import (
@@ -47,6 +48,10 @@ class BravePlugin(ChromiumMixin, BrowserPlugin):
         "browser/brave/extension", GENERIC_EXTENSION_RECORD_FIELDS
     )
 
+    BrowserPasswordRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
+        "browser/brave/password", GENERIC_PASSWORD_RECORD_FIELDS
+    )
+
     @export(record=BrowserHistoryRecord)
     def history(self) -> Iterator[BrowserHistoryRecord]:
         """Return browser history records for Brave."""
@@ -66,3 +71,8 @@ class BravePlugin(ChromiumMixin, BrowserPlugin):
     def extensions(self) -> Iterator[BrowserExtensionRecord]:
         """Return browser extension records for Brave."""
         yield from super().extensions("brave")
+
+    @export(record=BrowserPasswordRecord)
+    def passwords(self) -> Iterator[BrowserPasswordRecord]:
+        """Return browser password records for Brave."""
+        yield from super().passwords("brave")

dissect/target/plugins/apps/browser/browser.py

@@ -1,6 +1,10 @@
+from functools import cache
+
+from dissect.target.helpers import keychain
 from dissect.target.helpers.descriptor_extensions import UserRecordDescriptorExtension
 from dissect.target.helpers.record import create_extended_descriptor
 from dissect.target.plugin import NamespacePlugin
+from dissect.target.target import Target
 
 GENERIC_DOWNLOAD_RECORD_FIELDS = [
     ("datetime", "ts_start"),
@@ -63,6 +67,21 @@ GENERIC_HISTORY_RECORD_FIELDS = [
     ("uri", "from_url"),
     ("path", "source"),
 ]
+
+GENERIC_PASSWORD_RECORD_FIELDS = [
+    ("datetime", "ts_created"),
+    ("datetime", "ts_last_used"),
+    ("datetime", "ts_last_changed"),
+    ("string", "browser"),
+    ("varint", "id"),
+    ("uri", "url"),
+    ("string", "encrypted_username"),
+    ("string", "encrypted_password"),
+    ("string", "decrypted_username"),
+    ("string", "decrypted_password"),
+    ("path", "source"),
+]
+
 BrowserDownloadRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
     "browser/download", GENERIC_DOWNLOAD_RECORD_FIELDS
 )
@@ -75,11 +94,35 @@ BrowserHistoryRecord = create_extended_descriptor([UserRecordDescriptorExtension
 BrowserCookieRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
     "browser/cookie", GENERIC_COOKIE_FIELDS
 )
+BrowserPasswordRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
+    "browser/password", GENERIC_PASSWORD_RECORD_FIELDS
+)
 
 
 class BrowserPlugin(NamespacePlugin):
     __namespace__ = "browser"
 
+    def __init__(self, target: Target):
+        super().__init__(target)
+        self.keychain = cache(self.keychain)
+
+    def keychain(self) -> set:
+        """Retrieve a set of passphrases to use for decrypting saved browser credentials.
+
+        Always adds an empty passphrase as some browsers encrypt values using empty passphrases.
+
+        Returns:
+            Set of passphrase strings.
+        """
+        passphrases = set()
+        for provider in [self.__namespace__, "browser", "user", None]:
+            for key in keychain.get_keys_for_provider(provider) if provider else keychain.get_keys_without_provider():
+                if key.key_type == keychain.KeyType.PASSPHRASE:
+                    passphrases.add(key.value)
+
+        passphrases.add("")
+        return passphrases
+
 
 def try_idna(url: str) -> bytes:
     """Attempts to convert a possible Unicode url to ASCII using the IDNA standard.

dissect/target/plugins/apps/browser/chrome.py

@@ -8,6 +8,7 @@ from dissect.target.plugins.apps.browser.browser import (
     GENERIC_DOWNLOAD_RECORD_FIELDS,
     GENERIC_EXTENSION_RECORD_FIELDS,
     GENERIC_HISTORY_RECORD_FIELDS,
+    GENERIC_PASSWORD_RECORD_FIELDS,
     BrowserPlugin,
 )
 from dissect.target.plugins.apps.browser.chromium import (
@@ -49,6 +50,10 @@ class ChromePlugin(ChromiumMixin, BrowserPlugin):
         "browser/chrome/extension", GENERIC_EXTENSION_RECORD_FIELDS
     )
 
+    BrowserPasswordRecord = create_extended_descriptor([UserRecordDescriptorExtension])(
+        "browser/chrome/password", GENERIC_PASSWORD_RECORD_FIELDS
+    )
+
     @export(record=BrowserHistoryRecord)
     def history(self) -> Iterator[BrowserHistoryRecord]:
         """Return browser history records for Google Chrome."""
@@ -68,3 +73,8 @@ class ChromePlugin(ChromiumMixin, BrowserPlugin):
     def extensions(self) -> Iterator[BrowserExtensionRecord]:
         """Return browser extension records for Google Chrome."""
         yield from super().extensions("chrome")
+
+    @export(record=BrowserPasswordRecord)
+    def passwords(self) -> Iterator[BrowserPasswordRecord]:
+        """Return browser password records for Google Chrome."""
+        yield from super().passwords("chrome")