dissect.target 3.17.dev36__py3-none-any.whl → 3.18__py3-none-any.whl

dissect/target/exceptions.py

@@ -114,3 +114,7 @@ class RegistryCorruptError(RegistryError):
 
 class ConfigurationParsingError(Error):
     """An error occurred during configuration parsing."""
+
+
+class TargetPathNotFoundError(TargetError):
+    """The path to the target does not exist."""

dissect/target/filesystem.py

@@ -6,7 +6,6 @@ import logging
 import os
 import pathlib
 import stat
-import warnings
 from collections import defaultdict
 from typing import TYPE_CHECKING, Any, BinaryIO, Callable, Iterator, Optional, Type
 
@@ -67,15 +66,6 @@ class Filesystem:
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}>"
 
-    @classmethod
-    @property
-    def __fstype__(cls) -> str:
-        warnings.warn(
-            "The __fstype__ attribute is deprecated and will be removed in dissect.target 3.15. Use __type__ instead",
-            category=DeprecationWarning,
-        )
-        return cls.__type__
-
     def path(self, *args) -> fsutil.TargetPath:
         """Instantiate a new path-like object on this filesystem."""
         return fsutil.TargetPath(self, *args)

dissect/target/helpers/cache.py

@@ -147,7 +147,9 @@ class Cache:
             if os.access(cache_file, os.R_OK, effective_ids=bool(os.supports_effective_ids)):
                 if os.stat(cache_file).st_size != 0:
                     try:
-                        return self.open_reader(cache_file, output)
+                        reader = self.open_reader(cache_file, output)
+                        target.log.info("Using cache for function: %s", self.fname)
+                        return reader
                     except Exception as e:
                         target.log.warning("Cache will NOT be used. Error opening cache file: %s", cache_file)
                         target.log.debug("", exc_info=e)

dissect/target/helpers/hashutil.py

@@ -1,18 +1,11 @@
 from __future__ import annotations
 
 import hashlib
-import warnings
 from typing import TYPE_CHECKING, BinaryIO, Union
 
-from flow.record import Record
-
-from dissect.target.exceptions import FileNotFoundError
-
 if TYPE_CHECKING:
     from hashlib._hashlib import HASH
 
-    from dissect.target.target import Target
-
 BUFFER_SIZE = 32768
 
 
@@ -52,36 +45,3 @@ def custom(fh: BinaryIO, algos: list[Union[str, HASH]]) -> tuple[str]:
         ctx = algos
 
     return _hash(fh, ctx)
-
-
-def hash_uri(target: Target, path: str) -> tuple[str, str]:
-    """Hash the target path."""
-    warnings.warn(
-        (
-            "The hash_uri() function is deprecated, and will be removed in dissect.target 3.15. "
-            "Use target.fs.hash() instead"
-        ),
-        DeprecationWarning,
-    )
-
-    if path is None:
-        raise FileNotFoundError()
-
-    path = str(target.resolve(path))
-    return (path, target.fs.hash(path))
-
-
-def hash_uri_records(target: Target, record: Record) -> Record:
-    """Hash uri paths inside the record."""
-
-    from dissect.target.helpers.record_modifier import Modifier, get_modifier_function
-
-    warnings.warn(
-        (
-            "The hash_uri_records() function is deprecated, and will be removed in dissect.target 3.15. "
-            "Use hash_path_records() instead"
-        ),
-        DeprecationWarning,
-    )
-    func = get_modifier_function(Modifier.HASH)
-    return func(target, record)

dissect/target/helpers/protobuf.py

@@ -3,30 +3,26 @@ from __future__ import annotations
 from typing import Any, BinaryIO
 
 from dissect.cstruct.types.base import BaseType
-from dissect.cstruct.types.bytesinteger import BytesInteger
 
 
-class ProtobufVarint(BytesInteger):
+class ProtobufVarint(BaseType):
     """Implements a protobuf integer type for dissect.cstruct that can span a variable amount of bytes.
 
-    Mainly follows the cstruct BytesInteger implementation with minor tweaks
-    to support protobuf's msb varint implementation.
+    Supports protobuf's msb varint implementation.
 
     Resources:
         - https://protobuf.dev/programming-guides/encoding/
         - https://github.com/protocolbuffers/protobuf/blob/main/python/google/protobuf/internal/decoder.py
     """
 
-    def _read(self, stream: BinaryIO, context: dict[str, Any] = None) -> int:
+    @classmethod
+    def _read(cls, stream: BinaryIO, context: dict[str, Any] = None) -> int:
         return decode_varint(stream)
 
-    def _write(self, stream: BinaryIO, data: int) -> int:
+    @classmethod
+    def _write(cls, stream: BinaryIO, data: int) -> int:
         return stream.write(encode_varint(data))
 
-    _read_array = BaseType._read_array
-
-    _write_array = BaseType._write_array
-
 
 def decode_varint(stream: BinaryIO) -> int:
     """Reads a varint from the provided buffer stream.

dissect/target/helpers/record_modifier.py

@@ -62,13 +62,16 @@ MODIFIER_MAPPING = {
 
 def _resolve_path_types(target: Target, record: Record) -> Iterator[tuple[str, TargetPath]]:
     for field_name, field_type in record._field_types.items():
-        if not issubclass(field_type, fieldtypes.path):
+        if not issubclass(field_type, (fieldtypes.path, fieldtypes.command)):
             continue
 
         path = getattr(record, field_name, None)
         if path is None:
             continue
 
+        if isinstance(path, fieldtypes.command):
+            path = path.executable
+
         yield field_name, target.resolve(str(path))
 
 

dissect/target/helpers/ssh.py

@@ -1,9 +1,9 @@
 import base64
 import binascii
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 
-c_rfc4716_def = """
+rfc4716_def = """
 struct ssh_string {
     uint32 length;
     char value[length];
@@ -23,8 +23,7 @@ struct ssh_private_key {
 }
 """
 
-c_rfc4716 = cstruct.cstruct(endian=">")
-c_rfc4716.load(c_rfc4716_def)
+c_rfc4716 = cstruct(endian=">").load(rfc4716_def)
 
 RFC4716_MARKER_START = b"-----BEGIN OPENSSH PRIVATE KEY-----"
 RFC4716_MARKER_END = b"-----END OPENSSH PRIVATE KEY-----"

dissect/target/loaders/mqtt.py

@@ -1,13 +1,18 @@
 from __future__ import annotations
 
+import atexit
 import logging
+import math
+import os
 import ssl
+import sys
 import time
 import urllib
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
 from struct import pack, unpack_from
+from threading import Thread
 from typing import Any, Callable, Iterator, Optional, Union
 
 import paho.mqtt.client as mqtt
@@ -51,6 +56,34 @@ class SeekMessage:
     data: bytes = b""
 
 
+class MQTTTransferRatePerSecond:
+    def __init__(self, window_size: int = 10):
+        self.window_size = window_size
+        self.timestamps = []
+        self.bytes = []
+
+    def record(self, timestamp: float, byte_count: int) -> MQTTTransferRatePerSecond:
+        while self.timestamps and (timestamp - self.timestamps[0] > self.window_size):
+            self.timestamps.pop(0)
+            self.bytes.pop(0)
+
+        self.timestamps.append(timestamp)
+        self.bytes.append(byte_count)
+        return self
+
+    def value(self, current_time: float) -> float:
+        if not self.timestamps:
+            return 0
+
+        elapsed_time = current_time - self.timestamps[0]
+        if elapsed_time == 0:
+            return 0
+
+        total_bytes = self.bytes[-1] - self.bytes[0]
+
+        return total_bytes / elapsed_time
+
+
 class MQTTStream(AlignedStream):
     def __init__(self, stream: MQTTConnection, disk_id: int, size: Optional[int] = None):
         self.stream = stream
@@ -62,12 +95,108 @@ class MQTTStream(AlignedStream):
         return data
 
 
+class MQTTDiagnosticLine:
+    def __init__(self, connection: MQTTConnection, total_peers: int):
+        self.connection = connection
+        self.total_peers = total_peers
+        self._columns, self._rows = os.get_terminal_size(0)
+        atexit.register(self._detach)
+        self._attach()
+
+    def _attach(self) -> None:
+        # save cursor position
+        sys.stderr.write("\0337")
+        # set top and bottom margins of the scrolling region to default
+        sys.stderr.write("\033[r")
+        # restore cursor position
+        sys.stderr.write("\0338")
+        # move cursor down one line in the same column; if at the bottom, the screen scrolls up
+        sys.stderr.write("\033D")
+        # move cursor up one line in the same column; if at the top, screen scrolls down
+        sys.stderr.write("\033M")
+        # save cursor position again
+        sys.stderr.write("\0337")
+        # restrict scrolling to a region from the first line to one before the last line
+        sys.stderr.write(f"\033[1;{self._rows - 1}r")
+        # restore cursor position after setting scrolling region
+        sys.stderr.write("\0338")
+
+    def _detach(self) -> None:
+        # save cursor position
+        sys.stderr.write("\0337")
+        # move cursor to the specified position (last line, first column)
+        sys.stderr.write(f"\033[{self._rows};1H")
+        # clear from cursor to end of the line
+        sys.stderr.write("\033[K")
+        # reset scrolling region to include the entire display
+        sys.stderr.write("\033[r")
+        # restore cursor position
+        sys.stderr.write("\0338")
+        # ensure the written content is displayed (flush output)
+        sys.stderr.flush()
+
+    def display(self) -> None:
+        # prepare: set background color to blue and text color to white at the beginning of the line
+        prefix = "\x1b[44m\x1b[37m\r"
+        # reset all attributes (colors, styles) to their defaults afterwards
+        suffix = "\x1b[0m"
+        # separator to set background color to red and text style to bold
+        separator = "\x1b[41m\x1b[1m"
+        logo = "TARGETD"
+
+        start = time.time()
+        transfer_rate = MQTTTransferRatePerSecond(window_size=7)
+
+        while True:
+            time.sleep(0.05)
+            peers = "?"
+            try:
+                peers = len(self.connection.broker.peers(self.connection.host))
+            except Exception:
+                pass
+
+            recv = self.connection.broker.bytes_received
+            now = time.time()
+            transfer = transfer_rate.record(now, recv).value(now) / 1000  # convert to KB/s
+            failures = self.connection.retries
+            seconds_elapsed = round(now - start) % 60
+            minutes_elapsed = math.floor((now - start) / 60) % 60
+            hours_elapsed = math.floor((now - start) / 60**2)
+            timer = f"{hours_elapsed:02d}:{minutes_elapsed:02d}:{seconds_elapsed:02d}"
+            display = f"{timer} {peers}/{self.total_peers} peers {transfer:>8.2f} KB p/s {failures:>4} failures"
+            rest = self._columns - len(display)
+            padding = (rest - len(logo)) * " "
+
+            # save cursor position
+            sys.stderr.write("\0337")
+            # move cursor to specified position (last line, first column)
+            sys.stderr.write(f"\033[{self._rows};1H")
+            # disable line wrapping
+            sys.stderr.write("\033[?7l")
+            # reset all attributes
+            sys.stderr.write("\033[0m")
+            # write the display line with prefix, calculated display content, padding, separator, and logo
+            sys.stderr.write(prefix + display + padding + separator + logo + suffix)
+            # enable line wrapping again
+            sys.stderr.write("\033[?7h")
+            # restore cursor position
+            sys.stderr.write("\0338")
+            # flush output to ensure it is displayed
+            sys.stderr.flush()
+
+    def start(self) -> None:
+        t = Thread(target=self.display)
+        t.daemon = True
+        t.start()
+
+
 class MQTTConnection:
     broker = None
     host = None
     prev = -1
     factor = 1
     prefetch_factor_inc = 10
+    retries = 0
 
     def __init__(self, broker: Broker, host: str):
         self.broker = broker
@@ -125,6 +254,7 @@ class MQTTConnection:
                 # message might have not reached agent, resend...
                 self.broker.seek(self.host, disk_id, offset, flength, optimization_strategy)
                 attempts = 0
+                self.retries += 1
 
         return message.data
 
@@ -138,6 +268,8 @@ class Broker:
     mqtt_client = None
     connected = False
     case = None
+    bytes_received = 0
+    monitor = False
 
     diskinfo = {}
     index = {}
@@ -217,6 +349,9 @@ class Broker:
         if casename != self.case:
             return
 
+        if self.monitor:
+            self.bytes_received += len(msg.payload)
+
         if response == "DISKS":
             self._on_disk(hostname, msg.payload)
         elif response == "READ":
@@ -238,9 +373,12 @@ class Broker:
         self.mqtt_client.publish(f"{self.case}/{host}/INFO")
 
     def topology(self, host: str) -> None:
-        self.topo[host] = []
+        if host not in self.topo:
+            self.topo[host] = []
         self.mqtt_client.subscribe(f"{self.case}/{host}/ID")
         time.sleep(1)  # need some time to avoid race condition, i.e. MQTT might react too fast
+        # send a simple clear command (invalid, just clears the prev. msg) just in case TOPO is stale
+        self.mqtt_client.publish(f"{self.case}/{host}/CLR")
         self.mqtt_client.publish(f"{self.case}/{host}/TOPO")
 
     def connect(self) -> None:
@@ -272,6 +410,7 @@ class Broker:
 @arg("--mqtt-crt", dest="crt", help="client certificate file")
 @arg("--mqtt-ca", dest="ca", help="certificate authority file")
 @arg("--mqtt-command", dest="command", help="direct command to client(s)")
+@arg("--mqtt-diag", action="store_true", dest="diag", help="show MQTT diagnostic information")
 class MQTTLoader(Loader):
     """Load remote targets through a broker."""
 
@@ -292,6 +431,7 @@ class MQTTLoader(Loader):
     def find_all(path: Path, **kwargs) -> Iterator[str]:
         cls = MQTTLoader
         num_peers = 1
+
         if cls.broker is None:
             if (uri := kwargs.get("parsed_path")) is None:
                 raise LoaderError("No URI connection details have been passed.")
@@ -299,8 +439,13 @@ class MQTTLoader(Loader):
             cls.broker = Broker(**options)
             cls.broker.connect()
             num_peers = int(options.get("peers", 1))
+            cls.connection = MQTTConnection(cls.broker, path)
+            if options.get("diag", None):
+                cls.broker.monitor = True
+                MQTTDiagnosticLine(cls.connection, num_peers).start()
+        else:
+            cls.connection = MQTTConnection(cls.broker, path)
 
-        cls.connection = MQTTConnection(cls.broker, path)
         cls.peers = cls.connection.topo(num_peers)
         yield from cls.peers
 

dissect/target/loaders/raw.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
 from dissect.target import container
+from dissect.target.exceptions import TargetPathNotFoundError
 from dissect.target.loader import Loader
 from dissect.target.target import Target
 
@@ -8,6 +9,12 @@ from dissect.target.target import Target
 class RawLoader(Loader):
     """Load raw container files such as disk images."""
 
+    def __init__(self, path: Path, **kwargs):
+        if not path.exists():
+            raise TargetPathNotFoundError("Provided target path does not exist")
+
+        super().__init__(path, **kwargs)
+
     @staticmethod
     def detect(path: Path) -> bool:
         return not path.is_dir()

dissect/target/plugins/apps/av/mcafee.py

@@ -71,6 +71,9 @@ class McAfeePlugin(Plugin):
         """Return msc log history records from McAfee.
 
         Yields McAfeeMscLogRecord with the following fields:
+
+        .. code-block:: text
+
             hostname (string): The target hostname.
             domain (string): The target domain.
             ts (datetime): timestamp.

dissect/target/plugins/apps/av/sophos.py

@@ -56,6 +56,9 @@ class SophosPlugin(Plugin):
         """Return alert log records from Sophos Hitman Pro/Alert.
 
         Yields HitmanAlertRecord with the following fields:
+
+        .. code-block:: text
+
             ts (datetime): Timestamp.
             alert (string): Type of Alert.
             description (string): Short description of the alert.
@@ -85,6 +88,9 @@ class SophosPlugin(Plugin):
         """Return log history records from Sophos Home.
 
         Yields SophosLogRecord with the following fields:
+
+        .. code-block:: text
+
             ts (datetime): Timestamp.
             description (string): Short description of the alert.
             path (path): Path to the infected file (if available).

dissect/target/plugins/apps/av/symantec.py

@@ -293,6 +293,9 @@ class SymantecPlugin(Plugin):
         """Return log records.
 
         Yields SEPLogRecord with the following fields:
+
+        .. code-block:: text
+
             ts (datetime): Timestamp associated with the event.
             virus (string): Name of the virus.
             user (string): Name of the user associated with the event.
@@ -326,6 +329,9 @@ class SymantecPlugin(Plugin):
         """Return log firewall records.
 
         Yields SEPFirewallRecord with the following fields:
+
+        .. code-block:: text
+
             ts (datetime): Timestamp associated with the event.
             protocol (string): Protocol name associated with the firewall record.
             local_ip ("net.ipaddress"): Local IP address associated with the event.

dissect/target/plugins/apps/av/trendmicro.py

@@ -1,6 +1,6 @@
 from typing import Iterator
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 from dissect.util.ts import from_unix
 
 from dissect.target import Target
@@ -47,8 +47,7 @@ struct firewall_entry {
     char      _pad3[10];
 };
 """
-c_pfwlog = cstruct.cstruct()
-c_pfwlog.load(pfwlog_def)
+c_pfwlog = cstruct().load(pfwlog_def)
 
 
 class TrendMicroPlugin(Plugin):
@@ -71,6 +70,9 @@ class TrendMicroPlugin(Plugin):
         """Return Trend Micro Worry-free log history records.
 
         Yields TrendMicroWFLogRecord with the following fields:
+
+        .. code-block:: text
+
             hostname (string): The target hostname.
             domain (string): The target domain.
             ts (datetime): timestamp.
@@ -94,6 +96,9 @@ class TrendMicroPlugin(Plugin):
         """Return Trend Micro Worry-free firewall log history records.
 
         Yields TrendMicroWFFirewallRecord with the following fields:
+
+        .. code-block:: text
+
             hostname (string): The target hostname.
             domain (string): The target domain.
             ts (datetime): timestamp.

dissect/target/plugins/apps/browser/chromium.py

@@ -148,6 +148,9 @@ class ChromiumMixin:
             browser_name: The name of the browser as a string.
 
         Yields:
+
+        .. code-block:: text
+
             Records with the following fields:
                 ts (datetime): Visit timestamp.
                 browser (string): The browser from which the records are generated from.
@@ -209,6 +212,9 @@ class ChromiumMixin:
             browser_name: The name of the browser as a string.
 
         Yields:
+
+        .. code-block:: text
+
             Records with the following fields:
                 ts_created (datetime): Cookie created timestamp.
                 ts_last_accessed (datetime): Cookie last accessed timestamp.
@@ -284,6 +290,9 @@ class ChromiumMixin:
             browser_name: The name of the browser as a string.
 
         Yields:
+
+        .. code-block:: text
+
             Records with the following fields:
                 ts_start (datetime): Download start timestamp.
                 ts_end (datetime): Download end timestamp.
@@ -344,6 +353,9 @@ class ChromiumMixin:
             browser_name (str): Name of the browser to scan for extensions.
 
         Yields:
+
+        .. code-block:: text
+
             Records with the following fields:
                 ts_install (datetime): Extension install timestamp.
                 ts_update (datetime): Extension update timestamp.

dissect/target/plugins/apps/browser/firefox.py

@@ -132,6 +132,9 @@ class FirefoxPlugin(BrowserPlugin):
         """Return browser history records from Firefox.
 
         Yields BrowserHistoryRecord with the following fields:
+
+        .. code-block:: text
+
             ts (datetime): Visit timestamp.
             browser (string): The browser from which the records are generated from.
             id (string): Record ID.
@@ -193,6 +196,9 @@ class FirefoxPlugin(BrowserPlugin):
             browser_name: The name of the browser as a string.
 
         Yields:
+
+        .. code-block:: text
+
             Records with the following fields:
                 ts_created (datetime): Cookie created timestamp.
                 ts_last_accessed (datetime): Cookie last accessed timestamp.
@@ -232,6 +238,9 @@ class FirefoxPlugin(BrowserPlugin):
         """Return browser download records from Firefox.
 
         Yields BrowserDownloadRecord with the following fields:
+
+        .. code-block:: text
+
             ts_start (datetime): Download start timestamp.
             ts_end (datetime): Download end timestamp.
             browser (string): The browser from which the records are generated from.
@@ -315,7 +324,10 @@ class FirefoxPlugin(BrowserPlugin):
     def extensions(self) -> Iterator[BrowserExtensionRecord]:
         """Return browser extension records for Firefox.
 
-        Yields BrowserExtensionRecord with the following fields::
+        Yields BrowserExtensionRecord with the following fields:
+
+        .. code-block:: text
+
             ts_install (datetime): Extension install timestamp.
             ts_update (datetime): Extension update timestamp.
             browser (string): The browser from which the records are generated.

dissect/target/plugins/apps/browser/iexplore.py

@@ -131,6 +131,9 @@ class InternetExplorerPlugin(BrowserPlugin):
         """Return browser history records from Internet Explorer.
 
         Yields BrowserHistoryRecord with the following fields:
+
+        .. code-block:: text
+
             ts (datetime): Visit timestamp.
             browser (string): The browser from which the records are generated from.
             id (string): Record ID.
@@ -183,6 +186,9 @@ class InternetExplorerPlugin(BrowserPlugin):
         """Return browser downloads records from Internet Explorer.
 
         Yields BrowserDownloadRecord with the following fields:
+
+        .. code-block:: text
+
             ts_start (datetime): Download start timestamp.
             ts_end (datetime): Download end timestamp.
             browser (string): The browser from which the records are generated from.

dissect/target/plugins/apps/container/docker.py

@@ -88,7 +88,7 @@ struct entry {
 """
 
 c_local = cstruct(endian=">")
-c_local.addtype("varint", ProtobufVarint(c_local, "varint", size=None, signed=False, alignment=1))
+c_local.add_custom_type("varint", ProtobufVarint, size=None, alignment=1, signed=False)
 c_local.load(local_def, compiled=False)
 
 RE_DOCKER_NS = re.compile(r"\.(?P<nanoseconds>\d{7,})(?P<postfix>Z|\+\d{2}:\d{2})")