dissect.target 3.18.dev5__py3-none-any.whl → 3.18.dev7__py3-none-any.whl

dissect/target/helpers/protobuf.py

@@ -3,30 +3,26 @@ from __future__ import annotations
 from typing import Any, BinaryIO
 
 from dissect.cstruct.types.base import BaseType
-from dissect.cstruct.types.bytesinteger import BytesInteger
 
 
-class ProtobufVarint(BytesInteger):
+class ProtobufVarint(BaseType):
     """Implements a protobuf integer type for dissect.cstruct that can span a variable amount of bytes.
 
-    Mainly follows the cstruct BytesInteger implementation with minor tweaks
-    to support protobuf's msb varint implementation.
+    Supports protobuf's msb varint implementation.
 
     Resources:
         - https://protobuf.dev/programming-guides/encoding/
         - https://github.com/protocolbuffers/protobuf/blob/main/python/google/protobuf/internal/decoder.py
     """
 
-    def _read(self, stream: BinaryIO, context: dict[str, Any] = None) -> int:
+    @classmethod
+    def _read(cls, stream: BinaryIO, context: dict[str, Any] = None) -> int:
         return decode_varint(stream)
 
-    def _write(self, stream: BinaryIO, data: int) -> int:
+    @classmethod
+    def _write(cls, stream: BinaryIO, data: int) -> int:
         return stream.write(encode_varint(data))
 
-    _read_array = BaseType._read_array
-
-    _write_array = BaseType._write_array
-
 
 def decode_varint(stream: BinaryIO) -> int:
     """Reads a varint from the provided buffer stream.

dissect/target/helpers/ssh.py

@@ -1,9 +1,9 @@
 import base64
 import binascii
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 
-c_rfc4716_def = """
+rfc4716_def = """
 struct ssh_string {
     uint32 length;
     char value[length];
@@ -23,8 +23,7 @@ struct ssh_private_key {
 }
 """
 
-c_rfc4716 = cstruct.cstruct(endian=">")
-c_rfc4716.load(c_rfc4716_def)
+c_rfc4716 = cstruct(endian=">").load(rfc4716_def)
 
 RFC4716_MARKER_START = b"-----BEGIN OPENSSH PRIVATE KEY-----"
 RFC4716_MARKER_END = b"-----END OPENSSH PRIVATE KEY-----"

dissect/target/loaders/mqtt.py

@@ -1,13 +1,18 @@
 from __future__ import annotations
 
+import atexit
 import logging
+import math
+import os
 import ssl
+import sys
 import time
 import urllib
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
 from struct import pack, unpack_from
+from threading import Thread
 from typing import Any, Callable, Iterator, Optional, Union
 
 import paho.mqtt.client as mqtt
@@ -51,6 +56,34 @@ class SeekMessage:
     data: bytes = b""
 
 
+class MQTTTransferRatePerSecond:
+    def __init__(self, window_size: int = 10):
+        self.window_size = window_size
+        self.timestamps = []
+        self.bytes = []
+
+    def record(self, timestamp: float, byte_count: int) -> MQTTTransferRatePerSecond:
+        while self.timestamps and (timestamp - self.timestamps[0] > self.window_size):
+            self.timestamps.pop(0)
+            self.bytes.pop(0)
+
+        self.timestamps.append(timestamp)
+        self.bytes.append(byte_count)
+        return self
+
+    def value(self, current_time: float) -> float:
+        if not self.timestamps:
+            return 0
+
+        elapsed_time = current_time - self.timestamps[0]
+        if elapsed_time == 0:
+            return 0
+
+        total_bytes = self.bytes[-1] - self.bytes[0]
+
+        return total_bytes / elapsed_time
+
+
 class MQTTStream(AlignedStream):
     def __init__(self, stream: MQTTConnection, disk_id: int, size: Optional[int] = None):
         self.stream = stream
@@ -62,12 +95,108 @@ class MQTTStream(AlignedStream):
         return data
 
 
+class MQTTDiagnosticLine:
+    def __init__(self, connection: MQTTConnection, total_peers: int):
+        self.connection = connection
+        self.total_peers = total_peers
+        self._columns, self._rows = os.get_terminal_size(0)
+        atexit.register(self._detach)
+        self._attach()
+
+    def _attach(self) -> None:
+        # save cursor position
+        sys.stderr.write("\0337")
+        # set top and bottom margins of the scrolling region to default
+        sys.stderr.write("\033[r")
+        # restore cursor position
+        sys.stderr.write("\0338")
+        # move cursor down one line in the same column; if at the bottom, the screen scrolls up
+        sys.stderr.write("\033D")
+        # move cursor up one line in the same column; if at the top, screen scrolls down
+        sys.stderr.write("\033M")
+        # save cursor position again
+        sys.stderr.write("\0337")
+        # restrict scrolling to a region from the first line to one before the last line
+        sys.stderr.write(f"\033[1;{self._rows - 1}r")
+        # restore cursor position after setting scrolling region
+        sys.stderr.write("\0338")
+
+    def _detach(self) -> None:
+        # save cursor position
+        sys.stderr.write("\0337")
+        # move cursor to the specified position (last line, first column)
+        sys.stderr.write(f"\033[{self._rows};1H")
+        # clear from cursor to end of the line
+        sys.stderr.write("\033[K")
+        # reset scrolling region to include the entire display
+        sys.stderr.write("\033[r")
+        # restore cursor position
+        sys.stderr.write("\0338")
+        # ensure the written content is displayed (flush output)
+        sys.stderr.flush()
+
+    def display(self) -> None:
+        # prepare: set background color to blue and text color to white at the beginning of the line
+        prefix = "\x1b[44m\x1b[37m\r"
+        # reset all attributes (colors, styles) to their defaults afterwards
+        suffix = "\x1b[0m"
+        # separator to set background color to red and text style to bold
+        separator = "\x1b[41m\x1b[1m"
+        logo = "TARGETD"
+
+        start = time.time()
+        transfer_rate = MQTTTransferRatePerSecond(window_size=7)
+
+        while True:
+            time.sleep(0.05)
+            peers = "?"
+            try:
+                peers = len(self.connection.broker.peers(self.connection.host))
+            except Exception:
+                pass
+
+            recv = self.connection.broker.bytes_received
+            now = time.time()
+            transfer = transfer_rate.record(now, recv).value(now) / 1000  # convert to KB/s
+            failures = self.connection.retries
+            seconds_elapsed = round(now - start) % 60
+            minutes_elapsed = math.floor((now - start) / 60) % 60
+            hours_elapsed = math.floor((now - start) / 60**2)
+            timer = f"{hours_elapsed:02d}:{minutes_elapsed:02d}:{seconds_elapsed:02d}"
+            display = f"{timer} {peers}/{self.total_peers} peers {transfer:>8.2f} KB p/s {failures:>4} failures"
+            rest = self._columns - len(display)
+            padding = (rest - len(logo)) * " "
+
+            # save cursor position
+            sys.stderr.write("\0337")
+            # move cursor to specified position (last line, first column)
+            sys.stderr.write(f"\033[{self._rows};1H")
+            # disable line wrapping
+            sys.stderr.write("\033[?7l")
+            # reset all attributes
+            sys.stderr.write("\033[0m")
+            # write the display line with prefix, calculated display content, padding, separator, and logo
+            sys.stderr.write(prefix + display + padding + separator + logo + suffix)
+            # enable line wrapping again
+            sys.stderr.write("\033[?7h")
+            # restore cursor position
+            sys.stderr.write("\0338")
+            # flush output to ensure it is displayed
+            sys.stderr.flush()
+
+    def start(self) -> None:
+        t = Thread(target=self.display)
+        t.daemon = True
+        t.start()
+
+
 class MQTTConnection:
     broker = None
     host = None
     prev = -1
     factor = 1
     prefetch_factor_inc = 10
+    retries = 0
 
     def __init__(self, broker: Broker, host: str):
         self.broker = broker
@@ -125,6 +254,7 @@ class MQTTConnection:
                 # message might have not reached agent, resend...
                 self.broker.seek(self.host, disk_id, offset, flength, optimization_strategy)
                 attempts = 0
+                self.retries += 1
 
         return message.data
 
@@ -138,6 +268,8 @@ class Broker:
     mqtt_client = None
     connected = False
     case = None
+    bytes_received = 0
+    monitor = False
 
     diskinfo = {}
     index = {}
@@ -217,6 +349,9 @@ class Broker:
         if casename != self.case:
             return
 
+        if self.monitor:
+            self.bytes_received += len(msg.payload)
+
         if response == "DISKS":
             self._on_disk(hostname, msg.payload)
         elif response == "READ":
@@ -238,9 +373,12 @@ class Broker:
         self.mqtt_client.publish(f"{self.case}/{host}/INFO")
 
     def topology(self, host: str) -> None:
-        self.topo[host] = []
+        if host not in self.topo:
+            self.topo[host] = []
         self.mqtt_client.subscribe(f"{self.case}/{host}/ID")
         time.sleep(1)  # need some time to avoid race condition, i.e. MQTT might react too fast
+        # send a simple clear command (invalid, just clears the prev. msg) just in case TOPO is stale
+        self.mqtt_client.publish(f"{self.case}/{host}/CLR")
         self.mqtt_client.publish(f"{self.case}/{host}/TOPO")
 
     def connect(self) -> None:
@@ -272,6 +410,7 @@ class Broker:
 @arg("--mqtt-crt", dest="crt", help="client certificate file")
 @arg("--mqtt-ca", dest="ca", help="certificate authority file")
 @arg("--mqtt-command", dest="command", help="direct command to client(s)")
+@arg("--mqtt-diag", action="store_true", dest="diag", help="show MQTT diagnostic information")
 class MQTTLoader(Loader):
     """Load remote targets through a broker."""
 
@@ -292,6 +431,7 @@ class MQTTLoader(Loader):
     def find_all(path: Path, **kwargs) -> Iterator[str]:
         cls = MQTTLoader
         num_peers = 1
+
         if cls.broker is None:
             if (uri := kwargs.get("parsed_path")) is None:
                 raise LoaderError("No URI connection details have been passed.")
@@ -299,8 +439,13 @@ class MQTTLoader(Loader):
             cls.broker = Broker(**options)
             cls.broker.connect()
             num_peers = int(options.get("peers", 1))
+            cls.connection = MQTTConnection(cls.broker, path)
+            if options.get("diag", None):
+                cls.broker.monitor = True
+                MQTTDiagnosticLine(cls.connection, num_peers).start()
+        else:
+            cls.connection = MQTTConnection(cls.broker, path)
 
-        cls.connection = MQTTConnection(cls.broker, path)
         cls.peers = cls.connection.topo(num_peers)
         yield from cls.peers
 

dissect/target/plugins/apps/av/trendmicro.py

@@ -1,6 +1,6 @@
 from typing import Iterator
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 from dissect.util.ts import from_unix
 
 from dissect.target import Target
@@ -47,8 +47,7 @@ struct firewall_entry {
     char      _pad3[10];
 };
 """
-c_pfwlog = cstruct.cstruct()
-c_pfwlog.load(pfwlog_def)
+c_pfwlog = cstruct().load(pfwlog_def)
 
 
 class TrendMicroPlugin(Plugin):

dissect/target/plugins/apps/container/docker.py

@@ -88,7 +88,7 @@ struct entry {
 """
 
 c_local = cstruct(endian=">")
-c_local.addtype("varint", ProtobufVarint(c_local, "varint", size=None, signed=False, alignment=1))
+c_local.add_custom_type("varint", ProtobufVarint, size=None, alignment=1, signed=False)
 c_local.load(local_def, compiled=False)
 
 RE_DOCKER_NS = re.compile(r"\.(?P<nanoseconds>\d{7,})(?P<postfix>Z|\+\d{2}:\d{2})")

dissect/target/plugins/os/unix/locate/gnulocate.py

@@ -26,8 +26,7 @@ GNULocateRecord = TargetRecordDescriptor(
     ],
 )
 
-c_gnulocate = cstruct()
-c_gnulocate.load(gnulocate_def)
+c_gnulocate = cstruct().load(gnulocate_def)
 
 
 class GNULocateFile:

dissect/target/plugins/os/unix/locate/mlocate.py

@@ -20,10 +20,10 @@ struct header_config {
     int32 conf_size;
     int8 version;                            /* file format version */
     int8 require_visibility;
-    int8 pad[2];                             /* 32-bit total alignment */
+    int8 pad0[2];                             /* 32-bit total alignment */
     char root_database;
     char config_block[conf_size];
-    int8 pad;
+    int8 pad1;
 };
 
 enum DBE_TYPE: uint8 {                       /* database entry type */
@@ -68,8 +68,7 @@ MLocateRecord = TargetRecordDescriptor(
     ],
 )
 
-c_mlocate = cstruct(endian=">")
-c_mlocate.load(mlocate_def)
+c_mlocate = cstruct(endian=">").load(mlocate_def)
 
 
 class MLocateFile:

dissect/target/plugins/os/unix/locate/plocate.py

@@ -65,8 +65,7 @@ PLocateRecord = TargetRecordDescriptor(
     ],
 )
 
-c_plocate = cstruct()
-c_plocate.load(plocate_def)
+c_plocate = cstruct().load(plocate_def)
 
 
 class PLocateFile:

dissect/target/plugins/os/unix/log/atop.py

@@ -2,7 +2,7 @@ import zlib
 from io import BytesIO
 from typing import BinaryIO, Iterator
 
-from dissect.cstruct import Instance, cstruct
+from dissect.cstruct import cstruct
 
 from dissect.target.exceptions import UnsupportedPluginError
 from dissect.target.helpers.record import TargetRecordDescriptor
@@ -178,8 +178,7 @@ struct tstat {
 };
 """  # noqa: E501
 
-c_atop = cstruct()
-c_atop.load(atop_def)
+c_atop = cstruct().load(atop_def)
 c_atop.load(atop_tstat_def, align=True)
 
 AtopRecord = TargetRecordDescriptor(
@@ -226,7 +225,7 @@ class AtopFile:
         self.header = c_atop.rawheader(self.fh)
         self.version = self.version()
 
-    def __iter__(self) -> Iterator[Instance]:
+    def __iter__(self) -> Iterator[c_atop.tstat]:
         while True:
             try:
                 record = c_atop.rawrecord(self.fh)

dissect/target/plugins/os/unix/log/journal.py

@@ -1,8 +1,10 @@
+from __future__ import annotations
+
 import lzma
 from typing import BinaryIO, Callable, Iterator
 
 import zstandard
-from dissect.cstruct import Instance, cstruct
+from dissect.cstruct import cstruct
 from dissect.util import ts
 from dissect.util.compression import lz4
 
@@ -252,8 +254,7 @@ struct EntryArrayObject_Compact {
 };
 """  # noqa: E501
 
-c_journal = cstruct()
-c_journal.load(journal_def)
+c_journal = cstruct().load(journal_def)
 
 
 def get_optional(value: str, to_type: Callable):
@@ -314,7 +315,7 @@ class JournalFile:
 
         return key, value
 
-    def __iter__(self) -> Iterator[Instance]:
+    def __iter__(self) -> Iterator[dict[str, int | str]]:
         "Iterate over the entry objects to read payloads."
 
         for offset in self.entry_object_offsets():

dissect/target/plugins/os/unix/log/lastlog.py

@@ -1,6 +1,6 @@
 from typing import BinaryIO
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 from dissect.util import ts
 
 from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
@@ -36,8 +36,7 @@ struct entry {
 };
 """
 
-c_lastlog = cstruct.cstruct()
-c_lastlog.load(lastlog_def)
+c_lastlog = cstruct().load(lastlog_def)
 
 
 class LastLogFile:

dissect/target/plugins/os/unix/log/utmp.py

@@ -39,14 +39,14 @@ WtmpRecord = TargetRecordDescriptor(
     ],
 )
 
-c_utmp = """
+utmp_def = """
 #define UT_LINESIZE     32
 #define UT_NAMESIZE     32
 #define UT_HOSTSIZE     256
 
 typedef uint32 pid_t;
 
-enum Type : char {
+enum Type : uint8_t {
     EMPTY           = 0x0,
     RUN_LVL         = 0x1,
     BOOT_TIME       = 0x2,
@@ -84,8 +84,7 @@ struct entry {
 };
 """  # noqa: E501
 
-utmp = cstruct()
-utmp.load(c_utmp)
+c_utmp = cstruct().load(utmp_def)
 
 UTMP_ENTRY = namedtuple(
     "UTMPRecord",
@@ -122,11 +121,11 @@ class UtmpFile:
 
         while True:
             try:
-                entry = utmp.entry(byte_stream)
+                entry = c_utmp.entry(byte_stream)
 
                 r_type = ""
-                if entry.ut_type in utmp.Type.reverse:
-                    r_type = utmp.Type.reverse[entry.ut_type]
+                if entry.ut_type in c_utmp.Type:
+                    r_type = c_utmp.Type(entry.ut_type).name
 
                 ut_host = entry.ut_host.decode(errors="surrogateescape").strip("\x00")
                 ut_addr = None

dissect/target/plugins/os/windows/adpolicy.py

@@ -1,7 +1,7 @@
 from struct import unpack
 
 from defusedxml import ElementTree
-from dissect import cstruct
+from dissect.cstruct import cstruct
 from dissect.regf.c_regf import (
     REG_BINARY,
     REG_DWORD,
@@ -18,14 +18,13 @@ from dissect.target.exceptions import UnsupportedPluginError
 from dissect.target.helpers.record import TargetRecordDescriptor
 from dissect.target.plugin import Plugin, export
 
-c_def = """
+policy_def = """
 struct registry_policy_header {
     uint32   signature;
     uint32   version;
 };
 """
-c_adpolicy = cstruct.cstruct()
-c_adpolicy.load(c_def)
+c_adpolicy = cstruct().load(policy_def)
 
 ADPolicyRecord = TargetRecordDescriptor(
     "windows/adpolicy",

dissect/target/plugins/os/windows/credhist.py

@@ -53,8 +53,7 @@ struct entry {
 };
 """
 
-c_credhist = cstruct()
-c_credhist.load(credhist_def)
+c_credhist = cstruct().load(credhist_def)
 
 
 @dataclass

dissect/target/plugins/os/windows/datetime.py

@@ -3,7 +3,7 @@ from collections import namedtuple
 from datetime import datetime, timedelta, timezone, tzinfo
 from typing import Dict, Tuple
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 
 from dissect.target.exceptions import (
     RegistryError,
@@ -34,8 +34,7 @@ typedef struct _REG_TZI_FORMAT {
     SYSTEMTIME DaylightDate;
 } REG_TZI_FORMAT;
 """
-c_tz = cstruct.cstruct()
-c_tz.load(tz_def)
+c_tz = cstruct().load(tz_def)
 
 
 # Althoug calendar.SUNDAY is only officially documented since Python 3.10, it
@@ -63,7 +62,7 @@ ZERO = timedelta(0)
 HOUR = timedelta(hours=1)
 
 
-def parse_systemtime_transition(systemtime: cstruct.Instance, year: int) -> datetime:
+def parse_systemtime_transition(systemtime: c_tz._SYSTEMTIME, year: int) -> datetime:
     """Return the transition datetime for a given year using the SYSTEMTIME of a STD or DST transition date.
 
     The SYSTEMTIME date of a TZI structure needs to be used to calculate the actual date for a given year.

dissect/target/plugins/os/windows/defender.py

@@ -237,8 +237,7 @@ struct QuarantineEntryResourceField {
 };
 """
 
-c_defender = cstruct()
-c_defender.load(defender_def)
+c_defender = cstruct().load(defender_def)
 
 STREAM_ID = c_defender.STREAM_ID
 STREAM_ATTRIBUTES = c_defender.STREAM_ATTRIBUTES
@@ -381,7 +380,7 @@ class QuarantineEntryResource:
             self.last_access_time = ts.wintimestamp(int.from_bytes(field.Data, "little"))
         elif field.Identifier == FIELD_IDENTIFIER.LastWriteTime:
             self.last_write_time = ts.wintimestamp(int.from_bytes(field.Data, "little"))
-        elif field.Identifier not in FIELD_IDENTIFIER.values.values():
+        elif field.Identifier not in FIELD_IDENTIFIER:
             self.unknown_fields.append(field)
 
 
@@ -526,7 +525,7 @@ class MicrosoftDefenderPlugin(plugin.Plugin):
                     subdir = resource.resource_id[0:2]
                     resourcedata_location = resourcedata_directory.joinpath(subdir).joinpath(resource.resource_id)
                     if not resourcedata_location.exists():
-                        self.target.log.warning(f"Could not find a ResourceData file for {entry.resource_id}.")
+                        self.target.log.warning(f"Could not find a ResourceData file for {resource.resource_id}.")
                         continue
                     if not resourcedata_location.is_file():
                         self.target.log.warning(f"{resourcedata_location} is not a file!")

dissect/target/plugins/os/windows/dpapi/blob.py

@@ -36,8 +36,7 @@ struct DPAPIBlob {
 };
 """
 
-c_blob = cstruct()
-c_blob.load(blob_def)
+c_blob = cstruct().load(blob_def)
 
 
 class Blob:

dissect/target/plugins/os/windows/dpapi/master_key.py

@@ -29,7 +29,7 @@ struct DomainKey {
     DWORD   accessCheckLen;
     char    guid[16];
     char    encryptedSecret[secretLen];
-    char    accessCheckLen[accessCheckLen];
+    char    accessCheck[accessCheckLen];
 };
 
 struct CredHist {
@@ -66,8 +66,7 @@ struct MasterKeyFileHeader {
     QWORD   qwDomainKeySize;
 };
 """
-c_master_key = cstruct()
-c_master_key.load(master_key_def)
+c_master_key = cstruct().load(master_key_def)
 
 
 class MasterKey:

dissect/target/plugins/os/windows/notifications.py

@@ -91,8 +91,7 @@ typedef struct {
 } Chunk;                               // size: 0x23810
 """
 
-c_appdb = cstruct(endian="<")
-c_appdb.load(appdb_def)
+c_appdb = cstruct(endian="<").load(appdb_def)
 
 APPDB_MAGIC = b"DNPW"
 NUM_APPDB_CHUNKS = 256