dissect.target 3.18.dev15__py3-none-any.whl → 3.19__py3-none-any.whl

dissect/target/plugins/filesystem/ntfs/mft.py

@@ -1,12 +1,16 @@
+from __future__ import annotations
+
 import logging
-from typing import Callable
+from typing import Callable, Iterator
 
 from dissect.ntfs.attr import Attribute
 from dissect.ntfs.c_ntfs import FILE_RECORD_SEGMENT_IN_USE
 from dissect.ntfs.mft import MftRecord
+from flow.record import Record
 from flow.record.fieldtypes import windows_path
 
 from dissect.target.exceptions import UnsupportedPluginError
+from dissect.target.filesystems.ntfs import NtfsFilesystem
 from dissect.target.helpers.record import TargetRecordDescriptor
 from dissect.target.plugin import Plugin, arg, export
 from dissect.target.plugins.filesystem.ntfs.utils import (
@@ -53,7 +57,6 @@ FilesystemStdRecord = TargetRecordDescriptor(
     ],
 )
 
-
 FilesystemFilenameCompactRecord = TargetRecordDescriptor(
     "filesystem/ntfs/mft/filename/compact",
     [
@@ -90,6 +93,21 @@ FilesystemFilenameRecord = TargetRecordDescriptor(
     ],
 )
 
+FilesystemMACBRecord = TargetRecordDescriptor(
+    "filesystem/ntfs/mft/macb",
+    [
+        ("datetime", "ts"),
+        ("string", "macb"),
+        ("uint32", "filename_index"),
+        ("uint32", "segment"),
+        ("path", "path"),
+        ("string", "owner"),
+        ("filesize", "filesize"),
+        ("boolean", "resident"),
+        ("boolean", "inuse"),
+        ("string", "volume_uuid"),
+    ],
+)
 
 RECORD_TYPES = {
     InformationType.STANDARD_INFORMATION: FilesystemStdRecord,
@@ -104,9 +122,12 @@ COMPACT_RECORD_TYPES = {
 
 
 class MftPlugin(Plugin):
+    def __init__(self, target):
+        super().__init__(target)
+        self.ntfs_filesystems = {index: fs for index, fs in enumerate(self.target.filesystems) if fs.__type__ == "ntfs"}
+
     def check_compatible(self) -> None:
-        ntfs_filesystems = [fs for fs in self.target.filesystems if fs.__type__ == "ntfs"]
-        if not len(ntfs_filesystems):
+        if not len(self.ntfs_filesystems):
             raise UnsupportedPluginError("No NTFS filesystems found")
 
     @export(
@@ -117,8 +138,24 @@ class MftPlugin(Plugin):
             FilesystemFilenameCompactRecord,
         ]
     )
-    @arg("--compact", action="store_true", help="compacts the MFT entry timestamps into a single record")
-    def mft(self, compact: bool = False):
+    @arg(
+        "--compact",
+        group="fmt",
+        action="store_true",
+        help="compacts the MFT entry timestamps into a single record",
+    )
+    @arg("--fs", type=int, default=None, help="optional filesystem index, zero indexed")
+    @arg("--start", type=int, default=0, help="the first MFT segment number")
+    @arg("--end", type=int, default=-1, help="the last MFT segment number")
+    @arg(
+        "--macb",
+        group="fmt",
+        action="store_true",
+        help="compacts the MFT entry timestamps into aggregated records with MACB bitfield",
+    )
+    def mft(
+        self, compact: bool = False, fs: int | None = None, start: int = 0, end: int = -1, macb: bool = False
+    ) -> Iterator[Record]:
         """Return the MFT records of all NTFS filesystems.
 
         The Master File Table (MFT) contains primarily metadata about every file and folder on a NFTS filesystem.
@@ -133,41 +170,60 @@ class MftPlugin(Plugin):
             - https://docs.microsoft.com/en-us/windows/win32/fileio/master-file-table
         """
 
-        if compact:
-            record_formatter = compacted_formatter
-        else:
-            record_formatter = formatter
+        record_formatter = formatter
 
-        for fs in self.target.filesystems:
-            if fs.__type__ != "ntfs":
-                continue
+        def noaggr(records: list[Record]) -> Iterator[Record]:
+            yield from records
 
-            # If this filesystem is a "fake" NTFS filesystem, used to enhance a
-            # VirtualFilesystem, The driveletter (more accurate mount point)
-            # returned will be that of the VirtualFilesystem. This makes sure
-            # the paths returned in the records are actually reachable.
-            drive_letter = get_drive_letter(self.target, fs)
-            volume_uuid = get_volume_identifier(fs)
+        aggr = noaggr
 
+        if compact:
+            record_formatter = compacted_formatter
+        elif macb:
+            aggr = macb_aggr
+
+        if fs is not None:
             try:
-                for record in fs.ntfs.mft.segments():
-                    try:
-                        inuse = bool(record.header.Flags & FILE_RECORD_SEGMENT_IN_USE)
-                        owner, _ = get_owner_and_group(record, fs)
-                        resident = False
-                        size = None
-
-                        if not record.is_dir():
-                            for data_attribute in record.attributes.DATA:
-                                if data_attribute.name == "":
-                                    resident = data_attribute.resident
-                                    break
-
-                            size = get_record_size(record)
-
-                        for path in record.full_paths():
-                            path = f"{drive_letter}{path}"
-                            yield from self.mft_records(
+                filesystem = self.ntfs_filesystems[fs]
+            except KeyError:
+                self.target.log.error("NTFS filesystem with index number %s does not exist", fs)
+                return
+
+            yield from self.segments(filesystem, record_formatter, aggr, start, end)
+        else:
+            for filesystem in self.ntfs_filesystems.values():
+                yield from self.segments(filesystem, record_formatter, aggr, start, end)
+
+    def segments(
+        self, fs: NtfsFilesystem, record_formatter: Callable, aggr: Callable, start: int, end: int
+    ) -> Iterator[Record]:
+        # If this filesystem is a "fake" NTFS filesystem, used to enhance a
+        # VirtualFilesystem, The driveletter (more accurate mount point)
+        # returned will be that of the VirtualFilesystem. This makes sure
+        # the paths returned in the records are actually reachable.
+        drive_letter = get_drive_letter(self.target, fs)
+        volume_uuid = get_volume_identifier(fs)
+
+        try:
+            for record in fs.ntfs.mft.segments(start, end):
+                try:
+                    inuse = bool(record.header.Flags & FILE_RECORD_SEGMENT_IN_USE)
+                    owner, _ = get_owner_and_group(record, fs)
+                    resident = None
+                    size = None
+
+                    if not record.is_dir():
+                        for data_attribute in record.attributes.DATA:
+                            if data_attribute.name == "":
+                                resident = data_attribute.resident
+                                break
+
+                        size = get_record_size(record)
+
+                    for path in record.full_paths():
+                        path = f"{drive_letter}{path}"
+                        yield from aggr(
+                            self.mft_records(
                                 drive_letter=drive_letter,
                                 record=record,
                                 segment=record.segment,
@@ -179,12 +235,13 @@ class MftPlugin(Plugin):
                                 volume_uuid=volume_uuid,
                                 record_formatter=record_formatter,
                             )
-                    except Exception as e:
-                        self.target.log.warning("An error occured parsing MFT segment %d: %s", record.segment, str(e))
-                        self.target.log.debug("", exc_info=e)
+                        )
+                except Exception as e:
+                    self.target.log.warning("An error occured parsing MFT segment %d: %s", record.segment, str(e))
+                    self.target.log.debug("", exc_info=e)
 
-            except Exception:
-                log.exception("An error occured constructing FilesystemRecords")
+        except Exception:
+            log.exception("An error occured constructing FilesystemRecords")
 
     def mft_records(
         self,
@@ -198,7 +255,7 @@ class MftPlugin(Plugin):
         inuse: bool,
         volume_uuid: str,
         record_formatter: Callable,
-    ):
+    ) -> Iterator[Record]:
         for attr in record.attributes.STANDARD_INFORMATION:
             yield from record_formatter(
                 attr=attr,
@@ -255,7 +312,7 @@ class MftPlugin(Plugin):
             )
 
 
-def compacted_formatter(attr: Attribute, record_type: InformationType, **kwargs):
+def compacted_formatter(attr: Attribute, record_type: InformationType, **kwargs) -> Iterator[Record]:
     record_desc = COMPACT_RECORD_TYPES.get(record_type)
     yield record_desc(
         creation_time=attr.creation_time,
@@ -266,7 +323,7 @@ def compacted_formatter(attr: Attribute, record_type: InformationType, **kwargs)
     )
 
 
-def formatter(attr: Attribute, record_type: InformationType, **kwargs):
+def formatter(attr: Attribute, record_type: InformationType, **kwargs) -> Iterator[Record]:
     record_desc = RECORD_TYPES.get(record_type)
     for type, timestamp in [
         ("B", attr.creation_time),
@@ -275,3 +332,33 @@ def formatter(attr: Attribute, record_type: InformationType, **kwargs):
         ("A", attr.last_access_time),
     ]:
         yield record_desc(ts=timestamp, ts_type=type, **kwargs)
+
+
+def macb_aggr(records: list[Record]) -> Iterator[Record]:
+    def macb_set(bitfield, index, letter):
+        return bitfield[:index] + letter + bitfield[index + 1 :]
+
+    macbs = []
+    for record in records:
+        found = False
+
+        offset_std = int(record._desc.name == "filesystem/ntfs/mft/std") * 5
+        offset_ads = (int(record.ads) * 10) if offset_std == 0 else 0
+
+        field = "MACB".find(record.ts_type) + offset_std + offset_ads
+        for macb in macbs:
+            if macb.ts == record.ts:
+                macb.macb = macb_set(macb.macb, field, record.ts_type)
+                found = True
+                break
+
+        if found:
+            continue
+
+        macb = FilesystemMACBRecord.init_from_record(record)
+        macb.macb = "..../..../...."
+        macb.macb = macb_set(macb.macb, field, record.ts_type)
+
+        macbs.append(macb)
+
+    yield from macbs

dissect/target/plugins/filesystem/unix/capability.py

@@ -1,20 +1,20 @@
-import struct
 from enum import IntEnum
 from io import BytesIO
+from typing import Iterator
 
-from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
+from dissect.target.exceptions import UnsupportedPluginError
 from dissect.target.helpers.record import TargetRecordDescriptor
 from dissect.target.plugin import Plugin, export
-from dissect.target.plugins.filesystem.walkfs import generate_record
 
 CapabilityRecord = TargetRecordDescriptor(
     "filesystem/unix/capability",
     [
-        ("record", "record"),
+        ("datetime", "ts_mtime"),
+        ("path", "path"),
         ("string[]", "permitted"),
         ("string[]", "inheritable"),
         ("boolean", "effective"),
-        ("uint32", "rootid"),
+        ("uint32", "root_id"),
     ],
 )
 
@@ -82,88 +82,103 @@ class CapabilityPlugin(Plugin):
     """Plugin to yield files with capabilites set."""
 
     def check_compatible(self) -> None:
-        if not self.target.has_function("walkfs") or self.target.os == "windows":
-            raise UnsupportedPluginError("Unsupported plugin")
+        if not self.target.has_function("walkfs"):
+            raise UnsupportedPluginError("Need walkfs plugin")
+
+        if not any(fs.__type__ in ("extfs", "xfs") for fs in self.target.filesystems):
+            raise UnsupportedPluginError("Capability plugin only works on EXT and XFS filesystems")
 
     @export(record=CapabilityRecord)
-    def capability_binaries(self):
-        """Find all files that have capabilities set."""
-        for path_entries, _, files in self.target.fs.walk_ext("/"):
-            entries = [path_entries[-1]] + files
-            for entry in entries:
-                path = self.target.fs.path(entry.path)
-                try:
-                    record = generate_record(self.target, path)
-                except FileNotFoundError:
-                    continue
+    def capability_binaries(self) -> Iterator[CapabilityRecord]:
+        """Find all files that have capabilities set on files.
+
+        Resources:
+            - https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h
+        """
+
+        for entry in self.target.fs.recurse("/"):
+            if not entry.is_file() or entry.is_symlink():
+                continue
+
+            try:
+                attrs = [attr for attr in entry.lattr() if attr.name == "security.capability"]
+            except Exception as e:
+                self.target.log.warning("Failed to get attrs for entry %s", entry)
+                self.target.log.debug("", exc_info=e)
+                continue
+
+            for attr in attrs:
                 try:
-                    attrs = path.get().lattr()
-                except TypeError:
-                    # Virtual(File|Directory|Symlink) instances don't have a functional lattr()
-                    continue
-                except Exception:
-                    self.target.log.exception("Failed to get attrs for entry %s", entry)
-                    continue
-
-                for attr in attrs:
-                    if attr.name != "security.capability":
-                        continue
-
-                    buf = BytesIO(attr.value)
-
-                    # Reference: https://github.com/torvalds/linux/blob/master/include/uapi/linux/capability.h
-                    # The struct is small enough we can just use struct
-                    magic_etc = struct.unpack("<I", buf.read(4))[0]
-                    cap_revision = magic_etc & VFS_CAP_REVISION_MASK
-
-                    permitted_caps = []
-                    inheritable_caps = []
-                    rootid = None
-
-                    if cap_revision == VFS_CAP_REVISION_1:
-                        num_caps = VFS_CAP_U32_1
-                        data_len = (1 + 2 * VFS_CAP_U32_1) * 4
-                    elif cap_revision == VFS_CAP_REVISION_2:
-                        num_caps = VFS_CAP_U32_2
-                        data_len = (1 + 2 * VFS_CAP_U32_2) * 4
-                    elif cap_revision == VFS_CAP_REVISION_3:
-                        num_caps = VFS_CAP_U32_3
-                        data_len = (2 + 2 * VFS_CAP_U32_2) * 4
-                    else:
-                        self.target.log.error("Unexpected capability revision: %s", entry)
-                        continue
-
-                    if data_len != len(attr.value):
-                        self.target.log.error("Unexpected capability length: %s", entry)
-                        continue
-
-                    for _ in range(num_caps):
-                        permitted_val, inheritable_val = struct.unpack("<2I", buf.read(8))
-                        permitted_caps.append(permitted_val)
-                        inheritable_caps.append(inheritable_val)
-
-                    if cap_revision == VFS_CAP_REVISION_3:
-                        rootid = struct.unpack("<I", buf.read(4))[0]
-
-                    permitted = []
-                    inheritable = []
-
-                    for capability in Capabilities:
-                        for caps, results in [(permitted_caps, permitted), (inheritable_caps, inheritable)]:
-                            # CAP_TO_INDEX
-                            cap_index = capability.value >> 5
-                            if cap_index >= len(caps):
-                                # We loop over all capabilities, but might only have a version 1 caps list
-                                continue
-
-                            if caps[cap_index] & (1 << (capability.value & 31)) != 0:
-                                results.append(capability.name)
-
-                    yield CapabilityRecord(
-                        record=record,
-                        permitted=permitted,
-                        inheritable=inheritable,
-                        effective=magic_etc & VFS_CAP_FLAGS_EFFECTIVE != 0,
-                        rootid=rootid,
-                        _target=self.target,
-                    )
+                    permitted, inheritable, effective, root_id = parse_attr(attr.value)
+                except ValueError as e:
+                    self.target.log.warning("Could not parse attributes for entry %s: %s", entry, str(e.value))
+                    self.target.log.debug("", exc_info=e)
+
+                yield CapabilityRecord(
+                    ts_mtime=entry.lstat().st_mtime,
+                    path=self.target.fs.path(entry.path),
+                    permitted=permitted,
+                    inheritable=inheritable,
+                    effective=effective,
+                    root_id=root_id,
+                    _target=self.target,
+                )
+
+
+def parse_attr(attr: bytes) -> tuple[list[str], list[str], bool, int]:
+    """Efficiently parse a Linux xattr capability struct.
+
+    Returns:
+        A tuple of permitted capability names, inheritable capability names, effective flag and ``root_id``.
+    """
+    buf = BytesIO(attr)
+
+    # The struct is small enough we can just use int.from_bytes
+    magic_etc = int.from_bytes(buf.read(4), "little")
+    effective = magic_etc & VFS_CAP_FLAGS_EFFECTIVE != 0
+    cap_revision = magic_etc & VFS_CAP_REVISION_MASK
+
+    permitted_caps = []
+    inheritable_caps = []
+    root_id = None
+
+    if cap_revision == VFS_CAP_REVISION_1:
+        num_caps = VFS_CAP_U32_1
+        data_len = (1 + 2 * VFS_CAP_U32_1) * 4
+
+    elif cap_revision == VFS_CAP_REVISION_2:
+        num_caps = VFS_CAP_U32_2
+        data_len = (1 + 2 * VFS_CAP_U32_2) * 4
+
+    elif cap_revision == VFS_CAP_REVISION_3:
+        num_caps = VFS_CAP_U32_3
+        data_len = (2 + 2 * VFS_CAP_U32_2) * 4
+
+    else:
+        raise ValueError("Unexpected capability revision '%s'" % cap_revision)
+
+    if data_len != (actual_len := len(attr)):
+        raise ValueError("Unexpected capability length (%s vs %s)", data_len, actual_len)
+
+    for _ in range(num_caps):
+        permitted_caps.append(int.from_bytes(buf.read(4), "little"))
+        inheritable_caps.append(int.from_bytes(buf.read(4), "little"))
+
+    if cap_revision == VFS_CAP_REVISION_3:
+        root_id = int.from_bytes(buf.read(4), "little")
+
+    permitted = []
+    inheritable = []
+
+    for capability in Capabilities:
+        for caps, results in [(permitted_caps, permitted), (inheritable_caps, inheritable)]:
+            # CAP_TO_INDEX
+            cap_index = capability.value >> 5
+            if cap_index >= len(caps):
+                # We loop over all capabilities, but might only have a version 1 caps list
+                continue
+
+            if caps[cap_index] & (1 << (capability.value & 31)) != 0:
+                results.append(capability.name)
+
+    return permitted, inheritable, effective, root_id

dissect/target/plugins/filesystem/walkfs.py

@@ -1,12 +1,11 @@
-from typing import Iterable
+from typing import Iterator
 
 from dissect.util.ts import from_unix
 
 from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
-from dissect.target.filesystem import LayerFilesystemEntry
-from dissect.target.helpers.fsutil import TargetPath
+from dissect.target.filesystem import FilesystemEntry, LayerFilesystemEntry
 from dissect.target.helpers.record import TargetRecordDescriptor
-from dissect.target.plugin import Plugin, export
+from dissect.target.plugin import Plugin, arg, export
 from dissect.target.target import Target
 
 FilesystemRecord = TargetRecordDescriptor(
@@ -30,37 +29,49 @@ FilesystemRecord = TargetRecordDescriptor(
 class WalkFSPlugin(Plugin):
     def check_compatible(self) -> None:
         if not len(self.target.filesystems):
-            raise UnsupportedPluginError("No filesystems found")
+            raise UnsupportedPluginError("No filesystems to walk")
 
     @export(record=FilesystemRecord)
-    def walkfs(self) -> Iterable[FilesystemRecord]:
+    @arg("--walkfs-path", default="/", help="path to recursively walk")
+    def walkfs(self, walkfs_path: str = "/") -> Iterator[FilesystemRecord]:
         """Walk a target's filesystem and return all filesystem entries."""
-        for path_entries, _, files in self.target.fs.walk_ext("/"):
-            entries = [path_entries[-1]] + files
-            for entry in entries:
-                path = self.target.fs.path(entry.path)
-                try:
-                    record = generate_record(self.target, path)
-                except FileNotFoundError:
-                    continue
-                yield record
+        for entry in self.target.fs.recurse(walkfs_path):
+            try:
+                yield generate_record(self.target, entry)
 
+            except FileNotFoundError as e:
+                self.target.log.warning("File not found: %s", entry)
+                self.target.log.debug("", exc_info=e)
+            except Exception as e:
+                self.target.log.warning("Exception generating record for: %s", entry)
+                self.target.log.debug("", exc_info=e)
+                continue
+
+
+def generate_record(target: Target, entry: FilesystemEntry) -> FilesystemRecord:
+    """Generate a :class:`FilesystemRecord` from the given :class:`FilesystemEntry`.
+
+    Args:
+        target: :class:`Target` instance
+        entry: :class:`FilesystemEntry` instance
+
+    Returns:
+        Generated :class:`FilesystemRecord` for the given :class:`FilesystemEntry`.
+    """
+    stat = entry.lstat()
 
-def generate_record(target: Target, path: TargetPath) -> FilesystemRecord:
-    stat = path.lstat()
-    btime = from_unix(stat.st_birthtime) if stat.st_birthtime else None
-    entry = path.get()
     if isinstance(entry, LayerFilesystemEntry):
         fs_types = [sub_entry.fs.__type__ for sub_entry in entry.entries]
     else:
         fs_types = [entry.fs.__type__]
+
     return FilesystemRecord(
         atime=from_unix(stat.st_atime),
         mtime=from_unix(stat.st_mtime),
         ctime=from_unix(stat.st_ctime),
-        btime=btime,
+        btime=from_unix(stat.st_birthtime) if stat.st_birthtime else None,
         ino=stat.st_ino,
-        path=path,
+        path=entry.path,
         size=stat.st_size,
         mode=stat.st_mode,
         uid=stat.st_uid,