acquire 3.14.dev7__tar.gz → 3.14.dev9__tar.gz

{acquire-3.14.dev7/acquire.egg-info → acquire-3.14.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: acquire
-Version: 3.14.dev7
+Version: 3.14.dev9
 Summary: A tool to quickly gather forensic artifacts from disk images or a live system into a lightweight container
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3

{acquire-3.14.dev7 → acquire-3.14.dev9}/acquire/acquire.py

@@ -1,9 +1,10 @@
+from __future__ import annotations
+
 import argparse
 import enum
 import functools
 import io
 import itertools
-import json
 import logging
 import os
 import platform
@@ -16,16 +17,15 @@ import urllib.request
 from collections import defaultdict, namedtuple
 from itertools import product
 from pathlib import Path
-from typing import Iterator, Optional, Union
+from typing import BinaryIO, Callable, Iterator, Optional, Union
 
-from dissect.target import Target, exceptions
+from dissect.target import Target
 from dissect.target.filesystem import Filesystem
 from dissect.target.filesystems import ntfs
 from dissect.target.helpers import fsutil
-from dissect.target.loaders.remote import RemoteStreamConnection
-from dissect.target.loaders.targetd import TargetdLoader
 from dissect.target.plugins.apps.webserver import iis
 from dissect.target.plugins.os.windows.log import evt, evtx
+from dissect.target.tools.utils import args_to_uri
 from dissect.util.stream import RunlistStream
 
 from acquire.collector import Collector, get_full_formatted_report, get_report_summary
@@ -92,7 +92,6 @@ MODULE_LOOKUP = {}
 
 CLI_ARGS_MODULE = "cli-args"
 
-
 log = logging.getLogger("acquire")
 log.propagate = 0
 log_file_handler = None
@@ -149,46 +148,48 @@ MISC_MAPPING = {
 def from_user_home(target: Target, path: str) -> Iterator[str]:
     try:
         for user_details in target.user_details.all_with_home():
-            yield normalize_path(target, user_details.home_path.joinpath(path), lower_case=False)
+            yield user_details.home_path.joinpath(path).as_posix()
     except Exception as e:
         log.warning("Error occurred when requesting all user homes")
         log.debug("", exc_info=e)
 
     misc_user_homes = MISC_MAPPING.get(target.os, misc_unix_user_homes)
     for user_dir in misc_user_homes(target):
-        yield str(user_dir.joinpath(path))
+        yield user_dir.joinpath(path).as_posix()
 
 
-def iter_ntfs_filesystems(target: Target) -> Iterator[tuple[ntfs.NtfsFilesystem, str, str]]:
+def iter_ntfs_filesystems(target: Target) -> Iterator[tuple[ntfs.NtfsFilesystem, Optional[str], str, str]]:
     mount_lookup = defaultdict(list)
     for mount, fs in target.fs.mounts.items():
         mount_lookup[fs].append(mount)
 
-    sysvol = target.fs.mounts["sysvol"]
     for fs in target.filesystems:
-        if fs in mount_lookup:
-            mountpoints = ", ".join(mount_lookup[fs])
-        else:
-            mountpoints = "No mounts"
-
         # The attr check is needed to correctly collect fake NTFS filesystems
         # where the MFT etc. are added to a VirtualFilesystem. This happens for
         # instance when the target is an acquired tar target.
         if not isinstance(fs, ntfs.NtfsFilesystem) and not hasattr(fs, "ntfs"):
-            log.warning("Skipping %s (%s) - not an NTFS filesystem", fs, mountpoints)
+            log.warning("Skipping %s - not an NTFS filesystem", fs)
             continue
 
-        if fs == sysvol:
-            name = "sysvol"
-        elif fs in mount_lookup:
-            name = mount_lookup[fs][0]
+        if fs in mount_lookup:
+            mountpoints = mount_lookup[fs]
+
+            for main_mountpoint in mountpoints:
+                if main_mountpoint != "sysvol":
+                    break
+
+            name = main_mountpoint
+            mountpoints = ", ".join(mountpoints)
         else:
+            main_mountpoint = None
             name = f"vol-{fs.ntfs.serial:x}"
+            mountpoints = "No mounts"
+            log.warning("Unmounted NTFS filesystem found %s (%s)", fs, name)
 
-        yield fs, name, mountpoints
+        yield fs, main_mountpoint, name, mountpoints
 
 
-def iter_esxi_filesystems(target: Target) -> Iterator[tuple[str, str, Filesystem]]:
+def iter_esxi_filesystems(target: Target) -> Iterator[tuple[Filesystem, str, str, Optional[str]]]:
     for mount, fs in target.fs.mounts.items():
         if not mount.startswith("/vmfs/volumes/"):
             continue
@@ -200,11 +201,11 @@ def iter_esxi_filesystems(target: Target) -> Iterator[tuple[str, str, Filesystem
         elif fs.__type__ == "vmfs":
             name = fs.vmfs.label
 
-        yield uuid, name, fs
+        yield fs, mount, uuid, name
 
 
-def register_module(*args, **kwargs):
-    def wrapper(module_cls):
+def register_module(*args, **kwargs) -> Callable[[type[Module]], type[Module]]:
+    def wrapper(module_cls: type[Module]) -> type[Module]:
         name = module_cls.__name__
 
         if name in MODULES:
@@ -228,8 +229,8 @@ def register_module(*args, **kwargs):
     return wrapper
 
 
-def module_arg(*args, **kwargs):
-    def wrapper(module_cls):
+def module_arg(*args, **kwargs) -> Callable[[type[Module]], type[Module]]:
+    def wrapper(module_cls: type[Module]) -> type[Module]:
         if not hasattr(module_cls, "__cli_args__"):
             module_cls.__cli_args__ = []
         module_cls.__cli_args__.append((args, kwargs))
@@ -238,7 +239,7 @@ def module_arg(*args, **kwargs):
     return wrapper
 
 
-def local_module(cls):
+def local_module(cls: type[object]) -> object:
     """A decorator that sets property `__local__` on a module class to mark it for local target only"""
     cls.__local__ = True
     return cls
@@ -308,22 +309,25 @@ class NTFS(Module):
 
     @classmethod
     def _run(cls, target: Target, cli_args: argparse.Namespace, collector: Collector) -> None:
-        for fs, name, mountpoints in iter_ntfs_filesystems(target):
-            log.info("Acquiring %s (%s)", fs, mountpoints)
+        for fs, main_mountpoint, name, mountpoints in iter_ntfs_filesystems(target):
+            log.info("Acquiring from %s as %s (%s)", fs, name, mountpoints)
+
+            for filename in ("$MFT", "$Boot", "$Secure:$SDS"):
+                if main_mountpoint is not None:
+                    path = fsutil.join(main_mountpoint, filename)
+                    collector.collect_path(path)
 
-            collector.collect_file(fs.path("$MFT"), outpath=name + "/$MFT")
-            collector.collect_file(fs.path("$Boot"), outpath=name + "/$Boot")
+                else:
+                    # In case the NTFS filesystem is not mounted, which should not occur but
+                    # iter_ntfs_filesystems allows for the possibility, we fall back to raw file
+                    # collection.
+                    collector.collect_file_raw(filename, fs, name)
 
             cls.collect_usnjrnl(collector, fs, name)
-            cls.collect_ntfs_secure(collector, fs, name)
 
     @classmethod
     def collect_usnjrnl(cls, collector: Collector, fs: Filesystem, name: str) -> None:
-        try:
-            usnjrnl_path = fs.path("$Extend/$Usnjrnl:$J")
-            entry = usnjrnl_path.get()
-            journal = entry.open()
-
+        def usnjrnl_accessor(journal: BinaryIO) -> tuple[BinaryIO, int]:
             # If the filesystem is a virtual NTFS filesystem, journal will be
             # plain BinaryIO, not a RunlistStream.
             if isinstance(journal, RunlistStream):
@@ -331,57 +335,18 @@ class NTFS(Module):
                 while journal.runlist[i][0] is None:
                     journal.seek(journal.runlist[i][1] * journal.block_size, io.SEEK_CUR)
                     i += 1
+                size = journal.size - journal.tell()
+            else:
+                size = journal.size
 
-            # Use the same method to construct the output path as is used in
-            # collector.collect_file()
-            outpath = collector._output_path(f"{name}/$Extend/$Usnjrnl:$J")
-
-            collector.output.write(
-                outpath,
-                journal,
-                size=journal.size - journal.tell(),
-                entry=entry,
-            )
-            collector.report.add_file_collected(cls.__name__, usnjrnl_path)
-            result = "OK"
-        except exceptions.FileNotFoundError:
-            collector.report.add_file_missing(cls.__name__, usnjrnl_path)
-            result = "File not found"
-        except Exception as err:
-            log.debug("Failed to acquire UsnJrnl", exc_info=True)
-            collector.report.add_file_failed(cls.__name__, usnjrnl_path)
-            result = repr(err)
-
-        log.info("- Collecting file $Extend/$Usnjrnl:$J: %s", result)
-
-    @classmethod
-    def collect_ntfs_secure(cls, collector: Collector, fs: Filesystem, name: str) -> None:
-        try:
-            secure_path = fs.path("$Secure:$SDS")
-            entry = secure_path.get()
-            sds = entry.open()
-
-            # Use the same method to construct the output path as is used in
-            # collector.collect_file()
-            outpath = collector._output_path(f"{name}/$Secure:$SDS")
-
-            collector.output.write(
-                outpath,
-                sds,
-                size=sds.size,
-                entry=entry,
-            )
-            collector.report.add_file_collected(cls.__name__, secure_path)
-            result = "OK"
-        except FileNotFoundError:
-            collector.report.add_file_missing(cls.__name__, secure_path)
-            result = "File not found"
-        except Exception as err:
-            log.debug("Failed to acquire SDS", exc_info=True)
-            collector.report.add_file_failed(cls.__name__, secure_path)
-            result = repr(err)
+            return (journal, size)
 
-        log.info("- Collecting file $Secure:$SDS: %s", result)
+        collector.collect_file_raw(
+            "$Extend/$Usnjrnl:$J",
+            fs,
+            name,
+            file_accessor=usnjrnl_accessor,
+        )
 
 
 @register_module("-r", "--registry")
@@ -722,13 +687,20 @@ class RecycleBin(Module):
             patterns.extend(["$Recycle.Bin/$R*", "$Recycle.Bin/*/$R*", "RECYCLE*/D*"])
 
         with collector.file_filter(large_files_filter):
-            for fs, name, mountpoints in iter_ntfs_filesystems(target):
-                log.info("Acquiring recycle bin from %s (%s)", fs, mountpoints)
+            for fs, main_mountpoint, name, mountpoints in iter_ntfs_filesystems(target):
+                log.info("Acquiring recycle bin from %s as %s (%s)", fs, name, mountpoints)
 
                 for pattern in patterns:
-                    for entry in fs.path().glob(pattern):
-                        if entry.is_file():
-                            collector.collect_file(entry, outpath=fsutil.join(name, str(entry)))
+                    if main_mountpoint is not None:
+                        pattern = fsutil.join(main_mountpoint, pattern)
+                        collector.collect_glob(pattern)
+                    else:
+                        # In case the NTFS filesystem is not mounted, which should not occur but
+                        # iter_ntfs_filesystems allows for the possibility, we fall back to raw file
+                        # collection.
+                        for entry in fs.path().glob(pattern):
+                            if entry.is_file():
+                                collector.collect_file_raw(fs, entry, name)
 
 
 @register_module("--drivers")
@@ -1294,8 +1266,9 @@ class Boot(Module):
 
 
 def private_key_filter(path: fsutil.TargetPath) -> bool:
-    with path.open("rt") as file:
-        return "PRIVATE KEY" in file.readline()
+    if path.is_file() and not path.is_symlink():
+        with path.open("rt") as file:
+            return "PRIVATE KEY" in file.readline()
 
 
 @register_module("--home")
@@ -1441,21 +1414,24 @@ class Bootbanks(Module):
             "bootbank": "BOOTBANK1",
             "altbootbank": "BOOTBANK2",
         }
-        boot_fs = []
+        boot_fs = {}
 
         for boot_dir, boot_vol in boot_dirs.items():
             dir_path = target.fs.path(boot_dir)
             if dir_path.is_symlink() and dir_path.exists():
                 dst = dir_path.readlink()
-                boot_fs.append((dst.name, boot_vol, dst.get().top.fs))
+                fs = dst.get().top.fs
+                boot_fs[fs] = boot_vol
 
-        for uuid, name, fs in boot_fs:
-            log.info("Acquiring /vmfs/volumes/%s (%s)", uuid, name)
-            base = f"fs/{uuid}:{name}"
-            for path in fs.path("/").rglob("*"):
-                if not path.is_file():
-                    continue
-                collector.collect_file(path, outpath=path, base=base)
+        for fs, mountpoint, uuid, _ in iter_esxi_filesystems(target):
+            if fs in boot_fs:
+                name = boot_fs[fs]
+                log.info("Acquiring %s (%s)", mountpoint, name)
+                mountpoint_len = len(mountpoint)
+                base = f"fs/{uuid}:{name}"
+                for path in target.fs.path(mountpoint).rglob("*"):
+                    outpath = path.as_posix()[mountpoint_len:]
+                    collector.collect_path(path, outpath=outpath, base=base)
 
 
 @register_module("--esxi")
@@ -1478,16 +1454,16 @@ class VMFS(Module):
 
     @classmethod
     def _run(cls, target: Target, cli_args: argparse.Namespace, collector: Collector) -> None:
-        for uuid, name, fs in iter_esxi_filesystems(target):
+        for fs, mountpoint, uuid, name in iter_esxi_filesystems(target):
             if not fs.__type__ == "vmfs":
                 continue
 
-            log.info("Acquiring /vmfs/volumes/%s (%s)", uuid, name)
+            log.info("Acquiring %s (%s)", mountpoint, name)
+            mountpoint_len = len(mountpoint)
             base = f"fs/{uuid}:{name}"
-            for path in fs.path("/").glob("*.sf"):
-                if not path.is_file():
-                    continue
-                collector.collect_file(path, outpath=path, base=base)
+            for path in target.fs.path(mountpoint).glob("*.sf"):
+                outpath = path.as_posix()[mountpoint_len:]
+                collector.collect_path(path, outpath=outpath, base=base)
 
 
 @register_module("--activities-cache")
@@ -1658,45 +1634,6 @@ def print_acquire_warning(target: Target) -> None:
         log.warning("========================================== WARNING ==========================================")
 
 
-def modargs2json(args: argparse.Namespace) -> dict:
-    json_opts = {}
-    for module in MODULES.values():
-        cli_arg = module.__cli_args__[-1:][0][1]
-        if opt := cli_arg.get("dest"):
-            json_opts[opt] = getattr(args, opt)
-    return json_opts
-
-
-def acquire_target(target: Target, *args, **kwargs) -> list[str]:
-    if isinstance(target._loader, TargetdLoader):
-        files = acquire_target_targetd(target, *args, **kwargs)
-    else:
-        files = acquire_target_regular(target, *args, **kwargs)
-    return files
-
-
-def acquire_target_targetd(target: Target, args: argparse.Namespace, output_ts: Optional[str] = None) -> list[str]:
-    files = []
-    # debug logs contain references to flow objects and will give errors
-    logging.getLogger().setLevel(logging.CRITICAL)
-    if not len(target.hostname()):
-        log.error("Unable to initialize targetd.")
-        return files
-    json_opts = modargs2json(args)
-    json_opts["profile"] = args.profile
-    json_opts["file"] = args.file
-    json_opts["directory"] = args.directory
-    json_opts["glob"] = args.glob
-    m = {"targetd-meta": "acquire", "args": json_opts}
-    json_str = json.dumps(m)
-    targetd = target._loader.instance.client
-    targetd.send_message(json_str.encode("utf-8"))
-    targetd.sync()
-    for stream in targetd.streams:
-        files.append(stream.out_file)
-    return files
-
-
 def _add_modules_for_profile(choice: str, operating_system: str, profile: dict, msg: str) -> Optional[dict]:
     modules_selected = dict()
 
@@ -1712,7 +1649,7 @@ def _add_modules_for_profile(choice: str, operating_system: str, profile: dict,
     return modules_selected
 
 
-def acquire_target_regular(target: Target, args: argparse.Namespace, output_ts: Optional[str] = None) -> list[str]:
+def acquire_target(target: Target, args: argparse.Namespace, output_ts: Optional[str] = None) -> list[str]:
     acquire_gui = GUI()
     files = []
     output_ts = output_ts or get_utc_now_str()
@@ -1727,7 +1664,7 @@ def acquire_target_regular(target: Target, args: argparse.Namespace, output_ts:
     if log_file:
         files.append(log_file)
         if target.path.name == "local":
-            skip_list.add(normalize_path(target, log_file, resolve=True))
+            skip_list.add(normalize_path(target, log_file, resolve_parents=True, preserve_case=False))
 
     print_disks_overview(target)
     print_volumes_overview(target)
@@ -1817,7 +1754,7 @@ def acquire_target_regular(target: Target, args: argparse.Namespace, output_ts:
         log.info("Logging to file %s", log_path)
         files = [log_file_handler.baseFilename]
         if target.path.name == "local":
-            skip_list = {normalize_path(target, log_path, resolve=True)}
+            skip_list = {normalize_path(target, log_path, resolve_parents=True, preserve_case=False)}
 
     output_path = args.output or args.output_file
     if output_path.is_dir():
@@ -1833,7 +1770,7 @@ def acquire_target_regular(target: Target, args: argparse.Namespace, output_ts:
     )
     files.append(output.path)
     if target.path.name == "local":
-        skip_list.add(normalize_path(target, output.path, resolve=True))
+        skip_list.add(normalize_path(target, output.path, resolve_parents=True, preserve_case=False))
 
     log.info("Writing output to %s", output.path)
     if skip_list:
@@ -2092,7 +2029,7 @@ VOLATILE = {
 
 def main() -> None:
     parser = create_argument_parser(PROFILES, VOLATILE, MODULES)
-    args = parse_acquire_args(parser, config=CONFIG)
+    args, rest = parse_acquire_args(parser, config=CONFIG)
 
     # start GUI if requested through CLI / config
     flavour = None
@@ -2144,26 +2081,6 @@ def main() -> None:
         log.exception(err)
         parser.exit(1)
 
-    if args.targetd:
-        from targetd.tools.targetd import start_client
-
-        # set @auto hostname to real hostname
-        if args.targetd_hostname == "@auto":
-            args.targetd_hostname = f"/host/{Target.open('local').hostname}"
-
-        config = {
-            "function": args.targetd_func,
-            "topics": [args.targetd_hostname, args.targetd_groupname, args.targetd_globalname],
-            "link": args.targetd_link,
-            "address": args.targetd_ip,
-            "port": args.targetd_port,
-            "cacert_str": args.targetd_cacert,
-            "service": args.targetd_func == "agent",
-            "cacert": None,
-        }
-        start_client(args, presets=config)
-        return
-
     if args.upload:
         try:
             upload_files(args.upload, args.upload_plugin, args.no_proxy)
@@ -2171,43 +2088,44 @@ def main() -> None:
             log.exception("Failed to upload files")
         return
 
-    RemoteStreamConnection.configure(args.cagent_key, args.cagent_certificate)
-
-    target_path = args.target
-
-    if target_path == "local":
-        target_query = {}
-        if args.force_fallback:
-            target_query.update({"force-directory-fs": 1})
+    target_paths = []
+    for target_path in args.targets:
+        target_path = args_to_uri([target_path], args.loader, rest)[0] if args.loader else target_path
+        if target_path == "local":
+            target_query = {}
+            if args.force_fallback:
+                target_query.update({"force-directory-fs": 1})
 
-        if args.fallback:
-            target_query.update({"fallback-to-directory-fs": 1})
+            if args.fallback:
+                target_query.update({"fallback-to-directory-fs": 1})
 
-        target_query = urllib.parse.urlencode(target_query)
-        target_path = f"{target_path}?{target_query}"
-
-    log.info("Loading target %s", target_path)
+            target_query = urllib.parse.urlencode(target_query)
+            target_path = f"{target_path}?{target_query}"
+        target_paths.append(target_path)
 
     try:
-        target = Target.open(target_path)
-        log.info(target)
+        target_name = "Unknown"  # just in case open_all already fails
+        for target in Target.open_all(target_paths):
+            target_name = "Unknown"  # overwrite previous target name
+            target_name = target.name
+            log.info("Loading target %s", target_name)
+            log.info(target)
+            if target.os == "esxi" and target.name == "local":
+                # Loader found that we are running on an esxi host
+                # Perform operations to "enhance" memory
+                with esxi_memory_context_manager():
+                    acquire_children_and_targets(target, args)
+            else:
+                acquire_children_and_targets(target, args)
     except Exception:
         if not is_user_admin():
-            log.error("Failed to load target, try re-running as administrator/root.")
+            log.error("Failed to load target: %s, try re-running as administrator/root", target_name)
             acquire_gui.message("This application must be run as administrator.")
             acquire_gui.wait_for_quit()
             parser.exit(1)
-        log.exception("Failed to load target")
+        log.exception("Failed to load target: %s", target_name)
         raise
 
-    if target.os == "esxi" and target.name == "local":
-        # Loader found that we are running on an esxi host
-        # Perform operations to "enhance" memory
-        with esxi_memory_context_manager():
-            acquire_children_and_targets(target, args)
-    else:
-        acquire_children_and_targets(target, args)
-
 
 def load_child(target: Target, child_path: Path) -> None:
     log.info("")