dissect.archive 1.2.dev2__tar.gz → 1.3.dev1__tar.gz

dissect_archive-1.3.dev1/.gitattributes

@@ -0,0 +1 @@
+tests/data/* filter=lfs diff=lfs merge=lfs -text

{dissect_archive-1.2.dev2 → dissect_archive-1.3.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dissect.archive
-Version: 1.2.dev2
+Version: 1.3.dev1
 Summary: A Dissect module implementing parsers for various archive and backup formats
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -22,9 +22,12 @@ Requires-Python: ~=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: COPYRIGHT
-Requires-Dist: dissect.cstruct<5,>=4.dev
+Requires-Dist: dissect.cstruct<5,>=4
 Requires-Dist: dissect.util<4,>=3
+Provides-Extra: full
+Requires-Dist: rich; extra == "full"
 Provides-Extra: dev
+Requires-Dist: dissect.archive[full]; extra == "dev"
 Requires-Dist: dissect.cstruct<5.0.dev,>=4.0.dev; extra == "dev"
 Requires-Dist: dissect.util<4.0.dev,>=3.0.dev; extra == "dev"
 

dissect_archive-1.3.dev1/dissect/archive/c_vbk.py

@@ -0,0 +1,290 @@
+from dissect.cstruct import cstruct
+
+vbk_def = """
+#define PAGE_SIZE 4096
+
+/* Storage header */
+
+struct StorageHeader {
+    uint32              FormatVersion;              /* 0x0000 */
+    uint32              Initialized;                /* 0x0004 */
+    uint32              DigestTypeLength;           /* 0x0008 */
+    char                DigestType[251];            /* 0x000C */
+    uint32              SnapshotSlotFormat;         /* 0x0107 format > 5 -> crc32c */
+    uint32              StandardBlockSize;          /* 0x010B */
+    uint8               ClusterAlign;               /* 0x010F */
+    char                Unk0[16];                   /* 0x0120 */
+    char                ExternalStorageId[16];      /* 0x0130 */
+};
+
+/* Snapshot header */
+
+struct SnapshotSlotHeader {
+    uint32              CRC;
+    uint32              ContainsSnapshot;
+};
+
+struct DirectoryRootRecord {
+    int64               RootPage;                   /* Root page of the directory */
+    uint64              Count;                      /* Number of children */
+};
+
+struct BlocksStoreHeader {
+    int64               RootPage;                   /* Root of the blocks store */
+    uint64              Count;                      /* Number of blocks store entries */
+    int64               FreeRootPage;               /* Root of the free blocks tree */
+    int64               DeduplicationRootPage;      /* Root of the deduplication tree */
+    int64               Unk0;
+    int64               Unk1;
+};
+
+struct CryptoStoreRecord {
+    int64               RootPage;                   /* Root of the crypto store */
+};
+
+struct SnapshotDescriptor {
+    uint64              Version;                    /* Acts as a sequence number, highest is active slot */
+    uint64              StorageEOF;                 /* End of file, aka file size */
+    uint32              BanksCount;                 /* Number of banks */
+    DirectoryRootRecord DirectoryRoot;              /* Directory root record */
+    BlocksStoreHeader   BlocksStore;                /* Blocks store header */
+    CryptoStoreRecord   CryptoStore;                /* Crypto store record */
+    uint64              Unk0;
+    uint64              Unk1;
+};
+
+struct BankDescriptor {
+    uint32              CRC;
+    uint64              Offset;
+    uint32              Size;
+};
+
+struct BanksGrain {
+    uint32              MaxBanks;
+    uint32              StoredBanks;
+    // BankDescriptor      Banks[StoredBanks];
+};
+
+/* Block headers */
+
+struct BankHeader {
+    uint16              PageCount;
+    uint16              Flags;
+    char                Unk0[3064];
+    uint64              Unk1;
+    char                Unk2[1020];
+};
+
+struct BankHeaderV71 {
+    uint16              PageCount;
+    uint16              Flags;                      /* 2 == encrypted */
+    char                Unk0[3072];
+    char                KeySetId[16];
+    char                Unk1[16];
+    char                Unk2[16];
+    uint32              Unk3;
+    char                Unk4[968];
+};
+
+struct MetaBlobHeader {
+    int64               NextPage;
+    int32               Unk0;
+};
+
+struct Lz4BlockHeader {
+    uint32              Magic;                      /* 0xF800000F */
+    uint32              CRC;                        /* CRC32C of the compressed data */
+    uint32              SourceSize;
+};
+
+/* DirItem headers */
+struct BlocksVectorHeader {
+    uint64              RootPage;
+    uint64              Count;
+};
+
+struct SubFolderHeader {
+    uint64              RootPage;                   /* 0x94 */
+    uint32              Count;                      /* 0x9C */
+    char                Data[32];                   /* 0xA0 */
+};                                                  /* 0xC0 */
+
+struct ExtFibHeader {
+    uint16              UpdateInProgress;           /* 0x94 */
+    uint8               Unk3;                       /* 0x96 */
+    uint8               Format;                     /* 0x97 Bit 3 == 1 */
+    BlocksVectorHeader  BlocksVector;               /* 0x98 */
+    uint64              FibSize;                    /* 0xA8 */
+    uint64              Size;                       /* 0xB0 */
+    uint8               FsObjAttachState;           /* 0xB8 */
+    char                Data[7];                    /* 0xB9 */
+};                                                  /* 0xC0 */
+
+struct IntFibHeader {
+    uint16              UpdateInProgress;           /* 0x94 */
+    uint8               Unk3;                       /* 0x96 */
+    uint8               Format;                     /* 0x97 Bit 3 == 1 */
+    BlocksVectorHeader  BlocksVector;               /* 0x98 */
+    uint64              FibSize;                    /* 0xA8 */
+    uint64              Size;                       /* 0xB0 */
+    uint8               FsObjAttachState;           /* 0xB8 */
+    char                Data[7];                    /* 0xB9 */
+};                                                  /* 0xC0 */
+
+struct PatchHeader {
+    uint32              Unk0;                       /* 0x94 */
+    BlocksVectorHeader  BlocksVector;               /* 0x98 */
+    uint64              FibSize;                    /* 0xA8 Source file size */
+    uint64              Unk4;                       /* 0xB0 */
+    char                Data[8];                    /* 0xB8 */
+};                                                  /* 0xC0 */
+
+struct IncrementHeader {
+    uint32              Unk0;                       /* 0x94 */
+    BlocksVectorHeader  BlocksVector;               /* 0x98 */
+    uint64              FibSize;                    /* 0xA8 Original FIB size */
+    uint64              Unk4;                       /* 0xB0 */
+    char                Data[8];                    /* 0xB8 */
+};                                                  /* 0xC0 */
+
+enum DirItemType : uint32 {
+    None                = 0,
+    SubFolder           = 1,
+    ExtFib              = 2,
+    IntFib              = 3,
+    Patch               = 4,
+    Increment           = 5,
+};
+
+struct DirItemRecord {
+    DirItemType         Type;                       /* 0x00 */
+    uint32              NameLength;                 /* 0x04 */
+    char                Name[128];                  /* 0x08 */
+    int64               PropsRootPage;              /* 0x88 */
+    uint32              Unk1;                       /* 0x90 */
+    union {                                         /* 0x94 */
+        char            Data[44];
+        SubFolderHeader SubFolder;
+        ExtFibHeader    ExtFib;
+        IntFibHeader    IntFib;
+        PatchHeader     Patch;
+        IncrementHeader Increment;
+    };
+};
+
+/* Block descriptors */
+
+flag BlockFlags : uint8 {
+    None                = 0x00,
+    Updated             = 0x01,
+    CommitInProgress    = 0x02,
+};
+
+enum BlockLocationType : uint8 {
+    Normal              = 0x00,
+    Sparse              = 0x01,
+    Reserved            = 0x02,
+    Archived            = 0x03,                     /* CompressedSize | (CompressionType << 32) */
+    BlockInBlob         = 0x04,                     /* BlockId? & 0x3FFFFFF | (BlobId << 26) | ((Offset >> 9) << 42) */
+    BlockInBlobReserved = 0x05,                     /* BlockId? | 0xFFFFFFFFFC000000 */
+};
+
+enum CompressionType : int8 {
+    Plain               = -1,
+    RL                  =  2,
+    ZLH                 =  3,
+    ZLL                 =  4,
+    LZ4                 =  7,
+};
+
+struct MetaTableDescriptor {
+    int64               RootPage;
+    uint64              BlockSize;
+    uint64              Count;
+};
+
+struct StgBlockDescriptor {
+    uint8               Format;                     /* Format != 4 == legacy */
+    uint32              UsageCounter;
+    uint64              Offset;
+    uint32              AllocatedSize;
+    uint8               Deduplication;
+    char                Digest[16];
+    CompressionType     CompressionType;
+    uint8               Unk0;
+    uint32              CompressedSize;
+    uint32              SourceSize;
+};
+
+struct StgBlockDescriptorV7 {
+    uint8               Format;                     /* Format != 4 == legacy */
+    uint32              UsageCounter;
+    uint64              Offset;
+    uint32              AllocatedSize;
+    uint8               Deduplication;
+    char                Digest[16];
+    CompressionType     CompressionType;
+    uint8               Unk0;
+    uint32              CompressedSize;
+    uint32              SourceSize;
+    char                KeySetId[16];
+};
+
+struct FibBlockDescriptor {
+    uint32              BlockSize;
+    BlockLocationType   Type;
+    char                Digest[16];
+    // union {
+    //     struct {
+    //         uint32 ArchiveUsedSize;
+    //         uint8 ArchiveCompressionType;
+    //         uint8 Unk3;
+    //         uint16 Unk4;
+    //     } Archived;
+    //     uint64  Offset;
+    // };
+    uint64              BlockId;                    /* For performance reasons we just put a uint64 here, but this is actually a union */
+    BlockFlags          Flags;
+};
+
+struct FibBlockDescriptorV7 {
+    uint32              BlockSize;
+    BlockLocationType   Type;
+    char                Digest[16];
+    // union {
+    //     struct {
+    //         uint32   ArchiveUsedSize;
+    //         uint8    ArchiveCompressionType;
+    //         uint8    Unk3;
+    //         uint16   Unk4;
+    //     } Archived;
+    //     uint64       Offset;
+    // };
+    uint64              BlockId;                    /* For performance reasons we just put a uint64 here, but this is actually a union */
+    BlockFlags          Flags;
+    char                KeySetId[16];
+};
+
+struct PatchBlockDescriptor {
+};
+
+struct PatchBlockDescriptorV7 {
+};
+
+/* Property dictionary */
+
+enum PropertyType : int32 {
+    UInt32              = 1,
+    UInt64              = 2,
+    AString             = 3,
+    WString             = 4,
+    Binary              = 5,
+    Boolean             = 6,
+    End                 = -1,
+};
+"""  # noqa: E501
+
+c_vbk = cstruct().load(vbk_def)
+
+PAGE_SIZE = c_vbk.PAGE_SIZE
+"""VBK page size."""

dissect_archive-1.3.dev1/dissect/archive/c_vma.py

@@ -0,0 +1,60 @@
+from dissect.cstruct import cstruct
+
+vma_def = """
+#define VMA_BLOCK_BITS          12
+#define VMA_BLOCK_SIZE          (1 << VMA_BLOCK_BITS)
+#define VMA_CLUSTER_BITS        (VMA_BLOCK_BITS + 4)
+#define VMA_CLUSTER_SIZE        (1 << VMA_CLUSTER_BITS)
+
+#define VMA_EXTENT_HEADER_SIZE  512
+#define VMA_BLOCKS_PER_EXTENT   59
+#define VMA_MAX_CONFIGS         256
+
+#define VMA_MAX_EXTENT_SIZE     (VMA_EXTENT_HEADER_SIZE + VMA_CLUSTER_SIZE * VMA_BLOCKS_PER_EXTENT)
+
+/* File Format Definitions */
+
+struct VmaDeviceInfoHeader {
+    uint32  devname_ptr;    /* offset into blob_buffer table */
+    uint32  reserved0;
+    uint64  size;           /* device size in bytes */
+    uint64  reserved1;
+    uint64  reserved2;
+};
+
+struct VmaHeader {
+    char    magic[4];
+    uint32  version;
+    char    uuid[16];
+    int64   ctime;
+    char    md5sum[16];
+
+    uint32  blob_buffer_offset;
+    uint32  blob_buffer_size;
+    uint32  header_size;
+
+    char    _reserved1[1984];
+
+    uint32  config_names[VMA_MAX_CONFIGS]; /* offset into blob_buffer table */
+    uint32  config_data[VMA_MAX_CONFIGS];  /* offset into blob_buffer table */
+
+    char    _reserved2[4];
+
+    VmaDeviceInfoHeader     dev_info[256];
+};
+
+struct VmaExtentHeader {
+    char    magic[4];
+    uint16  reserved1;
+    uint16  block_count;
+    char    uuid[16];
+    char    md5sum[16];
+    uint64  blockinfo[VMA_BLOCKS_PER_EXTENT];
+};
+"""
+
+c_vma = cstruct(endian=">").load(vma_def)
+
+
+VMA_MAGIC = b"VMA\x00"
+VMA_EXTENT_MAGIC = b"VMAE"

dissect_archive-1.3.dev1/dissect/archive/exceptions.py

@@ -0,0 +1,22 @@
+class Error(Exception):
+    pass
+
+
+class InvalidHeaderError(Error):
+    pass
+
+
+class FileNotFoundError(Error, FileNotFoundError):
+    pass
+
+
+class IsADirectoryError(Error, IsADirectoryError):
+    pass
+
+
+class NotADirectoryError(Error, NotADirectoryError):
+    pass
+
+
+class NotAReparsePointError(Error):
+    pass

dissect_archive-1.3.dev1/dissect/archive/tools/backup.py

@@ -0,0 +1,208 @@
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+from dissect.archive.c_vma import c_vma
+from dissect.archive.vbk import VBK, DirItem
+from dissect.archive.vma import VMA, _iter_mask
+
+try:
+    from rich.logging import RichHandler
+    from rich.progress import (
+        BarColumn,
+        DownloadColumn,
+        Progress,
+        TextColumn,
+        TimeRemainingColumn,
+        TransferSpeedColumn,
+    )
+
+    progress = Progress(
+        TextColumn("[bold blue]{task.fields[filename]}", justify="right"),
+        BarColumn(bar_width=None),
+        "[progress.percentage]{task.percentage:>3.1f}%",
+        "•",
+        DownloadColumn(),
+        "•",
+        TransferSpeedColumn(),
+        "•",
+        TimeRemainingColumn(),
+        transient=True,
+    )
+except ImportError:
+    RichHandler = logging.StreamHandler
+
+    class Progress:
+        def __init__(self):
+            self.filename = None
+            self.total = None
+
+            self._task_id = 0
+            self._info = {}
+
+        def __enter__(self):
+            pass
+
+        def __exit__(self, *args, **kwargs) -> None:
+            sys.stderr.write("\n")
+            sys.stderr.flush()
+
+        def add_task(self, name: str, filename: str, total: int, **kwargs) -> int:
+            task_id = self._task_id
+            self._task_id += 1
+
+            self._info[task_id] = {"filename": filename, "total": total, "position": 0}
+
+            return task_id
+
+        def update(self, task_id: int, advance: int) -> None:
+            self._info[task_id]["position"] += advance
+            self.draw()
+
+        def draw(self) -> None:
+            infos = []
+            for info in self._info.values():
+                infos.append(f"{info['filename']} {(info['position'] / info['total']) * 100:0.2f}%")
+            sys.stderr.write("\r" + " | ".join(infos))
+            sys.stderr.flush()
+
+    progress = Progress()
+
+
+log = logging.getLogger(__name__)
+
+
+def setup_logging(logger: logging.Logger, verbosity: int) -> None:
+    if verbosity == 1:
+        level = logging.ERROR
+    elif verbosity == 2:
+        level = logging.WARNING
+    elif verbosity == 3:
+        level = logging.INFO
+    elif verbosity >= 4:
+        level = logging.DEBUG
+    else:
+        level = logging.CRITICAL
+
+    handler = RichHandler()
+    handler.setFormatter(logging.Formatter("%(message)s"))
+    handler.setLevel(level)
+    logger.addHandler(handler)
+    logger.setLevel(level)
+
+
+def extract_vma(vma: VMA, out_dir: Path) -> None:
+    log.info("Extracting config files")
+    for config_name, config_data in vma.configs().items():
+        out_file = out_dir.joinpath(config_name)
+
+        log.info("%s -> %s (%d bytes)", config_name, out_file, len(config_data))
+        out_file.write_bytes(config_data)
+
+    log.info("Extracting device data")
+    tasks = {}
+    handles = {}
+    for device in vma.devices():
+        task_id = progress.add_task("extract", filename=device.name, total=device.size)
+        tasks[device.id] = task_id
+        handles[device.id] = out_dir.joinpath(device.name).open("wb")
+
+    with progress:
+        try:
+            for extent in vma.extents():
+                vma.fh.seek(extent.data_offset)
+                for block_info in extent.header.blockinfo:
+                    cluster_num = block_info & 0xFFFFFFFF
+                    dev_id = (block_info >> 32) & 0xFF
+                    mask = block_info >> (32 + 16)
+
+                    if dev_id == 0:
+                        continue
+
+                    fh_out = handles[dev_id]
+                    fh_out.seek(cluster_num * c_vma.VMA_CLUSTER_SIZE)
+
+                    if mask == 0xFFFF:
+                        fh_out.write(vma.fh.read(c_vma.VMA_CLUSTER_SIZE))
+                    elif mask == 0:
+                        fh_out.write(b"\x00" * c_vma.VMA_CLUSTER_SIZE)
+                    else:
+                        for allocated, count in _iter_mask(mask, 16):
+                            if allocated:
+                                fh_out.write(vma.fh.read(count * c_vma.VMA_BLOCK_SIZE))
+                            else:
+                                fh_out.write(b"\x00" * count * c_vma.VMA_BLOCK_SIZE)
+
+                    progress.update(tasks[dev_id], advance=c_vma.VMA_CLUSTER_SIZE)
+        except Exception as e:
+            log.exception("Exception during extraction")
+            log.debug("", exc_info=e)
+        finally:
+            for handle in handles.values():
+                handle.close()
+
+
+def extract_vbk(vbk: VBK, out_dir: Path) -> None:
+    def extract_directory(directory: DirItem, out_dir: Path) -> None:
+        out_dir.mkdir(exist_ok=True)
+        for entry in directory.iterdir():
+            out_path = out_dir.joinpath(entry.name)
+            if entry.is_dir():
+                extract_directory(entry, out_path)
+            else:
+                task_id = progress.add_task("extract", filename=entry.name, total=entry.size)
+                with entry.open() as fh_in, out_path.open("wb") as fh_out:
+                    for chunk in iter(lambda: fh_in.read(vbk.block_size), b""):
+                        fh_out.write(chunk)
+                        progress.update(task_id, advance=len(chunk))
+
+    with progress:
+        try:
+            extract_directory(vbk.get("/"), out_dir)
+        except Exception as e:
+            log.exception("Exception during extraction")
+            log.debug("", exc_info=e)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Hypervisor backup extractor")
+    parser.add_argument("input", type=Path, help="path to backup file")
+    parser.add_argument("-o", "--output", type=Path, required=True, help="path to output directory")
+    parser.add_argument("-v", "--verbose", action="count", default=3, help="increase output verbosity")
+    args = parser.parse_args()
+
+    setup_logging(log, args.verbose)
+
+    in_file = args.input.resolve()
+    if not in_file.exists():
+        log.error("Input file does not exist: %s", in_file)
+        parser.exit()
+
+    out_dir = args.output.resolve()
+    if not out_dir.exists():
+        log.error("Output path does not exist: %s", out_dir)
+        parser.exit()
+
+    if not out_dir.is_dir():
+        log.error("Output path is not a directory: %s", out_dir)
+        parser.exit()
+
+    with in_file.open("rb") as fh:
+        for klass, extract in ((VMA, extract_vma), (VBK, extract_vbk)):
+            try:
+                backup = klass(fh)
+                extract(backup, out_dir)
+                break
+            except Exception as e:
+                log.debug("Failed to extract using %s", klass.__name__, exc_info=e)
+        else:
+            log.error("Unknown backup format")
+            parser.exit()
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        pass