dissect.hypervisor 3.16.dev1__tar.gz → 3.16.dev3__tar.gz

{dissect_hypervisor-3.16.dev1/dissect.hypervisor.egg-info → dissect_hypervisor-3.16.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dissect.hypervisor
-Version: 3.16.dev1
+Version: 3.16.dev3
 Summary: A Dissect module implementing parsers for various hypervisor disk, backup and configuration files
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -27,7 +27,6 @@ Requires-Dist: dissect.cstruct<5,>=4
 Requires-Dist: dissect.util<4,>=3
 Provides-Extra: full
 Requires-Dist: pycryptodome; extra == "full"
-Requires-Dist: rich; extra == "full"
 Provides-Extra: dev
 Requires-Dist: dissect.hypervisor[full]; extra == "dev"
 Requires-Dist: dissect.cstruct<5.0.dev,>=4.0.dev; extra == "dev"

{dissect_hypervisor-3.16.dev1 → dissect_hypervisor-3.16.dev3}/dissect/hypervisor/__init__.py

@@ -1,4 +1,3 @@
-from dissect.hypervisor.backup import vma, xva
 from dissect.hypervisor.descriptor import hyperv, ovf, pvs, vbox, vmx
 from dissect.hypervisor.disk import hdd, qcow2, vdi, vhd, vhdx, vmdk
 from dissect.hypervisor.util import envelope, vmtar
@@ -14,9 +13,7 @@ __all__ = [
     "vdi",
     "vhd",
     "vhdx",
-    "vma",
     "vmdk",
     "vmtar",
     "vmx",
-    "xva",
 ]

{dissect_hypervisor-3.16.dev1 → dissect_hypervisor-3.16.dev3}/dissect/hypervisor/disk/vmdk.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import ctypes
 import io
 import logging
@@ -7,7 +5,6 @@ import os
 import textwrap
 import zlib
 from bisect import bisect_right
-from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
 
@@ -62,13 +59,13 @@ class VMDK(AlignedStream):
                 if self.descriptor.attr["parentCID"] != "ffffffff":
                     self.parent = open_parent(path.parent, self.descriptor.attr["parentFileNameHint"])
 
-                for extent in self.descriptor.extents:
-                    if extent.type in ["SPARSE", "VMFSSPARSE", "SESPARSE"]:
-                        sdisk_fh = path.with_name(extent.filename).open("rb")
+                for _, size, extent_type, filename in self.descriptor.extents:
+                    if extent_type in ["SPARSE", "VMFSSPARSE", "SESPARSE"]:
+                        sdisk_fh = path.with_name(filename).open("rb")
                         self.disks.append(SparseDisk(sdisk_fh, parent=self.parent))
-                    elif extent.type in ["VMFS", "FLAT"]:
-                        rdisk_fh = path.with_name(extent.filename).open("rb")
-                        self.disks.append(RawDisk(rdisk_fh, extent.sectors * SECTOR_SIZE))
+                    elif extent_type in ["VMFS", "FLAT"]:
+                        rdisk_fh = path.with_name(filename).open("rb")
+                        self.disks.append(RawDisk(rdisk_fh, size * SECTOR_SIZE))
 
             elif magic in (COWD_MAGIC, VMDK_MAGIC, SESPARSE_MAGIC):
                 sparse_disk = SparseDisk(fh)
@@ -401,37 +398,8 @@ class SparseExtentHeader:
         return getattr(self.hdr, attr)
 
 
-@dataclass
-class ExtentDescriptor:
-    access_mode: str
-    sectors: int
-    type: str
-    filename: str | None = None
-    start_sector: int | None = None
-    partition_uuid: str | None = None
-    device_identifier: str | None = None
-
-    def __post_init__(self) -> None:
-        self._raw = " ".join(map(str, [v for v in self.__dict__.values() if v is not None]))
-        self.sectors = int(self.sectors)
-
-        if self.filename:
-            self.filename = self.filename.strip('"')
-
-        if self.start_sector:
-            self.start_sector = int(self.start_sector)
-
-    def __repr__(self) -> str:
-        return f"<ExtentDescriptor {self._raw}>"
-
-    def __str__(self) -> str:
-        return self._raw
-
-
 class DiskDescriptor:
-    def __init__(
-        self, attr: dict, extents: list[ExtentDescriptor], disk_db: dict, sectors: int, raw_config: str | None = None
-    ):
+    def __init__(self, attr, extents, disk_db, sectors, raw_config=None):
         self.attr = attr
         self.extents = extents
         self.ddb = disk_db
@@ -439,15 +407,9 @@ class DiskDescriptor:
         self.raw = raw_config
 
     @classmethod
-    def parse(cls, vmdk_config: str) -> DiskDescriptor:
-        """Return :class:`DiskDescriptor` based on the provided ``vmdk_config``.
-
-        Resources:
-            - https://github.com/libyal/libvmdk/blob/main/documentation/VMWare%20Virtual%20Disk%20Format%20(VMDK).asciidoc
-        """  # noqa: E501
-
+    def parse(cls, vmdk_config):
         descriptor_settings = {}
-        extents: list[ExtentDescriptor] = []
+        extents = []
         disk_db = {}
         sectors = 0
 
@@ -458,16 +420,11 @@ class DiskDescriptor:
                 continue
 
             if line.startswith("RW ") or line.startswith("RDONLY ") or line.startswith("NOACCESS "):
-                # Extent descriptors can have up to seven values according to libvmdk documentation.
-                parts = line.split(" ", maxsplit=6)
-
-                if len(parts) < 3:
-                    log.warning("Unexpected ExtentDescriptor format in vmdk config: %s, ignoring", line)
-                    continue
-
-                extent = ExtentDescriptor(*parts)
-                sectors += extent.sectors
-                extents.append(extent)
+                access_type, size, extent_type, filename = line.split(" ", 3)
+                filename = filename.strip('"')
+                size = int(size)
+                sectors += size
+                extents.append([access_type, size, extent_type, filename])
                 continue
 
             setting, _, value = line.partition("=")
@@ -481,33 +438,35 @@ class DiskDescriptor:
 
         return cls(descriptor_settings, extents, disk_db, sectors, vmdk_config)
 
-    def __str__(self) -> str:
-        str_template = textwrap.dedent(
-            """\
-        # Disk DescriptorFile
-        version=1
-        {}
-
-        # Extent Description
-        {}
+    def __str__(self):
+        str_template = """\
+                          # Disk DescriptorFile
+                          version=1
+                          {}
 
-        # The Disk Data Base
-        #DDB
+                          # Extent Description
+                          {}
 
-        {}"""
-        )
+                          # The Disk Data Base
+                          #DDB
 
+                          {}"""
+        str_template = textwrap.dedent(str_template)
         descriptor_settings = []
         for setting, value in self.attr.items():
-            if setting != "version":
-                descriptor_settings.append(f"{setting}={value}")
+            if setting == "version":
+                continue
+            descriptor_settings.append("{}={}".format(setting, value))
         descriptor_settings = "\n".join(descriptor_settings)
 
-        extents = "\n".join(map(str, self.extents))
+        extents = []
+        for access_type, size, extent_type, filename in self.extents:
+            extents.append('{} {} {} "{}"'.format(access_type, size, extent_type, filename))
+        extents = "\n".join(extents)
 
         disk_db = []
         for setting, value in self.ddb.items():
-            disk_db.append(f'{setting} = "{value}"')
+            disk_db.append('{} = "{}"'.format(setting, value))
         disk_db = "\n".join(disk_db)
 
         return str_template.format(descriptor_settings, extents, disk_db)

{dissect_hypervisor-3.16.dev1 → dissect_hypervisor-3.16.dev3/dissect.hypervisor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dissect.hypervisor
-Version: 3.16.dev1
+Version: 3.16.dev3
 Summary: A Dissect module implementing parsers for various hypervisor disk, backup and configuration files
 Author-email: Dissect Team <dissect@fox-it.com>
 License: Affero General Public License v3
@@ -27,7 +27,6 @@ Requires-Dist: dissect.cstruct<5,>=4
 Requires-Dist: dissect.util<4,>=3
 Provides-Extra: full
 Requires-Dist: pycryptodome; extra == "full"
-Requires-Dist: rich; extra == "full"
 Provides-Extra: dev
 Requires-Dist: dissect.hypervisor[full]; extra == "dev"
 Requires-Dist: dissect.cstruct<5.0.dev,>=4.0.dev; extra == "dev"

{dissect_hypervisor-3.16.dev1 → dissect_hypervisor-3.16.dev3}/dissect.hypervisor.egg-info/SOURCES.txt

@@ -12,10 +12,6 @@ dissect.hypervisor.egg-info/requires.txt
 dissect.hypervisor.egg-info/top_level.txt
 dissect/hypervisor/__init__.py
 dissect/hypervisor/exceptions.py
-dissect/hypervisor/backup/__init__.py
-dissect/hypervisor/backup/c_vma.py
-dissect/hypervisor/backup/vma.py
-dissect/hypervisor/backup/xva.py
 dissect/hypervisor/descriptor/__init__.py
 dissect/hypervisor/descriptor/c_hyperv.py
 dissect/hypervisor/descriptor/hyperv.py
@@ -38,7 +34,6 @@ dissect/hypervisor/disk/vhdx.py
 dissect/hypervisor/disk/vmdk.py
 dissect/hypervisor/tools/__init__.py
 dissect/hypervisor/tools/envelope.py
-dissect/hypervisor/tools/vma.py
 dissect/hypervisor/util/__init__.py
 dissect/hypervisor/util/envelope.py
 dissect/hypervisor/util/vmtar.py
@@ -52,7 +47,6 @@ tests/test_pvs.py
 tests/test_vbox.py
 tests/test_vhd.py
 tests/test_vhdx.py
-tests/test_vma.py
 tests/test_vmdk.py
 tests/test_vmtar.py
 tests/test_vmx.py
@@ -67,7 +61,6 @@ tests/data/local.tgz.ve
 tests/data/sesparse.vmdk.gz
 tests/data/test.VMRS
 tests/data/test.vgz
-tests/data/test.vma.gz
 tests/data/test.vmcx
 tests/data/expanding.hdd/DiskDescriptor.xml
 tests/data/expanding.hdd/expanding.hdd

{dissect_hypervisor-3.16.dev1 → dissect_hypervisor-3.16.dev3}/dissect.hypervisor.egg-info/entry_points.txt

@@ -1,3 +1,2 @@
 [console_scripts]
 envelope-decrypt = dissect.hypervisor.tools.envelope:main
-vma-extract = dissect.hypervisor.tools.vma:main

{dissect_hypervisor-3.16.dev1 → dissect_hypervisor-3.16.dev3}/dissect.hypervisor.egg-info/requires.txt

@@ -9,4 +9,3 @@ dissect.util<4.0.dev,>=3.0.dev
 
 [full]
 pycryptodome
-rich

{dissect_hypervisor-3.16.dev1 → dissect_hypervisor-3.16.dev3}/pyproject.toml

@@ -39,7 +39,6 @@ repository = "https://github.com/fox-it/dissect.hypervisor"
 [project.optional-dependencies]
 full = [
     "pycryptodome",
-    "rich",
 ]
 dev = [
     "dissect.hypervisor[full]",
@@ -48,7 +47,6 @@ dev = [
 ]
 
 [project.scripts]
-vma-extract = "dissect.hypervisor.tools.vma:main"
 envelope-decrypt = "dissect.hypervisor.tools.envelope:main"
 
 [tool.black]

{dissect_hypervisor-3.16.dev1 → dissect_hypervisor-3.16.dev3}/tests/conftest.py

@@ -79,11 +79,6 @@ def split_hdd() -> Iterator[str]:
     yield absolute_path("data/split.hdd")
 
 
-@pytest.fixture
-def simple_vma() -> Iterator[BinaryIO]:
-    yield from open_file_gz("data/test.vma.gz")
-
-
 @pytest.fixture
 def envelope() -> Iterator[BinaryIO]:
     yield from open_file("data/local.tgz.ve")

dissect_hypervisor-3.16.dev3/tests/test_vmdk.py

@@ -0,0 +1,20 @@
+from dissect.hypervisor.disk.c_vmdk import c_vmdk
+from dissect.hypervisor.disk.vmdk import VMDK
+
+
+def test_vmdk_sesparse(sesparse_vmdk):
+    vmdk = VMDK(sesparse_vmdk)
+
+    disk = vmdk.disks[0]
+
+    assert disk.is_sesparse
+    assert disk._grain_directory_size == 0x20000
+    assert disk._grain_table_size == 0x1000
+    assert disk._grain_entry_type == c_vmdk.uint64
+    assert disk._grain_directory[0] == 0x1000000000000000
+
+    header = disk.header
+    assert header.magic == c_vmdk.SESPARSE_CONST_HEADER_MAGIC
+    assert header.version == 0x200000001
+
+    assert vmdk.read(0x1000000) == b"a" * 0x1000000

dissect_hypervisor-3.16.dev1/dissect/hypervisor/backup/c_vma.py

@@ -1,60 +0,0 @@
-from dissect.cstruct import cstruct
-
-vma_def = """
-#define VMA_BLOCK_BITS          12
-#define VMA_BLOCK_SIZE          (1 << VMA_BLOCK_BITS)
-#define VMA_CLUSTER_BITS        (VMA_BLOCK_BITS + 4)
-#define VMA_CLUSTER_SIZE        (1 << VMA_CLUSTER_BITS)
-
-#define VMA_EXTENT_HEADER_SIZE  512
-#define VMA_BLOCKS_PER_EXTENT   59
-#define VMA_MAX_CONFIGS         256
-
-#define VMA_MAX_EXTENT_SIZE     (VMA_EXTENT_HEADER_SIZE + VMA_CLUSTER_SIZE * VMA_BLOCKS_PER_EXTENT)
-
-/* File Format Definitions */
-
-struct VmaDeviceInfoHeader {
-    uint32  devname_ptr;    /* offset into blob_buffer table */
-    uint32  reserved0;
-    uint64  size;           /* device size in bytes */
-    uint64  reserved1;
-    uint64  reserved2;
-};
-
-struct VmaHeader {
-    char    magic[4];
-    uint32  version;
-    char    uuid[16];
-    int64   ctime;
-    char    md5sum[16];
-
-    uint32  blob_buffer_offset;
-    uint32  blob_buffer_size;
-    uint32  header_size;
-
-    char    _reserved1[1984];
-
-    uint32  config_names[VMA_MAX_CONFIGS]; /* offset into blob_buffer table */
-    uint32  config_data[VMA_MAX_CONFIGS];  /* offset into blob_buffer table */
-
-    char    _reserved2[4];
-
-    VmaDeviceInfoHeader     dev_info[256];
-};
-
-struct VmaExtentHeader {
-    char    magic[4];
-    uint16  reserved1;
-    uint16  block_count;
-    char    uuid[16];
-    char    md5sum[16];
-    uint64  blockinfo[VMA_BLOCKS_PER_EXTENT];
-};
-"""
-
-c_vma = cstruct(endian=">").load(vma_def)
-
-
-VMA_MAGIC = b"VMA\x00"
-VMA_EXTENT_MAGIC = b"VMAE"

dissect_hypervisor-3.16.dev1/dissect/hypervisor/backup/vma.py

@@ -1,269 +0,0 @@
-# References:
-# - https://git.proxmox.com/?p=pve-qemu.git;a=blob;f=vma_spec.txt
-# - https://lists.gnu.org/archive/html/qemu-devel/2013-02/msg03667.html
-
-import hashlib
-import struct
-from collections import defaultdict
-from functools import lru_cache
-from uuid import UUID
-
-from dissect.util import ts
-from dissect.util.stream import AlignedStream
-
-from dissect.hypervisor.backup.c_vma import VMA_EXTENT_MAGIC, VMA_MAGIC, c_vma
-from dissect.hypervisor.exceptions import InvalidHeaderError
-
-
-class VMA:
-    """Proxmox VMA.
-
-    Parse and provide a readable object for devices in a Proxmox VMA backup file.
-    VMA is designed to be streamed for extraction, so we need to do some funny stuff to create a readable
-    object from it. Performance is not optimal, so it's generally advised to extract a VMA instead.
-    The vma-extract utility can be used for that.
-    """
-
-    def __init__(self, fh):
-        self.fh = fh
-
-        offset = fh.tell()
-        self.header = c_vma.VmaHeader(fh)
-        if self.header.magic != VMA_MAGIC:
-            raise InvalidHeaderError("Invalid VMA header magic")
-
-        fh.seek(offset)
-        header_data = bytearray(fh.read(self.header.header_size))
-        header_data[32:48] = b"\x00" * 16
-        if hashlib.md5(header_data).digest() != self.header.md5sum:
-            raise InvalidHeaderError("Invalid VMA checksum")
-
-        self.version = self.header.version
-        self.uuid = UUID(bytes=self.header.uuid)
-
-        blob_start = self.header.blob_buffer_offset
-        blob_end = self.header.blob_buffer_offset + self.header.blob_buffer_size
-        self._blob = memoryview(bytes(header_data))[blob_start:blob_end]
-
-        blob_offset = 1
-        self._blob_data = {}
-        while blob_offset + 2 <= self.header.blob_buffer_size:
-            # The header is in big endian, but this is little...
-            size = struct.unpack("<H", self._blob[blob_offset : blob_offset + 2])[0]
-            if blob_offset + 2 + size <= blob_end:
-                self._blob_data[blob_offset] = self._blob[blob_offset + 2 : blob_offset + 2 + size].tobytes()
-            blob_offset += size + 2
-
-        self._config = {}
-        for conf_name, conf_data in zip(self.header.config_names, self.header.config_data):
-            if (conf_name, conf_data) == (0, 0):
-                continue
-
-            self._config[self.blob_string(conf_name)] = self.blob_data(conf_data)
-
-        self._devices = {}
-        for dev_id, dev_info in enumerate(self.header.dev_info):
-            if dev_id == 0 or dev_info.devname_ptr == 0:
-                continue
-
-            self._devices[dev_id] = Device(self, dev_id, self.blob_string(dev_info.devname_ptr), dev_info.size)
-
-        self._extent = lru_cache(65536)(self._extent)
-
-    @property
-    def creation_time(self):
-        return ts.from_unix(self.header.ctime)
-
-    def blob_data(self, offset):
-        if offset not in self._blob_data:
-            raise KeyError(f"No blob data for offset {offset}")
-        return self._blob_data[offset]
-
-    def blob_string(self, offset):
-        return self.blob_data(offset).decode().rstrip("\x00")
-
-    def config(self, name):
-        return self._config[name]
-
-    def configs(self):
-        return self._config
-
-    def device(self, dev_id):
-        return self._devices[dev_id]
-
-    def devices(self):
-        return list(self._devices.values())
-
-    def _extent(self, offset):
-        return Extent(self.fh, offset)
-
-    def extents(self):
-        offset = self.header.header_size
-        while True:
-            try:
-                extent = self._extent(offset)
-            except EOFError:
-                break
-
-            yield extent
-
-            offset += c_vma.VMA_EXTENT_HEADER_SIZE + extent.size
-
-
-class Device:
-    def __init__(self, vma, dev_id, name, size):
-        self.vma = vma
-        self.id = dev_id
-        self.name = name
-        self.size = size
-
-    def __repr__(self):
-        return f"<Device id={self.id} name={self.name} size={self.size}>"
-
-    def open(self):
-        return DeviceDataStream(self)
-
-
-class Extent:
-    def __init__(self, fh, offset):
-        self.fh = fh
-        self.offset = offset
-        self.data_offset = offset + c_vma.VMA_EXTENT_HEADER_SIZE
-
-        self.fh.seek(offset)
-        header_data = bytearray(fh.read(c_vma.VMA_EXTENT_HEADER_SIZE))
-        self.header = c_vma.VmaExtentHeader(header_data)
-        if self.header.magic != VMA_EXTENT_MAGIC:
-            raise InvalidHeaderError("Invalid VMA extent header magic")
-
-        header_data[24:40] = b"\x00" * 16
-        if hashlib.md5(header_data).digest() != self.header.md5sum:
-            raise InvalidHeaderError("Invalid VMA extent checksum")
-
-        self.uuid = UUID(bytes=self.header.uuid)
-        self.size = self.header.block_count * c_vma.VMA_BLOCK_SIZE
-
-        # Keep track of the lowest and highest cluster we have for any device
-        # We can use this to speed up extent lookup later on
-        # There are at most 59 entries, so safe to parse ahead of use
-        self._min = {}
-        self._max = {}
-        self.blocks = defaultdict(list)
-        block_offset = self.data_offset
-        for block_info in self.header.blockinfo:
-            cluster_num = block_info & 0xFFFFFFFF
-            dev_id = (block_info >> 32) & 0xFF
-            mask = block_info >> (32 + 16)
-
-            if dev_id == 0:
-                continue
-
-            if dev_id not in self._min:
-                self._min[dev_id] = cluster_num
-                self._max[dev_id] = cluster_num
-            elif cluster_num < self._min[dev_id]:
-                self._min[dev_id] = cluster_num
-            elif cluster_num > self._max[dev_id]:
-                self._max[dev_id] = cluster_num
-
-            self.blocks[dev_id].append((cluster_num, mask, block_offset))
-
-            if mask == 0xFFFF:
-                block_offset += 16 * c_vma.VMA_BLOCK_SIZE
-            elif mask == 0:
-                pass
-            else:
-                block_offset += bin(mask).count("1") * c_vma.VMA_BLOCK_SIZE
-
-    def __repr__(self):
-        return f"<Extent offset=0x{self.offset:x} size=0x{self.size:x}>"
-
-
-class DeviceDataStream(AlignedStream):
-    def __init__(self, device):
-        self.device = device
-        self.vma = device.vma
-        super().__init__(size=device.size, align=c_vma.VMA_CLUSTER_SIZE)
-
-    def _read(self, offset, length):
-        cluster_offset = offset // c_vma.VMA_CLUSTER_SIZE
-        cluster_count = (length + c_vma.VMA_CLUSTER_SIZE - 1) // c_vma.VMA_CLUSTER_SIZE
-        block_count = (length + c_vma.VMA_BLOCK_SIZE - 1) // c_vma.VMA_BLOCK_SIZE
-
-        result = []
-        for _, mask, block_offset in _iter_clusters(self.vma, self.device.id, cluster_offset, cluster_count):
-            read_count = min(block_count, 16)
-
-            # Optimize reading fully set and fully sparse masks
-            if mask == 0xFFFF:
-                self.vma.fh.seek(block_offset)
-                result.append(self.vma.fh.read(c_vma.VMA_BLOCK_SIZE * read_count))
-            elif mask == 0:
-                result.append(b"\x00" * read_count * c_vma.VMA_BLOCK_SIZE)
-            else:
-                self.vma.fh.seek(block_offset)
-                for allocated, count in _iter_mask(mask, read_count):
-                    if allocated:
-                        result.append(self.vma.fh.read(c_vma.VMA_BLOCK_SIZE * count))
-                    else:
-                        result.append(b"\x00" * count * c_vma.VMA_BLOCK_SIZE)
-
-            block_count -= read_count
-            if block_count == 0:
-                break
-
-        return b"".join(result)
-
-
-def _iter_clusters(vma, dev_id, cluster, count):
-    # Find clusters and starting offsets in all extents
-    temp = {}
-    end = cluster + count
-
-    for extent in vma.extents():
-        if dev_id not in extent.blocks:
-            continue
-
-        if end < extent._min[dev_id] or cluster > extent._max[dev_id]:
-            continue
-
-        for cluster_num, mask, block_offset in extent.blocks[dev_id]:
-            if cluster_num == cluster:
-                yield cluster_num, mask, block_offset
-                cluster += 1
-
-                while cluster in temp:
-                    yield temp[cluster]
-                    del temp[cluster]
-                    cluster += 1
-            elif cluster < cluster_num <= end:
-                temp[cluster_num] = (cluster_num, mask, block_offset)
-
-            if cluster == end:
-                break
-
-        if cluster == end:
-            break
-
-    while cluster in temp:
-        yield temp[cluster]
-        del temp[cluster]
-        cluster += 1
-
-
-def _iter_mask(mask, length):
-    # Yield consecutive bitmask values
-    current_status = mask & 1
-    current_count = 0
-
-    for bit_idx in range(length):
-        status = (mask & (1 << bit_idx)) >> bit_idx
-        if status == current_status:
-            current_count += 1
-        else:
-            yield current_status, current_count
-            current_status = status
-            current_count = 1
-
-    if current_count:
-        yield current_status, current_count