dissect.volume 3.19.dev1__tar.gz → 3.19.dev3__tar.gz

{dissect_volume-3.19.dev1 → dissect_volume-3.19.dev3}/.git-blame-ignore-revs

@@ -4,3 +4,5 @@
 #
 # Change linter to Ruff (#44)
 8fefd76a54ff43b2fdbedbbb3cd330c27b1a0b14
+# Add stricter Ruff linting (#63)
+6bb48483a8eaa62fba114ae33643a19f7a91a171

{dissect_volume-3.19.dev1 → dissect_volume-3.19.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dissect.volume
-Version: 3.19.dev1
+Version: 3.19.dev3
 Summary: A Dissect module implementing a parser for different disk volume and partition systems, for example LVM2, GPT and MBR
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: AGPL-3.0-or-later

{dissect_volume-3.19.dev1 → dissect_volume-3.19.dev3}/dissect/volume/lvm/metadata.py

@@ -4,7 +4,7 @@ from dataclasses import dataclass
 from datetime import datetime  # noqa: TC003
 from functools import cache
 from types import UnionType
-from typing import BinaryIO, get_args, get_origin, get_type_hints
+from typing import TYPE_CHECKING, BinaryIO, get_args, get_origin, get_type_hints
 
 from dissect.util import ts
 from dissect.util.stream import MappingStream
@@ -14,6 +14,9 @@ from dissect.volume.exceptions import LVM2Error
 from dissect.volume.lvm.c_lvm2 import SECTOR_SIZE
 from dissect.volume.lvm.physical import LVM2Device  # noqa: TC001
 
+if TYPE_CHECKING:
+    from dissect.util.stream import RunlistStream
+
 
 @dataclass(init=False)
 class MetaBase:
@@ -131,11 +134,46 @@ class PhysicalVolume(MetaBase):
     def dev(self) -> LVM2Device | None:
         return self._dev
 
+    @property
+    def size(self) -> int:
+        """Return the size of the physical volume in bytes.
+
+        This can sometimes be larger than what the LVM2Device dictates, in which case the VolumeGroup overwrites it:
+            https://github.com/lvmteam/lvm2/blob/6e208b81ec587434097de3207fbd1ecd7a0afb8c/lib/format_text/layout.h#L44
+
+        From the code, this is also the value the pvck tool uses to update pv_header.device_size_xl.
+        """
+        # This size is also displayed when showing physical device information with pvdisplay:
+        #   https://github.com/lvmteam/lvm2/blob/6e208b81ec587434097de3207fbd1ecd7a0afb8c/lib/display/display.c#L291
+        if self.dev_size:
+            return self.dev_size * self.vg.extent_size
+        # The size of the stripes on this device + size of the metadata
+        return (self.pe_start + self.pe_count * self.vg.extent_size) * SECTOR_SIZE
+
     def _from_dict(self, obj: dict, name: str | None = None, parent: MetaBase | None = None) -> None:
         super()._from_dict(obj, name=name, parent=parent)
         self._name = name
         self._volume_group = parent
 
+    def open(self) -> BinaryIO:
+        """Opens the physical device.
+
+        Automatically uses the corect size as determined by :attr:`size`
+        """
+        if self.dev is None:
+            raise LVM2Error(f"Physical volume not found: {self.name} (id={self.id}, device={self.device})")
+
+        stream: RunlistStream = self.dev.open()
+        if stream.size >= self.size:
+            return stream
+
+        size = max(stream.size, self.size)
+        area = self.dev._data_area_descriptors[0]
+
+        stream.runlist = [(area.offset // SECTOR_SIZE, size // SECTOR_SIZE)]
+        stream.size = size
+        return stream
+
 
 @dataclass(init=False)
 class LogicalVolume(MetaBase):
@@ -343,12 +381,7 @@ class StripedSegment(Segment):
         offset = 0
         for pv_name, extent_offset in self.stripes:
             if pv_name not in opened_pv:
-                if (pv_dev := pv[pv_name].dev) is not None:
-                    pv_fh = pv_dev.open()
-                else:
-                    raise LVM2Error(
-                        f"Physical volume not found: {pv_name} (id={pv[pv_name].id}, device={pv[pv_name].device})"
-                    )
+                pv_fh = pv[pv_name].open()
                 opened_pv[pv_name] = pv_fh
             else:
                 pv_fh = opened_pv[pv_name]

{dissect_volume-3.19.dev1 → dissect_volume-3.19.dev3}/dissect/volume/lvm/physical.py

@@ -30,11 +30,17 @@ class LVM2Device:
             raise LVM2Error("Can't find physical volume label header")
 
         fh.seek(self.label_offset + self.label.offset)
+
         self.header = c_lvm.pv_header(fh)
         self.id = self.header.pv_uuid.decode()
         self.size = self.header.device_size
 
         self._data_area_descriptors = _read_descriptors(fh, c_lvm.disk_locn)
+
+        if len(self._data_area_descriptors) != 1:
+            # According to the LVM2 code, there should only be 1 data area descriptor for each Physical Volume
+            raise RuntimeError(f"There should be 1 data area descriptor, found {len(self._data_area_descriptors)}")
+
         self._metadata_area_descriptors = _read_descriptors(fh, c_lvm.disk_locn)
 
         self._metadata_areas = []
@@ -88,11 +94,10 @@ class LVM2Device:
         return b"".join(r)
 
     def open(self) -> BinaryIO:
-        runlist = [
-            (area.offset // SECTOR_SIZE, (area.size or self.size) // SECTOR_SIZE)
-            for area in self._data_area_descriptors
-        ]
-
+        # Only accounts for one data area descriptor
+        # When multiple descriptors get allowed in the future, this code should be adjusted to reflect that
+        area = self._data_area_descriptors[0]
+        runlist = [(area.offset // SECTOR_SIZE, self.size // SECTOR_SIZE)]
         return RunlistStream(self.fh, runlist, self.size, SECTOR_SIZE)
 
 

{dissect_volume-3.19.dev1 → dissect_volume-3.19.dev3}/dissect.volume.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dissect.volume
-Version: 3.19.dev1
+Version: 3.19.dev3
 Summary: A Dissect module implementing a parser for different disk volume and partition systems, for example LVM2, GPT and MBR
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: AGPL-3.0-or-later

{dissect_volume-3.19.dev1 → dissect_volume-3.19.dev3}/dissect.volume.egg-info/SOURCES.txt

@@ -85,6 +85,7 @@ tests/_data/dm/dm-thin-data.bin.gz
 tests/_data/dm/dm-thin-empty-data.bin.gz
 tests/_data/dm/dm-thin-empty-metadata.bin.gz
 tests/_data/dm/dm-thin-metadata.bin.gz
+tests/_data/lvm/lvm-inconsistent-sizes.bin.gz
 tests/_data/lvm/lvm-mirror-1.bin.gz
 tests/_data/lvm/lvm-mirror-2.bin.gz
 tests/_data/lvm/lvm-thin.bin.gz
@@ -130,6 +131,7 @@ tests/_docs/Makefile
 tests/_docs/__init__.py
 tests/_docs/conf.py
 tests/_docs/index.rst
+tests/_tools/lvm/generate.sh
 tests/ddf/__init__.py
 tests/ddf/test_ddf.py
 tests/disk/__init__.py

dissect_volume-3.19.dev3/tests/_tools/lvm/generate.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly TESTS_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+readonly OUT_DIR="${TESTS_ROOT}/_data/lvm"
+
+log()  { printf '[INFO] %s\n' "$*" >&2; }
+warn() { printf '[WARN] %s\n' "$*" >&2; }
+error()  { printf '[ERROR] %s\n' "$*" >&2; }
+have() { command -v "$1" >/dev/null 2>&1; }
+
+require_tools() {
+    local -a tools=(pvcreate mkfs.ext4 dd xxd)
+    local missing=0
+
+    for t in "${tools[@]}"; do
+        if ! have "$t"; then
+            error "Missing required tool: $t"
+            missing=1
+        fi
+    done
+
+    if (( missing != 0 )); then
+        error "One or more required tools are missing. Aborting."
+        exit 1
+    fi
+}
+
+function _create_lvm_inconsistent_sizes {
+  local filename="$1"
+  local uid=$(id -u)
+  dd if=/dev/zero of="$filename" bs=1M count=8 >&/dev/null
+  log "Created block device $filename"
+
+  local lo=$(sudo losetup --partscan --show --find "$filename")
+
+  log "Created loopback device $lo"
+  sudo pvcreate "$lo" >/dev/null
+  sudo vgcreate vghelp "$lo" >/dev/null
+  sudo lvcreate -l 100%FREE -n lv vghelp >/dev/null
+  sudo mkfs.ext4 /dev/vghelp/lv >&/dev/null
+
+  log "Created lvm2 structure"
+
+
+  sudo mount /dev/vghelp/lv /mnt/tmp-lvm --mkdir
+  sudo chmod -R o+rw /mnt/tmp-lvm
+  fallocate -l "2448KiB" /mnt/tmp-lvm/large-file
+  echo "A small file at the end of the disk" > /mnt/tmp-lvm/small-file
+
+  log "Created files on disk"
+
+  log "Cleanup lvm devices"
+  sudo umount /mnt/tmp-lvm
+  sudo lvchange -an vghelp/lv
+  sudo vgchange -an vghelp >/dev/null
+  sudo losetup -d "$lo"
+
+
+  log "Update size of the pv header"
+  # Update the size at this offset
+  printf "00000240: 0000 3000" | xxd -r - "$filename"
+
+  # Compress the resulting file
+  gzip "$filename"
+}
+
+
+main() {
+    require_tools
+
+    mkdir -p "${OUT_DIR}"
+
+    _create_lvm_inconsistent_sizes "${OUT_DIR}/lvm-inconsistent-sizes.bin"
+
+    log "All test cases generated under: ${OUT_DIR}"
+}
+
+main

{dissect_volume-3.19.dev1 → dissect_volume-3.19.dev3}/tests/conftest.py

@@ -45,6 +45,11 @@ def lvm_mirror() -> Iterator[list[BinaryIO]]:
     yield from open_files_gz(["_data/lvm/lvm-mirror-1.bin.gz", "_data/lvm/lvm-mirror-2.bin.gz"])
 
 
+@pytest.fixture
+def lvm_inconsistent_sizes() -> Iterator[BinaryIO]:
+    yield from open_file_gz("_data/lvm/lvm-inconsistent-sizes.bin.gz")
+
+
 @pytest.fixture
 def mbr() -> Iterator[BinaryIO]:
     yield from open_file("_data/disk/mbr.bin")

dissect_volume-3.19.dev3/tests/lvm/test_lvm.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+import io
+import re
+from typing import BinaryIO
+
+import pytest
+
+from dissect.volume.exceptions import LVM2Error
+from dissect.volume.lvm import LVM2
+from dissect.volume.lvm.c_lvm2 import c_lvm
+from dissect.volume.lvm.metadata import StripedSegment
+
+
+def test_lvm(lvm: BinaryIO) -> None:
+    vs = LVM2(lvm)
+
+    vg = vs.vg
+    assert vg.name == "vg_test"
+    assert vg.id == "8HfEjs-9DNH-0dy1-U5u8-EYBF-Vce4-8BcSWU"
+    assert len(vg.lv) == 1
+    assert len(vg.pv) == 1
+
+    lv = vg.lv[0]
+    assert lv.name == "lv_test"
+    assert lv.id == "TnYdWo-zRE9-wf2T-5nt0-M1aD-vtoP-fASCxK"
+    assert lv.type == "striped"
+    assert len(lv.segments) == 1
+
+    seg = lv.segments[0]
+    assert isinstance(seg, StripedSegment)
+    assert len(seg.stripes) == seg.stripe_count
+    assert seg.extent_count == 1
+    assert seg.extent_count * vg.extent_size == 8192
+
+    fh = lv.open()
+    assert fh.read(1024) == b"\xde\xad\xbe\xef" * 256
+    assert fh.read(1024) == b"\x00" * 1024
+
+    vg.pv[0]._dev = None
+    with pytest.raises(
+        LVM2Error,
+        match=re.escape(
+            "Physical volume not found: pv0 (id=2Svcy0-cRH2-3Xrz-87Fv-zNUI-9CoI-Ycoyql, device=/dev/loop1)"
+        ),
+    ):
+        fh = lv.open()
+
+
+def test_lvm_invalid() -> None:
+    buf = io.BytesIO(4096 * b"\x00")
+
+    with pytest.raises(LVM2Error):
+        LVM2(buf)
+
+
+def test_lvm_thin(lvm_thin: BinaryIO) -> None:
+    lvm = LVM2(lvm_thin)
+
+    for lv_name in ["lv-1", "lv-2"]:
+        lv = lvm.vg.logical_volumes[lv_name]
+        assert lv.type == "thin"
+
+        fh = lv.open()
+        for i in range(1, 513):
+            assert fh.read(4096) == i.to_bytes(2, "little") * 2048
+
+    pool = lvm.vg.logical_volumes["data"]
+    assert pool.type == "thin-pool"
+
+    with pytest.raises(
+        RuntimeError, match="Opening a thin-pool for reading is not possible, use open_pool\\(\\) instead"
+    ):
+        pool.open()
+
+
+def test_lvm_mirror(lvm_mirror: list[BinaryIO]) -> None:
+    lvm = LVM2(lvm_mirror)
+    lv = lvm.vg.logical_volumes["mirrormirror"]
+
+    assert lv.type == "mirror"
+
+    fh = lv.open()
+    for i in range(1, 513):
+        assert fh.read(4096) == i.to_bytes(2, "little") * 2048
+
+
+def test_lvm_multiple_data_areas() -> None:
+    """Test if LVM2 raises an Error if more than 1 data area descriptor is specified.
+
+    This test can be removed once lvm2 allows for multiple area descriptors
+    """
+    lvm_data = [
+        c_lvm.label_header(id=b"LABELONE", offset=len(c_lvm.label_header)),
+        c_lvm.pv_header(),
+        c_lvm.disk_locn(offset=1),
+        c_lvm.disk_locn(offset=2),
+        c_lvm.disk_locn(),
+    ]
+    data = io.BytesIO(b"".join(struct.dumps() for struct in lvm_data))
+    with pytest.raises(RuntimeError, match="There should be 1 data area descriptor, found 2"):
+        LVM2(data)
+
+
+def test_lvm_sizes_mismatch(lvm_inconsistent_sizes: BinaryIO) -> None:
+    """Test if opening the LVM2Device uses PhysicalVolume.size when LVM2Device.size < PhysicalVolume.size."""
+    lvm = LVM2(lvm_inconsistent_sizes)
+
+    lv = lvm.vg.logical_volumes["lv"]
+
+    # Size manually adjusted to be smaller than the actual data
+    # This is the offset the data of a text file starts
+    assert next(iter(lvm.devices.values())).size == 0x300000
+    physical_volume = lvm.vg.pv[0]
+    assert physical_volume.size == 0x8000000
+
+    fh = lv.open()
+    # Size of the stripe
+    assert fh.size == 0x400000
+
+    segment_stream = fh._runs[0][2]
+    pv_stream = segment_stream._runs[0][2]
+    # Check whether the underlying pv stream size is correct (The non adjusted size)
+    assert pv_stream.size == 0x8000000
+
+    fh.seek(0x3B8800)
+
+    expected_data = b"A small file at the end of the disk"
+    assert fh.read(len(expected_data)) == expected_data
+
+
+def test_lvm_sizes_mismatch_missing_dev_size(lvm_inconsistent_sizes: BinaryIO) -> None:
+    """Test if opening the LVM2Device uses the calculated disk size when PhysicalVolume.dev_size is missing."""
+    lvm = LVM2(lvm_inconsistent_sizes)
+
+    lv = lvm.vg.logical_volumes["lv"]
+
+    # Size manually adjusted to be smaller than the actual data
+    # This is the offset the data of a text file starts
+    assert next(iter(lvm.devices.values())).size == 0x300000
+    physical_volume = lvm.vg.pv[0]
+    # physical_volume.size will calculate size based on the the data offset
+    # and size of the stripes if dev_size is not defined
+    physical_volume.dev_size = None
+    assert physical_volume.size == 0x500000
+
+    fh = lv.open()
+    # Size of the stripe
+    assert fh.size == 0x400000
+
+    segment_stream = fh._runs[0][2]
+    pv_stream = segment_stream._runs[0][2]
+    assert pv_stream.size == 0x500000
+
+    fh.seek(0x3B8800)
+
+    expected_data = b"A small file at the end of the disk"
+    assert fh.read(len(expected_data)) == expected_data

dissect_volume-3.19.dev1/tests/lvm/test_lvm.py

@@ -1,84 +0,0 @@
-from __future__ import annotations
-
-import io
-import re
-from typing import BinaryIO
-
-import pytest
-
-from dissect.volume.exceptions import LVM2Error
-from dissect.volume.lvm import LVM2
-from dissect.volume.lvm.metadata import StripedSegment
-
-
-def test_lvm(lvm: BinaryIO) -> None:
-    vs = LVM2(lvm)
-
-    vg = vs.vg
-    assert vg.name == "vg_test"
-    assert vg.id == "8HfEjs-9DNH-0dy1-U5u8-EYBF-Vce4-8BcSWU"
-    assert len(vg.lv) == 1
-    assert len(vg.pv) == 1
-
-    lv = vg.lv[0]
-    assert lv.name == "lv_test"
-    assert lv.id == "TnYdWo-zRE9-wf2T-5nt0-M1aD-vtoP-fASCxK"
-    assert lv.type == "striped"
-    assert len(lv.segments) == 1
-
-    seg = lv.segments[0]
-    assert isinstance(seg, StripedSegment)
-    assert len(seg.stripes) == seg.stripe_count
-    assert seg.extent_count == 1
-    assert seg.extent_count * vg.extent_size == 8192
-
-    fh = lv.open()
-    assert fh.read(1024) == b"\xde\xad\xbe\xef" * 256
-    assert fh.read(1024) == b"\x00" * 1024
-
-    vg.pv[0]._dev = None
-    with pytest.raises(
-        LVM2Error,
-        match=re.escape(
-            "Physical volume not found: pv0 (id=2Svcy0-cRH2-3Xrz-87Fv-zNUI-9CoI-Ycoyql, device=/dev/loop1)"
-        ),
-    ):
-        fh = lv.open()
-
-
-def test_lvm_invalid() -> None:
-    buf = io.BytesIO(4096 * b"\x00")
-
-    with pytest.raises(LVM2Error):
-        LVM2(buf)
-
-
-def test_lvm_thin(lvm_thin: BinaryIO) -> None:
-    lvm = LVM2(lvm_thin)
-
-    for lv_name in ["lv-1", "lv-2"]:
-        lv = lvm.vg.logical_volumes[lv_name]
-        assert lv.type == "thin"
-
-        fh = lv.open()
-        for i in range(1, 513):
-            assert fh.read(4096) == i.to_bytes(2, "little") * 2048
-
-    pool = lvm.vg.logical_volumes["data"]
-    assert pool.type == "thin-pool"
-
-    with pytest.raises(
-        RuntimeError, match="Opening a thin-pool for reading is not possible, use open_pool\\(\\) instead"
-    ):
-        pool.open()
-
-
-def test_lvm_mirror(lvm_mirror: list[BinaryIO]) -> None:
-    lvm = LVM2(lvm_mirror)
-    lv = lvm.vg.logical_volumes["mirrormirror"]
-
-    assert lv.type == "mirror"
-
-    fh = lv.open()
-    for i in range(1, 513):
-        assert fh.read(4096) == i.to_bytes(2, "little") * 2048