dissect.target 3.25.dev6__tar.gz → 3.25.dev7__tar.gz

{dissect_target-3.25.dev6 → dissect_target-3.25.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dissect.target
-Version: 3.25.dev6
+Version: 3.25.dev7
 Summary: This module ties all other Dissect modules together, it provides a programming API and command line tools which allow easy access to various data sources inside disk images or file collections (a.k.a. targets)
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: AGPL-3.0-or-later
@@ -58,7 +58,7 @@ Requires-Dist: fusepy; extra == "full"
 Requires-Dist: pycryptodome; extra == "full"
 Requires-Dist: ruamel.yaml; extra == "full"
 Requires-Dist: tomli; python_version < "3.11" and extra == "full"
-Requires-Dist: zstandard; platform_python_implementation != "PyPy" and extra == "full"
+Requires-Dist: backports.zstd; python_version < "3.14" and extra == "full"
 Provides-Extra: dev
 Requires-Dist: dissect.target[full,yara]; extra == "dev"
 Requires-Dist: dissect.archive[dev]<2.0.dev,>=1.0.dev; extra == "dev"

{dissect_target-3.25.dev6 → dissect_target-3.25.dev7}/dissect/target/filesystems/vmtar.py

@@ -12,6 +12,14 @@ class VmtarFilesystem(TarFilesystem):
     __type__ = "vmtar"
 
     def __init__(self, fh: BinaryIO, base: str | None = None, *args, **kwargs):
+        """Load a vmtar file (modified tar version used on ESXi).
+
+        In most case vmtar files are compressed using zstd, xz or LZMA, and then using gzip.
+
+        Args:
+            fh: File-like object of a vmtar file.
+            base: Base path in the tar file to open the filesystem on.
+        """
         fh = open_decompress(fileobj=open_decompress(fileobj=fh))
         super().__init__(fh, base, *args, tarinfo=vmtar.VisorTarInfo, **kwargs)
 

{dissect_target-3.25.dev6 → dissect_target-3.25.dev7}/dissect/target/helpers/fsutil.py

@@ -29,7 +29,10 @@ except ImportError:
     HAS_BZ2 = False
 
 try:
-    import zstandard
+    if sys.version_info >= (3, 14):
+        from compression import zstd  # novermin
+    else:
+        from backports import zstd
 
     HAS_ZSTD = True
 except ImportError:
@@ -526,6 +529,9 @@ def open_decompress(
     Returns:
         An binary or text IO stream, depending on the mode with which the file was opened.
 
+    Raises:
+        ValueError: path and fileobj are mutually exclusive, but one of them is required
+
     Example:
         .. code-block:: python
 
@@ -565,9 +571,7 @@ def open_decompress(
         return bz2.open(file, mode, encoding=encoding, errors=errors, newline=newline)
 
     if HAS_ZSTD and magic[:4] in [b"\xfd\x2f\xb5\x28", b"\x28\xb5\x2f\xfd"]:
-        # stream_reader is not seekable, so we have to resort to the less
-        # efficient decompressor which returns bytes.
-        return io.BytesIO(zstandard.decompress(file.read()))
+        return zstd.open(file, mode=mode, encoding=encoding, errors=errors, newline=newline)
 
     if path:
         file.close()

{dissect_target-3.25.dev6 → dissect_target-3.25.dev7}/dissect/target/plugins/os/unix/esxi/_os.py

@@ -1,8 +1,6 @@
 from __future__ import annotations
 
-import gzip
 import json as jsonlib
-import lzma
 import struct
 import subprocess
 from configparser import ConfigParser
@@ -12,9 +10,9 @@ from typing import TYPE_CHECKING, Any, BinaryIO, TextIO
 
 from defusedxml import ElementTree
 from dissect.database.sqlite3 import SQLite3
-from dissect.hypervisor.util import vmtar
 
 from dissect.target.filesystems.nfs import NfsFilesystem
+from dissect.target.filesystems.vmtar import VmtarFilesystem
 from dissect.target.helpers.sunrpc import client
 from dissect.target.helpers.sunrpc.client import LocalPortPolicy
 
@@ -284,16 +282,10 @@ def _mount_modules(target: Target, sysvol: Filesystem, cfg: dict[str, str]) -> N
         if module_path.name.endswith((".tar.gz", ".tgz")):
             tfs = tar.TarFilesystem(module_path.open())
         elif module_path.suffix.startswith(".v"):
-            # Visor tar files are always gzipped
-            cfile = gzip.GzipFile(fileobj=module_path.open())
-            # Sometimes they are also xz compressed, check for XZ magic
-            # NOTE: The XZ layer may also contain file signatures
-            # Could be interesting to check.
-            if cfile.peek(6)[:6] == b"\xfd7zXZ\x00":
-                cfile = lzma.LZMAFile(cfile)  # noqa: SIM115
-
-            tfs = tar.TarFilesystem(cfile, tarinfo=vmtar.VisorTarInfo)
-
+            try:
+                tfs = VmtarFilesystem(module_path.open())
+            except Exception as e:
+                target.log.warning("%s, skipping file %s", str(e), module_path)
         if tfs:
             target.fs.append_layer().mount("/", tfs)
 

{dissect_target-3.25.dev6 → dissect_target-3.25.dev7}/dissect/target/plugins/os/unix/locate/plocate.py

@@ -1,10 +1,9 @@
 from __future__ import annotations
 
-import platform
+import sys
 from typing import TYPE_CHECKING, BinaryIO
 
 from dissect.cstruct import cstruct
-from dissect.util.stream import RangeStream
 
 from dissect.target.exceptions import UnsupportedPluginError
 from dissect.target.helpers.record import TargetRecordDescriptor
@@ -15,11 +14,10 @@ if TYPE_CHECKING:
     from collections.abc import Iterator
 
 try:
-    from zstandard import (
-        DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE,
-        ZstdCompressionDict,
-        ZstdDecompressor,
-    )
+    if sys.version_info >= (3, 14):
+        from compression import zstd  # novermin
+    else:
+        from backports import zstd
 
     HAS_ZSTD = True
 except ImportError:
@@ -54,10 +52,6 @@ struct header {
     uint8_t check_visibility;
     char padding[7];                         /* padding for alignment */
 };
-
-struct file {
-    char path[];
-};
 """  # noqa : E501
 
 PLocateRecord = TargetRecordDescriptor(
@@ -97,7 +91,6 @@ class PLocateFile:
     HEADER_SIZE = 0x70  # 0x8 bytes magic + 0x68 bytes header
     NUM_OVERFLOW_SLOTS = 16
     TRIGRAM_SIZE_BYTES = 16
-    DOCID_SIZE_BYTES = 8
 
     def __init__(self, fh: BinaryIO):
         self.fh = fh
@@ -107,56 +100,27 @@ class PLocateFile:
             raise ValueError(f"Invalid plocate file magic. Expected b'/x00plocate', got {magic}")
 
         self.header = c_plocate.header(self.fh)
-        self.dict_data = None
+        self.zstd_dict = None
 
         if self.header.zstd_dictionary_offset_bytes:
-            self.dict_data = ZstdCompressionDict(self.fh.read(self.header.zstd_dictionary_length_bytes))
+            self.zstd_dict = zstd.ZstdDict(self.fh.read(self.header.zstd_dictionary_length_bytes))
 
         self.compressed_length_bytes = (
             self.header.filename_index_offset_bytes - self.HEADER_SIZE - self.header.zstd_dictionary_length_bytes
         )
-        self.ctx = ZstdDecompressor(dict_data=self.dict_data)
-        self.buf = RangeStream(self.fh, self.fh.tell(), self.compressed_length_bytes)
-
-    def __iter__(self) -> Iterator[PLocateFile]:
-        # NOTE: This is a workaround for a PyPy bug
-        # We don't know what breaks, but PyPy + zstandard = unhappy times
-        # You just get random garbage data back instead of the decompressed data
-        # This weird dance of using a decompressobj and unused data is the only way that seems to work
-        # It's more expensive on memory, but at least it doesn't break
-        if platform.python_implementation() == "PyPy":
-            buf = self.buf.read()
-
-            def reader(ctx: ZstdDecompressor) -> Iterator[bytes]:
-                obj = ctx.decompressobj()
-
-                yield obj.decompress(buf)
-                while unused_data := obj.unused_data:
-                    obj = self.ctx.decompressobj()
-                    yield obj.decompress(unused_data)
-
-            it = reader(self.ctx)
-        else:
-            # NOTE: The end of a zstandard frame does not include a final `0x00`.
-            # This causes the c_plocate `file` struct to parse the last path and the first path on the next frame as one
-            # since cstruct will read it across frame boundaries waiting for a `0x00`.
-            def reader() -> Iterator[bytes]:
-                with self.ctx.stream_reader(self.buf) as reader:
-                    while chunk := reader.read(DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE):
-                        yield chunk
-
-            it = reader()
-
-        for chunk in it:
-            for path in chunk.split(b"\x00"):
+
+    def __iter__(self) -> Iterator[str]:
+        offsets = self.filename_index()
+        for i in range(self.header.num_docids):
+            self.fh.seek(offsets[i])
+            block = zstd.decompress(self.fh.read(offsets[i + 1] - offsets[i]), zstd_dict=self.zstd_dict)
+            for path in block.split(b"\x00"):
                 yield path.decode(errors="surrogateescape")
 
-    def filename_index(self) -> bytes:
+    def filename_index(self) -> list[int]:
         """Return the filename index of the plocate.db file."""
         self.fh.seek(self.header.filename_index_offset_bytes)
-        num_docids = self.header.num_docids
-        filename_index_size = num_docids * self.DOCID_SIZE_BYTES
-        return self.fh.read(filename_index_size)
+        return c_plocate.uint64[self.header.num_docids + 1](self.fh)
 
     def hashtable(self) -> bytes:
         """Return the hashtable of the plocate.db file."""
@@ -178,7 +142,7 @@ class PLocatePlugin(BaseLocatePlugin):
 
         if not HAS_ZSTD:
             raise UnsupportedPluginError(
-                "Please install `python-zstandard` or `pip install zstandard` to use the PLocatePlugin"
+                "Required dependency 'backports.zstd' is missing, install with `pip install backports.zstd` to use the PLocatePlugin"  # noqa: E501
             )
 
     @export(record=PLocateRecord)

{dissect_target-3.25.dev6 → dissect_target-3.25.dev7}/dissect/target/plugins/os/unix/log/journal.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
 import lzma
+import sys
 from typing import TYPE_CHECKING, Any, BinaryIO
 
-import zstandard
 from dissect.cstruct import cstruct
 from dissect.util import ts
 from dissect.util.compression import lz4
@@ -19,6 +19,11 @@ if TYPE_CHECKING:
 
     from dissect.target.target import Target
 
+if sys.version_info >= (3, 14):
+    from compression import zstd  # novermin
+else:
+    from backports import zstd
+
 
 log = get_logger(__name__)
 
@@ -401,7 +406,7 @@ class JournalFile:
                     data = lz4.decompress(data[8:])
 
                 elif data_object.flags & c_journal.ObjectFlag.OBJECT_COMPRESSED_ZSTD:
-                    data = zstandard.decompress(data)
+                    data = zstd.decompress(data)
 
                 key, value = self.decode_value(data)
                 event[key] = value

{dissect_target-3.25.dev6 → dissect_target-3.25.dev7}/dissect/target/tools/dump.py

@@ -31,7 +31,10 @@ except ImportError:
 
 
 try:
-    import zstandard
+    if sys.version_info >= (3, 14):
+        from compression import zstd  # novermin
+    else:
+        from backports import zstd
 
     HAS_ZSTD = True
 except ImportError:
@@ -602,9 +605,8 @@ def open_path(path: Path, mode: str, compression: Compression | None = None) ->
         fh = lz4.frame.open(path, mode)
     elif compression == Compression.ZSTD:
         if not HAS_ZSTD:
-            raise ValueError("Python module zstandard is not available")
-        cctx = zstandard.ZstdCompressor()
-        fh = cctx.stream_writer(path.open(mode))
+            raise ValueError("Python module backport.zstd is not available")
+        fh = zstd.open(path, mode)
     elif compression == Compression.NONE:
         fh = path.open(mode)
     else:

{dissect_target-3.25.dev6 → dissect_target-3.25.dev7}/dissect.target.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dissect.target
-Version: 3.25.dev6
+Version: 3.25.dev7
 Summary: This module ties all other Dissect modules together, it provides a programming API and command line tools which allow easy access to various data sources inside disk images or file collections (a.k.a. targets)
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: AGPL-3.0-or-later
@@ -58,7 +58,7 @@ Requires-Dist: fusepy; extra == "full"
 Requires-Dist: pycryptodome; extra == "full"
 Requires-Dist: ruamel.yaml; extra == "full"
 Requires-Dist: tomli; python_version < "3.11" and extra == "full"
-Requires-Dist: zstandard; platform_python_implementation != "PyPy" and extra == "full"
+Requires-Dist: backports.zstd; python_version < "3.14" and extra == "full"
 Provides-Extra: dev
 Requires-Dist: dissect.target[full,yara]; extra == "dev"
 Requires-Dist: dissect.archive[dev]<2.0.dev,>=1.0.dev; extra == "dev"

{dissect_target-3.25.dev6 → dissect_target-3.25.dev7}/dissect.target.egg-info/requires.txt

@@ -69,12 +69,12 @@ fusepy
 pycryptodome
 ruamel.yaml
 
-[full:platform_python_implementation != "PyPy"]
-zstandard
-
 [full:python_version < "3.11"]
 tomli
 
+[full:python_version < "3.14"]
+backports.zstd
+
 [mqtt]
 dissect.target[full]
 paho-mqtt==1.6.1

{dissect_target-3.25.dev6 → dissect_target-3.25.dev7}/pyproject.toml

@@ -71,7 +71,7 @@ full = [
     "pycryptodome",
     "ruamel.yaml",
     "tomli; python_version<'3.11'",
-    "zstandard; platform_python_implementation != 'PyPy'",
+    "backports.zstd; python_version < '3.14'"
 ]
 dev = [
     "dissect.target[full,yara]",

dissect_target-3.25.dev7/tests/filesystems/test_vmtar.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import os
+
+from dissect.target.filesystems.vmtar import VmtarFilesystem
+from dissect.target.helpers.fsutil import open_decompress
+from tests._utils import absolute_path
+
+
+def test_filesystems_vmtar() -> None:
+    vmtar_path = absolute_path("_data/filesystems/vmtar/simple.vmtar")
+
+    with vmtar_path.open("rb") as fh:
+        assert VmtarFilesystem.detect(fh)
+
+        fs = VmtarFilesystem(fh)
+        assert [f.name for f in fs.path("/").iterdir()] == ["hello_world.txt"]
+        assert fs.get("hello_world.txt").open().read() == b"hello_from_a_tar_file\n"
+
+
+def test_filesystems_vmtar_zstd() -> None:
+    vmtar_path = absolute_path("_data/filesystems/vmtar/a.vmtar.zst.gz")
+
+    with vmtar_path.open("rb") as fh:
+        zstd_fh = open_decompress(fileobj=fh)
+        vmtar_fh = open_decompress(fileobj=zstd_fh)
+        assert vmtar_fh.seek(0, os.SEEK_END) == 4608
+
+    with vmtar_path.open("rb") as fh:
+        assert VmtarFilesystem.detect(fh)
+
+        fs = VmtarFilesystem(fh)
+        assert fs.get("a/b/c/test.txt").open().read() == b"test_data\nmultiline.txt\n"

dissect_target-3.25.dev6/tests/filesystems/test_vmtar.py

@@ -1,17 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-from dissect.target.filesystems.vmtar import VmtarFilesystem
-from tests._utils import absolute_path
-
-
-def test_filesystems_vmtar() -> None:
-    vmtar_path = Path(absolute_path("_data/filesystems/vmtar/simple.vmtar"))
-
-    with vmtar_path.open("rb") as fh:
-        assert VmtarFilesystem.detect(fh)
-
-        fs = VmtarFilesystem(fh)
-        assert [f.name for f in fs.path("/").iterdir()] == ["hello_world.txt"]
-        assert fs.get("hello_world.txt").open().read() == b"hello_from_a_tar_file\n"