hbkit 0.5.2__tar.gz → 0.5.3__tar.gz

{hbkit-0.5.2 → hbkit-0.5.3}/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.5.3
+
+### Fixed
+
+- Archives stored without compression (`data_compress_type` 0, "none" in `doctor`) now
+  restore. Every chunk of such an archive failed before. Found and diagnosed by @tabrindle
+  on two 12 TB archives, with the patch this fix builds on (#9). A stored chunk that is
+  itself valid zlib data, such as a git object, comes back as stored: hbkit keeps the
+  decoding whose MD5 matches.
+
 ## 0.5.2
 
 No change to how hbkit reads or restores archives.

{hbkit-0.5.2 → hbkit-0.5.3}/FORMAT.md

@@ -210,6 +210,12 @@ The codec comes from `data_compress_type` in `_Syno_TaskConfig`. The vendor disp
 lz4-hc share a decompressor. A robust reader attempts LZ4 and falls back to zlib. [D for
 the mapping, V for LZ4]
 
+**0 = none**: the chunk is stored uncompressed. `compressed length` equals `uncompressed
+length` in every bucket record and the payload is the chunk's own bytes. Seen on two DSM
+archives from 2020 with compression off (issue #9). A stored payload can itself be valid
+zlib or LZ4 data (a git object is a zlib stream), so a reader keeps the decoding whose MD5
+matches, and tries the stored bytes first when the task says type 0. [V]
+
 Existing open-source tools implement only LZ4 and report corruption on some chunks; the
 zlib path is the likely explanation.
 

{hbkit-0.5.2 → hbkit-0.5.3}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.5
 Name: hbkit
-Version: 0.5.2
+Version: 0.5.3
 Summary: Recover files from Synology Hyper Backup (.hbk) archives without Synology software
-Project-URL: Homepage, https://yordilorenzo.github.io/hbkit/
-Project-URL: Documentation, https://yordilorenzo.github.io/hbkit/restore-without-nas/
+Project-URL: Homepage, https://hbkit.dev/
+Project-URL: Documentation, https://hbkit.dev/restore-without-nas/
 Project-URL: Source, https://github.com/YordiLorenzo/hbkit
 Project-URL: Issues, https://github.com/YordiLorenzo/hbkit/issues
 Project-URL: Format spec, https://github.com/YordiLorenzo/hbkit/blob/main/FORMAT.md
@@ -46,7 +46,7 @@ Point it at a backup on a local disk, an external drive, or a network mount, bro
 a tree, and pull out what you want. Works headless on Linux and macOS, including Apple silicon, where Synology's
 own Hyper Backup Explorer is awkward or unavailable.
 
-Website: **[yordilorenzo.github.io/hbkit](https://yordilorenzo.github.io/hbkit/)**
+Website: **[hbkit.dev](https://hbkit.dev/)**
 
 ```sh
 brew install YordiLorenzo/tap/hbkit     # macOS / Linux, pulls in liblz4 for you
@@ -79,9 +79,9 @@ for you. Otherwise: `brew install lz4`, `sudo apt install liblz4-1`, or
 ### Restore without a NAS
 
 The step-by-step guide is on the site:
-[Restore a Hyper Backup without a Synology NAS](https://yordilorenzo.github.io/hbkit/restore-without-nas/).
+[Restore a Hyper Backup without a Synology NAS](https://hbkit.dev/restore-without-nas/).
 If Hyper Backup says the backup is corrupted, start with
-[what you can still recover](https://yordilorenzo.github.io/hbkit/damaged-backup/).
+[what you can still recover](https://hbkit.dev/damaged-backup/).
 
 hbkit never writes to the archive. If it is your only copy, copy the whole `.hbk` folder
 to another disk first and run hbkit on the copy.

{hbkit-0.5.2 → hbkit-0.5.3}/README.md

@@ -12,7 +12,7 @@ Point it at a backup on a local disk, an external drive, or a network mount, bro
 a tree, and pull out what you want. Works headless on Linux and macOS, including Apple silicon, where Synology's
 own Hyper Backup Explorer is awkward or unavailable.
 
-Website: **[yordilorenzo.github.io/hbkit](https://yordilorenzo.github.io/hbkit/)**
+Website: **[hbkit.dev](https://hbkit.dev/)**
 
 ```sh
 brew install YordiLorenzo/tap/hbkit     # macOS / Linux, pulls in liblz4 for you
@@ -45,9 +45,9 @@ for you. Otherwise: `brew install lz4`, `sudo apt install liblz4-1`, or
 ### Restore without a NAS
 
 The step-by-step guide is on the site:
-[Restore a Hyper Backup without a Synology NAS](https://yordilorenzo.github.io/hbkit/restore-without-nas/).
+[Restore a Hyper Backup without a Synology NAS](https://hbkit.dev/restore-without-nas/).
 If Hyper Backup says the backup is corrupted, start with
-[what you can still recover](https://yordilorenzo.github.io/hbkit/damaged-backup/).
+[what you can still recover](https://hbkit.dev/damaged-backup/).
 
 hbkit never writes to the archive. If it is your only copy, copy the whole `.hbk` folder
 to another disk first and run hbkit on the copy.

{hbkit-0.5.2 → hbkit-0.5.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hbkit"
-version = "0.5.2"
+version = "0.5.3"
 description = "Recover files from Synology Hyper Backup (.hbk) archives without Synology software"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -34,8 +34,8 @@ dependencies = ["textual>=0.80", "pynacl>=1.5", "cryptography>=41"]
 dev = ["pytest>=7"]
 
 [project.urls]
-Homepage = "https://yordilorenzo.github.io/hbkit/"
-Documentation = "https://yordilorenzo.github.io/hbkit/restore-without-nas/"
+Homepage = "https://hbkit.dev/"
+Documentation = "https://hbkit.dev/restore-without-nas/"
 Source = "https://github.com/YordiLorenzo/hbkit"
 Issues = "https://github.com/YordiLorenzo/hbkit/issues"
 "Format spec" = "https://github.com/YordiLorenzo/hbkit/blob/main/FORMAT.md"

{hbkit-0.5.2 → hbkit-0.5.3}/src/hbkit/__init__.py

@@ -1,6 +1,6 @@
 """hbkit - recover files from Synology Hyper Backup (.hbk) archives without Synology software."""
 
-__version__ = "0.5.2"
+__version__ = "0.5.3"
 
 from .archive import (Archive, NeedPassword, UnsupportedArchive,  # noqa: F401
                       find_archive_root, is_archive)

{hbkit-0.5.2 → hbkit-0.5.3}/src/hbkit/archive.py

@@ -11,7 +11,8 @@ Format (reverse engineered; all integers BIG-ENDIAN):
        u32 comp_len, u32 offset, u32 uncomp_len, md5[16] of DECOMPRESSED chunk,
        u32 crc32 of record[0:28]
   -> same-named .bucket @offset, comp_len bytes: raw LZ4 BLOCK (zlib is the fallback codec;
-     SYNO::Backup::decompress dispatches type 1/2/4 = lz4 / lz4-hc / zlib)
+     SYNO::Backup::decompress dispatches type 1/2/4 = lz4 / lz4-hc / zlib; type 0 stores the
+     chunk uncompressed, comp_len == uncomp_len)
 
 Index families are <N>.idx shards concatenated in numeric order; records start at stream
 offset 64; only shard 0 carries the 64-byte header (magic 0x7053A86E).
@@ -127,15 +128,34 @@ def lz4():
     return _lz4
 
 
-def _decompress(raw: bytes, ulen: int) -> bytes | None:
-    """A chunk payload: a raw LZ4 block, else zlib (compress type 4). None if neither fits."""
+def _decodings(raw: bytes, ulen: int, stored_first: bool):
+    """The ways a chunk payload can decode to `ulen` bytes: a raw LZ4 block, zlib (compress
+    type 4), or the payload as stored (compress type 0, where the stored length equals the
+    uncompressed one). A stored payload can itself be valid LZ4 or zlib data (a git object
+    is a zlib stream), so the caller keeps the first decoding whose MD5 matches. An archive
+    whose task says type 0 tries the stored bytes first.
+    """
+    if stored_first and len(raw) == ulen:
+        yield raw
     dst = ctypes.create_string_buffer(ulen)
     if lz4().LZ4_decompress_safe(raw, dst, len(raw), ulen) == ulen:
-        return dst.raw
+        yield dst.raw
     try:
-        return zlib.decompress(raw)
+        z = zlib.decompress(raw)
     except zlib.error:
-        return None
+        z = None
+    if z is not None:
+        yield z
+    if not stored_first and len(raw) == ulen:
+        yield raw
+
+
+def _decode(raw: bytes, ulen: int, md5: bytes, verify: bool, stored_first: bool) -> bytes | None:
+    """The first decoding whose MD5 matches, or the first decoding at all when not verifying."""
+    for data in _decodings(raw, ulen, stored_first):
+        if not verify or hashlib.md5(data).digest() == md5:
+            return data
+    return None
 
 
 # ------------------------------------------------------------ path resolution
@@ -347,6 +367,7 @@ class Archive:
         self._bdat: OrderedDict[int, object] = OrderedDict()
         self._brec: dict[int, int] = {}
         self._bkey: dict[int, int] = {}      # bucket id -> index into crypto.data_keys
+        self._stored_first = self.task_config().get("data_compress_type") == "0"
         self._password = None
         # The version list is stored in the clear: check the selector before unlocking,
         # so a typo or a missing version is reported without asking for a password.
@@ -622,11 +643,10 @@ class Archive:
         md5 = rec[12:28]
         if self.crypto is not None:
             return self._decrypt(bid, boff, raw, ulen, md5)
-        data = _decompress(raw, ulen)
+        data = _decode(raw, ulen, md5, verify, self._stored_first)
         if data is None:
-            raise ValueError(f"decompress failed b{bid}@{boff} want={ulen}")
-        if verify and hashlib.md5(data).digest() != md5:
-            raise ValueError(f"MD5 mismatch b{bid}@{boff}")
+            raise ValueError(f"MD5 mismatch b{bid}@{boff}: no decoding matches (want={ulen})"
+                             if verify else f"decompress failed b{bid}@{boff} want={ulen}")
         return data
 
     def _decrypt(self, bid: int, boff: int, raw: bytes, ulen: int, md5: bytes) -> bytes:
@@ -640,8 +660,8 @@ class Archive:
         n = len(self.crypto.data_keys)
         first = self._bkey.get(bid, 0)
         for k in (first, *(i for i in range(n) if i != first)):
-            data = _decompress(self.crypto.decrypt_chunk(raw, k), ulen)
-            if data is not None and hashlib.md5(data).digest() == md5:
+            data = _decode(self.crypto.decrypt_chunk(raw, k), ulen, md5, True, self._stored_first)
+            if data is not None:
                 self._bkey[bid] = k
                 return data
         raise ValueError(f"no data key decodes b{bid}@{boff} ({n} tried, want={ulen})")

{hbkit-0.5.2 → hbkit-0.5.3}/tests/synth.py

@@ -89,8 +89,11 @@ class _Crypto:
 
 
 def build(root: str, versions: list[V], encrypted: bool = False,
-          with_version_info: bool = True) -> str:
-    """Write `<root>/synth.hbk` holding `versions` (version ids 1..n). Returns its path."""
+          with_version_info: bool = True, compress_type: int = 1) -> str:
+    """Write `<root>/synth.hbk` holding `versions` (version ids 1..n). Returns its path.
+
+    `compress_type` 1 stores LZ4 blocks; 0 stores chunks uncompressed, as some real archives do.
+    """
     arc = os.path.join(root, "synth.hbk")
     cfg, pool = f"{arc}/Config", f"{arc}/Pool"
     for d in (f"{cfg}/virtual_file.index", f"{cfg}/file_chunk1.index",
@@ -113,7 +116,7 @@ def build(root: str, versions: list[V], encrypted: bool = False,
                 keys = []
                 for c in parts:
                     if c not in chunks:
-                        comp = _lz4(c)
+                        comp = _lz4(c) if compress_type else c
                         if crypto:
                             comp = crypto.chunk(comp, vi)
                         rec = struct.pack(">III", len(comp), len(bdat), len(c)) + \
@@ -185,7 +188,7 @@ def build(root: str, versions: list[V], encrypted: bool = False,
         db.close()
 
     conf = ['[task_config]', 'name="synth"', 'host_name="synth-host"',
-            'data_compress_type=1', f'enable_data_encrypt={"true" if crypto else "false"}']
+            f'data_compress_type={compress_type}', f'enable_data_encrypt={"true" if crypto else "false"}']
     if crypto:
         conf.append(f'unikey="{UNIKEY}"')
         with open(f"{cfg}/public.pem", "wb") as fh:

{hbkit-0.5.2 → hbkit-0.5.3}/tests/test_hbkit.py

@@ -13,6 +13,7 @@ import subprocess
 import sys
 import tempfile
 import time
+import zlib
 
 import pytest
 
@@ -262,6 +263,7 @@ def _multi_key_archive():
     arc.crypto = ArchiveCrypto(pw, unikey, pub, [(nb.crypto_box_seal(k, pub),
                                                   nb.crypto_box_seal(iv, pub)) for k, iv in keys])
     arc._brec, arc._bkey, buckets, plain = {}, {}, {}, []
+    arc._stored_first = False              # an LZ4 archive (data_compress_type 1)
     lib.LZ4_compress_default.argtypes = [ctypes.c_char_p, ctypes.c_char_p,
                                          ctypes.c_int, ctypes.c_int]
     for bid, (key, iv) in enumerate([*keys, (os.urandom(32), os.urandom(16))]):
@@ -808,6 +810,26 @@ def _hbk(tmp_path, *args, password=None):
                           capture_output=True, text=True, env=env, input="")
 
 
+@pytest.mark.parametrize("encrypted", [False, True], ids=["plain", "encrypted"])
+def test_uncompressed_archive_restores(tmp_path, encrypted):
+    """Hyper Backup can store chunks without compression (data_compress_type 0): the
+    stored length equals the real one and the payload is the file's own bytes (issue #9).
+    The last chunk is itself a valid zlib stream, as a git object is: it must come back as
+    stored, not decompressed."""
+    synth = _synth()
+    body = [b"H" * 3000 + os.urandom(1000), os.urandom(5000), zlib.compress(b"blob 12\0" * 40)]
+    root = synth.build(str(tmp_path / "arc"), [synth.V({"firebird/raw.bak": body}, T0)],
+                       encrypted=encrypted, compress_type=0)
+    pw = synth.PASSWORD if encrypted else None
+    r = _hbk(tmp_path, root, "doctor", password=pw)
+    assert r.returncode == 0, r.stdout + r.stderr
+    assert "chunk codec : none" in r.stdout
+    out = tmp_path / "out"
+    r = _hbk(tmp_path, root, "get", "/db_staging/firebird/raw.bak", out, password=pw)
+    assert r.returncode == 0, r.stdout + r.stderr
+    assert (out / "db_staging/firebird/raw.bak").read_bytes() == b"".join(body)
+
+
 GLOB = "/db_staging/firebird/*.gbk"
 DB_ONE = "db_staging/firebird/db one.gbk"