hbkit 0.2.0__tar.gz → 0.2.1__tar.gz

{hbkit-0.2.0 → hbkit-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hbkit
-Version: 0.2.0
+Version: 0.2.1
 Summary: Recover files from Synology Hyper Backup (.hbk) archives without Synology software
 Project-URL: Homepage, https://github.com/YordiLorenzo/hbkit
 Project-URL: Source, https://github.com/YordiLorenzo/hbkit
@@ -35,9 +35,9 @@ Description-Content-Type: text/markdown
 
 Recover files from **Synology Hyper Backup (`.hbk`)** archives without any Synology software.
 
-Point it at a backup — on a local disk, an external drive, or an S3/R2 bucket mounted with
-rclone — browse it as a tree, and pull out what you want. Works headless on Linux and macOS,
-including Apple Silicon, where Synology's own Hyper Backup Explorer is awkward or unavailable.
+Point it at a backup on a local disk, an external drive, or a network mount, browse it as
+a tree, and pull out what you want. Works headless on Linux and macOS, including Apple Silicon, where Synology's
+own Hyper Backup Explorer is awkward or unavailable.
 
 ```sh
 brew install lz4          # or: sudo apt install liblz4-1
@@ -141,6 +141,30 @@ readable by rebuilding a random sample of real files with full checksum verifica
 - **Index cached** per archive in `~/.cache/hbkit`, rebuilt automatically when the archive
   changes. Browsing 1.1M files is instant after the first open.
 
+## Network mounts (rclone / S3 / R2)
+
+Opening an archive no longer measures every index shard. Shards are a fixed 8 MiB except
+the last, so offsets are computed instead — one directory listing and a single stat per
+index family, and `file_chunk<N>.index` families are opened only if a file references them.
+On a 3 TB archive that removed roughly **2,600 network round-trips** from startup.
+
+If you do mount a bucket, the flags matter more than anything hbkit does:
+
+```sh
+rclone mount r2:bucket ~/mnt/r2 --read-only \
+  --vfs-cache-mode off \      # range requests; 'full' downloads whole files
+  --dir-cache-time 72h         # first listing of ~2000 shards is slow, then cached
+```
+
+**Do not use `--vfs-cache-mode full`.** hbkit reads a 32-byte index record and a ~5 KB
+chunk at a time; in `full` mode each of those pulls an entire file, so a 10 KB extraction
+downloaded ~82 MB (an 8 MiB index shard plus a ~50 MB bucket) and took ten minutes.
+
+Even configured well, a network mount is dramatically slower than local storage — the
+access pattern is thousands of small scattered reads. **If you can, copy the archive to a
+local disk first.** Treat mounted-bucket recovery as workable for pulling out a handful of
+files, not for restoring terabytes.
+
 ## Performance
 
 Use `-j` to set worker processes (default 8). Threads do not help — extraction is

{hbkit-0.2.0 → hbkit-0.2.1}/README.md

@@ -6,9 +6,9 @@
 
 Recover files from **Synology Hyper Backup (`.hbk`)** archives without any Synology software.
 
-Point it at a backup — on a local disk, an external drive, or an S3/R2 bucket mounted with
-rclone — browse it as a tree, and pull out what you want. Works headless on Linux and macOS,
-including Apple Silicon, where Synology's own Hyper Backup Explorer is awkward or unavailable.
+Point it at a backup on a local disk, an external drive, or a network mount, browse it as
+a tree, and pull out what you want. Works headless on Linux and macOS, including Apple Silicon, where Synology's
+own Hyper Backup Explorer is awkward or unavailable.
 
 ```sh
 brew install lz4          # or: sudo apt install liblz4-1
@@ -112,6 +112,30 @@ readable by rebuilding a random sample of real files with full checksum verifica
 - **Index cached** per archive in `~/.cache/hbkit`, rebuilt automatically when the archive
   changes. Browsing 1.1M files is instant after the first open.
 
+## Network mounts (rclone / S3 / R2)
+
+Opening an archive no longer measures every index shard. Shards are a fixed 8 MiB except
+the last, so offsets are computed instead — one directory listing and a single stat per
+index family, and `file_chunk<N>.index` families are opened only if a file references them.
+On a 3 TB archive that removed roughly **2,600 network round-trips** from startup.
+
+If you do mount a bucket, the flags matter more than anything hbkit does:
+
+```sh
+rclone mount r2:bucket ~/mnt/r2 --read-only \
+  --vfs-cache-mode off \      # range requests; 'full' downloads whole files
+  --dir-cache-time 72h         # first listing of ~2000 shards is slow, then cached
+```
+
+**Do not use `--vfs-cache-mode full`.** hbkit reads a 32-byte index record and a ~5 KB
+chunk at a time; in `full` mode each of those pulls an entire file, so a 10 KB extraction
+downloaded ~82 MB (an 8 MiB index shard plus a ~50 MB bucket) and took ten minutes.
+
+Even configured well, a network mount is dramatically slower than local storage — the
+access pattern is thousands of small scattered reads. **If you can, copy the archive to a
+local disk first.** Treat mounted-bucket recovery as workable for pulling out a handful of
+files, not for restoring terabytes.
+
 ## Performance
 
 Use `-j` to set worker processes (default 8). Threads do not help — extraction is

{hbkit-0.2.0 → hbkit-0.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hbkit"
-version = "0.2.0"
+version = "0.2.1"
 description = "Recover files from Synology Hyper Backup (.hbk) archives without Synology software"
 readme = "README.md"
 requires-python = ">=3.10"

{hbkit-0.2.0 → hbkit-0.2.1}/src/hbkit/__init__.py

@@ -1,6 +1,6 @@
 """hbkit - recover files from Synology Hyper Backup (.hbk) archives without Synology software."""
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 from .archive import (Archive, NeedPassword, UnsupportedArchive,  # noqa: F401
                       find_archive_root, is_archive)

{hbkit-0.2.0 → hbkit-0.2.1}/src/hbkit/archive.py

@@ -45,6 +45,7 @@ BUCKET_BUF = int(os.environ.get("HBK_BUCKET_BUF", 1 << 22))   # read-ahead; plat
 INDEX_BUF = int(os.environ.get("HBK_INDEX_BUF", 1 << 16))     # chunk keys are mostly sequential;
                                                              # unbuffered = one syscall per 29 B
 MAGIC = 0x7053A86E
+SHARD_SIZE = 8 << 20        # every index shard is exactly 8 MiB except the last
 
 _LZ4_CANDIDATES = [
     os.environ.get("HBK_LZ4"),
@@ -112,7 +113,14 @@ def resolve(d: str, base: str) -> str:
 
 
 class Cat:
-    """Numbered <N>.idx[.gen] shards presented as a single logical byte stream."""
+    """Numbered <N>.idx[.gen] shards presented as a single logical byte stream.
+
+    Shards are a fixed SHARD_SIZE except the last, so offsets are computed rather than
+    measured. That matters enormously on network mounts: stat-ing every shard cost ~2,600
+    round-trips on a 3 TB archive (minutes before reading a byte). We now do one directory
+    listing plus a single stat of the final shard. If the assumption were ever wrong, a
+    short read surfaces as a chunk MD5 failure - loudly - never as silent corruption.
+    """
 
     def __init__(self, d: str):
         self.d = d
@@ -129,12 +137,11 @@ class Cat:
         if not found:
             raise FileNotFoundError(f"no .idx shards in {d}")
         self.files = [found[k][1] for k in sorted(found)]
-        self.sizes = [os.path.getsize(os.path.join(d, f)) for f in self.files]
-        self.starts, t = [], 0
-        for s in self.sizes:
-            self.starts.append(t)
-            t += s
-        self.total = t
+        n = len(self.files)
+        last = os.path.getsize(os.path.join(d, self.files[-1]))
+        self.sizes = [SHARD_SIZE] * (n - 1) + [last]
+        self.starts = [i * SHARD_SIZE for i in range(n)]
+        self.total = (n - 1) * SHARD_SIZE + last
         self._fh: dict[int, object] = {}
         # Shard 0 carries a 64-byte header that declares the record size, so readers do
         # not have to hardcode per-DSM-version layouts.
@@ -150,16 +157,26 @@ class Cat:
         self.declared_total = (w[5] << 32) | w[6]
 
     def read(self, off: int, n: int) -> bytes:
+        """Read n bytes at logical offset off, spanning shards as needed."""
         out = b""
-        for i, (f, st, sz) in enumerate(zip(self.files, self.starts, self.sizes)):
-            if off + n <= st or off >= st + sz:
-                continue
-            a, b = max(off, st), min(off + n, st + sz)
+        i = off // SHARD_SIZE
+        while n > 0 and i < len(self.files):
+            st = self.starts[i]
+            take = min(n, st + self.sizes[i] - off)
+            if take <= 0:
+                break
             fh = self._fh.get(i)
             if fh is None:
-                fh = self._fh[i] = open(os.path.join(self.d, f), "rb", buffering=INDEX_BUF)
-            fh.seek(a - st)
-            out += fh.read(b - a)
+                fh = self._fh[i] = open(os.path.join(self.d, self.files[i]), "rb",
+                                        buffering=INDEX_BUF)
+            fh.seek(off - st)
+            got = fh.read(take)
+            out += got
+            if len(got) < take:            # short shard: stop rather than mis-align
+                break
+            off += take
+            n -= take
+            i += 1
         return out
 
     def close(self):
@@ -168,6 +185,32 @@ class Cat:
         self._fh.clear()
 
 
+class _LazyCats(dict):
+    """file_chunk<N>.index families, opened only when a file actually references one."""
+
+    def __init__(self, dirs: dict[int, str]):
+        super().__init__()
+        self._dirs = dirs
+
+    def get(self, k, default=None):
+        if k in self:
+            return self[k]
+        d = self._dirs.get(k)
+        if d is None:
+            return default
+        self[k] = c = Cat(d)
+        return c
+
+    def __missing__(self, k):
+        c = self.get(k)
+        if c is None:
+            raise KeyError(k)
+        return c
+
+    def values(self):
+        return [v for v in dict.values(self)]
+
+
 def is_archive(d: str) -> bool:
     return os.path.isdir(os.path.join(d, "Pool")) and os.path.isdir(os.path.join(d, "Config"))
 
@@ -198,13 +241,14 @@ class Archive:
         c, p = f"{self.root}/Config", f"{self.root}/Pool"
         self.vf = Cat(f"{c}/virtual_file.index")
         self.ci = Cat(f"{p}/chunk_index")
-        self.fc: dict[int, Cat] = {}
+        self._fc_dirs: dict[int, str] = {}
         for e in os.listdir(c):
             m = re.match(r"^file_chunk(\d+)\.index$", e)
             if m and os.path.isdir(os.path.join(c, e)):
-                self.fc[int(m.group(1))] = Cat(os.path.join(c, e))
-        if not self.fc:
+                self._fc_dirs[int(m.group(1))] = os.path.join(c, e)
+        if not self._fc_dirs:
             raise FileNotFoundError(f"no file_chunk<N>.index directories in {c}")
+        self.fc: dict[int, Cat] = _LazyCats(self._fc_dirs)
         pools = [e for e in os.listdir(p) if e.isdigit() and os.path.isdir(os.path.join(p, e))]
         if not pools:
             raise FileNotFoundError(f"no numbered pool directory under {p}")

{hbkit-0.2.0 → hbkit-0.2.1}/tests/test_hbkit.py

@@ -229,3 +229,59 @@ def test_encrypted_names_and_content():
         data = arc.extract(ovf, size, verify=True)   # verify=True checks every chunk MD5
         assert len(data) == size, path
     arc.close()
+
+
+# ------------------------------------------------- shard maths (no archive needed)
+
+def _make_shards(tmp_path, sizes, shard_size, monkeypatch):
+    """Write synthetic <N>.idx.2 shards. Shard 0 carries a valid 64-byte header, as real
+    archives do; the header is part of the logical stream so offsets stay comparable."""
+    import struct
+
+    from hbkit import archive as A
+    monkeypatch.setattr(A, "SHARD_SIZE", shard_size)
+    total = sum(sizes)
+    header = struct.pack(">16I", A.MAGIC, 0, 0, 0, 32, total >> 32, total & 0xFFFFFFFF,
+                         *([0] * 9))
+    blob = b""
+    for i, n in enumerate(sizes):
+        if i == 0:
+            part = header + bytes((64 + j) % 251 for j in range(n - 64))
+        else:
+            part = bytes((len(blob) + j) % 251 for j in range(n))
+        (tmp_path / f"{i}.idx.2").write_bytes(part)
+        blob += part
+    return blob
+
+
+def test_cat_computes_offsets_without_stating_every_shard(tmp_path, monkeypatch):
+    """Sizes/offsets must be derived from the fixed shard size + one final stat."""
+    from hbkit import archive as A
+    blob = _make_shards(tmp_path, [128, 128, 10], 128, monkeypatch)
+    c = A.Cat(str(tmp_path))
+    assert len(c.files) == 3
+    assert c.total == len(blob) == 266
+    assert c.starts == [0, 128, 256]
+    assert c.sizes == [128, 128, 10]
+    assert c.record_size == 32          # read from the header, not guessed
+
+
+def test_cat_reads_across_shard_boundaries(tmp_path, monkeypatch):
+    """The multi-shard read path: every offset/length combination must match one blob."""
+    from hbkit import archive as A
+    blob = _make_shards(tmp_path, [128, 128, 10], 128, monkeypatch)
+    c = A.Cat(str(tmp_path))
+    for off in range(0, len(blob), 7):
+        for n in (1, 5, 127, 128, 129, 260):
+            assert c.read(off, n) == blob[off:off + n], f"off={off} n={n}"
+    assert c.read(0, len(blob)) == blob          # whole stream in one read
+    assert c.read(len(blob), 10) == b""          # past the end
+    c.close()
+
+
+def test_cat_single_shard_still_works(tmp_path, monkeypatch):
+    from hbkit import archive as A
+    blob = _make_shards(tmp_path, [100], 128, monkeypatch)
+    c = A.Cat(str(tmp_path))
+    assert c.total == 100 and c.read(70, 20) == blob[70:90]
+    c.close()